--- /dev/null
+Checks: '-readability-identifier-naming,-llvm-header-guard'
+InheritParentConfig: true
--- /dev/null
+*~
+darwin_fat
+clang_darwin
+multi_arch
+*.sw?
+*.pyc
# An important constraint of the build is that it only produces libraries
# based on the ability of the host toolchain to target various platforms.
-cmake_minimum_required(VERSION 3.4.3)
-
-if(POLICY CMP0075)
- cmake_policy(SET CMP0075 NEW)
-endif()
+cmake_minimum_required(VERSION 3.13.4)
# Check if compiler-rt is built as a standalone project.
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR OR COMPILER_RT_STANDALONE_BUILD)
mark_as_advanced(COMPILER_RT_BUILD_LIBFUZZER)
option(COMPILER_RT_BUILD_PROFILE "Build profile runtime" ON)
mark_as_advanced(COMPILER_RT_BUILD_PROFILE)
+option(COMPILER_RT_BUILD_MEMPROF "Build memory profiling runtime" ON)
+mark_as_advanced(COMPILER_RT_BUILD_MEMPROF)
option(COMPILER_RT_BUILD_XRAY_NO_PREINIT "Build xray with no preinit patching" OFF)
mark_as_advanced(COMPILER_RT_BUILD_XRAY_NO_PREINIT)
+option(COMPILER_RT_BUILD_ORC "Build ORC runtime" ON)
+mark_as_advanced(COMPILER_RT_BUILD_ORC)
set(COMPILER_RT_ASAN_SHADOW_SCALE ""
CACHE STRING "Override the shadow scale to be used in ASan runtime")
-D${COMPILER_RT_ASAN_SHADOW_SCALE_DEFINITION})
endif()
-set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS ON CACHE BOOL
- "Enable libc interceptors in HWASan (testing mode)")
+if(FUCHSIA)
+ set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS_DEFAULT OFF)
+else()
+ set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS_DEFAULT ON)
+endif()
+set(COMPILER_RT_HWASAN_WITH_INTERCEPTORS ${COMPILER_RT_HWASAN_WITH_INTERCEPTORS_DEFAULT} CACHE BOOL "Enable libc interceptors in HWASan (testing mode)")
set(COMPILER_RT_BAREMETAL_BUILD OFF CACHE BOOL
"Build for a bare-metal target.")
set_target_properties(intrinsics_gen PROPERTIES FOLDER "Compiler-RT Misc")
endif()
- if(CMAKE_VERSION VERSION_LESS 3.12)
- # Find Python interpreter.
- include(FindPythonInterp)
- if(NOT PYTHONINTERP_FOUND)
- message(FATAL_ERROR "
- Unable to find Python interpreter required testing. Please install Python
- or specify the PYTHON_EXECUTABLE CMake variable.")
+ find_package(Python3 COMPONENTS Interpreter)
+ if(NOT Python3_Interpreter_FOUND)
+ message(WARNING "Python3 not found, using python2 as a fallback")
+ find_package(Python2 COMPONENTS Interpreter REQUIRED)
+ if(Python2_VERSION VERSION_LESS 2.7)
+ message(SEND_ERROR "Python 2.7 or newer is required")
endif()
+ # Treat python2 as python3
add_executable(Python3::Interpreter IMPORTED)
set_target_properties(Python3::Interpreter PROPERTIES
- IMPORTED_LOCATION ${PYTHON_EXECUTABLE})
- set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
- else()
- find_package(Python3 COMPONENTS Interpreter)
- if(NOT Python3_Interpreter_FOUND)
- message(WARNING "Python3 not found, using python2 as a fallback")
- find_package(Python2 COMPONENTS Interpreter REQUIRED)
- if(Python2_VERSION VERSION_LESS 2.7)
- message(SEND_ERROR "Python 2.7 or newer is required")
- endif()
-
- # Treat python2 as python3
- add_executable(Python3::Interpreter IMPORTED)
- set_target_properties(Python3::Interpreter PROPERTIES
- IMPORTED_LOCATION ${Python2_EXECUTABLE})
- set(Python3_EXECUTABLE ${Python2_EXECUTABLE})
- endif()
+ IMPORTED_LOCATION ${Python2_EXECUTABLE})
+ set(Python3_EXECUTABLE ${Python2_EXECUTABLE})
endif()
# Ensure that fat libraries are built correctly on Darwin
- if(CMAKE_HOST_APPLE AND APPLE)
+ if(APPLE)
include(UseLibtool)
endif()
if ("${COMPILER_RT_DEFAULT_TARGET_TRIPLE}" MATCHES ".*hf$")
if (${COMPILER_RT_DEFAULT_TARGET_ARCH} MATCHES "^arm")
set(COMPILER_RT_DEFAULT_TARGET_ARCH "armhf")
+ CHECK_SYMBOL_EXISTS (__thumb__ "" COMPILER_RT_ARM_THUMB)
endif()
endif()
if ("${COMPILER_RT_DEFAULT_TARGET_TRIPLE}" MATCHES ".*android.*")
set(ANDROID 1)
+ string(REGEX MATCH "-target(=| +)[^ ]+android[a-z]*([0-9]+)" ANDROID_API_LEVEL "${CMAKE_C_FLAGS}")
+ set(ANDROID_API_LEVEL ${CMAKE_MATCH_2})
endif()
pythonize_bool(ANDROID)
-set(ANDROID_NDK_VERSION 18
- CACHE STRING "Set this to the Android NDK version that you are using")
-
set(COMPILER_RT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(COMPILER_RT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
option(SANITIZER_ALLOW_CXXABI "Allow use of C++ ABI details in ubsan" ON)
-set(SANITIZE_CAN_USE_CXXABI OFF)
+set(SANITIZER_CAN_USE_CXXABI OFF)
if (cxxabi_supported AND SANITIZER_ALLOW_CXXABI)
set(SANITIZER_CAN_USE_CXXABI ON)
endif()
pythonize_bool(SANITIZER_CAN_USE_CXXABI)
macro(handle_default_cxx_lib var)
- if (${var} STREQUAL "default")
+ # Specifying -stdlib= in CMAKE_CXX_FLAGS overrides the defaults.
+ if (CMAKE_CXX_FLAGS MATCHES "-stdlib=([a-zA-Z+]*)")
+ set(${var}_LIBNAME "${CMAKE_MATCH_1}")
+ set(${var}_SYSTEM 1)
+ elseif (${var} STREQUAL "default")
if (APPLE OR CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
set(${var}_LIBNAME "libc++")
set(${var}_SYSTEM 1)
add_definitions(-D__func__=__FUNCTION__)
endif()
-# Provide some common commmandline flags for Sanitizer runtimes.
+# Provide some common commandline flags for Sanitizer runtimes.
+if("${ANDROID_API_LEVEL}" GREATER_EQUAL 29)
+ list(APPEND SANITIZER_COMMON_CFLAGS -fno-emulated-tls)
+ string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " -fno-emulated-tls")
+endif()
if(NOT WIN32)
append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC SANITIZER_COMMON_CFLAGS)
endif()
append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 SANITIZER_COMMON_CFLAGS)
+if(CMAKE_CXX_COMPILER_ID MATCHES Clang)
+ list(APPEND THREAD_SAFETY_FLAGS
+ "-Werror=thread-safety"
+ "-Werror=thread-safety-reference"
+ "-Werror=thread-safety-beta"
+ )
+ list(APPEND SANITIZER_COMMON_CFLAGS ${THREAD_SAFETY_FLAGS})
+ string(REPLACE ";" " " thread_safety_flags_space_sep "${THREAD_SAFETY_FLAGS}")
+ string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${thread_safety_flags_space_sep}")
+endif()
+
# If we're using MSVC,
# always respect the optimization flags set by CMAKE_BUILD_TYPE instead.
if (NOT MSVC)
append_list_if(COMPILER_RT_HAS_WGNU_FLAG -Wno-gnu SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG -Wno-variadic-macros SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WC99_EXTENSIONS_FLAG -Wno-c99-extensions SANITIZER_COMMON_CFLAGS)
-append_list_if(COMPILER_RT_HAS_WNON_VIRTUAL_DTOR_FLAG -Wno-non-virtual-dtor SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WD4146_FLAG /wd4146 SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WD4291_FLAG /wd4291 SANITIZER_COMMON_CFLAGS)
append_list_if(COMPILER_RT_HAS_WD4391_FLAG /wd4391 SANITIZER_COMMON_CFLAGS)
endif()
append_list_if(COMPILER_RT_HAS_LIBC c SANITIZER_COMMON_LINK_LIBS)
-
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Fuchsia")
- list(APPEND SANITIZER_COMMON_LINK_FLAGS -Wl,-z,defs,-z,now,-z,relro)
list(APPEND SANITIZER_COMMON_LINK_LIBS zircon)
endif()
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Fuchsia")
+ set(SANITIZER_NO_UNDEFINED_SYMBOLS_DEFAULT ON)
+else()
+ set(SANITIZER_NO_UNDEFINED_SYMBOLS_DEFAULT OFF)
+endif()
+option(SANITIZER_NO_UNDEFINED_SYMBOLS "Report error on unresolved symbol references" ${SANITIZER_NO_UNDEFINED_SYMBOLS_DEFAULT})
+if (SANITIZER_NO_UNDEFINED_SYMBOLS)
+ list(APPEND SANITIZER_COMMON_LINK_FLAGS -Wl,-z,defs)
+endif()
+
+# TODO: COMPILER_RT_COMMON_CFLAGS and COMPILER_RT_COMMON_LINK_FLAGS are
+# intended for use in non-sanitizer runtimes such as libFuzzer, profile or XRay,
+# move these higher to include common flags, then derive SANITIZER_COMMON_CFLAGS
+# and SANITIZER_COMMON_LINK_FLAGS from those and append sanitizer-specific flags.
+set(COMPILER_RT_COMMON_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+set(COMPILER_RT_COMMON_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
+
+# We don't use the C++ standard library, so avoid including it by mistake.
+append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ SANITIZER_COMMON_CFLAGS)
+append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ SANITIZER_COMMON_LINK_FLAGS)
+
+# Remove -stdlib= which is unused when passing -nostdinc++...
+string(REGEX MATCHALL "-stdlib=[a-zA-Z+]*" stdlib_flag "${CMAKE_CXX_FLAGS}")
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
+# ...we need it to build some runtimes and tests so readd it where appropriate.
+list(APPEND COMPILER_RT_COMMON_CFLAGS ${stdlib_flag})
+list(APPEND COMPILER_RT_COMMON_LINK_FLAGS ${stdlib_flag})
+
macro(append_libcxx_libs var)
if (${var}_INTREE)
if (SANITIZER_USE_STATIC_LLVM_UNWINDER AND (TARGET unwind_static OR HAVE_LIBUNWIND))
append_list_if(COMPILER_RT_HAS_LIBSTDCXX stdc++ SANITIZER_TEST_CXX_LIBRARIES)
endif()
+# TODO: There's a lot of duplication across lib/*/tests/CMakeLists.txt files,
+# move some of the common flags to COMPILER_RT_UNITTEST_CFLAGS.
+
+# Unittests need access to C++ standard library.
+string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${stdlib_flag}")
+
+# When cross-compiling, COMPILER_RT_TEST_COMPILER_CFLAGS help in compilation
+# and linking of unittests.
+string(REPLACE " " ";" COMPILER_RT_UNITTEST_CFLAGS "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+set(COMPILER_RT_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_CFLAGS})
+
+# Unittests support.
+set(COMPILER_RT_GTEST_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest)
+set(COMPILER_RT_GTEST_SOURCE ${COMPILER_RT_GTEST_PATH}/src/gtest-all.cc)
+set(COMPILER_RT_GTEST_CFLAGS
+ -DGTEST_NO_LLVM_SUPPORT=1
+ -DGTEST_HAS_RTTI=0
+ -I${COMPILER_RT_GTEST_PATH}/include
+ -I${COMPILER_RT_GTEST_PATH}
+)
+if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+ # FreeBSD has its pthread functions marked with thread safety annotations, but
+ # googletest is not compatible with such annotations. Disable the thread
+ # safety warnings-as-errors until googletest has been fixed.
+ list(APPEND NO_THREAD_SAFETY_FLAGS ${THREAD_SAFETY_FLAGS})
+ list(TRANSFORM NO_THREAD_SAFETY_FLAGS REPLACE "error=" "no-")
+ list(APPEND COMPILER_RT_GTEST_CFLAGS ${NO_THREAD_SAFETY_FLAGS})
+endif()
+
+# Mocking support.
+set(COMPILER_RT_GMOCK_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock)
+set(COMPILER_RT_GMOCK_SOURCE ${COMPILER_RT_GMOCK_PATH}/src/gmock-all.cc)
+set(COMPILER_RT_GMOCK_CFLAGS
+ -DGTEST_NO_LLVM_SUPPORT=1
+ -DGTEST_HAS_RTTI=0
+ -I${COMPILER_RT_GMOCK_PATH}/include
+ -I${COMPILER_RT_GMOCK_PATH}
+)
+
+append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 COMPILER_RT_UNITTEST_CFLAGS)
+append_list_if(COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG -Wno-covered-switch-default COMPILER_RT_UNITTEST_CFLAGS)
+append_list_if(COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG -Wno-suggest-override COMPILER_RT_UNITTEST_CFLAGS)
+
+if(MSVC)
+ # gtest use a lot of stuff marked as deprecated on Windows.
+ list(APPEND COMPILER_RT_GTEST_CFLAGS -Wno-deprecated-declarations)
+endif()
+
# Warnings to turn off for all libraries, not just sanitizers.
append_string_if(COMPILER_RT_HAS_WUNUSED_PARAMETER_FLAG -Wno-unused-parameter CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /IGNORE:4221")
endif()
+if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_C_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
+ set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
+endif()
+
add_subdirectory(include)
option(COMPILER_RT_USE_LIBCXX
set(COMPILER_RT_HAS_LLD TRUE)
endif()
endif()
+
+if(ANDROID)
+ set(COMPILER_RT_HAS_LLD TRUE)
+ set(COMPILER_RT_TEST_USE_LLD TRUE)
+ append_list_if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG -fuse-ld=lld SANITIZER_COMMON_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LLD -fuse-ld=lld COMPILER_RT_UNITTEST_LINK_FLAGS)
+ if(COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
+ set(COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT FALSE)
+ endif()
+endif()
pythonize_bool(COMPILER_RT_HAS_LLD)
+pythonize_bool(COMPILER_RT_TEST_USE_LLD)
add_subdirectory(lib)
if(COMPILER_RT_INCLUDE_TESTS)
add_subdirectory(unittests)
add_subdirectory(test)
- if (COMPILER_RT_STANDALONE_BUILD)
+ # Don't build llvm-lit for runtimes-build, it will clean up map_config.
+ if (COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD)
# If we have a valid source tree, generate llvm-lit into the bin directory.
# The user can still choose to have the check targets *use* a different lit
# by specifying -DLLVM_EXTERNAL_LIT, but we generate it regardless.
function(add_asm_sources output)
set(${output} ${ARGN} PARENT_SCOPE)
- # Xcode will try to compile asm files as C ('clang -x c'), and that will fail.
- if (${CMAKE_GENERATOR} STREQUAL "Xcode")
- enable_language(ASM)
- else()
- # Pass ASM file directly to the C++ compiler.
+ # CMake doesn't pass the correct architecture for Apple prior to CMake 3.19. https://gitlab.kitware.com/cmake/cmake/-/issues/20771
+ # MinGW didn't work correctly with assembly prior to CMake 3.17. https://gitlab.kitware.com/cmake/cmake/-/merge_requests/4287 and https://reviews.llvm.org/rGb780df052dd2b246a760d00e00f7de9ebdab9d09
+ # Workaround these two issues by compiling as C.
+ # Same workaround used in libunwind. Also update there if changed here.
+ if((APPLE AND CMAKE_VERSION VERSION_LESS 3.19) OR (MINGW AND CMAKE_VERSION VERSION_LESS 3.17))
set_source_files_properties(${ARGN} PROPERTIES LANGUAGE C)
endif()
endfunction()
else()
if(ANDROID AND ${arch} STREQUAL "i386")
set(${output} "${name}-i686${COMPILER_RT_OS_SUFFIX}")
+ elseif("${arch}" MATCHES "^arm")
+ if(COMPILER_RT_DEFAULT_TARGET_ONLY)
+ set(triple "${COMPILER_RT_DEFAULT_TARGET_TRIPLE}")
+ else()
+ set(triple "${TARGET_TRIPLE}")
+ endif()
+ # When using arch-suffixed runtime library names, clang only looks for
+ # libraries named "arm" or "armhf", see getArchNameForCompilerRTLib in
+ # clang. Therefore, try to inspect both the arch name and the triple
+ # if it seems like we're building an armhf target.
+ if ("${arch}" MATCHES "hf$" OR "${triple}" MATCHES "hf$")
+ set(${output} "${name}-armhf${COMPILER_RT_OS_SUFFIX}")
+ else()
+ set(${output} "${name}-arm${COMPILER_RT_OS_SUFFIX}")
+ endif()
else()
set(${output} "${name}-${arch}${COMPILER_RT_OS_SUFFIX}")
endif()
# Adds static or shared runtime for a list of architectures and operating
# systems and puts it in the proper directory in the build and install trees.
# add_compiler_rt_runtime(<name>
-# {OBJECT|STATIC|SHARED}
+# {OBJECT|STATIC|SHARED|MODULE}
# ARCHS <architectures>
# OS <os list>
# SOURCES <source files>
# CFLAGS <compile flags>
# LINK_FLAGS <linker flags>
# DEFS <compile definitions>
+# DEPS <dependencies>
# LINK_LIBS <linked libraries> (only for shared library)
# OBJECT_LIBS <object libraries to use as sources>
# PARENT_TARGET <convenience parent target>
# ADDITIONAL_HEADERS <header files>)
function(add_compiler_rt_runtime name type)
- if(NOT type MATCHES "^(OBJECT|STATIC|SHARED)$")
- message(FATAL_ERROR "type argument must be OBJECT, STATIC or SHARED")
+ if(NOT type MATCHES "^(OBJECT|STATIC|SHARED|MODULE)$")
+ message(FATAL_ERROR
+ "type argument must be OBJECT, STATIC, SHARED or MODULE")
return()
endif()
cmake_parse_arguments(LIB
""
"PARENT_TARGET"
- "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS"
+ "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS"
${ARGN})
set(libnames)
# Until we support this some other way, build compiler-rt runtime without LTO
RUNTIME DESTINATION ${install_dir_${libname}}
${COMPONENT_OPTION})
endif()
+ if(LIB_DEPS)
+ add_dependencies(${libname} ${LIB_DEPS})
+ endif()
set_target_properties(${libname} PROPERTIES
OUTPUT_NAME ${output_name_${libname}})
set_target_properties(${libname} PROPERTIES FOLDER "Compiler-RT Runtime")
add_custom_command(TARGET ${libname}
POST_BUILD
COMMAND codesign --sign - $<TARGET_FILE:${libname}>
- WORKING_DIRECTORY ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
+ WORKING_DIRECTORY ${COMPILER_RT_OUTPUT_LIBRARY_DIR}
)
endif()
endif()
endif()
endfunction()
-# when cross compiling, COMPILER_RT_TEST_COMPILER_CFLAGS help
-# in compilation and linking of unittests.
-string(REPLACE " " ";" COMPILER_RT_UNITTEST_CFLAGS "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
-set(COMPILER_RT_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_CFLAGS})
-
-# Unittests support.
-set(COMPILER_RT_GTEST_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googletest)
-set(COMPILER_RT_GTEST_SOURCE ${COMPILER_RT_GTEST_PATH}/src/gtest-all.cc)
-set(COMPILER_RT_GTEST_CFLAGS
- -DGTEST_NO_LLVM_SUPPORT=1
- -DGTEST_HAS_RTTI=0
- -I${COMPILER_RT_GTEST_PATH}/include
- -I${COMPILER_RT_GTEST_PATH}
-)
-
-# Mocking support.
-set(COMPILER_RT_GMOCK_PATH ${LLVM_MAIN_SRC_DIR}/utils/unittest/googlemock)
-set(COMPILER_RT_GMOCK_SOURCE ${COMPILER_RT_GMOCK_PATH}/src/gmock-all.cc)
-set(COMPILER_RT_GMOCK_CFLAGS
- -DGTEST_NO_LLVM_SUPPORT=1
- -DGTEST_HAS_RTTI=0
- -I${COMPILER_RT_GMOCK_PATH}/include
- -I${COMPILER_RT_GMOCK_PATH}
-)
-
-append_list_if(COMPILER_RT_DEBUG -DSANITIZER_DEBUG=1 COMPILER_RT_UNITTEST_CFLAGS)
-append_list_if(COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG -Wno-covered-switch-default COMPILER_RT_UNITTEST_CFLAGS)
-
-if(MSVC)
- # gtest use a lot of stuff marked as deprecated on Windows.
- list(APPEND COMPILER_RT_GTEST_CFLAGS -Wno-deprecated-declarations)
-endif()
-
# Compile and register compiler-rt tests.
# generate_compiler_rt_tests(<output object files> <test_suite> <test_name>
# <test architecture>
add_custom_target(${target_name} DEPENDS ${dst_file})
# Install in Clang resource directory.
install(FILES ${file_name}
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/share
+ DESTINATION ${COMPILER_RT_INSTALL_DATA_DIR}
COMPONENT ${component})
add_dependencies(${component} ${target_name})
add_custom_target(${name} DEPENDS ${dst})
install(FILES ${dst}
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/bin)
+ DESTINATION ${COMPILER_RT_INSTALL_BINARY_DIR})
endmacro(add_compiler_rt_script src name)
# Builds custom version of libc++ and installs it in <prefix>.
if(LIBCXX_USE_TOOLCHAIN)
set(compiler_args -DCMAKE_C_COMPILER=${COMPILER_RT_TEST_COMPILER}
-DCMAKE_CXX_COMPILER=${COMPILER_RT_TEST_CXX_COMPILER})
- if(NOT COMPILER_RT_STANDALONE_BUILD AND NOT RUNTIMES_BUILD)
+ if(NOT COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD)
set(toolchain_deps $<TARGET_FILE:clang>)
set(force_deps DEPENDS $<TARGET_FILE:clang>)
endif()
CMAKE_OBJDUMP
CMAKE_STRIP
CMAKE_SYSROOT
+ LIBCXX_HAS_MUSL_LIBC
+ PYTHON_EXECUTABLE
+ Python3_EXECUTABLE
+ Python2_EXECUTABLE
CMAKE_SYSTEM_NAME)
foreach(variable ${PASSTHROUGH_VARIABLES})
get_property(is_value_set CACHE ${variable} PROPERTY VALUE SET)
--- /dev/null
+include(CMakeParseArguments)
+include(CompilerRTUtils)
+
+function(get_aix_libatomic_default_link_flags link_flags export_list)
+ set(linkopts
+ "-Wl,-H512 -Wl,-D0 \
+ -Wl,-T512 -Wl,-bhalt:4 -Wl,-bernotok \
+ -Wl,-bnoentry -Wl,-bexport:${export_list} \
+ -Wl,-bmodtype:SRE -Wl,-lc")
+ # Add `-Wl,-G`. Quoted from release notes of cmake-3.16.0
+ # > On AIX, runtime linking is no longer enabled by default.
+ # See https://cmake.org/cmake/help/latest/release/3.16.html
+ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.16.0")
+ set(linkopts "-Wl,-G" "${linkopts}")
+ endif()
+ set(${link_flags} ${linkopts} PARENT_SCOPE)
+endfunction()
+
+function(get_aix_libatomic_type type)
+ if(${CMAKE_VERSION} VERSION_LESS "3.16.0")
+ set(${type} SHARED PARENT_SCOPE)
+ else()
+ set(${type} MODULE PARENT_SCOPE)
+ endif()
+endfunction()
+
+macro(archive_aix_libatomic name)
+ cmake_parse_arguments(LIB
+ ""
+ ""
+ "ARCHS;PARENT_TARGET"
+ ${ARGN})
+ set(shared_libraries_to_archive "")
+ foreach (arch ${LIB_ARCHS})
+ if(CAN_TARGET_${arch})
+ set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/libatomic-${arch}.dir")
+ # FIXME: Target name should be kept consistent with definition
+ # in AddCompilerRT.cmake added by
+ # add_compiler_rt_runtime(<name> SHARED ...)
+ set(target ${name}-dynamic-${arch})
+ if(TARGET ${target})
+ file(MAKE_DIRECTORY ${output_dir})
+ add_custom_command(OUTPUT "${output_dir}/libatomic.so.1"
+ POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E
+ copy "$<TARGET_FILE:${target}>"
+ "${output_dir}/libatomic.so.1"
+ # If built with MODULE, F_LOADONLY is set.
+ # We have to remove this flag at POST_BUILD.
+ COMMAND ${CMAKE_STRIP} -X32_64 -E
+ "${output_dir}/libatomic.so.1"
+ DEPENDS ${target})
+ list(APPEND shared_libraries_to_archive "${output_dir}/libatomic.so.1")
+ endif()
+ endif()
+ endforeach()
+ if(shared_libraries_to_archive)
+ set(output_dir "")
+ set(install_dir "")
+ # If LLVM defines top level library directory, we want to deliver
+ # libatomic.a at top level. See `llvm/cmake/modules/AddLLVM.cmake'
+ # setting _install_rpath on AIX for reference.
+ if(LLVM_LIBRARY_OUTPUT_INTDIR AND CMAKE_INSTALL_PREFIX)
+ set(output_dir "${LLVM_LIBRARY_OUTPUT_INTDIR}")
+ set(install_dir "${CMAKE_INSTALL_PREFIX}/lib${LLVM_LIBDIR_SUFFIX}")
+ else()
+ get_compiler_rt_output_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} output_dir)
+ get_compiler_rt_install_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} install_dir)
+ endif()
+ add_custom_command(OUTPUT "${output_dir}/libatomic.a"
+ COMMAND ${CMAKE_AR} -X32_64 r "${output_dir}/libatomic.a"
+ ${shared_libraries_to_archive}
+ DEPENDS ${shared_libraries_to_archive})
+ install(FILES "${output_dir}/libatomic.a"
+ DESTINATION ${install_dir})
+ add_custom_target(aix-libatomic
+ DEPENDS "${output_dir}/libatomic.a")
+ endif()
+ add_dependencies(${LIB_PARENT_TARGET} aix-libatomic)
+endmacro()
if (TARGET CompilerRTUnitTestCheckCxx)
list(APPEND SOURCE_DEPS CompilerRTUnitTestCheckCxx)
endif()
- string(REGEX MATCH "[.](cc|cpp)$" is_cxx ${source_rpath})
- string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
- if(is_cxx)
- string(REPLACE " " ";" global_flags "${CMAKE_CXX_FLAGS}")
- else()
- string(REPLACE " " ";" global_flags "${CMAKE_C_FLAGS}")
- endif()
+ if(COMPILER_RT_STANDALONE_BUILD)
+ # Only add global flags in standalone build.
+ string(REGEX MATCH "[.](cc|cpp)$" is_cxx ${source_rpath})
+ if(is_cxx)
+ string(REPLACE " " ";" global_flags "${CMAKE_CXX_FLAGS}")
+ else()
+ string(REPLACE " " ";" global_flags "${CMAKE_C_FLAGS}")
+ endif()
- if (MSVC)
- translate_msvc_cflags(global_flags "${global_flags}")
- endif()
+ if (MSVC)
+ translate_msvc_cflags(global_flags "${global_flags}")
+ endif()
- if (APPLE)
- set(global_flags ${OSX_SYSROOT_FLAG} ${global_flags})
+ if (APPLE)
+ set(global_flags ${OSX_SYSROOT_FLAG} ${global_flags})
+ endif()
+
+ # Ignore unknown warnings. CMAKE_CXX_FLAGS may contain GCC-specific options
+ # which are not supported by Clang.
+ list(APPEND global_flags -Wno-unknown-warning-option)
+ set(compile_flags ${global_flags} ${SOURCE_CFLAGS})
+ else()
+ set(compile_flags ${SOURCE_CFLAGS})
endif()
+
+ string(REGEX MATCH "[.](m|mm)$" is_objc ${source_rpath})
if (is_objc)
- list(APPEND global_flags -ObjC)
+ list(APPEND compile_flags "-ObjC")
endif()
- # Ignore unknown warnings. CMAKE_CXX_FLAGS may contain GCC-specific options
- # which are not supported by Clang.
- list(APPEND global_flags -Wno-unknown-warning-option)
- set(compile_flags ${global_flags} ${SOURCE_CFLAGS})
add_custom_command(
OUTPUT ${object_file}
COMMAND ${COMPILER_RT_TEST_COMPILER} ${compile_flags} -c
COMMAND bash -c "${CMD}"
COMMENT "Checking that just-built clang can find C++ headers..."
VERBATIM)
- if (NOT COMPILER_RT_STANDALONE_BUILD AND NOT RUNTIMES_BUILD)
+ if (NOT COMPILER_RT_STANDALONE_BUILD AND NOT LLVM_RUNTIMES_BUILD)
ADD_DEPENDENCIES(CompilerRTUnitTestCheckCxx clang)
endif()
endif()
endforeach(cflag)
endif()
+ if ("${LIB_OS}" MATCHES ".*sim$")
+ # Pass an explicit -simulator environment to the -target option to ensure
+ # that we don't rely on the architecture to infer whether we're building
+ # for the simulator.
+ string(REGEX REPLACE "sim" "" base_os "${LIB_OS}")
+ list(APPEND builtin_cflags
+ -target "${LIB_ARCH}-apple-${base_os}${DARWIN_${LIBOS}_BUILTIN_MIN_VER}-simulator")
+ endif()
+
set_target_compile_flags(${libname}
${sysroot_flag}
${DARWIN_${LIB_OS}_BUILTIN_MIN_VER_FLAG}
set(CMAKE_CXX_FLAGS "")
set(CMAKE_ASM_FLAGS "")
- set(PROFILE_SOURCES ../profile/InstrProfiling
- ../profile/InstrProfilingBuffer
- ../profile/InstrProfilingPlatformDarwin
- ../profile/InstrProfilingWriter)
+ append_string_if(COMPILER_RT_HAS_ASM_LSE " -DHAS_ASM_LSE" CFLAGS)
+
+ set(PROFILE_SOURCES ../profile/InstrProfiling.c
+ ../profile/InstrProfilingBuffer.c
+ ../profile/InstrProfilingPlatformDarwin.c
+ ../profile/InstrProfilingWriter.c
+ ../profile/InstrProfilingInternal.c
+ ../profile/InstrProfilingVersionVar.c)
foreach (os ${ARGN})
list_intersect(DARWIN_BUILTIN_ARCHS DARWIN_${os}_BUILTIN_ARCHS BUILTIN_SUPPORTED_ARCH)
+
+ if((arm64 IN_LIST DARWIN_BUILTIN_ARCHS OR arm64e IN_LIST DARWIN_BUILTIN_ARCHS) AND NOT TARGET lse_builtin_symlinks)
+ add_custom_target(
+ lse_builtin_symlinks
+ BYPRODUCTS ${lse_builtins}
+ ${arm64_lse_commands}
+ )
+
+ set(deps_arm64 lse_builtin_symlinks)
+ set(deps_arm64e lse_builtin_symlinks)
+ endif()
+
foreach (arch ${DARWIN_BUILTIN_ARCHS})
darwin_find_excluded_builtins_list(${arch}_${os}_EXCLUDED_BUILTINS
OS ${os}
darwin_add_builtin_library(clang_rt builtins
OS ${os}
ARCH ${arch}
+ DEPS ${deps_${arch}}
SOURCES ${filtered_sources}
CFLAGS ${CFLAGS} -arch ${arch}
PARENT_TARGET builtins)
darwin_add_builtin_library(clang_rt cc_kext
OS ${os}
ARCH ${arch}
+ DEPS ${deps_${arch}}
SOURCES ${filtered_sources} ${PROFILE_SOURCES}
CFLAGS ${CFLAGS} -arch ${arch} -mkernel
DEFS KERNEL_USE
PARENT_TARGET builtins
LIPO_FLAGS ${${os}_cc_kext_lipo_flags}
DEPENDS ${${os}_cc_kext_libs}
- OUTPUT_DIR ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
- INSTALL_DIR ${COMPILER_RT_LIBRARY_INSTALL_DIR})
+ OUTPUT_DIR ${COMPILER_RT_OUTPUT_LIBRARY_DIR}
+ INSTALL_DIR ${COMPILER_RT_INSTALL_LIBRARY_DIR})
endif()
endforeach()
- # We put the x86 sim slices into the archives for their base OS
foreach (os ${ARGN})
- if(NOT ${os} MATCHES ".*sim$")
- darwin_lipo_libs(clang_rt.${os}
- PARENT_TARGET builtins
- LIPO_FLAGS ${${os}_builtins_lipo_flags} ${${os}sim_builtins_lipo_flags}
- DEPENDS ${${os}_builtins_libs} ${${os}sim_builtins_libs}
- OUTPUT_DIR ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
- INSTALL_DIR ${COMPILER_RT_LIBRARY_INSTALL_DIR})
- endif()
+ darwin_lipo_libs(clang_rt.${os}
+ PARENT_TARGET builtins
+ LIPO_FLAGS ${${os}_builtins_lipo_flags}
+ DEPENDS ${${os}_builtins_libs}
+ OUTPUT_DIR ${COMPILER_RT_OUTPUT_LIBRARY_DIR}
+ INSTALL_DIR ${COMPILER_RT_INSTALL_LIBRARY_DIR})
endforeach()
darwin_add_embedded_builtin_libraries()
endmacro()
set(DARWIN_macho_embedded_ARCHS armv6m armv7m armv7em armv7 i386 x86_64)
set(DARWIN_macho_embedded_LIBRARY_OUTPUT_DIR
- ${COMPILER_RT_OUTPUT_DIR}/lib/macho_embedded)
+ ${COMPILER_RT_OUTPUT_LIBRARY_DIR}/macho_embedded)
set(DARWIN_macho_embedded_LIBRARY_INSTALL_DIR
- ${COMPILER_RT_INSTALL_PATH}/lib/macho_embedded)
+ ${COMPILER_RT_INSTALL_LIBRARY_DIR}/macho_embedded)
set(CFLAGS_armv7 "-target thumbv7-apple-darwin-eabi")
set(CFLAGS_i386 "-march=pentium")
--- /dev/null
+# This macro mocks enough of the changes `LLVMConfig.cmake` makes so that
+# compiler-rt can successfully configure itself when a LLVM toolchain is
+# available but the corresponding CMake build files are not.
+#
+# The motivation for this is to be able to generate the compiler-rt
+# lit tests suites and run them against an arbitrary LLVM toolchain
+# which doesn't ship the LLVM CMake build files.
+macro(compiler_rt_mock_llvm_cmake_config)
+ message(STATUS "Attempting to mock the changes made by LLVMConfig.cmake")
+ compiler_rt_mock_llvm_cmake_config_set_cmake_path()
+ compiler_rt_mock_llvm_cmake_config_set_target_triple()
+ compiler_rt_mock_llvm_cmake_config_include_cmake_files()
+endmacro()
+
+macro(compiler_rt_mock_llvm_cmake_config_set_cmake_path)
+ # Point `LLVM_CMAKE_PATH` at the source tree in the monorepo.
+ set(LLVM_CMAKE_PATH "${LLVM_MAIN_SRC_DIR}/cmake/modules")
+ if (NOT EXISTS "${LLVM_CMAKE_PATH}")
+ message(FATAL_ERROR "LLVM_CMAKE_PATH (${LLVM_CMAKE_PATH}) does not exist")
+ endif()
+ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
+ message(STATUS "LLVM_CMAKE_PATH: \"${LLVM_CMAKE_PATH}\"")
+endmacro()
+
+function(compiler_rt_mock_llvm_cmake_config_set_target_triple)
+ # Various bits of compiler-rt depend on the `TARGET_TRIPLE`variable being
+ # defined. This function tries to set a sensible value for the variable.
+ # This is a function rather than a macro to avoid polluting the variable
+ # namespace.
+ set(COMPILER_OUTPUT "")
+
+ # If the user provides `COMPILER_RT_DEFAULT_TARGET_ONLY` and `CMAKE_C_COMPILER_TARGET`
+ # (see `construct_compiler_rt_default_triple`) then prefer that to examining the
+ # compiler.
+ if (COMPILER_RT_DEFAULT_TARGET_ONLY)
+ if (NOT "${CMAKE_C_COMPILER_TARGET}" STREQUAL "")
+ message(STATUS
+ "Using CMAKE_C_COMPILER_TARGET (${CMAKE_C_COMPILER_TARGET}) as TARGET_TRIPLE")
+ endif()
+ set(COMPILER_OUTPUT "${CMAKE_C_COMPILER_TARGET}")
+ endif()
+
+ # Try asking the compiler for its default target triple.
+ set(HAD_ERROR FALSE)
+ if ("${COMPILER_OUTPUT}" STREQUAL "")
+ if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang|GNU")
+ # Note: Clang also supports `-print-target-triple` but gcc doesn't
+ # support this flag.
+ execute_process(
+ COMMAND "${CMAKE_C_COMPILER}" -dumpmachine
+ RESULT_VARIABLE HAD_ERROR
+ OUTPUT_VARIABLE COMPILER_OUTPUT
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+ else()
+ message(FATAL_ERROR
+ "Fetching target triple from compiler \"${CMAKE_C_COMPILER_ID}\" "
+ "is not implemented.")
+ endif()
+ endif()
+
+ if (HAD_ERROR)
+ message(FATAL_ERROR "Fetching target triple from compiler failed")
+ endif()
+ set(TARGET_TRIPLE "${COMPILER_OUTPUT}")
+ message(STATUS "TARGET_TRIPLE: \"${TARGET_TRIPLE}\"")
+ if ("${TARGET_TRIPLE}" STREQUAL "")
+ message(FATAL_ERROR "TARGET_TRIPLE cannot be empty")
+ endif()
+ set(TARGET_TRIPLE "${TARGET_TRIPLE}" PARENT_SCOPE)
+endfunction()
+
+macro(compiler_rt_mock_llvm_cmake_config_include_cmake_files)
+ # Some compiler-rt CMake code needs to call code in this file.
+ include("${LLVM_CMAKE_PATH}/AddLLVM.cmake")
+endmacro()
check_symbol_exists(__i386__ "" __I386)
check_symbol_exists(__mips__ "" __MIPS)
check_symbol_exists(__mips64__ "" __MIPS64)
+ check_symbol_exists(__powerpc__ "" __PPC)
check_symbol_exists(__powerpc64__ "" __PPC64)
check_symbol_exists(__powerpc64le__ "" __PPC64LE)
check_symbol_exists(__riscv "" __RISCV)
elseif(__AARCH64)
add_default_target_arch(aarch64)
elseif(__X86_64)
- add_default_target_arch(x86_64)
+ if(CMAKE_SIZEOF_VOID_P EQUAL "4")
+ add_default_target_arch(x32)
+ elseif(CMAKE_SIZEOF_VOID_P EQUAL "8")
+ add_default_target_arch(x86_64)
+ else()
+ message(FATAL_ERROR "Unsupported pointer size for X86_64")
+ endif()
elseif(__I386)
add_default_target_arch(i386)
elseif(__MIPS64) # must be checked before __MIPS
add_default_target_arch(mips64)
elseif(__MIPS)
add_default_target_arch(mips)
- elseif(__PPC64)
+ elseif(__PPC64) # must be checked before __PPC
add_default_target_arch(powerpc64)
elseif(__PPC64LE)
add_default_target_arch(powerpc64le)
+ elseif(__PPC)
+ add_default_target_arch(powerpc)
elseif(__RISCV)
if(CMAKE_SIZEOF_VOID_P EQUAL "4")
add_default_target_arch(riscv32)
endif()
endmacro()
+function(get_compiler_rt_root_source_dir ROOT_DIR_VAR)
+ # Compute the path to the root of the Compiler-RT source tree
+ # regardless of how the project was configured.
+ #
+ # This function is useful because using `${CMAKE_SOURCE_DIR}`
+ # is error prone due to the numerous ways Compiler-RT can be
+ # configured.
+ #
+ # `ROOT_DIR_VAR` - the name of the variable to write the result to.
+ #
+ # TODO(dliew): When CMake min version is 3.17 or newer use
+ # `CMAKE_CURRENT_FUNCTION_LIST_DIR` instead.
+ if ("${ROOT_DIR_VAR}" STREQUAL "")
+ message(FATAL_ERROR "ROOT_DIR_VAR cannot be empty")
+ endif()
+
+ # Compiler-rt supports different source root paths.
+ # Handle each case here.
+ set(PATH_TO_COMPILER_RT_SOURCE_ROOT "")
+ if (DEFINED CompilerRTBuiltins_SOURCE_DIR)
+ # Compiler-RT Builtins standalone build.
+ # `llvm-project/compiler-rt/lib/builtins`
+ set(PATH_TO_COMPILER_RT_SOURCE_ROOT "${CompilerRTBuiltins_SOURCE_DIR}/../../")
+ elseif(DEFINED CompilerRT_SOURCE_DIR)
+ # Compiler-RT standalone build.
+ # `llvm-project/compiler-rt`
+ set(PATH_TO_COMPILER_RT_SOURCE_ROOT "${CompilerRT_SOURCE_DIR}")
+ elseif (EXISTS "${CMAKE_SOURCE_DIR}/../compiler-rt")
+ # In tree build with LLVM as the root project.
+ # See `llvm-project/projects/`.
+ # Assumes monorepo layout.
+ set(PATH_TO_COMPILER_RT_SOURCE_ROOT "${CMAKE_SOURCE_DIR}/../compiler-rt")
+ else()
+ message(FATAL_ERROR "Unhandled Compiler-RT source root configuration.")
+ endif()
+
+ get_filename_component(ROOT_DIR "${PATH_TO_COMPILER_RT_SOURCE_ROOT}" ABSOLUTE)
+ if (NOT EXISTS "${ROOT_DIR}")
+ message(FATAL_ERROR "Path \"${ROOT_DIR}\" doesn't exist")
+ endif()
+
+ # Sanity check: Make sure we can locate the current source file via the
+ # computed path.
+ set(PATH_TO_CURRENT_FILE "${ROOT_DIR}/cmake/Modules/CompilerRTUtils.cmake")
+ if (NOT EXISTS "${PATH_TO_CURRENT_FILE}")
+ message(FATAL_ERROR "Could not find \"${PATH_TO_CURRENT_FILE}\"")
+ endif()
+
+ set("${ROOT_DIR_VAR}" "${ROOT_DIR}" PARENT_SCOPE)
+endfunction()
+
macro(load_llvm_config)
if (NOT LLVM_CONFIG_PATH)
find_program(LLVM_CONFIG_PATH "llvm-config"
"Reconfigure with -DLLVM_CONFIG_PATH=path/to/llvm-config.")
endif()
endif()
+
+ # Compute path to LLVM sources assuming the monorepo layout.
+ # We don't set `LLVM_MAIN_SRC_DIR` directly to avoid overriding a user provided
+ # CMake cache value.
+ get_compiler_rt_root_source_dir(COMPILER_RT_ROOT_SRC_PATH)
+ get_filename_component(LLVM_MAIN_SRC_DIR_DEFAULT "${COMPILER_RT_ROOT_SRC_PATH}/../llvm" ABSOLUTE)
+ if (NOT EXISTS "${LLVM_MAIN_SRC_DIR_DEFAULT}")
+ # TODO(dliew): Remove this legacy fallback path.
+ message(WARNING
+ "LLVM source tree not found at \"${LLVM_MAIN_SRC_DIR_DEFAULT}\". "
+ "You are not using the monorepo layout. This configuration is DEPRECATED.")
+ endif()
+
+ set(FOUND_LLVM_CMAKE_PATH FALSE)
if (LLVM_CONFIG_PATH)
execute_process(
COMMAND ${LLVM_CONFIG_PATH} "--obj-root" "--bindir" "--libdir" "--src-root" "--includedir"
set(LLVM_BINARY_DIR ${BINARY_DIR} CACHE PATH "Path to LLVM build tree")
set(LLVM_LIBRARY_DIR ${LIBRARY_DIR} CACHE PATH "Path to llvm/lib")
- set(LLVM_MAIN_SRC_DIR ${MAIN_SRC_DIR} CACHE PATH "Path to LLVM source tree")
set(LLVM_TOOLS_BINARY_DIR ${TOOLS_BINARY_DIR} CACHE PATH "Path to llvm/bin")
set(LLVM_INCLUDE_DIR ${INCLUDE_DIR} CACHE PATH "Paths to LLVM headers")
+ if (NOT EXISTS "${LLVM_MAIN_SRC_DIR_DEFAULT}")
+ # TODO(dliew): Remove this legacy fallback path.
+ message(WARNING
+ "Consulting llvm-config for the LLVM source path "
+ "as a fallback. This behavior will be removed in the future.")
+ # We don't set `LLVM_MAIN_SRC_DIR` directly to avoid overriding a user
+ # provided CMake cache value.
+ set(LLVM_MAIN_SRC_DIR_DEFAULT "${MAIN_SRC_DIR}")
+ message(STATUS "Using LLVM source path (${LLVM_MAIN_SRC_DIR_DEFAULT}) from llvm-config")
+ endif()
+
# Detect if we have the LLVMXRay and TestingSupport library installed and
# available from llvm-config.
execute_process(
set(LLVM_CMAKE_PATH "${LLVM_BINARY_DIR_CMAKE_STYLE}/lib${LLVM_LIBDIR_SUFFIX}/cmake/llvm")
endif()
- list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
- # Get some LLVM variables from LLVMConfig.
- include("${LLVM_CMAKE_PATH}/LLVMConfig.cmake")
+ set(LLVM_CMAKE_INCLUDE_FILE "${LLVM_CMAKE_PATH}/LLVMConfig.cmake")
+ if (EXISTS "${LLVM_CMAKE_INCLUDE_FILE}")
+ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}")
+ # Get some LLVM variables from LLVMConfig.
+ include("${LLVM_CMAKE_INCLUDE_FILE}")
+ set(FOUND_LLVM_CMAKE_PATH TRUE)
+ else()
+ set(FOUND_LLVM_CMAKE_PATH FALSE)
+ message(WARNING "LLVM CMake path (${LLVM_CMAKE_INCLUDE_FILE}) reported by llvm-config does not exist")
+ endif()
+ unset(LLVM_CMAKE_INCLUDE_FILE)
set(LLVM_LIBRARY_OUTPUT_INTDIR
${LLVM_BINARY_DIR}/${CMAKE_CFG_INTDIR}/lib${LLVM_LIBDIR_SUFFIX})
endif()
+
+ # Finally set the cache variable now that `llvm-config` has also had a chance
+ # to set `LLVM_MAIN_SRC_DIR_DEFAULT`.
+ set(LLVM_MAIN_SRC_DIR "${LLVM_MAIN_SRC_DIR_DEFAULT}" CACHE PATH "Path to LLVM source tree")
+ message(STATUS "LLVM_MAIN_SRC_DIR: \"${LLVM_MAIN_SRC_DIR}\"")
+ if (NOT EXISTS "${LLVM_MAIN_SRC_DIR}")
+ # TODO(dliew): Make this a hard error
+ message(WARNING "LLVM_MAIN_SRC_DIR (${LLVM_MAIN_SRC_DIR}) does not exist. "
+ "You can override the inferred path by adding "
+ "`-DLLVM_MAIN_SRC_DIR=<path_to_llvm_src>` to your CMake invocation "
+ "where `<path_to_llvm_src>` is the path to the `llvm` directory in "
+ "the `llvm-project` repo. "
+ "This will be treated as error in the future.")
+ endif()
+
+ if (NOT FOUND_LLVM_CMAKE_PATH)
+ # This configuration tries to configure without the prescence of `LLVMConfig.cmake`. It is
+ # intended for testing purposes (generating the lit test suites) and will likely not support
+ # a build of the runtimes in compiler-rt.
+ include(CompilerRTMockLLVMCMakeConfig)
+ compiler_rt_mock_llvm_cmake_config()
+ endif()
+
endmacro()
macro(construct_compiler_rt_default_triple)
string(REPLACE "-" ";" TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE})
list(GET TARGET_TRIPLE_LIST 0 COMPILER_RT_DEFAULT_TARGET_ARCH)
+
+ # Map various forms of the architecture names to the canonical forms
+ # (as they are used by clang, see getArchNameForCompilerRTLib).
+ if("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "^i.86$")
+ # Android uses i686, but that's remapped at a later stage.
+ set(COMPILER_RT_DEFAULT_TARGET_ARCH "i386")
+ endif()
+
# Determine if test target triple is specified explicitly, and doesn't match the
# default.
if(NOT COMPILER_RT_DEFAULT_TARGET_TRIPLE STREQUAL TARGET_TRIPLE)
function(get_compiler_rt_install_dir arch install_dir)
if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
get_compiler_rt_target(${arch} target)
- set(${install_dir} ${COMPILER_RT_INSTALL_PATH}/lib/${target} PARENT_SCOPE)
+ set(${install_dir} ${COMPILER_RT_INSTALL_LIBRARY_DIR}/${target} PARENT_SCOPE)
else()
- set(${install_dir} ${COMPILER_RT_LIBRARY_INSTALL_DIR} PARENT_SCOPE)
+ set(${install_dir} ${COMPILER_RT_INSTALL_LIBRARY_DIR} PARENT_SCOPE)
endif()
endfunction()
function(get_compiler_rt_output_dir arch output_dir)
if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
get_compiler_rt_target(${arch} target)
- set(${output_dir} ${COMPILER_RT_OUTPUT_DIR}/lib/${target} PARENT_SCOPE)
+ set(${output_dir} ${COMPILER_RT_OUTPUT_LIBRARY_DIR}/${target} PARENT_SCOPE)
else()
- set(${output_dir} ${COMPILER_RT_LIBRARY_OUTPUT_DIR} PARENT_SCOPE)
+ set(${output_dir} ${COMPILER_RT_OUTPUT_LIBRARY_DIR} PARENT_SCOPE)
endif()
endfunction()
-cmake_minimum_required(VERSION 3.4.3)
+# TODO(phosek): We should use the runtimes build instead configured with
+# LLVM_ENABLE_RUNTIMES=libcxxabi;libcxx to avoid duplication of logic.
+
+cmake_minimum_required(VERSION 3.13.4)
project(custom-libcxx C CXX)
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
# Build static libcxxabi.
-set(LIBCXXABI_STANDALONE_BUILD 1)
set(LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "")
set(LIBCXXABI_ENABLE_EXCEPTIONS ON CACHE BOOL "")
set(LIBCXXABI_HERMETIC_STATIC_LIBRARY ON CACHE STRING "")
add_subdirectory(${COMPILER_RT_LIBCXXABI_PATH} ${CMAKE_CURRENT_BINARY_DIR}/cxxabi)
# Build static libcxx without exceptions.
-set(LIBCXX_STANDALONE_BUILD 1)
set(LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY OFF CACHE BOOL "")
set(LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
set(LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
set(CMAKE_${lang}_CREATE_STATIC_LIBRARY
"\"${CMAKE_LIBTOOL}\" -static ${LIBTOOL_NO_WARNING_FLAG} -o <TARGET> <LINK_FLAGS> <OBJECTS>")
endforeach()
+
+ # By default, CMake invokes ranlib on a static library after installing it.
+ # libtool will have produced the table of contents for us already, and ranlib
+ # does not understanding universal binaries, so skip this step. It's important
+ # to set it to empty instead of unsetting it to shadow the cache variable, and
+ # we don't want to unset the cache variable to not affect anything outside
+ # this scope.
+ set(CMAKE_RANLIB "")
endif()
# If DYLD_LIBRARY_PATH is set we need to set it on archiver commands
include(CheckIncludeFile)
include(CheckCXXSourceCompiles)
-include(TestBigEndian)
check_include_file(unwind.h HAVE_UNWIND_H)
"Path where built compiler-rt libraries should be stored.")
set(COMPILER_RT_EXEC_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin CACHE PATH
"Path where built compiler-rt executables should be stored.")
- set(COMPILER_RT_INSTALL_PATH ${CMAKE_INSTALL_PREFIX} CACHE PATH
- "Path where built compiler-rt libraries should be installed.")
+ set(COMPILER_RT_INSTALL_PATH "" CACHE PATH
+ "Prefix for directories where built compiler-rt artifacts should be installed.")
option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt unit tests." OFF)
option(COMPILER_RT_ENABLE_WERROR "Fail and stop if warning is triggered" OFF)
# Use a host compiler to compile/link tests.
set(COMPILER_RT_TEST_COMPILER_ID GNU)
endif()
+function(extend_install_path joined_path current_segment)
+ if("${current_segment}" STREQUAL "")
+ set(temp_path "${COMPILER_RT_INSTALL_PATH}")
+ elseif("${COMPILER_RT_INSTALL_PATH}" STREQUAL "")
+ set(temp_path "${current_segment}")
+ elseif(IS_ABSOLUTE "${current_segment}")
+ message(WARNING "Since \"${current_segment}\" is absolute, it overrides COMPILER_RT_INSTALL_PATH: \"${COMPILER_RT_INSTALL_PATH}\".")
+ set(temp_path "${current_segment}")
+ else()
+ set(temp_path "${COMPILER_RT_INSTALL_PATH}/${current_segment}")
+ endif()
+ set(${joined_path} "${temp_path}" PARENT_SCOPE)
+endfunction()
+
if(NOT DEFINED COMPILER_RT_OS_DIR)
string(TOLOWER ${CMAKE_SYSTEM_NAME} COMPILER_RT_OS_DIR)
endif()
if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
- set(COMPILER_RT_LIBRARY_OUTPUT_DIR
- ${COMPILER_RT_OUTPUT_DIR})
- set(COMPILER_RT_LIBRARY_INSTALL_DIR
- ${COMPILER_RT_INSTALL_PATH})
-else(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR)
- set(COMPILER_RT_LIBRARY_OUTPUT_DIR
+ set(COMPILER_RT_OUTPUT_LIBRARY_DIR
+ ${COMPILER_RT_OUTPUT_DIR}/lib)
+ extend_install_path(default_install_path lib)
+ set(COMPILER_RT_INSTALL_LIBRARY_DIR "${default_install_path}" CACHE PATH
+ "Path where built compiler-rt libraries should be installed.")
+else(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
+ set(COMPILER_RT_OUTPUT_LIBRARY_DIR
${COMPILER_RT_OUTPUT_DIR}/lib/${COMPILER_RT_OS_DIR})
- set(COMPILER_RT_LIBRARY_INSTALL_DIR
- ${COMPILER_RT_INSTALL_PATH}/lib/${COMPILER_RT_OS_DIR})
+ extend_install_path(default_install_path "lib/${COMPILER_RT_OS_DIR}")
+ set(COMPILER_RT_INSTALL_LIBRARY_DIR "${default_install_path}" CACHE PATH
+ "Path where built compiler-rt libraries should be installed.")
endif()
+extend_install_path(default_install_path bin)
+set(COMPILER_RT_INSTALL_BINARY_DIR "${default_install_path}" CACHE PATH
+ "Path where built compiler-rt executables should be installed.")
+extend_install_path(default_install_path include)
+set(COMPILER_RT_INSTALL_INCLUDE_DIR "${default_install_path}" CACHE PATH
+ "Path where compiler-rt headers should be installed.")
+extend_install_path(default_install_path share)
+set(COMPILER_RT_INSTALL_DATA_DIR "${default_install_path}" CACHE PATH
+ "Path where compiler-rt data files should be installed.")
if(APPLE)
# On Darwin if /usr/include/c++ doesn't exist, the user probably has Xcode but
add_default_target_arch(${COMPILER_RT_DEFAULT_TARGET_ARCH})
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "i[2-6]86|x86|amd64")
if(NOT MSVC)
- if(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
- if (CMAKE_SIZEOF_VOID_P EQUAL 4)
- test_target_arch(i386 __i386__ "-m32")
- else()
- test_target_arch(x86_64 "" "-m64")
- endif()
- else()
- test_target_arch(x86_64 "" "-m64")
- test_target_arch(i386 __i386__ "-m32")
- endif()
+ test_target_arch(x86_64 "" "-m64")
+ test_target_arch(i386 __i386__ "-m32")
else()
if (CMAKE_SIZEOF_VOID_P EQUAL 4)
test_target_arch(i386 "" "")
test_target_arch(x86_64 "" "")
endif()
endif()
+ elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "powerpc64le")
+ test_target_arch(powerpc64le "" "-m64")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "powerpc")
- # Strip out -nodefaultlibs when calling TEST_BIG_ENDIAN. Configuration
- # will fail with this option when building with a sanitizer.
- cmake_push_check_state()
- string(REPLACE "-nodefaultlibs" "" CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
- TEST_BIG_ENDIAN(HOST_IS_BIG_ENDIAN)
- cmake_pop_check_state()
-
- if(HOST_IS_BIG_ENDIAN)
- test_target_arch(powerpc64 "" "-m64")
- else()
- test_target_arch(powerpc64le "" "-m64")
+ if(CMAKE_SYSTEM_NAME MATCHES "AIX")
+ test_target_arch(powerpc "" "-m32")
endif()
+ test_target_arch(powerpc64 "" "-m64")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "s390x")
test_target_arch(s390x "" "")
elseif("${COMPILER_RT_DEFAULT_TARGET_ARCH}" MATCHES "sparc")
builtin_check_c_compiler_flag(-fno-builtin COMPILER_RT_HAS_FNO_BUILTIN_FLAG)
builtin_check_c_compiler_flag(-std=c11 COMPILER_RT_HAS_STD_C11_FLAG)
builtin_check_c_compiler_flag(-fvisibility=hidden COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG)
-builtin_check_c_compiler_flag(-fomit-frame-pointer COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG)
builtin_check_c_compiler_flag(-ffreestanding COMPILER_RT_HAS_FREESTANDING_FLAG)
builtin_check_c_compiler_flag(-fxray-instrument COMPILER_RT_HAS_XRAY_COMPILER_FLAG)
}
")
+builtin_check_c_compiler_source(COMPILER_RT_HAS_FLOAT16
+"
+_Float16 foo(_Float16 x) {
+ return x;
+}
+"
+)
+
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ASM_LSE
+"
+asm(\".arch armv8-a+lse\");
+asm(\"cas w0, w1, [x2]\");
+")
set(ARM64 aarch64)
-set(ARM32 arm armhf armv6m armv7m armv7em armv7 armv7s armv7k)
+set(ARM32 arm armhf armv6m armv7m armv7em armv7 armv7s armv7k armv8m.main armv8.1m.main)
set(HEXAGON hexagon)
set(X86 i386)
set(X86_64 x86_64)
set(MIPS32 mips mipsel)
set(MIPS64 mips64 mips64el)
+set(PPC32 powerpc)
set(PPC64 powerpc64 powerpc64le)
set(RISCV32 riscv32)
set(RISCV64 riscv64)
set(ALL_BUILTIN_SUPPORTED_ARCH
${X86} ${X86_64} ${ARM32} ${ARM64}
- ${HEXAGON} ${MIPS32} ${MIPS64} ${PPC64}
+ ${HEXAGON} ${MIPS32} ${MIPS64} ${PPC32} ${PPC64}
${RISCV32} ${RISCV64} ${SPARC} ${SPARCV9}
${WASM32} ${WASM64} ${VE})
set(DARWIN_watchos_BUILTIN_MIN_VER 2.0)
set(DARWIN_watchos_BUILTIN_MIN_VER_FLAG
${DARWIN_watchos_MIN_VER_FLAG}=${DARWIN_watchos_BUILTIN_MIN_VER})
- set(DARWIN_watchos_BUILTIN_ALL_POSSIBLE_ARCHS armv7 armv7k)
+ set(DARWIN_watchos_BUILTIN_ALL_POSSIBLE_ARCHS armv7 armv7k arm64_32)
set(DARWIN_watchossim_BUILTIN_ALL_POSSIBLE_ARCHS ${X86})
endif()
if(COMPILER_RT_ENABLE_TVOS)
include(CheckSymbolExists)
include(TestBigEndian)
-function(check_linker_flag flag out_var)
+function(compiler_rt_check_linker_flag flag out_var)
cmake_push_check_state()
set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${flag}")
check_cxx_compiler_flag("" ${out_var})
# CodeGen options.
check_c_compiler_flag(-ffreestanding COMPILER_RT_HAS_FFREESTANDING_FLAG)
+check_c_compiler_flag(-fomit-frame-pointer COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG)
check_c_compiler_flag(-std=c11 COMPILER_RT_HAS_STD_C11_FLAG)
check_cxx_compiler_flag(-fPIC COMPILER_RT_HAS_FPIC_FLAG)
check_cxx_compiler_flag(-fPIE COMPILER_RT_HAS_FPIE_FLAG)
check_cxx_compiler_flag("-Werror -Wvariadic-macros" COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG)
check_cxx_compiler_flag("-Werror -Wunused-parameter" COMPILER_RT_HAS_WUNUSED_PARAMETER_FLAG)
check_cxx_compiler_flag("-Werror -Wcovered-switch-default" COMPILER_RT_HAS_WCOVERED_SWITCH_DEFAULT_FLAG)
+check_cxx_compiler_flag("-Werror -Wsuggest-override" COMPILER_RT_HAS_WSUGGEST_OVERRIDE_FLAG)
check_cxx_compiler_flag(-Wno-pedantic COMPILER_RT_HAS_WNO_PEDANTIC)
check_cxx_compiler_flag(/W4 COMPILER_RT_HAS_W4_FLAG)
check_symbol_exists(__func__ "" COMPILER_RT_HAS_FUNC_SYMBOL)
# Includes.
-check_include_files("sys/auxv.h" COMPILER_RT_HAS_AUXV)
+check_cxx_compiler_flag(-nostdinc++ COMPILER_RT_HAS_NOSTDINCXX_FLAG)
+check_cxx_compiler_flag(-nostdlib++ COMPILER_RT_HAS_NOSTDLIBXX_FLAG)
+check_include_files("sys/auxv.h" COMPILER_RT_HAS_AUXV)
# Libraries.
check_library_exists(dl dlopen "" COMPILER_RT_HAS_LIBDL)
check_library_exists(execinfo backtrace "" COMPILER_RT_HAS_LIBEXECINFO)
# Look for terminfo library, used in unittests that depend on LLVMSupport.
+if(LLVM_ENABLE_TERMINFO STREQUAL FORCE_ON)
+ set(MAYBE_REQUIRED REQUIRED)
+else()
+ set(MAYBE_REQUIRED)
+endif()
if(LLVM_ENABLE_TERMINFO)
- foreach(library terminfo tinfo curses ncurses ncursesw)
- string(TOUPPER ${library} library_suffix)
- check_library_exists(
- ${library} setupterm "" COMPILER_RT_HAS_TERMINFO_${library_suffix})
- if(COMPILER_RT_HAS_TERMINFO_${library_suffix})
- set(COMPILER_RT_HAS_TERMINFO TRUE)
- set(COMPILER_RT_TERMINFO_LIB "${library}")
- break()
- endif()
- endforeach()
+ find_library(COMPILER_RT_TERMINFO_LIB NAMES terminfo tinfo curses ncurses ncursesw ${MAYBE_REQUIRED})
+endif()
+if(COMPILER_RT_TERMINFO_LIB)
+ set(LLVM_ENABLE_TERMINFO 1)
+else()
+ set(LLVM_ENABLE_TERMINFO 0)
endif()
if (ANDROID AND COMPILER_RT_HAS_LIBDL)
check_library_exists(stdc++ __cxa_throw "" COMPILER_RT_HAS_LIBSTDCXX)
# Linker flags.
-check_linker_flag("-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
-check_linker_flag("-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
+compiler_rt_check_linker_flag("-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
+compiler_rt_check_linker_flag("-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
+
+set(VERS_COMPAT_OPTION "-Wl,-z,gnu-version-script-compat")
+compiler_rt_check_linker_flag("${VERS_COMPAT_OPTION}" COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+
+set(DUMMY_VERS ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/dummy.vers)
+file(WRITE ${DUMMY_VERS} "{};")
+set(VERS_OPTION "-Wl,--version-script,${DUMMY_VERS}")
+if(COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
+ # Solaris 11.4 ld only supports --version-script with
+ # -z gnu-version-script-compat.
+ string(APPEND VERS_OPTION " ${VERS_COMPAT_OPTION}")
+endif()
+compiler_rt_check_linker_flag("${VERS_OPTION}" COMPILER_RT_HAS_VERSION_SCRIPT)
if(ANDROID)
- check_linker_flag("-Wl,-z,global" COMPILER_RT_HAS_Z_GLOBAL)
+ compiler_rt_check_linker_flag("-Wl,-z,global" COMPILER_RT_HAS_Z_GLOBAL)
check_library_exists(log __android_log_write "" COMPILER_RT_HAS_LIBLOG)
endif()
# specific architecture. When cross-compiling, this is controled via
# COMPILER_RT_TEST_COMPILER and COMPILER_RT_TEST_COMPILER_CFLAGS.
macro(get_test_cc_for_arch arch cc_out cflags_out)
- if(ANDROID OR ${arch} MATCHES "arm|aarch64")
+ if(ANDROID OR ${arch} MATCHES "arm|aarch64|riscv32|riscv64")
# This is only true if we are cross-compiling.
# Build all tests with host compiler and use host tools.
set(${cc_out} ${COMPILER_RT_TEST_COMPILER})
set(X86_64 x86_64)
set(MIPS32 mips mipsel)
set(MIPS64 mips64 mips64el)
+set(PPC32 powerpc)
set(PPC64 powerpc64 powerpc64le)
set(RISCV32 riscv32)
set(RISCV64 riscv64)
set(SPARCV9 sparcv9)
set(WASM32 wasm32)
set(WASM64 wasm64)
+set(VE ve)
if(APPLE)
set(ARM64 arm64)
set(ALL_SANITIZER_COMMON_SUPPORTED_ARCH ${X86} ${X86_64} ${PPC64} ${RISCV64}
${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64}
+set(ALL_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_CRT_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV32} ${RISCV64})
+set(ALL_CRT_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV32} ${RISCV64} ${VE})
set(ALL_DFSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64})
if(ANDROID)
endif()
if(OS_NAME MATCHES "Linux")
- set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64})
+ set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${S390X})
elseif (OS_NAME MATCHES "Windows")
set(ALL_FUZZER_SUPPORTED_ARCH ${X86} ${X86_64})
elseif(OS_NAME MATCHES "Android")
set(ALL_FUZZER_SUPPORTED_ARCH ${X86_64} ${ARM64})
endif()
-set(ALL_GWP_ASAN_SUPPORTED_ARCH ${X86} ${X86_64})
+set(ALL_GWP_ASAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64})
if(APPLE)
set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64})
else()
- set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32} ${PPC64} ${S390X})
+ set(ALL_LSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${MIPS64} ${ARM64} ${ARM32} ${PPC64} ${S390X} ${RISCV64})
endif()
set(ALL_MSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X})
set(ALL_HWASAN_SUPPORTED_ARCH ${X86_64} ${ARM64})
-set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC64}
+set(ALL_MEMPROF_SUPPORTED_ARCH ${X86_64})
+set(ALL_PROFILE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${PPC32} ${PPC64}
${MIPS32} ${MIPS64} ${S390X} ${SPARC} ${SPARCV9})
-set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64})
+set(ALL_TSAN_SUPPORTED_ARCH ${X86_64} ${MIPS64} ${ARM64} ${PPC64} ${S390X})
set(ALL_UBSAN_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${RISCV64}
${MIPS32} ${MIPS64} ${PPC64} ${S390X} ${SPARC} ${SPARCV9})
set(ALL_SAFESTACK_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM64} ${MIPS32} ${MIPS64})
set(ALL_CFI_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS64})
set(ALL_SCUDO_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${PPC64})
-set(ALL_SCUDO_STANDALONE_SUPPORTED_ARCH ${X86} ${X86_64})
+set(ALL_SCUDO_STANDALONE_SUPPORTED_ARCH ${X86} ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64} ${PPC64})
if(APPLE)
set(ALL_XRAY_SUPPORTED_ARCH ${X86_64})
else()
endif()
set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
+if (UNIX)
+set(ALL_ORC_SUPPORTED_ARCH ${X86_64})
+endif()
+
if(APPLE)
include(CompilerRTDarwinUtils)
set(TSAN_SUPPORTED_OS osx)
set(XRAY_SUPPORTED_OS osx)
set(FUZZER_SUPPORTED_OS osx)
+ set(ORC_SUPPORTED_OS osx)
# Note: In order to target x86_64h on OS X the minimum deployment target must
# be 10.8 or higher.
-lc++
-lc++abi)
- check_linker_flag("-fapplication-extension" COMPILER_RT_HAS_APP_EXTENSION)
+ compiler_rt_check_linker_flag("-fapplication-extension" COMPILER_RT_HAS_APP_EXTENSION)
if(COMPILER_RT_HAS_APP_EXTENSION)
list(APPEND DARWIN_COMMON_LINK_FLAGS "-fapplication-extension")
endif()
list_intersect(HWASAN_SUPPORTED_ARCH
ALL_HWASAN_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
+ list_intersect(MEMPROF_SUPPORTED_ARCH
+ ALL_MEMPROF_SUPPORTED_ARCH
+ SANITIZER_COMMON_SUPPORTED_ARCH)
list_intersect(PROFILE_SUPPORTED_ARCH
ALL_PROFILE_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
list_intersect(SHADOWCALLSTACK_SUPPORTED_ARCH
ALL_SHADOWCALLSTACK_SUPPORTED_ARCH
SANITIZER_COMMON_SUPPORTED_ARCH)
+ list_intersect(ORC_SUPPORTED_ARCH
+ ALL_ORC_SUPPORTED_ARCH
+ SANITIZER_COMMON_SUPPORTED_ARCH)
else()
filter_available_targets(CRT_SUPPORTED_ARCH ${ALL_CRT_SUPPORTED_ARCH})
filter_available_targets(LSAN_SUPPORTED_ARCH ${ALL_LSAN_SUPPORTED_ARCH})
filter_available_targets(MSAN_SUPPORTED_ARCH ${ALL_MSAN_SUPPORTED_ARCH})
filter_available_targets(HWASAN_SUPPORTED_ARCH ${ALL_HWASAN_SUPPORTED_ARCH})
+ filter_available_targets(MEMPROF_SUPPORTED_ARCH ${ALL_MEMPROF_SUPPORTED_ARCH})
filter_available_targets(PROFILE_SUPPORTED_ARCH ${ALL_PROFILE_SUPPORTED_ARCH})
filter_available_targets(TSAN_SUPPORTED_ARCH ${ALL_TSAN_SUPPORTED_ARCH})
filter_available_targets(UBSAN_SUPPORTED_ARCH ${ALL_UBSAN_SUPPORTED_ARCH})
filter_available_targets(SHADOWCALLSTACK_SUPPORTED_ARCH
${ALL_SHADOWCALLSTACK_SUPPORTED_ARCH})
filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH})
+ filter_available_targets(ORC_SUPPORTED_ARCH ${ALL_ORC_SUPPORTED_ARCH})
endif()
if (MSVC)
set(CAN_SYMBOLIZE 1)
endif()
-find_program(GOLD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.gold ld.gold ${LLVM_DEFAULT_TARGET_TRIPLE}-ld ld DOC "The gold linker")
+find_program(GNU_LD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.bfd ld.bfd DOC "GNU ld")
+find_program(GOLD_EXECUTABLE NAMES ${LLVM_DEFAULT_TARGET_TRIPLE}-ld.gold ld.gold DOC "GNU gold")
if(COMPILER_RT_SUPPORTED_ARCH)
list(REMOVE_DUPLICATES COMPILER_RT_SUPPORTED_ARCH)
list_replace(COMPILER_RT_SANITIZERS_TO_BUILD all "${ALL_SANITIZERS}")
if (SANITIZER_COMMON_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND
- (OS_NAME MATCHES "Android|Darwin|Linux|FreeBSD|NetBSD|OpenBSD|Fuchsia|SunOS" OR
+ (OS_NAME MATCHES "Android|Darwin|Linux|FreeBSD|NetBSD|Fuchsia|SunOS" OR
(OS_NAME MATCHES "Windows" AND NOT CYGWIN AND
(NOT MINGW OR CMAKE_CXX_COMPILER_ID MATCHES "Clang"))))
set(COMPILER_RT_HAS_SANITIZER_COMMON TRUE)
set(COMPILER_RT_HAS_INTERCEPTION FALSE)
endif()
-if (COMPILER_RT_HAS_SANITIZER_COMMON AND ASAN_SUPPORTED_ARCH AND
- NOT OS_NAME MATCHES "OpenBSD")
+if (COMPILER_RT_HAS_SANITIZER_COMMON AND ASAN_SUPPORTED_ARCH)
set(COMPILER_RT_HAS_ASAN TRUE)
else()
set(COMPILER_RT_HAS_ASAN FALSE)
endif()
if (COMPILER_RT_HAS_SANITIZER_COMMON AND LSAN_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Darwin|Linux|NetBSD|Fuchsia")
+ OS_NAME MATCHES "Android|Darwin|Linux|NetBSD|Fuchsia")
set(COMPILER_RT_HAS_LSAN TRUE)
else()
set(COMPILER_RT_HAS_LSAN FALSE)
set(COMPILER_RT_HAS_HWASAN FALSE)
endif()
+if (COMPILER_RT_HAS_SANITIZER_COMMON AND MEMPROF_SUPPORTED_ARCH AND
+ OS_NAME MATCHES "Linux")
+ set(COMPILER_RT_HAS_MEMPROF TRUE)
+else()
+ set(COMPILER_RT_HAS_MEMPROF FALSE)
+endif()
+
if (PROFILE_SUPPORTED_ARCH AND NOT LLVM_USE_SANITIZER AND
- OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android|Fuchsia|SunOS|NetBSD")
+ OS_NAME MATCHES "Darwin|Linux|FreeBSD|Windows|Android|Fuchsia|SunOS|NetBSD|AIX")
set(COMPILER_RT_HAS_PROFILE TRUE)
else()
set(COMPILER_RT_HAS_PROFILE FALSE)
endif()
if (COMPILER_RT_HAS_SANITIZER_COMMON AND UBSAN_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|OpenBSD|Windows|Android|Fuchsia|SunOS")
+ OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|Windows|Android|Fuchsia|SunOS")
set(COMPILER_RT_HAS_UBSAN TRUE)
else()
set(COMPILER_RT_HAS_UBSAN FALSE)
endif()
if (COMPILER_RT_HAS_SANITIZER_COMMON AND UBSAN_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Linux|FreeBSD|NetBSD|OpenBSD|Android|Darwin")
+ OS_NAME MATCHES "Linux|FreeBSD|NetBSD|Android|Darwin")
set(COMPILER_RT_HAS_UBSAN_MINIMAL TRUE)
else()
set(COMPILER_RT_HAS_UBSAN_MINIMAL FALSE)
endif()
if (COMPILER_RT_HAS_SANITIZER_COMMON AND XRAY_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|OpenBSD|Fuchsia")
+ OS_NAME MATCHES "Darwin|Linux|FreeBSD|NetBSD|Fuchsia")
set(COMPILER_RT_HAS_XRAY TRUE)
else()
set(COMPILER_RT_HAS_XRAY FALSE)
endif()
+if (ORC_SUPPORTED_ARCH)
+ set(COMPILER_RT_HAS_ORC TRUE)
+else()
+ set(COMPILER_RT_HAS_ORC FALSE)
+endif()
+
if (COMPILER_RT_HAS_SANITIZER_COMMON AND FUZZER_SUPPORTED_ARCH AND
- OS_NAME MATCHES "Android|Darwin|Linux|NetBSD|FreeBSD|OpenBSD|Fuchsia|Windows")
+ OS_NAME MATCHES "Android|Darwin|Linux|NetBSD|FreeBSD|Fuchsia|Windows")
set(COMPILER_RT_HAS_FUZZER TRUE)
else()
set(COMPILER_RT_HAS_FUZZER FALSE)
--- /dev/null
+.. _BuildingCompilerRT:
+
+===============
+Building Compiler-RT
+===============
+
+.. contents::
+ :local:
+
+.. _build instructions:
+
+The instructions on this page are aimed at vendors who ship Compiler-RT as part of an
+operating system distribution, a toolchain or similar shipping vehicules. If you
+are a user merely trying to use Compiler-RT in your program, you most likely want to
+refer to your vendor's documentation, or to the general documentation for using
+LLVM, Clang, the various santizers, etc.
+
+CMake Options
+=============
+
+Here are some of the CMake variables that are used often, along with a
+brief explanation and LLVM-specific notes. For full documentation, check the
+CMake docs or execute ``cmake --help-variable VARIABLE_NAME``.
+
+**CMAKE_BUILD_TYPE**:STRING
+ Sets the build type for ``make`` based generators. Possible values are
+ Release, Debug, RelWithDebInfo and MinSizeRel. On systems like Visual Studio
+ the user sets the build type with the IDE settings.
+
+**CMAKE_INSTALL_PREFIX**:PATH
+ Path where LLVM will be installed if "make install" is invoked or the
+ "INSTALL" target is built.
+
+**CMAKE_CXX_COMPILER**:STRING
+ The C++ compiler to use when building and testing Compiler-RT.
+
+
+.. _compiler-rt-specific options:
+
+Compiler-RT specific options
+-----------------------
+
+.. option:: COMPILER_RT_INSTALL_PATH:PATH
+
+ **Default**: ```` (empty relative path)
+
+ Prefix for directories where built Compiler-RT artifacts should be installed.
+ Can be an absolute path, like the default empty string, in which case it is
+ relative ``CMAKE_INSTALL_PREFIX``. If setting a relative path, make sure to
+ include the ``:PATH`` with your ``-D``, i.e. use
+ ``-DCOMPILER_RT_INSTALL_PATH:PATH=...`` not
+ ``-DCOMPILER_RT_INSTALL_PATH=...``, otherwise CMake will convert the
+ path to an absolute path.
+
+.. option:: COMPILER_RT_INSTALL_LIBRARY_DIR:PATH
+
+ **Default**: ``lib``
+
+ Path where built Compiler-RT libraries should be installed. If a relative
+ path, relative to ``COMPILER_RT_INSTALL_PATH``.
+
+.. option:: COMPILER_RT_INSTALL_BINARY_DIR:PATH
+
+ **Default**: ``bin``
+
+ Path where built Compiler-RT executables should be installed. If a relative
+ path, relative to ``COMPILER_RT_INSTALL_PATH``.
+
+.. option:: COMPILER_RT_INSTALL_INCLUDE_DIR:PATH
+
+ **Default**: ``include``
+
+ Path where Compiler-RT headers should be installed. If a relative
+ path, relative to ``COMPILER_RT_INSTALL_PATH``.
+
+.. option:: COMPILER_RT_INSTALL_DATA_DIR:PATH
+
+ **Default**: ``share``
+
+ Path where Compiler-RT data should be installed. If a relative
+ path, relative to ``COMPILER_RT_INSTALL_PATH``.
+
+.. _LLVM-specific variables:
+
+LLVM-specific options
+---------------------
+
+.. option:: LLVM_LIBDIR_SUFFIX:STRING
+
+ Extra suffix to append to the directory where libraries are to be
+ installed. On a 64-bit architecture, one could use ``-DLLVM_LIBDIR_SUFFIX=64``
+ to install libraries to ``/usr/lib64``.
)
endif(COMPILER_RT_BUILD_SANITIZERS)
+if (COMPILER_RT_BUILD_MEMPROF)
+ set(MEMPROF_HEADERS
+ sanitizer/memprof_interface.h
+ )
+endif(COMPILER_RT_BUILD_MEMPROF)
+
if (COMPILER_RT_BUILD_XRAY)
set(XRAY_HEADERS
xray/xray_interface.h
set(COMPILER_RT_HEADERS
${SANITIZER_HEADERS}
${FUZZER_HEADERS}
+ ${MEMPROF_HEADERS}
${XRAY_HEADERS}
${PROFILE_HEADERS})
install(FILES ${SANITIZER_HEADERS}
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/sanitizer)
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/sanitizer)
# Install fuzzer headers.
install(FILES ${FUZZER_HEADERS}
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/fuzzer)
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/fuzzer)
# Install xray headers.
install(FILES ${XRAY_HEADERS}
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/xray)
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/xray)
# Install profile headers.
install(FILES ${PROFILE_HEADERS}
COMPONENT compiler-rt-headers
PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/include/profile)
+ DESTINATION ${COMPILER_RT_INSTALL_INCLUDE_DIR}/profile)
if (NOT CMAKE_CONFIGURATION_TYPES) # don't add this for IDEs.
add_custom_target(install-compiler-rt-headers
#define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
#include <algorithm>
+#include <array>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <initializer_list>
+#include <limits>
#include <string>
#include <type_traits>
#include <utility>
// Returns a value from the given array.
template <typename T, size_t size> T PickValueInArray(const T (&array)[size]);
+ template <typename T, size_t size>
+ T PickValueInArray(const std::array<T, size> &array);
template <typename T> T PickValueInArray(std::initializer_list<const T> list);
// Writes data to the given destination and returns number of bytes written.
return array[ConsumeIntegralInRange<size_t>(0, size - 1)];
}
+template <typename T, size_t size>
+T FuzzedDataProvider::PickValueInArray(const std::array<T, size> &array) {
+ static_assert(size > 0, "The array must be non empty.");
+ return array[ConsumeIntegralInRange<size_t>(0, size - 1)];
+}
+
template <typename T>
T FuzzedDataProvider::PickValueInArray(std::initializer_list<const T> list) {
// TODO(Dor1s): switch to static_assert once C++14 is allowed.
return static_cast<TS>(value);
} else {
constexpr auto TS_min = std::numeric_limits<TS>::min();
- return TS_min + static_cast<char>(value - TS_min);
+ return TS_min + static_cast<TS>(value - TS_min);
}
}
|*
\*===----------------------------------------------------------------------===*/
/*
- * This is the master file that defines all the data structure, signature,
+ * This is the main file that defines all the data structure, signature,
* constant literals that are shared across profiling runtime library,
* compiler (instrumentation), and host tools (reader/writer). The entities
* defined in this file affect the profile runtime ABI, the raw profile format,
* or both.
*
- * The file has two identical copies. The master copy lives in LLVM and
+ * The file has two identical copies. The primary copy lives in LLVM and
* the other one sits in compiler-rt/lib/profile directory. To make changes
- * in this file, first modify the master copy and copy it over to compiler-rt.
+ * in this file, first modify the primary copy and copy it over to compiler-rt.
* Testing of any change in this file can start only after the two copies are
* synced up.
*
#endif
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
+INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
-#ifndef VALUE_RANGE_PROF
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
-#else /* VALUE_RANGE_PROF */
-VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \
- INSTR_PROF_COMMA
-VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx))
-#endif /*VALUE_RANGE_PROF */
#undef VALUE_PROF_FUNC_PARAM
#undef INSTR_PROF_COMMA
/* VALUE_PROF_FUNC_PARAM end */
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 5
+#define INSTR_PROF_RAW_VERSION 7
/* Indexed profile format version (start from 1). */
-#define INSTR_PROF_INDEX_VERSION 6
+#define INSTR_PROF_INDEX_VERSION 7
/* Coverage mapping format version (start from 0). */
-#define INSTR_PROF_COVMAP_VERSION 3
+#define INSTR_PROF_COVMAP_VERSION 5
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
* version for other variants of profile. We set the lowest bit of the upper 8
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
+#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
+#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
/* The variable that holds the name of the profile data
* specified via command line. */
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
-#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
-#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
- INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
#endif
#undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ * [0, 0]
+ * [1, 1]
+ * [2, 2]
+ * [3, 3]
+ * [4, 4]
+ * [5, 5]
+ * [6, 6]
+ * [7, 7]
+ * [8, 8]
+ * [9, 15]
+ * [16, 16]
+ * [17, 31]
+ * [32, 32]
+ * [33, 63]
+ * [64, 64]
+ * [65, 127]
+ * [128, 128]
+ * [129, 255]
+ * [256, 256]
+ * [257, 511]
+ * [512, 512]
+ * [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+#define INSTR_PROF_NUM_BUCKETS 22
+
+/*
+ * Clz and Popcount. This code was copied from
+ * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
+ * llvm/include/llvm/Support/MathExtras.h.
+ */
+#if defined(_MSC_VER) && !defined(__clang__)
+
+#include <intrin.h>
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) {
+ unsigned long LeadZeroIdx = 0;
+#if !defined(_M_ARM64) && !defined(_M_X64)
+ // Scan the high 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
+ return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
+ // from the MSB.
+ // Scan the low 32 bits.
+ if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
+ return (int)(63 - LeadZeroIdx);
+#else
+ if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
+#endif
+ return 64;
+}
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) {
+ // This code originates from https://reviews.llvm.org/rG30626254510f.
+ unsigned long long v = X;
+ v = v - ((v >> 1) & 0x5555555555555555ULL);
+ v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
+ v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
+ return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
+}
+
+#else
+
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
+int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
+
+#endif /* defined(_MSC_VER) && !defined(__clang__) */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked. Use the value as is.
+ return Value;
+ else if (Value >= 513)
+ // The last range is mapped to its lowest value.
+ return 513;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, use it as is.
+ return Value;
+ else
+ // Otherwise, take to the previous power of two + 1.
+ return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked.
+ return 1;
+ else if (InstProfPopcountll(Value) == 1)
+ // If it's a power of two, there's only one value.
+ return 1;
+ else
+ // Otherwise, there's more than one value in the range.
+ return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
/// \param addr Address to locate.
/// \param name Buffer to store the variable's name.
/// \param name_size Size in bytes of the variable's name buffer.
-/// \param region_address [out] Address of the region.
-/// \param region_size [out] Size of the region in bytes.
+/// \param[out] region_address Address of the region.
+/// \param[out] region_size Size of the region in bytes.
///
/// \returns Returns the category of the given pointer as a constant string.
const char *__asan_locate_address(void *addr, char *name, size_t name_size,
/// \param addr A heap address.
/// \param trace A buffer to store the stack trace.
/// \param size Size in bytes of the trace buffer.
-/// \param thread_id [out] The thread ID of the address.
+/// \param[out] thread_id The thread ID of the address.
///
/// \returns Returns the number of stored frames or 0 on error.
size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size,
/// \param addr A heap address.
/// \param trace A buffer to store the stack trace.
/// \param size Size in bytes of the trace buffer.
-/// \param thread_id [out] The thread ID of the address.
+/// \param[out] thread_id The thread ID of the address.
///
/// \returns Returns the number of stored frames or 0 on error.
size_t __asan_get_free_stack(void *addr, void **trace, size_t size,
/// Gets the current shadow memory mapping (useful for calling from the
/// debugger).
///
-/// \param shadow_scale [out] Shadow scale value.
-/// \param shadow_offset [out] Offset value.
+/// \param[out] shadow_scale Shadow scale value.
+/// \param[out] shadow_offset Offset value.
void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset);
/// This is an internal function that is called to report an error. However,
///
/// \param fake_stack An opaque handler to a fake stack.
/// \param addr Address to test.
-/// \param beg [out] Beginning of fake frame.
-/// \param end [out] End of fake frame.
+/// \param[out] beg Beginning of fake frame.
+/// \param[out] end End of fake frame.
/// \returns Stack address or NULL.
void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg,
void **end);
// Tell the tools to write their reports to the provided file descriptor
// (casted to void *).
void __sanitizer_set_report_fd(void *fd);
+// Get the current full report file path, if a path was specified by
+// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
+const char *__sanitizer_get_report_path();
// Notify the tools that the sandbox is going to be turned on. The reserved
// parameter will be used in the future to hold a structure with functions
/// signal callback runs during the switch, it will not benefit from stack
/// use-after-return detection.
///
-/// \param fake_stack_save [out] Fake stack save location.
+/// \param[out] fake_stack_save Fake stack save location.
/// \param bottom Bottom address of stack.
/// \param size Size of stack in bytes.
void __sanitizer_start_switch_fiber(void **fake_stack_save,
/// <c>__sanitizer_start_switch_fiber()</c>.
///
/// \param fake_stack_save Fake stack save location.
-/// \param bottom_old [out] Bottom address of old stack.
-/// \param size_old [out] Size of old stack in bytes.
+/// \param[out] bottom_old Bottom address of old stack.
+/// \param[out] size_old Size of old stack in bytes.
void __sanitizer_finish_switch_fiber(void *fake_stack_save,
const void **bottom_old,
size_t *size_old);
extern "C" {
#endif
-typedef uint16_t dfsan_label;
-
-/// Stores information associated with a specific label identifier. A label
-/// may be a base label created using dfsan_create_label, with associated
-/// text description and user data, or an automatically created union label,
-/// which represents the union of two label identifiers (which may themselves
-/// be base or union labels).
-struct dfsan_label_info {
- // Fields for union labels, set to 0 for base labels.
- dfsan_label l1;
- dfsan_label l2;
-
- // Fields for base labels.
- const char *desc;
- void *userdata;
-};
+typedef uint8_t dfsan_label;
+typedef uint32_t dfsan_origin;
/// Signature of the callback argument to dfsan_set_write_callback().
typedef void (*dfsan_write_callback_t)(int fd, const void *buf, size_t count);
-/// Computes the union of \c l1 and \c l2, possibly creating a union label in
-/// the process.
+/// Computes the union of \c l1 and \c l2, resulting in a union label.
dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2);
-/// Creates and returns a base label with the given description and user data.
-dfsan_label dfsan_create_label(const char *desc, void *userdata);
-
/// Sets the label for each address in [addr,addr+size) to \c label.
void dfsan_set_label(dfsan_label label, void *addr, size_t size);
/// value.
dfsan_label dfsan_get_label(long data);
+/// Retrieves the immediate origin associated with the given data. The returned
+/// origin may point to another origin.
+///
+/// The type of 'data' is arbitrary.
+dfsan_origin dfsan_get_origin(long data);
+
/// Retrieves the label associated with the data at the given address.
dfsan_label dfsan_read_label(const void *addr, size_t size);
-/// Retrieves a pointer to the dfsan_label_info struct for the given label.
-const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label);
-
/// Returns whether the given label label contains the label elem.
int dfsan_has_label(dfsan_label label, dfsan_label elem);
-/// If the given label label contains a label with the description desc, returns
-/// that label, else returns 0.
-dfsan_label dfsan_has_label_with_desc(dfsan_label label, const char *desc);
-
-/// Returns the number of labels allocated.
-size_t dfsan_get_label_count(void);
-
/// Flushes the DFSan shadow, i.e. forgets about all labels currently associated
-/// with the application memory. Will work only if there are no other
-/// threads executing DFSan-instrumented code concurrently.
-/// Use this call to start over the taint tracking within the same procces.
+/// with the application memory. Use this call to start over the taint tracking
+/// within the same process.
+///
+/// Note: If another thread is working with tainted data during the flush, that
+/// taint could still be written to shadow after the flush.
void dfsan_flush(void);
/// Sets a callback to be invoked on calls to write(). The callback is invoked
/// callback executes. Pass in NULL to remove any callback.
void dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback);
-/// Writes the labels currently used by the program to the given file
-/// descriptor. The lines of the output have the following format:
-///
-/// <label> <parent label 1> <parent label 2> <label description if any>
-void dfsan_dump_labels(int fd);
-
/// Interceptor hooks.
/// Whenever a dfsan's custom function is called the corresponding
/// hook is called it non-zero. The hooks should be defined by the user.
void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2,
size_t n, dfsan_label s1_label,
dfsan_label s2_label, dfsan_label n_label);
+
+/// Prints the origin trace of the label at the address addr to stderr. It also
+/// prints description at the beginning of the trace. If origin tracking is not
+/// on, or the address is not labeled, it prints nothing.
+void dfsan_print_origin_trace(const void *addr, const char *description);
+
+/// Prints the origin trace of the label at the address \p addr to a
+/// pre-allocated output buffer. If origin tracking is not on, or the address is
+/// not labeled, it prints nothing.
+///
+/// Typical usage:
+/// \code
+/// char kDescription[] = "...";
+/// char buf[1024];
+/// dfsan_sprint_origin_trace(&tainted_var, kDescription, buf, sizeof(buf));
+/// \endcode
+///
+/// Typical usage that handles truncation:
+/// \code
+/// char buf[1024];
+/// int len = dfsan_sprint_origin_trace(&var, nullptr, buf, sizeof(buf));
+///
+/// if (len < sizeof(buf)) {
+/// ProcessOriginTrace(buf);
+/// } else {
+/// char *tmpbuf = new char[len + 1];
+/// dfsan_sprint_origin_trace(&var, nullptr, tmpbuf, len + 1);
+/// ProcessOriginTrace(tmpbuf);
+/// delete[] tmpbuf;
+/// }
+/// \endcode
+///
+/// \param addr The tainted memory address whose origin we are printing.
+/// \param description A description printed at the beginning of the trace.
+/// \param [out] out_buf The output buffer to write the results to.
+/// \param out_buf_size The size of \p out_buf.
+///
+/// \returns The number of symbols that should have been written to \p out_buf
+/// (not including trailing null byte '\0'). Thus, the string is truncated iff
+/// return value is not less than \p out_buf_size.
+size_t dfsan_sprint_origin_trace(const void *addr, const char *description,
+ char *out_buf, size_t out_buf_size);
+
+/// Prints the stack trace leading to this call to a pre-allocated output
+/// buffer.
+///
+/// For usage examples, see dfsan_sprint_origin_trace.
+///
+/// \param [out] out_buf The output buffer to write the results to.
+/// \param out_buf_size The size of \p out_buf.
+///
+/// \returns The number of symbols that should have been written to \p out_buf
+/// (not including trailing null byte '\0'). Thus, the string is truncated iff
+/// return value is not less than \p out_buf_size.
+size_t dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size);
+
+/// Retrieves the very first origin associated with the data at the given
+/// address.
+dfsan_origin dfsan_get_init_origin(const void *addr);
+
+/// Returns the value of -dfsan-track-origins.
+/// * 0: do not track origins.
+/// * 1: track origins at memory store operations.
+/// * 2: track origins at memory load and store operations.
+int dfsan_get_track_origins(void);
#ifdef __cplusplus
} // extern "C"
* accessed through the pointer in x, or -1 if the whole range is good. */
intptr_t __hwasan_test_shadow(const volatile void *x, size_t size);
+ /* Sets the callback function to be called during HWASan error reporting. */
+ void __hwasan_set_error_report_callback(void (*callback)(const char *));
+
int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size);
void * __sanitizer_memalign(size_t alignment, size_t size);
void * __sanitizer_aligned_alloc(size_t alignment, size_t size);
--- /dev/null
+//===-- sanitizer/memprof_interface.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler (MemProf).
+//
+// Public interface header.
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_MEMPROF_INTERFACE_H
+#define SANITIZER_MEMPROF_INTERFACE_H
+
+#include <sanitizer/common_interface_defs.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/// Records access to a memory region (<c>[addr, addr+size)</c>).
+///
+/// This memory must be previously allocated by your program.
+///
+/// \param addr Start of memory region.
+/// \param size Size of memory region.
+void __memprof_record_access_range(void const volatile *addr, size_t size);
+
+/// Records access to a memory address <c><i>addr</i></c>.
+///
+/// This memory must be previously allocated by your program.
+///
+/// \param addr Accessed memory address
+void __memprof_record_access(void const volatile *addr);
+
+/// User-provided callback on MemProf errors.
+///
+/// You can provide a function that would be called immediately when MemProf
+/// detects an error. This is useful in cases when MemProf detects an error but
+/// your program crashes before the MemProf report is printed.
+void __memprof_on_error(void);
+
+/// Prints accumulated statistics to <c>stderr</c> (useful for calling from the
+/// debugger).
+void __memprof_print_accumulated_stats(void);
+
+/// User-provided default option settings.
+///
+/// You can provide your own implementation of this function to return a string
+/// containing MemProf runtime options (for example,
+/// <c>verbosity=1:print_stats=1</c>).
+///
+/// \returns Default options string.
+const char *__memprof_default_options(void);
+
+/// Prints the memory profile to the current profile file.
+///
+/// \returns 0 on success.
+int __memprof_profile_dump(void);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SANITIZER_MEMPROF_INTERFACE_H
call to __msan_scoped_disable_interceptor_checks. */
void __msan_scoped_enable_interceptor_checks(void);
+ void __msan_start_switch_fiber(const void *bottom, size_t size);
+ void __msan_finish_switch_fiber(const void **bottom_old, size_t *size_old);
+
#ifdef __cplusplus
} // extern "C"
#endif
// DO NOT EDIT! THIS FILE HAS BEEN GENERATED!
//
// Generated with: generate_netbsd_syscalls.awk
-// Generated date: 2019-12-24
-// Generated from: syscalls.master,v 1.296 2019/09/22 22:59:39 christos Exp
+// Generated date: 2020-09-10
+// Generated from: syscalls.master,v 1.306 2020/08/14 00:53:16 riastradh Exp
//
//===----------------------------------------------------------------------===//
#ifndef SANITIZER_NETBSD_SYSCALL_HOOKS_H
__sanitizer_syscall_pre_impl_dup2((long long)(from), (long long)(to))
#define __sanitizer_syscall_post_dup2(res, from, to) \
__sanitizer_syscall_post_impl_dup2(res, (long long)(from), (long long)(to))
-/* syscall 91 has been skipped */
+#define __sanitizer_syscall_pre_getrandom(buf, buflen, flags) \
+ __sanitizer_syscall_pre_impl_getrandom( \
+ (long long)(buf), (long long)(buflen), (long long)(flags))
+#define __sanitizer_syscall_post_getrandom(res, buf, buflen, flags) \
+ __sanitizer_syscall_post_impl_getrandom( \
+ res, (long long)(buf), (long long)(buflen), (long long)(flags))
#define __sanitizer_syscall_pre_fcntl(fd, cmd, arg) \
__sanitizer_syscall_pre_impl_fcntl((long long)(fd), (long long)(cmd), \
(long long)(arg))
#define __sanitizer_syscall_post_sysarch(res, op, parms) \
__sanitizer_syscall_post_impl_sysarch(res, (long long)(op), \
(long long)(parms))
-/* syscall 166 has been skipped */
-/* syscall 167 has been skipped */
-/* syscall 168 has been skipped */
+#define __sanitizer_syscall_pre___futex(uaddr, op, val, timeout, uaddr2, val2, \
+ val3) \
+ __sanitizer_syscall_pre_impl___futex((long long)(uaddr), (long long)(op), \
+ (long long)(val), (long long)(timeout), \
+ (long long)(uaddr2), (long long)(val2), \
+ (long long)(val3))
+#define __sanitizer_syscall_post___futex(res, uaddr, op, val, timeout, uaddr2, \
+ val2, val3) \
+ __sanitizer_syscall_post_impl___futex( \
+ res, (long long)(uaddr), (long long)(op), (long long)(val), \
+ (long long)(timeout), (long long)(uaddr2), (long long)(val2), \
+ (long long)(val3))
+#define __sanitizer_syscall_pre___futex_set_robust_list(head, len) \
+ __sanitizer_syscall_pre_impl___futex_set_robust_list((long long)(head), \
+ (long long)(len))
+#define __sanitizer_syscall_post___futex_set_robust_list(res, head, len) \
+ __sanitizer_syscall_post_impl___futex_set_robust_list( \
+ res, (long long)(head), (long long)(len))
+#define __sanitizer_syscall_pre___futex_get_robust_list(lwpid, headp, lenp) \
+ __sanitizer_syscall_pre_impl___futex_get_robust_list( \
+ (long long)(lwpid), (long long)(headp), (long long)(lenp))
+#define __sanitizer_syscall_post___futex_get_robust_list(res, lwpid, headp, \
+ lenp) \
+ __sanitizer_syscall_post_impl___futex_get_robust_list( \
+ res, (long long)(lwpid), (long long)(headp), (long long)(lenp))
#if !defined(_LP64)
#define __sanitizer_syscall_pre_compat_10_osemsys(which, a2, a3, a4, a5) \
__sanitizer_syscall_pre_impl_compat_10_osemsys( \
__sanitizer_syscall_post_impl___fhstatvfs190( \
res, (long long)(fhp), (long long)(fh_size), (long long)(buf), \
(long long)(flags))
+#define __sanitizer_syscall_pre___acl_get_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_delete_link(path, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_link((long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_link(res, path, type) \
+ __sanitizer_syscall_post_impl___acl_delete_link(res, (long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_aclcheck_link(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_link( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_link(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_link( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_get_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_get_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_get_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_get_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_get_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_set_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_set_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_set_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_set_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_delete_file(path, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_file((long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_file(res, path, type) \
+ __sanitizer_syscall_post_impl___acl_delete_file(res, (long long)(path), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_delete_fd(filedes, type) \
+ __sanitizer_syscall_pre_impl___acl_delete_fd((long long)(filedes), \
+ (long long)(type))
+#define __sanitizer_syscall_post___acl_delete_fd(res, filedes, type) \
+ __sanitizer_syscall_post_impl___acl_delete_fd(res, (long long)(filedes), \
+ (long long)(type))
+#define __sanitizer_syscall_pre___acl_aclcheck_file(path, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_file( \
+ (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_file(res, path, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_file( \
+ res, (long long)(path), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre___acl_aclcheck_fd(filedes, type, aclp) \
+ __sanitizer_syscall_pre_impl___acl_aclcheck_fd( \
+ (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_post___acl_aclcheck_fd(res, filedes, type, aclp) \
+ __sanitizer_syscall_post_impl___acl_aclcheck_fd( \
+ res, (long long)(filedes), (long long)(type), (long long)(aclp))
+#define __sanitizer_syscall_pre_lpathconf(path, name) \
+ __sanitizer_syscall_pre_impl_lpathconf((long long)(path), (long long)(name))
+#define __sanitizer_syscall_post_lpathconf(res, path, name) \
+ __sanitizer_syscall_post_impl_lpathconf(res, (long long)(path), \
+ (long long)(name))
/* Compat with older releases */
#define __sanitizer_syscall_pre_getvfsstat \
void __sanitizer_syscall_pre_impl_dup2(long long from, long long to);
void __sanitizer_syscall_post_impl_dup2(long long res, long long from,
long long to);
-/* syscall 91 has been skipped */
+void __sanitizer_syscall_pre_impl_getrandom(long long buf, long long buflen,
+ long long flags);
+void __sanitizer_syscall_post_impl_getrandom(long long res, long long buf,
+ long long buflen, long long flags);
void __sanitizer_syscall_pre_impl_fcntl(long long fd, long long cmd,
long long arg);
void __sanitizer_syscall_post_impl_fcntl(long long res, long long fd,
void __sanitizer_syscall_pre_impl_sysarch(long long op, long long parms);
void __sanitizer_syscall_post_impl_sysarch(long long res, long long op,
long long parms);
-/* syscall 166 has been skipped */
-/* syscall 167 has been skipped */
-/* syscall 168 has been skipped */
+void __sanitizer_syscall_pre_impl___futex(long long uaddr, long long op,
+ long long val, long long timeout,
+ long long uaddr2, long long val2,
+ long long val3);
+void __sanitizer_syscall_post_impl___futex(long long res, long long uaddr,
+ long long op, long long val,
+ long long timeout, long long uaddr2,
+ long long val2, long long val3);
+void __sanitizer_syscall_pre_impl___futex_set_robust_list(long long head,
+ long long len);
+void __sanitizer_syscall_post_impl___futex_set_robust_list(long long res,
+ long long head,
+ long long len);
+void __sanitizer_syscall_pre_impl___futex_get_robust_list(long long lwpid,
+ long long headp,
+ long long lenp);
+void __sanitizer_syscall_post_impl___futex_get_robust_list(long long res,
+ long long lwpid,
+ long long headp,
+ long long lenp);
#if !defined(_LP64)
void __sanitizer_syscall_pre_impl_compat_10_osemsys(long long which,
long long a2, long long a3,
long long fh_size,
long long buf,
long long flags);
+void __sanitizer_syscall_pre_impl___acl_get_link(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_link(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_link(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_link(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_delete_link(long long path,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_link(long long res,
+ long long path,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_link(long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_link(long long res,
+ long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_get_file(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_file(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_file(long long path, long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_file(long long res, long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_get_fd(long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_post_impl___acl_get_fd(long long res,
+ long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_pre_impl___acl_set_fd(long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_post_impl___acl_set_fd(long long res,
+ long long filedes,
+ long long type, long long aclp);
+void __sanitizer_syscall_pre_impl___acl_delete_file(long long path,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_file(long long res,
+ long long path,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_delete_fd(long long filedes,
+ long long type);
+void __sanitizer_syscall_post_impl___acl_delete_fd(long long res,
+ long long filedes,
+ long long type);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_file(long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_file(long long res,
+ long long path,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl___acl_aclcheck_fd(long long filedes,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_post_impl___acl_aclcheck_fd(long long res,
+ long long filedes,
+ long long type,
+ long long aclp);
+void __sanitizer_syscall_pre_impl_lpathconf(long long path, long long name);
+void __sanitizer_syscall_post_impl_lpathconf(long long res, long long path,
+ long long name);
#ifdef __cplusplus
} // extern "C"
// the corresponding __tsan_mutex_post_lock annotation.
static const unsigned __tsan_mutex_recursive_unlock = 1 << 7;
+// Convenient composed constants.
+static const unsigned __tsan_mutex_try_read_lock =
+ __tsan_mutex_read_lock | __tsan_mutex_try_lock;
+static const unsigned __tsan_mutex_try_read_lock_failed =
+ __tsan_mutex_try_read_lock | __tsan_mutex_try_lock_failed;
+
// Annotate creation of a mutex.
// Supported flags: mutex creation flags.
void __tsan_mutex_create(void *addr, unsigned flags);
// and freed by __tsan_destroy_fiber.
// - TSAN context of current fiber or thread can be obtained
// by calling __tsan_get_current_fiber.
-// - __tsan_switch_to_fiber should be called immediatly before switch
+// - __tsan_switch_to_fiber should be called immediately before switch
// to fiber, such as call of swapcontext.
// - Fiber name can be set by __tsan_set_fiber_name.
void *__tsan_get_current_fiber(void);
// Do not establish a happens-before relation between fibers
static const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0;
+// User-provided callback invoked on TSan initialization.
+void __tsan_on_initialize();
+
+// User-provided callback invoked on TSan shutdown.
+// `failed` - Nonzero if TSan did detect issues, zero otherwise.
+// Return `0` if TSan should exit as if no issues were detected. Return nonzero
+// if TSan should exit as if issues were detected.
+int __tsan_on_finalize(int failed);
+
#ifdef __cplusplus
} // extern "C"
#endif
#endif
// Part of ABI, do not change.
-// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic
+// https://github.com/llvm/llvm-project/blob/main/libcxx/include/atomic
typedef enum {
__tsan_memory_order_relaxed,
__tsan_memory_order_consume,
#
#TODO: Refactor sanitizer_common into smaller pieces (e.g. flag parsing, utils).
if (COMPILER_RT_HAS_SANITIZER_COMMON AND
- (COMPILER_RT_BUILD_SANITIZERS OR COMPILER_RT_BUILD_XRAY))
+ (COMPILER_RT_BUILD_SANITIZERS OR COMPILER_RT_BUILD_XRAY OR COMPILER_RT_BUILD_MEMPROF))
add_subdirectory(sanitizer_common)
endif()
endif()
endfunction()
-if(COMPILER_RT_BUILD_SANITIZERS)
+if(COMPILER_RT_BUILD_SANITIZERS OR COMPILER_RT_BUILD_MEMPROF)
compiler_rt_build_runtime(interception)
+endif()
+if(COMPILER_RT_BUILD_SANITIZERS)
if(COMPILER_RT_HAS_SANITIZER_COMMON)
add_subdirectory(stats)
add_subdirectory(lsan)
compiler_rt_build_runtime(fuzzer)
endif()
+if(COMPILER_RT_BUILD_MEMPROF AND COMPILER_RT_HAS_SANITIZER_COMMON)
+ compiler_rt_build_runtime(memprof)
+endif()
+
+if(COMPILER_RT_BUILD_ORC)
+ compiler_rt_build_runtime(orc)
+endif()
+
# It doesn't normally make sense to build runtimes when a sanitizer is enabled,
# so we don't add_subdirectory the runtimes in that case. However, the opposite
# is true for fuzzers that exercise parts of the runtime. So we add the fuzzer
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
asan_posix.cpp
asan_premap_shadow.cpp
asan_report.cpp
- asan_rtems.cpp
asan_rtl.cpp
asan_shadow_setup.cpp
asan_stack.cpp
asan_interface_internal.h
asan_internal.h
asan_lock.h
- asan_malloc_local.h
asan_mapping.h
- asan_mapping_myriad.h
asan_poisoning.h
asan_premap_shadow.h
asan_report.h
append_list_if(COMPILER_RT_HAS_LIBLOG log ASAN_DYNAMIC_LIBS)
append_list_if(MINGW "${MINGW_LIBRARIES}" ASAN_DYNAMIC_LIBS)
-if (TARGET cxx-headers OR HAVE_LIBCXX)
- set(ASAN_DEPS cxx-headers)
-endif()
-
# Compile ASan sources into an object library.
add_compiler_rt_object_libraries(RTAsan_dynamic
SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_DYNAMIC_CFLAGS}
- DEFS ${ASAN_DYNAMIC_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_DYNAMIC_DEFINITIONS})
if(NOT APPLE)
add_compiler_rt_object_libraries(RTAsan
SOURCES ${ASAN_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_CFLAGS}
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_cxx
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_CXX_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_CFLAGS}
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_object_libraries(RTAsan_preinit
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${ASAN_PREINIT_SOURCES}
ADDITIONAL_HEADERS ${ASAN_HEADERS}
CFLAGS ${ASAN_CFLAGS}
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
add_compiler_rt_object_libraries(RTAsan_dynamic_version_script_dummy
ARCHS ${ASAN_SUPPORTED_ARCH}
SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
CFLAGS ${ASAN_DYNAMIC_CFLAGS}
- DEFS ${ASAN_DYNAMIC_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_DYNAMIC_DEFINITIONS})
endif()
# Build ASan runtimes shipped with Clang.
PARENT_TARGET asan)
foreach(arch ${ASAN_SUPPORTED_ARCH})
- if (UNIX)
+ if (COMPILER_RT_HAS_VERSION_SCRIPT)
add_sanitizer_rt_version_list(clang_rt.asan-dynamic-${arch}
LIBS clang_rt.asan-${arch} clang_rt.asan_cxx-${arch}
EXTRA asan.syms.extra)
-Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.asan-dynamic-${arch}.vers)
# The Solaris 11.4 linker supports a subset of GNU ld version scripts,
# but requires a special option to enable it.
- if (OS_NAME MATCHES "SunOS")
+ if (COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
list(APPEND VERSION_SCRIPT_FLAG -Wl,-z,gnu-version-script-compat)
endif()
set_property(SOURCE
ARCHS ${arch}
SOURCES asan_win_weak_interception.cpp
CFLAGS ${ASAN_CFLAGS} -DSANITIZER_DYNAMIC
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
set(ASAN_DYNAMIC_WEAK_INTERCEPTION
AsanWeakInterception
UbsanWeakInterception
SOURCES asan_globals_win.cpp
asan_win_dll_thunk.cpp
CFLAGS ${ASAN_CFLAGS} -DSANITIZER_DLL_THUNK
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_runtime(clang_rt.asan_dll_thunk
STATIC
SOURCES asan_globals_win.cpp
asan_win_dynamic_runtime_thunk.cpp
CFLAGS ${ASAN_CFLAGS} ${DYNAMIC_RUNTIME_THUNK_CFLAGS}
- DEFS ${ASAN_COMMON_DEFINITIONS}
- DEPS ${ASAN_DEPS})
+ DEFS ${ASAN_COMMON_DEFINITIONS})
add_compiler_rt_runtime(clang_rt.asan_dynamic_runtime_thunk
STATIC
endforeach()
endif()
-add_compiler_rt_resource_file(asan_blacklist asan_blacklist.txt asan)
+add_compiler_rt_resource_file(asan_ignorelist asan_ignorelist.txt asan)
add_subdirectory(scripts)
//===----------------------------------------------------------------------===//
#include "asan_allocator.h"
+
#include "asan_mapping.h"
#include "asan_poisoning.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_thread.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_list.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_quarantine.h"
-#include "lsan/lsan_common.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
namespace __asan {
static AsanAllocator &get_allocator();
+static void AtomicContextStore(volatile atomic_uint64_t *atomic_context,
+ u32 tid, u32 stack) {
+ u64 context = tid;
+ context <<= 32;
+ context += stack;
+ atomic_store(atomic_context, context, memory_order_relaxed);
+}
+
+static void AtomicContextLoad(const volatile atomic_uint64_t *atomic_context,
+ u32 &tid, u32 &stack) {
+ u64 context = atomic_load(atomic_context, memory_order_relaxed);
+ stack = context;
+ context >>= 32;
+ tid = context;
+}
+
// The memory chunk allocated from the underlying allocator looks like this:
// L L L L L L H H U U U U U U R R
// L -- left redzone words (0 or more bytes)
// ---------------------|
// M -- magic value kAllocBegMagic
// B -- address of ChunkHeader pointing to the first 'H'
-static const uptr kAllocBegMagic = 0xCC6E96B9;
-
-struct ChunkHeader {
- // 1-st 8 bytes.
- u32 chunk_state : 8; // Must be first.
- u32 alloc_tid : 24;
-
- u32 free_tid : 24;
- u32 from_memalign : 1;
- u32 alloc_type : 2;
- u32 rz_log : 3;
- u32 lsan_tag : 2;
- // 2-nd 8 bytes
- // This field is used for small sizes. For large sizes it is equal to
- // SizeClassMap::kMaxSize and the actual size is stored in the
- // SecondaryAllocator's metadata.
- u32 user_requested_size : 29;
+
+class ChunkHeader {
+ public:
+ atomic_uint8_t chunk_state;
+ u8 alloc_type : 2;
+ u8 lsan_tag : 2;
+
// align < 8 -> 0
// else -> log2(min(align, 512)) - 2
- u32 user_requested_alignment_log : 3;
- u32 alloc_context_id;
+ u8 user_requested_alignment_log : 3;
+
+ private:
+ u16 user_requested_size_hi;
+ u32 user_requested_size_lo;
+ atomic_uint64_t alloc_context_id;
+
+ public:
+ uptr UsedSize() const {
+ uptr R = user_requested_size_lo;
+ if (sizeof(uptr) > sizeof(user_requested_size_lo))
+ R += (uptr)user_requested_size_hi << (8 * sizeof(user_requested_size_lo));
+ return R;
+ }
+
+ void SetUsedSize(uptr size) {
+ user_requested_size_lo = size;
+ if (sizeof(uptr) > sizeof(user_requested_size_lo)) {
+ size >>= (8 * sizeof(user_requested_size_lo));
+ user_requested_size_hi = size;
+ CHECK_EQ(user_requested_size_hi, size);
+ }
+ }
+
+ void SetAllocContext(u32 tid, u32 stack) {
+ AtomicContextStore(&alloc_context_id, tid, stack);
+ }
+
+ void GetAllocContext(u32 &tid, u32 &stack) const {
+ AtomicContextLoad(&alloc_context_id, tid, stack);
+ }
};
-struct ChunkBase : ChunkHeader {
- // Header2, intersects with user memory.
- u32 free_context_id;
+class ChunkBase : public ChunkHeader {
+ atomic_uint64_t free_context_id;
+
+ public:
+ void SetFreeContext(u32 tid, u32 stack) {
+ AtomicContextStore(&free_context_id, tid, stack);
+ }
+
+ void GetFreeContext(u32 &tid, u32 &stack) const {
+ AtomicContextLoad(&free_context_id, tid, stack);
+ }
};
static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
COMPILER_CHECK(kChunkHeaderSize == 16);
COMPILER_CHECK(kChunkHeader2Size <= 16);
-// Every chunk of memory allocated by this allocator can be in one of 3 states:
-// CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.
-// CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
-// CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone.
enum {
- CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it.
- CHUNK_ALLOCATED = 2,
- CHUNK_QUARANTINE = 3
+ // Either just allocated by underlying allocator, but AsanChunk is not yet
+ // ready, or almost returned to undelying allocator and AsanChunk is already
+ // meaningless.
+ CHUNK_INVALID = 0,
+ // The chunk is allocated and not yet freed.
+ CHUNK_ALLOCATED = 2,
+ // The chunk was freed and put into quarantine zone.
+ CHUNK_QUARANTINE = 3,
};
-struct AsanChunk: ChunkBase {
+class AsanChunk : public ChunkBase {
+ public:
uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
- uptr UsedSize(bool locked_version = false) {
- if (user_requested_size != SizeClassMap::kMaxSize)
- return user_requested_size;
- return *reinterpret_cast<uptr *>(
- get_allocator().GetMetaData(AllocBeg(locked_version)));
+ bool AddrIsInside(uptr addr) {
+ return (addr >= Beg()) && (addr < Beg() + UsedSize());
}
- void *AllocBeg(bool locked_version = false) {
- if (from_memalign) {
- if (locked_version)
- return get_allocator().GetBlockBeginFastLocked(
- reinterpret_cast<void *>(this));
- return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this));
- }
- return reinterpret_cast<void*>(Beg() - RZLog2Size(rz_log));
+};
+
+class LargeChunkHeader {
+ static constexpr uptr kAllocBegMagic =
+ FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
+ atomic_uintptr_t magic;
+ AsanChunk *chunk_header;
+
+ public:
+ AsanChunk *Get() const {
+ return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic
+ ? chunk_header
+ : nullptr;
}
- bool AddrIsInside(uptr addr, bool locked_version = false) {
- return (addr >= Beg()) && (addr < Beg() + UsedSize(locked_version));
+
+ void Set(AsanChunk *p) {
+ if (p) {
+ chunk_header = p;
+ atomic_store(&magic, kAllocBegMagic, memory_order_release);
+ return;
+ }
+
+ uptr old = kAllocBegMagic;
+ if (!atomic_compare_exchange_strong(&magic, &old, 0,
+ memory_order_release)) {
+ CHECK_EQ(old, kAllocBegMagic);
+ }
}
};
}
void Recycle(AsanChunk *m) {
- CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
- atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed);
- CHECK_NE(m->alloc_tid, kInvalidTid);
- CHECK_NE(m->free_tid, kInvalidTid);
- PoisonShadow(m->Beg(),
- RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
- kAsanHeapLeftRedzoneMagic);
- void *p = reinterpret_cast<void *>(m->AllocBeg());
+ void *p = get_allocator().GetBlockBegin(m);
if (p != m) {
- uptr *alloc_magic = reinterpret_cast<uptr *>(p);
- CHECK_EQ(alloc_magic[0], kAllocBegMagic);
// Clear the magic value, as allocator internals may overwrite the
// contents of deallocated chunk, confusing GetAsanChunk lookup.
- alloc_magic[0] = 0;
- CHECK_EQ(alloc_magic[1], reinterpret_cast<uptr>(m));
+ reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr);
+ }
+
+ u8 old_chunk_state = CHUNK_QUARANTINE;
+ if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
+ CHUNK_INVALID, memory_order_acquire)) {
+ CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
}
+ PoisonShadow(m->Beg(),
+ RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY),
+ kAsanHeapLeftRedzoneMagic);
+
// Statistics.
AsanStats &thread_stats = GetCurrentThreadStats();
thread_stats.real_frees++;
// This could be a user-facing chunk (with redzones), or some internal
// housekeeping chunk, like TransferBatch. Start by assuming the former.
AsanChunk *ac = GetAsanChunk((void *)chunk);
- uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)ac);
- uptr beg = ac->Beg();
- uptr end = ac->Beg() + ac->UsedSize(true);
- uptr chunk_end = chunk + allocated_size;
- if (chunk < beg && beg < end && end <= chunk_end &&
- ac->chunk_state == CHUNK_ALLOCATED) {
- // Looks like a valid AsanChunk in use, poison redzones only.
- PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
- uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
- FastPoisonShadowPartialRightRedzone(
- end_aligned_down, end - end_aligned_down,
- chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
- } else {
- // This is either not an AsanChunk or freed or quarantined AsanChunk.
- // In either case, poison everything.
- PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
+ uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)chunk);
+ if (ac && atomic_load(&ac->chunk_state, memory_order_acquire) ==
+ CHUNK_ALLOCATED) {
+ uptr beg = ac->Beg();
+ uptr end = ac->Beg() + ac->UsedSize();
+ uptr chunk_end = chunk + allocated_size;
+ if (chunk < beg && beg < end && end <= chunk_end) {
+ // Looks like a valid AsanChunk in use, poison redzones only.
+ PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic);
+ uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY);
+ FastPoisonShadowPartialRightRedzone(
+ end_aligned_down, end - end_aligned_down,
+ chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic);
+ return;
+ }
}
+
+ // This is either not an AsanChunk or freed or quarantined AsanChunk.
+ // In either case, poison everything.
+ PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
}
void ReInitialize(const AllocatorOptions &options) {
// -------------------- Helper methods. -------------------------
uptr ComputeRZLog(uptr user_requested_size) {
- u32 rz_log =
- user_requested_size <= 64 - 16 ? 0 :
- user_requested_size <= 128 - 32 ? 1 :
- user_requested_size <= 512 - 64 ? 2 :
- user_requested_size <= 4096 - 128 ? 3 :
- user_requested_size <= (1 << 14) - 256 ? 4 :
- user_requested_size <= (1 << 15) - 512 ? 5 :
- user_requested_size <= (1 << 16) - 1024 ? 6 : 7;
- u32 min_rz = atomic_load(&min_redzone, memory_order_acquire);
- u32 max_rz = atomic_load(&max_redzone, memory_order_acquire);
- return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz));
+ u32 rz_log = user_requested_size <= 64 - 16 ? 0
+ : user_requested_size <= 128 - 32 ? 1
+ : user_requested_size <= 512 - 64 ? 2
+ : user_requested_size <= 4096 - 128 ? 3
+ : user_requested_size <= (1 << 14) - 256 ? 4
+ : user_requested_size <= (1 << 15) - 512 ? 5
+ : user_requested_size <= (1 << 16) - 1024 ? 6
+ : 7;
+ u32 hdr_log = RZSize2Log(RoundUpToPowerOfTwo(sizeof(ChunkHeader)));
+ u32 min_log = RZSize2Log(atomic_load(&min_redzone, memory_order_acquire));
+ u32 max_log = RZSize2Log(atomic_load(&max_redzone, memory_order_acquire));
+ return Min(Max(rz_log, Max(min_log, hdr_log)), Max(max_log, hdr_log));
}
static uptr ComputeUserRequestedAlignmentLog(uptr user_requested_alignment) {
// We have an address between two chunks, and we want to report just one.
AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk,
AsanChunk *right_chunk) {
+ if (!left_chunk)
+ return right_chunk;
+ if (!right_chunk)
+ return left_chunk;
// Prefer an allocated chunk over freed chunk and freed chunk
// over available chunk.
- if (left_chunk->chunk_state != right_chunk->chunk_state) {
- if (left_chunk->chunk_state == CHUNK_ALLOCATED)
+ u8 left_state = atomic_load(&left_chunk->chunk_state, memory_order_relaxed);
+ u8 right_state =
+ atomic_load(&right_chunk->chunk_state, memory_order_relaxed);
+ if (left_state != right_state) {
+ if (left_state == CHUNK_ALLOCATED)
return left_chunk;
- if (right_chunk->chunk_state == CHUNK_ALLOCATED)
+ if (right_state == CHUNK_ALLOCATED)
return right_chunk;
- if (left_chunk->chunk_state == CHUNK_QUARANTINE)
+ if (left_state == CHUNK_QUARANTINE)
return left_chunk;
- if (right_chunk->chunk_state == CHUNK_QUARANTINE)
+ if (right_state == CHUNK_QUARANTINE)
return right_chunk;
}
// Same chunk_state: choose based on offset.
bool UpdateAllocationStack(uptr addr, BufferedStackTrace *stack) {
AsanChunk *m = GetAsanChunkByAddr(addr);
if (!m) return false;
- if (m->chunk_state != CHUNK_ALLOCATED) return false;
+ if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED)
+ return false;
if (m->Beg() != addr) return false;
- atomic_store((atomic_uint32_t *)&m->alloc_context_id, StackDepotPut(*stack),
- memory_order_relaxed);
+ AsanThread *t = GetCurrentThread();
+ m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack));
return true;
}
uptr needed_size = rounded_size + rz_size;
if (alignment > min_alignment)
needed_size += alignment;
- bool using_primary_allocator = true;
// If we are allocating from the secondary allocator, there will be no
// automatic right redzone, so add the right redzone manually.
- if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) {
+ if (!PrimaryAllocator::CanAllocate(needed_size, alignment))
needed_size += rz_size;
- using_primary_allocator = false;
- }
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
size > max_user_defined_malloc_size) {
uptr alloc_beg = reinterpret_cast<uptr>(allocated);
uptr alloc_end = alloc_beg + needed_size;
- uptr beg_plus_redzone = alloc_beg + rz_size;
- uptr user_beg = beg_plus_redzone;
+ uptr user_beg = alloc_beg + rz_size;
if (!IsAligned(user_beg, alignment))
user_beg = RoundUpTo(user_beg, alignment);
uptr user_end = user_beg + size;
uptr chunk_beg = user_beg - kChunkHeaderSize;
AsanChunk *m = reinterpret_cast<AsanChunk *>(chunk_beg);
m->alloc_type = alloc_type;
- m->rz_log = rz_log;
- u32 alloc_tid = t ? t->tid() : 0;
- m->alloc_tid = alloc_tid;
- CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield?
- m->free_tid = kInvalidTid;
- m->from_memalign = user_beg != beg_plus_redzone;
- if (alloc_beg != chunk_beg) {
- CHECK_LE(alloc_beg+ 2 * sizeof(uptr), chunk_beg);
- reinterpret_cast<uptr *>(alloc_beg)[0] = kAllocBegMagic;
- reinterpret_cast<uptr *>(alloc_beg)[1] = chunk_beg;
- }
- if (using_primary_allocator) {
- CHECK(size);
- m->user_requested_size = size;
- CHECK(allocator.FromPrimary(allocated));
- } else {
- CHECK(!allocator.FromPrimary(allocated));
- m->user_requested_size = SizeClassMap::kMaxSize;
- uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(allocated));
- meta[0] = size;
- meta[1] = chunk_beg;
- }
+ CHECK(size);
+ m->SetUsedSize(size);
m->user_requested_alignment_log = user_requested_alignment_log;
- m->alloc_context_id = StackDepotPut(*stack);
+ m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack));
uptr size_rounded_down_to_granularity =
RoundDownTo(size, SHADOW_GRANULARITY);
: __lsan::kDirectlyLeaked;
#endif
// Must be the last mutation of metadata in this function.
- atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release);
+ atomic_store(&m->chunk_state, CHUNK_ALLOCATED, memory_order_release);
+ if (alloc_beg != chunk_beg) {
+ CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
+ reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
+ }
ASAN_MALLOC_HOOK(res, size);
return res;
}
// Set quarantine flag if chunk is allocated, issue ASan error report on
// available and quarantined chunks. Return true on success, false otherwise.
bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr,
- BufferedStackTrace *stack) {
+ BufferedStackTrace *stack) {
u8 old_chunk_state = CHUNK_ALLOCATED;
// Flip the chunk_state atomically to avoid race on double-free.
- if (!atomic_compare_exchange_strong((atomic_uint8_t *)m, &old_chunk_state,
+ if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
CHUNK_QUARANTINE,
memory_order_acquire)) {
ReportInvalidFree(ptr, old_chunk_state, stack);
return false;
}
CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state);
+ // It was a user data.
+ m->SetFreeContext(kInvalidTid, 0);
return true;
}
// Expects the chunk to already be marked as quarantined by using
// AtomicallySetQuarantineFlagIfAllocated.
void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack) {
- CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE);
- CHECK_GE(m->alloc_tid, 0);
- if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area.
- CHECK_EQ(m->free_tid, kInvalidTid);
+ CHECK_EQ(atomic_load(&m->chunk_state, memory_order_relaxed),
+ CHUNK_QUARANTINE);
AsanThread *t = GetCurrentThread();
- m->free_tid = t ? t->tid() : 0;
- m->free_context_id = StackDepotPut(*stack);
+ m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack));
Flags &fl = *flags();
if (fl.max_free_fill_size > 0) {
void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true);
if (new_ptr) {
- u8 chunk_state = m->chunk_state;
+ u8 chunk_state = atomic_load(&m->chunk_state, memory_order_acquire);
if (chunk_state != CHUNK_ALLOCATED)
ReportInvalidFree(old_ptr, chunk_state, stack);
CHECK_NE(REAL(memcpy), nullptr);
// -------------------------- Chunk lookup ----------------------
// Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
+ // Returns nullptr if AsanChunk is not yet initialized just after
+ // get_allocator().Allocate(), or is being destroyed just before
+ // get_allocator().Deallocate().
AsanChunk *GetAsanChunk(void *alloc_beg) {
- if (!alloc_beg) return nullptr;
- if (!allocator.FromPrimary(alloc_beg)) {
- uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(alloc_beg));
- AsanChunk *m = reinterpret_cast<AsanChunk *>(meta[1]);
- return m;
+ if (!alloc_beg)
+ return nullptr;
+ AsanChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
+ if (!p) {
+ if (!allocator.FromPrimary(alloc_beg))
+ return nullptr;
+ p = reinterpret_cast<AsanChunk *>(alloc_beg);
}
- uptr *alloc_magic = reinterpret_cast<uptr *>(alloc_beg);
- if (alloc_magic[0] == kAllocBegMagic)
- return reinterpret_cast<AsanChunk *>(alloc_magic[1]);
- return reinterpret_cast<AsanChunk *>(alloc_beg);
+ u8 state = atomic_load(&p->chunk_state, memory_order_relaxed);
+ // It does not guaranty that Chunk is initialized, but it's
+ // definitely not for any other value.
+ if (state == CHUNK_ALLOCATED || state == CHUNK_QUARANTINE)
+ return p;
+ return nullptr;
}
AsanChunk *GetAsanChunkByAddr(uptr p) {
uptr AllocationSize(uptr p) {
AsanChunk *m = GetAsanChunkByAddr(p);
if (!m) return 0;
- if (m->chunk_state != CHUNK_ALLOCATED) return 0;
+ if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED)
+ return 0;
if (m->Beg() != p) return 0;
return m->UsedSize();
}
AsanChunkView FindHeapChunkByAddress(uptr addr) {
AsanChunk *m1 = GetAsanChunkByAddr(addr);
- if (!m1) return AsanChunkView(m1);
sptr offset = 0;
- if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) {
+ if (!m1 || AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) {
// The address is in the chunk's left redzone, so maybe it is actually
// a right buffer overflow from the other chunk to the left.
// Search a bit to the left to see if there is another chunk.
quarantine.PrintStats();
}
- void ForceLock() {
+ void ForceLock() ACQUIRE(fallback_mutex) {
allocator.ForceLock();
fallback_mutex.Lock();
}
- void ForceUnlock() {
+ void ForceUnlock() RELEASE(fallback_mutex) {
fallback_mutex.Unlock();
allocator.ForceUnlock();
}
}
bool AsanChunkView::IsValid() const {
- return chunk_ && chunk_->chunk_state != CHUNK_AVAILABLE;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) !=
+ CHUNK_INVALID;
}
bool AsanChunkView::IsAllocated() const {
- return chunk_ && chunk_->chunk_state == CHUNK_ALLOCATED;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) ==
+ CHUNK_ALLOCATED;
}
bool AsanChunkView::IsQuarantined() const {
- return chunk_ && chunk_->chunk_state == CHUNK_QUARANTINE;
+ return chunk_ && atomic_load(&chunk_->chunk_state, memory_order_relaxed) ==
+ CHUNK_QUARANTINE;
}
uptr AsanChunkView::Beg() const { return chunk_->Beg(); }
uptr AsanChunkView::End() const { return Beg() + UsedSize(); }
u32 AsanChunkView::UserRequestedAlignment() const {
return Allocator::ComputeUserAlignment(chunk_->user_requested_alignment_log);
}
-uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; }
-uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; }
+
+uptr AsanChunkView::AllocTid() const {
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetAllocContext(tid, stack);
+ return tid;
+}
+
+uptr AsanChunkView::FreeTid() const {
+ if (!IsQuarantined())
+ return kInvalidTid;
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetFreeContext(tid, stack);
+ return tid;
+}
+
AllocType AsanChunkView::GetAllocType() const {
return (AllocType)chunk_->alloc_type;
}
return res;
}
-u32 AsanChunkView::GetAllocStackId() const { return chunk_->alloc_context_id; }
-u32 AsanChunkView::GetFreeStackId() const { return chunk_->free_context_id; }
+u32 AsanChunkView::GetAllocStackId() const {
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetAllocContext(tid, stack);
+ return stack;
+}
+
+u32 AsanChunkView::GetFreeStackId() const {
+ if (!IsQuarantined())
+ return 0;
+ u32 tid = 0;
+ u32 stack = 0;
+ chunk_->GetFreeContext(tid, stack);
+ return stack;
+}
StackTrace AsanChunkView::GetAllocStack() const {
return GetStackTraceFromId(GetAllocStackId());
return instance.AllocationSize(reinterpret_cast<uptr>(ptr));
}
-void asan_mz_force_lock() {
- instance.ForceLock();
-}
+void asan_mz_force_lock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceLock(); }
-void asan_mz_force_unlock() {
+void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS {
instance.ForceUnlock();
}
instance.SetRssLimitExceeded(limit_exceeded);
}
-} // namespace __asan
+} // namespace __asan
// --- Implementation of LSan-specific functions --- {{{1
namespace __lsan {
*end = *begin + sizeof(__asan::get_allocator());
}
-uptr PointsIntoChunk(void* p) {
+uptr PointsIntoChunk(void *p) {
uptr addr = reinterpret_cast<uptr>(p);
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(addr);
- if (!m) return 0;
- uptr chunk = m->Beg();
- if (m->chunk_state != __asan::CHUNK_ALLOCATED)
+ if (!m || atomic_load(&m->chunk_state, memory_order_acquire) !=
+ __asan::CHUNK_ALLOCATED)
return 0;
- if (m->AddrIsInside(addr, /*locked_version=*/true))
+ uptr chunk = m->Beg();
+ if (m->AddrIsInside(addr))
return chunk;
- if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(/*locked_version*/ true),
- addr))
+ if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(), addr))
return chunk;
return 0;
}
-// Debug code. Delete once issue #1193 is chased down.
-extern "C" SANITIZER_WEAK_ATTRIBUTE const char *__lsan_current_stage;
-
uptr GetUserBegin(uptr chunk) {
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk);
- if (!m)
- Printf(
- "ASAN is about to crash with a CHECK failure.\n"
- "The ASAN developers are trying to chase down this bug,\n"
- "so if you've encountered this bug please let us know.\n"
- "See also: https://github.com/google/sanitizers/issues/1193\n"
- "chunk: %p caller %p __lsan_current_stage %s\n",
- chunk, GET_CALLER_PC(), __lsan_current_stage);
- CHECK(m);
- return m->Beg();
+ return m ? m->Beg() : 0;
}
LsanMetadata::LsanMetadata(uptr chunk) {
- metadata_ = reinterpret_cast<void *>(chunk - __asan::kChunkHeaderSize);
+ metadata_ = chunk ? reinterpret_cast<void *>(chunk - __asan::kChunkHeaderSize)
+ : nullptr;
}
bool LsanMetadata::allocated() const {
+ if (!metadata_)
+ return false;
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->chunk_state == __asan::CHUNK_ALLOCATED;
+ return atomic_load(&m->chunk_state, memory_order_relaxed) ==
+ __asan::CHUNK_ALLOCATED;
}
ChunkTag LsanMetadata::tag() const {
uptr LsanMetadata::requested_size() const {
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->UsedSize(/*locked_version=*/true);
+ return m->UsedSize();
}
u32 LsanMetadata::stack_trace_id() const {
__asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_);
- return m->alloc_context_id;
+ u32 tid = 0;
+ u32 stack = 0;
+ m->GetAllocContext(tid, stack);
+ return stack;
}
void ForEachChunk(ForEachChunkCallback callback, void *arg) {
IgnoreObjectResult IgnoreObjectLocked(const void *p) {
uptr addr = reinterpret_cast<uptr>(p);
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr);
- if (!m) return kIgnoreObjectInvalid;
- if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) {
- if (m->lsan_tag == kIgnored)
- return kIgnoreObjectAlreadyIgnored;
- m->lsan_tag = __lsan::kIgnored;
- return kIgnoreObjectSuccess;
- } else {
+ if (!m ||
+ (atomic_load(&m->chunk_state, memory_order_acquire) !=
+ __asan::CHUNK_ALLOCATED) ||
+ !m->AddrIsInside(addr)) {
return kIgnoreObjectInvalid;
}
+ if (m->lsan_tag == kIgnored)
+ return kIgnoreObjectAlreadyIgnored;
+ m->lsan_tag = __lsan::kIgnored;
+ return kIgnoreObjectSuccess;
+}
+
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
+ // Look for the arg pointer of threads that have been created or are running.
+ // This is necessary to prevent false positive leaks due to the AsanThread
+ // holding the only live reference to a heap object. This can happen because
+ // the `pthread_create()` interceptor doesn't wait for the child thread to
+ // start before returning and thus loosing the the only live reference to the
+ // heap object on the stack.
+
+ __asan::AsanThreadContext *atctx =
+ reinterpret_cast<__asan::AsanThreadContext *>(tctx);
+ __asan::AsanThread *asan_thread = atctx->thread;
+
+ // Note ThreadStatusRunning is required because there is a small window where
+ // the thread status switches to `ThreadStatusRunning` but the `arg` pointer
+ // still isn't on the stack yet.
+ if (atctx->status != ThreadStatusCreated &&
+ atctx->status != ThreadStatusRunning)
+ return;
+
+ uptr thread_arg = reinterpret_cast<uptr>(asan_thread->get_arg());
+ if (!thread_arg)
+ return;
+
+ auto ptrsVec = reinterpret_cast<InternalMmapVector<uptr> *>(ptrs);
+ ptrsVec->push_back(thread_arg);
}
+
} // namespace __lsan
// ---------------------- Interface ---------------- {{{1
#define ASAN_ALLOCATOR_H
#include "asan_flags.h"
-#include "asan_internal.h"
#include "asan_interceptors.h"
+#include "asan_internal.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_list.h"
+#include "sanitizer_common/sanitizer_platform.h"
namespace __asan {
FROM_NEW_BR = 3 // Memory block came from operator new [ ]
};
-struct AsanChunk;
+class AsanChunk;
struct AllocatorOptions {
u32 quarantine_size_mb;
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
typedef VeryCompactSizeClassMap SizeClassMap;
+#elif SANITIZER_RISCV64
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
+typedef VeryDenseSizeClassMap SizeClassMap;
# elif defined(__aarch64__)
// AArch64/SANITIZER_CAN_USE_ALLOCATOR64 is only for 42-bit VMA
// so no need to different values for different VMA.
struct AP32 {
static const uptr kSpaceBeg = 0;
static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;
- static const uptr kMetadataSize = 16;
+ static const uptr kMetadataSize = 0;
typedef __asan::SizeClassMap SizeClassMap;
static const uptr kRegionSizeLog = 20;
using AddressSpaceView = AddressSpaceViewTy;
CHECK(context);
asanThreadRegistry().CheckLocked();
// No need to announce the main thread.
- if (context->tid == 0 || context->announced) {
+ if (context->tid == kMainTid || context->announced) {
return;
}
context->announced = true;
- InternalScopedString str(1024);
+ InternalScopedString str;
str.append("Thread %s", AsanThreadIdAndName(context).c_str());
if (context->parent_tid == kInvalidTid) {
str.append(" created by unknown thread\n");
} else if (AddrIsInLowShadow(addr)) {
*shadow_kind = kShadowKindLow;
} else {
- CHECK(0 && "Address is not in memory and not in shadow?");
return false;
}
return true;
static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) {
Decorator d;
- InternalScopedString str(4096);
+ InternalScopedString str;
str.append("%s", d.Location());
switch (descr.access_type) {
case kAccessTypeLeft:
else if (addr >= prev_var_end && addr - prev_var_end >= var.beg - addr_end)
pos_descr = "underflows";
}
- InternalScopedString str(1024);
+ InternalScopedString str;
str.append(" [%zd, %zd)", var.beg, var_end);
// Render variable name.
str.append(" '");
// Global descriptions
static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size,
const __asan_global &g) {
- InternalScopedString str(4096);
+ InternalScopedString str;
Decorator d;
str.append("%s", d.Location());
if (addr < g.beg) {
return;
}
data.kind = kAddressKindWild;
- addr = 0;
+ data.wild.addr = addr;
+ data.wild.access_size = access_size;
+}
+
+void WildAddressDescription::Print() const {
+ Printf("Address %p is a wild pointer inside of access range of size %p.\n",
+ addr, access_size);
}
void PrintAddressDescription(uptr addr, uptr access_size,
bool GetStackAddressInformation(uptr addr, uptr access_size,
StackAddressDescription *descr);
+struct WildAddressDescription {
+ uptr addr;
+ uptr access_size;
+
+ void Print() const;
+};
+
struct GlobalAddressDescription {
uptr addr;
// Assume address is close to at most four globals.
HeapAddressDescription heap;
StackAddressDescription stack;
GlobalAddressDescription global;
- uptr addr;
+ WildAddressDescription wild;
};
};
uptr Address() const {
switch (data.kind) {
case kAddressKindWild:
- return data.addr;
+ return data.wild.addr;
case kAddressKindShadow:
return data.shadow.addr;
case kAddressKindHeap:
void Print(const char *bug_descr = nullptr) const {
switch (data.kind) {
case kAddressKindWild:
- Printf("Address %p is a wild pointer.\n", data.addr);
+ data.wild.Print();
return;
case kAddressKindShadow:
return data.shadow.Print();
Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
global1.beg);
Printf("%s", d.Default());
- InternalScopedString g1_loc(256), g2_loc(256);
+ InternalScopedString g1_loc;
+ InternalScopedString g2_loc;
PrintGlobalLocation(&g1_loc, global1);
PrintGlobalLocation(&g2_loc, global2);
Printf(" [1] size=%zd '%s' %s\n", global1.size,
Report(
"HINT: if you don't care about these errors you may set "
"ASAN_OPTIONS=detect_odr_violation=0\n");
- InternalScopedString error_msg(256);
+ InternalScopedString error_msg;
error_msg.append("%s: global '%s' at %s", scariness.GetDescription(),
MaybeDemangleGlobalName(global1.name), g1_loc.data());
ReportErrorSummary(error_msg.data());
PrintShadowByte(str, " ASan internal: ", kAsanInternalHeapMagic);
PrintShadowByte(str, " Left alloca redzone: ", kAsanAllocaLeftMagic);
PrintShadowByte(str, " Right alloca redzone: ", kAsanAllocaRightMagic);
- PrintShadowByte(str, " Shadow gap: ", kAsanShadowGap);
}
static void PrintShadowBytes(InternalScopedString *str, const char *before,
uptr shadow_addr = MemToShadow(addr);
const uptr n_bytes_per_row = 16;
uptr aligned_shadow = shadow_addr & ~(n_bytes_per_row - 1);
- InternalScopedString str(4096 * 8);
+ InternalScopedString str;
str.append("Shadow bytes around the buggy address:\n");
for (int i = -5; i <= 5; i++) {
uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row;
void FakeStack::Destroy(int tid) {
PoisonAll(0);
if (Verbosity() >= 2) {
- InternalScopedString str(kNumberOfSizeClasses * 50);
+ InternalScopedString str;
for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++)
str.append("%zd: %zd/%zd; ", class_id, hint_position_[class_id],
NumberOfFrames(stack_size_log(), class_id));
static FakeStack *GetFakeStack() {
AsanThread *t = GetCurrentThread();
if (!t) return nullptr;
- return t->fake_stack();
+ return t->get_or_create_fake_stack();
}
static FakeStack *GetFakeStackFast() {
return GetFakeStack();
}
-ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) {
+static FakeStack *GetFakeStackFastAlways() {
+ if (FakeStack *fs = GetTLSFakeStack())
+ return fs;
+ return GetFakeStack();
+}
+
+static ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) {
FakeStack *fs = GetFakeStackFast();
if (!fs) return 0;
uptr local_stack;
return ptr;
}
-ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) {
+static ALWAYS_INLINE uptr OnMallocAlways(uptr class_id, uptr size) {
+ FakeStack *fs = GetFakeStackFastAlways();
+ if (!fs)
+ return 0;
+ uptr local_stack;
+ uptr real_stack = reinterpret_cast<uptr>(&local_stack);
+ FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack);
+ if (!ff)
+ return 0; // Out of fake stack.
+ uptr ptr = reinterpret_cast<uptr>(ff);
+ SetShadow(ptr, size, class_id, 0);
+ return ptr;
+}
+
+static ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) {
FakeStack::Deallocate(ptr, class_id);
SetShadow(ptr, size, class_id, kMagic8);
}
// ---------------------- Interface ---------------- {{{1
using namespace __asan;
-#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
- __asan_stack_malloc_##class_id(uptr size) { \
- return OnMalloc(class_id, size); \
- } \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \
- uptr ptr, uptr size) { \
- OnFree(ptr, class_id, size); \
+#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
+ __asan_stack_malloc_##class_id(uptr size) { \
+ return OnMalloc(class_id, size); \
+ } \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \
+ __asan_stack_malloc_always_##class_id(uptr size) { \
+ return OnMallocAlways(class_id, size); \
+ } \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \
+ uptr ptr, uptr size) { \
+ OnFree(ptr, class_id, size); \
}
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(0)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(8)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(9)
DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(10)
+
extern "C" {
+// TODO: remove this method and fix tests that use it by setting
+// -asan-use-after-return=never, after modal UAR flag lands
+// (https://github.com/google/sanitizers/issues/1394)
SANITIZER_INTERFACE_ATTRIBUTE
void *__asan_get_current_fake_stack() { return GetFakeStackFast(); }
Flags asan_flags_dont_use_directly; // use via flags().
-static const char *MaybeCallAsanDefaultOptions() {
- return (&__asan_default_options) ? __asan_default_options() : "";
-}
-
static const char *MaybeUseAsanDefaultOptionsCompileDefinition() {
#ifdef ASAN_DEFAULT_OPTIONS
return SANITIZER_STRINGIFY(ASAN_DEFAULT_OPTIONS);
asan_parser.ParseString(asan_compile_def);
// Override from user-specified string.
- const char *asan_default_options = MaybeCallAsanDefaultOptions();
+ const char *asan_default_options = __asan_default_options();
asan_parser.ParseString(asan_default_options);
#if CAN_SANITIZE_UB
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
#endif
#if CAN_SANITIZE_LEAKS
- const char *lsan_default_options = __lsan::MaybeCallLsanDefaultOptions();
+ const char *lsan_default_options = __lsan_default_options();
lsan_parser.ParseString(lsan_default_options);
#endif
CHECK_LE(f->max_redzone, 2048);
CHECK(IsPowerOfTwo(f->redzone));
CHECK(IsPowerOfTwo(f->max_redzone));
- if (SANITIZER_RTEMS) {
- CHECK(!f->unmap_shadow_on_exit);
- CHECK(!f->protect_shadow_gap);
- }
// quarantine_size is deprecated but we still honor it.
// quarantine_size can not be used together with quarantine_size_mb.
"295.*.")
ASAN_FLAG(bool, unmap_shadow_on_exit, false,
"If set, explicitly unmaps the (huge) shadow at exit.")
-ASAN_FLAG(bool, protect_shadow_gap, !SANITIZER_RTEMS,
- "If set, mprotect the shadow gap")
+ASAN_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap")
ASAN_FLAG(bool, print_stats, false,
"Print various statistics after printing an error message or if "
"atexit=1.")
void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
static inline size_t AsanThreadMmapSize() {
- return RoundUpTo(sizeof(AsanThread), PAGE_SIZE);
+ return RoundUpTo(sizeof(AsanThread), _zx_system_get_page_size());
}
struct AsanThread::InitOptions {
// Shared setup between thread creation and startup for the initial thread.
static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid,
uptr user_id, bool detached,
- const char *name, uptr stack_bottom,
- uptr stack_size) {
+ const char *name) {
// In lieu of AsanThread::Create.
AsanThread *thread = (AsanThread *)MmapOrDie(AsanThreadMmapSize(), __func__);
asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args);
asanThreadRegistry().SetThreadName(tid, name);
- // On other systems, AsanThread::Init() is called from the new
- // thread itself. But on Fuchsia we already know the stack address
- // range beforehand, so we can do most of the setup right now.
- const AsanThread::InitOptions options = {stack_bottom, stack_size};
- thread->Init(&options);
-
return thread;
}
_zx_object_get_property(thrd_get_zx_handle(self), ZX_PROP_NAME, name,
sizeof(name)) == ZX_OK
? name
- : nullptr,
- __sanitizer::MainThreadStackBase, __sanitizer::MainThreadStackSize);
+ : nullptr);
+ // We need to set the current thread before calling AsanThread::Init() below,
+ // since it reads the thread ID.
SetCurrentThread(t);
+ DCHECK_EQ(t->tid(), 0);
+
+ const AsanThread::InitOptions options = {__sanitizer::MainThreadStackBase,
+ __sanitizer::MainThreadStackSize};
+ t->Init(&options);
+
return t;
}
GET_STACK_TRACE_THREAD;
u32 parent_tid = GetCurrentTidOrInvalid();
- return CreateAsanThread(&stack, parent_tid, user_id, detached, name,
- stack_bottom, stack_size);
+ AsanThread *thread =
+ CreateAsanThread(&stack, parent_tid, user_id, detached, name);
+
+ // On other systems, AsanThread::Init() is called from the new
+ // thread itself. But on Fuchsia we already know the stack address
+ // range beforehand, so we can do most of the setup right now.
+ const AsanThread::InitOptions options = {stack_bottom, stack_size};
+ thread->Init(&options);
+ return thread;
}
// This is called after creating a new thread (in the creating thread),
return false;
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ __sanitizer_fill_shadow(p, size, 0, 0);
+}
+
} // namespace __asan
// These are declared (in extern "C") by <zircon/sanitizer.h>.
--- /dev/null
+# Ignorelist for AddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of ignorelist
+# at compile-time using -fsanitize-ignorelist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
+# global:*global_with_bad_access_or_initialization*
+# global:*global_with_initialization_issues*=init
+# type:*Namespace::ClassName*=init
+
+# Stack buffer overflow in VC/INCLUDE/xlocnum, see http://goo.gl/L4qqUG
+fun:*_Find_elem@*@std*
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_libc.h"
-// There is no general interception at all on Fuchsia and RTEMS.
+// There is no general interception at all on Fuchsia.
// Only the functions in asan_interceptors_memintrinsics.cpp are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#if !SANITIZER_FUCHSIA
-#if SANITIZER_POSIX
-#include "sanitizer_common/sanitizer_posix.h"
-#endif
+# if SANITIZER_POSIX
+# include "sanitizer_common/sanitizer_posix.h"
+# endif
-#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \
- ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
-#include <unwind.h>
-#endif
+# if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \
+ ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION
+# include <unwind.h>
+# endif
-#if defined(__i386) && SANITIZER_LINUX
-#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1"
-#elif defined(__mips__) && SANITIZER_LINUX
-#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2"
-#endif
+# if defined(__i386) && SANITIZER_LINUX
+# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1"
+# elif defined(__mips__) && SANITIZER_LINUX
+# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2"
+# endif
namespace __asan {
(void) ctx; \
#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
-#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
ASAN_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
+ ASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)
#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
ASAN_WRITE_RANGE(ctx, ptr, size)
#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
#include "sanitizer_common/sanitizer_common_syscalls.inc"
#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
-struct ThreadStartParam {
- atomic_uintptr_t t;
- atomic_uintptr_t is_registered;
-};
-
#if ASAN_INTERCEPT_PTHREAD_CREATE
static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
- ThreadStartParam *param = reinterpret_cast<ThreadStartParam *>(arg);
- AsanThread *t = nullptr;
- while ((t = reinterpret_cast<AsanThread *>(
- atomic_load(¶m->t, memory_order_acquire))) == nullptr)
- internal_sched_yield();
+ AsanThread *t = (AsanThread *)arg;
SetCurrentThread(t);
- return t->ThreadStart(GetTid(), ¶m->is_registered);
+ return t->ThreadStart(GetTid());
}
INTERCEPTOR(int, pthread_create, void *thread,
int detached = 0;
if (attr)
REAL(pthread_attr_getdetachstate)(attr, &detached);
- ThreadStartParam param;
- atomic_store(¶m.t, 0, memory_order_relaxed);
- atomic_store(¶m.is_registered, 0, memory_order_relaxed);
+
+ u32 current_tid = GetCurrentTidOrInvalid();
+ AsanThread *t =
+ AsanThread::Create(start_routine, arg, current_tid, &stack, detached);
+
int result;
{
// Ignore all allocations made by pthread_create: thread stack/TLS may be
#if CAN_SANITIZE_LEAKS
__lsan::ScopedInterceptorDisabler disabler;
#endif
- result = REAL(pthread_create)(thread, attr, asan_thread_start, ¶m);
+ result = REAL(pthread_create)(thread, attr, asan_thread_start, t);
}
- if (result == 0) {
- u32 current_tid = GetCurrentTidOrInvalid();
- AsanThread *t =
- AsanThread::Create(start_routine, arg, current_tid, &stack, detached);
- atomic_store(¶m.t, reinterpret_cast<uptr>(t), memory_order_release);
- // Wait until the AsanThread object is initialized and the ThreadRegistry
- // entry is in "started" state. One reason for this is that after this
- // interceptor exits, the child thread's stack may be the only thing holding
- // the |arg| pointer. This may cause LSan to report a leak if leak checking
- // happens at a point when the interceptor has already exited, but the stack
- // range for the child thread is not yet known.
- while (atomic_load(¶m.is_registered, memory_order_acquire) == 0)
- internal_sched_yield();
+ if (result != 0) {
+ // If the thread didn't start delete the AsanThread to avoid leaking it.
+ // Note AsanThreadContexts never get destroyed so the AsanThreadContext
+ // that was just created for the AsanThread is wasted.
+ t->Destroy();
}
return result;
}
// Intercept threading-related functions
#if ASAN_INTERCEPT_PTHREAD_CREATE
+// TODO: this should probably have an unversioned fallback for newer arches?
#if defined(ASAN_PTHREAD_CREATE_VERSION)
ASAN_INTERCEPT_FUNC_VER(pthread_create, ASAN_PTHREAD_CREATE_VERSION);
#else
#ifndef ASAN_INTERCEPTORS_H
#define ASAN_INTERCEPTORS_H
-#include "asan_internal.h"
#include "asan_interceptors_memintrinsics.h"
+#include "asan_internal.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_platform.h"
#include "sanitizer_common/sanitizer_platform_interceptors.h"
namespace __asan {
} // namespace __asan
-// There is no general interception at all on Fuchsia and RTEMS.
+// There is no general interception at all on Fuchsia.
// Only the functions in asan_interceptors_memintrinsics.h are
// really defined to replace libc functions.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#if !SANITIZER_FUCHSIA
// Use macro to describe if specific function should be
// intercepted on a given platform.
# define ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX 0
#endif
-#if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_SOLARIS
+#if SANITIZER_GLIBC || SANITIZER_SOLARIS
# define ASAN_INTERCEPT_SWAPCONTEXT 1
#else
# define ASAN_INTERCEPT_SWAPCONTEXT 0
# define ASAN_INTERCEPT_SIGLONGJMP 0
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
# define ASAN_INTERCEPT___LONGJMP_CHK 1
#else
# define ASAN_INTERCEPT___LONGJMP_CHK 0
# define ASAN_INTERCEPT_ATEXIT 0
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
# define ASAN_INTERCEPT___STRDUP 1
#else
# define ASAN_INTERCEPT___STRDUP 0
#endif
-#if SANITIZER_LINUX && (defined(__arm__) || defined(__aarch64__) || \
- defined(__i386__) || defined(__x86_64__))
+#if SANITIZER_LINUX && \
+ (defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \
+ defined(__x86_64__) || SANITIZER_RISCV64)
# define ASAN_INTERCEPT_VFORK 1
#else
# define ASAN_INTERCEPT_VFORK 0
DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
#if !SANITIZER_MAC
-#define ASAN_INTERCEPT_FUNC(name) \
- do { \
- if (!INTERCEPT_FUNCTION(name)) \
- VReport(1, "AddressSanitizer: failed to intercept '%s'\n'", #name); \
+#define ASAN_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "AddressSanitizer: failed to intercept '%s'\n", #name); \
} while (0)
#define ASAN_INTERCEPT_FUNC_VER(name, ver) \
do { \
VReport(1, "AddressSanitizer: failed to intercept '%s@@%s'\n", #name, \
#ver); \
} while (0)
+#define ASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
+ VReport(1, "AddressSanitizer: failed to intercept '%s@@%s' or '%s'\n", \
+ #name, #ver, #name); \
+ } while (0)
+
#else
// OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION.
#define ASAN_INTERCEPT_FUNC(name)
ASAN_MEMMOVE_IMPL(nullptr, to, from, size);
}
-#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
+#if SANITIZER_FUCHSIA
-// Fuchsia and RTEMS don't use sanitizer_common_interceptors.inc, but
+// Fuchsia doesn't use sanitizer_common_interceptors.inc, but
// the only things there it wants are these three. Just define them
// as aliases here rather than repeating the contents.
extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]];
extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]];
-#endif // SANITIZER_FUCHSIA || SANITIZER_RTEMS
+#endif // SANITIZER_FUCHSIA
#define COMMON_INTERCEPTOR_HANDLE_VFORK __asan_handle_vfork
#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S"
-#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
#endif
NO_EXEC_STACK_DIRECTIVE
INTERFACE_FUNCTION(__asan_stack_malloc_8)
INTERFACE_FUNCTION(__asan_stack_malloc_9)
INTERFACE_FUNCTION(__asan_stack_malloc_10)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_0)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_1)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_2)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_3)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_4)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_5)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_6)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_7)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_8)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_9)
+INTERFACE_FUNCTION(__asan_stack_malloc_always_10)
INTERFACE_FUNCTION(__asan_store1)
INTERFACE_FUNCTION(__asan_store2)
INTERFACE_FUNCTION(__asan_store4)
SANITIZER_INTERFACE_ATTRIBUTE void __asan_print_accumulated_stats();
- SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
- const char* __asan_default_options();
+ SANITIZER_INTERFACE_ATTRIBUTE
+ const char *__asan_default_options();
SANITIZER_INTERFACE_ATTRIBUTE
extern uptr __asan_shadow_memory_dynamic_address;
// If set, values like allocator chunk size, as well as defaults for some flags
// will be changed towards less memory overhead.
#ifndef ASAN_LOW_MEMORY
-# if SANITIZER_IOS || SANITIZER_ANDROID || SANITIZER_RTEMS
-# define ASAN_LOW_MEMORY 1
-# else
-# define ASAN_LOW_MEMORY 0
-# endif
+# if SANITIZER_IOS || SANITIZER_ANDROID
+# define ASAN_LOW_MEMORY 1
+# else
+# define ASAN_LOW_MEMORY 0
+# endif
#endif
#ifndef ASAN_DYNAMIC
// asan_malloc_linux.cpp / asan_malloc_mac.cpp
void ReplaceSystemMalloc();
-// asan_linux.cpp / asan_mac.cpp / asan_rtems.cpp / asan_win.cpp
+// asan_linux.cpp / asan_mac.cpp / asan_win.cpp
uptr FindDynamicShadowStart();
void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
void *AsanDlSymNext(const char *sym);
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name);
-
// Returns `true` iff most of ASan init process should be skipped due to the
// ASan library being loaded via `dlopen()`. Platforms may perform any
// `dlopen()` specific initialization inside this function.
const int kAsanIntraObjectRedzone = 0xbb;
const int kAsanAllocaLeftMagic = 0xca;
const int kAsanAllocaRightMagic = 0xcb;
-// Used to populate the shadow gap for systems without memory
-// protection there (i.e. Myriad).
-const int kAsanShadowGap = 0xcc;
static const uptr kCurrentStackFrameMagic = 0x41B58AB3;
static const uptr kRetiredStackFrameMagic = 0x45E0360E;
#else
#include <sys/ucontext.h>
#include <link.h>
+extern ElfW(Dyn) _DYNAMIC[];
#endif
// x86-64 FreeBSD 9.2 and older define 'ucontext_t' incorrectly in
void *AsanDoesNotSupportStaticLinkage() {
// This will fail to link with -static.
- return &_DYNAMIC; // defined in link.h
-}
-
-static void UnmapFromTo(uptr from, uptr to) {
- CHECK(to >= from);
- if (to == from) return;
- uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
- if (UNLIKELY(internal_iserror(res))) {
- Report(
- "ERROR: AddresSanitizer failed to unmap 0x%zx (%zd) bytes at address "
- "%p\n",
- to - from, to - from, from);
- CHECK("unable to unmap" && 0);
- }
+ return &_DYNAMIC;
}
#if ASAN_PREMAP_SHADOW
-uptr FindPremappedShadowStart() {
+uptr FindPremappedShadowStart(uptr shadow_size_bytes) {
uptr granularity = GetMmapGranularity();
uptr shadow_start = reinterpret_cast<uptr>(&__asan_shadow);
uptr premap_shadow_size = PremapShadowSize();
- uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
+ uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
// We may have mapped too much. Release extra memory.
UnmapFromTo(shadow_start + shadow_size, shadow_start + premap_shadow_size);
return shadow_start;
#endif
uptr FindDynamicShadowStart() {
+ uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd);
#if ASAN_PREMAP_SHADOW
if (!PremapShadowFailed())
- return FindPremappedShadowStart();
+ return FindPremappedShadowStart(shadow_size_bytes);
#endif
- uptr granularity = GetMmapGranularity();
- uptr alignment = granularity * 8;
- uptr left_padding = granularity;
- uptr shadow_size = RoundUpTo(kHighShadowEnd, granularity);
- uptr map_size = shadow_size + left_padding + alignment;
-
- uptr map_start = (uptr)MmapNoAccess(map_size);
- CHECK_NE(map_start, ~(uptr)0);
-
- uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
- UnmapFromTo(map_start, shadow_start - left_padding);
- UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
-
- return shadow_start;
+ return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
#if SANITIZER_ANDROID
// FIXME: should we do anything for Android?
void AsanCheckDynamicRTPrereqs() {}
}
uptr FindDynamicShadowStart() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = 8 * granularity;
- uptr left_padding = granularity;
- uptr space_size = kHighShadowEnd + left_padding;
-
- uptr largest_gap_found = 0;
- uptr max_occupied_addr = 0;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
- uptr shadow_start =
- FindAvailableMemoryRange(space_size, alignment, granularity,
- &largest_gap_found, &max_occupied_addr);
- // If the shadow doesn't fit, restrict the address space to make it fit.
- if (shadow_start == 0) {
- VReport(
- 2,
- "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
- largest_gap_found, max_occupied_addr);
- uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
- if (new_max_vm < max_occupied_addr) {
- Report("Unable to find a memory range for dynamic shadow.\n");
- Report(
- "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
- "new_max_vm = %p\n",
- space_size, largest_gap_found, max_occupied_addr, new_max_vm);
- CHECK(0 && "cannot place shadow");
- }
- RestrictMemoryToMaxAddress(new_max_vm);
- kHighMemEnd = new_max_vm - 1;
- space_size = kHighShadowEnd + left_padding;
- VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
- shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
- nullptr, nullptr);
- if (shadow_start == 0) {
- Report("Unable to find a memory range after restricting VM.\n");
- CHECK(0 && "cannot place shadow after restricting vm");
- }
- }
- CHECK_NE((uptr)0, shadow_start);
- CHECK(IsAligned(shadow_start, alignment));
- return shadow_start;
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
// No-op. Mac does not support static linkage anyway.
op(globals, size / sizeof(__asan_global));
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
UNIMPLEMENTED();
}
#include "sanitizer_common/sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \
- SANITIZER_NETBSD || SANITIZER_RTEMS || SANITIZER_SOLARIS
+ SANITIZER_NETBSD || SANITIZER_SOLARIS
-#include "sanitizer_common/sanitizer_allocator_checks.h"
-#include "sanitizer_common/sanitizer_errno.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include "asan_allocator.h"
-#include "asan_interceptors.h"
-#include "asan_internal.h"
-#include "asan_malloc_local.h"
-#include "asan_stack.h"
+# include "asan_allocator.h"
+# include "asan_interceptors.h"
+# include "asan_internal.h"
+# include "asan_stack.h"
+# include "sanitizer_common/sanitizer_allocator_checks.h"
+# include "sanitizer_common/sanitizer_errno.h"
+# include "sanitizer_common/sanitizer_tls_get_addr.h"
// ---------------------- Replacement functions ---------------- {{{1
using namespace __asan;
static uptr allocated_for_dlsym;
static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024;
+static const uptr kDlsymAllocPoolSize = 1024;
static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-static INLINE bool IsInDlsymAllocPool(const void *ptr) {
+static inline bool IsInDlsymAllocPool(const void *ptr) {
uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
}
return 0;
}
-#if SANITIZER_RTEMS
-void* MemalignFromLocalPool(uptr alignment, uptr size) {
- void *ptr = nullptr;
- alignment = Max(alignment, kWordSize);
- PosixMemalignFromLocalPool(&ptr, alignment, size);
- return ptr;
-}
-
-bool IsFromLocalPool(const void *ptr) {
- return IsInDlsymAllocPool(ptr);
-}
-#endif
-
-static INLINE bool MaybeInDlsym() {
+static inline bool MaybeInDlsym() {
// Fuchsia doesn't use dlsym-based interceptors.
return !SANITIZER_FUCHSIA && asan_init_is_running;
}
-static INLINE bool UseLocalPool() {
- return EarlyMalloc() || MaybeInDlsym();
-}
+static inline bool UseLocalPool() { return MaybeInDlsym(); }
static void *ReallocFromLocalPool(void *ptr, uptr size) {
const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
}
INTERCEPTOR(void, free, void *ptr) {
- GET_STACK_TRACE_FREE;
if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
DeallocateFromLocalPool(ptr);
return;
}
+ GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#if SANITIZER_INTERCEPT_CFREE
INTERCEPTOR(void, cfree, void *ptr) {
- GET_STACK_TRACE_FREE;
if (UNLIKELY(IsInDlsymAllocPool(ptr)))
return;
+ GET_STACK_TRACE_FREE;
asan_free(ptr, &stack, FROM_MALLOC);
}
#endif // SANITIZER_INTERCEPT_CFREE
// || `[0x2000000000, 0x23ffffffff]` || LowShadow ||
// || `[0x0000000000, 0x1fffffffff]` || LowMem ||
//
+// Default Linux/RISCV64 Sv39 mapping:
+// || `[0x1555550000, 0x3fffffffff]` || HighMem ||
+// || `[0x0fffffa000, 0x1555555fff]` || HighShadow ||
+// || `[0x0effffa000, 0x0fffff9fff]` || ShadowGap ||
+// || `[0x0d55550000, 0x0effff9fff]` || LowShadow ||
+// || `[0x0000000000, 0x0d5554ffff]` || LowMem ||
+//
// Default Linux/AArch64 (39-bit VMA) mapping:
// || `[0x2000000000, 0x7fffffffff]` || highmem ||
// || `[0x1400000000, 0x1fffffffff]` || highshadow ||
// || `[0x36000000, 0x39ffffff]` || ShadowGap ||
// || `[0x30000000, 0x35ffffff]` || LowShadow ||
// || `[0x00000000, 0x2fffffff]` || LowMem ||
-//
-// Shadow mapping on Myriad2 (for shadow scale 5):
-// || `[0x9ff80000, 0x9fffffff]` || ShadowGap ||
-// || `[0x9f000000, 0x9ff7ffff]` || LowShadow ||
-// || `[0x80000000, 0x9effffff]` || LowMem ||
-// || `[0x00000000, 0x7fffffff]` || Ignored ||
#if defined(ASAN_SHADOW_SCALE)
static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE;
#else
-static const u64 kDefaultShadowScale = SANITIZER_MYRIAD2 ? 5 : 3;
+static const u64 kDefaultShadowScale = 3;
#endif
static const u64 kDefaultShadowSentinel = ~(uptr)0;
static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000
static const u64 kDefaultShort64bitShadowOffset =
0x7FFFFFFF & (~0xFFFULL << kDefaultShadowScale); // < 2G.
static const u64 kAArch64_ShadowOffset64 = 1ULL << 36;
+static const u64 kRiscv64_ShadowOffset64 = 0xd55550000;
static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000;
static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37;
static const u64 kPPC64_ShadowOffset64 = 1ULL << 44;
static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000
static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000
-static const u64 kMyriadMemoryOffset32 = 0x80000000ULL;
-static const u64 kMyriadMemorySize32 = 0x20000000ULL;
-static const u64 kMyriadMemoryEnd32 =
- kMyriadMemoryOffset32 + kMyriadMemorySize32 - 1;
-static const u64 kMyriadShadowOffset32 =
- (kMyriadMemoryOffset32 + kMyriadMemorySize32 -
- (kMyriadMemorySize32 >> kDefaultShadowScale));
-static const u64 kMyriadCacheBitMask32 = 0x40000000ULL;
-
#define SHADOW_SCALE kDefaultShadowScale
#if SANITIZER_FUCHSIA
# define SHADOW_OFFSET kWindowsShadowOffset32
# elif SANITIZER_IOS
# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
-# elif SANITIZER_MYRIAD2
-# define SHADOW_OFFSET kMyriadShadowOffset32
# else
# define SHADOW_OFFSET kDefaultShadowOffset32
# endif
#else
# if SANITIZER_IOS
# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+# elif SANITIZER_MAC && defined(__aarch64__)
+# define SHADOW_OFFSET __asan_shadow_memory_dynamic_address
+#elif SANITIZER_RISCV64
+#define SHADOW_OFFSET kRiscv64_ShadowOffset64
# elif defined(__aarch64__)
# define SHADOW_OFFSET kAArch64_ShadowOffset64
# elif defined(__powerpc64__)
} // namespace __asan
-#if SANITIZER_MYRIAD2
-#include "asan_mapping_myriad.h"
-#elif defined(__sparc__) && SANITIZER_WORDSIZE == 64
-#include "asan_mapping_sparc64.h"
+#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
+# include "asan_mapping_sparc64.h"
#else
#define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET))
} // namespace __asan
-#endif // SANITIZER_MYRIAD2
+#endif
namespace __asan {
+static inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; }
+
static inline bool AddrIsInMem(uptr a) {
PROFILE_ASAN_MAPPING();
return AddrIsInLowMem(a) || AddrIsInMidMem(a) || AddrIsInHighMem(a) ||
static inline bool AddressIsPoisoned(uptr a) {
PROFILE_ASAN_MAPPING();
- if (SANITIZER_MYRIAD2 && !AddrIsInMem(a) && !AddrIsInShadow(a))
- return false;
const uptr kAccessSize = 1;
u8 *shadow_address = (u8*)MEM_TO_SHADOW(a);
s8 shadow_value = *shadow_address;
// Interceptors for operators new and delete.
//===----------------------------------------------------------------------===//
+#include <stddef.h>
+
#include "asan_allocator.h"
#include "asan_internal.h"
-#include "asan_malloc_local.h"
#include "asan_report.h"
#include "asan_stack.h"
-
#include "interception/interception.h"
-#include <stddef.h>
-
// C++ operators can't have dllexport attributes on Windows. We export them
// anyway by passing extra -export flags to the linker, which is exactly that
// dllexport would normally do. We need to export them in order to make the
#endif
#undef COMMENT_EXPORT
#else
-#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE
+#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
#endif
using namespace __asan;
// For local pool allocation, align to SHADOW_GRANULARITY to match asan
// allocator behavior.
#define OPERATOR_NEW_BODY(type, nothrow) \
- MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \
GET_STACK_TRACE_MALLOC; \
void *res = asan_memalign(0, size, &stack, type); \
if (!nothrow && UNLIKELY(!res)) \
ReportOutOfMemory(size, &stack); \
return res;
#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
- MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \
GET_STACK_TRACE_MALLOC; \
void *res = asan_memalign((uptr)align, size, &stack, type); \
if (!nothrow && UNLIKELY(!res)) \
#endif // !SANITIZER_MAC
#define OPERATOR_DELETE_BODY(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, 0, 0, &stack, type);
#define OPERATOR_DELETE_BODY_SIZE(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, size, 0, &stack, type);
#define OPERATOR_DELETE_BODY_ALIGN(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
- if (IS_FROM_LOCAL_POOL(ptr)) return;\
- GET_STACK_TRACE_FREE;\
+ GET_STACK_TRACE_FREE; \
asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
#if !SANITIZER_MAC
}
};
-void FlushUnneededASanShadowMemory(uptr p, uptr size) {
- // Since asan's mapping is compacting, the shadow chunk may be
- // not page-aligned, so we only flush the page-aligned portion.
- ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
-}
-
void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
uptr end = ptr + size;
if (Verbosity()) {
}
uptr __asan_region_is_poisoned(uptr beg, uptr size) {
- if (!size) return 0;
+ if (!size)
+ return 0;
uptr end = beg + size;
- if (SANITIZER_MYRIAD2) {
- // On Myriad, address not in DRAM range need to be treated as
- // unpoisoned.
- if (!AddrIsInMem(beg) && !AddrIsInShadow(beg)) return 0;
- if (!AddrIsInMem(end) && !AddrIsInShadow(end)) return 0;
- } else {
- if (!AddrIsInMem(beg)) return beg;
- if (!AddrIsInMem(end)) return end;
- }
+ if (!AddrIsInMem(beg))
+ return beg;
+ if (!AddrIsInMem(end))
+ return end;
CHECK_LT(beg, end);
uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY);
uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY);
// First check the first and the last application bytes,
// then check the SHADOW_GRANULARITY-aligned region by calling
// mem_is_zero on the corresponding shadow.
- if (!__asan::AddressIsPoisoned(beg) &&
- !__asan::AddressIsPoisoned(end - 1) &&
+ if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) &&
(shadow_end <= shadow_beg ||
__sanitizer::mem_is_zero((const char *)shadow_beg,
shadow_end - shadow_beg)))
&stack);
}
CHECK_LE(end - beg,
- FIRST_32_SECOND_64(1UL << 30, 1ULL << 34)); // Sanity check.
+ FIRST_32_SECOND_64(1UL << 30, 1ULL << 40)); // Sanity check.
uptr a = RoundDownTo(Min(old_mid, new_mid), granularity);
uptr c = RoundUpTo(Max(old_mid, new_mid), granularity);
// probably provide higher-level interface for these operations.
// For now, just memset on Windows.
if (value || SANITIZER_WINDOWS == 1 ||
- // RTEMS doesn't have have pages, let alone a fast way to zero
- // them, so default to memset.
- SANITIZER_RTEMS == 1 ||
shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg);
} else {
if (signal_stack.ss_flags != SS_ONSTACK)
return false;
- // Since we're on the signal altnerate stack, we cannot find the DEFAULT
+ // Since we're on the signal alternate stack, we cannot find the DEFAULT
// stack bottom using a local variable.
uptr default_bottom, tls_addr, tls_size, stack_size;
GetThreadStackAndTls(/*main=*/false, &default_bottom, &stack_size, &tls_addr,
// Returns an address aligned to 8 pages, such that one page on the left and
// PremapShadowSize() bytes on the right of it are mapped r/o.
uptr PremapShadow() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = granularity * 8;
- uptr left_padding = granularity;
- uptr shadow_size = PremapShadowSize();
- uptr map_size = shadow_size + left_padding + alignment;
-
- uptr map_start = (uptr)MmapNoAccess(map_size);
- CHECK_NE(map_start, ~(uptr)0);
-
- uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
- uptr shadow_end = shadow_start + shadow_size;
- internal_munmap(reinterpret_cast<void *>(map_start),
- shadow_start - left_padding - map_start);
- internal_munmap(reinterpret_cast<void *>(shadow_end),
- map_start + map_size - shadow_end);
- return shadow_start;
+ return MapDynamicShadow(PremapShadowSize(), /*mmap_alignment_scale*/ 3,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
bool PremapShadowFailed() {
if (common_flags()->print_cmdline)
PrintCmdline();
- if (common_flags()->print_module_map == 2) PrintModuleMap();
+ if (common_flags()->print_module_map == 2)
+ DumpProcessMap();
// Copy the message buffer so that we could start logging without holding a
// lock that gets aquired during printing.
return false;
}
-static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) {
+static inline void CheckForInvalidPointerPair(void *p1, void *p2) {
switch (flags()->detect_invalid_pointer_pairs) {
case 0:
return;
#include "asan_activation.h"
#include "asan_allocator.h"
+#include "asan_fake_stack.h"
#include "asan_interceptors.h"
#include "asan_interface_internal.h"
#include "asan_internal.h"
#include "asan_stats.h"
#include "asan_suppressions.h"
#include "asan_thread.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
-#include "lsan/lsan_common.h"
#include "ubsan/ubsan_init.h"
#include "ubsan/ubsan_platform.h"
// Don't die twice - run a busy loop.
while (1) { }
}
- if (common_flags()->print_module_map >= 1) PrintModuleMap();
+ if (common_flags()->print_module_map >= 1)
+ DumpProcessMap();
if (flags()->sleep_before_dying) {
Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
SleepForSeconds(flags()->sleep_before_dying);
}
}
-static void AsanCheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2) {
- Report("AddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
- line, cond, (uptr)v1, (uptr)v2);
-
- // Print a stack trace the first time we come here. Otherwise, we probably
- // failed a CHECK during symbolization.
- static atomic_uint32_t num_calls;
- if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) == 0) {
- PRINT_CURRENT_STACK_CHECK();
- }
-
- Die();
+static void CheckUnwind() {
+ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
+ stack.Print();
}
// -------------------------- Globals --------------------- {{{1
Die();
}
+NOINLINE
+static void ReportGenericErrorWrapper(uptr addr, bool is_write, int size,
+ int exp_arg, bool fatal) {
+ if (__asan_test_only_reported_buggy_pointer) {
+ *__asan_test_only_reported_buggy_pointer = addr;
+ } else {
+ GET_CALLER_PC_BP_SP;
+ ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal);
+ }
+}
+
// --------------- LowLevelAllocateCallbac ---------- {{{1
static void OnLowLevelAllocate(uptr ptr, uptr size) {
PoisonShadow(ptr, size, kAsanInternalHeapMagic);
ASAN_REPORT_ERROR_N(store, true)
#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \
- if (SANITIZER_MYRIAD2 && !AddrIsInMem(addr) && !AddrIsInShadow(addr)) \
- return; \
- uptr sp = MEM_TO_SHADOW(addr); \
- uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
- : *reinterpret_cast<u16 *>(sp); \
- if (UNLIKELY(s)) { \
- if (UNLIKELY(size >= SHADOW_GRANULARITY || \
- ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \
- (s8)s)) { \
- if (__asan_test_only_reported_buggy_pointer) { \
- *__asan_test_only_reported_buggy_pointer = addr; \
- } else { \
- GET_CALLER_PC_BP_SP; \
- ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, \
- fatal); \
- } \
- } \
- }
+ uptr sp = MEM_TO_SHADOW(addr); \
+ uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \
+ : *reinterpret_cast<u16 *>(sp); \
+ if (UNLIKELY(s)) { \
+ if (UNLIKELY(size >= SHADOW_GRANULARITY || \
+ ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \
+ (s8)s)) { \
+ ReportGenericErrorWrapper(addr, is_write, size, exp_arg, fatal); \
+ } \
+ }
#define ASAN_MEMORY_ACCESS_CALLBACK(type, is_write, size) \
extern "C" NOINLINE INTERFACE_ATTRIBUTE \
}
static void InitializeHighMemEnd() {
-#if !SANITIZER_MYRIAD2
#if !ASAN_FIXED_MAPPING
kHighMemEnd = GetMaxUserVirtualAddress();
// Increase kHighMemEnd to make sure it's properly
// aligned together with kHighMemBeg:
- kHighMemEnd |= SHADOW_GRANULARITY * GetMmapGranularity() - 1;
+ kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
#endif // !ASAN_FIXED_MAPPING
CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0);
-#endif // !SANITIZER_MYRIAD2
}
void PrintAddressSpaceLayout() {
// Install tool-specific callbacks in sanitizer_common.
AddDieCallback(AsanDie);
- SetCheckFailedCallback(AsanCheckFailed);
+ SetCheckUnwindCallback(CheckUnwind);
SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
__sanitizer_set_report_path(common_flags()->log_path);
type, top, bottom, top - bottom, top - bottom);
return;
}
- PoisonShadow(bottom, top - bottom, 0);
+ PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0);
}
static void UnpoisonDefaultStack() {
const uptr page_size = GetPageSizeCached();
top = curr_thread->stack_top();
bottom = ((uptr)&local_stack - page_size) & ~(page_size - 1);
- } else if (SANITIZER_RTEMS) {
- // Give up On RTEMS.
- return;
} else {
CHECK(!SANITIZER_FUCHSIA);
// If we haven't seen this thread, try asking the OS for stack bounds.
static void UnpoisonFakeStack() {
AsanThread *curr_thread = GetCurrentThread();
- if (curr_thread && curr_thread->has_fake_stack())
- curr_thread->fake_stack()->HandleNoReturn();
+ if (!curr_thread)
+ return;
+ FakeStack *stack = curr_thread->get_fake_stack();
+ if (!stack)
+ return;
+ stack->HandleNoReturn();
}
} // namespace __asan
#include "sanitizer_common/sanitizer_platform.h"
-// asan_fuchsia.cpp and asan_rtems.cpp have their own
-// InitializeShadowMemory implementation.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+// asan_fuchsia.cpp has their own InitializeShadowMemory implementation.
+#if !SANITIZER_FUCHSIA
-#include "asan_internal.h"
-#include "asan_mapping.h"
+# include "asan_internal.h"
+# include "asan_mapping.h"
namespace __asan {
-// ---------------------- mmap -------------------- {{{1
-// Reserve memory range [beg, end].
-// We need to use inclusive range because end+1 may not be representable.
-void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
- CHECK_EQ((beg % GetMmapGranularity()), 0);
- CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
- uptr size = end - beg + 1;
- DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- if (!MmapFixedSuperNoReserve(beg, size, name)) {
- Report(
- "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
- "Perhaps you're using ulimit -v\n",
- size);
- Abort();
- }
- if (common_flags()->use_madv_dontdump) DontDumpShadowMemory(beg, size);
-}
-
static void ProtectGap(uptr addr, uptr size) {
if (!flags()->protect_shadow_gap) {
// The shadow gap is unprotected, so there is a chance that someone
"unprotected gap shadow");
return;
}
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
- // A few pages at the start of the address space can not be protected.
- // But we really want to protect as much as possible, to prevent this memory
- // being returned as a result of a non-FIXED mmap().
- if (addr == kZeroBaseShadowStart) {
- uptr step = GetMmapGranularity();
- while (size > step && addr < kZeroBaseMaxShadowStart) {
- addr += step;
- size -= step;
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res) return;
- }
- }
-
- Report(
- "ERROR: Failed to protect the shadow gap. "
- "ASan cannot proceed correctly. ABORTING.\n");
- DumpProcessMap();
- Die();
+ __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
+ kZeroBaseMaxShadowStart);
}
static void MaybeReportLinuxPIEBug() {
-#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__aarch64__))
+#if SANITIZER_LINUX && \
+ (defined(__x86_64__) || defined(__aarch64__) || SANITIZER_RISCV64)
Report("This might be related to ELF_ET_DYN_BASE change in Linux 4.12.\n");
Report(
"See https://github.com/google/sanitizers/issues/856 for possible "
// |kDefaultShadowSentinel|.
bool full_shadow_is_available = false;
if (shadow_start == kDefaultShadowSentinel) {
- __asan_shadow_memory_dynamic_address = 0;
- CHECK_EQ(0, kLowShadowBeg);
shadow_start = FindDynamicShadowStart();
if (SANITIZER_LINUX) full_shadow_is_available = true;
}
} // namespace __asan
-#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#endif // !SANITIZER_FUCHSIA
if (SANITIZER_MIPS && t &&
!IsValidFrame(bp, t->stack_top(), t->stack_bottom()))
return;
- Unwind(max_depth, pc, bp, context, 0, 0, false);
+ Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,
+ t ? t->stack_bottom() : 0, false);
}
// ------------------ Interface -------------- {{{1
stack.Unwind(pc, bp, nullptr, \
common_flags()->fast_unwind_on_fatal)
-#define GET_STACK_TRACE_SIGNAL(sig) \
- BufferedStackTrace stack; \
- stack.Unwind((sig).pc, (sig).bp, (sig).context, \
- common_flags()->fast_unwind_on_fatal)
-
#define GET_STACK_TRACE_FATAL_HERE \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
-#define GET_STACK_TRACE_CHECK_HERE \
- GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check)
-
#define GET_STACK_TRACE_THREAD \
GET_STACK_TRACE(kStackTraceMax, true)
stack.Print(); \
}
-#define PRINT_CURRENT_STACK_CHECK() \
- { \
- GET_STACK_TRACE_CHECK_HERE; \
- stack.Print(); \
- }
-
#endif // ASAN_STACK_H
// in TSD and can't reliably tell when no more TSD destructors will
// be called. It would be wrong to reuse AsanThreadContext for another
// thread before all TSD destructors will be called for it.
- asan_thread_registry = new(thread_registry_placeholder) ThreadRegistry(
- GetAsanThreadContext, kMaxNumberOfThreads, kMaxNumberOfThreads);
+ asan_thread_registry =
+ new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext);
initialized = true;
}
return *asan_thread_registry;
int tid = this->tid();
VReport(1, "T%d exited\n", tid);
- malloc_storage().CommitBack();
- if (common_flags()->use_sigaltstack) UnsetAlternateSignalStack();
- asanThreadRegistry().FinishThread(tid);
- FlushToDeadThreadStats(&stats_);
- // We also clear the shadow on thread destruction because
- // some code may still be executing in later TSD destructors
- // and we don't want it to have any poisoned stack.
- ClearShadowForThreadStackAndTLS();
- DeleteFakeStack(tid);
+ bool was_running =
+ (asanThreadRegistry().FinishThread(tid) == ThreadStatusRunning);
+ if (was_running) {
+ if (AsanThread *thread = GetCurrentThread())
+ CHECK_EQ(this, thread);
+ malloc_storage().CommitBack();
+ if (common_flags()->use_sigaltstack)
+ UnsetAlternateSignalStack();
+ FlushToDeadThreadStats(&stats_);
+ // We also clear the shadow on thread destruction because
+ // some code may still be executing in later TSD destructors
+ // and we don't want it to have any poisoned stack.
+ ClearShadowForThreadStackAndTLS();
+ DeleteFakeStack(tid);
+ } else {
+ CHECK_NE(this, GetCurrentThread());
+ }
uptr size = RoundUpTo(sizeof(AsanThread), GetPageSizeCached());
UnmapOrDie(this, size);
- DTLS_Destroy();
+ if (was_running)
+ DTLS_Destroy();
}
void AsanThread::StartSwitchFiber(FakeStack **fake_stack_save, uptr bottom,
return bounds.top - bounds.bottom;
}
-// We want to create the FakeStack lazyly on the first use, but not eralier
+// We want to create the FakeStack lazily on the first use, but not earlier
// than the stack size is known and the procedure has to be async-signal safe.
FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
uptr stack_size = this->stack_size();
stack_size_log =
Max(stack_size_log, static_cast<uptr>(flags()->min_uar_stack_size_log));
fake_stack_ = FakeStack::Create(stack_size_log);
+ DCHECK_EQ(GetCurrentThread(), this);
SetTLSFakeStack(fake_stack_);
return fake_stack_;
}
}
void AsanThread::Init(const InitOptions *options) {
+ DCHECK_NE(tid(), kInvalidTid);
next_stack_top_ = next_stack_bottom_ = 0;
atomic_store(&stack_switching_, false, memory_order_release);
CHECK_EQ(this->stack_size(), 0U);
}
ClearShadowForThreadStackAndTLS();
fake_stack_ = nullptr;
- if (__asan_option_detect_stack_use_after_return)
+ if (__asan_option_detect_stack_use_after_return &&
+ tid() == GetCurrentTidOrInvalid()) {
+ // AsyncSignalSafeLazyInitFakeStack makes use of threadlocals and must be
+ // called from the context of the thread it is initializing, not its parent.
+ // Most platforms call AsanThread::Init on the newly-spawned thread, but
+ // Fuchsia calls this function from the parent thread. To support that
+ // approach, we avoid calling AsyncSignalSafeLazyInitFakeStack here; it will
+ // be called by the new thread when it first attempts to access the fake
+ // stack.
AsyncSignalSafeLazyInitFakeStack();
+ }
int local = 0;
VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
(void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
&local);
}
-// Fuchsia and RTEMS don't use ThreadStart.
-// asan_fuchsia.c/asan_rtems.c define CreateMainThread and
-// SetThreadStackAndTls.
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+// Fuchsia doesn't use ThreadStart.
+// asan_fuchsia.c definies CreateMainThread and SetThreadStackAndTls.
+#if !SANITIZER_FUCHSIA
-thread_return_t AsanThread::ThreadStart(
- tid_t os_id, atomic_uintptr_t *signal_thread_is_registered) {
+thread_return_t AsanThread::ThreadStart(tid_t os_id) {
Init();
asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr);
- if (signal_thread_is_registered)
- atomic_store(signal_thread_is_registered, 1, memory_order_release);
if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
AsanThread *CreateMainThread() {
AsanThread *main_thread = AsanThread::Create(
- /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ 0,
+ /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ kMainTid,
/* stack */ nullptr, /* detached */ true);
SetCurrentThread(main_thread);
- main_thread->ThreadStart(internal_getpid(),
- /* signal_thread_is_registered */ nullptr);
+ main_thread->ThreadStart(internal_getpid());
return main_thread;
}
DCHECK_EQ(options, nullptr);
uptr tls_size = 0;
uptr stack_size = 0;
- GetThreadStackAndTls(tid() == 0, &stack_bottom_, &stack_size, &tls_begin_,
- &tls_size);
- stack_top_ = stack_bottom_ + stack_size;
+ GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size,
+ &tls_begin_, &tls_size);
+ stack_top_ = RoundDownTo(stack_bottom_ + stack_size, SHADOW_GRANULARITY);
tls_end_ = tls_begin_ + tls_size;
dtls_ = DTLS_Get();
}
}
-#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
+#endif // !SANITIZER_FUCHSIA
void AsanThread::ClearShadowForThreadStackAndTLS() {
if (stack_top_ != stack_bottom_)
uptr bottom = 0;
if (AddrIsInStack(addr)) {
bottom = stack_bottom();
- } else if (has_fake_stack()) {
- bottom = fake_stack()->AddrIsInFakeStack(addr);
+ } else if (FakeStack *fake_stack = get_fake_stack()) {
+ bottom = fake_stack->AddrIsInFakeStack(addr);
CHECK(bottom);
access->offset = addr - bottom;
access->frame_pc = ((uptr*)bottom)[2];
uptr bottom = 0;
if (AddrIsInStack(addr)) {
bottom = stack_bottom();
- } else if (has_fake_stack()) {
- bottom = fake_stack()->AddrIsInFakeStack(addr);
- CHECK(bottom);
+ } else if (FakeStack *fake_stack = get_fake_stack()) {
+ bottom = fake_stack->AddrIsInFakeStack(addr);
+ if (bottom == 0) {
+ return 0;
+ }
} else {
return 0;
}
static bool ThreadStackContainsAddress(ThreadContextBase *tctx_base,
void *addr) {
- AsanThreadContext *tctx = static_cast<AsanThreadContext*>(tctx_base);
+ AsanThreadContext *tctx = static_cast<AsanThreadContext *>(tctx_base);
AsanThread *t = tctx->thread;
- if (!t) return false;
- if (t->AddrIsInStack((uptr)addr)) return true;
- if (t->has_fake_stack() && t->fake_stack()->AddrIsInFakeStack((uptr)addr))
+ if (!t)
+ return false;
+ if (t->AddrIsInStack((uptr)addr))
return true;
- return false;
+ FakeStack *fake_stack = t->get_fake_stack();
+ if (!fake_stack)
+ return false;
+ return fake_stack->AddrIsInFakeStack((uptr)addr);
}
AsanThread *GetCurrentThread() {
- if (SANITIZER_RTEMS && !asan_inited)
- return nullptr;
-
AsanThreadContext *context =
reinterpret_cast<AsanThreadContext *>(AsanTSDGet());
if (!context) {
// address. We are not entirely sure that we have correct main thread
// limits, so only do this magic on Android, and only if the found thread
// is the main thread.
- AsanThreadContext *tctx = GetThreadContextByTidLocked(0);
+ AsanThreadContext *tctx = GetThreadContextByTidLocked(kMainTid);
if (tctx && ThreadStackContainsAddress(tctx, &context)) {
SetCurrentThread(tctx->thread);
return tctx->thread;
void EnsureMainThreadIDIsCorrect() {
AsanThreadContext *context =
reinterpret_cast<AsanThreadContext *>(AsanTSDGet());
- if (context && (context->tid == 0))
+ if (context && (context->tid == kMainTid))
context->os_id = GetTid();
}
void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback,
void *arg) {
__asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
- if (t && t->has_fake_stack())
- t->fake_stack()->ForEachFakeFrame(callback, arg);
+ if (!t)
+ return;
+ __asan::FakeStack *fake_stack = t->get_fake_stack();
+ if (!fake_stack)
+ return;
+ fake_stack->ForEachFakeFrame(callback, arg);
}
void LockThreadRegistry() {
namespace __asan {
-const u32 kInvalidTid = 0xffffff; // Must fit into 24 bits.
-const u32 kMaxNumberOfThreads = (1 << 22); // 4M
-
class AsanThread;
// These objects are created for every thread and are never deleted,
// so we can find them by tid even if the thread is long dead.
-class AsanThreadContext : public ThreadContextBase {
+class AsanThreadContext final : public ThreadContextBase {
public:
explicit AsanThreadContext(int tid)
: ThreadContextBase(tid), announced(false),
struct InitOptions;
void Init(const InitOptions *options = nullptr);
- thread_return_t ThreadStart(tid_t os_id,
- atomic_uintptr_t *signal_thread_is_registered);
+ thread_return_t ThreadStart(tid_t os_id);
uptr stack_top();
uptr stack_bottom();
void FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old,
uptr *size_old);
- bool has_fake_stack() {
- return !atomic_load(&stack_switching_, memory_order_relaxed) &&
- (reinterpret_cast<uptr>(fake_stack_) > 1);
+ FakeStack *get_fake_stack() {
+ if (atomic_load(&stack_switching_, memory_order_relaxed))
+ return nullptr;
+ if (reinterpret_cast<uptr>(fake_stack_) <= 1)
+ return nullptr;
+ return fake_stack_;
}
- FakeStack *fake_stack() {
- if (!__asan_option_detect_stack_use_after_return)
- return nullptr;
+ FakeStack *get_or_create_fake_stack() {
if (atomic_load(&stack_switching_, memory_order_relaxed))
return nullptr;
- if (!has_fake_stack())
+ if (reinterpret_cast<uptr>(fake_stack_) <= 1)
return AsyncSignalSafeLazyInitFakeStack();
return fake_stack_;
}
void *extra_spill_area() { return &extra_spill_area_; }
+ void *get_arg() { return arg_; }
+
private:
// NOTE: There is no AsanThread constructor. It is allocated
// via mmap() and *must* be valid in zero-initialized state.
static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
AsanThread *t = (AsanThread *)arg;
SetCurrentThread(t);
- return t->ThreadStart(GetTid(), /* signal_thread_is_registered */ nullptr);
+ return t->ThreadStart(GetTid());
}
INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security,
UNIMPLEMENTED();
}
+void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Since asan's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
// ---------------------- TSD ---------------- {{{
static bool tsd_key_inited = false;
}
uptr FindDynamicShadowStart() {
- uptr granularity = GetMmapGranularity();
- uptr alignment = 8 * granularity;
- uptr left_padding = granularity;
- uptr space_size = kHighShadowEnd + left_padding;
- uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
- granularity, nullptr, nullptr);
- CHECK_NE((uptr)0, shadow_start);
- CHECK(IsAligned(shadow_start, alignment));
- return shadow_start;
+ return MapDynamicShadow(MemToShadowSize(kHighMemEnd), SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
}
void AsanCheckDynamicRTPrereqs() {}
local _ARCH=
local _ARCH64=
if [[ $_ABI == x86* ]]; then
- _ARCH=i386
+ _ARCH=i686
elif [[ $_ABI == armeabi* ]]; then
_ARCH=arm
elif [[ $_ABI == arm64-v8a* ]]; then
ASAN_OPTIONS=$ASAN_OPTIONS \\
ASAN_ACTIVATION_OPTIONS=include_if_exists=/data/local/tmp/asan.options.%b \\
LD_PRELOAD=$_ld_preload \\
-exec $_to \$@
+exec $_to "\$@"
EOF
}
where it is necessary to handle site-specific quirks (e.g. binaries with debug
symbols only accessible via a remote service) without having to modify the
script itself.
-
+
"""
import argparse
import bisect
def is_valid_arch(s):
return s in ["i386", "x86_64", "x86_64h", "arm", "armv6", "armv7", "armv7s",
- "armv7k", "arm64", "powerpc64", "powerpc64le", "s390x", "s390"]
+ "armv7k", "arm64", "powerpc64", "powerpc64le", "s390x", "s390",
+ "riscv64"]
def guess_arch(addr):
# Guess which arch we're running. 10 = len('0x') + 8 hex digits.
def open_llvm_symbolizer(self):
cmd = [self.symbolizer_path,
- '--use-symbol-table=true',
- '--demangle=%s' % demangle,
+ ('--demangle' if demangle else '--no-demangle'),
'--functions=linkage',
- '--inlining=true',
+ '--inlines',
'--default-arch=%s' % self.default_arch]
if self.system == 'Darwin':
for hint in self.dsym_hints:
# EPIPE happens if addr2line exits early (which some implementations do
# if an invalid file is passed).
if e.errno == errno.EPIPE:
- logging.debug("addr2line exited early (broken pipe), returncode=%d" % self.pipe.poll())
+ logging.debug(f"addr2line exited early (broken pipe) returncode={self.pipe.poll()}")
else:
logging.debug("unexpected I/O exception communicating with addr2line", exc_info=e)
lines.append(('??', '??:0'))
-fno-rtti
-O2
-Wno-format
- -Werror=sign-compare
- -Wno-non-virtual-dtor
- )
+ -Werror=sign-compare)
append_list_if(COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG -Wno-variadic-macros ASAN_UNITTEST_COMMON_CFLAGS)
# This will ensure the target linker is used
# Use -D instead of definitions to please custom compile command.
list(APPEND ASAN_UNITTEST_COMMON_CFLAGS
${COMPILER_RT_ASAN_SHADOW_SCALE_FLAG}
- -DASAN_HAS_BLACKLIST=1
+ -DASAN_HAS_IGNORELIST=1
-DASAN_HAS_EXCEPTIONS=1
-DASAN_UAR=0
)
list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS})
endif()
-set(ASAN_BLACKLIST_FILE "${CMAKE_CURRENT_SOURCE_DIR}/asan_test.ignore")
+set(ASAN_IGNORELIST_FILE "${CMAKE_CURRENT_SOURCE_DIR}/asan_test.ignore")
set(ASAN_UNITTEST_INSTRUMENTED_CFLAGS
${ASAN_UNITTEST_COMMON_CFLAGS}
-fsanitize=address
- "-fsanitize-blacklist=${ASAN_BLACKLIST_FILE}"
+ "-fsanitize-ignorelist=${ASAN_IGNORELIST_FILE}"
)
if(NOT MSVC)
list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS --driver-mode=g++)
# Closure to keep the values.
function(generate_asan_tests test_objects test_suite testname)
generate_compiler_rt_tests(${test_objects} ${test_suite} ${testname} ${arch}
- COMPILE_DEPS ${ASAN_UNITTEST_HEADERS} ${ASAN_BLACKLIST_FILE}
+ COMPILE_DEPS ${ASAN_UNITTEST_HEADERS} ${ASAN_IGNORELIST_FILE}
DEPS gtest asan
KIND ${TEST_KIND}
${ARGN}
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
+ $<TARGET_OBJECTS:RTLSanCommon.${arch}>
$<TARGET_OBJECTS:RTUbsan.${arch}>
$<TARGET_OBJECTS:RTUbsan_cxx.${arch}>
${COMPILER_RT_GTEST_SOURCE}
BAD_ACCESS(array, 40);
BAD_ACCESS(array, 60);
BAD_ACCESS(array, 79);
- char value;
- EXPECT_DEATH(value = Ident(array[40]), kUseAfterPoisonErrorMessage);
+ EXPECT_DEATH(Ident(array[40]), kUseAfterPoisonErrorMessage);
__asan_unpoison_memory_region(array + 40, 40);
// access previously poisoned memory.
GOOD_ACCESS(array, 40);
TEST(AddressSanitizer, BCmpOOBTest) {
#if (defined(__linux__) && !defined(__ANDROID__) && defined(_GNU_SOURCE)) || \
- defined(__NetBSD__) || defined(__FreeBSD__) || defined(__OpenBSD__)
+ defined(__NetBSD__) || defined(__FreeBSD__)
CmpOOBTestCommon<bcmp>();
#endif
}
delete [] Ident(p);
}
-#if ASAN_HAS_BLACKLIST
+#if ASAN_HAS_IGNORELIST
TEST(AddressSanitizer, IgnoreTest) {
int *x = Ident(new int);
delete Ident(x);
*x = 0;
}
-#endif // ASAN_HAS_BLACKLIST
+#endif // ASAN_HAS_IGNORELIST
struct StructWithBitField {
int bf1:1;
siglongjmp(buf, 1);
}
-#if !defined(__ANDROID__) && !defined(__arm__) && \
- !defined(__aarch64__) && !defined(__mips__) && \
- !defined(__mips64) && !defined(__s390__)
+#if !defined(__ANDROID__) && !defined(__arm__) && !defined(__aarch64__) && \
+ !defined(__mips__) && !defined(__mips64) && !defined(__s390__) && \
+ !defined(__riscv)
NOINLINE void BuiltinLongJmpFunc1(jmp_buf buf) {
// create three red zones for these two stack objects.
int a;
#endif // !defined(__ANDROID__) && !defined(__arm__) &&
// !defined(__aarch64__) && !defined(__mips__)
// !defined(__mips64) && !defined(__s390__)
+ // !defined(__riscv)
TEST(AddressSanitizer, UnderscopeLongJmpTest) {
static jmp_buf buf;
return MallocAndMemsetString(size, 'z');
}
-#if defined(__linux__) && !defined(__ANDROID__)
+#if SANITIZER_GLIBC
#define READ_TEST(READ_N_BYTES) \
char *x = new char[10]; \
int fd = open("/proc/self/stat", O_RDONLY); \
TEST(AddressSanitizer, read) {
READ_TEST(read(fd, x, 15));
}
-#endif // defined(__linux__) && !defined(__ANDROID__)
+#endif // SANITIZER_GLIBC
// This test case fails
// Clang optimizes memcpy/memset calls which lead to unaligned access
-# blacklisted functions for instrumented ASan unit test
+# ignorelisted functions for instrumented ASan unit test
fun:*IgnoreTest*
fun:*SomeOtherFunc*
# error "please define ASAN_HAS_EXCEPTIONS"
#endif
-#ifndef ASAN_HAS_BLACKLIST
-# error "please define ASAN_HAS_BLACKLIST"
+#ifndef ASAN_HAS_IGNORELIST
+# error "please define ASAN_HAS_IGNORELIST"
#endif
#ifndef ASAN_NEEDS_SEGV
# architecture-specific code in various subdirectories.
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
- cmake_minimum_required(VERSION 3.4.3)
+ cmake_minimum_required(VERSION 3.13.4)
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
project(CompilerRTBuiltins C ASM)
set(COMPILER_RT_STANDALONE_BUILD TRUE)
set(COMPILER_RT_BUILTINS_STANDALONE_BUILD TRUE)
if(APPLE)
include(CompilerRTDarwinUtils)
endif()
- if(CMAKE_HOST_APPLE AND APPLE)
+ if(APPLE)
include(UseLibtool)
endif()
include(AddCompilerRT)
+
+ if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -X32_64 qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> -X32_64 q <TARGET> <LINK_FLAGS> <OBJECTS>")
+ set(CMAKE_C_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
+ set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> -X32_64 <TARGET>")
+ endif()
endif()
if (COMPILER_RT_STANDALONE_BUILD)
include(builtin-config-ix)
+if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ include(CompilerRTAIXUtils)
+endif()
+
+option(COMPILER_RT_BUILTINS_HIDE_SYMBOLS
+ "Do not export any symbols from the static library." ON)
+
# TODO: Need to add a mechanism for logging errors when builtin source files are
# added to a sub-directory and not this CMakeLists file.
set(GENERIC_SOURCES
divdi3.c
divmoddi4.c
divmodsi4.c
+ divmodti4.c
divsc3.c
divsf3.c
divsi3.c
umodti3.c
)
+# TODO: Several "tf" files (and divtc3.c, but not multc3.c) are in
+# GENERIC_SOURCES instead of here.
set(GENERIC_TF_SOURCES
addtf3.c
comparetf2.c
divtc3.c
divtf3.c
extenddftf2.c
+ extendhftf2.c
extendsftf2.c
fixtfdi.c
fixtfsi.c
powitf2.c
subtf3.c
trunctfdf2.c
+ trunctfhf2.c
trunctfsf2.c
)
)
endif()
-# These sources work on all x86 variants, but only x86 variants.
+# These files are used on 32-bit and 64-bit x86.
set(x86_ARCH_SOURCES
cpu_model.c
+ )
+
+if (NOT MSVC)
+ set(x86_ARCH_SOURCES
+ ${x86_ARCH_SOURCES}
+ i386/fp_mode.c
+ )
+endif ()
+
+# Implement extended-precision builtins, assuming long double is 80 bits.
+# long double is not 80 bits on Android or MSVC.
+set(x86_80_BIT_SOURCES
divxc3.c
fixxfdi.c
fixxfti.c
powixf2.c
)
-if (NOT MSVC)
- set(x86_ARCH_SOURCES
- ${x86_ARCH_SOURCES}
- i386/fp_mode.c
- )
-endif ()
-
if (NOT MSVC)
set(x86_64_SOURCES
${GENERIC_SOURCES}
${x86_ARCH_SOURCES}
x86_64/floatdidf.c
x86_64/floatdisf.c
- x86_64/floatdixf.c
x86_64/floatundidf.S
x86_64/floatundisf.S
- x86_64/floatundixf.S
)
+ if (NOT ANDROID)
+ set(x86_64_SOURCES
+ ${x86_64_SOURCES}
+ ${x86_80_BIT_SOURCES}
+ x86_64/floatdixf.c
+ x86_64/floatundixf.S
+ )
+ endif()
+
# Darwin x86_64 Haswell
set(x86_64h_SOURCES ${x86_64_SOURCES})
i386/divdi3.S
i386/floatdidf.S
i386/floatdisf.S
- i386/floatdixf.S
i386/floatundidf.S
i386/floatundisf.S
- i386/floatundixf.S
i386/lshrdi3.S
i386/moddi3.S
i386/muldi3.S
i386/umoddi3.S
)
+ if (NOT ANDROID)
+ set(i386_SOURCES
+ ${i386_SOURCES}
+ ${x86_80_BIT_SOURCES}
+ i386/floatdixf.S
+ i386/floatundixf.S
+ )
+ endif()
+
if (WIN32)
set(i386_SOURCES
${i386_SOURCES}
${x86_ARCH_SOURCES}
x86_64/floatdidf.c
x86_64/floatdisf.c
- x86_64/floatdixf.c
)
set(i386_SOURCES ${GENERIC_SOURCES} ${x86_ARCH_SOURCES})
endif () # if (NOT MSVC)
arm/restore_vfp_d8_d15_regs.S
arm/save_vfp_d8_d15_regs.S
)
-set(arm_Thumb1_VFPv2_SOURCES
+set(arm_Thumb1_VFPv2_DP_SOURCES
arm/adddf3vfp.S
- arm/addsf3vfp.S
arm/divdf3vfp.S
- arm/divsf3vfp.S
arm/eqdf2vfp.S
- arm/eqsf2vfp.S
arm/extendsfdf2vfp.S
arm/fixdfsivfp.S
- arm/fixsfsivfp.S
arm/fixunsdfsivfp.S
- arm/fixunssfsivfp.S
arm/floatsidfvfp.S
- arm/floatsisfvfp.S
arm/floatunssidfvfp.S
- arm/floatunssisfvfp.S
arm/gedf2vfp.S
- arm/gesf2vfp.S
arm/gtdf2vfp.S
- arm/gtsf2vfp.S
arm/ledf2vfp.S
- arm/lesf2vfp.S
arm/ltdf2vfp.S
- arm/ltsf2vfp.S
arm/muldf3vfp.S
- arm/mulsf3vfp.S
arm/nedf2vfp.S
arm/negdf2vfp.S
- arm/negsf2vfp.S
- arm/nesf2vfp.S
arm/subdf3vfp.S
- arm/subsf3vfp.S
arm/truncdfsf2vfp.S
arm/unorddf2vfp.S
+)
+set(arm_Thumb1_VFPv2_SP_SOURCES
+ arm/addsf3vfp.S
+ arm/divsf3vfp.S
+ arm/eqsf2vfp.S
+ arm/fixsfsivfp.S
+ arm/fixunssfsivfp.S
+ arm/floatsisfvfp.S
+ arm/floatunssisfvfp.S
+ arm/gesf2vfp.S
+ arm/gtsf2vfp.S
+ arm/lesf2vfp.S
+ arm/ltsf2vfp.S
+ arm/mulsf3vfp.S
+ arm/negsf2vfp.S
+ arm/nesf2vfp.S
+ arm/subsf3vfp.S
arm/unordsf2vfp.S
)
set(arm_Thumb1_icache_SOURCES
set(arm_Thumb1_SOURCES
${arm_Thumb1_JT_SOURCES}
${arm_Thumb1_SjLj_EH_SOURCES}
- ${arm_Thumb1_VFPv2_SOURCES}
+ ${arm_Thumb1_VFPv2_DP_SOURCES}
+ ${arm_Thumb1_VFPv2_SP_SOURCES}
${arm_Thumb1_icache_SOURCES}
)
arm/aeabi_uldivmod.S
arm/chkstk.S
mingw_fixfloat.c
- ${GENERIC_SOURCES}
+ ${arm_SOURCES}
)
elseif(NOT WIN32)
# TODO the EABI sources should only be added to EABI targets
set(aarch64_SOURCES
${GENERIC_TF_SOURCES}
${GENERIC_SOURCES}
+ cpu_model.c
aarch64/fp_mode.c
)
+# Generate outline atomics helpers from lse.S base
+set(OA_HELPERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/outline_atomic_helpers.dir")
+file(MAKE_DIRECTORY "${OA_HELPERS_DIR}")
+
+if(CMAKE_HOST_UNIX)
+ set(COMPILER_RT_LINK_OR_COPY create_symlink)
+else()
+ set(COMPILER_RT_LINK_OR_COPY copy)
+endif()
+
+foreach(pat cas swp ldadd ldclr ldeor ldset)
+ foreach(size 1 2 4 8 16)
+ foreach(model 1 2 3 4)
+ if(pat STREQUAL "cas" OR NOT size STREQUAL "16")
+ set(helper_asm "${OA_HELPERS_DIR}/outline_atomic_${pat}${size}_${model}.S")
+ list(APPEND lse_builtins "${helper_asm}")
+ list(APPEND arm64_lse_commands COMMAND ${CMAKE_COMMAND} -E ${COMPILER_RT_LINK_OR_COPY} "${CMAKE_CURRENT_SOURCE_DIR}/aarch64/lse.S" "${helper_asm}")
+ set_source_files_properties("${helper_asm}"
+ PROPERTIES
+ COMPILE_DEFINITIONS "L_${pat};SIZE=${size};MODEL=${model}"
+ INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}"
+ )
+ list(APPEND aarch64_SOURCES "${helper_asm}")
+ endif()
+ endforeach(model)
+ endforeach(size)
+endforeach(pat)
+
if (MINGW)
set(aarch64_SOURCES
${aarch64_SOURCES}
set(armv7k_SOURCES ${arm_SOURCES})
set(arm64_SOURCES ${aarch64_SOURCES})
set(arm64e_SOURCES ${aarch64_SOURCES})
+set(arm64_32_SOURCES ${aarch64_SOURCES})
# macho_embedded archs
set(armv6m_SOURCES ${thumb1_SOURCES})
set(armv7m_SOURCES ${arm_SOURCES})
set(armv7em_SOURCES ${arm_SOURCES})
+set(armv8m.main_SOURCES ${arm_SOURCES})
+set(armv8.1m.main_SOURCES ${arm_SOURCES})
# hexagon arch
set(hexagon_SOURCES
set(mips64el_SOURCES ${GENERIC_TF_SOURCES}
${mips_SOURCES})
+set(powerpc_SOURCES ${GENERIC_SOURCES})
+
set(powerpc64_SOURCES
ppc/divtc3.c
- ppc/fixtfti.c
ppc/fixtfdi.c
- ppc/fixunstfti.c
ppc/fixunstfdi.c
- ppc/floattitf.c
ppc/floatditf.c
ppc/floatunditf.c
ppc/gcc_qadd.c
ppc/multc3.c
${GENERIC_SOURCES}
)
+# These routines require __int128, which isn't supported on AIX.
+if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ set(powerpc64_SOURCES
+ ppc/floattitf.c
+ ppc/fixtfti.c
+ ppc/fixunstfti.c
+ ${powerpc64_SOURCES}
+ )
+endif()
set(powerpc64le_SOURCES ${powerpc64_SOURCES})
-set(riscv_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
+set(riscv_SOURCES
+ riscv/save.S
+ riscv/restore.S
+ ${GENERIC_SOURCES}
+ ${GENERIC_TF_SOURCES}
+)
set(riscv32_SOURCES
riscv/mulsi3.S
${riscv_SOURCES}
)
-set(riscv64_SOURCES ${riscv_SOURCES})
+set(riscv64_SOURCES
+ riscv/muldi3.S
+ ${riscv_SOURCES}
+)
set(sparc_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
set(sparcv9_SOURCES ${GENERIC_SOURCES} ${GENERIC_TF_SOURCES})
else ()
set(BUILTIN_CFLAGS "")
+ append_list_if(COMPILER_RT_HAS_FLOAT16 -DCOMPILER_RT_HAS_FLOAT16 BUILTIN_CFLAGS)
+
append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 BUILTIN_CFLAGS)
# These flags would normally be added to CMAKE_C_FLAGS by the llvm
append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC BUILTIN_CFLAGS)
endif()
append_list_if(COMPILER_RT_HAS_FNO_BUILTIN_FLAG -fno-builtin BUILTIN_CFLAGS)
- if(NOT ANDROID)
+ if(COMPILER_RT_BUILTINS_HIDE_SYMBOLS)
append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG -fvisibility=hidden BUILTIN_CFLAGS)
endif()
if(NOT COMPILER_RT_DEBUG)
set(BUILTIN_DEFS "")
- if(NOT ANDROID)
+ if(COMPILER_RT_BUILTINS_HIDE_SYMBOLS)
append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG VISIBILITY_HIDDEN BUILTIN_DEFS)
endif()
+ append_list_if(COMPILER_RT_HAS_ASM_LSE HAS_ASM_LSE BUILTIN_DEFS)
+
foreach (arch ${BUILTIN_SUPPORTED_ARCH})
if (CAN_TARGET_${arch})
+ set(BUILTIN_CFLAGS_${arch} ${BUILTIN_CFLAGS})
# For ARM archs, exclude any VFP builtins if VFP is not supported
- if (${arch} MATCHES "^(arm|armhf|armv7|armv7s|armv7k|armv7m|armv7em)$")
+ if (${arch} MATCHES "^(arm|armhf|armv7|armv7s|armv7k|armv7m|armv7em|armv8m.main|armv8.1m.main)$")
string(REPLACE ";" " " _TARGET_${arch}_CFLAGS "${TARGET_${arch}_CFLAGS}")
- check_compile_definition(__VFP_FP__ "${CMAKE_C_FLAGS} ${_TARGET_${arch}_CFLAGS}" COMPILER_RT_HAS_${arch}_VFP)
+ check_compile_definition(__ARM_FP "${CMAKE_C_FLAGS} ${_TARGET_${arch}_CFLAGS}" COMPILER_RT_HAS_${arch}_VFP)
if(NOT COMPILER_RT_HAS_${arch}_VFP)
- list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_SOURCES} ${arm_Thumb1_SjLj_EH_SOURCES})
+ list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES} ${arm_Thumb1_VFPv2_SP_SOURCES} ${arm_Thumb1_SjLj_EH_SOURCES})
+ else()
+ # Exclude any double-precision builtins if VFP is single-precision-only
+ try_compile_only(COMPILER_RT_HAS_${arch}_VFP_DP
+ SOURCE "#if !(__ARM_FP & 0x8)
+ #error No double-precision support!
+ #endif
+ int main() { return 0; }")
+ if(NOT COMPILER_RT_HAS_${arch}_VFP_DP)
+ list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES})
+ endif()
endif()
endif()
# Needed for clear_cache on debug mode, due to r7's usage in inline asm.
# Release mode already sets it via -O2/3, Debug mode doesn't.
if (${arch} STREQUAL "armhf")
- list(APPEND BUILTIN_CFLAGS -fomit-frame-pointer -DCOMPILER_RT_ARMHF_TARGET)
+ list(APPEND BUILTIN_CFLAGS_${arch} -fomit-frame-pointer -DCOMPILER_RT_ARMHF_TARGET)
endif()
# For RISCV32, we must force enable int128 for compiling long
# double routines.
if("${arch}" STREQUAL "riscv32")
- list(APPEND BUILTIN_CFLAGS -fforce-enable-int128)
+ list(APPEND BUILTIN_CFLAGS_${arch} -fforce-enable-int128)
+ endif()
+
+ if(arch STREQUAL "aarch64")
+ add_custom_target(
+ lse_builtin_symlinks
+ BYPRODUCTS ${lse_builtins}
+ ${arm64_lse_commands}
+ )
+
+ set(deps_aarch64 lse_builtin_symlinks)
endif()
add_compiler_rt_runtime(clang_rt.builtins
STATIC
ARCHS ${arch}
+ DEPS ${deps_${arch}}
SOURCES ${${arch}_SOURCES}
DEFS ${BUILTIN_DEFS}
- CFLAGS ${BUILTIN_CFLAGS}
+ CFLAGS ${BUILTIN_CFLAGS_${arch}}
PARENT_TARGET builtins)
endif ()
endforeach ()
endif ()
+option(COMPILER_RT_BUILD_STANDALONE_LIBATOMIC
+ "Build standalone shared atomic library."
+ OFF)
+
+if(COMPILER_RT_BUILD_STANDALONE_LIBATOMIC)
+ add_custom_target(builtins-standalone-atomic)
+ set(BUILTIN_DEPS "")
+ set(BUILTIN_TYPE SHARED)
+ if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ if(NOT COMPILER_RT_LIBATOMIC_LINK_FLAGS)
+ get_aix_libatomic_default_link_flags(COMPILER_RT_LIBATOMIC_LINK_FLAGS
+ "${CMAKE_CURRENT_SOURCE_DIR}/ppc/atomic.exp")
+ endif()
+ # The compiler needs builtins to link any other binaries, so let
+ # clang_rt.atomic be built after builtins.
+ set(BUILTIN_DEPS builtins)
+ # For different versions of cmake, SHARED behaves differently. For some
+ # versions, we might need MODULE rather than SHARED.
+ get_aix_libatomic_type(BUILTIN_TYPE)
+ endif()
+ foreach (arch ${BUILTIN_SUPPORTED_ARCH})
+ if(CAN_TARGET_${arch})
+ add_compiler_rt_runtime(clang_rt.atomic
+ ${BUILTIN_TYPE}
+ ARCHS ${arch}
+ SOURCES atomic.c
+ LINK_FLAGS ${COMPILER_RT_LIBATOMIC_LINK_FLAGS}
+ DEPS ${BUILTIN_DEPS}
+ PARENT_TARGET builtins-standalone-atomic)
+ endif()
+ endforeach()
+ # FIXME: On AIX, we have to archive built shared libraries into a static
+ # archive, i.e., libatomic.a. Once cmake adds support of such usage for AIX,
+ # this ad-hoc part can be removed.
+ if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+ archive_aix_libatomic(clang_rt.atomic
+ ARCHS ${BUILTIN_SUPPORTED_ARCH}
+ PARENT_TARGET builtins-standalone-atomic)
+ endif()
+ add_dependencies(compiler-rt builtins-standalone-atomic)
+endif()
+
add_dependencies(compiler-rt builtins)
tu_int __udivmodti4(tu_int a, tu_int b, tu_int* rem); // a / b, *rem = a % b unsigned
su_int __udivmodsi4(su_int a, su_int b, su_int* rem); // a / b, *rem = a % b unsigned
si_int __divmodsi4(si_int a, si_int b, si_int* rem); // a / b, *rem = a % b signed
+di_int __divmoddi4(di_int a, di_int b, di_int* rem); // a / b, *rem = a % b signed
+ti_int __divmodti4(ti_int a, ti_int b, ti_int* rem); // a / b, *rem = a % b signed
#ifndef __ARM_FP
// For soft float targets, allow changing rounding mode by overriding the weak
// __aarch64_fe_default_rmode symbol.
-FE_ROUND_MODE __attribute__((weak)) __aarch64_fe_default_rmode = FE_TONEAREST;
+CRT_FE_ROUND_MODE __attribute__((weak)) __aarch64_fe_default_rmode =
+ CRT_FE_TONEAREST;
#endif
-FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround() {
#ifdef __ARM_FP
uint64_t fpcr;
__asm__ __volatile__("mrs %0, fpcr" : "=r" (fpcr));
fpcr = fpcr >> AARCH64_RMODE_SHIFT & AARCH64_RMODE_MASK;
switch (fpcr) {
case AARCH64_UPWARD:
- return FE_UPWARD;
+ return CRT_FE_UPWARD;
case AARCH64_DOWNWARD:
- return FE_DOWNWARD;
+ return CRT_FE_DOWNWARD;
case AARCH64_TOWARDZERO:
- return FE_TOWARDZERO;
+ return CRT_FE_TOWARDZERO;
case AARCH64_TONEAREST:
default:
- return FE_TONEAREST;
+ return CRT_FE_TONEAREST;
}
#else
return __aarch64_fe_default_rmode;
--- /dev/null
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "assembly.h"
+
+// Out-of-line LSE atomics helpers. Ported from libgcc library.
+// N = {1, 2, 4, 8}
+// M = {1, 2, 4, 8, 16}
+// ORDER = {'relax', 'acq', 'rel', 'acq_rel'}
+// Routines implemented:
+//
+// iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
+// iN __aarch64_swpN_ORDER(iN val, iN *ptr)
+// iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
+// iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
+// iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
+// iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)
+//
+// Routines may modify temporary registers tmp0, tmp1, tmp2,
+// return value x0 and the flags only.
+
+#ifdef __aarch64__
+
+#ifdef HAS_ASM_LSE
+.arch armv8-a+lse
+#else
+.arch armv8-a
+#endif
+
+#if !defined(__APPLE__)
+HIDDEN(__aarch64_have_lse_atomics)
+#else
+HIDDEN(___aarch64_have_lse_atomics)
+#endif
+
+// Generate mnemonics for
+// L_cas: SIZE: 1,2,4,8,16 MODEL: 1,2,3,4
+// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8 MODEL: 1,2,3,4
+
+#if SIZE == 1
+#define S b
+#define UXT uxtb
+#define B 0x00000000
+#elif SIZE == 2
+#define S h
+#define UXT uxth
+#define B 0x40000000
+#elif SIZE == 4 || SIZE == 8 || SIZE == 16
+#define S
+#define UXT mov
+#if SIZE == 4
+#define B 0x80000000
+#elif SIZE == 8
+#define B 0xc0000000
+#endif
+#else
+#error
+#endif // SIZE
+
+#if MODEL == 1
+#define SUFF _relax
+#define A
+#define L
+#define M 0x000000
+#define N 0x000000
+#elif MODEL == 2
+#define SUFF _acq
+#define A a
+#define L
+#define M 0x400000
+#define N 0x800000
+#elif MODEL == 3
+#define SUFF _rel
+#define A
+#define L l
+#define M 0x008000
+#define N 0x400000
+#elif MODEL == 4
+#define SUFF _acq_rel
+#define A a
+#define L l
+#define M 0x408000
+#define N 0xc00000
+#else
+#error
+#endif // MODEL
+
+// Define register size.
+#define x(N) GLUE2(x, N)
+#define w(N) GLUE2(w, N)
+#if SIZE < 8
+#define s(N) w(N)
+#else
+#define s(N) x(N)
+#endif
+
+#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)
+#define LDXR GLUE4(ld, A, xr, S)
+#define STXR GLUE4(st, L, xr, S)
+
+// Define temporary registers.
+#define tmp0 16
+#define tmp1 17
+#define tmp2 15
+
+// Macro for branch to label if no LSE available
+.macro JUMP_IF_NOT_LSE label
+#if !defined(__APPLE__)
+ adrp x(tmp0), __aarch64_have_lse_atomics
+ ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]
+#else
+ adrp x(tmp0), ___aarch64_have_lse_atomics@page
+ ldrb w(tmp0), [x(tmp0), ___aarch64_have_lse_atomics@pageoff]
+#endif
+ cbz w(tmp0), \label
+.endm
+
+#ifdef L_cas
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas))
+ JUMP_IF_NOT_LSE 8f
+#if SIZE < 16
+#ifdef HAS_ASM_LSE
+#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2]
+#else
+#define CAS .inst 0x08a07c41 + B + M
+#endif
+ CAS // s(0), s(1), [x2]
+ ret
+8:
+ UXT s(tmp0), s(0)
+0:
+ LDXR s(0), [x2]
+ cmp s(0), s(tmp0)
+ bne 1f
+ STXR w(tmp1), s(1), [x2]
+ cbnz w(tmp1), 0b
+1:
+ ret
+#else
+#define LDXP GLUE3(ld, A, xp)
+#define STXP GLUE3(st, L, xp)
+#ifdef HAS_ASM_LSE
+#define CASP GLUE3(casp, A, L) x0, x1, x2, x3, [x4]
+#else
+#define CASP .inst 0x48207c82 + M
+#endif
+
+ CASP // x0, x1, x2, x3, [x4]
+ ret
+8:
+ mov x(tmp0), x0
+ mov x(tmp1), x1
+0:
+ LDXP x0, x1, [x4]
+ cmp x0, x(tmp0)
+ ccmp x1, x(tmp1), #0, eq
+ bne 1f
+ STXP w(tmp2), x2, x3, [x4]
+ cbnz w(tmp2), 0b
+1:
+ ret
+#endif
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))
+#endif // L_cas
+
+#ifdef L_swp
+#ifdef HAS_ASM_LSE
+#define SWP GLUE4(swp, A, L, S) s(0), s(0), [x1]
+#else
+#define SWP .inst 0x38208020 + B + N
+#endif
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))
+ JUMP_IF_NOT_LSE 8f
+ SWP // s(0), s(0), [x1]
+ ret
+8:
+ mov s(tmp0), s(0)
+0:
+ LDXR s(0), [x1]
+ STXR w(tmp1), s(tmp0), [x1]
+ cbnz w(tmp1), 0b
+ ret
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))
+#endif // L_swp
+
+#if defined(L_ldadd) || defined(L_ldclr) || \
+ defined(L_ldeor) || defined(L_ldset)
+
+#ifdef L_ldadd
+#define LDNM ldadd
+#define OP add
+#define OPN 0x0000
+#elif defined(L_ldclr)
+#define LDNM ldclr
+#define OP bic
+#define OPN 0x1000
+#elif defined(L_ldeor)
+#define LDNM ldeor
+#define OP eor
+#define OPN 0x2000
+#elif defined(L_ldset)
+#define LDNM ldset
+#define OP orr
+#define OPN 0x3000
+#else
+#error
+#endif
+
+#ifdef HAS_ASM_LSE
+#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1]
+#else
+#define LDOP .inst 0x38200020 + OPN + B + N
+#endif
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))
+ JUMP_IF_NOT_LSE 8f
+ LDOP // s(0), s(0), [x1]
+ ret
+8:
+ mov s(tmp0), s(0)
+0:
+ LDXR s(0), [x1]
+ OP s(tmp1), s(0), s(tmp0)
+ STXR w(tmp2), s(tmp1), [x1]
+ cbnz w(tmp2), 0b
+ ret
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
+#endif // L_ldadd L_ldclr L_ldeor L_ldset
+
+NO_EXEC_STACK_DIRECTIVE
+
+// GNU property note for BTI and PAC
+GNU_PROPERTY_BTI_PAC
+
+#endif // __aarch64__
ARM_DOWNWARD | ARM_TOWARDZERO)
#define ARM_RMODE_SHIFT 22
-#define ARM_INEXACT 0x1000
+#define ARM_INEXACT 0x10
#ifndef __ARM_FP
// For soft float targets, allow changing rounding mode by overriding the weak
// __arm_fe_default_rmode symbol.
-FE_ROUND_MODE __attribute__((weak)) __arm_fe_default_rmode = FE_TONEAREST;
+CRT_FE_ROUND_MODE __attribute__((weak)) __arm_fe_default_rmode =
+ CRT_FE_TONEAREST;
#endif
-FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround() {
#ifdef __ARM_FP
uint32_t fpscr;
__asm__ __volatile__("vmrs %0, fpscr" : "=r" (fpscr));
fpscr = fpscr >> ARM_RMODE_SHIFT & ARM_RMODE_MASK;
switch (fpscr) {
case ARM_UPWARD:
- return FE_UPWARD;
+ return CRT_FE_UPWARD;
case ARM_DOWNWARD:
- return FE_DOWNWARD;
+ return CRT_FE_DOWNWARD;
case ARM_TOWARDZERO:
- return FE_TOWARDZERO;
+ return CRT_FE_TOWARDZERO;
case ARM_TONEAREST:
default:
- return FE_TONEAREST;
+ return CRT_FE_TONEAREST;
}
#else
return __arm_fe_default_rmode;
#ifndef COMPILERRT_ASSEMBLY_H
#define COMPILERRT_ASSEMBLY_H
-#if defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__)
-#define SEPARATOR @
+#if defined(__APPLE__) && defined(__aarch64__)
+#define SEPARATOR %%
#else
#define SEPARATOR ;
#endif
#define HIDDEN(name) .hidden name
#define LOCAL_LABEL(name) .L_##name
#define FILE_LEVEL_DIRECTIVE
-#if defined(__arm__)
+#if defined(__arm__) || defined(__aarch64__)
#define SYMBOL_IS_FUNC(name) .type name,%function
#else
#define SYMBOL_IS_FUNC(name) .type name,@function
#endif
#define CONST_SECTION .section .rodata
-#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
defined(__linux__)
#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
#else
#endif
+#if defined(__arm__) || defined(__aarch64__)
+#define FUNC_ALIGN \
+ .text SEPARATOR \
+ .balign 16 SEPARATOR
+#else
+#define FUNC_ALIGN
+#endif
+
+// BTI and PAC gnu property note
+#define NT_GNU_PROPERTY_TYPE_0 5
+#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
+#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
+#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
+
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
+#else
+#define BTI_FLAG 0
+#endif
+
+#if __ARM_FEATURE_PAC_DEFAULT & 3
+#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC
+#else
+#define PAC_FLAG 0
+#endif
+
+#define GNU_PROPERTY(type, value) \
+ .pushsection .note.gnu.property, "a" SEPARATOR \
+ .p2align 3 SEPARATOR \
+ .word 4 SEPARATOR \
+ .word 16 SEPARATOR \
+ .word NT_GNU_PROPERTY_TYPE_0 SEPARATOR \
+ .asciz "GNU" SEPARATOR \
+ .word type SEPARATOR \
+ .word 4 SEPARATOR \
+ .word value SEPARATOR \
+ .word 0 SEPARATOR \
+ .popsection
+
+#if BTI_FLAG != 0
+#define BTI_C hint #34
+#define BTI_J hint #36
+#else
+#define BTI_C
+#define BTI_J
+#endif
+
+#if (BTI_FLAG | PAC_FLAG) != 0
+#define GNU_PROPERTY_BTI_PAC \
+ GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG)
+#else
+#define GNU_PROPERTY_BTI_PAC
+#endif
+
+#if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM)
+#define CFI_START .cfi_startproc
+#define CFI_END .cfi_endproc
+#else
+#define CFI_START
+#define CFI_END
+#endif
+
#if defined(__arm__)
// Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
#define DEFINE_CODE_STATE
#endif
-#define GLUE2(a, b) a##b
-#define GLUE(a, b) GLUE2(a, b)
+#define GLUE2_(a, b) a##b
+#define GLUE(a, b) GLUE2_(a, b)
+#define GLUE2(a, b) GLUE2_(a, b)
+#define GLUE3_(a, b, c) a##b##c
+#define GLUE3(a, b, c) GLUE3_(a, b, c)
+#define GLUE4_(a, b, c, d) a##b##c##d
+#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d)
+
#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
#ifdef VISIBILITY_HIDDEN
#define DECLARE_SYMBOL_VISIBILITY(name) \
HIDDEN(SYMBOL_NAME(name)) SEPARATOR
+#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) \
+ HIDDEN(name) SEPARATOR
#else
#define DECLARE_SYMBOL_VISIBILITY(name)
+#define DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name)
#endif
#define DEFINE_COMPILERRT_FUNCTION(name) \
DECLARE_FUNC_ENCODING \
name:
+#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \
+ DEFINE_CODE_STATE \
+ FUNC_ALIGN \
+ .globl name SEPARATOR \
+ SYMBOL_IS_FUNC(name) SEPARATOR \
+ DECLARE_SYMBOL_VISIBILITY_UNMANGLED(name) SEPARATOR \
+ CFI_START SEPARATOR \
+ DECLARE_FUNC_ENCODING \
+ name: SEPARATOR BTI_C
+
#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
#ifdef __ELF__
#define END_COMPILERRT_FUNCTION(name) \
.size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
+ CFI_END SEPARATOR \
+ .size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
#else
#define END_COMPILERRT_FUNCTION(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name) \
+ CFI_END
#endif
#endif // COMPILERRT_ASSEMBLY_H
//===----------------------------------------------------------------------===//
#include <stdbool.h>
+#include <stddef.h>
#include <stdint.h>
-#include <string.h>
#include "assembly.h"
+// We use __builtin_mem* here to avoid dependencies on libc-provided headers.
+#define memcpy __builtin_memcpy
+#define memcmp __builtin_memcmp
+
// Clang objects if you redefine a builtin. This little hack allows us to
// define a function with the same name as an intrinsic.
#pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load)
#pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange)
#pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \
__atomic_compare_exchange)
+#pragma redefine_extname __atomic_is_lock_free_c SYMBOL_NAME( \
+ __atomic_is_lock_free)
/// Number of locks. This allocates one page on 32-bit platforms, two on
/// 64-bit. This can be specified externally if a different trade between
// defined. Each platform should define the Lock type, and corresponding
// lock() and unlock() functions.
////////////////////////////////////////////////////////////////////////////////
-#ifdef __FreeBSD__
+#if defined(__FreeBSD__) || defined(__DragonFly__)
#include <errno.h>
// clang-format off
#include <sys/types.h>
return locks + (hash & SPINLOCK_MASK);
}
-/// Macros for determining whether a size is lock free. Clang can not yet
-/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are
-/// not lock free.
-#define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1)
-#define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2)
-#define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4)
-#define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8)
-#define IS_LOCK_FREE_16 0
+/// Macros for determining whether a size is lock free.
+#define ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(size, p) \
+ (__atomic_always_lock_free(size, p) || \
+ (__atomic_always_lock_free(size, 0) && ((uintptr_t)p % size) == 0))
+#define IS_LOCK_FREE_1(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(1, p)
+#define IS_LOCK_FREE_2(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(2, p)
+#define IS_LOCK_FREE_4(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(4, p)
+#define IS_LOCK_FREE_8(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(8, p)
+#define IS_LOCK_FREE_16(p) ATOMIC_ALWAYS_LOCK_FREE_OR_ALIGNED_LOCK_FREE(16, p)
/// Macro that calls the compiler-generated lock-free versions of functions
/// when they exist.
-#define LOCK_FREE_CASES() \
+#define TRY_LOCK_FREE_CASE(n, type, ptr) \
+ case n: \
+ if (IS_LOCK_FREE_##n(ptr)) { \
+ LOCK_FREE_ACTION(type); \
+ } \
+ break;
+#ifdef __SIZEOF_INT128__
+#define TRY_LOCK_FREE_CASE_16(p) TRY_LOCK_FREE_CASE(16, __uint128_t, p)
+#else
+#define TRY_LOCK_FREE_CASE_16(p) /* __uint128_t not available */
+#endif
+
+#define LOCK_FREE_CASES(ptr) \
do { \
switch (size) { \
- case 1: \
- if (IS_LOCK_FREE_1) { \
- LOCK_FREE_ACTION(uint8_t); \
- } \
- break; \
- case 2: \
- if (IS_LOCK_FREE_2) { \
- LOCK_FREE_ACTION(uint16_t); \
- } \
- break; \
- case 4: \
- if (IS_LOCK_FREE_4) { \
- LOCK_FREE_ACTION(uint32_t); \
- } \
- break; \
- case 8: \
- if (IS_LOCK_FREE_8) { \
- LOCK_FREE_ACTION(uint64_t); \
- } \
- break; \
- case 16: \
- if (IS_LOCK_FREE_16) { \
- /* FIXME: __uint128_t isn't available on 32 bit platforms. \
- LOCK_FREE_ACTION(__uint128_t);*/ \
- } \
+ TRY_LOCK_FREE_CASE(1, uint8_t, ptr) \
+ TRY_LOCK_FREE_CASE(2, uint16_t, ptr) \
+ TRY_LOCK_FREE_CASE(4, uint32_t, ptr) \
+ TRY_LOCK_FREE_CASE(8, uint64_t, ptr) \
+ TRY_LOCK_FREE_CASE_16(ptr) /* __uint128_t may not be supported */ \
+ default: \
break; \
} \
} while (0)
+/// Whether atomic operations for the given size (and alignment) are lock-free.
+bool __atomic_is_lock_free_c(size_t size, void *ptr) {
+#define LOCK_FREE_ACTION(type) return true;
+ LOCK_FREE_CASES(ptr);
+#undef LOCK_FREE_ACTION
+ return false;
+}
+
/// An atomic load operation. This is atomic with respect to the source
/// pointer only.
void __atomic_load_c(int size, void *src, void *dest, int model) {
#define LOCK_FREE_ACTION(type) \
*((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(src);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(src);
lock(l);
#define LOCK_FREE_ACTION(type) \
__c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(dest);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(dest);
lock(l);
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \
failure)
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
*(type *)old = \
__c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \
return;
- LOCK_FREE_CASES();
+ LOCK_FREE_CASES(ptr);
#undef LOCK_FREE_ACTION
Lock *l = lock_for_pointer(ptr);
lock(l);
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_load_##n(type *src, int model) { \
- if (lockfree) \
+ if (lockfree(src)) \
return __c11_atomic_load((_Atomic(type) *)src, model); \
Lock *l = lock_for_pointer(src); \
lock(l); \
#define OPTIMISED_CASE(n, lockfree, type) \
void __atomic_store_##n(type *dest, type val, int model) { \
- if (lockfree) { \
+ if (lockfree(dest)) { \
__c11_atomic_store((_Atomic(type) *)dest, val, model); \
return; \
} \
#define OPTIMISED_CASE(n, lockfree, type) \
type __atomic_exchange_##n(type *dest, type val, int model) { \
- if (lockfree) \
+ if (lockfree(dest)) \
return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \
Lock *l = lock_for_pointer(dest); \
lock(l); \
#define OPTIMISED_CASE(n, lockfree, type) \
bool __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \
int success, int failure) { \
- if (lockfree) \
+ if (lockfree(ptr)) \
return __c11_atomic_compare_exchange_strong( \
(_Atomic(type) *)ptr, expected, desired, success, failure); \
Lock *l = lock_for_pointer(ptr); \
////////////////////////////////////////////////////////////////////////////////
#define ATOMIC_RMW(n, lockfree, type, opname, op) \
type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \
- if (lockfree) \
+ if (lockfree(ptr)) \
return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \
Lock *l = lock_for_pointer(ptr); \
lock(l); \
#define DOUBLE_PRECISION
#include "fp_lib.h"
-enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 };
+#include "fp_compare_impl.inc"
-COMPILER_RT_ABI enum LE_RESULT __ledf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- // If either a or b is NaN, they are unordered.
- if (aAbs > infRep || bAbs > infRep)
- return LE_UNORDERED;
-
- // If a and b are both zeros, they are equal.
- if ((aAbs | bAbs) == 0)
- return LE_EQUAL;
-
- // If at least one of a and b is positive, we get the same result comparing
- // a and b as signed integers as we would with a floating-point compare.
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- }
-
- // Otherwise, both are negative, so we need to flip the sense of the
- // comparison to get the correct result. (This assumes a twos- or ones-
- // complement integer representation; if integers are represented in a
- // sign-magnitude representation, then this flip is incorrect).
- else {
- if (aInt > bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __ledf2(fp_t a, fp_t b) { return __leXf2__(a, b); }
#if defined(__ELF__)
// Alias for libgcc compatibility
COMPILER_RT_ALIAS(__ledf2, __ltdf2)
COMPILER_RT_ALIAS(__ledf2, __nedf2)
-enum GE_RESULT {
- GE_LESS = -1,
- GE_EQUAL = 0,
- GE_GREATER = 1,
- GE_UNORDERED = -1 // Note: different from LE_UNORDERED
-};
-
-COMPILER_RT_ABI enum GE_RESULT __gedf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- if (aAbs > infRep || bAbs > infRep)
- return GE_UNORDERED;
- if ((aAbs | bAbs) == 0)
- return GE_EQUAL;
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- } else {
- if (aInt > bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __gedf2(fp_t a, fp_t b) { return __geXf2__(a, b); }
COMPILER_RT_ALIAS(__gedf2, __gtdf2)
-COMPILER_RT_ABI int
-__unorddf2(fp_t a, fp_t b) {
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
- return aAbs > infRep || bAbs > infRep;
+COMPILER_RT_ABI CMP_RESULT __unorddf2(fp_t a, fp_t b) {
+ return __unordXf2__(a, b);
}
#if defined(__ARM_EABI__)
#define SINGLE_PRECISION
#include "fp_lib.h"
-enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 };
+#include "fp_compare_impl.inc"
-COMPILER_RT_ABI enum LE_RESULT __lesf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- // If either a or b is NaN, they are unordered.
- if (aAbs > infRep || bAbs > infRep)
- return LE_UNORDERED;
-
- // If a and b are both zeros, they are equal.
- if ((aAbs | bAbs) == 0)
- return LE_EQUAL;
-
- // If at least one of a and b is positive, we get the same result comparing
- // a and b as signed integers as we would with a fp_ting-point compare.
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- }
-
- // Otherwise, both are negative, so we need to flip the sense of the
- // comparison to get the correct result. (This assumes a twos- or ones-
- // complement integer representation; if integers are represented in a
- // sign-magnitude representation, then this flip is incorrect).
- else {
- if (aInt > bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __lesf2(fp_t a, fp_t b) { return __leXf2__(a, b); }
#if defined(__ELF__)
// Alias for libgcc compatibility
COMPILER_RT_ALIAS(__lesf2, __ltsf2)
COMPILER_RT_ALIAS(__lesf2, __nesf2)
-enum GE_RESULT {
- GE_LESS = -1,
- GE_EQUAL = 0,
- GE_GREATER = 1,
- GE_UNORDERED = -1 // Note: different from LE_UNORDERED
-};
-
-COMPILER_RT_ABI enum GE_RESULT __gesf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- if (aAbs > infRep || bAbs > infRep)
- return GE_UNORDERED;
- if ((aAbs | bAbs) == 0)
- return GE_EQUAL;
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- } else {
- if (aInt > bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __gesf2(fp_t a, fp_t b) { return __geXf2__(a, b); }
COMPILER_RT_ALIAS(__gesf2, __gtsf2)
-COMPILER_RT_ABI int
-__unordsf2(fp_t a, fp_t b) {
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
- return aAbs > infRep || bAbs > infRep;
+COMPILER_RT_ABI CMP_RESULT __unordsf2(fp_t a, fp_t b) {
+ return __unordXf2__(a, b);
}
#if defined(__ARM_EABI__)
#include "fp_lib.h"
#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
-enum LE_RESULT { LE_LESS = -1, LE_EQUAL = 0, LE_GREATER = 1, LE_UNORDERED = 1 };
+#include "fp_compare_impl.inc"
-COMPILER_RT_ABI enum LE_RESULT __letf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- // If either a or b is NaN, they are unordered.
- if (aAbs > infRep || bAbs > infRep)
- return LE_UNORDERED;
-
- // If a and b are both zeros, they are equal.
- if ((aAbs | bAbs) == 0)
- return LE_EQUAL;
-
- // If at least one of a and b is positive, we get the same result comparing
- // a and b as signed integers as we would with a floating-point compare.
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- } else {
- // Otherwise, both are negative, so we need to flip the sense of the
- // comparison to get the correct result. (This assumes a twos- or ones-
- // complement integer representation; if integers are represented in a
- // sign-magnitude representation, then this flip is incorrect).
- if (aInt > bInt)
- return LE_LESS;
- else if (aInt == bInt)
- return LE_EQUAL;
- else
- return LE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __letf2(fp_t a, fp_t b) { return __leXf2__(a, b); }
#if defined(__ELF__)
// Alias for libgcc compatibility
COMPILER_RT_ALIAS(__letf2, __lttf2)
COMPILER_RT_ALIAS(__letf2, __netf2)
-enum GE_RESULT {
- GE_LESS = -1,
- GE_EQUAL = 0,
- GE_GREATER = 1,
- GE_UNORDERED = -1 // Note: different from LE_UNORDERED
-};
-
-COMPILER_RT_ABI enum GE_RESULT __getf2(fp_t a, fp_t b) {
-
- const srep_t aInt = toRep(a);
- const srep_t bInt = toRep(b);
- const rep_t aAbs = aInt & absMask;
- const rep_t bAbs = bInt & absMask;
-
- if (aAbs > infRep || bAbs > infRep)
- return GE_UNORDERED;
- if ((aAbs | bAbs) == 0)
- return GE_EQUAL;
- if ((aInt & bInt) >= 0) {
- if (aInt < bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- } else {
- if (aInt > bInt)
- return GE_LESS;
- else if (aInt == bInt)
- return GE_EQUAL;
- else
- return GE_GREATER;
- }
-}
+COMPILER_RT_ABI CMP_RESULT __getf2(fp_t a, fp_t b) { return __geXf2__(a, b); }
COMPILER_RT_ALIAS(__getf2, __gttf2)
-COMPILER_RT_ABI int __unordtf2(fp_t a, fp_t b) {
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
- return aAbs > infRep || bAbs > infRep;
+COMPILER_RT_ABI CMP_RESULT __unordtf2(fp_t a, fp_t b) {
+ return __unordXf2__(a, b);
}
#endif
//
// This file is based on LLVM's lib/Support/Host.cpp.
// It implements the operating system Host concept and builtin
-// __cpu_model for the compiler_rt library, for x86 only.
+// __cpu_model for the compiler_rt library for x86 and
+// __aarch64_have_lse_atomics for AArch64.
//
//===----------------------------------------------------------------------===//
+#if defined(HAVE_INIT_PRIORITY)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
+#elif __has_attribute(__constructor__)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
+#else
+// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
+// this runs during initialization.
+#define CONSTRUCTOR_ATTRIBUTE
+#endif
+
#if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \
defined(_M_X64)) && \
(defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
INTEL_GOLDMONT,
INTEL_GOLDMONT_PLUS,
INTEL_TREMONT,
+ AMDFAM19H,
CPU_TYPE_MAX
};
INTEL_COREI7_CASCADELAKE,
INTEL_COREI7_TIGERLAKE,
INTEL_COREI7_COOPERLAKE,
+ INTEL_COREI7_SAPPHIRERAPIDS,
+ INTEL_COREI7_ALDERLAKE,
+ AMDFAM19H_ZNVER3,
+ INTEL_COREI7_ROCKETLAKE,
CPU_SUBTYPE_MAX
};
*Subtype = INTEL_COREI7_SKYLAKE;
break;
+ // Rocketlake:
+ case 0xa7:
+ CPU = "rocketlake";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_ROCKETLAKE;
+ break;
+
// Skylake Xeon:
case 0x55:
*Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_ICELAKE_SERVER;
break;
+ // Sapphire Rapids:
+ case 0x8f:
+ CPU = "sapphirerapids";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_SAPPHIRERAPIDS;
+ break;
+
case 0x1c: // Most 45 nm Intel Atom processors
case 0x26: // 45 nm Atom Lincroft
case 0x27: // 32 nm Atom Medfield
break; // 00h-0Fh: Zen1
}
break;
+ case 25:
+ CPU = "znver3";
+ *Type = AMDFAM19H;
+ if (Model <= 0x0f) {
+ *Subtype = AMDFAM19H_ZNVER3;
+ break; // 00h-0Fh: Zen3
+ }
+ break;
default:
break; // Unknown AMD CPU.
}
#undef setFeature
}
-#if defined(HAVE_INIT_PRIORITY)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
-#elif __has_attribute(__constructor__)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
-#else
-// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
-// this runs during initialization.
-#define CONSTRUCTOR_ATTRIBUTE
-#endif
-
#ifndef _WIN32
__attribute__((visibility("hidden")))
#endif
return 0;
}
-
+#elif defined(__aarch64__)
+// LSE support detection for out-of-line atomics
+// using HWCAP and Auxiliary vector
+_Bool __aarch64_have_lse_atomics
+ __attribute__((visibility("hidden"), nocommon));
+#if defined(__has_include)
+#if __has_include(<sys/auxv.h>)
+#include <sys/auxv.h>
+#ifndef AT_HWCAP
+#define AT_HWCAP 16
#endif
+#ifndef HWCAP_ATOMICS
+#define HWCAP_ATOMICS (1 << 8)
+#endif
+static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
+ unsigned long hwcap = getauxval(AT_HWCAP);
+ __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0;
+}
+#endif // defined(__has_include)
+#endif // __has_include(<sys/auxv.h>)
+#endif // defined(__aarch64__)
COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c,
double __d) {
int __ilogbw = 0;
- double __logbw = __compiler_rt_logb(crt_fmax(crt_fabs(__c), crt_fabs(__d)));
+ double __logbw = __compiler_rt_logb(__compiler_rt_fmax(crt_fabs(__c),
+ crt_fabs(__d)));
if (crt_isfinite(__logbw)) {
__ilogbw = (int)__logbw;
- __c = crt_scalbn(__c, -__ilogbw);
- __d = crt_scalbn(__d, -__ilogbw);
+ __c = __compiler_rt_scalbn(__c, -__ilogbw);
+ __d = __compiler_rt_scalbn(__d, -__ilogbw);
}
double __denom = __c * __c + __d * __d;
Dcomplex z;
- COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
+ COMPLEX_REAL(z) =
+ __compiler_rt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw);
COMPLEX_IMAGINARY(z) =
- crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
+ __compiler_rt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw);
if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) {
if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) {
COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a;
// This file implements double-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define DOUBLE_PRECISION
-#include "fp_lib.h"
-
-COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
-
- // Align the significand of b as a Q31 fixed-point number in the range
- // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- const uint32_t q31b = bSignificand >> 21;
- uint32_t recip32 = UINT32_C(0x7504f333) - q31b;
- // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2)
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint32_t correction32;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
- correction32 = -((uint64_t)recip32 * q31b >> 32);
- recip32 = (uint64_t)recip32 * correction32 >> 31;
-
- // The reciprocal may have overflowed to zero if the upper half of b is
- // exactly 1.0. This would sabatoge the full-width final stage of the
- // computation that follows, so we adjust the reciprocal down by one bit.
- recip32--;
-
- // We need to perform one more iteration to get us to 56 binary digits.
- // The last iteration needs to happen with extra precision.
- const uint32_t q63blo = bSignificand << 11;
- uint64_t correction, reciprocal;
- correction = -((uint64_t)recip32 * q31b + ((uint64_t)recip32 * q63blo >> 32));
- uint32_t cHi = correction >> 32;
- uint32_t cLo = correction;
- reciprocal = (uint64_t)recip32 * cHi + ((uint64_t)recip32 * cLo >> 32);
-
- // Adjust the final 64-bit reciprocal estimate downward to ensure that it is
- // strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-56, lies in the
- // interval [0.5, 1.0), and is strictly smaller than the true reciprocal
- // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b
- // in Q53 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0.5, 2.0)
- // 3. The error in q is bounded away from 2^-53 (actually, we have a
- // couple of bits to spare, but this is all we need).
-
- // We need a 64 x 64 multiply high to compute q, which isn't a basic
- // operation in C, so we need to be a little bit fussy.
- rep_t quotient, quotientLo;
- wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo);
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- if (quotient < (implicitBit << 1)) {
- residual = (aSignificand << 53) - quotient * bSignificand;
- quotientExponent--;
- } else {
- quotient >>= 1;
- residual = (aSignificand << 52) - quotient * bSignificand;
- }
-
- const int writtenExponent = quotientExponent + exponentBias;
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- }
+#define NUMBER_OF_HALF_ITERATIONS 3
+#define NUMBER_OF_FULL_ITERATIONS 1
- else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- }
+#include "fp_div_impl.inc"
- else {
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- const double result = fromRep(absResult | quotientSign);
- return result;
- }
-}
+COMPILER_RT_ABI fp_t __divdf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
// Returns: a / b
-COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) {
- const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
- di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0
- di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- return (__udivmoddi4(a, b, (du_int *)0) ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t di_int
+#define fixuint_t du_int
+#define COMPUTE_UDIV(a, b) __udivmoddi4((a), (b), (du_int *)0)
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI di_int __divdi3(di_int a, di_int b) { return __divXi3(a, b); }
// Returns: a / b, *rem = a % b
COMPILER_RT_ABI di_int __divmoddi4(di_int a, di_int b, di_int *rem) {
- di_int d = __divdi3(a, b);
- *rem = a - (d * b);
- return d;
+ const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
+ di_int s_a = a >> bits_in_dword_m1; // s_a = a < 0 ? -1 : 0
+ di_int s_b = b >> bits_in_dword_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ du_int r;
+ di_int q = (__udivmoddi4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
}
// Returns: a / b, *rem = a % b
COMPILER_RT_ABI si_int __divmodsi4(si_int a, si_int b, si_int *rem) {
- si_int d = __divsi3(a, b);
- *rem = a - (d * b);
- return d;
+ const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1;
+ si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0
+ si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ su_int r;
+ si_int q = (__udivmodsi4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
}
--- /dev/null
+//===-- divmodti4.c - Implement __divmodti4 -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements __divmodti4 for the compiler_rt library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+#ifdef CRT_HAS_128BIT
+
+// Returns: a / b, *rem = a % b
+
+COMPILER_RT_ABI ti_int __divmodti4(ti_int a, ti_int b, ti_int *rem) {
+ const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
+ ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0
+ ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0
+ a = (a ^ s_a) - s_a; // negate if s_a == -1
+ b = (b ^ s_b) - s_b; // negate if s_b == -1
+ s_b ^= s_a; // sign of quotient
+ tu_int r;
+ ti_int q = (__udivmodti4(a, b, &r) ^ s_b) - s_b; // negate if s_b == -1
+ *rem = (r ^ s_a) - s_a; // negate if s_a == -1
+ return q;
+}
+
+#endif // CRT_HAS_128BIT
COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) {
int __ilogbw = 0;
float __logbw =
- __compiler_rt_logbf(crt_fmaxf(crt_fabsf(__c), crt_fabsf(__d)));
+ __compiler_rt_logbf(__compiler_rt_fmaxf(crt_fabsf(__c), crt_fabsf(__d)));
if (crt_isfinite(__logbw)) {
__ilogbw = (int)__logbw;
- __c = crt_scalbnf(__c, -__ilogbw);
- __d = crt_scalbnf(__d, -__ilogbw);
+ __c = __compiler_rt_scalbnf(__c, -__ilogbw);
+ __d = __compiler_rt_scalbnf(__d, -__ilogbw);
}
float __denom = __c * __c + __d * __d;
Fcomplex z;
- COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw);
+ COMPLEX_REAL(z) =
+ __compiler_rt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw);
COMPLEX_IMAGINARY(z) =
- crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw);
+ __compiler_rt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw);
if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) {
if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) {
COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a;
// This file implements single-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define SINGLE_PRECISION
-#include "fp_lib.h"
-
-COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
- // 0x7504F333 / 2^32 + 1 = 3/4 + 1/sqrt(2)
-
- // Align the significand of b as a Q31 fixed-point number in the range
- // [1, 2.0) and get a Q32 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- uint32_t q31b = bSignificand << 8;
- uint32_t reciprocal = UINT32_C(0x7504f333) - q31b;
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint32_t correction;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
- correction = -((uint64_t)reciprocal * q31b >> 32);
- reciprocal = (uint64_t)reciprocal * correction >> 31;
-
- // Adust the final 32-bit reciprocal estimate downward to ensure that it is
- // strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-28, lies in the
- // interval [0x1.000000eep-1, 0x1.fffffffcp-1], and is strictly smaller
- // than the true reciprocal of b. Multiplying a by this reciprocal thus
- // gives a numerical q = a/b in Q24 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0x1.000000eep-1, 0x1.fffffffcp0)
- // 3. The error in q is at most 2^-24 + 2^-27 -- the 2^24 term comes
- // from the fact that we truncate the product, and the 2^27 term
- // is the error in the reciprocal of b scaled by the maximum
- // possible value of a. As a consequence of this error bound,
- // either q or nextafter(q) is the correctly rounded.
- rep_t quotient = (uint64_t)reciprocal * (aSignificand << 1) >> 32;
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- if (quotient < (implicitBit << 1)) {
- residual = (aSignificand << 24) - quotient * bSignificand;
- quotientExponent--;
- } else {
- quotient >>= 1;
- residual = (aSignificand << 23) - quotient * bSignificand;
- }
-
- const int writtenExponent = quotientExponent + exponentBias;
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- }
+#define NUMBER_OF_HALF_ITERATIONS 0
+#define NUMBER_OF_FULL_ITERATIONS 3
+#define USE_NATIVE_FULL_ITERATIONS
- else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- }
+#include "fp_div_impl.inc"
- else {
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- return fromRep(absResult | quotientSign);
- }
-}
+COMPILER_RT_ABI fp_t __divsf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
// Returns: a / b
-COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) {
- const int bits_in_word_m1 = (int)(sizeof(si_int) * CHAR_BIT) - 1;
- si_int s_a = a >> bits_in_word_m1; // s_a = a < 0 ? -1 : 0
- si_int s_b = b >> bits_in_word_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- //
- // On CPUs without unsigned hardware division support,
- // this calls __udivsi3 (notice the cast to su_int).
- // On CPUs with unsigned hardware division support,
- // this uses the unsigned division instruction.
- //
- return ((su_int)a / (su_int)b ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t si_int
+#define fixuint_t su_int
+// On CPUs without unsigned hardware division support,
+// this calls __udivsi3 (notice the cast to su_int).
+// On CPUs with unsigned hardware division support,
+// this uses the unsigned division instruction.
+#define COMPUTE_UDIV(a, b) ((su_int)(a) / (su_int)(b))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI si_int __divsi3(si_int a, si_int b) { return __divXi3(a, b); }
#if defined(__ARM_EABI__)
COMPILER_RT_ALIAS(__divsi3, __aeabi_idiv)
long double __c, long double __d) {
int __ilogbw = 0;
long double __logbw =
- __compiler_rt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d)));
+ __compiler_rt_logbl(__compiler_rt_fmaxl(crt_fabsl(__c), crt_fabsl(__d)));
if (crt_isfinite(__logbw)) {
__ilogbw = (int)__logbw;
- __c = crt_scalbnl(__c, -__ilogbw);
- __d = crt_scalbnl(__d, -__ilogbw);
+ __c = __compiler_rt_scalbnl(__c, -__ilogbw);
+ __d = __compiler_rt_scalbnl(__d, -__ilogbw);
}
long double __denom = __c * __c + __d * __d;
Lcomplex z;
- COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw);
+ COMPLEX_REAL(z) =
+ __compiler_rt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw);
COMPLEX_IMAGINARY(z) =
- crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw);
+ __compiler_rt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw);
if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) {
if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) {
COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a;
// This file implements quad-precision soft-float division
// with the IEEE-754 default rounding (to nearest, ties to even).
//
-// For simplicity, this implementation currently flushes denormals to zero.
-// It should be a fairly straightforward exercise to implement gradual
-// underflow with correct rounding.
-//
//===----------------------------------------------------------------------===//
#define QUAD_PRECISION
#include "fp_lib.h"
#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
-COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) {
-
- const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
- const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
- const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
-
- rep_t aSignificand = toRep(a) & significandMask;
- rep_t bSignificand = toRep(b) & significandMask;
- int scale = 0;
-
- // Detect if a or b is zero, denormal, infinity, or NaN.
- if (aExponent - 1U >= maxExponent - 1U ||
- bExponent - 1U >= maxExponent - 1U) {
-
- const rep_t aAbs = toRep(a) & absMask;
- const rep_t bAbs = toRep(b) & absMask;
-
- // NaN / anything = qNaN
- if (aAbs > infRep)
- return fromRep(toRep(a) | quietBit);
- // anything / NaN = qNaN
- if (bAbs > infRep)
- return fromRep(toRep(b) | quietBit);
-
- if (aAbs == infRep) {
- // infinity / infinity = NaN
- if (bAbs == infRep)
- return fromRep(qnanRep);
- // infinity / anything else = +/- infinity
- else
- return fromRep(aAbs | quotientSign);
- }
-
- // anything else / infinity = +/- 0
- if (bAbs == infRep)
- return fromRep(quotientSign);
-
- if (!aAbs) {
- // zero / zero = NaN
- if (!bAbs)
- return fromRep(qnanRep);
- // zero / anything else = +/- zero
- else
- return fromRep(quotientSign);
- }
- // anything else / zero = +/- infinity
- if (!bAbs)
- return fromRep(infRep | quotientSign);
-
- // One or both of a or b is denormal. The other (if applicable) is a
- // normal number. Renormalize one or both of a and b, and set scale to
- // include the necessary exponent adjustment.
- if (aAbs < implicitBit)
- scale += normalize(&aSignificand);
- if (bAbs < implicitBit)
- scale -= normalize(&bSignificand);
- }
-
- // Set the implicit significand bit. If we fell through from the
- // denormal path it was already set by normalize( ), but setting it twice
- // won't hurt anything.
- aSignificand |= implicitBit;
- bSignificand |= implicitBit;
- int quotientExponent = aExponent - bExponent + scale;
-
- // Align the significand of b as a Q63 fixed-point number in the range
- // [1, 2.0) and get a Q64 approximate reciprocal using a small minimax
- // polynomial approximation: reciprocal = 3/4 + 1/sqrt(2) - b/2. This
- // is accurate to about 3.5 binary digits.
- const uint64_t q63b = bSignificand >> 49;
- uint64_t recip64 = UINT64_C(0x7504f333F9DE6484) - q63b;
- // 0x7504f333F9DE6484 / 2^64 + 1 = 3/4 + 1/sqrt(2)
-
- // Now refine the reciprocal estimate using a Newton-Raphson iteration:
- //
- // x1 = x0 * (2 - x0 * b)
- //
- // This doubles the number of correct binary digits in the approximation
- // with each iteration.
- uint64_t correction64;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
- correction64 = -((rep_t)recip64 * q63b >> 64);
- recip64 = (rep_t)recip64 * correction64 >> 63;
-
- // The reciprocal may have overflowed to zero if the upper half of b is
- // exactly 1.0. This would sabatoge the full-width final stage of the
- // computation that follows, so we adjust the reciprocal down by one bit.
- recip64--;
-
- // We need to perform one more iteration to get us to 112 binary digits;
- // The last iteration needs to happen with extra precision.
- const uint64_t q127blo = bSignificand << 15;
- rep_t correction, reciprocal;
-
- // NOTE: This operation is equivalent to __multi3, which is not implemented
- // in some architechure
- rep_t r64q63, r64q127, r64cH, r64cL, dummy;
- wideMultiply((rep_t)recip64, (rep_t)q63b, &dummy, &r64q63);
- wideMultiply((rep_t)recip64, (rep_t)q127blo, &dummy, &r64q127);
-
- correction = -(r64q63 + (r64q127 >> 64));
-
- uint64_t cHi = correction >> 64;
- uint64_t cLo = correction;
-
- wideMultiply((rep_t)recip64, (rep_t)cHi, &dummy, &r64cH);
- wideMultiply((rep_t)recip64, (rep_t)cLo, &dummy, &r64cL);
-
- reciprocal = r64cH + (r64cL >> 64);
-
- // Adjust the final 128-bit reciprocal estimate downward to ensure that it
- // is strictly smaller than the infinitely precise exact reciprocal. Because
- // the computation of the Newton-Raphson step is truncating at every step,
- // this adjustment is small; most of the work is already done.
- reciprocal -= 2;
-
- // The numerical reciprocal is accurate to within 2^-112, lies in the
- // interval [0.5, 1.0), and is strictly smaller than the true reciprocal
- // of b. Multiplying a by this reciprocal thus gives a numerical q = a/b
- // in Q127 with the following properties:
- //
- // 1. q < a/b
- // 2. q is in the interval [0.5, 2.0)
- // 3. The error in q is bounded away from 2^-113 (actually, we have a
- // couple of bits to spare, but this is all we need).
-
- // We need a 128 x 128 multiply high to compute q, which isn't a basic
- // operation in C, so we need to be a little bit fussy.
- rep_t quotient, quotientLo;
- wideMultiply(aSignificand << 2, reciprocal, "ient, "ientLo);
-
- // Two cases: quotient is in [0.5, 1.0) or quotient is in [1.0, 2.0).
- // In either case, we are going to compute a residual of the form
- //
- // r = a - q*b
- //
- // We know from the construction of q that r satisfies:
- //
- // 0 <= r < ulp(q)*b
- //
- // If r is greater than 1/2 ulp(q)*b, then q rounds up. Otherwise, we
- // already have the correct result. The exact halfway case cannot occur.
- // We also take this time to right shift quotient if it falls in the [1,2)
- // range and adjust the exponent accordingly.
- rep_t residual;
- rep_t qb;
- if (quotient < (implicitBit << 1)) {
- wideMultiply(quotient, bSignificand, &dummy, &qb);
- residual = (aSignificand << 113) - qb;
- quotientExponent--;
- } else {
- quotient >>= 1;
- wideMultiply(quotient, bSignificand, &dummy, &qb);
- residual = (aSignificand << 112) - qb;
- }
+#define NUMBER_OF_HALF_ITERATIONS 4
+#define NUMBER_OF_FULL_ITERATIONS 1
- const int writtenExponent = quotientExponent + exponentBias;
+#include "fp_div_impl.inc"
- if (writtenExponent >= maxExponent) {
- // If we have overflowed the exponent, return infinity.
- return fromRep(infRep | quotientSign);
- } else if (writtenExponent < 1) {
- if (writtenExponent == 0) {
- // Check whether the rounded result is normal.
- const bool round = (residual << 1) > bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Round.
- absResult += round;
- if (absResult & ~significandMask) {
- // The rounded result is normal; return it.
- return fromRep(absResult | quotientSign);
- }
- }
- // Flush denormals to zero. In the future, it would be nice to add
- // code to round them correctly.
- return fromRep(quotientSign);
- } else {
- const bool round = (residual << 1) >= bSignificand;
- // Clear the implicit bit.
- rep_t absResult = quotient & significandMask;
- // Insert the exponent.
- absResult |= (rep_t)writtenExponent << significandBits;
- // Round.
- absResult += round;
- // Insert the sign and return.
- const fp_t result = fromRep(absResult | quotientSign);
- return result;
- }
-}
+COMPILER_RT_ABI fp_t __divtf3(fp_t a, fp_t b) { return __divXf3__(a, b); }
#endif
// Returns: a / b
-COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) {
- const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
- ti_int s_a = a >> bits_in_tword_m1; // s_a = a < 0 ? -1 : 0
- ti_int s_b = b >> bits_in_tword_m1; // s_b = b < 0 ? -1 : 0
- a = (a ^ s_a) - s_a; // negate if s_a == -1
- b = (b ^ s_b) - s_b; // negate if s_b == -1
- s_a ^= s_b; // sign of quotient
- return (__udivmodti4(a, b, (tu_int *)0) ^ s_a) - s_a; // negate if s_a == -1
-}
+#define fixint_t ti_int
+#define fixuint_t tu_int
+#define COMPUTE_UDIV(a, b) __udivmodti4((a), (b), (tu_int *)0)
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI ti_int __divti3(ti_int a, ti_int b) { return __divXi3(a, b); }
#endif // CRT_HAS_128BIT
// Use a forwarding definition and noinline to implement a poor man's alias,
// as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) {
+COMPILER_RT_ABI NOINLINE float __extendhfsf2(src_t a) {
return __extendXfYf2__(a);
}
-COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { return __extendhfsf2(a); }
+COMPILER_RT_ABI float __gnu_h2f_ieee(src_t a) { return __extendhfsf2(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); }
+AEABI_RTABI float __aeabi_h2f(src_t a) { return __extendhfsf2(a); }
#else
COMPILER_RT_ALIAS(__extendhfsf2, __aeabi_h2f)
#endif
--- /dev/null
+//===-- lib/extendhftf2.c - half -> quad conversion ---------------*- C -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
+ defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_HALF
+#define DST_QUAD
+#include "fp_extend_impl.inc"
+
+COMPILER_RT_ABI long double __extendhftf2(_Float16 a) {
+ return __extendXfYf2__(a);
+}
+
+#endif
#define DOUBLE_PRECISION
#include "fp_lib.h"
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; can set the invalid
// flag as a side-effect of computation.
#define SINGLE_PRECISION
#include "fp_lib.h"
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; can set the invalid
// flag as a side-effect of computation.
#define DOUBLE_PRECISION
#include "fp_lib.h"
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; can set the invalid
// flag as a side-effect of computation.
#define SINGLE_PRECISION
#include "fp_lib.h"
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; can set the invalid
// flag as a side-effect of computation.
// seee eeee eeee mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm
// mmmm
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; we'll set the inexact
// flag as a side-effect of this computation.
#include "int_lib.h"
-#ifndef __SOFT_FP__
+#ifndef __SOFTFP__
// Support for systems that have hardware floating-point; we'll set the inexact
// flag as a side-effect of this computation.
// Perform the final rounding. The result may overflow to infinity, but
// that is the correct result in that case.
switch (__fe_getround()) {
- case FE_TONEAREST:
+ case CRT_FE_TONEAREST:
if (roundGuardSticky > 0x4)
result++;
if (roundGuardSticky == 0x4)
result += result & 1;
break;
- case FE_DOWNWARD:
+ case CRT_FE_DOWNWARD:
if (resultSign && roundGuardSticky) result++;
break;
- case FE_UPWARD:
+ case CRT_FE_UPWARD:
if (!resultSign && roundGuardSticky) result++;
break;
- case FE_TOWARDZERO:
+ case CRT_FE_TOWARDZERO:
break;
}
if (roundGuardSticky)
--- /dev/null
+//===-- lib/fp_compare_impl.inc - Floating-point comparison -------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "fp_lib.h"
+
+// GCC uses long (at least for x86_64) as the return type of the comparison
+// functions. We need to ensure that the return value is sign-extended in the
+// same way as GCC expects (since otherwise GCC-generated __builtin_isinf
+// returns true for finite 128-bit floating-point numbers).
+#ifdef __aarch64__
+// AArch64 GCC overrides libgcc_cmp_return to use int instead of long.
+typedef int CMP_RESULT;
+#elif __SIZEOF_POINTER__ == 8 && __SIZEOF_LONG__ == 4
+// LLP64 ABIs use long long instead of long.
+typedef long long CMP_RESULT;
+#else
+// Otherwise the comparison functions return long.
+typedef long CMP_RESULT;
+#endif
+
+#if !defined(__clang__) && defined(__GNUC__)
+// GCC uses a special __libgcc_cmp_return__ mode to define the return type, so
+// check that we are ABI-compatible when compiling the builtins with GCC.
+typedef int GCC_CMP_RESULT __attribute__((__mode__(__libgcc_cmp_return__)));
+_Static_assert(sizeof(GCC_CMP_RESULT) == sizeof(CMP_RESULT),
+ "SOFTFP ABI not compatible with GCC");
+#endif
+
+enum {
+ LE_LESS = -1,
+ LE_EQUAL = 0,
+ LE_GREATER = 1,
+ LE_UNORDERED = 1,
+};
+
+static inline CMP_RESULT __leXf2__(fp_t a, fp_t b) {
+ const srep_t aInt = toRep(a);
+ const srep_t bInt = toRep(b);
+ const rep_t aAbs = aInt & absMask;
+ const rep_t bAbs = bInt & absMask;
+
+ // If either a or b is NaN, they are unordered.
+ if (aAbs > infRep || bAbs > infRep)
+ return LE_UNORDERED;
+
+ // If a and b are both zeros, they are equal.
+ if ((aAbs | bAbs) == 0)
+ return LE_EQUAL;
+
+ // If at least one of a and b is positive, we get the same result comparing
+ // a and b as signed integers as we would with a floating-point compare.
+ if ((aInt & bInt) >= 0) {
+ if (aInt < bInt)
+ return LE_LESS;
+ else if (aInt == bInt)
+ return LE_EQUAL;
+ else
+ return LE_GREATER;
+ } else {
+ // Otherwise, both are negative, so we need to flip the sense of the
+ // comparison to get the correct result. (This assumes a twos- or ones-
+ // complement integer representation; if integers are represented in a
+ // sign-magnitude representation, then this flip is incorrect).
+ if (aInt > bInt)
+ return LE_LESS;
+ else if (aInt == bInt)
+ return LE_EQUAL;
+ else
+ return LE_GREATER;
+ }
+}
+
+enum {
+ GE_LESS = -1,
+ GE_EQUAL = 0,
+ GE_GREATER = 1,
+ GE_UNORDERED = -1 // Note: different from LE_UNORDERED
+};
+
+static inline CMP_RESULT __geXf2__(fp_t a, fp_t b) {
+ const srep_t aInt = toRep(a);
+ const srep_t bInt = toRep(b);
+ const rep_t aAbs = aInt & absMask;
+ const rep_t bAbs = bInt & absMask;
+
+ if (aAbs > infRep || bAbs > infRep)
+ return GE_UNORDERED;
+ if ((aAbs | bAbs) == 0)
+ return GE_EQUAL;
+ if ((aInt & bInt) >= 0) {
+ if (aInt < bInt)
+ return GE_LESS;
+ else if (aInt == bInt)
+ return GE_EQUAL;
+ else
+ return GE_GREATER;
+ } else {
+ if (aInt > bInt)
+ return GE_LESS;
+ else if (aInt == bInt)
+ return GE_EQUAL;
+ else
+ return GE_GREATER;
+ }
+}
+
+static inline CMP_RESULT __unordXf2__(fp_t a, fp_t b) {
+ const rep_t aAbs = toRep(a) & absMask;
+ const rep_t bAbs = toRep(b) & absMask;
+ return aAbs > infRep || bAbs > infRep;
+}
--- /dev/null
+//===-- fp_div_impl.inc - Floating point division -----------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements soft-float division with the IEEE-754 default
+// rounding (to nearest, ties to even).
+//
+//===----------------------------------------------------------------------===//
+
+#include "fp_lib.h"
+
+// The __divXf3__ function implements Newton-Raphson floating point division.
+// It uses 3 iterations for float32, 4 for float64 and 5 for float128,
+// respectively. Due to number of significant bits being roughly doubled
+// every iteration, the two modes are supported: N full-width iterations (as
+// it is done for float32 by default) and (N-1) half-width iteration plus one
+// final full-width iteration. It is expected that half-width integer
+// operations (w.r.t rep_t size) can be performed faster for some hardware but
+// they require error estimations to be computed separately due to larger
+// computational errors caused by truncating intermediate results.
+
+// Half the bit-size of rep_t
+#define HW (typeWidth / 2)
+// rep_t-sized bitmask with lower half of bits set to ones
+#define loMask (REP_C(-1) >> HW)
+
+#if NUMBER_OF_FULL_ITERATIONS < 1
+#error At least one full iteration is required
+#endif
+
+static __inline fp_t __divXf3__(fp_t a, fp_t b) {
+
+ const unsigned int aExponent = toRep(a) >> significandBits & maxExponent;
+ const unsigned int bExponent = toRep(b) >> significandBits & maxExponent;
+ const rep_t quotientSign = (toRep(a) ^ toRep(b)) & signBit;
+
+ rep_t aSignificand = toRep(a) & significandMask;
+ rep_t bSignificand = toRep(b) & significandMask;
+ int scale = 0;
+
+ // Detect if a or b is zero, denormal, infinity, or NaN.
+ if (aExponent - 1U >= maxExponent - 1U ||
+ bExponent - 1U >= maxExponent - 1U) {
+
+ const rep_t aAbs = toRep(a) & absMask;
+ const rep_t bAbs = toRep(b) & absMask;
+
+ // NaN / anything = qNaN
+ if (aAbs > infRep)
+ return fromRep(toRep(a) | quietBit);
+ // anything / NaN = qNaN
+ if (bAbs > infRep)
+ return fromRep(toRep(b) | quietBit);
+
+ if (aAbs == infRep) {
+ // infinity / infinity = NaN
+ if (bAbs == infRep)
+ return fromRep(qnanRep);
+ // infinity / anything else = +/- infinity
+ else
+ return fromRep(aAbs | quotientSign);
+ }
+
+ // anything else / infinity = +/- 0
+ if (bAbs == infRep)
+ return fromRep(quotientSign);
+
+ if (!aAbs) {
+ // zero / zero = NaN
+ if (!bAbs)
+ return fromRep(qnanRep);
+ // zero / anything else = +/- zero
+ else
+ return fromRep(quotientSign);
+ }
+ // anything else / zero = +/- infinity
+ if (!bAbs)
+ return fromRep(infRep | quotientSign);
+
+ // One or both of a or b is denormal. The other (if applicable) is a
+ // normal number. Renormalize one or both of a and b, and set scale to
+ // include the necessary exponent adjustment.
+ if (aAbs < implicitBit)
+ scale += normalize(&aSignificand);
+ if (bAbs < implicitBit)
+ scale -= normalize(&bSignificand);
+ }
+
+ // Set the implicit significand bit. If we fell through from the
+ // denormal path it was already set by normalize( ), but setting it twice
+ // won't hurt anything.
+ aSignificand |= implicitBit;
+ bSignificand |= implicitBit;
+
+ int writtenExponent = (aExponent - bExponent + scale) + exponentBias;
+
+ const rep_t b_UQ1 = bSignificand << (typeWidth - significandBits - 1);
+
+ // Align the significand of b as a UQ1.(n-1) fixed-point number in the range
+ // [1.0, 2.0) and get a UQ0.n approximate reciprocal using a small minimax
+ // polynomial approximation: x0 = 3/4 + 1/sqrt(2) - b/2.
+ // The max error for this approximation is achieved at endpoints, so
+ // abs(x0(b) - 1/b) <= abs(x0(1) - 1/1) = 3/4 - 1/sqrt(2) = 0.04289...,
+ // which is about 4.5 bits.
+ // The initial approximation is between x0(1.0) = 0.9571... and x0(2.0) = 0.4571...
+
+ // Then, refine the reciprocal estimate using a quadratically converging
+ // Newton-Raphson iteration:
+ // x_{n+1} = x_n * (2 - x_n * b)
+ //
+ // Let b be the original divisor considered "in infinite precision" and
+ // obtained from IEEE754 representation of function argument (with the
+ // implicit bit set). Corresponds to rep_t-sized b_UQ1 represented in
+ // UQ1.(W-1).
+ //
+ // Let b_hw be an infinitely precise number obtained from the highest (HW-1)
+ // bits of divisor significand (with the implicit bit set). Corresponds to
+ // half_rep_t-sized b_UQ1_hw represented in UQ1.(HW-1) that is a **truncated**
+ // version of b_UQ1.
+ //
+ // Let e_n := x_n - 1/b_hw
+ // E_n := x_n - 1/b
+ // abs(E_n) <= abs(e_n) + (1/b_hw - 1/b)
+ // = abs(e_n) + (b - b_hw) / (b*b_hw)
+ // <= abs(e_n) + 2 * 2^-HW
+
+ // rep_t-sized iterations may be slower than the corresponding half-width
+ // variant depending on the handware and whether single/double/quad precision
+ // is selected.
+ // NB: Using half-width iterations increases computation errors due to
+ // rounding, so error estimations have to be computed taking the selected
+ // mode into account!
+#if NUMBER_OF_HALF_ITERATIONS > 0
+ // Starting with (n-1) half-width iterations
+ const half_rep_t b_UQ1_hw = bSignificand >> (significandBits + 1 - HW);
+
+ // C is (3/4 + 1/sqrt(2)) - 1 truncated to W0 fractional bits as UQ0.HW
+ // with W0 being either 16 or 32 and W0 <= HW.
+ // That is, C is the aforementioned 3/4 + 1/sqrt(2) constant (from which
+ // b/2 is subtracted to obtain x0) wrapped to [0, 1) range.
+#if defined(SINGLE_PRECISION)
+ // Use 16-bit initial estimation in case we are using half-width iterations
+ // for float32 division. This is expected to be useful for some 16-bit
+ // targets. Not used by default as it requires performing more work during
+ // rounding and would hardly help on regular 32- or 64-bit targets.
+ const half_rep_t C_hw = HALF_REP_C(0x7504);
+#else
+ // HW is at least 32. Shifting into the highest bits if needed.
+ const half_rep_t C_hw = HALF_REP_C(0x7504F333) << (HW - 32);
+#endif
+
+ // b >= 1, thus an upper bound for 3/4 + 1/sqrt(2) - b/2 is about 0.9572,
+ // so x0 fits to UQ0.HW without wrapping.
+ half_rep_t x_UQ0_hw = C_hw - (b_UQ1_hw /* exact b_hw/2 as UQ0.HW */);
+ // An e_0 error is comprised of errors due to
+ // * x0 being an inherently imprecise first approximation of 1/b_hw
+ // * C_hw being some (irrational) number **truncated** to W0 bits
+ // Please note that e_0 is calculated against the infinitely precise
+ // reciprocal of b_hw (that is, **truncated** version of b).
+ //
+ // e_0 <= 3/4 - 1/sqrt(2) + 2^-W0
+
+ // By construction, 1 <= b < 2
+ // f(x) = x * (2 - b*x) = 2*x - b*x^2
+ // f'(x) = 2 * (1 - b*x)
+ //
+ // On the [0, 1] interval, f(0) = 0,
+ // then it increses until f(1/b) = 1 / b, maximum on (0, 1),
+ // then it decreses to f(1) = 2 - b
+ //
+ // Let g(x) = x - f(x) = b*x^2 - x.
+ // On (0, 1/b), g(x) < 0 <=> f(x) > x
+ // On (1/b, 1], g(x) > 0 <=> f(x) < x
+ //
+ // For half-width iterations, b_hw is used instead of b.
+ REPEAT_N_TIMES(NUMBER_OF_HALF_ITERATIONS, {
+ // corr_UQ1_hw can be **larger** than 2 - b_hw*x by at most 1*Ulp
+ // of corr_UQ1_hw.
+ // "0.0 - (...)" is equivalent to "2.0 - (...)" in UQ1.(HW-1).
+ // On the other hand, corr_UQ1_hw should not overflow from 2.0 to 0.0 provided
+ // no overflow occurred earlier: ((rep_t)x_UQ0_hw * b_UQ1_hw >> HW) is
+ // expected to be strictly positive because b_UQ1_hw has its highest bit set
+ // and x_UQ0_hw should be rather large (it converges to 1/2 < 1/b_hw <= 1).
+ half_rep_t corr_UQ1_hw = 0 - ((rep_t)x_UQ0_hw * b_UQ1_hw >> HW);
+
+ // Now, we should multiply UQ0.HW and UQ1.(HW-1) numbers, naturally
+ // obtaining an UQ1.(HW-1) number and proving its highest bit could be
+ // considered to be 0 to be able to represent it in UQ0.HW.
+ // From the above analysis of f(x), if corr_UQ1_hw would be represented
+ // without any intermediate loss of precision (that is, in twice_rep_t)
+ // x_UQ0_hw could be at most [1.]000... if b_hw is exactly 1.0 and strictly
+ // less otherwise. On the other hand, to obtain [1.]000..., one have to pass
+ // 1/b_hw == 1.0 to f(x), so this cannot occur at all without overflow (due
+ // to 1.0 being not representable as UQ0.HW).
+ // The fact corr_UQ1_hw was virtually round up (due to result of
+ // multiplication being **first** truncated, then negated - to improve
+ // error estimations) can increase x_UQ0_hw by up to 2*Ulp of x_UQ0_hw.
+ x_UQ0_hw = (rep_t)x_UQ0_hw * corr_UQ1_hw >> (HW - 1);
+ // Now, either no overflow occurred or x_UQ0_hw is 0 or 1 in its half_rep_t
+ // representation. In the latter case, x_UQ0_hw will be either 0 or 1 after
+ // any number of iterations, so just subtract 2 from the reciprocal
+ // approximation after last iteration.
+
+ // In infinite precision, with 0 <= eps1, eps2 <= U = 2^-HW:
+ // corr_UQ1_hw = 2 - (1/b_hw + e_n) * b_hw + 2*eps1
+ // = 1 - e_n * b_hw + 2*eps1
+ // x_UQ0_hw = (1/b_hw + e_n) * (1 - e_n*b_hw + 2*eps1) - eps2
+ // = 1/b_hw - e_n + 2*eps1/b_hw + e_n - e_n^2*b_hw + 2*e_n*eps1 - eps2
+ // = 1/b_hw + 2*eps1/b_hw - e_n^2*b_hw + 2*e_n*eps1 - eps2
+ // e_{n+1} = -e_n^2*b_hw + 2*eps1/b_hw + 2*e_n*eps1 - eps2
+ // = 2*e_n*eps1 - (e_n^2*b_hw + eps2) + 2*eps1/b_hw
+ // \------ >0 -------/ \-- >0 ---/
+ // abs(e_{n+1}) <= 2*abs(e_n)*U + max(2*e_n^2 + U, 2 * U)
+ })
+ // For initial half-width iterations, U = 2^-HW
+ // Let abs(e_n) <= u_n * U,
+ // then abs(e_{n+1}) <= 2 * u_n * U^2 + max(2 * u_n^2 * U^2 + U, 2 * U)
+ // u_{n+1} <= 2 * u_n * U + max(2 * u_n^2 * U + 1, 2)
+
+ // Account for possible overflow (see above). For an overflow to occur for the
+ // first time, for "ideal" corr_UQ1_hw (that is, without intermediate
+ // truncation), the result of x_UQ0_hw * corr_UQ1_hw should be either maximum
+ // value representable in UQ0.HW or less by 1. This means that 1/b_hw have to
+ // be not below that value (see g(x) above), so it is safe to decrement just
+ // once after the final iteration. On the other hand, an effective value of
+ // divisor changes after this point (from b_hw to b), so adjust here.
+ x_UQ0_hw -= 1U;
+ rep_t x_UQ0 = (rep_t)x_UQ0_hw << HW;
+ x_UQ0 -= 1U;
+
+#else
+ // C is (3/4 + 1/sqrt(2)) - 1 truncated to 32 fractional bits as UQ0.n
+ const rep_t C = REP_C(0x7504F333) << (typeWidth - 32);
+ rep_t x_UQ0 = C - b_UQ1;
+ // E_0 <= 3/4 - 1/sqrt(2) + 2 * 2^-32
+#endif
+
+ // Error estimations for full-precision iterations are calculated just
+ // as above, but with U := 2^-W and taking extra decrementing into account.
+ // We need at least one such iteration.
+
+#ifdef USE_NATIVE_FULL_ITERATIONS
+ REPEAT_N_TIMES(NUMBER_OF_FULL_ITERATIONS, {
+ rep_t corr_UQ1 = 0 - ((twice_rep_t)x_UQ0 * b_UQ1 >> typeWidth);
+ x_UQ0 = (twice_rep_t)x_UQ0 * corr_UQ1 >> (typeWidth - 1);
+ })
+#else
+#if NUMBER_OF_FULL_ITERATIONS != 1
+#error Only a single emulated full iteration is supported
+#endif
+#if !(NUMBER_OF_HALF_ITERATIONS > 0)
+ // Cannot normally reach here: only one full-width iteration is requested and
+ // the total number of iterations should be at least 3 even for float32.
+#error Check NUMBER_OF_HALF_ITERATIONS, NUMBER_OF_FULL_ITERATIONS and USE_NATIVE_FULL_ITERATIONS.
+#endif
+ // Simulating operations on a twice_rep_t to perform a single final full-width
+ // iteration. Using ad-hoc multiplication implementations to take advantage
+ // of particular structure of operands.
+ rep_t blo = b_UQ1 & loMask;
+ // x_UQ0 = x_UQ0_hw * 2^HW - 1
+ // x_UQ0 * b_UQ1 = (x_UQ0_hw * 2^HW) * (b_UQ1_hw * 2^HW + blo) - b_UQ1
+ //
+ // <--- higher half ---><--- lower half --->
+ // [x_UQ0_hw * b_UQ1_hw]
+ // + [ x_UQ0_hw * blo ]
+ // - [ b_UQ1 ]
+ // = [ result ][.... discarded ...]
+ rep_t corr_UQ1 = 0U - ( (rep_t)x_UQ0_hw * b_UQ1_hw
+ + ((rep_t)x_UQ0_hw * blo >> HW)
+ - REP_C(1)); // account for *possible* carry
+ rep_t lo_corr = corr_UQ1 & loMask;
+ rep_t hi_corr = corr_UQ1 >> HW;
+ // x_UQ0 * corr_UQ1 = (x_UQ0_hw * 2^HW) * (hi_corr * 2^HW + lo_corr) - corr_UQ1
+ x_UQ0 = ((rep_t)x_UQ0_hw * hi_corr << 1)
+ + ((rep_t)x_UQ0_hw * lo_corr >> (HW - 1))
+ - REP_C(2); // 1 to account for the highest bit of corr_UQ1 can be 1
+ // 1 to account for possible carry
+ // Just like the case of half-width iterations but with possibility
+ // of overflowing by one extra Ulp of x_UQ0.
+ x_UQ0 -= 1U;
+ // ... and then traditional fixup by 2 should work
+
+ // On error estimation:
+ // abs(E_{N-1}) <= (u_{N-1} + 2 /* due to conversion e_n -> E_n */) * 2^-HW
+ // + (2^-HW + 2^-W))
+ // abs(E_{N-1}) <= (u_{N-1} + 3.01) * 2^-HW
+
+ // Then like for the half-width iterations:
+ // With 0 <= eps1, eps2 < 2^-W
+ // E_N = 4 * E_{N-1} * eps1 - (E_{N-1}^2 * b + 4 * eps2) + 4 * eps1 / b
+ // abs(E_N) <= 2^-W * [ 4 * abs(E_{N-1}) + max(2 * abs(E_{N-1})^2 * 2^W + 4, 8)) ]
+ // abs(E_N) <= 2^-W * [ 4 * (u_{N-1} + 3.01) * 2^-HW + max(4 + 2 * (u_{N-1} + 3.01)^2, 8) ]
+#endif
+
+ // Finally, account for possible overflow, as explained above.
+ x_UQ0 -= 2U;
+
+ // u_n for different precisions (with N-1 half-width iterations):
+ // W0 is the precision of C
+ // u_0 = (3/4 - 1/sqrt(2) + 2^-W0) * 2^HW
+
+ // Estimated with bc:
+ // define half1(un) { return 2.0 * (un + un^2) / 2.0^hw + 1.0; }
+ // define half2(un) { return 2.0 * un / 2.0^hw + 2.0; }
+ // define full1(un) { return 4.0 * (un + 3.01) / 2.0^hw + 2.0 * (un + 3.01)^2 + 4.0; }
+ // define full2(un) { return 4.0 * (un + 3.01) / 2.0^hw + 8.0; }
+
+ // | f32 (0 + 3) | f32 (2 + 1) | f64 (3 + 1) | f128 (4 + 1)
+ // u_0 | < 184224974 | < 2812.1 | < 184224974 | < 791240234244348797
+ // u_1 | < 15804007 | < 242.7 | < 15804007 | < 67877681371350440
+ // u_2 | < 116308 | < 2.81 | < 116308 | < 499533100252317
+ // u_3 | < 7.31 | | < 7.31 | < 27054456580
+ // u_4 | | | | < 80.4
+ // Final (U_N) | same as u_3 | < 72 | < 218 | < 13920
+
+ // Add 2 to U_N due to final decrement.
+
+#if defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 2 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(74)
+#elif defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 0 && NUMBER_OF_FULL_ITERATIONS == 3
+#define RECIPROCAL_PRECISION REP_C(10)
+#elif defined(DOUBLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 3 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(220)
+#elif defined(QUAD_PRECISION) && NUMBER_OF_HALF_ITERATIONS == 4 && NUMBER_OF_FULL_ITERATIONS == 1
+#define RECIPROCAL_PRECISION REP_C(13922)
+#else
+#error Invalid number of iterations
+#endif
+
+ // Suppose 1/b - P * 2^-W < x < 1/b + P * 2^-W
+ x_UQ0 -= RECIPROCAL_PRECISION;
+ // Now 1/b - (2*P) * 2^-W < x < 1/b
+ // FIXME Is x_UQ0 still >= 0.5?
+
+ rep_t quotient_UQ1, dummy;
+ wideMultiply(x_UQ0, aSignificand << 1, "ient_UQ1, &dummy);
+ // Now, a/b - 4*P * 2^-W < q < a/b for q=<quotient_UQ1:dummy> in UQ1.(SB+1+W).
+
+ // quotient_UQ1 is in [0.5, 2.0) as UQ1.(SB+1),
+ // adjust it to be in [1.0, 2.0) as UQ1.SB.
+ rep_t residualLo;
+ if (quotient_UQ1 < (implicitBit << 1)) {
+ // Highest bit is 0, so just reinterpret quotient_UQ1 as UQ1.SB,
+ // effectively doubling its value as well as its error estimation.
+ residualLo = (aSignificand << (significandBits + 1)) - quotient_UQ1 * bSignificand;
+ writtenExponent -= 1;
+ aSignificand <<= 1;
+ } else {
+ // Highest bit is 1 (the UQ1.(SB+1) value is in [1, 2)), convert it
+ // to UQ1.SB by right shifting by 1. Least significant bit is omitted.
+ quotient_UQ1 >>= 1;
+ residualLo = (aSignificand << significandBits) - quotient_UQ1 * bSignificand;
+ }
+ // NB: residualLo is calculated above for the normal result case.
+ // It is re-computed on denormal path that is expected to be not so
+ // performance-sensitive.
+
+ // Now, q cannot be greater than a/b and can differ by at most 8*P * 2^-W + 2^-SB
+ // Each NextAfter() increments the floating point value by at least 2^-SB
+ // (more, if exponent was incremented).
+ // Different cases (<---> is of 2^-SB length, * = a/b that is shown as a midpoint):
+ // q
+ // | | * | | | | |
+ // <---> 2^t
+ // | | | | | * | |
+ // q
+ // To require at most one NextAfter(), an error should be less than 1.5 * 2^-SB.
+ // (8*P) * 2^-W + 2^-SB < 1.5 * 2^-SB
+ // (8*P) * 2^-W < 0.5 * 2^-SB
+ // P < 2^(W-4-SB)
+ // Generally, for at most R NextAfter() to be enough,
+ // P < (2*R - 1) * 2^(W-4-SB)
+ // For f32 (0+3): 10 < 32 (OK)
+ // For f32 (2+1): 32 < 74 < 32 * 3, so two NextAfter() are required
+ // For f64: 220 < 256 (OK)
+ // For f128: 4096 * 3 < 13922 < 4096 * 5 (three NextAfter() are required)
+
+ // If we have overflowed the exponent, return infinity
+ if (writtenExponent >= maxExponent)
+ return fromRep(infRep | quotientSign);
+
+ // Now, quotient_UQ1_SB <= the correctly-rounded result
+ // and may need taking NextAfter() up to 3 times (see error estimates above)
+ // r = a - b * q
+ rep_t absResult;
+ if (writtenExponent > 0) {
+ // Clear the implicit bit
+ absResult = quotient_UQ1 & significandMask;
+ // Insert the exponent
+ absResult |= (rep_t)writtenExponent << significandBits;
+ residualLo <<= 1;
+ } else {
+ // Prevent shift amount from being negative
+ if (significandBits + writtenExponent < 0)
+ return fromRep(quotientSign);
+
+ absResult = quotient_UQ1 >> (-writtenExponent + 1);
+
+ // multiplied by two to prevent shift amount to be negative
+ residualLo = (aSignificand << (significandBits + writtenExponent)) - (absResult * bSignificand << 1);
+ }
+
+ // Round
+ residualLo += absResult & 1; // tie to even
+ // The above line conditionally turns the below LT comparison into LTE
+ absResult += residualLo > bSignificand;
+#if defined(QUAD_PRECISION) || (defined(SINGLE_PRECISION) && NUMBER_OF_HALF_ITERATIONS > 0)
+ // Do not round Infinity to NaN
+ absResult += absResult < infRep && residualLo > (2 + 1) * bSignificand;
+#endif
+#if defined(QUAD_PRECISION)
+ absResult += absResult < infRep && residualLo > (4 + 1) * bSignificand;
+#endif
+ return fromRep(absResult | quotientSign);
+}
}
#elif defined SRC_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 src_t;
+#else
typedef uint16_t src_t;
+#endif
typedef uint16_t src_rep_t;
#define SRC_REP_C UINT16_C
static const int srcSigBits = 10;
#if defined SINGLE_PRECISION
+typedef uint16_t half_rep_t;
typedef uint32_t rep_t;
+typedef uint64_t twice_rep_t;
typedef int32_t srep_t;
typedef float fp_t;
+#define HALF_REP_C UINT16_C
#define REP_C UINT32_C
#define significandBits 23
#elif defined DOUBLE_PRECISION
+typedef uint32_t half_rep_t;
typedef uint64_t rep_t;
typedef int64_t srep_t;
typedef double fp_t;
+#define HALF_REP_C UINT32_C
#define REP_C UINT64_C
#define significandBits 52
#elif defined QUAD_PRECISION
#if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__)
#define CRT_LDBL_128BIT
+typedef uint64_t half_rep_t;
typedef __uint128_t rep_t;
typedef __int128_t srep_t;
typedef long double fp_t;
+#define HALF_REP_C UINT64_C
#define REP_C (__uint128_t)
// Note: Since there is no explicit way to tell compiler the constant is a
// 128-bit integer, we let the constant be casted to 128-bit integer
return exp - exponentBias - shift; // Unbias exponent
}
}
+
+// Avoid using scalbn from libm. Unlike libc/libm scalbn, this function never
+// sets errno on underflow/overflow.
+static __inline fp_t __compiler_rt_scalbnX(fp_t x, int y) {
+ const rep_t rep = toRep(x);
+ int exp = (rep & exponentMask) >> significandBits;
+
+ if (x == 0.0 || exp == maxExponent)
+ return x; // +/- 0.0, NaN, or inf: return x
+
+ // Normalize subnormal input.
+ rep_t sig = rep & significandMask;
+ if (exp == 0) {
+ exp += normalize(&sig);
+ sig &= ~implicitBit; // clear the implicit bit again
+ }
+
+ if (__builtin_sadd_overflow(exp, y, &exp)) {
+ // Saturate the exponent, which will guarantee an underflow/overflow below.
+ exp = (y >= 0) ? INT_MAX : INT_MIN;
+ }
+
+ // Return this value: [+/-] 1.sig * 2 ** (exp - exponentBias).
+ const rep_t sign = rep & signBit;
+ if (exp >= maxExponent) {
+ // Overflow, which could produce infinity or the largest-magnitude value,
+ // depending on the rounding mode.
+ return fromRep(sign | ((rep_t)(maxExponent - 1) << significandBits)) * 2.0f;
+ } else if (exp <= 0) {
+ // Subnormal or underflow. Use floating-point multiply to handle truncation
+ // correctly.
+ fp_t tmp = fromRep(sign | (REP_C(1) << significandBits) | sig);
+ exp += exponentBias - 1;
+ if (exp < 1)
+ exp = 1;
+ tmp *= fromRep((rep_t)exp << significandBits);
+ return tmp;
+ } else
+ return fromRep(sign | ((rep_t)exp << significandBits) | sig);
+}
+
+// Avoid using fmax from libm.
+static __inline fp_t __compiler_rt_fmaxX(fp_t x, fp_t y) {
+ // If either argument is NaN, return the other argument. If both are NaN,
+ // arbitrarily return the second one. Otherwise, if both arguments are +/-0,
+ // arbitrarily return the first one.
+ return (crt_isnan(x) || x < y) ? y : x;
+}
+
#endif
#if defined(SINGLE_PRECISION)
+
static __inline fp_t __compiler_rt_logbf(fp_t x) {
return __compiler_rt_logbX(x);
}
+static __inline fp_t __compiler_rt_scalbnf(fp_t x, int y) {
+ return __compiler_rt_scalbnX(x, y);
+}
+static __inline fp_t __compiler_rt_fmaxf(fp_t x, fp_t y) {
+#if defined(__aarch64__)
+ // Use __builtin_fmaxf which turns into an fmaxnm instruction on AArch64.
+ return __builtin_fmaxf(x, y);
+#else
+ // __builtin_fmaxf frequently turns into a libm call, so inline the function.
+ return __compiler_rt_fmaxX(x, y);
+#endif
+}
+
#elif defined(DOUBLE_PRECISION)
+
static __inline fp_t __compiler_rt_logb(fp_t x) {
return __compiler_rt_logbX(x);
}
+static __inline fp_t __compiler_rt_scalbn(fp_t x, int y) {
+ return __compiler_rt_scalbnX(x, y);
+}
+static __inline fp_t __compiler_rt_fmax(fp_t x, fp_t y) {
+#if defined(__aarch64__)
+ // Use __builtin_fmax which turns into an fmaxnm instruction on AArch64.
+ return __builtin_fmax(x, y);
+#else
+ // __builtin_fmax frequently turns into a libm call, so inline the function.
+ return __compiler_rt_fmaxX(x, y);
+#endif
+}
+
#elif defined(QUAD_PRECISION)
+
#if defined(CRT_LDBL_128BIT)
static __inline fp_t __compiler_rt_logbl(fp_t x) {
return __compiler_rt_logbX(x);
}
+static __inline fp_t __compiler_rt_scalbnl(fp_t x, int y) {
+ return __compiler_rt_scalbnX(x, y);
+}
+static __inline fp_t __compiler_rt_fmaxl(fp_t x, fp_t y) {
+ return __compiler_rt_fmaxX(x, y);
+}
#else
// The generic implementation only works for ieee754 floating point. For other
// floating point types, continue to rely on the libm implementation for now.
static __inline long double __compiler_rt_logbl(long double x) {
return crt_logbl(x);
}
-#endif
-#endif
+static __inline long double __compiler_rt_scalbnl(long double x, int y) {
+ return crt_scalbnl(x, y);
+}
+static __inline long double __compiler_rt_fmaxl(long double x, long double y) {
+ return crt_fmaxl(x, y);
+}
+#endif // CRT_LDBL_128BIT
+
+#endif // *_PRECISION
#endif // FP_LIB_HEADER
#include "fp_mode.h"
// IEEE-754 default rounding (to nearest, ties to even).
-FE_ROUND_MODE __fe_getround() {
- return FE_TONEAREST;
-}
+CRT_FE_ROUND_MODE __fe_getround() { return CRT_FE_TONEAREST; }
int __fe_raise_inexact() {
return 0;
#define FP_MODE
typedef enum {
- FE_TONEAREST,
- FE_DOWNWARD,
- FE_UPWARD,
- FE_TOWARDZERO
-} FE_ROUND_MODE;
+ CRT_FE_TONEAREST,
+ CRT_FE_DOWNWARD,
+ CRT_FE_UPWARD,
+ CRT_FE_TOWARDZERO
+} CRT_FE_ROUND_MODE;
-FE_ROUND_MODE __fe_getround(void);
+CRT_FE_ROUND_MODE __fe_getround(void);
int __fe_raise_inexact(void);
#endif // FP_MODE_H
static const int dstSigBits = 23;
#elif defined DST_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 dst_t;
+#else
typedef uint16_t dst_t;
+#endif
typedef uint16_t dst_rep_t;
#define DST_REP_C UINT16_C
static const int dstSigBits = 10;
//===----------------------------------------------------------------------===//
#include "int_lib.h"
+#include <stddef.h>
#include <unwind.h>
#if defined(__arm__) && !defined(__ARM_DWARF_EH__) && \
#include "unwind-ehabi-helpers.h"
#endif
+#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)
+#include <windows.h>
+#include <winnt.h>
+
+EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, void *, PCONTEXT,
+ PDISPATCHER_CONTEXT,
+ _Unwind_Personality_Fn);
+#endif
+
// Pointer encodings documented at:
// http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html
#define DW_EH_PE_indirect 0x80 // gcc extension
// read a uleb128 encoded value and advance pointer
-static uintptr_t readULEB128(const uint8_t **data) {
- uintptr_t result = 0;
- uintptr_t shift = 0;
+static size_t readULEB128(const uint8_t **data) {
+ size_t result = 0;
+ size_t shift = 0;
unsigned char byte;
const uint8_t *p = *data;
do {
COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
_Unwind_State state, struct _Unwind_Exception *exceptionObject,
struct _Unwind_Context *context)
+#elif defined(__SEH__)
+static _Unwind_Reason_Code __gcc_personality_imp(
+ int version, _Unwind_Action actions, uint64_t exceptionClass,
+ struct _Unwind_Exception *exceptionObject, struct _Unwind_Context *context)
#else
COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
int version, _Unwind_Action actions, uint64_t exceptionClass,
const uint8_t *p = callSiteTableStart;
while (p < callSiteTableEnd) {
uintptr_t start = readEncodedPointer(&p, callSiteEncoding);
- uintptr_t length = readEncodedPointer(&p, callSiteEncoding);
- uintptr_t landingPad = readEncodedPointer(&p, callSiteEncoding);
+ size_t length = readEncodedPointer(&p, callSiteEncoding);
+ size_t landingPad = readEncodedPointer(&p, callSiteEncoding);
readULEB128(&p); // action value not used for C code
if (landingPad == 0)
continue; // no landing pad for this entry
// No landing pad found, continue unwinding.
return continueUnwind(exceptionObject, context);
}
+
+#if defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)
+COMPILER_RT_ABI EXCEPTION_DISPOSITION
+__gcc_personality_seh0(PEXCEPTION_RECORD ms_exc, void *this_frame,
+ PCONTEXT ms_orig_context, PDISPATCHER_CONTEXT ms_disp) {
+ return _GCC_specific_handler(ms_exc, this_frame, ms_orig_context, ms_disp,
+ __gcc_personality_imp);
+}
+#endif
#define X87_TOWARDZERO 0x0c00
#define X87_RMODE_MASK (X87_TONEAREST | X87_UPWARD | X87_DOWNWARD | X87_TOWARDZERO)
-FE_ROUND_MODE __fe_getround() {
+CRT_FE_ROUND_MODE __fe_getround() {
// Assume that the rounding mode state for the fpu agrees with the SSE unit.
unsigned short cw;
__asm__ __volatile__ ("fnstcw %0" : "=m" (cw));
switch (cw & X87_RMODE_MASK) {
case X87_TONEAREST:
- return FE_TONEAREST;
+ return CRT_FE_TONEAREST;
case X87_DOWNWARD:
- return FE_DOWNWARD;
+ return CRT_FE_DOWNWARD;
case X87_UPWARD:
- return FE_UPWARD;
+ return CRT_FE_UPWARD;
case X87_TOWARDZERO:
- return FE_TOWARDZERO;
+ return CRT_FE_TOWARDZERO;
}
- return FE_TONEAREST;
+ return CRT_FE_TONEAREST;
}
int __fe_raise_inexact() {
}
return r;
}
+
+#ifdef COMPUTE_UDIV
+static __inline fixint_t __divXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1;
+ fixint_t s_a = a >> N; // s_a = a < 0 ? -1 : 0
+ fixint_t s_b = b >> N; // s_b = b < 0 ? -1 : 0
+ fixuint_t a_u = (fixuint_t)(a ^ s_a) + (-s_a); // negate if s_a == -1
+ fixuint_t b_u = (fixuint_t)(b ^ s_b) + (-s_b); // negate if s_b == -1
+ s_a ^= s_b; // sign of quotient
+ return (COMPUTE_UDIV(a_u, b_u) ^ s_a) + (-s_a); // negate if s_a == -1
+}
+#endif // COMPUTE_UDIV
+
+#ifdef ASSIGN_UMOD
+static __inline fixint_t __modXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT) - 1;
+ fixint_t s = b >> N; // s = b < 0 ? -1 : 0
+ fixuint_t b_u = (fixuint_t)(b ^ s) + (-s); // negate if s == -1
+ s = a >> N; // s = a < 0 ? -1 : 0
+ fixuint_t a_u = (fixuint_t)(a ^ s) + (-s); // negate if s == -1
+ fixuint_t res;
+ ASSIGN_UMOD(res, a_u, b_u);
+ return (res ^ s) + (-s); // negate if s == -1
+}
+#endif // ASSIGN_UMOD
#error Unsupported target
#endif
-#if defined(__NetBSD__) && (defined(_KERNEL) || defined(_STANDALONE))
+#if (defined(__FreeBSD__) || defined(__NetBSD__)) && \
+ (defined(_KERNEL) || defined(_STANDALONE))
//
// Kernel and boot environment can't use normal headers,
// so use the equivalent system headers.
+// NB: FreeBSD (and OpenBSD) deprecate machine/limits.h in
+// favour of sys/limits.h, so prefer the former, but fall
+// back on the latter if not available since NetBSD only has
+// the latter.
//
+#if defined(__has_include) && __has_include(<sys/limits.h>)
+#include <sys/limits.h>
+#else
#include <machine/limits.h>
+#endif
#include <sys/stdint.h>
#include <sys/types.h>
#else
#endif
#define __builtin_clzl __builtin_clzll
+
+bool __inline __builtin_sadd_overflow(int x, int y, int *result) {
+ if ((x < 0) != (y < 0)) {
+ *result = x + y;
+ return false;
+ }
+ int tmp = (unsigned int)x + (unsigned int)y;
+ if ((tmp < 0) != (x < 0))
+ return true;
+ *result = tmp;
+ return false;
+}
+
#endif // defined(_MSC_VER) && !defined(__clang__)
#endif // INT_LIB_H
#endif
#if defined(_MSC_VER) && !defined(__clang__)
-#define crt_fmax(x, y) __max((x), (y))
-#define crt_fmaxf(x, y) __max((x), (y))
#define crt_fmaxl(x, y) __max((x), (y))
#else
-#define crt_fmax(x, y) __builtin_fmax((x), (y))
-#define crt_fmaxf(x, y) __builtin_fmaxf((x), (y))
#define crt_fmaxl(x, y) __builtin_fmaxl((x), (y))
#endif
#endif
#if defined(_MSC_VER) && !defined(__clang__)
-#define crt_scalbn(x, y) scalbn((x), (y))
-#define crt_scalbnf(x, y) scalbnf((x), (y))
#define crt_scalbnl(x, y) scalbnl((x), (y))
#else
-#define crt_scalbn(x, y) __builtin_scalbn((x), (y))
-#define crt_scalbnf(x, y) __builtin_scalbnf((x), (y))
#define crt_scalbnl(x, y) __builtin_scalbnl((x), (y))
#endif
--- /dev/null
+//===-- int_mulo_impl.inc - Implement __mulo[sdt]i4 ---------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper used by __mulosi4, __mulodi4 and __muloti4.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+// Returns: a * b
+
+// Effects: sets *overflow to 1 if a * b overflows
+
+static __inline fixint_t __muloXi4(fixint_t a, fixint_t b, int *overflow) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
+ const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MAX = ~MIN;
+ *overflow = 0;
+ fixint_t result = a * b;
+ if (a == MIN) {
+ if (b != 0 && b != 1)
+ *overflow = 1;
+ return result;
+ }
+ if (b == MIN) {
+ if (a != 0 && a != 1)
+ *overflow = 1;
+ return result;
+ }
+ fixint_t sa = a >> (N - 1);
+ fixint_t abs_a = (a ^ sa) - sa;
+ fixint_t sb = b >> (N - 1);
+ fixint_t abs_b = (b ^ sb) - sb;
+ if (abs_a < 2 || abs_b < 2)
+ return result;
+ if (sa == sb) {
+ if (abs_a > MAX / abs_b)
+ *overflow = 1;
+ } else {
+ if (abs_a > MIN / -abs_b)
+ *overflow = 1;
+ }
+ return result;
+}
--- /dev/null
+//===-- int_mulv_impl.inc - Implement __mulv[sdt]i3 ---------------*- C -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper used by __mulvsi3, __mulvdi3 and __mulvti3.
+//
+//===----------------------------------------------------------------------===//
+
+#include "int_lib.h"
+
+// Returns: a * b
+
+// Effects: aborts if a * b overflows
+
+static __inline fixint_t __mulvXi3(fixint_t a, fixint_t b) {
+ const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
+ const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MAX = ~MIN;
+ if (a == MIN) {
+ if (b == 0 || b == 1)
+ return a * b;
+ compilerrt_abort();
+ }
+ if (b == MIN) {
+ if (a == 0 || a == 1)
+ return a * b;
+ compilerrt_abort();
+ }
+ fixint_t sa = a >> (N - 1);
+ fixint_t abs_a = (a ^ sa) - sa;
+ fixint_t sb = b >> (N - 1);
+ fixint_t abs_b = (b ^ sb) - sb;
+ if (abs_a < 2 || abs_b < 2)
+ return a * b;
+ if (sa == sb) {
+ if (abs_a > MAX / abs_b)
+ compilerrt_abort();
+ } else {
+ if (abs_a > MIN / -abs_b)
+ compilerrt_abort();
+ }
+ return a * b;
+}
#endif // CRT_HAS_128BIT
+// FreeBSD's boot environment does not support using floating-point and poisons
+// the float and double keywords.
+#if defined(__FreeBSD__) && defined(_STANDALONE)
+#define CRT_HAS_FLOATING_POINT 0
+#else
+#define CRT_HAS_FLOATING_POINT 1
+#endif
+
+#if CRT_HAS_FLOATING_POINT
typedef union {
su_int u;
float f;
udwords u;
double f;
} double_bits;
+#endif
typedef struct {
#if _YUGA_LITTLE_ENDIAN
#define HAS_80_BIT_LONG_DOUBLE 0
#endif
+#if CRT_HAS_FLOATING_POINT
typedef union {
uqwords u;
long double f;
#define COMPLEX_REAL(x) (x).real
#define COMPLEX_IMAGINARY(x) (x).imaginary
#endif
+#endif
#endif // INT_TYPES_H
NORETURN extern void __assert_rtn(const char *func, const char *file, int line,
const char *message);
-#ifndef _WIN32
__attribute__((weak))
__attribute__((visibility("hidden")))
-#endif
void __compilerrt_abort_impl(const char *file, int line, const char *function) {
__assert_rtn(function, file, line, "libcompiler_rt abort");
}
-#elif __Fuchsia__
-
-#ifndef _WIN32
-__attribute__((weak))
-__attribute__((visibility("hidden")))
-#endif
-void __compilerrt_abort_impl(const char *file, int line, const char *function) {
- __builtin_trap();
-}
-
#else
-// Get the system definition of abort()
+#ifdef _WIN32
#include <stdlib.h>
+#endif
#ifndef _WIN32
__attribute__((weak))
__attribute__((visibility("hidden")))
#endif
void __compilerrt_abort_impl(const char *file, int line, const char *function) {
+#if !__STDC_HOSTED__
+ // Avoid depending on libc when compiling with -ffreestanding.
+ __builtin_trap();
+#elif defined(_WIN32)
abort();
+#else
+ __builtin_abort();
+#endif
}
#endif
#define COMPILE_TIME_ASSERT2(expr, cnt) \
typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED
+// Force unrolling the code specified to be repeated N times.
+#define REPEAT_0_TIMES(code_to_repeat) /* do nothing */
+#define REPEAT_1_TIMES(code_to_repeat) code_to_repeat
+#define REPEAT_2_TIMES(code_to_repeat) \
+ REPEAT_1_TIMES(code_to_repeat) \
+ code_to_repeat
+#define REPEAT_3_TIMES(code_to_repeat) \
+ REPEAT_2_TIMES(code_to_repeat) \
+ code_to_repeat
+#define REPEAT_4_TIMES(code_to_repeat) \
+ REPEAT_3_TIMES(code_to_repeat) \
+ code_to_repeat
+
+#define REPEAT_N_TIMES_(N, code_to_repeat) REPEAT_##N##_TIMES(code_to_repeat)
+#define REPEAT_N_TIMES(N, code_to_repeat) REPEAT_N_TIMES_(N, code_to_repeat)
+
#endif // INT_UTIL_H
// Returns: a % b
-COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) {
- const int bits_in_dword_m1 = (int)(sizeof(di_int) * CHAR_BIT) - 1;
- di_int s = b >> bits_in_dword_m1; // s = b < 0 ? -1 : 0
- b = (b ^ s) - s; // negate if s == -1
- s = a >> bits_in_dword_m1; // s = a < 0 ? -1 : 0
- a = (a ^ s) - s; // negate if s == -1
- du_int r;
- __udivmoddi4(a, b, &r);
- return ((di_int)r ^ s) - s; // negate if s == -1
-}
+#define fixint_t di_int
+#define fixuint_t du_int
+#define ASSIGN_UMOD(res, a, b) __udivmoddi4((a), (b), &(res))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI di_int __moddi3(di_int a, di_int b) { return __modXi3(a, b); }
// Returns: a % b
-COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) {
- const int bits_in_tword_m1 = (int)(sizeof(ti_int) * CHAR_BIT) - 1;
- ti_int s = b >> bits_in_tword_m1; // s = b < 0 ? -1 : 0
- b = (b ^ s) - s; // negate if s == -1
- s = a >> bits_in_tword_m1; // s = a < 0 ? -1 : 0
- a = (a ^ s) - s; // negate if s == -1
- tu_int r;
- __udivmodti4(a, b, &r);
- return ((ti_int)r ^ s) - s; // negate if s == -1
-}
+#define fixint_t ti_int
+#define fixuint_t tu_int
+#define ASSIGN_UMOD(res, a, b) __udivmodti4((a), (b), &(res))
+#include "int_div_impl.inc"
+
+COMPILER_RT_ABI ti_int __modti3(ti_int a, ti_int b) { return __modXi3(a, b); }
#endif // CRT_HAS_128BIT
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t di_int
+#include "int_mulo_impl.inc"
// Returns: a * b
// Effects: sets *overflow to 1 if a * b overflows
COMPILER_RT_ABI di_int __mulodi4(di_int a, di_int b, int *overflow) {
- const int N = (int)(sizeof(di_int) * CHAR_BIT);
- const di_int MIN = (di_int)1 << (N - 1);
- const di_int MAX = ~MIN;
- *overflow = 0;
- di_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- di_int sa = a >> (N - 1);
- di_int abs_a = (a ^ sa) - sa;
- di_int sb = b >> (N - 1);
- di_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t si_int
+#include "int_mulo_impl.inc"
// Returns: a * b
// Effects: sets *overflow to 1 if a * b overflows
COMPILER_RT_ABI si_int __mulosi4(si_int a, si_int b, int *overflow) {
- const int N = (int)(sizeof(si_int) * CHAR_BIT);
- const si_int MIN = (si_int)1 << (N - 1);
- const si_int MAX = ~MIN;
- *overflow = 0;
- si_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- si_int sa = a >> (N - 1);
- si_int abs_a = (a ^ sa) - sa;
- si_int sb = b >> (N - 1);
- si_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
// Effects: sets *overflow to 1 if a * b overflows
+#define fixint_t ti_int
+#include "int_mulo_impl.inc"
+
COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) {
- const int N = (int)(sizeof(ti_int) * CHAR_BIT);
- const ti_int MIN = (ti_int)1 << (N - 1);
- const ti_int MAX = ~MIN;
- *overflow = 0;
- ti_int result = a * b;
- if (a == MIN) {
- if (b != 0 && b != 1)
- *overflow = 1;
- return result;
- }
- if (b == MIN) {
- if (a != 0 && a != 1)
- *overflow = 1;
- return result;
- }
- ti_int sa = a >> (N - 1);
- ti_int abs_a = (a ^ sa) - sa;
- ti_int sb = b >> (N - 1);
- ti_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return result;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- *overflow = 1;
- } else {
- if (abs_a > MIN / -abs_b)
- *overflow = 1;
- }
- return result;
+ return __muloXi4(a, b, overflow);
}
#endif // CRT_HAS_128BIT
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t di_int
+#include "int_mulv_impl.inc"
// Returns: a * b
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) {
- const int N = (int)(sizeof(di_int) * CHAR_BIT);
- const di_int MIN = (di_int)1 << (N - 1);
- const di_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- di_int sa = a >> (N - 1);
- di_int abs_a = (a ^ sa) - sa;
- di_int sb = b >> (N - 1);
- di_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+COMPILER_RT_ABI di_int __mulvdi3(di_int a, di_int b) { return __mulvXi3(a, b); }
//
//===----------------------------------------------------------------------===//
-#include "int_lib.h"
+#define fixint_t si_int
+#include "int_mulv_impl.inc"
// Returns: a * b
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) {
- const int N = (int)(sizeof(si_int) * CHAR_BIT);
- const si_int MIN = (si_int)1 << (N - 1);
- const si_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- si_int sa = a >> (N - 1);
- si_int abs_a = (a ^ sa) - sa;
- si_int sb = b >> (N - 1);
- si_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+COMPILER_RT_ABI si_int __mulvsi3(si_int a, si_int b) { return __mulvXi3(a, b); }
// Effects: aborts if a * b overflows
-COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) {
- const int N = (int)(sizeof(ti_int) * CHAR_BIT);
- const ti_int MIN = (ti_int)1 << (N - 1);
- const ti_int MAX = ~MIN;
- if (a == MIN) {
- if (b == 0 || b == 1)
- return a * b;
- compilerrt_abort();
- }
- if (b == MIN) {
- if (a == 0 || a == 1)
- return a * b;
- compilerrt_abort();
- }
- ti_int sa = a >> (N - 1);
- ti_int abs_a = (a ^ sa) - sa;
- ti_int sb = b >> (N - 1);
- ti_int abs_b = (b ^ sb) - sb;
- if (abs_a < 2 || abs_b < 2)
- return a * b;
- if (sa == sb) {
- if (abs_a > MAX / abs_b)
- compilerrt_abort();
- } else {
- if (abs_a > MIN / -abs_b)
- compilerrt_abort();
- }
- return a * b;
-}
+#define fixint_t ti_int
+#include "int_mulv_impl.inc"
+
+COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { return __mulvXi3(a, b); }
#endif // CRT_HAS_128BIT
// These three variables hold the host's OS version.
static int32_t GlobalMajor, GlobalMinor, GlobalSubminor;
static dispatch_once_t DispatchOnceCounter;
+static dispatch_once_t CompatibilityDispatchOnceCounter;
+
+// _availability_version_check darwin API support.
+typedef uint32_t dyld_platform_t;
+
+typedef struct {
+ dyld_platform_t platform;
+ uint32_t version;
+} dyld_build_version_t;
+
+typedef bool (*AvailabilityVersionCheckFuncTy)(uint32_t count,
+ dyld_build_version_t versions[]);
+
+static AvailabilityVersionCheckFuncTy AvailabilityVersionCheck;
// We can't include <CoreFoundation/CoreFoundation.h> directly from here, so
// just forward declare everything that we need from it.
CFStringEncoding);
typedef void (*CFReleaseFuncTy)(CFTypeRef);
-// Find and parse the SystemVersion.plist file.
-static void parseSystemVersionPList(void *Unused) {
- (void)Unused;
+static void _initializeAvailabilityCheck(bool LoadPlist) {
+ if (AvailabilityVersionCheck && !LoadPlist) {
+ // New API is supported and we're not being asked to load the plist,
+ // exit early!
+ return;
+ }
+
+ // Use the new API if it's is available.
+ AvailabilityVersionCheck = (AvailabilityVersionCheckFuncTy)dlsym(
+ RTLD_DEFAULT, "_availability_version_check");
+
+ if (AvailabilityVersionCheck && !LoadPlist) {
+ // New API is supported and we're not being asked to load the plist,
+ // exit early!
+ return;
+ }
+ // Still load the PLIST to ensure that the existing calls to
+ // __isOSVersionAtLeast still work even with new compiler-rt and old OSes.
+
// Load CoreFoundation dynamically
const void *NullAllocator = dlsym(RTLD_DEFAULT, "kCFAllocatorNull");
if (!NullAllocator)
fclose(PropertyList);
}
+// Find and parse the SystemVersion.plist file.
+static void compatibilityInitializeAvailabilityCheck(void *Unused) {
+ (void)Unused;
+ _initializeAvailabilityCheck(/*LoadPlist=*/true);
+}
+
+static void initializeAvailabilityCheck(void *Unused) {
+ (void)Unused;
+ _initializeAvailabilityCheck(/*LoadPlist=*/false);
+}
+
+// This old API entry point is no longer used by Clang for Darwin. We still need
+// to keep it around to ensure that object files that reference it are still
+// usable when linked with new compiler-rt.
int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
// Populate the global version variables, if they haven't already.
- dispatch_once_f(&DispatchOnceCounter, NULL, parseSystemVersionPList);
+ dispatch_once_f(&CompatibilityDispatchOnceCounter, NULL,
+ compatibilityInitializeAvailabilityCheck);
if (Major < GlobalMajor)
return 1;
return Subminor <= GlobalSubminor;
}
+static inline uint32_t ConstructVersion(uint32_t Major, uint32_t Minor,
+ uint32_t Subminor) {
+ return ((Major & 0xffff) << 16) | ((Minor & 0xff) << 8) | (Subminor & 0xff);
+}
+
+int32_t __isPlatformVersionAtLeast(uint32_t Platform, uint32_t Major,
+ uint32_t Minor, uint32_t Subminor) {
+ dispatch_once_f(&DispatchOnceCounter, NULL, initializeAvailabilityCheck);
+
+ if (!AvailabilityVersionCheck) {
+ return __isOSVersionAtLeast(Major, Minor, Subminor);
+ }
+ dyld_build_version_t Versions[] = {
+ {Platform, ConstructVersion(Major, Minor, Subminor)}};
+ return AvailabilityVersionCheck(1, Versions);
+}
+
+#elif __ANDROID__
+
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/system_properties.h>
+
+static int SdkVersion;
+static int IsPreRelease;
+
+static void readSystemProperties(void) {
+ char buf[PROP_VALUE_MAX];
+
+ if (__system_property_get("ro.build.version.sdk", buf) == 0) {
+ // When the system property doesn't exist, defaults to future API level.
+ SdkVersion = __ANDROID_API_FUTURE__;
+ } else {
+ SdkVersion = atoi(buf);
+ }
+
+ if (__system_property_get("ro.build.version.codename", buf) == 0) {
+ IsPreRelease = 1;
+ } else {
+ IsPreRelease = strcmp(buf, "REL") != 0;
+ }
+ return;
+}
+
+int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) {
+ (int32_t) Minor;
+ (int32_t) Subminor;
+ static pthread_once_t once = PTHREAD_ONCE_INIT;
+ pthread_once(&once, readSystemProperties);
+
+ return SdkVersion >= Major ||
+ (IsPreRelease && Major == __ANDROID_API_FUTURE__);
+}
+
#else
// Silence an empty translation unit warning.
COMPILER_RT_ABI int __paritydi2(di_int a) {
dwords x;
x.all = a;
- return __paritysi2(x.s.high ^ x.s.low);
+ su_int x2 = x.s.high ^ x.s.low;
+ x2 ^= x2 >> 16;
+ x2 ^= x2 >> 8;
+ x2 ^= x2 >> 4;
+ return (0x6996 >> (x2 & 0xF)) & 1;
}
COMPILER_RT_ABI int __parityti2(ti_int a) {
twords x;
+ dwords x2;
x.all = a;
- return __paritydi2(x.s.high ^ x.s.low);
+ x2.all = x.s.high ^ x.s.low;
+ su_int x3 = x2.s.high ^ x2.s.low;
+ x3 ^= x3 >> 16;
+ x3 ^= x3 >> 8;
+ x3 ^= x3 >> 4;
+ return (0x6996 >> (x3 & 0xF)) & 1;
}
#endif // CRT_HAS_128BIT
--- /dev/null
+__atomic_compare_exchange
+__atomic_compare_exchange_1
+__atomic_compare_exchange_2
+__atomic_compare_exchange_4
+__atomic_compare_exchange_8
+__atomic_exchange
+__atomic_exchange_1
+__atomic_exchange_2
+__atomic_exchange_4
+__atomic_exchange_8
+__atomic_fetch_add_1
+__atomic_fetch_add_2
+__atomic_fetch_add_4
+__atomic_fetch_add_8
+__atomic_fetch_and_1
+__atomic_fetch_and_2
+__atomic_fetch_and_4
+__atomic_fetch_and_8
+__atomic_fetch_or_1
+__atomic_fetch_or_2
+__atomic_fetch_or_4
+__atomic_fetch_or_8
+__atomic_fetch_sub_1
+__atomic_fetch_sub_2
+__atomic_fetch_sub_4
+__atomic_fetch_sub_8
+__atomic_fetch_xor_1
+__atomic_fetch_xor_2
+__atomic_fetch_xor_4
+__atomic_fetch_xor_8
+__atomic_is_lock_free
+__atomic_load
+__atomic_load_1
+__atomic_load_2
+__atomic_load_4
+__atomic_load_8
+__atomic_store
+__atomic_store_1
+__atomic_store_2
+__atomic_store_4
+__atomic_store_8
int ilogbw = 0;
const double logbw =
- __compiler_rt_logb(crt_fmax(crt_fabs(cDD.s.hi), crt_fabs(dDD.s.hi)));
+ __compiler_rt_logb(__compiler_rt_fmax(crt_fabs(cDD.s.hi),
+ crt_fabs(dDD.s.hi)));
if (crt_isfinite(logbw)) {
ilogbw = (int)logbw;
- cDD.s.hi = crt_scalbn(cDD.s.hi, -ilogbw);
- cDD.s.lo = crt_scalbn(cDD.s.lo, -ilogbw);
- dDD.s.hi = crt_scalbn(dDD.s.hi, -ilogbw);
- dDD.s.lo = crt_scalbn(dDD.s.lo, -ilogbw);
+ cDD.s.hi = __compiler_rt_scalbn(cDD.s.hi, -ilogbw);
+ cDD.s.lo = __compiler_rt_scalbn(cDD.s.lo, -ilogbw);
+ dDD.s.hi = __compiler_rt_scalbn(dDD.s.hi, -ilogbw);
+ dDD.s.lo = __compiler_rt_scalbn(dDD.s.lo, -ilogbw);
}
const long double denom =
DD real = {.ld = __gcc_qdiv(realNumerator, denom)};
DD imag = {.ld = __gcc_qdiv(imagNumerator, denom)};
- real.s.hi = crt_scalbn(real.s.hi, -ilogbw);
- real.s.lo = crt_scalbn(real.s.lo, -ilogbw);
- imag.s.hi = crt_scalbn(imag.s.hi, -ilogbw);
- imag.s.lo = crt_scalbn(imag.s.lo, -ilogbw);
+ real.s.hi = __compiler_rt_scalbn(real.s.hi, -ilogbw);
+ real.s.lo = __compiler_rt_scalbn(real.s.lo, -ilogbw);
+ imag.s.hi = __compiler_rt_scalbn(imag.s.hi, -ilogbw);
+ imag.s.lo = __compiler_rt_scalbn(imag.s.lo, -ilogbw);
if (crt_isnan(real.s.hi) && crt_isnan(imag.s.hi)) {
DD aDD = {.ld = a};
//
//===----------------------------------------------------------------------===//
-#if !defined(__riscv_mul)
+#ifndef __mulxi3
+#error "__mulxi3 must be defined to use this generic implementation"
+#endif
+
.text
.align 2
slli a2, a2, 1
bnez a1, .L1
ret
-#endif
--- /dev/null
+//===-- restore.S - restore up to 12 callee-save registers ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Multiple entry points depending on number of registers to restore
+//
+//===----------------------------------------------------------------------===//
+
+// All of the entry points are in the same section since we rely on many of
+// them falling through into each other and don't want the linker to
+// accidentally split them up, garbage collect, or reorder them.
+//
+// The entry points are grouped up into 2s for rv64 and 4s for rv32 since this
+// is the minimum grouping which will maintain the required 16-byte stack
+// alignment.
+
+ .text
+
+#if __riscv_xlen == 32
+
+ .globl __riscv_restore_12
+ .type __riscv_restore_12,@function
+__riscv_restore_12:
+ lw s11, 12(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_11/10/9/8
+
+ .globl __riscv_restore_11
+ .type __riscv_restore_11,@function
+ .globl __riscv_restore_10
+ .type __riscv_restore_10,@function
+ .globl __riscv_restore_9
+ .type __riscv_restore_9,@function
+ .globl __riscv_restore_8
+ .type __riscv_restore_8,@function
+__riscv_restore_11:
+__riscv_restore_10:
+__riscv_restore_9:
+__riscv_restore_8:
+ lw s10, 0(sp)
+ lw s9, 4(sp)
+ lw s8, 8(sp)
+ lw s7, 12(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_7/6/5/4
+
+ .globl __riscv_restore_7
+ .type __riscv_restore_7,@function
+ .globl __riscv_restore_6
+ .type __riscv_restore_6,@function
+ .globl __riscv_restore_5
+ .type __riscv_restore_5,@function
+ .globl __riscv_restore_4
+ .type __riscv_restore_4,@function
+__riscv_restore_7:
+__riscv_restore_6:
+__riscv_restore_5:
+__riscv_restore_4:
+ lw s6, 0(sp)
+ lw s5, 4(sp)
+ lw s4, 8(sp)
+ lw s3, 12(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_3/2/1/0
+
+ .globl __riscv_restore_3
+ .type __riscv_restore_3,@function
+ .globl __riscv_restore_2
+ .type __riscv_restore_2,@function
+ .globl __riscv_restore_1
+ .type __riscv_restore_1,@function
+ .globl __riscv_restore_0
+ .type __riscv_restore_0,@function
+__riscv_restore_3:
+__riscv_restore_2:
+__riscv_restore_1:
+__riscv_restore_0:
+ lw s2, 0(sp)
+ lw s1, 4(sp)
+ lw s0, 8(sp)
+ lw ra, 12(sp)
+ addi sp, sp, 16
+ ret
+
+#elif __riscv_xlen == 64
+
+ .globl __riscv_restore_12
+ .type __riscv_restore_12,@function
+__riscv_restore_12:
+ ld s11, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_11/10/9/8
+
+ .globl __riscv_restore_11
+ .type __riscv_restore_11,@function
+ .globl __riscv_restore_10
+ .type __riscv_restore_10,@function
+__riscv_restore_11:
+__riscv_restore_10:
+ ld s10, 0(sp)
+ ld s9, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_9/8
+
+ .globl __riscv_restore_9
+ .type __riscv_restore_9,@function
+ .globl __riscv_restore_8
+ .type __riscv_restore_8,@function
+__riscv_restore_9:
+__riscv_restore_8:
+ ld s8, 0(sp)
+ ld s7, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_7/6
+
+ .globl __riscv_restore_7
+ .type __riscv_restore_7,@function
+ .globl __riscv_restore_6
+ .type __riscv_restore_6,@function
+__riscv_restore_7:
+__riscv_restore_6:
+ ld s6, 0(sp)
+ ld s5, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_5/4
+
+ .globl __riscv_restore_5
+ .type __riscv_restore_5,@function
+ .globl __riscv_restore_4
+ .type __riscv_restore_4,@function
+__riscv_restore_5:
+__riscv_restore_4:
+ ld s4, 0(sp)
+ ld s3, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_3/2
+
+ .globl __riscv_restore_3
+ .type __riscv_restore_3,@function
+ .globl __riscv_restore_2
+ .type __riscv_restore_2,@function
+ .globl __riscv_restore_1
+ .type __riscv_restore_1,@function
+ .globl __riscv_restore_0
+ .type __riscv_restore_0,@function
+__riscv_restore_3:
+__riscv_restore_2:
+ ld s2, 0(sp)
+ ld s1, 8(sp)
+ addi sp, sp, 16
+ // fallthrough into __riscv_restore_1/0
+
+__riscv_restore_1:
+__riscv_restore_0:
+ ld s0, 0(sp)
+ ld ra, 8(sp)
+ addi sp, sp, 16
+ ret
+
+#else
+# error "xlen must be 32 or 64 for save-restore implementation
+#endif
--- /dev/null
+//===-- save.S - save up to 12 callee-saved registers ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Multiple entry points depending on number of registers to save
+//
+//===----------------------------------------------------------------------===//
+
+// The entry points are grouped up into 2s for rv64 and 4s for rv32 since this
+// is the minimum grouping which will maintain the required 16-byte stack
+// alignment.
+
+ .text
+
+#if __riscv_xlen == 32
+
+ .globl __riscv_save_12
+ .type __riscv_save_12,@function
+__riscv_save_12:
+ addi sp, sp, -64
+ mv t1, zero
+ sw s11, 12(sp)
+ j .Lriscv_save_11_8
+
+ .globl __riscv_save_11
+ .type __riscv_save_11,@function
+ .globl __riscv_save_10
+ .type __riscv_save_10,@function
+ .globl __riscv_save_9
+ .type __riscv_save_9,@function
+ .globl __riscv_save_8
+ .type __riscv_save_8,@function
+__riscv_save_11:
+__riscv_save_10:
+__riscv_save_9:
+__riscv_save_8:
+ addi sp, sp, -64
+ li t1, 16
+.Lriscv_save_11_8:
+ sw s10, 16(sp)
+ sw s9, 20(sp)
+ sw s8, 24(sp)
+ sw s7, 28(sp)
+ j .Lriscv_save_7_4
+
+ .globl __riscv_save_7
+ .type __riscv_save_7,@function
+ .globl __riscv_save_6
+ .type __riscv_save_6,@function
+ .globl __riscv_save_5
+ .type __riscv_save_5,@function
+ .globl __riscv_save_4
+ .type __riscv_save_4,@function
+__riscv_save_7:
+__riscv_save_6:
+__riscv_save_5:
+__riscv_save_4:
+ addi sp, sp, -64
+ li t1, 32
+.Lriscv_save_7_4:
+ sw s6, 32(sp)
+ sw s5, 36(sp)
+ sw s4, 40(sp)
+ sw s3, 44(sp)
+ sw s2, 48(sp)
+ sw s1, 52(sp)
+ sw s0, 56(sp)
+ sw ra, 60(sp)
+ add sp, sp, t1
+ jr t0
+
+ .globl __riscv_save_3
+ .type __riscv_save_3,@function
+ .globl __riscv_save_2
+ .type __riscv_save_2,@function
+ .globl __riscv_save_1
+ .type __riscv_save_1,@function
+ .globl __riscv_save_0
+ .type __riscv_save_0,@function
+__riscv_save_3:
+__riscv_save_2:
+__riscv_save_1:
+__riscv_save_0:
+ addi sp, sp, -16
+ sw s2, 0(sp)
+ sw s1, 4(sp)
+ sw s0, 8(sp)
+ sw ra, 12(sp)
+ jr t0
+
+#elif __riscv_xlen == 64
+
+ .globl __riscv_save_12
+ .type __riscv_save_12,@function
+__riscv_save_12:
+ addi sp, sp, -112
+ mv t1, zero
+ sd s11, 8(sp)
+ j .Lriscv_save_11_10
+
+ .globl __riscv_save_11
+ .type __riscv_save_11,@function
+ .globl __riscv_save_10
+ .type __riscv_save_10,@function
+__riscv_save_11:
+__riscv_save_10:
+ addi sp, sp, -112
+ li t1, 16
+.Lriscv_save_11_10:
+ sd s10, 16(sp)
+ sd s9, 24(sp)
+ j .Lriscv_save_9_8
+
+ .globl __riscv_save_9
+ .type __riscv_save_9,@function
+ .globl __riscv_save_8
+ .type __riscv_save_8,@function
+__riscv_save_9:
+__riscv_save_8:
+ addi sp, sp, -112
+ li t1, 32
+.Lriscv_save_9_8:
+ sd s8, 32(sp)
+ sd s7, 40(sp)
+ j .Lriscv_save_7_6
+
+ .globl __riscv_save_7
+ .type __riscv_save_7,@function
+ .globl __riscv_save_6
+ .type __riscv_save_6,@function
+__riscv_save_7:
+__riscv_save_6:
+ addi sp, sp, -112
+ li t1, 48
+.Lriscv_save_7_6:
+ sd s6, 48(sp)
+ sd s5, 56(sp)
+ j .Lriscv_save_5_4
+
+ .globl __riscv_save_5
+ .type __riscv_save_5,@function
+ .globl __riscv_save_4
+ .type __riscv_save_4,@function
+__riscv_save_5:
+__riscv_save_4:
+ addi sp, sp, -112
+ li t1, 64
+.Lriscv_save_5_4:
+ sd s4, 64(sp)
+ sd s3, 72(sp)
+ j .Lriscv_save_3_2
+
+ .globl __riscv_save_3
+ .type __riscv_save_3,@function
+ .globl __riscv_save_2
+ .type __riscv_save_2,@function
+__riscv_save_3:
+__riscv_save_2:
+ addi sp, sp, -112
+ li t1, 80
+.Lriscv_save_3_2:
+ sd s2, 80(sp)
+ sd s1, 88(sp)
+ sd s0, 96(sp)
+ sd ra, 104(sp)
+ add sp, sp, t1
+ jr t0
+
+ .globl __riscv_save_1
+ .type __riscv_save_1,@function
+ .globl __riscv_save_0
+ .type __riscv_save_0,@function
+ addi sp, sp, -16
+ sd s0, 0(sp)
+ sd ra, 8(sp)
+ jr t0
+
+#else
+# error "xlen must be 32 or 64 for save-restore implementation
+#endif
#define DST_HALF
#include "fp_trunc_impl.inc"
-COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
+COMPILER_RT_ABI dst_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
+AEABI_RTABI dst_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
#else
COMPILER_RT_ALIAS(__truncdfhf2, __aeabi_d2h)
#endif
// Use a forwarding definition and noinline to implement a poor man's alias,
// as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) {
+COMPILER_RT_ABI NOINLINE dst_t __truncsfhf2(float a) {
return __truncXfYf2__(a);
}
-COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
+COMPILER_RT_ABI dst_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
#if defined(__ARM_EABI__)
#if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
+AEABI_RTABI dst_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
#else
COMPILER_RT_ALIAS(__truncsfhf2, __aeabi_f2h)
#endif
--- /dev/null
+//===-- lib/trunctfhf2.c - quad -> half conversion ----------------*- C -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
+ defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_QUAD
+#define DST_HALF
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI _Float16 __trunctfhf2(long double a) {
+ return __truncXfYf2__(a);
+}
+
+#endif
endforeach()
endif()
-add_compiler_rt_resource_file(cfi_blacklist cfi_blacklist.txt cfi)
+add_compiler_rt_resource_file(cfi_ignorelist cfi_ignorelist.txt cfi)
THREADLOCAL int in_loader;
BlockingMutex shadow_update_lock(LINKER_INITIALIZED);
-void EnterLoader() {
+void EnterLoader() NO_THREAD_SAFETY_ANALYSIS {
if (in_loader == 0) {
shadow_update_lock.Lock();
}
++in_loader;
}
-void ExitLoader() {
+void ExitLoader() NO_THREAD_SAFETY_ANALYSIS {
CHECK(in_loader > 0);
--in_loader;
UpdateShadow();
__ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
RegisterCommonFlags(&ubsan_parser);
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
#endif
--- /dev/null
+[cfi-unrelated-cast]
+# The specification of std::get_temporary_buffer mandates a cast to
+# uninitialized T* (libstdc++, MSVC stdlib).
+fun:_ZSt20get_temporary_buffer*
+fun:*get_temporary_buffer@.*@std@@*
+
+# STL address-of magic (libstdc++).
+fun:*__addressof*
+
+# Windows C++ stdlib headers that contain bad unrelated casts.
+src:*xmemory0
+src:*xstddef
+
+# std::_Sp_counted_ptr_inplace::_Sp_counted_ptr_inplace() (libstdc++).
+# This ctor is used by std::make_shared and needs to cast to uninitialized T*
+# in order to call std::allocator_traits<T>::construct.
+fun:_ZNSt23_Sp_counted_ptr_inplace*
__asm__(".pushsection .init,\"ax\",@progbits\n\t"
"call " __USER_LABEL_PREFIX__ "__do_init\n\t"
".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .init,\"ax\",%progbits\n\t"
+ "call " __USER_LABEL_PREFIX__ "__do_init\n\t"
+ ".popsection");
#elif defined(__arm__) || defined(__aarch64__)
__asm__(".pushsection .init,\"ax\",%progbits\n\t"
"bl " __USER_LABEL_PREFIX__ "__do_init\n\t"
"bl " __USER_LABEL_PREFIX__ "__do_fini\n\t"
"nop\n\t"
".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "call " __USER_LABEL_PREFIX__ "__do_fini\n\t"
+ ".popsection");
#elif defined(__sparc__)
__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
"call " __USER_LABEL_PREFIX__ "__do_fini\n\t"
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
# Runtime library sources and build flags.
set(DFSAN_RTL_SOURCES
dfsan.cpp
+ dfsan_allocator.cpp
+ dfsan_chained_origin_depot.cpp
dfsan_custom.cpp
dfsan_interceptors.cpp
+ dfsan_new_delete.cpp
+ dfsan_thread.cpp
)
set(DFSAN_RTL_HEADERS
dfsan.h
+ dfsan_allocator.h
+ dfsan_chained_origin_depot.h
dfsan_flags.inc
+ dfsan_flags.h
dfsan_platform.h
+ dfsan_thread.h
)
set(DFSAN_COMMON_CFLAGS ${SANITIZER_COMMON_CFLAGS})
$<TARGET_OBJECTS:RTInterception.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
+ $<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
ADDITIONAL_HEADERS ${DFSAN_RTL_HEADERS}
CFLAGS ${DFSAN_CFLAGS}
PARENT_TARGET dfsan)
DEPENDS done_abilist.txt libc_ubuntu1404_abilist.txt)
add_dependencies(dfsan dfsan_abilist)
install(FILES ${dfsan_abilist_filename}
- DESTINATION ${COMPILER_RT_INSTALL_PATH}/share)
+ DESTINATION ${COMPILER_RT_INSTALL_DATA_DIR})
// prefixed __dfsan_.
//===----------------------------------------------------------------------===//
+#include "dfsan/dfsan.h"
+
+#include "dfsan/dfsan_chained_origin_depot.h"
+#include "dfsan/dfsan_flags.h"
+#include "dfsan/dfsan_origin.h"
+#include "dfsan/dfsan_thread.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_file.h"
-#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
-
-#include "dfsan/dfsan.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
using namespace __dfsan;
-typedef atomic_uint16_t atomic_dfsan_label;
-static const dfsan_label kInitializingLabel = -1;
-
-static const uptr kNumLabels = 1 << (sizeof(dfsan_label) * 8);
+Flags __dfsan::flags_data;
-static atomic_dfsan_label __dfsan_last_label;
-static dfsan_label_info __dfsan_label_info[kNumLabels];
+// The size of TLS variables. These constants must be kept in sync with the ones
+// in DataFlowSanitizer.cpp.
+static const int kDFsanArgTlsSize = 800;
+static const int kDFsanRetvalTlsSize = 800;
+static const int kDFsanArgOriginTlsSize = 800;
-Flags __dfsan::flags_data;
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
+ __dfsan_retval_tls[kDFsanRetvalTlsSize / sizeof(u64)];
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32 __dfsan_retval_origin_tls;
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
+ __dfsan_arg_tls[kDFsanArgTlsSize / sizeof(u64)];
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32
+ __dfsan_arg_origin_tls[kDFsanArgOriginTlsSize / sizeof(u32)];
-SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_retval_tls;
-SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_arg_tls[64];
+// Instrumented code may set this value in terms of -dfsan-track-origins.
+// * undefined or 0: do not track origins.
+// * 1: track origins at memory store operations.
+// * 2: track origins at memory load and store operations.
+// TODO: track callsites.
+extern "C" SANITIZER_WEAK_ATTRIBUTE const int __dfsan_track_origins;
-SANITIZER_INTERFACE_ATTRIBUTE uptr __dfsan_shadow_ptr_mask;
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE int dfsan_get_track_origins() {
+ return &__dfsan_track_origins ? __dfsan_track_origins : 0;
+}
// On Linux/x86_64, memory is laid out as follows:
//
-// +--------------------+ 0x800000000000 (top of memory)
-// | application memory |
-// +--------------------+ 0x700000008000 (kAppAddr)
-// | |
-// | unused |
-// | |
-// +--------------------+ 0x200200000000 (kUnusedAddr)
-// | union table |
-// +--------------------+ 0x200000000000 (kUnionTableAddr)
-// | shadow memory |
-// +--------------------+ 0x000000010000 (kShadowAddr)
-// | reserved by kernel |
-// +--------------------+ 0x000000000000
-//
-// To derive a shadow memory address from an application memory address,
-// bits 44-46 are cleared to bring the address into the range
-// [0x000000008000,0x100000000000). Then the address is shifted left by 1 to
-// account for the double byte representation of shadow labels and move the
-// address into the shadow memory range. See the function shadow_for below.
-
-// On Linux/MIPS64, memory is laid out as follows:
-//
-// +--------------------+ 0x10000000000 (top of memory)
-// | application memory |
-// +--------------------+ 0xF000008000 (kAppAddr)
-// | |
-// | unused |
-// | |
-// +--------------------+ 0x2200000000 (kUnusedAddr)
-// | union table |
-// +--------------------+ 0x2000000000 (kUnionTableAddr)
-// | shadow memory |
-// +--------------------+ 0x0000010000 (kShadowAddr)
-// | reserved by kernel |
-// +--------------------+ 0x0000000000
-
-// On Linux/AArch64 (39-bit VMA), memory is laid out as follow:
-//
-// +--------------------+ 0x8000000000 (top of memory)
-// | application memory |
-// +--------------------+ 0x7000008000 (kAppAddr)
-// | |
-// | unused |
-// | |
-// +--------------------+ 0x1200000000 (kUnusedAddr)
-// | union table |
-// +--------------------+ 0x1000000000 (kUnionTableAddr)
-// | shadow memory |
-// +--------------------+ 0x0000010000 (kShadowAddr)
-// | reserved by kernel |
-// +--------------------+ 0x0000000000
-
-// On Linux/AArch64 (42-bit VMA), memory is laid out as follow:
+// +--------------------+ 0x800000000000 (top of memory)
+// | application 3 |
+// +--------------------+ 0x700000000000
+// | invalid |
+// +--------------------+ 0x610000000000
+// | origin 1 |
+// +--------------------+ 0x600000000000
+// | application 2 |
+// +--------------------+ 0x510000000000
+// | shadow 1 |
+// +--------------------+ 0x500000000000
+// | invalid |
+// +--------------------+ 0x400000000000
+// | origin 3 |
+// +--------------------+ 0x300000000000
+// | shadow 3 |
+// +--------------------+ 0x200000000000
+// | origin 2 |
+// +--------------------+ 0x110000000000
+// | invalid |
+// +--------------------+ 0x100000000000
+// | shadow 2 |
+// +--------------------+ 0x010000000000
+// | application 1 |
+// +--------------------+ 0x000000000000
//
-// +--------------------+ 0x40000000000 (top of memory)
-// | application memory |
-// +--------------------+ 0x3ff00008000 (kAppAddr)
-// | |
-// | unused |
-// | |
-// +--------------------+ 0x1200000000 (kUnusedAddr)
-// | union table |
-// +--------------------+ 0x8000000000 (kUnionTableAddr)
-// | shadow memory |
-// +--------------------+ 0x0000010000 (kShadowAddr)
-// | reserved by kernel |
-// +--------------------+ 0x0000000000
-
-// On Linux/AArch64 (48-bit VMA), memory is laid out as follow:
-//
-// +--------------------+ 0x1000000000000 (top of memory)
-// | application memory |
-// +--------------------+ 0xffff00008000 (kAppAddr)
-// | unused |
-// +--------------------+ 0xaaaab0000000 (top of PIE address)
-// | application PIE |
-// +--------------------+ 0xaaaaa0000000 (top of PIE address)
-// | |
-// | unused |
-// | |
-// +--------------------+ 0x1200000000 (kUnusedAddr)
-// | union table |
-// +--------------------+ 0x8000000000 (kUnionTableAddr)
-// | shadow memory |
-// +--------------------+ 0x0000010000 (kShadowAddr)
-// | reserved by kernel |
-// +--------------------+ 0x0000000000
-
-typedef atomic_dfsan_label dfsan_union_table_t[kNumLabels][kNumLabels];
-
-#ifdef DFSAN_RUNTIME_VMA
-// Runtime detected VMA size.
-int __dfsan::vmaSize;
-#endif
+// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
+// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
-static uptr UnusedAddr() {
- return MappingArchImpl<MAPPING_UNION_TABLE_ADDR>()
- + sizeof(dfsan_union_table_t);
-}
-
-static atomic_dfsan_label *union_table(dfsan_label l1, dfsan_label l2) {
- return &(*(dfsan_union_table_t *) UnionTableAddr())[l1][l2];
-}
-
-// Checks we do not run out of labels.
-static void dfsan_check_label(dfsan_label label) {
- if (label == kInitializingLabel) {
- Report("FATAL: DataFlowSanitizer: out of labels\n");
- Die();
- }
-}
-
-// Resolves the union of two unequal labels. Nonequality is a precondition for
-// this function (the instrumentation pass inlines the equality test).
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-dfsan_label __dfsan_union(dfsan_label l1, dfsan_label l2) {
- if (flags().fast16labels)
- return l1 | l2;
- DCHECK_NE(l1, l2);
-
- if (l1 == 0)
- return l2;
- if (l2 == 0)
- return l1;
-
- if (l1 > l2)
- Swap(l1, l2);
-
- atomic_dfsan_label *table_ent = union_table(l1, l2);
- // We need to deal with the case where two threads concurrently request
- // a union of the same pair of labels. If the table entry is uninitialized,
- // (i.e. 0) use a compare-exchange to set the entry to kInitializingLabel
- // (i.e. -1) to mark that we are initializing it.
- dfsan_label label = 0;
- if (atomic_compare_exchange_strong(table_ent, &label, kInitializingLabel,
- memory_order_acquire)) {
- // Check whether l2 subsumes l1. We don't need to check whether l1
- // subsumes l2 because we are guaranteed here that l1 < l2, and (at least
- // in the cases we are interested in) a label may only subsume labels
- // created earlier (i.e. with a lower numerical value).
- if (__dfsan_label_info[l2].l1 == l1 ||
- __dfsan_label_info[l2].l2 == l1) {
- label = l2;
- } else {
- label =
- atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
- dfsan_check_label(label);
- __dfsan_label_info[label].l1 = l1;
- __dfsan_label_info[label].l2 = l2;
- }
- atomic_store(table_ent, label, memory_order_release);
- } else if (label == kInitializingLabel) {
- // Another thread is initializing the entry. Wait until it is finished.
- do {
- internal_sched_yield();
- label = atomic_load(table_ent, memory_order_acquire);
- } while (label == kInitializingLabel);
- }
+dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
+ dfsan_label label = ls[0];
+ for (uptr i = 1; i != n; ++i)
+ label |= ls[i];
return label;
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
- dfsan_label label = ls[0];
- for (uptr i = 1; i != n; ++i) {
- dfsan_label next_label = ls[i];
- if (label != next_label)
- label = __dfsan_union(label, next_label);
+// Return the union of all the n labels from addr at the high 32 bit, and the
+// origin of the first taint byte at the low 32 bit.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
+__dfsan_load_label_and_origin(const void *addr, uptr n) {
+ dfsan_label label = 0;
+ u64 ret = 0;
+ uptr p = (uptr)addr;
+ dfsan_label *s = shadow_for((void *)p);
+ for (uptr i = 0; i < n; ++i) {
+ dfsan_label l = s[i];
+ if (!l)
+ continue;
+ label |= l;
+ if (!ret)
+ ret = *(dfsan_origin *)origin_for((void *)(p + i));
}
- return label;
+ return ret | (u64)label << 32;
}
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
Die();
}
-// Like __dfsan_union, but for use from the client or custom functions. Hence
-// the equality comparison is done here before calling __dfsan_union.
+// Resolves the union of two labels.
SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
dfsan_union(dfsan_label l1, dfsan_label l2) {
- if (l1 == l2)
- return l1;
- return __dfsan_union(l1, l2);
+ return l1 | l2;
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-dfsan_label dfsan_create_label(const char *desc, void *userdata) {
- dfsan_label label =
- atomic_fetch_add(&__dfsan_last_label, 1, memory_order_relaxed) + 1;
- dfsan_check_label(label);
- __dfsan_label_info[label].l1 = __dfsan_label_info[label].l2 = 0;
- __dfsan_label_info[label].desc = desc;
- __dfsan_label_info[label].userdata = userdata;
- return label;
+static const uptr kOriginAlign = sizeof(dfsan_origin);
+static const uptr kOriginAlignMask = ~(kOriginAlign - 1UL);
+
+static uptr OriginAlignUp(uptr u) {
+ return (u + kOriginAlign - 1) & kOriginAlignMask;
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-void __dfsan_set_label(dfsan_label label, void *addr, uptr size) {
- for (dfsan_label *labelp = shadow_for(addr); size != 0; --size, ++labelp) {
+static uptr OriginAlignDown(uptr u) { return u & kOriginAlignMask; }
+
+// Return the origin of the first taint byte in the size bytes from the address
+// addr.
+static dfsan_origin GetOriginIfTainted(uptr addr, uptr size) {
+ for (uptr i = 0; i < size; ++i, ++addr) {
+ dfsan_label *s = shadow_for((void *)addr);
+
+ if (*s) {
+ // Validate address region.
+ CHECK(MEM_IS_SHADOW(s));
+ return *(dfsan_origin *)origin_for((void *)addr);
+ }
+ }
+ return 0;
+}
+
+// For platforms which support slow unwinder only, we need to restrict the store
+// context size to 1, basically only storing the current pc, because the slow
+// unwinder which is based on libunwind is not async signal safe and causes
+// random freezes in forking applications as well as in signal handlers.
+// DFSan supports only Linux. So we do not restrict the store context size.
+#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
+ BufferedStackTrace stack; \
+ stack.Unwind(pc, bp, nullptr, true, flags().store_context_size);
+
+#define PRINT_CALLER_STACK_TRACE \
+ { \
+ GET_CALLER_PC_BP_SP; \
+ (void)sp; \
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
+ stack.Print(); \
+ }
+
+// Return a chain with the previous ID id and the current stack.
+// from_init = true if this is the first chain of an origin tracking path.
+static u32 ChainOrigin(u32 id, StackTrace *stack, bool from_init = false) {
+ // StackDepot is not async signal safe. Do not create new chains in a signal
+ // handler.
+ DFsanThread *t = GetCurrentThread();
+ if (t && t->InSignalHandler())
+ return id;
+
+ // As an optimization the origin of an application byte is updated only when
+ // its shadow is non-zero. Because we are only interested in the origins of
+ // taint labels, it does not matter what origin a zero label has. This reduces
+ // memory write cost. MSan does similar optimization. The following invariant
+ // may not hold because of some bugs. We check the invariant to help debug.
+ if (!from_init && id == 0 && flags().check_origin_invariant) {
+ Printf(" DFSan found invalid origin invariant\n");
+ PRINT_CALLER_STACK_TRACE
+ }
+
+ Origin o = Origin::FromRawId(id);
+ stack->tag = StackTrace::TAG_UNKNOWN;
+ Origin chained = Origin::CreateChainedOrigin(o, stack);
+ return chained.raw_id();
+}
+
+static void ChainAndWriteOriginIfTainted(uptr src, uptr size, uptr dst,
+ StackTrace *stack) {
+ dfsan_origin o = GetOriginIfTainted(src, size);
+ if (o) {
+ o = ChainOrigin(o, stack);
+ *(dfsan_origin *)origin_for((void *)dst) = o;
+ }
+}
+
+// Copy the origins of the size bytes from src to dst. The source and target
+// memory ranges cannot be overlapped. This is used by memcpy. stack records the
+// stack trace of the memcpy. When dst and src are not 4-byte aligned properly,
+// origins at the unaligned address boundaries may be overwritten because four
+// contiguous bytes share the same origin.
+static void CopyOrigin(const void *dst, const void *src, uptr size,
+ StackTrace *stack) {
+ uptr d = (uptr)dst;
+ uptr beg = OriginAlignDown(d);
+ // Copy left unaligned origin if that memory is tainted.
+ if (beg < d) {
+ ChainAndWriteOriginIfTainted((uptr)src, beg + kOriginAlign - d, beg, stack);
+ beg += kOriginAlign;
+ }
+
+ uptr end = OriginAlignDown(d + size);
+ // If both ends fall into the same 4-byte slot, we are done.
+ if (end < beg)
+ return;
+
+ // Copy right unaligned origin if that memory is tainted.
+ if (end < d + size)
+ ChainAndWriteOriginIfTainted((uptr)src + (end - d), (d + size) - end, end,
+ stack);
+
+ if (beg >= end)
+ return;
+
+ // Align src up.
+ uptr src_a = OriginAlignUp((uptr)src);
+ dfsan_origin *src_o = origin_for((void *)src_a);
+ u32 *src_s = (u32 *)shadow_for((void *)src_a);
+ dfsan_origin *src_end = origin_for((void *)(src_a + (end - beg)));
+ dfsan_origin *dst_o = origin_for((void *)beg);
+ dfsan_origin last_src_o = 0;
+ dfsan_origin last_dst_o = 0;
+ for (; src_o < src_end; ++src_o, ++src_s, ++dst_o) {
+ if (!*src_s)
+ continue;
+ if (*src_o != last_src_o) {
+ last_src_o = *src_o;
+ last_dst_o = ChainOrigin(last_src_o, stack);
+ }
+ *dst_o = last_dst_o;
+ }
+}
+
+// Copy the origins of the size bytes from src to dst. The source and target
+// memory ranges may be overlapped. So the copy is done in a reverse order.
+// This is used by memmove. stack records the stack trace of the memmove.
+static void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
+ StackTrace *stack) {
+ uptr d = (uptr)dst;
+ uptr end = OriginAlignDown(d + size);
+
+ // Copy right unaligned origin if that memory is tainted.
+ if (end < d + size)
+ ChainAndWriteOriginIfTainted((uptr)src + (end - d), (d + size) - end, end,
+ stack);
+
+ uptr beg = OriginAlignDown(d);
+
+ if (beg + kOriginAlign < end) {
+ // Align src up.
+ uptr src_a = OriginAlignUp((uptr)src);
+ void *src_end = (void *)(src_a + end - beg - kOriginAlign);
+ dfsan_origin *src_end_o = origin_for(src_end);
+ u32 *src_end_s = (u32 *)shadow_for(src_end);
+ dfsan_origin *src_begin_o = origin_for((void *)src_a);
+ dfsan_origin *dst = origin_for((void *)(end - kOriginAlign));
+ dfsan_origin last_src_o = 0;
+ dfsan_origin last_dst_o = 0;
+ for (; src_end_o >= src_begin_o; --src_end_o, --src_end_s, --dst) {
+ if (!*src_end_s)
+ continue;
+ if (*src_end_o != last_src_o) {
+ last_src_o = *src_end_o;
+ last_dst_o = ChainOrigin(last_src_o, stack);
+ }
+ *dst = last_dst_o;
+ }
+ }
+
+ // Copy left unaligned origin if that memory is tainted.
+ if (beg < d)
+ ChainAndWriteOriginIfTainted((uptr)src, beg + kOriginAlign - d, beg, stack);
+}
+
+// Copy or move the origins of the len bytes from src to dst. The source and
+// target memory ranges may or may not be overlapped. This is used by memory
+// transfer operations. stack records the stack trace of the memory transfer
+// operation.
+static void MoveOrigin(const void *dst, const void *src, uptr size,
+ StackTrace *stack) {
+ // Validate address regions.
+ if (!MEM_IS_SHADOW(shadow_for(dst)) ||
+ !MEM_IS_SHADOW(shadow_for((void *)((uptr)dst + size))) ||
+ !MEM_IS_SHADOW(shadow_for(src)) ||
+ !MEM_IS_SHADOW(shadow_for((void *)((uptr)src + size)))) {
+ CHECK(false);
+ return;
+ }
+ // If destination origin range overlaps with source origin range, move
+ // origins by copying origins in a reverse order; otherwise, copy origins in
+ // a normal order. The orders of origin transfer are consistent with the
+ // orders of how memcpy and memmove transfer user data.
+ uptr src_aligned_beg = reinterpret_cast<uptr>(src) & ~3UL;
+ uptr src_aligned_end = (reinterpret_cast<uptr>(src) + size) & ~3UL;
+ uptr dst_aligned_beg = reinterpret_cast<uptr>(dst) & ~3UL;
+ if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
+ return ReverseCopyOrigin(dst, src, size, stack);
+ return CopyOrigin(dst, src, size, stack);
+}
+
+// Set the size bytes from the addres dst to be the origin value.
+static void SetOrigin(const void *dst, uptr size, u32 origin) {
+ if (size == 0)
+ return;
+
+ // Origin mapping is 4 bytes per 4 bytes of application memory.
+ // Here we extend the range such that its left and right bounds are both
+ // 4 byte aligned.
+ uptr x = unaligned_origin_for((uptr)dst);
+ uptr beg = OriginAlignDown(x);
+ uptr end = OriginAlignUp(x + size); // align up.
+ u64 origin64 = ((u64)origin << 32) | origin;
+ // This is like memset, but the value is 32-bit. We unroll by 2 to write
+ // 64 bits at once. May want to unroll further to get 128-bit stores.
+ if (beg & 7ULL) {
+ if (*(u32 *)beg != origin)
+ *(u32 *)beg = origin;
+ beg += 4;
+ }
+ for (uptr addr = beg; addr < (end & ~7UL); addr += 8) {
+ if (*(u64 *)addr == origin64)
+ continue;
+ *(u64 *)addr = origin64;
+ }
+ if (end & 7ULL)
+ if (*(u32 *)(end - kOriginAlign) != origin)
+ *(u32 *)(end - kOriginAlign) = origin;
+}
+
+static void WriteShadowInRange(dfsan_label label, uptr beg_shadow_addr,
+ uptr end_shadow_addr) {
+ // TODO: After changing dfsan_label to 8bit, use internal_memset when label
+ // is not 0.
+ dfsan_label *labelp = (dfsan_label *)beg_shadow_addr;
+ if (label) {
+ for (; (uptr)labelp < end_shadow_addr; ++labelp) *labelp = label;
+ return;
+ }
+
+ for (; (uptr)labelp < end_shadow_addr; ++labelp) {
// Don't write the label if it is already the value we need it to be.
// In a program where most addresses are not labeled, it is common that
// a page of shadow memory is entirely zeroed. The Linux copy-on-write
// the value written does not change the value in memory. Avoiding the
// write when both |label| and |*labelp| are zero dramatically reduces
// the amount of real memory used by large programs.
- if (label == *labelp)
+ if (!*labelp)
continue;
- *labelp = label;
+ *labelp = 0;
+ }
+}
+
+static void WriteShadowWithSize(dfsan_label label, uptr shadow_addr,
+ uptr size) {
+ WriteShadowInRange(label, shadow_addr, shadow_addr + size * sizeof(label));
+}
+
+#define RET_CHAIN_ORIGIN(id) \
+ GET_CALLER_PC_BP_SP; \
+ (void)sp; \
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp); \
+ return ChainOrigin(id, &stack);
+
+// Return a new origin chain with the previous ID id and the current stack
+// trace.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
+__dfsan_chain_origin(dfsan_origin id) {
+ RET_CHAIN_ORIGIN(id)
+}
+
+// Return a new origin chain with the previous ID id and the current stack
+// trace if the label is tainted.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
+__dfsan_chain_origin_if_tainted(dfsan_label label, dfsan_origin id) {
+ if (!label)
+ return id;
+ RET_CHAIN_ORIGIN(id)
+}
+
+// Copy or move the origins of the len bytes from src to dst.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_mem_origin_transfer(
+ const void *dst, const void *src, uptr len) {
+ if (src == dst)
+ return;
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ MoveOrigin(dst, src, len, &stack);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(const void *dst,
+ const void *src,
+ uptr len) {
+ __dfsan_mem_origin_transfer(dst, src, len);
+}
+
+namespace __dfsan {
+
+bool dfsan_inited = false;
+bool dfsan_init_is_running = false;
+
+void dfsan_copy_memory(void *dst, const void *src, uptr size) {
+ internal_memcpy(dst, src, size);
+ internal_memcpy((void *)shadow_for(dst), (const void *)shadow_for(src),
+ size * sizeof(dfsan_label));
+ if (dfsan_get_track_origins())
+ dfsan_mem_origin_transfer(dst, src, size);
+}
+
+} // namespace __dfsan
+
+// If the label s is tainted, set the size bytes from the address p to be a new
+// origin chain with the previous ID o and the current stack trace. This is
+// used by instrumentation to reduce code size when too much code is inserted.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
+ dfsan_label s, void *p, uptr size, dfsan_origin o) {
+ if (UNLIKELY(s)) {
+ GET_CALLER_PC_BP_SP;
+ (void)sp;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ SetOrigin(p, size, ChainOrigin(o, &stack));
}
}
+// Releases the pages within the origin address range.
+static void ReleaseOrigins(void *addr, uptr size) {
+ const uptr beg_origin_addr = (uptr)__dfsan::origin_for(addr);
+ const void *end_addr = (void *)((uptr)addr + size);
+ const uptr end_origin_addr = (uptr)__dfsan::origin_for(end_addr);
+
+ if (end_origin_addr - beg_origin_addr <
+ common_flags()->clear_shadow_mmap_threshold)
+ return;
+
+ const uptr page_size = GetPageSizeCached();
+ const uptr beg_aligned = RoundUpTo(beg_origin_addr, page_size);
+ const uptr end_aligned = RoundDownTo(end_origin_addr, page_size);
+
+ if (!MmapFixedSuperNoReserve(beg_aligned, end_aligned - beg_aligned))
+ Die();
+}
+
+// Releases the pages within the shadow address range, and sets
+// the shadow addresses not on the pages to be 0.
+static void ReleaseOrClearShadows(void *addr, uptr size) {
+ const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(addr);
+ const void *end_addr = (void *)((uptr)addr + size);
+ const uptr end_shadow_addr = (uptr)__dfsan::shadow_for(end_addr);
+
+ if (end_shadow_addr - beg_shadow_addr <
+ common_flags()->clear_shadow_mmap_threshold)
+ return WriteShadowWithSize(0, beg_shadow_addr, size);
+
+ const uptr page_size = GetPageSizeCached();
+ const uptr beg_aligned = RoundUpTo(beg_shadow_addr, page_size);
+ const uptr end_aligned = RoundDownTo(end_shadow_addr, page_size);
+
+ if (beg_aligned >= end_aligned) {
+ WriteShadowWithSize(0, beg_shadow_addr, size);
+ } else {
+ if (beg_aligned != beg_shadow_addr)
+ WriteShadowInRange(0, beg_shadow_addr, beg_aligned);
+ if (end_aligned != end_shadow_addr)
+ WriteShadowInRange(0, end_aligned, end_shadow_addr);
+ if (!MmapFixedSuperNoReserve(beg_aligned, end_aligned - beg_aligned))
+ Die();
+ }
+}
+
+void SetShadow(dfsan_label label, void *addr, uptr size, dfsan_origin origin) {
+ if (0 != label) {
+ const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(addr);
+ WriteShadowWithSize(label, beg_shadow_addr, size);
+ if (dfsan_get_track_origins())
+ SetOrigin(addr, size, origin);
+ return;
+ }
+
+ if (dfsan_get_track_origins())
+ ReleaseOrigins(addr, size);
+
+ ReleaseOrClearShadows(addr, size);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_set_label(
+ dfsan_label label, dfsan_origin origin, void *addr, uptr size) {
+ SetShadow(label, addr, size, origin);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
void dfsan_set_label(dfsan_label label, void *addr, uptr size) {
- __dfsan_set_label(label, addr, size);
+ dfsan_origin init_origin = 0;
+ if (label && dfsan_get_track_origins()) {
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ init_origin = ChainOrigin(0, &stack, true);
+ }
+ SetShadow(label, addr, size, init_origin);
}
SANITIZER_INTERFACE_ATTRIBUTE
void dfsan_add_label(dfsan_label label, void *addr, uptr size) {
+ if (0 == label)
+ return;
+
+ if (dfsan_get_track_origins()) {
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ dfsan_origin init_origin = ChainOrigin(0, &stack, true);
+ SetOrigin(addr, size, init_origin);
+ }
+
for (dfsan_label *labelp = shadow_for(addr); size != 0; --size, ++labelp)
- if (*labelp != label)
- *labelp = __dfsan_union(*labelp, label);
+ *labelp |= label;
}
// Unlike the other dfsan interface functions the behavior of this function
return data_label;
}
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfso_dfsan_get_label(
+ long data, dfsan_label data_label, dfsan_label *ret_label,
+ dfsan_origin data_origin, dfsan_origin *ret_origin) {
+ *ret_label = 0;
+ *ret_origin = 0;
+ return data_label;
+}
+
+// This function is used if dfsan_get_origin is called when origin tracking is
+// off.
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfsw_dfsan_get_origin(
+ long data, dfsan_label data_label, dfsan_label *ret_label) {
+ *ret_label = 0;
+ return 0;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfso_dfsan_get_origin(
+ long data, dfsan_label data_label, dfsan_label *ret_label,
+ dfsan_origin data_origin, dfsan_origin *ret_origin) {
+ *ret_label = 0;
+ *ret_origin = 0;
+ return data_origin;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
dfsan_read_label(const void *addr, uptr size) {
if (size == 0)
return __dfsan_union_load(shadow_for(addr), size);
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE
-const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label) {
- return &__dfsan_label_info[label];
+SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
+dfsan_read_origin_of_first_taint(const void *addr, uptr size) {
+ return GetOriginIfTainted((uptr)addr, size);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_label_origin(dfsan_label label,
+ dfsan_origin origin,
+ void *addr,
+ uptr size) {
+ __dfsan_set_label(label, origin, addr, size);
}
extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
dfsan_has_label(dfsan_label label, dfsan_label elem) {
- if (label == elem)
- return true;
- const dfsan_label_info *info = dfsan_get_label_info(label);
- if (info->l1 != 0) {
- return dfsan_has_label(info->l1, elem) || dfsan_has_label(info->l2, elem);
- } else {
- return false;
+ return (label & elem) == elem;
+}
+
+class Decorator : public __sanitizer::SanitizerCommonDecorator {
+ public:
+ Decorator() : SanitizerCommonDecorator() {}
+ const char *Origin() const { return Magenta(); }
+};
+
+namespace {
+
+void PrintNoOriginTrackingWarning() {
+ Decorator d;
+ Printf(
+ " %sDFSan: origin tracking is not enabled. Did you specify the "
+ "-dfsan-track-origins=1 option?%s\n",
+ d.Warning(), d.Default());
+}
+
+void PrintNoTaintWarning(const void *address) {
+ Decorator d;
+ Printf(" %sDFSan: no tainted value at %x%s\n", d.Warning(), address,
+ d.Default());
+}
+
+void PrintInvalidOriginWarning(dfsan_label label, const void *address) {
+ Decorator d;
+ Printf(
+ " %sTaint value 0x%x (at %p) has invalid origin tracking. This can "
+ "be a DFSan bug.%s\n",
+ d.Warning(), label, address, d.Default());
+}
+
+bool PrintOriginTraceToStr(const void *addr, const char *description,
+ InternalScopedString *out) {
+ CHECK(out);
+ CHECK(dfsan_get_track_origins());
+ Decorator d;
+
+ const dfsan_label label = *__dfsan::shadow_for(addr);
+ CHECK(label);
+
+ const dfsan_origin origin = *__dfsan::origin_for(addr);
+
+ out->append(" %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
+ d.Origin(), label, addr, description ? description : "",
+ d.Default());
+
+ Origin o = Origin::FromRawId(origin);
+ bool found = false;
+
+ while (o.isChainedOrigin()) {
+ StackTrace stack;
+ dfsan_origin origin_id = o.raw_id();
+ o = o.getNextChainedOrigin(&stack);
+ if (o.isChainedOrigin())
+ out->append(
+ " %sOrigin value: 0x%x, Taint value was stored to memory at%s\n",
+ d.Origin(), origin_id, d.Default());
+ else
+ out->append(" %sOrigin value: 0x%x, Taint value was created at%s\n",
+ d.Origin(), origin_id, d.Default());
+
+ // Includes a trailing newline, so no need to add it again.
+ stack.PrintTo(out);
+ found = true;
}
+
+ return found;
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
-dfsan_has_label_with_desc(dfsan_label label, const char *desc) {
- const dfsan_label_info *info = dfsan_get_label_info(label);
- if (info->l1 != 0) {
- return dfsan_has_label_with_desc(info->l1, desc) ||
- dfsan_has_label_with_desc(info->l2, desc);
- } else {
- return internal_strcmp(desc, info->desc) == 0;
+} // namespace
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
+ const void *addr, const char *description) {
+ if (!dfsan_get_track_origins()) {
+ PrintNoOriginTrackingWarning();
+ return;
+ }
+
+ const dfsan_label label = *__dfsan::shadow_for(addr);
+ if (!label) {
+ PrintNoTaintWarning(addr);
+ return;
}
+
+ InternalScopedString trace;
+ bool success = PrintOriginTraceToStr(addr, description, &trace);
+
+ if (trace.length())
+ Printf("%s", trace.data());
+
+ if (!success)
+ PrintInvalidOriginWarning(label, addr);
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
-dfsan_get_label_count(void) {
- dfsan_label max_label_allocated =
- atomic_load(&__dfsan_last_label, memory_order_relaxed);
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE size_t
+dfsan_sprint_origin_trace(const void *addr, const char *description,
+ char *out_buf, size_t out_buf_size) {
+ CHECK(out_buf);
- return static_cast<uptr>(max_label_allocated);
+ if (!dfsan_get_track_origins()) {
+ PrintNoOriginTrackingWarning();
+ return 0;
+ }
+
+ const dfsan_label label = *__dfsan::shadow_for(addr);
+ if (!label) {
+ PrintNoTaintWarning(addr);
+ return 0;
+ }
+
+ InternalScopedString trace;
+ bool success = PrintOriginTraceToStr(addr, description, &trace);
+
+ if (!success) {
+ PrintInvalidOriginWarning(label, addr);
+ return 0;
+ }
+
+ if (out_buf_size) {
+ internal_strncpy(out_buf, trace.data(), out_buf_size - 1);
+ out_buf[out_buf_size - 1] = '\0';
+ }
+
+ return trace.length();
}
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
-dfsan_dump_labels(int fd) {
- dfsan_label last_label =
- atomic_load(&__dfsan_last_label, memory_order_relaxed);
-
- for (uptr l = 1; l <= last_label; ++l) {
- char buf[64];
- internal_snprintf(buf, sizeof(buf), "%u %u %u ", l,
- __dfsan_label_info[l].l1, __dfsan_label_info[l].l2);
- WriteToFile(fd, buf, internal_strlen(buf));
- if (__dfsan_label_info[l].l1 == 0 && __dfsan_label_info[l].desc) {
- WriteToFile(fd, __dfsan_label_info[l].desc,
- internal_strlen(__dfsan_label_info[l].desc));
- }
- WriteToFile(fd, "\n", 1);
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
+dfsan_get_init_origin(const void *addr) {
+ if (!dfsan_get_track_origins())
+ return 0;
+
+ const dfsan_label label = *__dfsan::shadow_for(addr);
+ if (!label)
+ return 0;
+
+ const dfsan_origin origin = *__dfsan::origin_for(addr);
+
+ Origin o = Origin::FromRawId(origin);
+ dfsan_origin origin_id = o.raw_id();
+ while (o.isChainedOrigin()) {
+ StackTrace stack;
+ origin_id = o.raw_id();
+ o = o.getNextChainedOrigin(&stack);
+ }
+ return origin_id;
+}
+
+void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
+ void *context,
+ bool request_fast,
+ u32 max_depth) {
+ using namespace __dfsan;
+ DFsanThread *t = GetCurrentThread();
+ if (!t || !StackTrace::WillUseFastUnwind(request_fast)) {
+ return Unwind(max_depth, pc, bp, context, 0, 0, false);
}
+ Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_stack_trace() {
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ stack.Print();
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE size_t
+dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size) {
+ CHECK(out_buf);
+ GET_CALLER_PC_BP;
+ GET_STORE_STACK_TRACE_PC_BP(pc, bp);
+ return stack.PrintTo(out_buf, out_buf_size);
}
void Flags::SetDefaults() {
static void InitializeFlags() {
SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.intercept_tls_get_addr = true;
+ OverrideCommonFlags(cf);
+ }
flags().SetDefaults();
FlagParser parser;
if (common_flags()->help) parser.PrintFlagDescriptions();
}
-static void InitializePlatformEarly() {
- AvoidCVE_2016_2143();
-#ifdef DFSAN_RUNTIME_VMA
- __dfsan::vmaSize =
- (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
- if (__dfsan::vmaSize == 39 || __dfsan::vmaSize == 42 ||
- __dfsan::vmaSize == 48) {
- __dfsan_shadow_ptr_mask = ShadowMask();
- } else {
- Printf("FATAL: DataFlowSanitizer: unsupported VMA range\n");
- Printf("FATAL: Found %d - Supported 39, 42, and 48\n", __dfsan::vmaSize);
- Die();
+SANITIZER_INTERFACE_ATTRIBUTE
+void dfsan_clear_arg_tls(uptr offset, uptr size) {
+ internal_memset((void *)((uptr)__dfsan_arg_tls + offset), 0, size);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void dfsan_clear_thread_local_state() {
+ internal_memset(__dfsan_arg_tls, 0, sizeof(__dfsan_arg_tls));
+ internal_memset(__dfsan_retval_tls, 0, sizeof(__dfsan_retval_tls));
+
+ if (dfsan_get_track_origins()) {
+ internal_memset(__dfsan_arg_origin_tls, 0, sizeof(__dfsan_arg_origin_tls));
+ internal_memset(&__dfsan_retval_origin_tls, 0,
+ sizeof(__dfsan_retval_origin_tls));
}
-#endif
}
-static void dfsan_fini() {
- if (internal_strcmp(flags().dump_labels_at_exit, "") != 0) {
- fd_t fd = OpenFile(flags().dump_labels_at_exit, WrOnly);
- if (fd == kInvalidFd) {
- Report("WARNING: DataFlowSanitizer: unable to open output file %s\n",
- flags().dump_labels_at_exit);
- return;
+extern "C" void dfsan_flush() {
+ const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+ for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
+ uptr start = kMemoryLayout[i].start;
+ uptr end = kMemoryLayout[i].end;
+ uptr size = end - start;
+ MappingDesc::Type type = kMemoryLayout[i].type;
+
+ if (type != MappingDesc::SHADOW && type != MappingDesc::ORIGIN)
+ continue;
+
+ // Check if the segment should be mapped based on platform constraints.
+ if (start >= maxVirtualAddress)
+ continue;
+
+ if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name)) {
+ Printf("FATAL: DataFlowSanitizer: failed to clear memory region\n");
+ Die();
}
+ }
+}
+
+// TODO: CheckMemoryLayoutSanity is based on msan.
+// Consider refactoring these into a shared implementation.
+static void CheckMemoryLayoutSanity() {
+ uptr prev_end = 0;
+ for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
+ uptr start = kMemoryLayout[i].start;
+ uptr end = kMemoryLayout[i].end;
+ MappingDesc::Type type = kMemoryLayout[i].type;
+ CHECK_LT(start, end);
+ CHECK_EQ(prev_end, start);
+ CHECK(addr_is_type(start, type));
+ CHECK(addr_is_type((start + end) / 2, type));
+ CHECK(addr_is_type(end - 1, type));
+ if (type == MappingDesc::APP) {
+ uptr addr = start;
+ CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
+ CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
+ CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
- Report("INFO: DataFlowSanitizer: dumping labels to %s\n",
- flags().dump_labels_at_exit);
- dfsan_dump_labels(fd);
- CloseFile(fd);
+ addr = (start + end) / 2;
+ CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
+ CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
+ CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
+
+ addr = end - 1;
+ CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
+ CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
+ CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
+ }
+ prev_end = end;
}
}
-extern "C" void dfsan_flush() {
- UnmapOrDie((void*)ShadowAddr(), UnusedAddr() - ShadowAddr());
- if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr()))
- Die();
+// TODO: CheckMemoryRangeAvailability is based on msan.
+// Consider refactoring these into a shared implementation.
+static bool CheckMemoryRangeAvailability(uptr beg, uptr size) {
+ if (size > 0) {
+ uptr end = beg + size - 1;
+ if (!MemoryRangeIsAvailable(beg, end)) {
+ Printf("FATAL: Memory range %p - %p is not available.\n", beg, end);
+ return false;
+ }
+ }
+ return true;
+}
+
+// TODO: ProtectMemoryRange is based on msan.
+// Consider refactoring these into a shared implementation.
+static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) {
+ if (size > 0) {
+ void *addr = MmapFixedNoAccess(beg, size, name);
+ if (beg == 0 && addr) {
+ // Depending on the kernel configuration, we may not be able to protect
+ // the page at address zero.
+ uptr gap = 16 * GetPageSizeCached();
+ beg += gap;
+ size -= gap;
+ addr = MmapFixedNoAccess(beg, size, name);
+ }
+ if ((uptr)addr != beg) {
+ uptr end = beg + size - 1;
+ Printf("FATAL: Cannot protect memory range %p - %p (%s).\n", beg, end,
+ name);
+ return false;
+ }
+ }
+ return true;
}
-static void dfsan_init(int argc, char **argv, char **envp) {
+// TODO: InitShadow is based on msan.
+// Consider refactoring these into a shared implementation.
+bool InitShadow(bool init_origins) {
+ // Let user know mapping parameters first.
+ VPrintf(1, "dfsan_init %p\n", &__dfsan::dfsan_init);
+ for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
+ VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
+ kMemoryLayout[i].end - 1);
+
+ CheckMemoryLayoutSanity();
+
+ if (!MEM_IS_APP(&__dfsan::dfsan_init)) {
+ Printf("FATAL: Code %p is out of application range. Non-PIE build?\n",
+ (uptr)&__dfsan::dfsan_init);
+ return false;
+ }
+
+ const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
+
+ for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
+ uptr start = kMemoryLayout[i].start;
+ uptr end = kMemoryLayout[i].end;
+ uptr size = end - start;
+ MappingDesc::Type type = kMemoryLayout[i].type;
+
+ // Check if the segment should be mapped based on platform constraints.
+ if (start >= maxVirtualAddress)
+ continue;
+
+ bool map = type == MappingDesc::SHADOW ||
+ (init_origins && type == MappingDesc::ORIGIN);
+ bool protect = type == MappingDesc::INVALID ||
+ (!init_origins && type == MappingDesc::ORIGIN);
+ CHECK(!(map && protect));
+ if (!map && !protect)
+ CHECK(type == MappingDesc::APP);
+ if (map) {
+ if (!CheckMemoryRangeAvailability(start, size))
+ return false;
+ if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name))
+ return false;
+ if (common_flags()->use_madv_dontdump)
+ DontDumpShadowMemory(start, size);
+ }
+ if (protect) {
+ if (!CheckMemoryRangeAvailability(start, size))
+ return false;
+ if (!ProtectMemoryRange(start, size, kMemoryLayout[i].name))
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static void DFsanInit(int argc, char **argv, char **envp) {
+ CHECK(!dfsan_init_is_running);
+ if (dfsan_inited)
+ return;
+ dfsan_init_is_running = true;
+ SanitizerToolName = "DataflowSanitizer";
+
+ AvoidCVE_2016_2143();
+
InitializeFlags();
- ::InitializePlatformEarly();
+ CheckASLR();
- if (!MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr()))
- Die();
+ InitShadow(dfsan_get_track_origins());
+
+ initialize_interceptors();
- // Protect the region of memory we don't use, to preserve the one-to-one
- // mapping from application to shadow memory. But if ASLR is disabled, Linux
- // will load our executable in the middle of our unused region. This mostly
- // works so long as the program doesn't use too much memory. We support this
- // case by disabling memory protection when ASLR is disabled.
- uptr init_addr = (uptr)&dfsan_init;
- if (!(init_addr >= UnusedAddr() && init_addr < AppAddr()))
- MmapFixedNoAccess(UnusedAddr(), AppAddr() - UnusedAddr());
+ // Set up threads
+ DFsanTSDInit(DFsanTSDDtor);
- InitializeInterceptors();
+ dfsan_allocator_init();
- // Register the fini callback to run when the program terminates successfully
- // or it is killed by the runtime.
- Atexit(dfsan_fini);
- AddDieCallback(dfsan_fini);
+ DFsanThread *main_thread = DFsanThread::Create(nullptr, nullptr, nullptr);
+ SetCurrentThread(main_thread);
+ main_thread->ThreadStart();
- __dfsan_label_info[kInitializingLabel].desc = "<init label>";
+ dfsan_init_is_running = false;
+ dfsan_inited = true;
}
+namespace __dfsan {
+
+void dfsan_init() { DFsanInit(0, nullptr, nullptr); }
+
+} // namespace __dfsan
+
#if SANITIZER_CAN_USE_PREINIT_ARRAY
-__attribute__((section(".preinit_array"), used))
-static void (*dfsan_init_ptr)(int, char **, char **) = dfsan_init;
+__attribute__((section(".preinit_array"),
+ used)) static void (*dfsan_init_ptr)(int, char **,
+ char **) = DFsanInit;
#endif
#define DFSAN_H
#include "sanitizer_common/sanitizer_internal_defs.h"
+
#include "dfsan_platform.h"
+using __sanitizer::u32;
+using __sanitizer::u8;
using __sanitizer::uptr;
-using __sanitizer::u16;
// Copy declarations from public sanitizer/dfsan_interface.h header here.
-typedef u16 dfsan_label;
-
-struct dfsan_label_info {
- dfsan_label l1;
- dfsan_label l2;
- const char *desc;
- void *userdata;
-};
+typedef u8 dfsan_label;
+typedef u32 dfsan_origin;
extern "C" {
void dfsan_add_label(dfsan_label label, void *addr, uptr size);
void dfsan_set_label(dfsan_label label, void *addr, uptr size);
dfsan_label dfsan_read_label(const void *addr, uptr size);
dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2);
+// Zero out [offset, offset+size) from __dfsan_arg_tls.
+void dfsan_clear_arg_tls(uptr offset, uptr size);
+// Zero out the TLS storage.
+void dfsan_clear_thread_local_state();
+
+// Return the origin associated with the first taint byte in the size bytes
+// from the address addr.
+dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, uptr size);
+
+// Set the data within [addr, addr+size) with label and origin.
+void dfsan_set_label_origin(dfsan_label label, dfsan_origin origin, void *addr,
+ uptr size);
+
+// Copy or move the origins of the len bytes from src to dst.
+void dfsan_mem_origin_transfer(const void *dst, const void *src, uptr len);
} // extern "C"
template <typename T>
namespace __dfsan {
-void InitializeInterceptors();
+extern bool dfsan_inited;
+extern bool dfsan_init_is_running;
+
+void initialize_interceptors();
inline dfsan_label *shadow_for(void *ptr) {
- return (dfsan_label *) ((((uptr) ptr) & ShadowMask()) << 1);
+ return (dfsan_label *)MEM_TO_SHADOW(ptr);
}
inline const dfsan_label *shadow_for(const void *ptr) {
return shadow_for(const_cast<void *>(ptr));
}
-struct Flags {
-#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
-#include "dfsan_flags.inc"
-#undef DFSAN_FLAG
+inline uptr unaligned_origin_for(uptr ptr) { return MEM_TO_ORIGIN(ptr); }
- void SetDefaults();
-};
+inline dfsan_origin *origin_for(void *ptr) {
+ auto aligned_addr = unaligned_origin_for(reinterpret_cast<uptr>(ptr)) &
+ ~(sizeof(dfsan_origin) - 1);
+ return reinterpret_cast<dfsan_origin *>(aligned_addr);
+}
-extern Flags flags_data;
-inline Flags &flags() {
- return flags_data;
+inline const dfsan_origin *origin_for(const void *ptr) {
+ return origin_for(const_cast<void *>(ptr));
}
+void dfsan_copy_memory(void *dst, const void *src, uptr size);
+
+void dfsan_allocator_init();
+void dfsan_deallocate(void *ptr);
+
+void *dfsan_malloc(uptr size);
+void *dfsan_calloc(uptr nmemb, uptr size);
+void *dfsan_realloc(void *ptr, uptr size);
+void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size);
+void *dfsan_valloc(uptr size);
+void *dfsan_pvalloc(uptr size);
+void *dfsan_aligned_alloc(uptr alignment, uptr size);
+void *dfsan_memalign(uptr alignment, uptr size);
+int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size);
+
+void dfsan_init();
+
} // namespace __dfsan
#endif // DFSAN_H
dfsan_*
__dfsan_*
__dfsw_*
+__dfso_*
--- /dev/null
+//===-- dfsan_allocator.cpp -------------------------- --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataflowSanitizer.
+//
+// DataflowSanitizer allocator.
+//===----------------------------------------------------------------------===//
+
+#include "dfsan_allocator.h"
+
+#include "dfsan.h"
+#include "dfsan_flags.h"
+#include "dfsan_thread.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_errno.h"
+
+namespace __dfsan {
+
+struct Metadata {
+ uptr requested_size;
+};
+
+struct DFsanMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
+ void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
+};
+
+static const uptr kAllocatorSpace = 0x700000000000ULL;
+static const uptr kMaxAllowedMallocSize = 8UL << 30;
+
+struct AP64 { // Allocator64 parameters. Deliberately using a short name.
+ static const uptr kSpaceBeg = kAllocatorSpace;
+ static const uptr kSpaceSize = 0x40000000000; // 4T.
+ static const uptr kMetadataSize = sizeof(Metadata);
+ typedef DefaultSizeClassMap SizeClassMap;
+ typedef DFsanMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+ using AddressSpaceView = LocalAddressSpaceView;
+};
+
+typedef SizeClassAllocator64<AP64> PrimaryAllocator;
+
+typedef CombinedAllocator<PrimaryAllocator> Allocator;
+typedef Allocator::AllocatorCache AllocatorCache;
+
+static Allocator allocator;
+static AllocatorCache fallback_allocator_cache;
+static StaticSpinMutex fallback_mutex;
+
+static uptr max_malloc_size;
+
+void dfsan_allocator_init() {
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+ if (common_flags()->max_allocation_size_mb)
+ max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
+ kMaxAllowedMallocSize);
+ else
+ max_malloc_size = kMaxAllowedMallocSize;
+}
+
+AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
+ CHECK(ms);
+ CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
+ return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
+}
+
+void DFsanThreadLocalMallocStorage::CommitBack() {
+ allocator.SwallowCache(GetAllocatorCache(this));
+}
+
+static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
+ if (size > max_malloc_size) {
+ if (AllocatorMayReturnNull()) {
+ Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
+ size);
+ return nullptr;
+ }
+ BufferedStackTrace stack;
+ ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
+ }
+ DFsanThread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocated = allocator.Allocate(cache, size, alignment);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, size, alignment);
+ }
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportOutOfMemory(size, &stack);
+ }
+ Metadata *meta =
+ reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
+ meta->requested_size = size;
+ if (zeroise) {
+ internal_memset(allocated, 0, size);
+ dfsan_set_label(0, allocated, size);
+ } else if (flags().zero_in_malloc) {
+ dfsan_set_label(0, allocated, size);
+ }
+ return allocated;
+}
+
+void dfsan_deallocate(void *p) {
+ CHECK(p);
+ Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
+ uptr size = meta->requested_size;
+ meta->requested_size = 0;
+ if (flags().zero_in_free)
+ dfsan_set_label(0, p, size);
+ DFsanThread *t = GetCurrentThread();
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocator.Deallocate(cache, p);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, p);
+ }
+}
+
+void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
+ Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));
+ uptr old_size = meta->requested_size;
+ uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
+ if (new_size <= actually_allocated_size) {
+ // We are not reallocating here.
+ meta->requested_size = new_size;
+ if (new_size > old_size && flags().zero_in_malloc)
+ dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);
+ return old_p;
+ }
+ uptr memcpy_size = Min(new_size, old_size);
+ void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);
+ if (new_p) {
+ dfsan_copy_memory(new_p, old_p, memcpy_size);
+ dfsan_deallocate(old_p);
+ }
+ return new_p;
+}
+
+void *DFsanCalloc(uptr nmemb, uptr size) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportCallocOverflow(nmemb, size, &stack);
+ }
+ return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);
+}
+
+static uptr AllocationSize(const void *p) {
+ if (!p)
+ return 0;
+ const void *beg = allocator.GetBlockBegin(p);
+ if (beg != p)
+ return 0;
+ Metadata *b = (Metadata *)allocator.GetMetaData(p);
+ return b->requested_size;
+}
+
+void *dfsan_malloc(uptr size) {
+ return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
+}
+
+void *dfsan_calloc(uptr nmemb, uptr size) {
+ return SetErrnoOnNull(DFsanCalloc(nmemb, size));
+}
+
+void *dfsan_realloc(void *ptr, uptr size) {
+ if (!ptr)
+ return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
+ if (size == 0) {
+ dfsan_deallocate(ptr);
+ return nullptr;
+ }
+ return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));
+}
+
+void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportReallocArrayOverflow(nmemb, size, &stack);
+ }
+ return dfsan_realloc(ptr, nmemb * size);
+}
+
+void *dfsan_valloc(uptr size) {
+ return SetErrnoOnNull(
+ DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));
+}
+
+void *dfsan_pvalloc(uptr size) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportPvallocOverflow(size, &stack);
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));
+}
+
+void *dfsan_aligned_alloc(uptr alignment, uptr size) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
+ }
+ return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
+}
+
+void *dfsan_memalign(uptr alignment, uptr size) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ BufferedStackTrace stack;
+ ReportInvalidAllocationAlignment(alignment, &stack);
+ }
+ return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
+}
+
+int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ if (AllocatorMayReturnNull())
+ return errno_EINVAL;
+ BufferedStackTrace stack;
+ ReportInvalidPosixMemalignAlignment(alignment, &stack);
+ }
+ void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);
+ if (UNLIKELY(!ptr))
+ // OOM error is already taken care of by DFsanAllocate.
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+} // namespace __dfsan
+
+using namespace __dfsan;
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatAllocated];
+}
+
+uptr __sanitizer_get_heap_size() {
+ uptr stats[AllocatorStatCount];
+ allocator.GetStats(stats);
+ return stats[AllocatorStatMapped];
+}
+
+uptr __sanitizer_get_free_bytes() { return 1; }
+
+uptr __sanitizer_get_unmapped_bytes() { return 1; }
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
+
+uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
--- /dev/null
+//===-- dfsan_allocator.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataflowSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DFSAN_ALLOCATOR_H
+#define DFSAN_ALLOCATOR_H
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __dfsan {
+
+struct DFsanThreadLocalMallocStorage {
+ ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque.
+ void CommitBack();
+
+ private:
+ // These objects are allocated via mmap() and are zero-initialized.
+ DFsanThreadLocalMallocStorage() {}
+};
+
+} // namespace __dfsan
+#endif // DFSAN_ALLOCATOR_H
--- /dev/null
+//===-- dfsan_chained_origin_depot.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataFlowSanitizer.
+//
+// A storage for chained origins.
+//===----------------------------------------------------------------------===//
+
+#include "dfsan_chained_origin_depot.h"
+
+namespace __dfsan {
+
+static ChainedOriginDepot chainedOriginDepot;
+
+ChainedOriginDepot* GetChainedOriginDepot() { return &chainedOriginDepot; }
+
+} // namespace __dfsan
--- /dev/null
+//===-- dfsan_chained_origin_depot.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataFlowSanitizer.
+//
+// A storage for chained origins.
+//===----------------------------------------------------------------------===//
+
+#ifndef DFSAN_CHAINED_ORIGIN_DEPOT_H
+#define DFSAN_CHAINED_ORIGIN_DEPOT_H
+
+#include "sanitizer_common/sanitizer_chained_origin_depot.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __dfsan {
+
+ChainedOriginDepot* GetChainedOriginDepot();
+
+} // namespace __dfsan
+
+#endif // DFSAN_CHAINED_ORIGIN_DEPOT_H
-//===-- dfsan.cpp ---------------------------------------------------------===//
+//===-- dfsan_custom.cpp --------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// This file defines the custom functions listed in done_abilist.txt.
//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "sanitizer_common/sanitizer_linux.h"
-
-#include "dfsan/dfsan.h"
-
#include <arpa/inet.h>
#include <assert.h>
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/epoll.h>
#include <sys/resource.h>
#include <sys/select.h>
+#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
+#include "dfsan/dfsan.h"
+#include "dfsan/dfsan_chained_origin_depot.h"
+#include "dfsan/dfsan_flags.h"
+#include "dfsan/dfsan_thread.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
using namespace __dfsan;
#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) \
#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void f(__VA_ARGS__);
+// Async-safe, non-reentrant spin lock.
+class SignalSpinLocker {
+ public:
+ SignalSpinLocker() {
+ sigset_t all_set;
+ sigfillset(&all_set);
+ pthread_sigmask(SIG_SETMASK, &all_set, &saved_thread_mask_);
+ sigactions_mu.Lock();
+ }
+ ~SignalSpinLocker() {
+ sigactions_mu.Unlock();
+ pthread_sigmask(SIG_SETMASK, &saved_thread_mask_, nullptr);
+ }
+
+ private:
+ static StaticSpinMutex sigactions_mu;
+ sigset_t saved_thread_mask_;
+
+ SignalSpinLocker(const SignalSpinLocker &) = delete;
+ SignalSpinLocker &operator=(const SignalSpinLocker &) = delete;
+};
+
+StaticSpinMutex SignalSpinLocker::sigactions_mu;
+
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE int
__dfsw_stat(const char *path, struct stat *buf, dfsan_label path_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_stat(
+ const char *path, struct stat *buf, dfsan_label path_label,
+ dfsan_label buf_label, dfsan_label *ret_label, dfsan_origin path_origin,
+ dfsan_origin buf_origin, dfsan_origin *ret_origin) {
+ int ret = __dfsw_stat(path, buf, path_label, buf_label, ret_label);
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_fstat(int fd, struct stat *buf,
dfsan_label fd_label,
dfsan_label buf_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_fstat(
+ int fd, struct stat *buf, dfsan_label fd_label, dfsan_label buf_label,
+ dfsan_label *ret_label, dfsan_origin fd_origin, dfsan_origin buf_origin,
+ dfsan_origin *ret_origin) {
+ int ret = __dfsw_fstat(fd, buf, fd_label, buf_label, ret_label);
+ return ret;
+}
+
+static char *dfsan_strchr_with_label(const char *s, int c, size_t *bytes_read,
+ dfsan_label s_label, dfsan_label c_label,
+ dfsan_label *ret_label) {
+ char *match_pos = nullptr;
+ for (size_t i = 0;; ++i) {
+ if (s[i] == c || s[i] == 0) {
+ // If s[i] is the \0 at the end of the string, and \0 is not the
+ // character we are searching for, then return null.
+ *bytes_read = i + 1;
+ match_pos = s[i] == 0 && c != 0 ? nullptr : const_cast<char *>(s + i);
+ break;
+ }
+ }
+ if (flags().strict_data_dependencies)
+ *ret_label = s_label;
+ else
+ *ret_label = dfsan_union(dfsan_read_label(s, *bytes_read),
+ dfsan_union(s_label, c_label));
+ return match_pos;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strchr(const char *s, int c,
dfsan_label s_label,
dfsan_label c_label,
dfsan_label *ret_label) {
- for (size_t i = 0;; ++i) {
- if (s[i] == c || s[i] == 0) {
- if (flags().strict_data_dependencies) {
- *ret_label = s_label;
+ size_t bytes_read;
+ return dfsan_strchr_with_label(s, c, &bytes_read, s_label, c_label,
+ ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strchr(
+ const char *s, int c, dfsan_label s_label, dfsan_label c_label,
+ dfsan_label *ret_label, dfsan_origin s_origin, dfsan_origin c_origin,
+ dfsan_origin *ret_origin) {
+ size_t bytes_read;
+ char *r =
+ dfsan_strchr_with_label(s, c, &bytes_read, s_label, c_label, ret_label);
+ if (flags().strict_data_dependencies) {
+ *ret_origin = s_origin;
+ } else if (*ret_label) {
+ dfsan_origin o = dfsan_read_origin_of_first_taint(s, bytes_read);
+ *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+ }
+ return r;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strpbrk(const char *s,
+ const char *accept,
+ dfsan_label s_label,
+ dfsan_label accept_label,
+ dfsan_label *ret_label) {
+ const char *ret = strpbrk(s, accept);
+ if (flags().strict_data_dependencies) {
+ *ret_label = ret ? s_label : 0;
+ } else {
+ size_t s_bytes_read = (ret ? ret - s : strlen(s)) + 1;
+ *ret_label =
+ dfsan_union(dfsan_read_label(s, s_bytes_read),
+ dfsan_union(dfsan_read_label(accept, strlen(accept) + 1),
+ dfsan_union(s_label, accept_label)));
+ }
+ return const_cast<char *>(ret);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strpbrk(
+ const char *s, const char *accept, dfsan_label s_label,
+ dfsan_label accept_label, dfsan_label *ret_label, dfsan_origin s_origin,
+ dfsan_origin accept_origin, dfsan_origin *ret_origin) {
+ const char *ret = __dfsw_strpbrk(s, accept, s_label, accept_label, ret_label);
+ if (flags().strict_data_dependencies) {
+ if (ret)
+ *ret_origin = s_origin;
+ } else {
+ if (*ret_label) {
+ size_t s_bytes_read = (ret ? ret - s : strlen(s)) + 1;
+ dfsan_origin o = dfsan_read_origin_of_first_taint(s, s_bytes_read);
+ if (o) {
+ *ret_origin = o;
} else {
- *ret_label = dfsan_union(dfsan_read_label(s, i + 1),
- dfsan_union(s_label, c_label));
+ o = dfsan_read_origin_of_first_taint(accept, strlen(accept) + 1);
+ *ret_origin = o ? o : (s_label ? s_origin : accept_origin);
}
+ }
+ }
+ return const_cast<char *>(ret);
+}
- // If s[i] is the \0 at the end of the string, and \0 is not the
- // character we are searching for, then return null.
- if (s[i] == 0 && c != 0) {
- return nullptr;
- }
- return const_cast<char *>(s + i);
+static int dfsan_memcmp_bcmp(const void *s1, const void *s2, size_t n,
+ size_t *bytes_read) {
+ const char *cs1 = (const char *) s1, *cs2 = (const char *) s2;
+ for (size_t i = 0; i != n; ++i) {
+ if (cs1[i] != cs2[i]) {
+ *bytes_read = i + 1;
+ return cs1[i] - cs2[i];
}
}
+ *bytes_read = n;
+ return 0;
+}
+
+static dfsan_label dfsan_get_memcmp_label(const void *s1, const void *s2,
+ size_t pos) {
+ if (flags().strict_data_dependencies)
+ return 0;
+ return dfsan_union(dfsan_read_label(s1, pos), dfsan_read_label(s2, pos));
+}
+
+static void dfsan_get_memcmp_origin(const void *s1, const void *s2, size_t pos,
+ dfsan_label *ret_label,
+ dfsan_origin *ret_origin) {
+ *ret_label = dfsan_get_memcmp_label(s1, s2, pos);
+ if (*ret_label == 0)
+ return;
+ dfsan_origin o = dfsan_read_origin_of_first_taint(s1, pos);
+ *ret_origin = o ? o : dfsan_read_origin_of_first_taint(s2, pos);
+}
+
+static int dfsan_memcmp_bcmp_label(const void *s1, const void *s2, size_t n,
+ dfsan_label *ret_label) {
+ size_t bytes_read;
+ int r = dfsan_memcmp_bcmp(s1, s2, n, &bytes_read);
+ *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+ return r;
+}
+
+static int dfsan_memcmp_bcmp_origin(const void *s1, const void *s2, size_t n,
+ dfsan_label *ret_label,
+ dfsan_origin *ret_origin) {
+ size_t bytes_read;
+ int r = dfsan_memcmp_bcmp(s1, s2, n, &bytes_read);
+ dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+ return r;
}
DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, uptr caller_pc,
dfsan_label s1_label, dfsan_label s2_label,
dfsan_label n_label)
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_memcmp, uptr caller_pc,
+ const void *s1, const void *s2, size_t n,
+ dfsan_label s1_label, dfsan_label s2_label,
+ dfsan_label n_label, dfsan_origin s1_origin,
+ dfsan_origin s2_origin, dfsan_origin n_origin)
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_memcmp(const void *s1, const void *s2,
size_t n, dfsan_label s1_label,
dfsan_label s2_label,
dfsan_label *ret_label) {
CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, GET_CALLER_PC(), s1, s2, n,
s1_label, s2_label, n_label);
- const char *cs1 = (const char *) s1, *cs2 = (const char *) s2;
- for (size_t i = 0; i != n; ++i) {
- if (cs1[i] != cs2[i]) {
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(cs1, i + 1),
- dfsan_read_label(cs2, i + 1));
- }
- return cs1[i] - cs2[i];
- }
- }
+ return dfsan_memcmp_bcmp_label(s1, s2, n, ret_label);
+}
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(cs1, n),
- dfsan_read_label(cs2, n));
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_memcmp(
+ const void *s1, const void *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
+ CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_memcmp, GET_CALLER_PC(), s1,
+ s2, n, s1_label, s2_label, n_label, s1_origin,
+ s2_origin, n_origin);
+ return dfsan_memcmp_bcmp_origin(s1, s2, n, ret_label, ret_origin);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_bcmp(const void *s1, const void *s2,
+ size_t n, dfsan_label s1_label,
+ dfsan_label s2_label,
+ dfsan_label n_label,
+ dfsan_label *ret_label) {
+ return dfsan_memcmp_bcmp_label(s1, s2, n, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_bcmp(
+ const void *s1, const void *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
+ return dfsan_memcmp_bcmp_origin(s1, s2, n, ret_label, ret_origin);
+}
+
+// When n == 0, compare strings without byte limit.
+// When n > 0, compare the first (at most) n bytes of s1 and s2.
+static int dfsan_strncmp(const char *s1, const char *s2, size_t n,
+ size_t *bytes_read) {
+ for (size_t i = 0;; ++i) {
+ if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0 || (n > 0 && i == n - 1)) {
+ *bytes_read = i + 1;
+ return s1[i] - s2[i];
+ }
}
- return 0;
}
DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, uptr caller_pc,
const char *s1, const char *s2,
dfsan_label s1_label, dfsan_label s2_label)
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strcmp, uptr caller_pc,
+ const char *s1, const char *s2,
+ dfsan_label s1_label, dfsan_label s2_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin)
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strcmp(const char *s1, const char *s2,
dfsan_label s1_label,
dfsan_label s2_label,
dfsan_label *ret_label) {
CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, GET_CALLER_PC(), s1, s2,
s1_label, s2_label);
- for (size_t i = 0;; ++i) {
- if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0) {
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
- dfsan_read_label(s2, i + 1));
- }
- return s1[i] - s2[i];
- }
- }
- return 0;
+ size_t bytes_read;
+ int r = dfsan_strncmp(s1, s2, 0, &bytes_read);
+ *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+ return r;
}
-SANITIZER_INTERFACE_ATTRIBUTE int
-__dfsw_strcasecmp(const char *s1, const char *s2, dfsan_label s1_label,
- dfsan_label s2_label, dfsan_label *ret_label) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strcmp(
+ const char *s1, const char *s2, dfsan_label s1_label, dfsan_label s2_label,
+ dfsan_label *ret_label, dfsan_origin s1_origin, dfsan_origin s2_origin,
+ dfsan_origin *ret_origin) {
+ CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strcmp, GET_CALLER_PC(), s1,
+ s2, s1_label, s2_label, s1_origin, s2_origin);
+ size_t bytes_read;
+ int r = dfsan_strncmp(s1, s2, 0, &bytes_read);
+ dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+ return r;
+}
+
+// When n == 0, compare strings without byte limit.
+// When n > 0, compare the first (at most) n bytes of s1 and s2.
+static int dfsan_strncasecmp(const char *s1, const char *s2, size_t n,
+ size_t *bytes_read) {
for (size_t i = 0;; ++i) {
char s1_lower = tolower(s1[i]);
char s2_lower = tolower(s2[i]);
- if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0) {
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
- dfsan_read_label(s2, i + 1));
- }
+ if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0 ||
+ (n > 0 && i == n - 1)) {
+ *bytes_read = i + 1;
return s1_lower - s2_lower;
}
}
- return 0;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strcasecmp(const char *s1,
+ const char *s2,
+ dfsan_label s1_label,
+ dfsan_label s2_label,
+ dfsan_label *ret_label) {
+ size_t bytes_read;
+ int r = dfsan_strncasecmp(s1, s2, 0, &bytes_read);
+ *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+ return r;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strcasecmp(
+ const char *s1, const char *s2, dfsan_label s1_label, dfsan_label s2_label,
+ dfsan_label *ret_label, dfsan_origin s1_origin, dfsan_origin s2_origin,
+ dfsan_origin *ret_origin) {
+ size_t bytes_read;
+ int r = dfsan_strncasecmp(s1, s2, 0, &bytes_read);
+ dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+ return r;
}
DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, uptr caller_pc,
dfsan_label s1_label, dfsan_label s2_label,
dfsan_label n_label)
+DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strncmp, uptr caller_pc,
+ const char *s1, const char *s2, size_t n,
+ dfsan_label s1_label, dfsan_label s2_label,
+ dfsan_label n_label, dfsan_origin s1_origin,
+ dfsan_origin s2_origin, dfsan_origin n_origin)
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncmp(const char *s1, const char *s2,
size_t n, dfsan_label s1_label,
dfsan_label s2_label,
CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, GET_CALLER_PC(), s1, s2,
n, s1_label, s2_label, n_label);
- for (size_t i = 0;; ++i) {
- if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0 || i == n - 1) {
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
- dfsan_read_label(s2, i + 1));
- }
- return s1[i] - s2[i];
- }
- }
- return 0;
+ size_t bytes_read;
+ int r = dfsan_strncmp(s1, s2, n, &bytes_read);
+ *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+ return r;
}
-SANITIZER_INTERFACE_ATTRIBUTE int
-__dfsw_strncasecmp(const char *s1, const char *s2, size_t n,
- dfsan_label s1_label, dfsan_label s2_label,
- dfsan_label n_label, dfsan_label *ret_label) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strncmp(
+ const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
if (n == 0) {
*ret_label = 0;
return 0;
}
- for (size_t i = 0;; ++i) {
- char s1_lower = tolower(s1[i]);
- char s2_lower = tolower(s2[i]);
+ CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_origin_strncmp, GET_CALLER_PC(),
+ s1, s2, n, s1_label, s2_label, n_label, s1_origin,
+ s2_origin, n_origin);
- if (s1_lower != s2_lower || s1[i] == 0 || s2[i] == 0 || i == n - 1) {
- if (flags().strict_data_dependencies) {
- *ret_label = 0;
- } else {
- *ret_label = dfsan_union(dfsan_read_label(s1, i + 1),
- dfsan_read_label(s2, i + 1));
- }
- return s1_lower - s2_lower;
- }
+ size_t bytes_read;
+ int r = dfsan_strncmp(s1, s2, n, &bytes_read);
+ dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+ return r;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncasecmp(
+ const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label) {
+ if (n == 0) {
+ *ret_label = 0;
+ return 0;
}
- return 0;
+
+ size_t bytes_read;
+ int r = dfsan_strncasecmp(s1, s2, n, &bytes_read);
+ *ret_label = dfsan_get_memcmp_label(s1, s2, bytes_read);
+ return r;
}
-SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_calloc(size_t nmemb, size_t size,
- dfsan_label nmemb_label,
- dfsan_label size_label,
- dfsan_label *ret_label) {
- void *p = calloc(nmemb, size);
- dfsan_set_label(0, p, nmemb * size);
- *ret_label = 0;
- return p;
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_strncasecmp(
+ const char *s1, const char *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
+ if (n == 0) {
+ *ret_label = 0;
+ return 0;
+ }
+
+ size_t bytes_read;
+ int r = dfsan_strncasecmp(s1, s2, n, &bytes_read);
+ dfsan_get_memcmp_origin(s1, s2, bytes_read, ret_label, ret_origin);
+ return r;
}
+
SANITIZER_INTERFACE_ATTRIBUTE size_t
__dfsw_strlen(const char *s, dfsan_label s_label, dfsan_label *ret_label) {
size_t ret = strlen(s);
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE size_t __dfso_strlen(const char *s,
+ dfsan_label s_label,
+ dfsan_label *ret_label,
+ dfsan_origin s_origin,
+ dfsan_origin *ret_origin) {
+ size_t ret = __dfsw_strlen(s, s_label, ret_label);
+ if (!flags().strict_data_dependencies)
+ *ret_origin = dfsan_read_origin_of_first_taint(s, ret + 1);
+ return ret;
+}
+
+static void *dfsan_memmove(void *dest, const void *src, size_t n) {
+ dfsan_label *sdest = shadow_for(dest);
+ const dfsan_label *ssrc = shadow_for(src);
+ internal_memmove((void *)sdest, (const void *)ssrc, n * sizeof(dfsan_label));
+ return internal_memmove(dest, src, n);
+}
+
+static void *dfsan_memmove_with_origin(void *dest, const void *src, size_t n) {
+ dfsan_mem_origin_transfer(dest, src, n);
+ return dfsan_memmove(dest, src, n);
+}
static void *dfsan_memcpy(void *dest, const void *src, size_t n) {
dfsan_label *sdest = shadow_for(dest);
return internal_memcpy(dest, src, n);
}
+static void *dfsan_memcpy_with_origin(void *dest, const void *src, size_t n) {
+ dfsan_mem_origin_transfer(dest, src, n);
+ return dfsan_memcpy(dest, src, n);
+}
+
static void dfsan_memset(void *s, int c, dfsan_label c_label, size_t n) {
internal_memset(s, c, n);
dfsan_set_label(c_label, s, n);
}
+static void dfsan_memset_with_origin(void *s, int c, dfsan_label c_label,
+ dfsan_origin c_origin, size_t n) {
+ internal_memset(s, c, n);
+ dfsan_set_label_origin(c_label, c_origin, s, n);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
void *__dfsw_memcpy(void *dest, const void *src, size_t n,
dfsan_label dest_label, dfsan_label src_label,
return dfsan_memcpy(dest, src, n);
}
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memcpy(void *dest, const void *src, size_t n,
+ dfsan_label dest_label, dfsan_label src_label,
+ dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin dest_origin, dfsan_origin src_origin,
+ dfsan_origin n_origin, dfsan_origin *ret_origin) {
+ *ret_label = dest_label;
+ *ret_origin = dest_origin;
+ return dfsan_memcpy_with_origin(dest, src, n);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfsw_memmove(void *dest, const void *src, size_t n,
+ dfsan_label dest_label, dfsan_label src_label,
+ dfsan_label n_label, dfsan_label *ret_label) {
+ *ret_label = dest_label;
+ return dfsan_memmove(dest, src, n);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memmove(void *dest, const void *src, size_t n,
+ dfsan_label dest_label, dfsan_label src_label,
+ dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin dest_origin, dfsan_origin src_origin,
+ dfsan_origin n_origin, dfsan_origin *ret_origin) {
+ *ret_label = dest_label;
+ *ret_origin = dest_origin;
+ return dfsan_memmove_with_origin(dest, src, n);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
void *__dfsw_memset(void *s, int c, size_t n,
dfsan_label s_label, dfsan_label c_label,
return s;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__dfso_memset(void *s, int c, size_t n, dfsan_label s_label,
+ dfsan_label c_label, dfsan_label n_label,
+ dfsan_label *ret_label, dfsan_origin s_origin,
+ dfsan_origin c_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
+ dfsan_memset_with_origin(s, c, c_label, c_origin, n);
+ *ret_label = s_label;
+ *ret_origin = s_origin;
+ return s;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strcat(char *dest, const char *src,
+ dfsan_label dest_label,
+ dfsan_label src_label,
+ dfsan_label *ret_label) {
+ size_t dest_len = strlen(dest);
+ char *ret = strcat(dest, src); // NOLINT
+ dfsan_label *sdest = shadow_for(dest + dest_len);
+ const dfsan_label *ssrc = shadow_for(src);
+ internal_memcpy((void *)sdest, (const void *)ssrc,
+ strlen(src) * sizeof(dfsan_label));
+ *ret_label = dest_label;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strcat(
+ char *dest, const char *src, dfsan_label dest_label, dfsan_label src_label,
+ dfsan_label *ret_label, dfsan_origin dest_origin, dfsan_origin src_origin,
+ dfsan_origin *ret_origin) {
+ size_t dest_len = strlen(dest);
+ char *ret = strcat(dest, src); // NOLINT
+ dfsan_label *sdest = shadow_for(dest + dest_len);
+ const dfsan_label *ssrc = shadow_for(src);
+ size_t src_len = strlen(src);
+ dfsan_mem_origin_transfer(dest + dest_len, src, src_len);
+ internal_memcpy((void *)sdest, (const void *)ssrc,
+ src_len * sizeof(dfsan_label));
+ *ret_label = dest_label;
+ *ret_origin = dest_origin;
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE char *
__dfsw_strdup(const char *s, dfsan_label s_label, dfsan_label *ret_label) {
size_t len = strlen(s);
return static_cast<char *>(p);
}
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strdup(const char *s,
+ dfsan_label s_label,
+ dfsan_label *ret_label,
+ dfsan_origin s_origin,
+ dfsan_origin *ret_origin) {
+ size_t len = strlen(s);
+ void *p = malloc(len + 1);
+ dfsan_memcpy_with_origin(p, s, len + 1);
+ *ret_label = 0;
+ return static_cast<char *>(p);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE char *
__dfsw_strncpy(char *s1, const char *s2, size_t n, dfsan_label s1_label,
dfsan_label s2_label, dfsan_label n_label,
return s1;
}
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strncpy(
+ char *s1, const char *s2, size_t n, dfsan_label s1_label,
+ dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label,
+ dfsan_origin s1_origin, dfsan_origin s2_origin, dfsan_origin n_origin,
+ dfsan_origin *ret_origin) {
+ size_t len = strlen(s2);
+ if (len < n) {
+ dfsan_memcpy_with_origin(s1, s2, len + 1);
+ dfsan_memset_with_origin(s1 + len + 1, 0, 0, 0, n - len - 1);
+ } else {
+ dfsan_memcpy_with_origin(s1, s2, n);
+ }
+
+ *ret_label = s1_label;
+ *ret_origin = s1_origin;
+ return s1;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE ssize_t
__dfsw_pread(int fd, void *buf, size_t count, off_t offset,
dfsan_label fd_label, dfsan_label buf_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_pread(
+ int fd, void *buf, size_t count, off_t offset, dfsan_label fd_label,
+ dfsan_label buf_label, dfsan_label count_label, dfsan_label offset_label,
+ dfsan_label *ret_label, dfsan_origin fd_origin, dfsan_origin buf_origin,
+ dfsan_origin count_origin, dfsan_label offset_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_pread(fd, buf, count, offset, fd_label, buf_label, count_label,
+ offset_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE ssize_t
__dfsw_read(int fd, void *buf, size_t count,
dfsan_label fd_label, dfsan_label buf_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_read(
+ int fd, void *buf, size_t count, dfsan_label fd_label,
+ dfsan_label buf_label, dfsan_label count_label, dfsan_label *ret_label,
+ dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_read(fd, buf, count, fd_label, buf_label, count_label,
+ ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_clock_gettime(clockid_t clk_id,
struct timespec *tp,
dfsan_label clk_id_label,
return ret;
}
-static void unpoison(const void *ptr, uptr size) {
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_clock_gettime(
+ clockid_t clk_id, struct timespec *tp, dfsan_label clk_id_label,
+ dfsan_label tp_label, dfsan_label *ret_label, dfsan_origin clk_id_origin,
+ dfsan_origin tp_origin, dfsan_origin *ret_origin) {
+ return __dfsw_clock_gettime(clk_id, tp, clk_id_label, tp_label, ret_label);
+}
+
+static void dfsan_set_zero_label(const void *ptr, uptr size) {
dfsan_set_label(0, const_cast<void *>(ptr), size);
}
void *handle = dlopen(filename, flag);
link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE(handle);
if (map)
- ForEachMappedRegion(map, unpoison);
+ ForEachMappedRegion(map, dfsan_set_zero_label);
*ret_label = 0;
return handle;
}
-struct pthread_create_info {
- void *(*start_routine_trampoline)(void *, void *, dfsan_label, dfsan_label *);
- void *start_routine;
- void *arg;
-};
+SANITIZER_INTERFACE_ATTRIBUTE void *__dfso_dlopen(
+ const char *filename, int flag, dfsan_label filename_label,
+ dfsan_label flag_label, dfsan_label *ret_label,
+ dfsan_origin filename_origin, dfsan_origin flag_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_dlopen(filename, flag, filename_label, flag_label, ret_label);
+}
-static void *pthread_create_cb(void *p) {
- pthread_create_info pci(*(pthread_create_info *)p);
- free(p);
- dfsan_label ret_label;
- return pci.start_routine_trampoline(pci.start_routine, pci.arg, 0,
- &ret_label);
+static void *DFsanThreadStartFunc(void *arg) {
+ DFsanThread *t = (DFsanThread *)arg;
+ SetCurrentThread(t);
+ return t->ThreadStart();
+}
+
+static int dfsan_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
+ void *start_routine_trampoline,
+ void *start_routine, void *arg,
+ dfsan_label *ret_label,
+ bool track_origins = false) {
+ pthread_attr_t myattr;
+ if (!attr) {
+ pthread_attr_init(&myattr);
+ attr = &myattr;
+ }
+
+ // Ensure that the thread stack is large enough to hold all TLS data.
+ AdjustStackSize((void *)(const_cast<pthread_attr_t *>(attr)));
+
+ DFsanThread *t =
+ DFsanThread::Create(start_routine_trampoline,
+ (thread_callback_t)start_routine, arg, track_origins);
+ int res = pthread_create(thread, attr, DFsanThreadStartFunc, t);
+
+ if (attr == &myattr)
+ pthread_attr_destroy(&myattr);
+ *ret_label = 0;
+ return res;
}
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_create(
void *start_routine, void *arg, dfsan_label thread_label,
dfsan_label attr_label, dfsan_label start_routine_label,
dfsan_label arg_label, dfsan_label *ret_label) {
- pthread_create_info *pci =
- (pthread_create_info *)malloc(sizeof(pthread_create_info));
- pci->start_routine_trampoline = start_routine_trampoline;
- pci->start_routine = start_routine;
- pci->arg = arg;
- int rv = pthread_create(thread, attr, pthread_create_cb, (void *)pci);
- if (rv != 0)
- free(pci);
+ return dfsan_pthread_create(thread, attr, (void *)start_routine_trampoline,
+ start_routine, arg, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_pthread_create(
+ pthread_t *thread, const pthread_attr_t *attr,
+ void *(*start_routine_trampoline)(void *, void *, dfsan_label,
+ dfsan_label *, dfsan_origin,
+ dfsan_origin *),
+ void *start_routine, void *arg, dfsan_label thread_label,
+ dfsan_label attr_label, dfsan_label start_routine_label,
+ dfsan_label arg_label, dfsan_label *ret_label, dfsan_origin thread_origin,
+ dfsan_origin attr_origin, dfsan_origin start_routine_origin,
+ dfsan_origin arg_origin, dfsan_origin *ret_origin) {
+ return dfsan_pthread_create(thread, attr, (void *)start_routine_trampoline,
+ start_routine, arg, ret_label, true);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_join(pthread_t thread,
+ void **retval,
+ dfsan_label thread_label,
+ dfsan_label retval_label,
+ dfsan_label *ret_label) {
+ int ret = pthread_join(thread, retval);
+ if (ret == 0 && retval)
+ dfsan_set_label(0, retval, sizeof(*retval));
*ret_label = 0;
- return rv;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_pthread_join(
+ pthread_t thread, void **retval, dfsan_label thread_label,
+ dfsan_label retval_label, dfsan_label *ret_label,
+ dfsan_origin thread_origin, dfsan_origin retval_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_pthread_join(thread, retval, thread_label, retval_label,
+ ret_label);
}
struct dl_iterate_phdr_info {
void *data;
};
+struct dl_iterate_phdr_origin_info {
+ int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
+ size_t size, void *data, dfsan_label info_label,
+ dfsan_label size_label, dfsan_label data_label,
+ dfsan_label *ret_label, dfsan_origin info_origin,
+ dfsan_origin size_origin, dfsan_origin data_origin,
+ dfsan_origin *ret_origin);
+ void *callback;
+ void *data;
+};
+
int dl_iterate_phdr_cb(struct dl_phdr_info *info, size_t size, void *data) {
dl_iterate_phdr_info *dipi = (dl_iterate_phdr_info *)data;
dfsan_set_label(0, *info);
0, &ret_label);
}
+int dl_iterate_phdr_origin_cb(struct dl_phdr_info *info, size_t size,
+ void *data) {
+ dl_iterate_phdr_origin_info *dipi = (dl_iterate_phdr_origin_info *)data;
+ dfsan_set_label(0, *info);
+ dfsan_set_label(0, const_cast<char *>(info->dlpi_name),
+ strlen(info->dlpi_name) + 1);
+ dfsan_set_label(
+ 0, const_cast<char *>(reinterpret_cast<const char *>(info->dlpi_phdr)),
+ sizeof(*info->dlpi_phdr) * info->dlpi_phnum);
+ dfsan_label ret_label;
+ dfsan_origin ret_origin;
+ return dipi->callback_trampoline(dipi->callback, info, size, dipi->data, 0, 0,
+ 0, &ret_label, 0, 0, 0, &ret_origin);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_dl_iterate_phdr(
int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
size_t size, void *data, dfsan_label info_label,
return dl_iterate_phdr(dl_iterate_phdr_cb, &dipi);
}
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_dl_iterate_phdr(
+ int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
+ size_t size, void *data, dfsan_label info_label,
+ dfsan_label size_label, dfsan_label data_label,
+ dfsan_label *ret_label, dfsan_origin info_origin,
+ dfsan_origin size_origin,
+ dfsan_origin data_origin,
+ dfsan_origin *ret_origin),
+ void *callback, void *data, dfsan_label callback_label,
+ dfsan_label data_label, dfsan_label *ret_label,
+ dfsan_origin callback_origin, dfsan_origin data_origin,
+ dfsan_origin *ret_origin) {
+ dl_iterate_phdr_origin_info dipi = {callback_trampoline, callback, data};
+ *ret_label = 0;
+ return dl_iterate_phdr(dl_iterate_phdr_origin_cb, &dipi);
+}
+
+// This function is only available for glibc 2.27 or newer. Mark it weak so
+// linking succeeds with older glibcs.
+SANITIZER_WEAK_ATTRIBUTE void _dl_get_tls_static_info(size_t *sizep,
+ size_t *alignp);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __dfsw__dl_get_tls_static_info(
+ size_t *sizep, size_t *alignp, dfsan_label sizep_label,
+ dfsan_label alignp_label) {
+ assert(_dl_get_tls_static_info);
+ _dl_get_tls_static_info(sizep, alignp);
+ dfsan_set_label(0, sizep, sizeof(*sizep));
+ dfsan_set_label(0, alignp, sizeof(*alignp));
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __dfso__dl_get_tls_static_info(
+ size_t *sizep, size_t *alignp, dfsan_label sizep_label,
+ dfsan_label alignp_label, dfsan_origin sizep_origin,
+ dfsan_origin alignp_origin) {
+ __dfsw__dl_get_tls_static_info(sizep, alignp, sizep_label, alignp_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
dfsan_label buf_label, dfsan_label *ret_label) {
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
+ dfsan_label buf_label, dfsan_label *ret_label,
+ dfsan_origin timep_origin, dfsan_origin buf_origin,
+ dfsan_origin *ret_origin) {
+ char *ret = ctime_r(timep, buf);
+ if (ret) {
+ dfsan_set_label_origin(
+ dfsan_read_label(timep, sizeof(time_t)),
+ dfsan_read_origin_of_first_taint(timep, sizeof(time_t)), buf,
+ strlen(buf) + 1);
+ *ret_label = buf_label;
+ *ret_origin = buf_origin;
+ } else {
+ *ret_label = 0;
+ }
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_fgets(char *s, int size, FILE *stream, dfsan_label s_label,
dfsan_label size_label, dfsan_label stream_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_fgets(char *s, int size, FILE *stream, dfsan_label s_label,
+ dfsan_label size_label, dfsan_label stream_label,
+ dfsan_label *ret_label, dfsan_origin s_origin,
+ dfsan_origin size_origin, dfsan_origin stream_origin,
+ dfsan_origin *ret_origin) {
+ char *ret = __dfsw_fgets(s, size, stream, s_label, size_label, stream_label,
+ ret_label);
+ if (ret)
+ *ret_origin = s_origin;
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_getcwd(char *buf, size_t size, dfsan_label buf_label,
dfsan_label size_label, dfsan_label *ret_label) {
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_getcwd(char *buf, size_t size, dfsan_label buf_label,
+ dfsan_label size_label, dfsan_label *ret_label,
+ dfsan_origin buf_origin, dfsan_origin size_origin,
+ dfsan_origin *ret_origin) {
+ char *ret = __dfsw_getcwd(buf, size, buf_label, size_label, ret_label);
+ if (ret)
+ *ret_origin = buf_origin;
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_get_current_dir_name(dfsan_label *ret_label) {
char *ret = get_current_dir_name();
- if (ret) {
+ if (ret)
dfsan_set_label(0, ret, strlen(ret) + 1);
- }
*ret_label = 0;
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+char *__dfso_get_current_dir_name(dfsan_label *ret_label,
+ dfsan_origin *ret_origin) {
+ return __dfsw_get_current_dir_name(ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_gethostname(char *name, size_t len, dfsan_label name_label,
dfsan_label len_label, dfsan_label *ret_label) {
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_gethostname(char *name, size_t len, dfsan_label name_label,
+ dfsan_label len_label, dfsan_label *ret_label,
+ dfsan_origin name_origin, dfsan_origin len_origin,
+ dfsan_label *ret_origin) {
+ return __dfsw_gethostname(name, len, name_label, len_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_getrlimit(int resource, struct rlimit *rlim,
dfsan_label resource_label, dfsan_label rlim_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getrlimit(int resource, struct rlimit *rlim,
+ dfsan_label resource_label, dfsan_label rlim_label,
+ dfsan_label *ret_label, dfsan_origin resource_origin,
+ dfsan_origin rlim_origin, dfsan_origin *ret_origin) {
+ return __dfsw_getrlimit(resource, rlim, resource_label, rlim_label,
+ ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_getrusage(int who, struct rusage *usage, dfsan_label who_label,
dfsan_label usage_label, dfsan_label *ret_label) {
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getrusage(int who, struct rusage *usage, dfsan_label who_label,
+ dfsan_label usage_label, dfsan_label *ret_label,
+ dfsan_origin who_origin, dfsan_origin usage_origin,
+ dfsan_label *ret_origin) {
+ return __dfsw_getrusage(who, usage, who_label, usage_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
char *__dfsw_strcpy(char *dest, const char *src, dfsan_label dst_label,
dfsan_label src_label, dfsan_label *ret_label) {
}
SANITIZER_INTERFACE_ATTRIBUTE
-long int __dfsw_strtol(const char *nptr, char **endptr, int base,
- dfsan_label nptr_label, dfsan_label endptr_label,
- dfsan_label base_label, dfsan_label *ret_label) {
- char *tmp_endptr;
- long int ret = strtol(nptr, &tmp_endptr, base);
- if (endptr) {
- *endptr = tmp_endptr;
+char *__dfso_strcpy(char *dest, const char *src, dfsan_label dst_label,
+ dfsan_label src_label, dfsan_label *ret_label,
+ dfsan_origin dst_origin, dfsan_origin src_origin,
+ dfsan_origin *ret_origin) {
+ char *ret = strcpy(dest, src); // NOLINT
+ if (ret) {
+ size_t str_len = strlen(src) + 1;
+ dfsan_mem_origin_transfer(dest, src, str_len);
+ internal_memcpy(shadow_for(dest), shadow_for(src),
+ sizeof(dfsan_label) * str_len);
}
+ *ret_label = dst_label;
+ *ret_origin = dst_origin;
+ return ret;
+}
+
+static long int dfsan_strtol(const char *nptr, char **endptr, int base,
+ char **tmp_endptr) {
+ assert(tmp_endptr);
+ long int ret = strtol(nptr, tmp_endptr, base);
+ if (endptr)
+ *endptr = *tmp_endptr;
+ return ret;
+}
+
+static void dfsan_strtolong_label(const char *nptr, const char *tmp_endptr,
+ dfsan_label base_label,
+ dfsan_label *ret_label) {
if (tmp_endptr > nptr) {
// If *tmp_endptr is '\0' include its label as well.
*ret_label = dfsan_union(
} else {
*ret_label = 0;
}
+}
+
+static void dfsan_strtolong_origin(const char *nptr, const char *tmp_endptr,
+ dfsan_label base_label,
+ dfsan_label *ret_label,
+ dfsan_origin base_origin,
+ dfsan_origin *ret_origin) {
+ if (tmp_endptr > nptr) {
+ // When multiple inputs are tainted, we propagate one of its origins.
+ // Because checking if base_label is tainted does not need additional
+ // computation, we prefer to propagating base_origin.
+ *ret_origin = base_label
+ ? base_origin
+ : dfsan_read_origin_of_first_taint(
+ nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1));
+ }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long int __dfsw_strtol(const char *nptr, char **endptr, int base,
+ dfsan_label nptr_label, dfsan_label endptr_label,
+ dfsan_label base_label, dfsan_label *ret_label) {
+ char *tmp_endptr;
+ long int ret = dfsan_strtol(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
return ret;
}
SANITIZER_INTERFACE_ATTRIBUTE
-double __dfsw_strtod(const char *nptr, char **endptr,
+long int __dfso_strtol(const char *nptr, char **endptr, int base,
dfsan_label nptr_label, dfsan_label endptr_label,
- dfsan_label *ret_label) {
+ dfsan_label base_label, dfsan_label *ret_label,
+ dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+ dfsan_origin base_origin, dfsan_origin *ret_origin) {
char *tmp_endptr;
- double ret = strtod(nptr, &tmp_endptr);
- if (endptr) {
- *endptr = tmp_endptr;
- }
+ long int ret = dfsan_strtol(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+ ret_origin);
+ return ret;
+}
+
+static double dfsan_strtod(const char *nptr, char **endptr, char **tmp_endptr) {
+ assert(tmp_endptr);
+ double ret = strtod(nptr, tmp_endptr);
+ if (endptr)
+ *endptr = *tmp_endptr;
+ return ret;
+}
+
+static void dfsan_strtod_label(const char *nptr, const char *tmp_endptr,
+ dfsan_label *ret_label) {
if (tmp_endptr > nptr) {
// If *tmp_endptr is '\0' include its label as well.
*ret_label = dfsan_read_label(
} else {
*ret_label = 0;
}
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+double __dfsw_strtod(const char *nptr, char **endptr, dfsan_label nptr_label,
+ dfsan_label endptr_label, dfsan_label *ret_label) {
+ char *tmp_endptr;
+ double ret = dfsan_strtod(nptr, endptr, &tmp_endptr);
+ dfsan_strtod_label(nptr, tmp_endptr, ret_label);
return ret;
}
SANITIZER_INTERFACE_ATTRIBUTE
-long long int __dfsw_strtoll(const char *nptr, char **endptr, int base,
- dfsan_label nptr_label, dfsan_label endptr_label,
- dfsan_label base_label, dfsan_label *ret_label) {
+double __dfso_strtod(const char *nptr, char **endptr, dfsan_label nptr_label,
+ dfsan_label endptr_label, dfsan_label *ret_label,
+ dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+ dfsan_origin *ret_origin) {
char *tmp_endptr;
- long long int ret = strtoll(nptr, &tmp_endptr, base);
- if (endptr) {
- *endptr = tmp_endptr;
- }
+ double ret = dfsan_strtod(nptr, endptr, &tmp_endptr);
+ dfsan_strtod_label(nptr, tmp_endptr, ret_label);
if (tmp_endptr > nptr) {
// If *tmp_endptr is '\0' include its label as well.
- *ret_label = dfsan_union(
- base_label,
- dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
+ *ret_origin = dfsan_read_origin_of_first_taint(
+ nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1));
} else {
- *ret_label = 0;
+ *ret_origin = 0;
}
return ret;
}
+static long long int dfsan_strtoll(const char *nptr, char **endptr, int base,
+ char **tmp_endptr) {
+ assert(tmp_endptr);
+ long long int ret = strtoll(nptr, tmp_endptr, base);
+ if (endptr)
+ *endptr = *tmp_endptr;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long int __dfsw_strtoll(const char *nptr, char **endptr, int base,
+ dfsan_label nptr_label, dfsan_label endptr_label,
+ dfsan_label base_label, dfsan_label *ret_label) {
+ char *tmp_endptr;
+ long long int ret = dfsan_strtoll(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long int __dfso_strtoll(const char *nptr, char **endptr, int base,
+ dfsan_label nptr_label, dfsan_label endptr_label,
+ dfsan_label base_label, dfsan_label *ret_label,
+ dfsan_origin nptr_origin,
+ dfsan_origin endptr_origin,
+ dfsan_origin base_origin,
+ dfsan_origin *ret_origin) {
+ char *tmp_endptr;
+ long long int ret = dfsan_strtoll(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+ ret_origin);
+ return ret;
+}
+
+static unsigned long int dfsan_strtoul(const char *nptr, char **endptr,
+ int base, char **tmp_endptr) {
+ assert(tmp_endptr);
+ unsigned long int ret = strtoul(nptr, tmp_endptr, base);
+ if (endptr)
+ *endptr = *tmp_endptr;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+unsigned long int __dfsw_strtoul(const char *nptr, char **endptr, int base,
+ dfsan_label nptr_label, dfsan_label endptr_label,
+ dfsan_label base_label, dfsan_label *ret_label) {
+ char *tmp_endptr;
+ unsigned long int ret = dfsan_strtoul(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
-unsigned long int __dfsw_strtoul(const char *nptr, char **endptr, int base,
- dfsan_label nptr_label, dfsan_label endptr_label,
- dfsan_label base_label, dfsan_label *ret_label) {
+unsigned long int __dfso_strtoul(
+ const char *nptr, char **endptr, int base, dfsan_label nptr_label,
+ dfsan_label endptr_label, dfsan_label base_label, dfsan_label *ret_label,
+ dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+ dfsan_origin base_origin, dfsan_origin *ret_origin) {
char *tmp_endptr;
- unsigned long int ret = strtoul(nptr, &tmp_endptr, base);
- if (endptr) {
- *endptr = tmp_endptr;
- }
- if (tmp_endptr > nptr) {
- // If *tmp_endptr is '\0' include its label as well.
- *ret_label = dfsan_union(
- base_label,
- dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
- } else {
- *ret_label = 0;
- }
+ unsigned long int ret = dfsan_strtoul(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+ ret_origin);
+ return ret;
+}
+
+static long long unsigned int dfsan_strtoull(const char *nptr, char **endptr,
+ int base, char **tmp_endptr) {
+ assert(tmp_endptr);
+ long long unsigned int ret = strtoull(nptr, tmp_endptr, base);
+ if (endptr)
+ *endptr = *tmp_endptr;
return ret;
}
SANITIZER_INTERFACE_ATTRIBUTE
long long unsigned int __dfsw_strtoull(const char *nptr, char **endptr,
- dfsan_label nptr_label,
- int base, dfsan_label endptr_label,
+ int base, dfsan_label nptr_label,
+ dfsan_label endptr_label,
dfsan_label base_label,
dfsan_label *ret_label) {
char *tmp_endptr;
- long long unsigned int ret = strtoull(nptr, &tmp_endptr, base);
- if (endptr) {
- *endptr = tmp_endptr;
- }
- if (tmp_endptr > nptr) {
- // If *tmp_endptr is '\0' include its label as well.
- *ret_label = dfsan_union(
- base_label,
- dfsan_read_label(nptr, tmp_endptr - nptr + (*tmp_endptr ? 0 : 1)));
- } else {
- *ret_label = 0;
- }
+ long long unsigned int ret = dfsan_strtoull(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+long long unsigned int __dfso_strtoull(
+ const char *nptr, char **endptr, int base, dfsan_label nptr_label,
+ dfsan_label endptr_label, dfsan_label base_label, dfsan_label *ret_label,
+ dfsan_origin nptr_origin, dfsan_origin endptr_origin,
+ dfsan_origin base_origin, dfsan_origin *ret_origin) {
+ char *tmp_endptr;
+ long long unsigned int ret = dfsan_strtoull(nptr, endptr, base, &tmp_endptr);
+ dfsan_strtolong_label(nptr, tmp_endptr, base_label, ret_label);
+ dfsan_strtolong_origin(nptr, tmp_endptr, base_label, ret_label, base_origin,
+ ret_origin);
return ret;
}
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+time_t __dfso_time(time_t *t, dfsan_label t_label, dfsan_label *ret_label,
+ dfsan_origin t_origin, dfsan_origin *ret_origin) {
+ return __dfsw_time(t, t_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_inet_pton(int af, const char *src, void *dst, dfsan_label af_label,
dfsan_label src_label, dfsan_label dst_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_inet_pton(int af, const char *src, void *dst, dfsan_label af_label,
+ dfsan_label src_label, dfsan_label dst_label,
+ dfsan_label *ret_label, dfsan_origin af_origin,
+ dfsan_origin src_origin, dfsan_origin dst_origin,
+ dfsan_origin *ret_origin) {
+ int ret = inet_pton(af, src, dst);
+ if (ret == 1) {
+ int src_len = strlen(src) + 1;
+ dfsan_set_label_origin(
+ dfsan_read_label(src, src_len),
+ dfsan_read_origin_of_first_taint(src, src_len), dst,
+ af == AF_INET ? sizeof(struct in_addr) : sizeof(in6_addr));
+ }
+ *ret_label = 0;
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
struct tm *__dfsw_localtime_r(const time_t *timep, struct tm *result,
dfsan_label timep_label, dfsan_label result_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+struct tm *__dfso_localtime_r(const time_t *timep, struct tm *result,
+ dfsan_label timep_label, dfsan_label result_label,
+ dfsan_label *ret_label, dfsan_origin timep_origin,
+ dfsan_origin result_origin,
+ dfsan_origin *ret_origin) {
+ struct tm *ret = localtime_r(timep, result);
+ if (ret) {
+ dfsan_set_label_origin(
+ dfsan_read_label(timep, sizeof(time_t)),
+ dfsan_read_origin_of_first_taint(timep, sizeof(time_t)), result,
+ sizeof(struct tm));
+ *ret_label = result_label;
+ *ret_origin = result_origin;
+ } else {
+ *ret_label = 0;
+ }
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_getpwuid_r(id_t uid, struct passwd *pwd,
char *buf, size_t buflen, struct passwd **result,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_getpwuid_r(id_t uid, struct passwd *pwd, char *buf, size_t buflen,
+ struct passwd **result, dfsan_label uid_label,
+ dfsan_label pwd_label, dfsan_label buf_label,
+ dfsan_label buflen_label, dfsan_label result_label,
+ dfsan_label *ret_label, dfsan_origin uid_origin,
+ dfsan_origin pwd_origin, dfsan_origin buf_origin,
+ dfsan_origin buflen_origin, dfsan_origin result_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_getpwuid_r(uid, pwd, buf, buflen, result, uid_label, pwd_label,
+ buf_label, buflen_label, result_label, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfsw_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
+ int timeout, dfsan_label epfd_label,
+ dfsan_label events_label, dfsan_label maxevents_label,
+ dfsan_label timeout_label, dfsan_label *ret_label) {
+ int ret = epoll_wait(epfd, events, maxevents, timeout);
+ if (ret > 0)
+ dfsan_set_label(0, events, ret * sizeof(*events));
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
+ int timeout, dfsan_label epfd_label,
+ dfsan_label events_label, dfsan_label maxevents_label,
+ dfsan_label timeout_label, dfsan_label *ret_label,
+ dfsan_origin epfd_origin, dfsan_origin events_origin,
+ dfsan_origin maxevents_origin,
+ dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+ return __dfsw_epoll_wait(epfd, events, maxevents, timeout, epfd_label,
+ events_label, maxevents_label, timeout_label,
+ ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_poll(struct pollfd *fds, nfds_t nfds, int timeout,
dfsan_label dfs_label, dfsan_label nfds_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_poll(struct pollfd *fds, nfds_t nfds, int timeout,
+ dfsan_label dfs_label, dfsan_label nfds_label,
+ dfsan_label timeout_label, dfsan_label *ret_label,
+ dfsan_origin dfs_origin, dfsan_origin nfds_origin,
+ dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+ return __dfsw_poll(fds, nfds, timeout, dfs_label, nfds_label, timeout_label,
+ ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_select(int nfds, fd_set *readfds, fd_set *writefds,
fd_set *exceptfds, struct timeval *timeout,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_select(int nfds, fd_set *readfds, fd_set *writefds,
+ fd_set *exceptfds, struct timeval *timeout,
+ dfsan_label nfds_label, dfsan_label readfds_label,
+ dfsan_label writefds_label, dfsan_label exceptfds_label,
+ dfsan_label timeout_label, dfsan_label *ret_label,
+ dfsan_origin nfds_origin, dfsan_origin readfds_origin,
+ dfsan_origin writefds_origin, dfsan_origin exceptfds_origin,
+ dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+ return __dfsw_select(nfds, readfds, writefds, exceptfds, timeout, nfds_label,
+ readfds_label, writefds_label, exceptfds_label,
+ timeout_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask,
dfsan_label pid_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask,
+ dfsan_label pid_label,
+ dfsan_label cpusetsize_label,
+ dfsan_label mask_label, dfsan_label *ret_label,
+ dfsan_origin pid_origin,
+ dfsan_origin cpusetsize_origin,
+ dfsan_origin mask_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_sched_getaffinity(pid, cpusetsize, mask, pid_label,
+ cpusetsize_label, mask_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_sigemptyset(sigset_t *set, dfsan_label set_label,
dfsan_label *ret_label) {
int ret = sigemptyset(set);
dfsan_set_label(0, set, sizeof(sigset_t));
+ *ret_label = 0;
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sigemptyset(sigset_t *set, dfsan_label set_label,
+ dfsan_label *ret_label, dfsan_origin set_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_sigemptyset(set, set_label, ret_label);
+}
+
+class SignalHandlerScope {
+ public:
+ SignalHandlerScope() {
+ if (DFsanThread *t = GetCurrentThread())
+ t->EnterSignalHandler();
+ }
+ ~SignalHandlerScope() {
+ if (DFsanThread *t = GetCurrentThread())
+ t->LeaveSignalHandler();
+ }
+};
+
+// Clear DFSan runtime TLS state at the end of a scope.
+//
+// Implementation must be async-signal-safe and use small data size, because
+// instances of this class may live on the signal handler stack.
+//
+// DFSan uses TLS to pass metadata of arguments and return values. When an
+// instrumented function accesses the TLS, if a signal callback happens, and the
+// callback calls other instrumented functions with updating the same TLS, the
+// TLS is in an inconsistent state after the callback ends. This may cause
+// either under-tainting or over-tainting.
+//
+// The current implementation simply resets TLS at restore. This prevents from
+// over-tainting. Although under-tainting may still happen, a taint flow can be
+// found eventually if we run a DFSan-instrumented program multiple times. The
+// alternative option is saving the entire TLS. However the TLS storage takes
+// 2k bytes, and signal calls could be nested. So it does not seem worth.
+class ScopedClearThreadLocalState {
+ public:
+ ScopedClearThreadLocalState() {}
+ ~ScopedClearThreadLocalState() { dfsan_clear_thread_local_state(); }
+};
+
+// SignalSpinLocker::sigactions_mu guarantees atomicity of sigaction() calls.
+const int kMaxSignals = 1024;
+static atomic_uintptr_t sigactions[kMaxSignals];
+
+static void SignalHandler(int signo) {
+ SignalHandlerScope signal_handler_scope;
+ ScopedClearThreadLocalState scoped_clear_tls;
+
+ // Clear shadows for all inputs provided by system. This is why DFSan
+ // instrumentation generates a trampoline function to each function pointer,
+ // and uses the trampoline to clear shadows. However sigaction does not use
+ // a function pointer directly, so we have to do this manually.
+ dfsan_clear_arg_tls(0, sizeof(dfsan_label));
+
+ typedef void (*signal_cb)(int x);
+ signal_cb cb =
+ (signal_cb)atomic_load(&sigactions[signo], memory_order_relaxed);
+ cb(signo);
+}
+
+static void SignalAction(int signo, siginfo_t *si, void *uc) {
+ SignalHandlerScope signal_handler_scope;
+ ScopedClearThreadLocalState scoped_clear_tls;
+
+ // Clear shadows for all inputs provided by system. Similar to SignalHandler.
+ dfsan_clear_arg_tls(0, 3 * sizeof(dfsan_label));
+ dfsan_set_label(0, si, sizeof(*si));
+ dfsan_set_label(0, uc, sizeof(ucontext_t));
+
+ typedef void (*sigaction_cb)(int, siginfo_t *, void *);
+ sigaction_cb cb =
+ (sigaction_cb)atomic_load(&sigactions[signo], memory_order_relaxed);
+ cb(signo, si, uc);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_sigaction(int signum, const struct sigaction *act,
struct sigaction *oldact, dfsan_label signum_label,
dfsan_label act_label, dfsan_label oldact_label,
dfsan_label *ret_label) {
- int ret = sigaction(signum, act, oldact);
+ CHECK_LT(signum, kMaxSignals);
+ SignalSpinLocker lock;
+ uptr old_cb = atomic_load(&sigactions[signum], memory_order_relaxed);
+ struct sigaction new_act;
+ struct sigaction *pnew_act = act ? &new_act : nullptr;
+ if (act) {
+ internal_memcpy(pnew_act, act, sizeof(struct sigaction));
+ if (pnew_act->sa_flags & SA_SIGINFO) {
+ uptr cb = (uptr)(pnew_act->sa_sigaction);
+ if (cb != (uptr)SIG_IGN && cb != (uptr)SIG_DFL) {
+ atomic_store(&sigactions[signum], cb, memory_order_relaxed);
+ pnew_act->sa_sigaction = SignalAction;
+ }
+ } else {
+ uptr cb = (uptr)(pnew_act->sa_handler);
+ if (cb != (uptr)SIG_IGN && cb != (uptr)SIG_DFL) {
+ atomic_store(&sigactions[signum], cb, memory_order_relaxed);
+ pnew_act->sa_handler = SignalHandler;
+ }
+ }
+ }
+
+ int ret = sigaction(signum, pnew_act, oldact);
+
+ if (ret == 0 && oldact) {
+ if (oldact->sa_flags & SA_SIGINFO) {
+ if (oldact->sa_sigaction == SignalAction)
+ oldact->sa_sigaction = (decltype(oldact->sa_sigaction))old_cb;
+ } else {
+ if (oldact->sa_handler == SignalHandler)
+ oldact->sa_handler = (decltype(oldact->sa_handler))old_cb;
+ }
+ }
+
if (oldact) {
dfsan_set_label(0, oldact, sizeof(struct sigaction));
}
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sigaction(int signum, const struct sigaction *act,
+ struct sigaction *oldact, dfsan_label signum_label,
+ dfsan_label act_label, dfsan_label oldact_label,
+ dfsan_label *ret_label, dfsan_origin signum_origin,
+ dfsan_origin act_origin, dfsan_origin oldact_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_sigaction(signum, act, oldact, signum_label, act_label,
+ oldact_label, ret_label);
+}
+
+static sighandler_t dfsan_signal(int signum, sighandler_t handler,
+ dfsan_label *ret_label) {
+ CHECK_LT(signum, kMaxSignals);
+ SignalSpinLocker lock;
+ uptr old_cb = atomic_load(&sigactions[signum], memory_order_relaxed);
+ if (handler != SIG_IGN && handler != SIG_DFL) {
+ atomic_store(&sigactions[signum], (uptr)handler, memory_order_relaxed);
+ handler = &SignalHandler;
+ }
+
+ sighandler_t ret = signal(signum, handler);
+
+ if (ret == SignalHandler)
+ ret = (sighandler_t)old_cb;
+
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+sighandler_t __dfsw_signal(int signum,
+ void *(*handler_trampoline)(void *, int, dfsan_label,
+ dfsan_label *),
+ sighandler_t handler, dfsan_label signum_label,
+ dfsan_label handler_label, dfsan_label *ret_label) {
+ return dfsan_signal(signum, handler, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+sighandler_t __dfso_signal(
+ int signum,
+ void *(*handler_trampoline)(void *, int, dfsan_label, dfsan_label *,
+ dfsan_origin, dfsan_origin *),
+ sighandler_t handler, dfsan_label signum_label, dfsan_label handler_label,
+ dfsan_label *ret_label, dfsan_origin signum_origin,
+ dfsan_origin handler_origin, dfsan_origin *ret_origin) {
+ return dfsan_signal(signum, handler, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfsw_sigaltstack(const stack_t *ss, stack_t *old_ss, dfsan_label ss_label,
+ dfsan_label old_ss_label, dfsan_label *ret_label) {
+ int ret = sigaltstack(ss, old_ss);
+ if (ret != -1 && old_ss)
+ dfsan_set_label(0, old_ss, sizeof(*old_ss));
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sigaltstack(const stack_t *ss, stack_t *old_ss, dfsan_label ss_label,
+ dfsan_label old_ss_label, dfsan_label *ret_label,
+ dfsan_origin ss_origin, dfsan_origin old_ss_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_sigaltstack(ss, old_ss, ss_label, old_ss_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE
int __dfsw_gettimeofday(struct timeval *tv, struct timezone *tz,
dfsan_label tv_label, dfsan_label tz_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_gettimeofday(struct timeval *tv, struct timezone *tz,
+ dfsan_label tv_label, dfsan_label tz_label,
+ dfsan_label *ret_label, dfsan_origin tv_origin,
+ dfsan_origin tz_origin, dfsan_origin *ret_origin) {
+ return __dfsw_gettimeofday(tv, tz, tv_label, tz_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE void *__dfsw_memchr(void *s, int c, size_t n,
dfsan_label s_label,
dfsan_label c_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE void *__dfso_memchr(
+ void *s, int c, size_t n, dfsan_label s_label, dfsan_label c_label,
+ dfsan_label n_label, dfsan_label *ret_label, dfsan_origin s_origin,
+ dfsan_origin c_origin, dfsan_origin n_origin, dfsan_origin *ret_origin) {
+ void *ret = __dfsw_memchr(s, c, n, s_label, c_label, n_label, ret_label);
+ if (flags().strict_data_dependencies) {
+ if (ret)
+ *ret_origin = s_origin;
+ } else {
+ size_t len =
+ ret ? reinterpret_cast<char *>(ret) - reinterpret_cast<char *>(s) + 1
+ : n;
+ dfsan_origin o = dfsan_read_origin_of_first_taint(s, len);
+ *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+ }
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strrchr(char *s, int c,
dfsan_label s_label,
dfsan_label c_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strrchr(
+ char *s, int c, dfsan_label s_label, dfsan_label c_label,
+ dfsan_label *ret_label, dfsan_origin s_origin, dfsan_origin c_origin,
+ dfsan_origin *ret_origin) {
+ char *ret = __dfsw_strrchr(s, c, s_label, c_label, ret_label);
+ if (flags().strict_data_dependencies) {
+ if (ret)
+ *ret_origin = s_origin;
+ } else {
+ size_t s_len = strlen(s) + 1;
+ dfsan_origin o = dfsan_read_origin_of_first_taint(s, s_len);
+ *ret_origin = o ? o : (s_label ? s_origin : c_origin);
+ }
+
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strstr(char *haystack, char *needle,
dfsan_label haystack_label,
dfsan_label needle_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE char *__dfso_strstr(char *haystack, char *needle,
+ dfsan_label haystack_label,
+ dfsan_label needle_label,
+ dfsan_label *ret_label,
+ dfsan_origin haystack_origin,
+ dfsan_origin needle_origin,
+ dfsan_origin *ret_origin) {
+ char *ret =
+ __dfsw_strstr(haystack, needle, haystack_label, needle_label, ret_label);
+ if (flags().strict_data_dependencies) {
+ if (ret)
+ *ret_origin = haystack_origin;
+ } else {
+ size_t needle_len = strlen(needle);
+ size_t len = ret ? ret + needle_len - haystack : strlen(haystack) + 1;
+ dfsan_origin o = dfsan_read_origin_of_first_taint(haystack, len);
+ if (o) {
+ *ret_origin = o;
+ } else {
+ o = dfsan_read_origin_of_first_taint(needle, needle_len + 1);
+ *ret_origin = o ? o : (haystack_label ? haystack_origin : needle_origin);
+ }
+ }
+
+ return ret;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_nanosleep(const struct timespec *req,
struct timespec *rem,
dfsan_label req_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_nanosleep(
+ const struct timespec *req, struct timespec *rem, dfsan_label req_label,
+ dfsan_label rem_label, dfsan_label *ret_label, dfsan_origin req_origin,
+ dfsan_origin rem_origin, dfsan_origin *ret_origin) {
+ return __dfsw_nanosleep(req, rem, req_label, rem_label, ret_label);
+}
+
+static void clear_msghdr_labels(size_t bytes_written, struct msghdr *msg) {
+ dfsan_set_label(0, msg, sizeof(*msg));
+ dfsan_set_label(0, msg->msg_name, msg->msg_namelen);
+ dfsan_set_label(0, msg->msg_control, msg->msg_controllen);
+ for (size_t i = 0; bytes_written > 0; ++i) {
+ assert(i < msg->msg_iovlen);
+ struct iovec *iov = &msg->msg_iov[i];
+ size_t iov_written =
+ bytes_written < iov->iov_len ? bytes_written : iov->iov_len;
+ dfsan_set_label(0, iov->iov_base, iov_written);
+ bytes_written -= iov_written;
+ }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_recvmmsg(
+ int sockfd, struct mmsghdr *msgvec, unsigned int vlen, int flags,
+ struct timespec *timeout, dfsan_label sockfd_label,
+ dfsan_label msgvec_label, dfsan_label vlen_label, dfsan_label flags_label,
+ dfsan_label timeout_label, dfsan_label *ret_label) {
+ int ret = recvmmsg(sockfd, msgvec, vlen, flags, timeout);
+ for (int i = 0; i < ret; ++i) {
+ dfsan_set_label(0, &msgvec[i].msg_len, sizeof(msgvec[i].msg_len));
+ clear_msghdr_labels(msgvec[i].msg_len, &msgvec[i].msg_hdr);
+ }
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_recvmmsg(
+ int sockfd, struct mmsghdr *msgvec, unsigned int vlen, int flags,
+ struct timespec *timeout, dfsan_label sockfd_label,
+ dfsan_label msgvec_label, dfsan_label vlen_label, dfsan_label flags_label,
+ dfsan_label timeout_label, dfsan_label *ret_label,
+ dfsan_origin sockfd_origin, dfsan_origin msgvec_origin,
+ dfsan_origin vlen_origin, dfsan_origin flags_origin,
+ dfsan_origin timeout_origin, dfsan_origin *ret_origin) {
+ return __dfsw_recvmmsg(sockfd, msgvec, vlen, flags, timeout, sockfd_label,
+ msgvec_label, vlen_label, flags_label, timeout_label,
+ ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfsw_recvmsg(
+ int sockfd, struct msghdr *msg, int flags, dfsan_label sockfd_label,
+ dfsan_label msg_label, dfsan_label flags_label, dfsan_label *ret_label) {
+ ssize_t ret = recvmsg(sockfd, msg, flags);
+ if (ret >= 0)
+ clear_msghdr_labels(ret, msg);
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfso_recvmsg(
+ int sockfd, struct msghdr *msg, int flags, dfsan_label sockfd_label,
+ dfsan_label msg_label, dfsan_label flags_label, dfsan_label *ret_label,
+ dfsan_origin sockfd_origin, dfsan_origin msg_origin,
+ dfsan_origin flags_origin, dfsan_origin *ret_origin) {
+ return __dfsw_recvmsg(sockfd, msg, flags, sockfd_label, msg_label,
+ flags_label, ret_label);
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int
__dfsw_socketpair(int domain, int type, int protocol, int sv[2],
dfsan_label domain_label, dfsan_label type_label,
return ret;
}
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_socketpair(
+ int domain, int type, int protocol, int sv[2], dfsan_label domain_label,
+ dfsan_label type_label, dfsan_label protocol_label, dfsan_label sv_label,
+ dfsan_label *ret_label, dfsan_origin domain_origin,
+ dfsan_origin type_origin, dfsan_origin protocol_origin,
+ dfsan_origin sv_origin, dfsan_origin *ret_origin) {
+ return __dfsw_socketpair(domain, type, protocol, sv, domain_label, type_label,
+ protocol_label, sv_label, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockopt(
+ int sockfd, int level, int optname, void *optval, socklen_t *optlen,
+ dfsan_label sockfd_label, dfsan_label level_label,
+ dfsan_label optname_label, dfsan_label optval_label,
+ dfsan_label optlen_label, dfsan_label *ret_label) {
+ int ret = getsockopt(sockfd, level, optname, optval, optlen);
+ if (ret != -1 && optval && optlen) {
+ dfsan_set_label(0, optlen, sizeof(*optlen));
+ dfsan_set_label(0, optval, *optlen);
+ }
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getsockopt(
+ int sockfd, int level, int optname, void *optval, socklen_t *optlen,
+ dfsan_label sockfd_label, dfsan_label level_label,
+ dfsan_label optname_label, dfsan_label optval_label,
+ dfsan_label optlen_label, dfsan_label *ret_label,
+ dfsan_origin sockfd_origin, dfsan_origin level_origin,
+ dfsan_origin optname_origin, dfsan_origin optval_origin,
+ dfsan_origin optlen_origin, dfsan_origin *ret_origin) {
+ return __dfsw_getsockopt(sockfd, level, optname, optval, optlen, sockfd_label,
+ level_label, optname_label, optval_label,
+ optlen_label, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockname(
+ int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+ dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+ dfsan_label *ret_label) {
+ socklen_t origlen = addrlen ? *addrlen : 0;
+ int ret = getsockname(sockfd, addr, addrlen);
+ if (ret != -1 && addr && addrlen) {
+ socklen_t written_bytes = origlen < *addrlen ? origlen : *addrlen;
+ dfsan_set_label(0, addrlen, sizeof(*addrlen));
+ dfsan_set_label(0, addr, written_bytes);
+ }
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getsockname(
+ int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+ dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+ dfsan_label *ret_label, dfsan_origin sockfd_origin,
+ dfsan_origin addr_origin, dfsan_origin addrlen_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_getsockname(sockfd, addr, addrlen, sockfd_label, addr_label,
+ addrlen_label, ret_label);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getpeername(
+ int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+ dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+ dfsan_label *ret_label) {
+ socklen_t origlen = addrlen ? *addrlen : 0;
+ int ret = getpeername(sockfd, addr, addrlen);
+ if (ret != -1 && addr && addrlen) {
+ socklen_t written_bytes = origlen < *addrlen ? origlen : *addrlen;
+ dfsan_set_label(0, addrlen, sizeof(*addrlen));
+ dfsan_set_label(0, addr, written_bytes);
+ }
+ *ret_label = 0;
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_getpeername(
+ int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+ dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+ dfsan_label *ret_label, dfsan_origin sockfd_origin,
+ dfsan_origin addr_origin, dfsan_origin addrlen_origin,
+ dfsan_origin *ret_origin) {
+ return __dfsw_getpeername(sockfd, addr, addrlen, sockfd_label, addr_label,
+ addrlen_label, ret_label);
+}
+
// Type of the trampoline function passed to the custom version of
// dfsan_set_write_callback.
typedef void (*write_trampoline_t)(
int fd, const void *buf, ssize_t count,
dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label);
+typedef void (*write_origin_trampoline_t)(
+ void *callback, int fd, const void *buf, ssize_t count,
+ dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label,
+ dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin);
+
// Calls to dfsan_set_write_callback() set the values in this struct.
// Calls to the custom version of write() read (and invoke) them.
static struct {
void *write_callback = nullptr;
} write_callback_info;
+static struct {
+ write_origin_trampoline_t write_callback_trampoline = nullptr;
+ void *write_callback = nullptr;
+} write_origin_callback_info;
+
SANITIZER_INTERFACE_ATTRIBUTE void
__dfsw_dfsan_set_write_callback(
write_trampoline_t write_callback_trampoline,
write_callback_info.write_callback = write_callback;
}
+SANITIZER_INTERFACE_ATTRIBUTE void __dfso_dfsan_set_write_callback(
+ write_origin_trampoline_t write_callback_trampoline, void *write_callback,
+ dfsan_label write_callback_label, dfsan_label *ret_label,
+ dfsan_origin write_callback_origin, dfsan_origin *ret_origin) {
+ write_origin_callback_info.write_callback_trampoline =
+ write_callback_trampoline;
+ write_origin_callback_info.write_callback = write_callback;
+}
+
SANITIZER_INTERFACE_ATTRIBUTE int
__dfsw_write(int fd, const void *buf, size_t count,
dfsan_label fd_label, dfsan_label buf_label,
*ret_label = 0;
return write(fd, buf, count);
}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfso_write(
+ int fd, const void *buf, size_t count, dfsan_label fd_label,
+ dfsan_label buf_label, dfsan_label count_label, dfsan_label *ret_label,
+ dfsan_origin fd_origin, dfsan_origin buf_origin, dfsan_origin count_origin,
+ dfsan_origin *ret_origin) {
+ if (write_origin_callback_info.write_callback) {
+ write_origin_callback_info.write_callback_trampoline(
+ write_origin_callback_info.write_callback, fd, buf, count, fd_label,
+ buf_label, count_label, fd_origin, buf_origin, count_origin);
+ }
+
+ *ret_label = 0;
+ return write(fd, buf, count);
+}
} // namespace __dfsan
// Type used to extract a dfsan_label with va_arg()
// positional arguments.
static int format_buffer(char *str, size_t size, const char *fmt,
dfsan_label *va_labels, dfsan_label *ret_label,
+ dfsan_origin *va_origins, dfsan_origin *ret_origin,
va_list ap) {
Formatter formatter(str, fmt, size);
default:
retval = formatter.format(va_arg(ap, int));
}
- dfsan_set_label(*va_labels++, formatter.str_cur(),
- formatter.num_written_bytes(retval));
+ if (va_origins == nullptr)
+ dfsan_set_label(*va_labels++, formatter.str_cur(),
+ formatter.num_written_bytes(retval));
+ else
+ dfsan_set_label_origin(*va_labels++, *va_origins++,
+ formatter.str_cur(),
+ formatter.num_written_bytes(retval));
end_fmt = true;
break;
} else {
retval = formatter.format(va_arg(ap, double));
}
- dfsan_set_label(*va_labels++, formatter.str_cur(),
- formatter.num_written_bytes(retval));
+ if (va_origins == nullptr)
+ dfsan_set_label(*va_labels++, formatter.str_cur(),
+ formatter.num_written_bytes(retval));
+ else
+ dfsan_set_label_origin(*va_labels++, *va_origins++,
+ formatter.str_cur(),
+ formatter.num_written_bytes(retval));
end_fmt = true;
break;
case 'c':
retval = formatter.format(va_arg(ap, int));
- dfsan_set_label(*va_labels++, formatter.str_cur(),
- formatter.num_written_bytes(retval));
+ if (va_origins == nullptr)
+ dfsan_set_label(*va_labels++, formatter.str_cur(),
+ formatter.num_written_bytes(retval));
+ else
+ dfsan_set_label_origin(*va_labels++, *va_origins++,
+ formatter.str_cur(),
+ formatter.num_written_bytes(retval));
end_fmt = true;
break;
case 's': {
char *arg = va_arg(ap, char *);
retval = formatter.format(arg);
+ if (va_origins) {
+ va_origins++;
+ dfsan_mem_origin_transfer(formatter.str_cur(), arg,
+ formatter.num_written_bytes(retval));
+ }
va_labels++;
internal_memcpy(shadow_for(formatter.str_cur()), shadow_for(arg),
sizeof(dfsan_label) *
case 'p':
retval = formatter.format(va_arg(ap, void *));
- dfsan_set_label(*va_labels++, formatter.str_cur(),
- formatter.num_written_bytes(retval));
+ if (va_origins == nullptr)
+ dfsan_set_label(*va_labels++, formatter.str_cur(),
+ formatter.num_written_bytes(retval));
+ else
+ dfsan_set_label_origin(*va_labels++, *va_origins++,
+ formatter.str_cur(),
+ formatter.num_written_bytes(retval));
end_fmt = true;
break;
int *ptr = va_arg(ap, int *);
*ptr = (int)formatter.str_off;
va_labels++;
+ if (va_origins)
+ va_origins++;
dfsan_set_label(0, ptr, sizeof(ptr));
end_fmt = true;
break;
case '*':
formatter.width = va_arg(ap, int);
va_labels++;
+ if (va_origins)
+ va_origins++;
break;
default:
}
*ret_label = 0;
+ if (ret_origin)
+ *ret_origin = 0;
// Number of bytes written in total.
return formatter.str_off;
dfsan_label *ret_label, ...) {
va_list ap;
va_start(ap, ret_label);
- int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, ap);
+ int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, nullptr,
+ nullptr, ap);
+ va_end(ap);
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_sprintf(char *str, const char *format, dfsan_label str_label,
+ dfsan_label format_label, dfsan_label *va_labels,
+ dfsan_label *ret_label, dfsan_origin str_origin,
+ dfsan_origin format_origin, dfsan_origin *va_origins,
+ dfsan_origin *ret_origin, ...) {
+ va_list ap;
+ va_start(ap, ret_origin);
+ int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, va_origins,
+ ret_origin, ap);
va_end(ap);
return ret;
}
dfsan_label *ret_label, ...) {
va_list ap;
va_start(ap, ret_label);
- int ret = format_buffer(str, size, format, va_labels, ret_label, ap);
+ int ret = format_buffer(str, size, format, va_labels, ret_label, nullptr,
+ nullptr, ap);
+ va_end(ap);
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfso_snprintf(char *str, size_t size, const char *format,
+ dfsan_label str_label, dfsan_label size_label,
+ dfsan_label format_label, dfsan_label *va_labels,
+ dfsan_label *ret_label, dfsan_origin str_origin,
+ dfsan_origin size_origin, dfsan_origin format_origin,
+ dfsan_origin *va_origins, dfsan_origin *ret_origin, ...) {
+ va_list ap;
+ va_start(ap, ret_origin);
+ int ret = format_buffer(str, size, format, va_labels, ret_label, va_origins,
+ ret_origin, ap);
va_end(ap);
return ret;
}
+static void BeforeFork() {
+ StackDepotLockAll();
+ GetChainedOriginDepot()->LockAll();
+}
+
+static void AfterFork() {
+ GetChainedOriginDepot()->UnlockAll();
+ StackDepotUnlockAll();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+pid_t __dfsw_fork(dfsan_label *ret_label) {
+ pid_t pid = fork();
+ *ret_label = 0;
+ return pid;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+pid_t __dfso_fork(dfsan_label *ret_label, dfsan_origin *ret_origin) {
+ BeforeFork();
+ pid_t pid = __dfsw_fork(ret_label);
+ AfterFork();
+ return pid;
+}
+
// Default empty implementations (weak). Users should redefine them.
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard, u32 *) {}
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_guard_init, u32 *,
--- /dev/null
+//===-- dfsan_flags.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataFlowSanitizer.
+//
+// DFSan flags.
+//===----------------------------------------------------------------------===//
+
+#ifndef DFSAN_FLAGS_H
+#define DFSAN_FLAGS_H
+
+namespace __dfsan {
+
+struct Flags {
+#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "dfsan_flags.inc"
+#undef DFSAN_FLAG
+
+ void SetDefaults();
+};
+
+extern Flags flags_data;
+inline Flags &flags() { return flags_data; }
+
+} // namespace __dfsan
+
+#endif // DFSAN_FLAGS_H
// DFSAN_FLAG(Type, Name, DefaultValue, Description)
// See COMMON_FLAG in sanitizer_flags.inc for more details.
-DFSAN_FLAG(bool, warn_unimplemented, true,
+DFSAN_FLAG(bool, warn_unimplemented, false,
"Whether to warn on unimplemented functions.")
DFSAN_FLAG(bool, warn_nonzero_labels, false,
"Whether to warn on unimplemented functions.")
"(e.g., when comparing strings, ignore the fact that the output of the"
"comparison might be data-dependent on the content of the strings). This"
"applies only to the custom functions defined in 'custom.c'.")
-DFSAN_FLAG(const char *, dump_labels_at_exit, "", "The path of the file where "
- "to dump the labels when the "
- "program terminates.")
-DFSAN_FLAG(bool, fast16labels, false,
- "Enables experimental mode where DFSan supports only 16 power-of-2 labels "
- "(1, 2, 4, 8, ... 32768) and the label union is computed as a bit-wise OR."
-)
+DFSAN_FLAG(
+ int, origin_history_size, Origin::kMaxDepth,
+ "The limit of origin chain length. Non-positive values mean unlimited.")
+DFSAN_FLAG(
+ int, origin_history_per_stack_limit, 20000,
+ "The limit of origin node's references count. "
+ "Non-positive values mean unlimited.")
+DFSAN_FLAG(int, store_context_size, 20,
+ "The depth limit of origin tracking stack traces.")
+DFSAN_FLAG(bool, check_origin_invariant, false,
+ "Whether to check if the origin invariant holds.")
+DFSAN_FLAG(bool, zero_in_malloc, true,
+ "Whether to zero shadow space of new allocated memory.")
+DFSAN_FLAG(bool, zero_in_free, true,
+ "Whether to zero shadow space of deallocated memory.")
// Interceptors for standard library functions.
//===----------------------------------------------------------------------===//
+#include <sys/syscall.h>
+#include <unistd.h>
+
#include "dfsan/dfsan.h"
+#include "dfsan/dfsan_thread.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_posix.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
using namespace __sanitizer;
+namespace {
+
+bool interceptors_initialized;
+
+} // namespace
+
+INTERCEPTOR(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size) {
+ return __dfsan::dfsan_reallocarray(ptr, nmemb, size);
+}
+
+INTERCEPTOR(void *, __libc_memalign, SIZE_T alignment, SIZE_T size) {
+ void *ptr = __dfsan::dfsan_memalign(alignment, size);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+INTERCEPTOR(void *, aligned_alloc, SIZE_T alignment, SIZE_T size) {
+ return __dfsan::dfsan_aligned_alloc(alignment, size);
+}
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
+ if (UNLIKELY(!__dfsan::dfsan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return __dfsan::dfsan_calloc(nmemb, size);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!__dfsan::dfsan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = __dfsan::dfsan_malloc(copy_size);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return __dfsan::dfsan_realloc(ptr, size);
+}
+
+INTERCEPTOR(void *, malloc, SIZE_T size) {
+ if (UNLIKELY(!__dfsan::dfsan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return __dfsan::dfsan_malloc(size);
+}
+
+INTERCEPTOR(void, free, void *ptr) {
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ return __dfsan::dfsan_deallocate(ptr);
+}
+
+INTERCEPTOR(void, cfree, void *ptr) {
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ return __dfsan::dfsan_deallocate(ptr);
+}
+
+INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
+ CHECK_NE(memptr, 0);
+ int res = __dfsan::dfsan_posix_memalign(memptr, alignment, size);
+ if (!res)
+ dfsan_set_label(0, memptr, sizeof(*memptr));
+ return res;
+}
+
+INTERCEPTOR(void *, memalign, SIZE_T alignment, SIZE_T size) {
+ return __dfsan::dfsan_memalign(alignment, size);
+}
+
+INTERCEPTOR(void *, valloc, SIZE_T size) { return __dfsan::dfsan_valloc(size); }
+
+INTERCEPTOR(void *, pvalloc, SIZE_T size) {
+ return __dfsan::dfsan_pvalloc(size);
+}
+
+INTERCEPTOR(void, mallinfo, __sanitizer_struct_mallinfo *sret) {
+ internal_memset(sret, 0, sizeof(*sret));
+ dfsan_set_label(0, sret, sizeof(*sret));
+}
+
+INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; }
+
+INTERCEPTOR(void, malloc_stats, void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+
+INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+#define ENSURE_DFSAN_INITED() \
+ do { \
+ CHECK(!__dfsan::dfsan_init_is_running); \
+ if (!__dfsan::dfsan_inited) { \
+ __dfsan::dfsan_init(); \
+ } \
+ } while (0)
+
+#define COMMON_INTERCEPTOR_ENTER(func, ...) \
+ if (__dfsan::dfsan_init_is_running) \
+ return REAL(func)(__VA_ARGS__); \
+ ENSURE_DFSAN_INITED(); \
+ dfsan_set_label(0, __errno_location(), sizeof(int)); /* NOLINT */
+
INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
int fd, OFF_T offset) {
+ if (common_flags()->detect_write_exec)
+ ReportMmapWriteExec(prot);
+ if (!__dfsan::dfsan_inited)
+ return (void *)internal_mmap(addr, length, prot, flags, fd, offset);
+ COMMON_INTERCEPTOR_ENTER(mmap, addr, length, prot, flags, fd, offset);
void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
- if (res != (void*)-1)
- dfsan_set_label(0, res, RoundUpTo(length, GetPageSize()));
+ if (res != (void *)-1) {
+ dfsan_set_label(0, res, RoundUpTo(length, GetPageSizeCached()));
+ }
return res;
}
INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
int fd, OFF64_T offset) {
+ if (common_flags()->detect_write_exec)
+ ReportMmapWriteExec(prot);
+ if (!__dfsan::dfsan_inited)
+ return (void *)internal_mmap(addr, length, prot, flags, fd, offset);
+ COMMON_INTERCEPTOR_ENTER(mmap64, addr, length, prot, flags, fd, offset);
void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
- if (res != (void*)-1)
- dfsan_set_label(0, res, RoundUpTo(length, GetPageSize()));
+ if (res != (void *)-1) {
+ dfsan_set_label(0, res, RoundUpTo(length, GetPageSizeCached()));
+ }
+ return res;
+}
+
+INTERCEPTOR(int, munmap, void *addr, SIZE_T length) {
+ if (!__dfsan::dfsan_inited)
+ return internal_munmap(addr, length);
+ COMMON_INTERCEPTOR_ENTER(munmap, addr, length);
+ int res = REAL(munmap)(addr, length);
+ if (res != -1)
+ dfsan_set_label(0, addr, RoundUpTo(length, GetPageSizeCached()));
+ return res;
+}
+
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
+ if (__dfsan::DFsanThread *t = __dfsan::GetCurrentThread()) { \
+ *begin = t->tls_begin(); \
+ *end = t->tls_end(); \
+ } else { \
+ *begin = *end = 0; \
+ }
+#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ptr, size) \
+ dfsan_set_label(0, ptr, size)
+
+INTERCEPTOR(void *, __tls_get_addr, void *arg) {
+ COMMON_INTERCEPTOR_ENTER(__tls_get_addr, arg);
+ void *res = REAL(__tls_get_addr)(arg);
+ uptr tls_begin, tls_end;
+ COMMON_INTERCEPTOR_GET_TLS_RANGE(&tls_begin, &tls_end);
+ DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, tls_begin, tls_end);
+ if (dtv) {
+ // New DTLS block has been allocated.
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE((void *)dtv->beg, dtv->size);
+ }
return res;
}
namespace __dfsan {
-void InitializeInterceptors() {
- static int inited = 0;
- CHECK_EQ(inited, 0);
+void initialize_interceptors() {
+ CHECK(!interceptors_initialized);
+ INTERCEPT_FUNCTION(aligned_alloc);
+ INTERCEPT_FUNCTION(calloc);
+ INTERCEPT_FUNCTION(cfree);
+ INTERCEPT_FUNCTION(free);
+ INTERCEPT_FUNCTION(mallinfo);
+ INTERCEPT_FUNCTION(malloc);
+ INTERCEPT_FUNCTION(malloc_stats);
+ INTERCEPT_FUNCTION(malloc_usable_size);
+ INTERCEPT_FUNCTION(mallopt);
+ INTERCEPT_FUNCTION(memalign);
INTERCEPT_FUNCTION(mmap);
INTERCEPT_FUNCTION(mmap64);
- inited = 1;
+ INTERCEPT_FUNCTION(munmap);
+ INTERCEPT_FUNCTION(posix_memalign);
+ INTERCEPT_FUNCTION(pvalloc);
+ INTERCEPT_FUNCTION(realloc);
+ INTERCEPT_FUNCTION(reallocarray);
+ INTERCEPT_FUNCTION(valloc);
+ INTERCEPT_FUNCTION(__tls_get_addr);
+ INTERCEPT_FUNCTION(__libc_memalign);
+
+ interceptors_initialized = true;
}
} // namespace __dfsan
--- /dev/null
+//===-- dfsan_new_delete.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataflowSanitizer.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+#include "dfsan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+
+using namespace __dfsan;
+
+// Fake std::nothrow_t and std::align_val_t to avoid including <new>.
+namespace std {
+struct nothrow_t {};
+enum class align_val_t : size_t {};
+} // namespace std
+
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+ void *res = dfsan_malloc(size); \
+ if (!nothrow && UNLIKELY(!res)) { \
+ BufferedStackTrace stack; \
+ ReportOutOfMemory(size, &stack); \
+ } \
+ return res
+#define OPERATOR_NEW_BODY_ALIGN(nothrow) \
+ void *res = dfsan_memalign((uptr)align, size); \
+ if (!nothrow && UNLIKELY(!res)) { \
+ BufferedStackTrace stack; \
+ ReportOutOfMemory(size, &stack); \
+ } \
+ return res;
+
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const &) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const &) {
+ OPERATOR_NEW_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align) {
+ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align) {
+ OPERATOR_NEW_BODY_ALIGN(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_NEW_BODY_ALIGN(true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY \
+ if (ptr) \
+ dfsan_deallocate(ptr)
+
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr)NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size)NOEXCEPT { OPERATOR_DELETE_BODY; }
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align)NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size, std::align_val_t align)NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size,
+ std::align_val_t align) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
--- /dev/null
+//===-- dfsan_origin.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataFlowSanitizer.
+//
+// Origin id utils.
+//===----------------------------------------------------------------------===//
+
+#ifndef DFSAN_ORIGIN_H
+#define DFSAN_ORIGIN_H
+
+#include "dfsan_chained_origin_depot.h"
+#include "dfsan_flags.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+namespace __dfsan {
+
+// Origin handling.
+//
+// Origin is a 32-bit identifier that is attached to any taint value in the
+// program and describes how this memory came to be tainted.
+//
+// Chained origin id is like:
+// zzzz xxxx xxxx xxxx
+//
+// Chained origin id describes an event of storing a taint value to
+// memory. The xxx part is a value of ChainedOriginDepot, which is a mapping of
+// (stack_id, prev_id) -> id, where
+// * stack_id describes the event.
+// StackDepot keeps a mapping between those and corresponding stack traces.
+// * prev_id is another origin id that describes the earlier part of the
+// taint value history. 0 prev_id indicates the start of a chain.
+// Following a chain of prev_id provides the full recorded history of a taint
+// value.
+//
+// This, effectively, defines a forest where nodes are points in value history
+// marked with origin ids, and edges are events that are marked with stack_id.
+//
+// The "zzzz" bits of chained origin id are used to store the length of the
+// origin chain.
+
+class Origin {
+ public:
+ static bool isValidId(u32 id) { return id != 0; }
+
+ u32 raw_id() const { return raw_id_; }
+
+ bool isChainedOrigin() const { return Origin::isValidId(raw_id_); }
+
+ u32 getChainedId() const {
+ CHECK(Origin::isValidId(raw_id_));
+ return raw_id_ & kChainedIdMask;
+ }
+
+ // Returns the next origin in the chain and the current stack trace.
+ //
+ // It scans a partition of StackDepot linearly, and is used only by origin
+ // tracking report.
+ Origin getNextChainedOrigin(StackTrace *stack) const {
+ CHECK(Origin::isValidId(raw_id_));
+ u32 prev_id;
+ u32 stack_id = GetChainedOriginDepot()->Get(getChainedId(), &prev_id);
+ if (stack)
+ *stack = StackDepotGet(stack_id);
+ return Origin(prev_id);
+ }
+
+ static Origin CreateChainedOrigin(Origin prev, StackTrace *stack) {
+ int depth = prev.isChainedOrigin() ? prev.depth() : -1;
+ // depth is the length of the chain minus 1.
+ // origin_history_size of 0 means unlimited depth.
+ if (flags().origin_history_size > 0) {
+ ++depth;
+ if (depth >= flags().origin_history_size || depth > kMaxDepth)
+ return prev;
+ }
+
+ StackDepotHandle h = StackDepotPut_WithHandle(*stack);
+ if (!h.valid())
+ return prev;
+
+ if (flags().origin_history_per_stack_limit > 0) {
+ int use_count = h.use_count();
+ if (use_count > flags().origin_history_per_stack_limit)
+ return prev;
+ }
+
+ u32 chained_id;
+ bool inserted =
+ GetChainedOriginDepot()->Put(h.id(), prev.raw_id(), &chained_id);
+ CHECK((chained_id & kChainedIdMask) == chained_id);
+
+ if (inserted && flags().origin_history_per_stack_limit > 0)
+ h.inc_use_count_unsafe();
+
+ return Origin((depth << kDepthShift) | chained_id);
+ }
+
+ static Origin FromRawId(u32 id) { return Origin(id); }
+
+ private:
+ static const int kDepthBits = 4;
+ static const int kDepthShift = 32 - kDepthBits;
+
+ static const u32 kChainedIdMask = ((u32)-1) >> kDepthBits;
+
+ u32 raw_id_;
+
+ explicit Origin(u32 raw_id) : raw_id_(raw_id) {}
+
+ int depth() const {
+ CHECK(isChainedOrigin());
+ return (raw_id_ >> kDepthShift) & ((1 << kDepthBits) - 1);
+ }
+
+ public:
+ static const int kMaxDepth = (1 << kDepthBits) - 1;
+};
+
+} // namespace __dfsan
+
+#endif // DFSAN_ORIGIN_H
#ifndef DFSAN_PLATFORM_H
#define DFSAN_PLATFORM_H
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_platform.h"
+
namespace __dfsan {
-#if defined(__x86_64__)
-struct Mapping {
- static const uptr kShadowAddr = 0x10000;
- static const uptr kUnionTableAddr = 0x200000000000;
- static const uptr kAppAddr = 0x700000008000;
- static const uptr kShadowMask = ~0x700000000000;
-};
-#elif defined(__mips64)
-struct Mapping {
- static const uptr kShadowAddr = 0x10000;
- static const uptr kUnionTableAddr = 0x2000000000;
- static const uptr kAppAddr = 0xF000008000;
- static const uptr kShadowMask = ~0xF000000000;
-};
-#elif defined(__aarch64__)
-struct Mapping39 {
- static const uptr kShadowAddr = 0x10000;
- static const uptr kUnionTableAddr = 0x1000000000;
- static const uptr kAppAddr = 0x7000008000;
- static const uptr kShadowMask = ~0x7800000000;
-};
+using __sanitizer::uptr;
-struct Mapping42 {
- static const uptr kShadowAddr = 0x10000;
- static const uptr kUnionTableAddr = 0x8000000000;
- static const uptr kAppAddr = 0x3ff00008000;
- static const uptr kShadowMask = ~0x3c000000000;
-};
+// TODO: The memory mapping code to setup a 1:1 shadow is based on msan.
+// Consider refactoring these into a shared implementation.
-struct Mapping48 {
- static const uptr kShadowAddr = 0x10000;
- static const uptr kUnionTableAddr = 0x8000000000;
- static const uptr kAppAddr = 0xffff00008000;
- static const uptr kShadowMask = ~0xfffff0000000;
+struct MappingDesc {
+ uptr start;
+ uptr end;
+ enum Type { INVALID, APP, SHADOW, ORIGIN } type;
+ const char *name;
};
-extern int vmaSize;
-# define DFSAN_RUNTIME_VMA 1
-#else
-# error "DFSan not supported for this platform!"
-#endif
-
-enum MappingType {
- MAPPING_SHADOW_ADDR,
- MAPPING_UNION_TABLE_ADDR,
- MAPPING_APP_ADDR,
- MAPPING_SHADOW_MASK
-};
+#if SANITIZER_LINUX && SANITIZER_WORDSIZE == 64
-template<typename Mapping, int Type>
-uptr MappingImpl(void) {
- switch (Type) {
- case MAPPING_SHADOW_ADDR: return Mapping::kShadowAddr;
- case MAPPING_UNION_TABLE_ADDR: return Mapping::kUnionTableAddr;
- case MAPPING_APP_ADDR: return Mapping::kAppAddr;
- case MAPPING_SHADOW_MASK: return Mapping::kShadowMask;
- }
-}
+// All of the following configurations are supported.
+// ASLR disabled: main executable and DSOs at 0x555550000000
+// PIE and ASLR: main executable and DSOs at 0x7f0000000000
+// non-PIE: main executable below 0x100000000, DSOs at 0x7f0000000000
+// Heap at 0x700000000000.
+const MappingDesc kMemoryLayout[] = {
+ {0x000000000000ULL, 0x010000000000ULL, MappingDesc::APP, "app-1"},
+ {0x010000000000ULL, 0x100000000000ULL, MappingDesc::SHADOW, "shadow-2"},
+ {0x100000000000ULL, 0x110000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x110000000000ULL, 0x200000000000ULL, MappingDesc::ORIGIN, "origin-2"},
+ {0x200000000000ULL, 0x300000000000ULL, MappingDesc::SHADOW, "shadow-3"},
+ {0x300000000000ULL, 0x400000000000ULL, MappingDesc::ORIGIN, "origin-3"},
+ {0x400000000000ULL, 0x500000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x500000000000ULL, 0x510000000000ULL, MappingDesc::SHADOW, "shadow-1"},
+ {0x510000000000ULL, 0x600000000000ULL, MappingDesc::APP, "app-2"},
+ {0x600000000000ULL, 0x610000000000ULL, MappingDesc::ORIGIN, "origin-1"},
+ {0x610000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"},
+ {0x700000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}};
+# define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL)
+# define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL)
-template<int Type>
-uptr MappingArchImpl(void) {
-#ifdef __aarch64__
- switch (vmaSize) {
- case 39: return MappingImpl<Mapping39, Type>();
- case 42: return MappingImpl<Mapping42, Type>();
- case 48: return MappingImpl<Mapping48, Type>();
- }
- DCHECK(0);
- return 0;
#else
- return MappingImpl<Mapping, Type>();
+# error "Unsupported platform"
#endif
-}
-ALWAYS_INLINE
-uptr ShadowAddr() {
- return MappingArchImpl<MAPPING_SHADOW_ADDR>();
-}
+const uptr kMemoryLayoutSize = sizeof(kMemoryLayout) / sizeof(kMemoryLayout[0]);
-ALWAYS_INLINE
-uptr UnionTableAddr() {
- return MappingArchImpl<MAPPING_UNION_TABLE_ADDR>();
-}
+#define MEM_TO_ORIGIN(mem) (SHADOW_TO_ORIGIN(MEM_TO_SHADOW((mem))))
-ALWAYS_INLINE
-uptr AppAddr() {
- return MappingArchImpl<MAPPING_APP_ADDR>();
+#ifndef __clang__
+__attribute__((optimize("unroll-loops")))
+#endif
+inline bool
+addr_is_type(uptr addr, MappingDesc::Type mapping_type) {
+// It is critical for performance that this loop is unrolled (because then it is
+// simplified into just a few constant comparisons).
+#ifdef __clang__
+# pragma unroll
+#endif
+ for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
+ if (kMemoryLayout[i].type == mapping_type &&
+ addr >= kMemoryLayout[i].start && addr < kMemoryLayout[i].end)
+ return true;
+ return false;
}
-ALWAYS_INLINE
-uptr ShadowMask() {
- return MappingArchImpl<MAPPING_SHADOW_MASK>();
-}
+#define MEM_IS_APP(mem) addr_is_type((uptr)(mem), MappingDesc::APP)
+#define MEM_IS_SHADOW(mem) addr_is_type((uptr)(mem), MappingDesc::SHADOW)
+#define MEM_IS_ORIGIN(mem) addr_is_type((uptr)(mem), MappingDesc::ORIGIN)
} // namespace __dfsan
--- /dev/null
+#include "dfsan_thread.h"
+
+#include <pthread.h>
+
+#include "dfsan.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __dfsan {
+
+DFsanThread *DFsanThread::Create(void *start_routine_trampoline,
+ thread_callback_t start_routine, void *arg,
+ bool track_origins) {
+ uptr PageSize = GetPageSizeCached();
+ uptr size = RoundUpTo(sizeof(DFsanThread), PageSize);
+ DFsanThread *thread = (DFsanThread *)MmapOrDie(size, __func__);
+ thread->start_routine_trampoline_ = start_routine_trampoline;
+ thread->start_routine_ = start_routine;
+ thread->arg_ = arg;
+ thread->track_origins_ = track_origins;
+ thread->destructor_iterations_ = GetPthreadDestructorIterations();
+
+ return thread;
+}
+
+void DFsanThread::SetThreadStackAndTls() {
+ uptr tls_size = 0;
+ uptr stack_size = 0;
+ GetThreadStackAndTls(IsMainThread(), &stack_.bottom, &stack_size, &tls_begin_,
+ &tls_size);
+ stack_.top = stack_.bottom + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+
+ int local;
+ CHECK(AddrIsInStack((uptr)&local));
+}
+
+void DFsanThread::ClearShadowForThreadStackAndTLS() {
+ dfsan_set_label(0, (void *)stack_.bottom, stack_.top - stack_.bottom);
+ if (tls_begin_ != tls_end_)
+ dfsan_set_label(0, (void *)tls_begin_, tls_end_ - tls_begin_);
+ DTLS *dtls = DTLS_Get();
+ CHECK_NE(dtls, 0);
+ ForEachDVT(dtls, [](const DTLS::DTV &dtv, int id) {
+ dfsan_set_label(0, (void *)(dtv.beg), dtv.size);
+ });
+}
+
+void DFsanThread::Init() {
+ SetThreadStackAndTls();
+ ClearShadowForThreadStackAndTLS();
+}
+
+void DFsanThread::TSDDtor(void *tsd) {
+ DFsanThread *t = (DFsanThread *)tsd;
+ t->Destroy();
+}
+
+void DFsanThread::Destroy() {
+ malloc_storage().CommitBack();
+ // We also clear the shadow on thread destruction because
+ // some code may still be executing in later TSD destructors
+ // and we don't want it to have any poisoned stack.
+ ClearShadowForThreadStackAndTLS();
+ uptr size = RoundUpTo(sizeof(DFsanThread), GetPageSizeCached());
+ UnmapOrDie(this, size);
+ DTLS_Destroy();
+}
+
+thread_return_t DFsanThread::ThreadStart() {
+ Init();
+
+ if (!start_routine_) {
+ // start_routine_ == 0 if we're on the main thread or on one of the
+ // OS X libdispatch worker threads. But nobody is supposed to call
+ // ThreadStart() for the worker threads.
+ return 0;
+ }
+
+ CHECK(start_routine_trampoline_);
+
+ typedef void *(*thread_callback_trampoline_t)(void *, void *, dfsan_label,
+ dfsan_label *);
+ typedef void *(*thread_callback_origin_trampoline_t)(
+ void *, void *, dfsan_label, dfsan_label *, dfsan_origin, dfsan_origin *);
+
+ dfsan_label ret_label;
+ if (!track_origins_)
+ return ((thread_callback_trampoline_t)
+ start_routine_trampoline_)((void *)start_routine_, arg_, 0,
+ &ret_label);
+
+ dfsan_origin ret_origin;
+ return ((thread_callback_origin_trampoline_t)
+ start_routine_trampoline_)((void *)start_routine_, arg_, 0,
+ &ret_label, 0, &ret_origin);
+}
+
+DFsanThread::StackBounds DFsanThread::GetStackBounds() const {
+ return {stack_.bottom, stack_.top};
+}
+
+uptr DFsanThread::stack_top() { return GetStackBounds().top; }
+
+uptr DFsanThread::stack_bottom() { return GetStackBounds().bottom; }
+
+bool DFsanThread::AddrIsInStack(uptr addr) {
+ const auto bounds = GetStackBounds();
+ return addr >= bounds.bottom && addr < bounds.top;
+}
+
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+
+void DFsanTSDInit(void (*destructor)(void *tsd)) {
+ CHECK(!tsd_key_inited);
+ tsd_key_inited = true;
+ CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+}
+
+static THREADLOCAL DFsanThread *dfsan_current_thread;
+
+DFsanThread *GetCurrentThread() { return dfsan_current_thread; }
+
+void SetCurrentThread(DFsanThread *t) {
+ // Make sure we do not reset the current DFsanThread.
+ CHECK_EQ(0, dfsan_current_thread);
+ dfsan_current_thread = t;
+ // Make sure that DFsanTSDDtor gets called at the end.
+ CHECK(tsd_key_inited);
+ pthread_setspecific(tsd_key, t);
+}
+
+void DFsanTSDDtor(void *tsd) {
+ DFsanThread *t = (DFsanThread *)tsd;
+ if (t->destructor_iterations_ > 1) {
+ t->destructor_iterations_--;
+ CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+ return;
+ }
+ dfsan_current_thread = nullptr;
+ // Make sure that signal handler can not see a stale current thread pointer.
+ atomic_signal_fence(memory_order_seq_cst);
+ DFsanThread::TSDDtor(tsd);
+}
+
+} // namespace __dfsan
--- /dev/null
+//===-- dfsan_thread.h -------------------------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of DataFlowSanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DFSAN_THREAD_H
+#define DFSAN_THREAD_H
+
+#include "dfsan_allocator.h"
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace __dfsan {
+
+class DFsanThread {
+ public:
+ // NOTE: There is no DFsanThread constructor. It is allocated
+ // via mmap() and *must* be valid in zero-initialized state.
+
+ static DFsanThread *Create(void *start_routine_trampoline,
+ thread_callback_t start_routine, void *arg,
+ bool track_origins = false);
+ static void TSDDtor(void *tsd);
+ void Destroy();
+
+ void Init(); // Should be called from the thread itself.
+ thread_return_t ThreadStart();
+
+ uptr stack_top();
+ uptr stack_bottom();
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ bool IsMainThread() { return start_routine_ == nullptr; }
+
+ bool InSignalHandler() { return in_signal_handler_; }
+ void EnterSignalHandler() { in_signal_handler_++; }
+ void LeaveSignalHandler() { in_signal_handler_--; }
+
+ DFsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+
+ int destructor_iterations_;
+
+ private:
+ void SetThreadStackAndTls();
+ void ClearShadowForThreadStackAndTLS();
+ struct StackBounds {
+ uptr bottom;
+ uptr top;
+ };
+ StackBounds GetStackBounds() const;
+
+ bool AddrIsInStack(uptr addr);
+
+ void *start_routine_trampoline_;
+ thread_callback_t start_routine_;
+ void *arg_;
+ bool track_origins_;
+
+ StackBounds stack_;
+
+ uptr tls_begin_;
+ uptr tls_end_;
+
+ unsigned in_signal_handler_;
+
+ DFsanThreadLocalMallocStorage malloc_storage_;
+};
+
+DFsanThread *GetCurrentThread();
+void SetCurrentThread(DFsanThread *t);
+void DFsanTSDInit(void (*destructor)(void *tsd));
+void DFsanTSDDtor(void *tsd);
+
+} // namespace __dfsan
+
+#endif // DFSAN_THREAD_H
fun:dfsan_set_write_callback=custom
fun:dfsan_flush=uninstrumented
fun:dfsan_flush=discard
+fun:dfsan_print_origin_trace=uninstrumented
+fun:dfsan_print_origin_trace=discard
+fun:dfsan_sprint_origin_trace=uninstrumented
+fun:dfsan_sprint_origin_trace=discard
+fun:dfsan_sprint_stack_trace=uninstrumented
+fun:dfsan_sprint_stack_trace=discard
+fun:dfsan_get_origin=uninstrumented
+fun:dfsan_get_origin=custom
+fun:dfsan_get_init_origin=uninstrumented
+fun:dfsan_get_init_origin=discard
+fun:dfsan_get_track_origins=uninstrumented
+fun:dfsan_get_track_origins=discard
###############################################################################
# glibc
###############################################################################
+# Functions of memory allocators
+fun:__libc_memalign=discard
+fun:aligned_alloc=discard
+fun:calloc=discard
+fun:cfree=discard
+fun:mallinfo=discard
fun:malloc=discard
fun:free=discard
+fun:malloc_stats=discard
+fun:malloc_usable_size=discard
+fun:mallopt=discard
+fun:memalign=discard
+fun:posix_memalign=discard
+fun:pvalloc=discard
+fun:realloc=discard
+fun:reallocarray=discard
+fun:valloc=discard
# Functions that return a value that depends on the input, but the output might
# not be necessarily data-dependent on the input.
fun:toupper=functional
# Functions that return a value that is data-dependent on the input.
+fun:__isinf=functional
+fun:__isinff=functional
+fun:__signbit=functional
+fun:__signbitf=functional
+fun:__signbitl=functional
fun:btowc=functional
fun:exp=functional
fun:exp2=functional
+fun:expf=functional
+fun:expl=functional
fun:fabs=functional
fun:finite=functional
+fun:finitef=functional
+fun:finitel=functional
fun:floor=functional
fun:fmod=functional
fun:isinf=functional
+fun:isinff=functional
+fun:isinfl=functional
fun:isnan=functional
+fun:isnanf=functional
+fun:isnanl=functional
fun:log=functional
+fun:log1p=functional
+fun:log1pf=functional
+fun:log1pl=functional
+fun:log2=functional
+fun:log2f=functional
+fun:log2l=functional
fun:modf=functional
+fun:nextafter=functional
+fun:nextafterf=functional
+fun:nextafterl=functional
+fun:nexttoward=functional
+fun:nexttowardf=functional
+fun:nexttowardl=functional
fun:pow=functional
+fun:powf=functional
+fun:powl=functional
fun:round=functional
fun:sqrt=functional
+fun:sqrtf=functional
+fun:sqrtl=functional
fun:wctob=functional
# Functions that produce an output that does not depend on the input (shadow is
# zeroed automatically).
fun:__assert_fail=discard
+fun:__cmsg_nxthdr=discard
fun:__ctype_b_loc=discard
fun:__cxa_atexit=discard
fun:__errno_location=discard
fun:close=discard
fun:closedir=discard
fun:connect=discard
+fun:creat=discard
fun:dladdr=discard
fun:dlclose=discard
+fun:epoll_create=discard
+fun:epoll_create1=discard
+fun:epoll_ctl=discard
fun:fclose=discard
fun:feof=discard
fun:ferror=discard
fun:mmap=discard
fun:munmap=discard
fun:open=discard
+fun:openat=discard
fun:pipe=discard
fun:posix_fadvise=discard
-fun:posix_memalign=discard
fun:prctl=discard
fun:printf=discard
fun:pthread_sigmask=discard
fun:syscall=discard
fun:unlink=discard
fun:uselocale=discard
+fun:vfprintf=discard
# Functions that produce output does not depend on the input (need to zero the
# shadow manually).
-fun:calloc=custom
+fun:_dl_get_tls_static_info=custom
fun:clock_gettime=custom
fun:dlopen=custom
+fun:epoll_wait=custom
fun:fgets=custom
fun:fstat=custom
fun:getcwd=custom
fun:get_current_dir_name=custom
fun:gethostname=custom
+fun:getpeername=custom
fun:getrlimit=custom
fun:getrusage=custom
+fun:getsockname=custom
+fun:getsockopt=custom
fun:nanosleep=custom
fun:pread=custom
fun:read=custom
+fun:recvmmsg=custom
+fun:recvmsg=custom
+fun:sigaltstack=custom
fun:socketpair=custom
fun:stat=custom
fun:time=custom
fun:inet_pton=custom
fun:localtime_r=custom
fun:memcpy=custom
+fun:memmove=custom
fun:memset=custom
fun:strcpy=custom
fun:strdup=custom
fun:strtoll=custom
fun:strtoul=custom
fun:strtoull=custom
+fun:strcat=custom
# Functions that produce an output that is computed from the input, but is not
# necessarily data dependent.
+fun:bcmp=custom
fun:memchr=custom
fun:memcmp=custom
fun:strcasecmp=custom
fun:strlen=custom
fun:strncasecmp=custom
fun:strncmp=custom
+fun:strpbrk=custom
fun:strrchr=custom
fun:strstr=custom
fun:select=custom
fun:sigemptyset=custom
fun:sigaction=custom
+fun:signal=custom
fun:gettimeofday=custom
# sprintf-like
fun:asprintf=discard
fun:qsort=discard
+# fork
+fun:fork=custom
+
###############################################################################
# pthread
###############################################################################
+fun:__pthread_register_cancel=discard
+fun:__pthread_unregister_cancel=discard
+fun:pthread_attr_destroy=discard
+fun:pthread_attr_getaffinity_np=discard
+fun:pthread_attr_getdetachstate=discard
+fun:pthread_attr_getguardsize=discard
+fun:pthread_attr_getinheritsched=discard
+fun:pthread_attr_getschedparam=discard
+fun:pthread_attr_getschedpolicy=discard
+fun:pthread_attr_getscope=discard
+fun:pthread_attr_getstack=discard
+fun:pthread_attr_getstackaddr=disacrd
+fun:pthread_attr_getstacksize=discard
+fun:pthread_attr_init=discard
+fun:pthread_attr_setaffinity_np=discard
+fun:pthread_attr_setdetachstate=discard
+fun:pthread_attr_setguardsize=discard
+fun:pthread_attr_setinheritsched=discard
+fun:pthread_attr_setschedparam=discard
+fun:pthread_attr_setschedpolicy=discard
+fun:pthread_attr_setscope=discard
+fun:pthread_attr_setstack=discard
+fun:pthread_attr_setstackaddr=discard
+fun:pthread_attr_setstacksize=discard
fun:pthread_equal=discard
+fun:pthread_getschedparam=discard
fun:pthread_getspecific=discard
fun:pthread_key_create=discard
fun:pthread_key_delete=discard
fun:pthread_mutexattr_destroy=discard
fun:pthread_mutexattr_init=discard
fun:pthread_mutexattr_settype=discard
+fun:pthread_rwlock_destroy=discard
+fun:pthread_rwlock_init=discard
+fun:pthread_rwlock_rdlock=discard
+fun:pthread_rwlock_timedrdlock=discard
+fun:pthread_rwlock_timedwrlock=discard
+fun:pthread_rwlock_tryrdlock=discard
+fun:pthread_rwlock_trywrlock=discard
+fun:pthread_rwlock_wrlock=discard
+fun:pthread_rwlock_unlock=discard
+fun:pthread_setschedparam=discard
+fun:pthread_setname_np=discard
fun:pthread_once=discard
fun:pthread_self=discard
fun:pthread_setspecific=discard
# Functions that take a callback (wrap the callback manually).
fun:pthread_create=custom
+# Functions that produce output does not depend on the input (need to zero the
+# shadow manually).
+fun:pthread_join=custom
+
###############################################################################
# libffi/libgo
###############################################################################
fun:__sanitizer_cov_pcs_init=uninstrumented
fun:__sanitizer_cov_pcs_init=discard
+fun:__sanitizer_get_current_allocated_bytes=uninstrumented
+fun:__sanitizer_get_current_allocated_bytes=discard
+fun:__sanitizer_get_heap_size=uninstrumented
+fun:__sanitizer_get_heap_size=discard
+fun:__sanitizer_get_free_bytes=uninstrumented
+fun:__sanitizer_get_free_bytes=discard
+fun:__sanitizer_get_unmapped_bytes=uninstrumented
+fun:__sanitizer_get_unmapped_bytes=discard
+fun:__sanitizer_get_estimated_allocated_size=uninstrumented
+fun:__sanitizer_get_estimated_allocated_size=discard
+fun:__sanitizer_get_ownership=uninstrumented
+fun:__sanitizer_get_ownership=discard
+fun:__sanitizer_get_allocated_size=uninstrumented
+fun:__sanitizer_get_allocated_size=discard
+fun:__sanitizer_print_stack_trace=uninstrumented
+fun:__sanitizer_print_stack_trace=discard
+
+fun:TcmallocSlab_Internal_PushBatch_FixedShift=uninstrumented
+fun:TcmallocSlab_Internal_PushBatch_FixedShift=discard
+fun:TcmallocSlab_Internal_PushBatch_FixedShift_VCPU=uninstrumented
+fun:TcmallocSlab_Internal_PushBatch_FixedShift_VCPU=discard
+fun:TcmallocSlab_Internal_PerCpuCmpxchg64=uninstrumented
+fun:TcmallocSlab_Internal_PerCpuCmpxchg64=discard
+fun:TcmallocSlab_Internal_PerCpuCmpxchg64_VCPU=uninstrumented
+fun:TcmallocSlab_Internal_PerCpuCmpxchg64_VCPU=discard
+fun:TcmallocSlab_Internal_PopBatch_FixedShift=uninstrumented
+fun:TcmallocSlab_Internal_PopBatch_FixedShift=discard
+fun:TcmallocSlab_Internal_PopBatch_FixedShift_VCPU=uninstrumented
+fun:TcmallocSlab_Internal_PopBatch_FixedShift_VCPU=discard
+
# Ignores the dfsan wrappers.
fun:__dfsw_*=uninstrumented
fun:__dfsw_*=discard
+fun:__dfso_*=uninstrumented
+fun:__dfso_*=discard
# Ignore __sanitizer_cov_trace* because they are implemented elsewhere.
trap on_exit EXIT
grep -E "^fun:.*=custom" ${DFSAN_ABI_LIST} \
- | grep -v "dfsan_get_label\|__sanitizer_cov_trace" \
+ | grep -v "dfsan_get_label\|dfsan_get_origin\|__sanitizer_cov_trace" \
| sed "s/^fun:\(.*\)=custom.*/\1/" | sort > $DIFF_A
grep -E "__dfsw.*\(" ${DFSAN_CUSTOM_WRAPPERS} \
| grep -v "__sanitizer_cov_trace" \
- | sed "s/.*__dfsw_\(.*\)(.*/\1/" | sort > $DIFF_B
+ | sed "s/.*__dfsw_\(.*\)(.*/\1/" | sort | uniq > $DIFF_B
diff -u $DIFF_A $DIFF_B > ${DIFFOUT}
if [ $? -ne 0 ]
then
grep -E __dfsw_ ${DFSAN_CUSTOM_WRAPPERS} \
| grep -v "__sanitizer_cov_trace" \
- | sed "s/.*__dfsw_\([^(]*\).*/\1/" | sort > $DIFF_A
+ | sed "s/.*__dfsw_\([^(]*\).*/\1/" | sort | uniq > $DIFF_A
grep -E "^[[:space:]]*test_.*\(\);" ${DFSAN_CUSTOM_TESTS} \
| sed "s/.*test_\(.*\)();/\1/" | sort > $DIFF_B
diff -u $DIFF_A $DIFF_B > ${DIFFOUT}
FuzzerUtil.h
FuzzerValueBitMap.h)
+include_directories(../../include)
+
CHECK_CXX_SOURCE_COMPILES("
static thread_local int blah;
int main() {
}
" HAS_THREAD_LOCAL)
-set(LIBFUZZER_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+set(LIBFUZZER_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
if(OS_NAME MATCHES "Linux|Fuchsia" AND
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
- list(APPEND LIBFUZZER_CFLAGS -nostdinc++ -D_LIBCPP_ABI_VERSION=Fuzzer)
- # Remove -stdlib= which is unused when passing -nostdinc++.
- string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+ list(APPEND LIBFUZZER_CFLAGS -D_LIBCPP_ABI_VERSION=Fuzzer)
+ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS)
elseif(TARGET cxx-headers OR HAVE_LIBCXX)
+ # libFuzzer uses C++ standard library headers.
set(LIBFUZZER_DEPS cxx-headers)
endif()
CFLAGS ${LIBFUZZER_CFLAGS}
DEPS ${LIBFUZZER_DEPS})
+add_compiler_rt_object_libraries(RTfuzzer_interceptors
+ OS ${FUZZER_SUPPORTED_OS}
+ ARCHS ${FUZZER_SUPPORTED_ARCH}
+ SOURCES FuzzerInterceptors.cpp
+ CFLAGS ${LIBFUZZER_CFLAGS}
+ DEPS ${LIBFUZZER_DEPS})
+
add_compiler_rt_runtime(clang_rt.fuzzer
STATIC
OS ${FUZZER_SUPPORTED_OS}
CFLAGS ${LIBFUZZER_CFLAGS}
PARENT_TARGET fuzzer)
+add_compiler_rt_runtime(clang_rt.fuzzer_interceptors
+ STATIC
+ OS ${FUZZER_SUPPORTED_OS}
+ ARCHS ${FUZZER_SUPPORTED_ARCH}
+ OBJECT_LIBS RTfuzzer_interceptors
+ CFLAGS ${LIBFUZZER_CFLAGS}
+ PARENT_TARGET fuzzer)
+
if(OS_NAME MATCHES "Linux|Fuchsia" AND
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
add_dependencies(RTfuzzer_main.${arch} libcxx_fuzzer_${arch}-build)
+ target_compile_options(RTfuzzer_interceptors.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
+ add_dependencies(RTfuzzer_interceptors.${arch} libcxx_fuzzer_${arch}-build)
partially_link_libcxx(fuzzer_no_main ${LIBCXX_${arch}_PREFIX} ${arch})
+ partially_link_libcxx(fuzzer_interceptors ${LIBCXX_${arch}_PREFIX} ${arch})
partially_link_libcxx(fuzzer ${LIBCXX_${arch}_PREFIX} ${arch})
endforeach()
endif()
inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); }
inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); }
-inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); }
inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); }
} // namespace fuzzer
return 64;
}
-inline uint32_t Clz(uint32_t X) {
- unsigned long LeadZeroIdx = 0;
- if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx;
- return 32;
-}
-
inline int Popcountll(unsigned long long X) {
#if !defined(_M_ARM) && !defined(_M_X64)
return __popcnt(X) + __popcnt(X >> 32);
#include "FuzzerSHA1.h"
#include "FuzzerTracePC.h"
#include <algorithm>
+#include <chrono>
#include <numeric>
#include <random>
#include <unordered_set>
struct InputInfo {
Unit U; // The actual input data.
+ std::chrono::microseconds TimeOfUnit;
uint8_t Sha1[kSHA1NumBytes]; // Checksum.
// Number of features that this input has and no smaller input has.
size_t NumFeatures = 0;
// Stats.
size_t NumExecutedMutations = 0;
size_t NumSuccessfullMutations = 0;
+ bool NeverReduce = false;
bool MayDeleteFile = false;
bool Reduced = false;
bool HasFocusFunction = false;
// Power schedule.
bool NeedsEnergyUpdate = false;
double Energy = 0.0;
- size_t SumIncidence = 0;
+ double SumIncidence = 0.0;
Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
// Delete feature Idx and its frequency from FeatureFreqs.
}
// Assign more energy to a high-entropy seed, i.e., that reveals more
- // information about the globally rare features in the neighborhood
- // of the seed. Since we do not know the entropy of a seed that has
- // never been executed we assign fresh seeds maximum entropy and
- // let II->Energy approach the true entropy from above.
- void UpdateEnergy(size_t GlobalNumberOfFeatures) {
+ // information about the globally rare features in the neighborhood of the
+ // seed. Since we do not know the entropy of a seed that has never been
+ // executed we assign fresh seeds maximum entropy and let II->Energy approach
+ // the true entropy from above. If ScalePerExecTime is true, the computed
+ // entropy is scaled based on how fast this input executes compared to the
+ // average execution time of inputs. The faster an input executes, the more
+ // energy gets assigned to the input.
+ void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime,
+ std::chrono::microseconds AverageUnitExecutionTime) {
Energy = 0.0;
- SumIncidence = 0;
+ SumIncidence = 0.0;
// Apply add-one smoothing to locally discovered features.
for (auto F : FeatureFreqs) {
- size_t LocalIncidence = F.second + 1;
- Energy -= LocalIncidence * logl(LocalIncidence);
+ double LocalIncidence = F.second + 1;
+ Energy -= LocalIncidence * log(LocalIncidence);
SumIncidence += LocalIncidence;
}
// Apply add-one smoothing to locally undiscovered features.
- // PreciseEnergy -= 0; // since logl(1.0) == 0)
- SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size());
+ // PreciseEnergy -= 0; // since log(1.0) == 0)
+ SumIncidence +=
+ static_cast<double>(GlobalNumberOfFeatures - FeatureFreqs.size());
// Add a single locally abundant feature apply add-one smoothing.
- size_t AbdIncidence = NumExecutedMutations + 1;
- Energy -= AbdIncidence * logl(AbdIncidence);
+ double AbdIncidence = static_cast<double>(NumExecutedMutations + 1);
+ Energy -= AbdIncidence * log(AbdIncidence);
SumIncidence += AbdIncidence;
// Normalize.
if (SumIncidence != 0)
- Energy = (Energy / SumIncidence) + logl(SumIncidence);
+ Energy = Energy / SumIncidence + log(SumIncidence);
+
+ if (ScalePerExecTime) {
+ // Scaling to favor inputs with lower execution time.
+ uint32_t PerfScore = 100;
+ if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 10)
+ PerfScore = 10;
+ else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 4)
+ PerfScore = 25;
+ else if (TimeOfUnit.count() > AverageUnitExecutionTime.count() * 2)
+ PerfScore = 50;
+ else if (TimeOfUnit.count() * 3 > AverageUnitExecutionTime.count() * 4)
+ PerfScore = 75;
+ else if (TimeOfUnit.count() * 4 < AverageUnitExecutionTime.count())
+ PerfScore = 300;
+ else if (TimeOfUnit.count() * 3 < AverageUnitExecutionTime.count())
+ PerfScore = 200;
+ else if (TimeOfUnit.count() * 2 < AverageUnitExecutionTime.count())
+ PerfScore = 150;
+
+ Energy *= PerfScore;
+ }
}
// Increment the frequency of the feature Idx.
bool Enabled;
size_t NumberOfRarestFeatures;
size_t FeatureFrequencyThreshold;
+ bool ScalePerExecTime;
};
class InputCorpus {
bool empty() const { return Inputs.empty(); }
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
- bool HasFocusFunction,
+ bool HasFocusFunction, bool NeverReduce,
+ std::chrono::microseconds TimeOfUnit,
const Vector<uint32_t> &FeatureSet,
const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
+ // Inputs.size() is cast to uint32_t below.
+ assert(Inputs.size() < std::numeric_limits<uint32_t>::max());
Inputs.push_back(new InputInfo());
InputInfo &II = *Inputs.back();
II.U = U;
II.NumFeatures = NumFeatures;
+ II.NeverReduce = NeverReduce;
+ II.TimeOfUnit = TimeOfUnit;
II.MayDeleteFile = MayDeleteFile;
II.UniqFeatureSet = FeatureSet;
II.HasFocusFunction = HasFocusFunction;
// Assign maximal energy to the new seed.
II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size());
- II.SumIncidence = RareFeatures.size();
+ II.SumIncidence = static_cast<double>(RareFeatures.size());
II.NeedsEnergyUpdate = false;
std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end());
ComputeSHA1(U.data(), U.size(), II.Sha1);
return II;
}
+ InputInfo &ChooseUnitToCrossOverWith(Random &Rand, bool UniformDist) {
+ if (!UniformDist) {
+ return ChooseUnitToMutate(Rand);
+ }
+ InputInfo &II = *Inputs[Rand(Inputs.size())];
+ assert(!II.U.empty());
+ return II;
+ }
+
// Returns an index of random unit from the corpus to mutate.
size_t ChooseUnitIdxToMutate(Random &Rand) {
UpdateCorpusDistribution(Rand);
// Zero energy seeds will never be fuzzed and remain zero energy.
if (II->Energy > 0.0) {
II->SumIncidence += 1;
- II->Energy += logl(II->SumIncidence) / II->SumIncidence;
+ II->Energy += log(II->SumIncidence) / II->SumIncidence;
}
}
NumUpdatedFeatures++;
if (FeatureDebug)
Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize);
- SmallestElementPerFeature[Idx] = Inputs.size();
+ // Inputs.size() is guaranteed to be less than UINT32_MAX by AddToCorpus.
+ SmallestElementPerFeature[Idx] = static_cast<uint32_t>(Inputs.size());
InputSizesPerFeature[Idx] = NewSize;
return true;
}
static const bool FeatureDebug = false;
- size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; }
+ uint32_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; }
void ValidateFeatureSet() {
if (FeatureDebug)
Weights.resize(N);
std::iota(Intervals.begin(), Intervals.end(), 0);
+ std::chrono::microseconds AverageUnitExecutionTime(0);
+ for (auto II : Inputs) {
+ AverageUnitExecutionTime += II->TimeOfUnit;
+ }
+ AverageUnitExecutionTime /= N;
+
bool VanillaSchedule = true;
if (Entropic.Enabled) {
for (auto II : Inputs) {
if (II->NeedsEnergyUpdate && II->Energy != 0.0) {
II->NeedsEnergyUpdate = false;
- II->UpdateEnergy(RareFeatures.size());
+ II->UpdateEnergy(RareFeatures.size(), Entropic.ScalePerExecTime,
+ AverageUnitExecutionTime);
}
}
if (VanillaSchedule) {
for (size_t i = 0; i < N; i++)
- Weights[i] = Inputs[i]->NumFeatures
- ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1)
- : 0.;
+ Weights[i] =
+ Inputs[i]->NumFeatures
+ ? static_cast<double>((i + 1) *
+ (Inputs[i]->HasFocusFunction ? 1000 : 1))
+ : 0.;
}
if (FeatureDebug) {
CoveredBlocks.push_back(BB);
}
if (CoveredBlocks.empty()) return false;
+ // Ensures no CoverageVector is longer than UINT32_MAX.
uint32_t NumBlocks = CoveredBlocks.back();
CoveredBlocks.pop_back();
for (auto BB : CoveredBlocks)
Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx,
FunctionNames[FocusFuncIdx].c_str());
for (size_t i = 0; i < NumFunctions; i++) {
- if (!Weights[i]) continue;
+ if (Weights[i] == 0.0)
+ continue;
Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i,
Weights[i], Coverage.GetNumberOfBlocks(i),
Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0),
return 1;
}
- static char DFSanEnv[] = "DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0";
+ static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0";
putenv(DFSanEnv);
MkDir(DirPath);
for (auto &F : CorporaFiles) {
const Vector<SizedFile> &CorporaFiles);
class BlockCoverage {
- public:
+public:
+ // These functions guarantee no CoverageVector is longer than UINT32_MAX.
bool AppendCoverage(std::istream &IN);
bool AppendCoverage(const std::string &S);
uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) {
auto It = Functions.find(FunctionId);
- if (It == Functions.end()) return 0;
+ if (It == Functions.end())
+ return 0;
const auto &Counters = It->second;
if (BasicBlockId < Counters.size())
return Counters[BasicBlockId];
auto It = Functions.find(FunctionId);
if (It == Functions.end()) return 0;
const auto &Counters = It->second;
- return Counters.size();
+ return static_cast<uint32_t>(Counters.size());
}
uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) {
Vector<double> FunctionWeights(size_t NumFunctions) const;
void clear() { Functions.clear(); }
- private:
-
+private:
typedef Vector<uint32_t> CoverageVector;
uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const {
}
uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const {
- return Counters.size() - NumberOfCoveredBlocks(Counters);
+ return static_cast<uint32_t>(Counters.size()) -
+ NumberOfCoveredBlocks(Counters);
}
uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const {
public:
static const size_t kMaxSize = kMaxSizeT;
FixedWord() {}
- FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); }
+ FixedWord(const uint8_t *B, size_t S) { Set(B, S); }
- void Set(const uint8_t *B, uint8_t S) {
+ void Set(const uint8_t *B, size_t S) {
+ static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(),
+ "FixedWord::kMaxSizeT cannot fit in a uint8_t.");
assert(S <= kMaxSize);
memcpy(Data, B, S);
- Size = S;
+ Size = static_cast<uint8_t>(S);
}
bool operator==(const FixedWord<kMaxSize> &w) const {
// binary can test for its existence.
#if LIBFUZZER_MSVC
extern "C" void __libfuzzer_is_present() {}
+#if defined(_M_IX86) || defined(__i386__)
+#pragma comment(linker, "/include:___libfuzzer_is_present")
+#else
#pragma comment(linker, "/include:__libfuzzer_is_present")
+#endif
#else
extern "C" __attribute__((used)) void __libfuzzer_is_present() {}
#endif // LIBFUZZER_MSVC
const char *Str = FlagValue(Param, Name);
if (Str) {
if (FlagDescriptions[F].IntFlag) {
- int Val = MyStol(Str);
- *FlagDescriptions[F].IntFlag = Val;
+ auto Val = MyStol(Str);
+ *FlagDescriptions[F].IntFlag = static_cast<int>(Val);
if (Flags.verbosity >= 2)
Printf("Flag: %s %d\n", Name, Val);
return true;
} else if (FlagDescriptions[F].UIntFlag) {
- unsigned int Val = std::stoul(Str);
- *FlagDescriptions[F].UIntFlag = Val;
+ auto Val = std::stoul(Str);
+ *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(Val);
if (Flags.verbosity >= 2)
Printf("Flag: %s %u\n", Name, Val);
return true;
}
}
+static void ValidateDirectoryExists(const std::string &Path,
+ bool CreateDirectory) {
+ if (Path.empty()) {
+ Printf("ERROR: Provided directory path is an empty string\n");
+ exit(1);
+ }
+
+ if (IsDirectory(Path))
+ return;
+
+ if (CreateDirectory) {
+ if (!MkDirRecursive(Path)) {
+ Printf("ERROR: Failed to create directory \"%s\"\n", Path.c_str());
+ exit(1);
+ }
+ return;
+ }
+
+ Printf("ERROR: The required directory \"%s\" does not exist\n", Path.c_str());
+ exit(1);
+}
+
std::string CloneArgsWithoutX(const Vector<std::string> &Args,
const char *X1, const char *X2) {
std::string Cmd;
if (MaxLen && MaxLen < U.size())
U.resize(MaxLen);
F->ExecuteCallback(U.data(), U.size());
- F->TryDetectingAMemoryLeak(U.data(), U.size(), true);
+ if (Flags.print_full_coverage) {
+ // Leak detection is not needed when collecting full coverage data.
+ F->TPCUpdateObservedPCs();
+ } else {
+ F->TryDetectingAMemoryLeak(U.data(), U.size(), true);
+ }
return 0;
}
Options.Verbosity = Flags.verbosity;
Options.MaxLen = Flags.max_len;
Options.LenControl = Flags.len_control;
+ Options.KeepSeed = Flags.keep_seed;
Options.UnitTimeoutSec = Flags.timeout;
Options.ErrorExitCode = Flags.error_exitcode;
Options.TimeoutExitCode = Flags.timeout_exitcode;
Options.IgnoreCrashes = Flags.ignore_crashes;
Options.MaxTotalTimeSec = Flags.max_total_time;
Options.DoCrossOver = Flags.cross_over;
+ Options.CrossOverUniformDist = Flags.cross_over_uniform_dist;
Options.MutateDepth = Flags.mutate_depth;
Options.ReduceDepth = Flags.reduce_depth;
Options.UseCounters = Flags.use_counters;
Options.MallocLimitMb = Options.RssLimitMb;
if (Flags.runs >= 0)
Options.MaxNumberOfRuns = Flags.runs;
- if (!Inputs->empty() && !Flags.minimize_crash_internal_step)
- Options.OutputCorpus = (*Inputs)[0];
+ if (!Inputs->empty() && !Flags.minimize_crash_internal_step) {
+ // Ensure output corpus assumed to be the first arbitrary argument input
+ // is not a path to an existing file.
+ std::string OutputCorpusDir = (*Inputs)[0];
+ if (!IsFile(OutputCorpusDir)) {
+ Options.OutputCorpus = OutputCorpusDir;
+ ValidateDirectoryExists(Options.OutputCorpus, Flags.create_missing_dirs);
+ }
+ }
Options.ReportSlowUnits = Flags.report_slow_units;
- if (Flags.artifact_prefix)
+ if (Flags.artifact_prefix) {
Options.ArtifactPrefix = Flags.artifact_prefix;
- if (Flags.exact_artifact_path)
+
+ // Since the prefix could be a full path to a file name prefix, assume
+ // that if the path ends with the platform's separator that a directory
+ // is desired
+ std::string ArtifactPathDir = Options.ArtifactPrefix;
+ if (!IsSeparator(ArtifactPathDir[ArtifactPathDir.length() - 1])) {
+ ArtifactPathDir = DirName(ArtifactPathDir);
+ }
+ ValidateDirectoryExists(ArtifactPathDir, Flags.create_missing_dirs);
+ }
+ if (Flags.exact_artifact_path) {
Options.ExactArtifactPath = Flags.exact_artifact_path;
+ ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
+ Flags.create_missing_dirs);
+ }
Vector<Unit> Dictionary;
if (Flags.dict)
if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
Options.PrintFinalStats = Flags.print_final_stats;
Options.PrintCorpusStats = Flags.print_corpus_stats;
Options.PrintCoverage = Flags.print_coverage;
+ Options.PrintFullCoverage = Flags.print_full_coverage;
if (Flags.exit_on_src_pos)
Options.ExitOnSrcPos = Flags.exit_on_src_pos;
if (Flags.exit_on_item)
Options.FocusFunction = Flags.focus_function;
if (Flags.data_flow_trace)
Options.DataFlowTrace = Flags.data_flow_trace;
- if (Flags.features_dir)
+ if (Flags.features_dir) {
Options.FeaturesDir = Flags.features_dir;
+ ValidateDirectoryExists(Options.FeaturesDir, Flags.create_missing_dirs);
+ }
+ if (Flags.mutation_graph_file)
+ Options.MutationGraphFile = Flags.mutation_graph_file;
if (Flags.collect_data_flow)
Options.CollectDataFlow = Flags.collect_data_flow;
if (Flags.stop_file)
(size_t)Flags.entropic_feature_frequency_threshold;
Options.EntropicNumberOfRarestFeatures =
(size_t)Flags.entropic_number_of_rarest_features;
- if (Options.Entropic) {
- if (!Options.FocusFunction.empty()) {
- Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot "
- "be used together.\n");
- exit(1);
- }
+ Options.EntropicScalePerExecTime = Flags.entropic_scale_per_exec_time;
+ if (!Options.FocusFunction.empty())
+ Options.Entropic = false; // FocusFunction overrides entropic scheduling.
+ if (Options.Entropic)
Printf("INFO: Running with entropic power schedule (0x%X, %d).\n",
Options.EntropicFeatureFrequencyThreshold,
Options.EntropicNumberOfRarestFeatures);
- }
struct EntropicOptions Entropic;
Entropic.Enabled = Options.Entropic;
Entropic.FeatureFrequencyThreshold =
Options.EntropicFeatureFrequencyThreshold;
Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures;
+ Entropic.ScalePerExecTime = Options.EntropicScalePerExecTime;
unsigned Seed = Flags.seed;
// Initialize Seed.
if (Seed == 0)
- Seed =
- std::chrono::system_clock::now().time_since_epoch().count() + GetPid();
+ Seed = static_cast<unsigned>(
+ std::chrono::system_clock::now().time_since_epoch().count() + GetPid());
if (Flags.verbosity)
Printf("INFO: Seed: %u\n", Seed);
#endif // LIBFUZZER_EMSCRIPTEN
Options.HandleAbrt = Flags.handle_abrt;
+ Options.HandleAlrm = !Flags.minimize_crash;
Options.HandleBus = Flags.handle_bus;
Options.HandleFpe = Flags.handle_fpe;
Options.HandleIll = Flags.handle_ill;
Options.HandleXfsz = Flags.handle_xfsz;
Options.HandleUsr1 = Flags.handle_usr1;
Options.HandleUsr2 = Flags.handle_usr2;
+ Options.HandleWinExcept = Flags.handle_winexcept;
+
SetSignalHandler(Options);
std::atexit(Fuzzer::StaticExitCallback);
exit(0); // Don't let F destroy itself.
}
+extern "C" ATTRIBUTE_INTERFACE int
+LLVMFuzzerRunDriver(int *argc, char ***argv,
+ int (*UserCb)(const uint8_t *Data, size_t Size)) {
+ return FuzzerDriver(argc, argv, UserCb);
+}
+
// Storage for global ExternalFunctions object.
ExternalFunctions *EF = nullptr;
//===----------------------------------------------------------------------===//
#include "FuzzerPlatform.h"
#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \
- LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN
+ LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN
#include "FuzzerExtFunctions.h"
#include "FuzzerIO.h"
#include <cstdint>
#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \
- LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN
+ LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN
__attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters;
__attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters;
FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files "
"to use as an additional seed corpus. Alternatively, an \"@\" followed by "
"the name of a file containing the comma-separated list.")
+FUZZER_FLAG_INT(keep_seed, 0, "If 1, keep seed inputs in the corpus even if "
+ "they do not produce new coverage. When used with |reduce_inputs==1|, the "
+ "seed inputs will never be reduced. This option can be useful when seeds are"
+ "not properly formed for the fuzz target but still have useful snippets.")
FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.")
+FUZZER_FLAG_INT(cross_over_uniform_dist, 0, "Experimental. If 1, use a "
+ "uniform probability distribution when choosing inputs to cross over with. "
+ "Some of the inputs in the corpus may never get chosen for mutation "
+ "depending on the input mutation scheduling policy. With this flag, all "
+ "inputs, regardless of the input mutation scheduling policy, can be chosen "
+ "as an input to cross over with. This can be particularly useful with "
+ "|keep_seed==1|; all the initial seed inputs, even though they do not "
+ "increase coverage because they are not properly formed, will still be "
+ "chosen as an input to cross over with.")
+
FUZZER_FLAG_INT(mutate_depth, 5,
"Apply this number of consecutive mutations to each input.")
FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. "
"Every time a new input is added to the corpus, a corresponding file in the features_dir"
" is created containing the unique features of that input."
" Features are stored in binary format.")
+FUZZER_FLAG_STRING(mutation_graph_file, "Saves a graph (in DOT format) to"
+ " mutation_graph_file. The graph contains a vertex for each input that has"
+ " unique coverage; directed edges are provided between parents and children"
+ " where the child has unique coverage, and are recorded with the type of"
+ " mutation that caused the child.")
FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters")
FUZZER_FLAG_INT(use_memmem, 1,
"Use hints from intercepting memmem, strstr, etc")
"If 1, print statistics on corpus elements at exit.")
FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text"
" at exit.")
+FUZZER_FLAG_INT(print_full_coverage, 0, "If 1, print full coverage information "
+ "(all branches) as text at exit.")
FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.")
FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.")
FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.")
FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.")
FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.")
FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.")
+FUZZER_FLAG_INT(handle_winexcept, 1, "If 1, try to intercept uncaught Windows "
+ "Visual C++ Exceptions.")
FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
"if 2, close stderr; if 3, close both. "
"Be careful, this will also close e.g. stderr of asan.")
FUZZER_FLAG_STRING(focus_function, "Experimental. "
"Fuzzing will focus on inputs that trigger calls to this function. "
"If -focus_function=auto and -data_flow_trace is used, libFuzzer "
- "will choose the focus functions automatically.")
-FUZZER_FLAG_INT(entropic, 0, "Experimental. Enables entropic power schedule.")
+ "will choose the focus functions automatically. Disables -entropic when "
+ "specified.")
+FUZZER_FLAG_INT(entropic, 1, "Enables entropic power schedule.")
FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If "
"entropic is enabled, all features which are observed less often than "
"the specified value are considered as rare.")
"entropic is enabled, we keep track of the frequencies only for the "
"Top-X least abundant features (union features that are considered as "
"rare).")
+FUZZER_FLAG_INT(entropic_scale_per_exec_time, 0, "Experimental. If 1, "
+ "the Entropic power schedule gets scaled based on the input execution "
+ "time. Inputs with lower execution time get scheduled more (up to 30x). "
+ "Note that, if 1, fuzzer stops from being deterministic even if a "
+ "non-zero random seed is given.")
FUZZER_FLAG_INT(analyze_dict, 0, "Experimental")
FUZZER_DEPRECATED_FLAG(use_clang_coverage)
FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace")
FUZZER_FLAG_STRING(collect_data_flow,
"Experimental: collect the data flow trace")
+
+FUZZER_FLAG_INT(create_missing_dirs, 0, "Automatically attempt to create "
+ "directories for arguments that would normally expect them to already "
+ "exist (i.e. artifact_prefix, exact_artifact_path, features_dir, corpus)")
CollectDFT(SF);
}
auto Time2 = std::chrono::system_clock::now();
- Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
+ auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
+ assert(DftTimeInSeconds < std::numeric_limits<int>::max());
+ Job->DftTimeInSeconds = static_cast<int>(DftTimeInSeconds);
}
if (!Seeds.empty()) {
Job->SeedListPath =
else
Env.MainCorpusDir = CorpusDirs[0];
- auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
- CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features,
- {}, &Env.Cov,
- CFPath, false);
- RemoveFile(CFPath);
+ if (Options.KeepSeed) {
+ for (auto &File : SeedFiles)
+ Env.Files.push_back(File.File);
+ } else {
+ auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
+ Set<uint32_t> NewFeatures, NewCov;
+ CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features,
+ &NewFeatures, Env.Cov, &NewCov, CFPath, false);
+ Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
+ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
+ RemoveFile(CFPath);
+ }
Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
Env.Files.size(), Env.TempDir.c_str());
fclose(Out);
}
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
- long *Epoch, size_t MaxSize, bool ExitOnError) {
+void AppendToFile(const std::string &Data, const std::string &Path) {
+ AppendToFile(reinterpret_cast<const uint8_t *>(Data.data()), Data.size(),
+ Path);
+}
+
+void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) {
+ FILE *Out = fopen(Path.c_str(), "a");
+ if (!Out)
+ return;
+ fwrite(Data, sizeof(Data[0]), Size, Out);
+ fclose(Out);
+}
+
+void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+ size_t MaxSize, bool ExitOnError,
+ Vector<std::string> *VPaths) {
long E = Epoch ? *Epoch : 0;
Vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024)
Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path);
auto S = FileToVector(X, MaxSize, ExitOnError);
- if (!S.empty())
+ if (!S.empty()) {
V->push_back(S);
+ if (VPaths)
+ VPaths->push_back(X);
+ }
}
}
-
void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) {
Vector<std::string> Files;
ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true);
fflush(OutputFile);
}
+static bool MkDirRecursiveInner(const std::string &Leaf) {
+ // Prevent chance of potential infinite recursion
+ if (Leaf == ".")
+ return true;
+
+ const std::string &Dir = DirName(Leaf);
+
+ if (IsDirectory(Dir)) {
+ MkDir(Leaf);
+ return IsDirectory(Leaf);
+ }
+
+ bool ret = MkDirRecursiveInner(Dir);
+ if (!ret) {
+ // Give up early if a previous MkDir failed
+ return ret;
+ }
+
+ MkDir(Leaf);
+ return IsDirectory(Leaf);
+}
+
+bool MkDirRecursive(const std::string &Dir) {
+ if (Dir.empty())
+ return false;
+
+ if (IsDirectory(Dir))
+ return true;
+
+ return MkDirRecursiveInner(Dir);
+}
+
void RmDirRecursive(const std::string &Dir) {
IterateDirRecursive(
Dir, [](const std::string &Path) {},
void WriteToFile(const std::string &Data, const std::string &Path);
void WriteToFile(const Unit &U, const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
- long *Epoch, size_t MaxSize, bool ExitOnError);
+void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path);
+void AppendToFile(const std::string &Data, const std::string &Path);
+
+void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+ size_t MaxSize, bool ExitOnError,
+ Vector<std::string> *VPaths = 0);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
// Platform specific functions:
bool IsFile(const std::string &Path);
+bool IsDirectory(const std::string &Path);
size_t FileSize(const std::string &Path);
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
Vector<std::string> *V, bool TopDir);
+bool MkDirRecursive(const std::string &Dir);
void RmDirRecursive(const std::string &Dir);
// Iterate files and dirs inside Dir, recursively.
void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V);
char GetSeparator();
+bool IsSeparator(char C);
// Similar to the basename utility: returns the file name w/o the dir prefix.
std::string Basename(const std::string &Path);
return S_ISREG(St.st_mode);
}
-static bool IsDirectory(const std::string &Path) {
+bool IsDirectory(const std::string &Path) {
struct stat St;
if (stat(Path.c_str(), &St))
return false;
return '/';
}
+bool IsSeparator(char C) {
+ return C == '/';
+}
+
FILE* OpenFile(int Fd, const char* Mode) {
return fdopen(Fd, Mode);
}
}
void RawPrint(const char *Str) {
- write(2, Str, strlen(Str));
+ (void)write(2, Str, strlen(Str));
}
void MkDir(const std::string &Path) {
return FileAttrs & FILE_ATTRIBUTE_DIRECTORY;
}
+bool IsDirectory(const std::string &Path) {
+ DWORD Att = GetFileAttributesA(Path.c_str());
+
+ if (Att == INVALID_FILE_ATTRIBUTES) {
+ Printf("GetFileAttributesA() failed for \"%s\" (Error code: %lu).\n",
+ Path.c_str(), GetLastError());
+ return false;
+ }
+
+ return IsDir(Att);
+}
+
std::string Basename(const std::string &Path) {
size_t Pos = Path.find_last_of("/\\");
if (Pos == std::string::npos) return Path;
return _get_osfhandle(fd);
}
-static bool IsSeparator(char C) {
+bool IsSeparator(char C) {
return C == '\\' || C == '/';
}
--- /dev/null
+//===-- FuzzerInterceptors.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Intercept certain libc functions to aid fuzzing.
+// Linked only when other RTs that define their own interceptors are not linked.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+
+#if LIBFUZZER_LINUX
+
+#define GET_CALLER_PC() __builtin_return_address(0)
+
+#define PTR_TO_REAL(x) real_##x
+#define REAL(x) __interception::PTR_TO_REAL(x)
+#define FUNC_TYPE(x) x##_type
+#define DEFINE_REAL(ret_type, func, ...) \
+ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
+ namespace __interception { \
+ FUNC_TYPE(func) PTR_TO_REAL(func); \
+ }
+
+#include <cassert>
+#include <cstddef> // for size_t
+#include <cstdint>
+#include <dlfcn.h> // for dlsym()
+
+static void *getFuncAddr(const char *name, uintptr_t wrapper_addr) {
+ void *addr = dlsym(RTLD_NEXT, name);
+ if (!addr) {
+ // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
+ // later in the library search order than the DSO that we are trying to
+ // intercept, which means that we cannot intercept this function. We still
+ // want the address of the real definition, though, so look it up using
+ // RTLD_DEFAULT.
+ addr = dlsym(RTLD_DEFAULT, name);
+
+ // In case `name' is not loaded, dlsym ends up finding the actual wrapper.
+ // We don't want to intercept the wrapper and have it point to itself.
+ if (reinterpret_cast<uintptr_t>(addr) == wrapper_addr)
+ addr = nullptr;
+ }
+ return addr;
+}
+
+static int FuzzerInited = 0;
+static bool FuzzerInitIsRunning;
+
+static void fuzzerInit();
+
+static void ensureFuzzerInited() {
+ assert(!FuzzerInitIsRunning);
+ if (!FuzzerInited) {
+ fuzzerInit();
+ }
+}
+
+static int internal_strcmp_strncmp(const char *s1, const char *s2, bool strncmp,
+ size_t n) {
+ size_t i = 0;
+ while (true) {
+ if (strncmp) {
+ if (i == n)
+ break;
+ i++;
+ }
+ unsigned c1 = *s1;
+ unsigned c2 = *s2;
+ if (c1 != c2)
+ return (c1 < c2) ? -1 : 1;
+ if (c1 == 0)
+ break;
+ s1++;
+ s2++;
+ }
+ return 0;
+}
+
+static int internal_strncmp(const char *s1, const char *s2, size_t n) {
+ return internal_strcmp_strncmp(s1, s2, true, n);
+}
+
+static int internal_strcmp(const char *s1, const char *s2) {
+ return internal_strcmp_strncmp(s1, s2, false, 0);
+}
+
+static int internal_memcmp(const void *s1, const void *s2, size_t n) {
+ const uint8_t *t1 = static_cast<const uint8_t *>(s1);
+ const uint8_t *t2 = static_cast<const uint8_t *>(s2);
+ for (size_t i = 0; i < n; ++i, ++t1, ++t2)
+ if (*t1 != *t2)
+ return *t1 < *t2 ? -1 : 1;
+ return 0;
+}
+
+static size_t internal_strlen(const char *s) {
+ size_t i = 0;
+ while (s[i])
+ i++;
+ return i;
+}
+
+static char *internal_strstr(const char *haystack, const char *needle) {
+ // This is O(N^2), but we are not using it in hot places.
+ size_t len1 = internal_strlen(haystack);
+ size_t len2 = internal_strlen(needle);
+ if (len1 < len2)
+ return nullptr;
+ for (size_t pos = 0; pos <= len1 - len2; pos++) {
+ if (internal_memcmp(haystack + pos, needle, len2) == 0)
+ return const_cast<char *>(haystack) + pos;
+ }
+ return nullptr;
+}
+
+extern "C" {
+
+// Weak hooks forward-declared to avoid dependency on
+// <sanitizer/common_interface_defs.h>.
+void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1,
+ const void *s2, size_t n, int result);
+void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1,
+ const char *s2, size_t n, int result);
+void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1,
+ const char *s2, size_t n, int result);
+void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1,
+ const char *s2, int result);
+void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1,
+ const char *s2, int result);
+void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1,
+ const char *s2, char *result);
+void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1,
+ const char *s2, char *result);
+void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1,
+ const void *s2, size_t len2, void *result);
+
+DEFINE_REAL(int, bcmp, const void *, const void *, size_t)
+DEFINE_REAL(int, memcmp, const void *, const void *, size_t)
+DEFINE_REAL(int, strncmp, const char *, const char *, size_t)
+DEFINE_REAL(int, strcmp, const char *, const char *)
+DEFINE_REAL(int, strncasecmp, const char *, const char *, size_t)
+DEFINE_REAL(int, strcasecmp, const char *, const char *)
+DEFINE_REAL(char *, strstr, const char *, const char *)
+DEFINE_REAL(char *, strcasestr, const char *, const char *)
+DEFINE_REAL(void *, memmem, const void *, size_t, const void *, size_t)
+
+ATTRIBUTE_INTERFACE int bcmp(const char *s1, const char *s2, size_t n) {
+ if (!FuzzerInited)
+ return internal_memcmp(s1, s2, n);
+ int result = REAL(bcmp)(s1, s2, n);
+ __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE int memcmp(const void *s1, const void *s2, size_t n) {
+ if (!FuzzerInited)
+ return internal_memcmp(s1, s2, n);
+ int result = REAL(memcmp)(s1, s2, n);
+ __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE int strncmp(const char *s1, const char *s2, size_t n) {
+ if (!FuzzerInited)
+ return internal_strncmp(s1, s2, n);
+ int result = REAL(strncmp)(s1, s2, n);
+ __sanitizer_weak_hook_strncmp(GET_CALLER_PC(), s1, s2, n, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE int strcmp(const char *s1, const char *s2) {
+ if (!FuzzerInited)
+ return internal_strcmp(s1, s2);
+ int result = REAL(strcmp)(s1, s2);
+ __sanitizer_weak_hook_strcmp(GET_CALLER_PC(), s1, s2, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE int strncasecmp(const char *s1, const char *s2, size_t n) {
+ ensureFuzzerInited();
+ int result = REAL(strncasecmp)(s1, s2, n);
+ __sanitizer_weak_hook_strncasecmp(GET_CALLER_PC(), s1, s2, n, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE int strcasecmp(const char *s1, const char *s2) {
+ ensureFuzzerInited();
+ int result = REAL(strcasecmp)(s1, s2);
+ __sanitizer_weak_hook_strcasecmp(GET_CALLER_PC(), s1, s2, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE char *strstr(const char *s1, const char *s2) {
+ if (!FuzzerInited)
+ return internal_strstr(s1, s2);
+ char *result = REAL(strstr)(s1, s2);
+ __sanitizer_weak_hook_strstr(GET_CALLER_PC(), s1, s2, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE char *strcasestr(const char *s1, const char *s2) {
+ ensureFuzzerInited();
+ char *result = REAL(strcasestr)(s1, s2);
+ __sanitizer_weak_hook_strcasestr(GET_CALLER_PC(), s1, s2, result);
+ return result;
+}
+
+ATTRIBUTE_INTERFACE
+void *memmem(const void *s1, size_t len1, const void *s2, size_t len2) {
+ ensureFuzzerInited();
+ void *result = REAL(memmem)(s1, len1, s2, len2);
+ __sanitizer_weak_hook_memmem(GET_CALLER_PC(), s1, len1, s2, len2, result);
+ return result;
+}
+
+__attribute__((section(".preinit_array"),
+ used)) static void (*__local_fuzzer_preinit)(void) = fuzzerInit;
+
+} // extern "C"
+
+static void fuzzerInit() {
+ assert(!FuzzerInitIsRunning);
+ if (FuzzerInited)
+ return;
+ FuzzerInitIsRunning = true;
+
+ REAL(bcmp) = reinterpret_cast<memcmp_type>(
+ getFuncAddr("bcmp", reinterpret_cast<uintptr_t>(&bcmp)));
+ REAL(memcmp) = reinterpret_cast<memcmp_type>(
+ getFuncAddr("memcmp", reinterpret_cast<uintptr_t>(&memcmp)));
+ REAL(strncmp) = reinterpret_cast<strncmp_type>(
+ getFuncAddr("strncmp", reinterpret_cast<uintptr_t>(&strncmp)));
+ REAL(strcmp) = reinterpret_cast<strcmp_type>(
+ getFuncAddr("strcmp", reinterpret_cast<uintptr_t>(&strcmp)));
+ REAL(strncasecmp) = reinterpret_cast<strncasecmp_type>(
+ getFuncAddr("strncasecmp", reinterpret_cast<uintptr_t>(&strncasecmp)));
+ REAL(strcasecmp) = reinterpret_cast<strcasecmp_type>(
+ getFuncAddr("strcasecmp", reinterpret_cast<uintptr_t>(&strcasecmp)));
+ REAL(strstr) = reinterpret_cast<strstr_type>(
+ getFuncAddr("strstr", reinterpret_cast<uintptr_t>(&strstr)));
+ REAL(strcasestr) = reinterpret_cast<strcasestr_type>(
+ getFuncAddr("strcasestr", reinterpret_cast<uintptr_t>(&strcasestr)));
+ REAL(memmem) = reinterpret_cast<memmem_type>(
+ getFuncAddr("memmem", reinterpret_cast<uintptr_t>(&memmem)));
+
+ FuzzerInitIsRunning = false;
+ FuzzerInited = 1;
+}
+
+#endif
void ExecuteCallback(const uint8_t *Data, size_t Size);
bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
- InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr);
+ InputInfo *II = nullptr, bool ForceAddToCorpus = false,
+ bool *FoundUniqFeatures = nullptr);
+ void TPCUpdateObservedPCs();
// Merge Corpora[1:] into Corpora[0].
void Merge(const Vector<std::string> &Corpora);
}
void Fuzzer::PrintFinalStats() {
+ if (Options.PrintFullCoverage)
+ TPC.PrintCoverage(/*PrintAllCounters=*/true);
if (Options.PrintCoverage)
- TPC.PrintCoverage();
+ TPC.PrintCoverage(/*PrintAllCounters=*/false);
if (Options.PrintCorpusStats)
Corpus.PrintStats();
if (!Options.PrintFinalStats)
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec)
return;
Vector<Unit> AdditionalCorpus;
- ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
- &EpochOfLastReadOfOutputCorpus, MaxSize,
- /*ExitOnError*/ false);
+ Vector<std::string> AdditionalCorpusPaths;
+ ReadDirToVectorOfUnits(
+ Options.OutputCorpus.c_str(), &AdditionalCorpus,
+ &EpochOfLastReadOfOutputCorpus, MaxSize,
+ /*ExitOnError*/ false,
+ (Options.Verbosity >= 2 ? &AdditionalCorpusPaths : nullptr));
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
bool Reloaded = false;
- for (auto &U : AdditionalCorpus) {
+ for (size_t i = 0; i != AdditionalCorpus.size(); ++i) {
+ auto &U = AdditionalCorpus[i];
if (U.size() > MaxSize)
U.resize(MaxSize);
if (!Corpus.HasUnit(U)) {
if (RunOne(U.data(), U.size())) {
CheckExitOnSrcPosOrItem();
Reloaded = true;
+ if (Options.Verbosity >= 2)
+ Printf("Reloaded %s\n", AdditionalCorpusPaths[i].c_str());
}
}
}
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
secondsSinceProcessStartUp() >= 2)
PrintStats("pulse ");
- if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 &&
- TimeOfUnit >= Options.ReportSlowUnits) {
+ auto Threshhold =
+ static_cast<long>(static_cast<double>(TimeOfLongestUnitInSeconds) * 1.1);
+ if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) {
TimeOfLongestUnitInSeconds = TimeOfUnit;
Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
DirPlusFile(FeaturesDir, NewFile));
}
+static void WriteEdgeToMutationGraphFile(const std::string &MutationGraphFile,
+ const InputInfo *II,
+ const InputInfo *BaseII,
+ const std::string &MS) {
+ if (MutationGraphFile.empty())
+ return;
+
+ std::string Sha1 = Sha1ToString(II->Sha1);
+
+ std::string OutputString;
+
+ // Add a new vertex.
+ OutputString.append("\"");
+ OutputString.append(Sha1);
+ OutputString.append("\"\n");
+
+ // Add a new edge if there is base input.
+ if (BaseII) {
+ std::string BaseSha1 = Sha1ToString(BaseII->Sha1);
+ OutputString.append("\"");
+ OutputString.append(BaseSha1);
+ OutputString.append("\" -> \"");
+ OutputString.append(Sha1);
+ OutputString.append("\" [label=\"");
+ OutputString.append(MS);
+ OutputString.append("\"];\n");
+ }
+
+ AppendToFile(OutputString, MutationGraphFile);
+}
+
bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
- InputInfo *II, bool *FoundUniqFeatures) {
+ InputInfo *II, bool ForceAddToCorpus,
+ bool *FoundUniqFeatures) {
if (!Size)
return false;
+ // Largest input length should be INT_MAX.
+ assert(Size < std::numeric_limits<uint32_t>::max());
ExecuteCallback(Data, Size);
+ auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime);
UniqFeatureSetTmp.clear();
size_t FoundUniqFeaturesOfII = 0;
size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
- TPC.CollectFeatures([&](size_t Feature) {
- if (Corpus.AddFeature(Feature, Size, Options.Shrink))
+ TPC.CollectFeatures([&](uint32_t Feature) {
+ if (Corpus.AddFeature(Feature, static_cast<uint32_t>(Size), Options.Shrink))
UniqFeatureSetTmp.push_back(Feature);
if (Options.Entropic)
Corpus.UpdateFeatureFrequency(II, Feature);
- if (Options.ReduceInputs && II)
+ if (Options.ReduceInputs && II && !II->NeverReduce)
if (std::binary_search(II->UniqFeatureSet.begin(),
II->UniqFeatureSet.end(), Feature))
FoundUniqFeaturesOfII++;
*FoundUniqFeatures = FoundUniqFeaturesOfII;
PrintPulseAndReportSlowInput(Data, Size);
size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore;
- if (NumNewFeatures) {
+ if (NumNewFeatures || ForceAddToCorpus) {
TPC.UpdateObservedPCs();
- auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures,
- MayDeleteFile, TPC.ObservedFocusFunction(),
- UniqFeatureSetTmp, DFT, II);
+ auto NewII =
+ Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile,
+ TPC.ObservedFocusFunction(), ForceAddToCorpus,
+ TimeOfUnit, UniqFeatureSetTmp, DFT, II);
WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1),
NewII->UniqFeatureSet);
+ WriteEdgeToMutationGraphFile(Options.MutationGraphFile, NewII, II,
+ MD.MutationSequence());
return true;
}
if (II && FoundUniqFeaturesOfII &&
return false;
}
+void Fuzzer::TPCUpdateObservedPCs() { TPC.UpdateObservedPCs(); }
+
size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
assert(InFuzzingThread());
*Data = CurrentUnitData;
!memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2);
}
-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
+// This method is not inlined because it would cause a test to fail where it
+// is part of the stack unwinding. See D97975 for details.
+ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data,
+ size_t Size) {
TPC.RecordInitialStack();
TotalNumberOfRuns++;
assert(InFuzzingThread());
PrintStats(Text, "");
if (Options.Verbosity) {
Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize());
- MD.PrintMutationSequence();
+ MD.PrintMutationSequence(Options.Verbosity >= 2);
Printf("\n");
}
}
MD.StartMutationSequence();
auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
- if (Options.DoCrossOver)
- MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U);
+ if (Options.DoCrossOver) {
+ auto &CrossOverII = Corpus.ChooseUnitToCrossOverWith(
+ MD.GetRand(), Options.CrossOverUniformDist);
+ MD.SetCrossOverWith(&CrossOverII.U);
+ }
const auto &U = II.U;
memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1));
assert(CurrentUnitData);
bool FoundUniqFeatures = false;
bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II,
- &FoundUniqFeatures);
+ /*ForceAddToCorpus*/ false, &FoundUniqFeatures);
TryDetectingAMemoryLeak(CurrentUnitData, Size,
/*DuringInitialCorpusExecution*/ false);
if (NewCov) {
for (auto &SF : CorporaFiles) {
auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false);
assert(U.size() <= MaxInputLen);
- RunOne(U.data(), U.size());
+ RunOne(U.data(), U.size(), /*MayDeleteFile*/ false, /*II*/ nullptr,
+ /*ForceAddToCorpus*/ Options.KeepSeed,
+ /*FoundUniqFeatures*/ nullptr);
CheckExitOnSrcPosOrItem();
TryDetectingAMemoryLeak(U.data(), U.size(),
/*DuringInitialCorpusExecution*/ true);
while (std::getline(IS, Line, '\n')) {
std::istringstream ISS1(Line);
std::string Marker;
- size_t N;
- ISS1 >> Marker;
- ISS1 >> N;
+ uint32_t N;
+ if (!(ISS1 >> Marker) || !(ISS1 >> N))
+ return false;
if (Marker == "STARTED") {
// STARTED FILE_ID FILE_SIZE
if (ExpectedStartMarker != N)
const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
Vector<std::string> *NewFiles) {
NewFiles->clear();
+ NewFeatures->clear();
+ NewCov->clear();
assert(NumFilesInFirstCorpus <= Files.size());
Set<uint32_t> AllFeatures = InitialFeatures;
namespace fuzzer {
const size_t Dictionary::kMaxDictSize;
+static const size_t kMaxMutationsToPrint = 10;
static void PrintASCII(const Word &W, const char *PrintAfter) {
PrintASCII(W.data(), W.size(), PrintAfter);
}
static char RandCh(Random &Rand) {
- if (Rand.RandBool()) return Rand(256);
+ if (Rand.RandBool())
+ return static_cast<char>(Rand(256));
const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00";
return Special[Rand(sizeof(Special) - 1)];
}
size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size,
size_t MaxSize) {
- return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand());
+ if (EF->__msan_unpoison)
+ EF->__msan_unpoison(Data, Size);
+ if (EF->__msan_unpoison_param)
+ EF->__msan_unpoison_param(4);
+ return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize,
+ Rand.Rand<unsigned int>());
}
size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size,
return 0;
CustomCrossOverInPlaceHere.resize(MaxSize);
auto &U = CustomCrossOverInPlaceHere;
+
+ if (EF->__msan_unpoison) {
+ EF->__msan_unpoison(Data, Size);
+ EF->__msan_unpoison(Other.data(), Other.size());
+ EF->__msan_unpoison(U.data(), U.size());
+ }
+ if (EF->__msan_unpoison_param)
+ EF->__msan_unpoison_param(7);
size_t NewSize = EF->LLVMFuzzerCustomCrossOver(
- Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand());
+ Data, Size, Other.data(), Other.size(), U.data(), U.size(),
+ Rand.Rand<unsigned int>());
+
if (!NewSize)
return 0;
assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit");
// Insert new values at Data[Idx].
memmove(Data + Idx + N, Data + Idx, Size - Idx);
// Give preference to 0x00 and 0xff.
- uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255);
+ uint8_t Byte = static_cast<uint8_t>(
+ Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255));
for (size_t i = 0; i < N; i++)
Data[Idx + i] = Byte;
return Size + N;
Size += W.size();
} else { // Overwrite some bytes with W.
if (W.size() > Size) return 0;
- size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size());
+ size_t Idx =
+ UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1 - W.size());
memcpy(Data + Idx, W.data(), W.size());
}
return Size;
T Arg1, T Arg2, const uint8_t *Data, size_t Size) {
if (Rand.RandBool()) Arg1 = Bswap(Arg1);
if (Rand.RandBool()) Arg2 = Bswap(Arg2);
- T Arg1Mutation = Arg1 + Rand(-1, 1);
- T Arg2Mutation = Arg2 + Rand(-1, 1);
+ T Arg1Mutation = static_cast<T>(Arg1 + Rand(-1, 1));
+ T Arg2Mutation = static_cast<T>(Arg2 + Rand(-1, 1));
return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation,
sizeof(Arg1), Data, Size);
}
DictionaryEntry DE;
switch (Rand(4)) {
case 0: {
- auto X = TPC.TORC8.Get(Rand.Rand());
+ auto X = TPC.TORC8.Get(Rand.Rand<size_t>());
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 1: {
- auto X = TPC.TORC4.Get(Rand.Rand());
+ auto X = TPC.TORC4.Get(Rand.Rand<size_t>());
if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool())
DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size);
else
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 2: {
- auto X = TPC.TORCW.Get(Rand.Rand());
+ auto X = TPC.TORCW.Get(Rand.Rand<size_t>());
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 3: if (Options.UseMemmem) {
- auto X = TPC.MMT.Get(Rand.Rand());
- DE = DictionaryEntry(X);
+ auto X = TPC.MMT.Get(Rand.Rand<size_t>());
+ DE = DictionaryEntry(X);
} break;
default:
assert(0);
assert(Off + sizeof(T) <= Size);
T Val;
if (Off < 64 && !Rand(4)) {
- Val = Size;
+ Val = static_cast<T>(Size);
if (Rand.RandBool())
Val = Bswap(Val);
} else {
memcpy(&Val, Data + Off, sizeof(Val));
- T Add = Rand(21);
+ T Add = static_cast<T>(Rand(21));
Add -= 10;
if (Rand.RandBool())
Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes.
if (!CrossOverWith) return 0;
const Unit &O = *CrossOverWith;
if (O.empty()) return 0;
- MutateInPlaceHere.resize(MaxSize);
- auto &U = MutateInPlaceHere;
size_t NewSize = 0;
switch(Rand(3)) {
case 0:
- NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size());
+ MutateInPlaceHere.resize(MaxSize);
+ NewSize = CrossOver(Data, Size, O.data(), O.size(),
+ MutateInPlaceHere.data(), MaxSize);
+ memcpy(Data, MutateInPlaceHere.data(), NewSize);
break;
case 1:
- NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize);
+ NewSize = InsertPartOf(O.data(), O.size(), Data, Size, MaxSize);
if (!NewSize)
- NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size());
+ NewSize = CopyPartOf(O.data(), O.size(), Data, Size);
break;
case 2:
- NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size());
+ NewSize = CopyPartOf(O.data(), O.size(), Data, Size);
break;
default: assert(0);
}
assert(NewSize > 0 && "CrossOver returned empty unit");
assert(NewSize <= MaxSize && "CrossOver returned overisized unit");
- memcpy(Data, U.data(), NewSize);
return NewSize;
}
assert(DE->GetW().size());
// Linear search is fine here as this happens seldom.
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
- PersistentAutoDictionary.push_back({DE->GetW(), 1});
+ PersistentAutoDictionary.push_back(*DE);
}
}
Printf("###### End of recommended dictionary. ######\n");
}
-void MutationDispatcher::PrintMutationSequence() {
+void MutationDispatcher::PrintMutationSequence(bool Verbose) {
Printf("MS: %zd ", CurrentMutatorSequence.size());
- for (auto M : CurrentMutatorSequence)
- Printf("%s-", M.Name);
+ size_t EntriesToPrint =
+ Verbose ? CurrentMutatorSequence.size()
+ : std::min(kMaxMutationsToPrint, CurrentMutatorSequence.size());
+ for (size_t i = 0; i < EntriesToPrint; i++)
+ Printf("%s-", CurrentMutatorSequence[i].Name);
if (!CurrentDictionaryEntrySequence.empty()) {
Printf(" DE: ");
- for (auto DE : CurrentDictionaryEntrySequence) {
+ EntriesToPrint = Verbose ? CurrentDictionaryEntrySequence.size()
+ : std::min(kMaxMutationsToPrint,
+ CurrentDictionaryEntrySequence.size());
+ for (size_t i = 0; i < EntriesToPrint; i++) {
Printf("\"");
- PrintASCII(DE->GetW(), "\"-");
+ PrintASCII(CurrentDictionaryEntrySequence[i]->GetW(), "\"-");
}
}
}
+std::string MutationDispatcher::MutationSequence() {
+ std::string MS;
+ for (auto M : CurrentMutatorSequence) {
+ MS += M.Name;
+ MS += "-";
+ }
+ return MS;
+}
+
size_t MutationDispatcher::Mutate(uint8_t *Data, size_t Size, size_t MaxSize) {
return MutateImpl(Data, Size, MaxSize, Mutators);
}
~MutationDispatcher() {}
/// Indicate that we are about to start a new sequence of mutations.
void StartMutationSequence();
- /// Print the current sequence of mutations.
- void PrintMutationSequence();
+ /// Print the current sequence of mutations. Only prints the full sequence
+ /// when Verbose is true.
+ void PrintMutationSequence(bool Verbose = true);
+ /// Return the current sequence of mutations.
+ std::string MutationSequence();
/// Indicate that the current sequence of mutations was successful.
void RecordSuccessfulMutationSequence();
/// Mutates data by invoking user-provided mutator.
size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize);
/// Mutates data by inserting several repeated bytes.
size_t Mutate_InsertRepeatedBytes(uint8_t *Data, size_t Size, size_t MaxSize);
- /// Mutates data by chanding one byte.
+ /// Mutates data by changing one byte.
size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize);
- /// Mutates data by chanding one bit.
+ /// Mutates data by changing one bit.
size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize);
/// Mutates data by copying/inserting a part of data into a different place.
size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize);
// Dictionary provided by the user via -dict=DICT_FILE.
Dictionary ManualDictionary;
- // Temporary dictionary modified by the fuzzer itself,
- // recreated periodically.
- Dictionary TempAutoDictionary;
// Persistent dictionary modified by the fuzzer, consists of
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
int Verbosity = 1;
size_t MaxLen = 0;
size_t LenControl = 1000;
+ bool KeepSeed = false;
int UnitTimeoutSec = 300;
int TimeoutExitCode = 70;
int OOMExitCode = 71;
int RssLimitMb = 0;
int MallocLimitMb = 0;
bool DoCrossOver = true;
+ bool CrossOverUniformDist = false;
int MutateDepth = 5;
bool ReduceDepth = false;
bool UseCounters = false;
size_t MaxNumberOfRuns = -1L;
int ReportSlowUnits = 10;
bool OnlyASCII = false;
- bool Entropic = false;
+ bool Entropic = true;
size_t EntropicFeatureFrequencyThreshold = 0xFF;
size_t EntropicNumberOfRarestFeatures = 100;
+ bool EntropicScalePerExecTime = false;
std::string OutputCorpus;
std::string ArtifactPrefix = "./";
std::string ExactArtifactPath;
std::string DataFlowTrace;
std::string CollectDataFlow;
std::string FeaturesDir;
+ std::string MutationGraphFile;
std::string StopFile;
bool SaveArtifacts = true;
bool PrintNEW = true; // Print a status line when new units are found;
bool PrintFinalStats = false;
bool PrintCorpusStats = false;
bool PrintCoverage = false;
+ bool PrintFullCoverage = false;
bool DumpCoverage = false;
bool DetectLeaks = true;
int PurgeAllocatorIntervalSec = 1;
int TraceMalloc = 0;
bool HandleAbrt = false;
+ bool HandleAlrm = false;
bool HandleBus = false;
bool HandleFpe = false;
bool HandleIll = false;
bool HandleXfsz = false;
bool HandleUsr1 = false;
bool HandleUsr2 = false;
+ bool HandleWinExcept = false;
};
} // namespace fuzzer
#define LIBFUZZER_LINUX 1
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 0
#elif __APPLE__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 0
#elif __NetBSD__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 1
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 0
#elif __FreeBSD__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 1
-#define LIBFUZZER_OPENBSD 0
-#define LIBFUZZER_WINDOWS 0
-#define LIBFUZZER_EMSCRIPTEN 0
-#elif __OpenBSD__
-#define LIBFUZZER_APPLE 0
-#define LIBFUZZER_FUCHSIA 0
-#define LIBFUZZER_LINUX 0
-#define LIBFUZZER_NETBSD 0
-#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 1
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 0
#elif _WIN32
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 1
#define LIBFUZZER_EMSCRIPTEN 0
#elif __Fuchsia__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 0
#elif __EMSCRIPTEN__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#define LIBFUZZER_WINDOWS 0
#define LIBFUZZER_EMSCRIPTEN 1
#else
#define LIBFUZZER_POSIX \
(LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \
- LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN)
+ LIBFUZZER_FREEBSD || LIBFUZZER_EMSCRIPTEN)
#ifdef __x86_64
#if __has_attribute(target)
public:
Random(unsigned int seed) : std::minstd_rand(seed) {}
result_type operator()() { return this->std::minstd_rand::operator()(); }
- size_t Rand() { return this->operator()(); }
- size_t RandBool() { return Rand() % 2; }
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type Rand() {
+ return static_cast<T>(this->operator()());
+ }
+ size_t RandBool() { return this->operator()() % 2; }
size_t SkewTowardsLast(size_t n) {
size_t T = this->operator()(n * n);
- size_t Res = sqrt(T);
+ size_t Res = static_cast<size_t>(sqrt(T));
return Res;
}
- size_t operator()(size_t n) { return n ? Rand() % n : 0; }
- intptr_t operator()(intptr_t From, intptr_t To) {
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type operator()(T n) {
+ return n ? Rand<T>() % n : 0;
+ }
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type
+ operator()(T From, T To) {
assert(From < To);
- intptr_t RangeSize = To - From + 1;
- return operator()(RangeSize) + From;
+ auto RangeSize = static_cast<unsigned long long>(To) -
+ static_cast<unsigned long long>(From) + 1;
+ return static_cast<T>(this->operator()(RangeSize) + From);
}
};
s->state[4] += e;
}
-void sha1_addUncounted(sha1nfo *s, uint8_t data) {
- uint8_t * const b = (uint8_t*) s->buffer;
+// Adds the least significant byte of |data|.
+void sha1_addUncounted(sha1nfo *s, uint32_t data) {
+ uint8_t *const b = (uint8_t *)s->buffer;
#ifdef SHA_BIG_ENDIAN
- b[s->bufferOffset] = data;
+ b[s->bufferOffset] = static_cast<uint8_t>(data);
#else
- b[s->bufferOffset ^ 3] = data;
+ b[s->bufferOffset ^ 3] = static_cast<uint8_t>(data);
#endif
s->bufferOffset++;
if (s->bufferOffset == BLOCK_LENGTH) {
}
if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin())
Printf("INFO: %zd Extra Counters\n", NumExtraCounters);
+
+ size_t MaxFeatures = CollectFeatures([](uint32_t) {});
+ if (MaxFeatures > std::numeric_limits<uint32_t>::max())
+ Printf("WARNING: The coverage PC tables may produce up to %zu features.\n"
+ "This exceeds the maximum 32-bit value. Some features may be\n"
+ "ignored, and fuzzing may become less precise. If possible,\n"
+ "consider refactoring the fuzzer into several smaller fuzzers\n"
+ "linked against only a portion of the current target.\n",
+ MaxFeatures);
}
ATTRIBUTE_NO_SANITIZE_ALL
return FocusFunctionCounterPtr && *FocusFunctionCounterPtr;
}
-void TracePC::PrintCoverage() {
+void TracePC::PrintCoverage(bool PrintAllCounters) {
if (!EF->__sanitizer_symbolize_pc ||
!EF->__sanitizer_get_module_and_offset_for_pc) {
Printf("INFO: __sanitizer_symbolize_pc or "
" not printing coverage\n");
return;
}
- Printf("COVERAGE:\n");
+ Printf(PrintAllCounters ? "FULL COVERAGE:\n" : "COVERAGE:\n");
auto CoveredFunctionCallback = [&](const PCTableEntry *First,
const PCTableEntry *Last,
uintptr_t Counter) {
std::string LineStr = DescribePC("%l", VisualizePC);
size_t NumEdges = Last - First;
Vector<uintptr_t> UncoveredPCs;
+ Vector<uintptr_t> CoveredPCs;
for (auto TE = First; TE < Last; TE++)
if (!ObservedPCs.count(TE))
UncoveredPCs.push_back(TE->PC);
- Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter);
- Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges);
- Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(),
- LineStr.c_str());
- if (Counter)
+ else
+ CoveredPCs.push_back(TE->PC);
+
+ if (PrintAllCounters) {
+ Printf("U");
for (auto PC : UncoveredPCs)
- Printf(" UNCOVERED_PC: %s\n",
- DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str());
+ Printf(DescribePC(" %l", GetNextInstructionPc(PC)).c_str());
+ Printf("\n");
+
+ Printf("C");
+ for (auto PC : CoveredPCs)
+ Printf(DescribePC(" %l", GetNextInstructionPc(PC)).c_str());
+ Printf("\n");
+ } else {
+ Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter);
+ Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges);
+ Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(),
+ LineStr.c_str());
+ if (Counter)
+ for (auto PC : UncoveredPCs)
+ Printf(" UNCOVERED_PC: %s\n",
+ DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str());
+ }
};
IterateCoveredFunctions(CoveredFunctionCallback);
uint8_t HammingDistance = 0;
for (; I < Len; I++) {
if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) {
- HammingDistance = Popcountll(B1[I] ^ B2[I]);
+ HammingDistance = static_cast<uint8_t>(Popcountll(B1[I] ^ B2[I]));
break;
}
}
void Add(const uint8_t *Data, size_t Size) {
if (Size <= 2) return;
Size = std::min(Size, Word::GetMaxSize());
- size_t Idx = SimpleFastHash(Data, Size) % kSize;
+ auto Idx = SimpleFastHash(Data, Size) % kSize;
MemMemWords[Idx].Set(Data, Size);
}
const Word &Get(size_t Idx) {
void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; }
void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; }
void UpdateObservedPCs();
- template <class Callback> void CollectFeatures(Callback CB) const;
+ template <class Callback> size_t CollectFeatures(Callback CB) const;
void ResetMaps() {
ValueProfileMap.Reset();
void PrintModuleInfo();
- void PrintCoverage();
+ void PrintCoverage(bool PrintAllCounters);
template<class CallBack>
void IterateCoveredFunctions(CallBack CB);
Handle8bitCounter(FirstFeature, P - Begin, V);
// Iterate by Step bytes at a time.
- for (; P < End; P += Step)
- if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P))
+ for (; P + Step <= End; P += Step)
+ if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) {
+ Bundle = HostToLE(Bundle);
for (size_t I = 0; I < Step; I++, Bundle >>= 8)
if (uint8_t V = Bundle & 0xff)
Handle8bitCounter(FirstFeature, P - Begin + I, V);
+ }
// Iterate by 1 byte until the end.
for (; P < End; P++)
return Bit;
}
-template <class Callback> // void Callback(size_t Feature)
-ATTRIBUTE_NO_SANITIZE_ADDRESS
-ATTRIBUTE_NOINLINE
-void TracePC::CollectFeatures(Callback HandleFeature) const {
+template <class Callback> // void Callback(uint32_t Feature)
+ATTRIBUTE_NO_SANITIZE_ADDRESS ATTRIBUTE_NOINLINE size_t
+TracePC::CollectFeatures(Callback HandleFeature) const {
auto Handle8bitCounter = [&](size_t FirstFeature,
size_t Idx, uint8_t Counter) {
if (UseCounters)
- HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter));
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx * 8 +
+ CounterToFeature(Counter)));
else
- HandleFeature(FirstFeature + Idx);
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx));
};
size_t FirstFeature = 0;
if (UseValueProfileMask) {
ValueProfileMap.ForEach([&](size_t Idx) {
- HandleFeature(FirstFeature + Idx);
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx));
});
FirstFeature += ValueProfileMap.SizeInBits();
}
// Step function, grows similar to 8 * Log_2(A).
- auto StackDepthStepFunction = [](uint32_t A) -> uint32_t {
- if (!A) return A;
- uint32_t Log2 = Log(A);
- if (Log2 < 3) return A;
+ auto StackDepthStepFunction = [](size_t A) -> size_t {
+ if (!A)
+ return A;
+ auto Log2 = Log(A);
+ if (Log2 < 3)
+ return A;
Log2 -= 3;
return (Log2 + 1) * 8 + ((A >> Log2) & 7);
};
assert(StackDepthStepFunction(1024 * 4) == 80);
assert(StackDepthStepFunction(1024 * 1024) == 144);
- if (auto MaxStackOffset = GetMaxStackOffset())
- HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8));
+ if (auto MaxStackOffset = GetMaxStackOffset()) {
+ HandleFeature(static_cast<uint32_t>(
+ FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)));
+ FirstFeature += StackDepthStepFunction(std::numeric_limits<size_t>::max());
+ }
+
+ return FirstFeature;
}
extern TracePC TPC;
char Hex[] = "0xAA";
Hex[2] = Str[Pos + 2];
Hex[3] = Str[Pos + 3];
- U->push_back(strtol(Hex, nullptr, 16));
+ U->push_back(static_cast<uint8_t>(strtol(Hex, nullptr, 16)));
Pos += 3;
continue;
}
return N;
}
-size_t SimpleFastHash(const uint8_t *Data, size_t Size) {
- size_t Res = 0;
+uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial) {
+ uint64_t Res = Initial;
+ const uint8_t *Bytes = static_cast<const uint8_t *>(Data);
for (size_t i = 0; i < Size; i++)
- Res = Res * 11 + Data[i];
+ Res = Res * 11 + Bytes[i];
return Res;
}
std::string SearchRegexCmd(const std::string &Regex);
-size_t SimpleFastHash(const uint8_t *Data, size_t Size);
+uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial = 0);
-inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; }
+inline size_t Log(size_t X) {
+ return static_cast<size_t>((sizeof(unsigned long long) * 8) - Clzll(X) - 1);
+}
inline size_t PageSize() { return 4096; }
inline uint8_t *RoundUpByPage(uint8_t *P) {
return reinterpret_cast<uint8_t *>(X);
}
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+template <typename T> T HostToLE(T X) { return X; }
+#else
+template <typename T> T HostToLE(T X) { return Bswap(X); }
+#endif
+
} // namespace fuzzer
#endif // LLVM_FUZZER_UTIL_H
}
}
-void InterruptHandler() {
- fd_set readfds;
- // Ctrl-C sends ETX in Zircon.
- do {
- FD_ZERO(&readfds);
- FD_SET(STDIN_FILENO, &readfds);
- select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, nullptr);
- } while(!FD_ISSET(STDIN_FILENO, &readfds) || getchar() != 0x03);
- Fuzzer::StaticInterruptCallback();
-}
-
// CFAOffset is used to reference the stack pointer before entering the
// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping
// to the trampoline we copy all the registers onto the stack. We need to make
Printf("%s", Buf);
// Set up alarm handler if needed.
- if (Options.UnitTimeoutSec > 0) {
+ if (Options.HandleAlrm && Options.UnitTimeoutSec > 0) {
std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1);
T.detach();
}
- // Set up interrupt handler if needed.
- if (Options.HandleInt || Options.HandleTerm) {
- std::thread T(InterruptHandler);
- T.detach();
- }
+ // Options.HandleInt and Options.HandleTerm are not supported on Fuchsia
// Early exit if no crash handler needed.
if (!Options.HandleSegv && !Options.HandleBus && !Options.HandleIll &&
return rc;
}
- return Info.return_code;
+ return static_cast<int>(Info.return_code);
}
bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) {
//===----------------------------------------------------------------------===//
#include "FuzzerPlatform.h"
#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \
- LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN
+ LIBFUZZER_EMSCRIPTEN
#include "FuzzerCommand.h"
#include <stdlib.h>
return;
}
- sigact = {};
- sigact.sa_flags = SA_SIGINFO;
- sigact.sa_sigaction = callback;
- if (sigaction(signum, &sigact, 0)) {
+ struct sigaction new_sigact = {};
+ // Address sanitizer needs SA_ONSTACK (causing the signal handler to run on a
+ // dedicated stack) in order to be able to detect stack overflows; keep the
+ // flag if it's set.
+ new_sigact.sa_flags = SA_SIGINFO | (sigact.sa_flags & SA_ONSTACK);
+ new_sigact.sa_sigaction = callback;
+ if (sigaction(signum, &new_sigact, nullptr)) {
Printf("libFuzzer: sigaction failed with %d\n", errno);
exit(1);
}
void SetSignalHandler(const FuzzingOptions& Options) {
// setitimer is not implemented in emscripten.
- if (Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN)
+ if (Options.HandleAlrm && Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN)
SetTimer(Options.UnitTimeoutSec / 2 + 1);
if (Options.HandleInt)
SetSigaction(SIGINT, InterruptHandler);
if (getrusage(RUSAGE_SELF, &usage))
return 0;
if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD ||
- LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) {
+ LIBFUZZER_EMSCRIPTEN) {
// ru_maxrss is in KiB
return usage.ru_maxrss >> 10;
} else if (LIBFUZZER_APPLE) {
if (HandlerOpt->HandleFpe)
Fuzzer::StaticCrashSignalCallback();
break;
- // TODO: handle (Options.HandleXfsz)
+ // This is an undocumented exception code corresponding to a Visual C++
+ // Exception.
+ //
+ // See: https://devblogs.microsoft.com/oldnewthing/20100730-00/?p=13273
+ case 0xE06D7363:
+ if (HandlerOpt->HandleWinExcept)
+ Fuzzer::StaticCrashSignalCallback();
+ break;
+ // TODO: Handle (Options.HandleXfsz)
}
return EXCEPTION_CONTINUE_SEARCH;
}
void SetSignalHandler(const FuzzingOptions& Options) {
HandlerOpt = &Options;
- if (Options.UnitTimeoutSec > 0)
+ if (Options.HandleAlrm && Options.UnitTimeoutSec > 0)
Timer.SetTimer(Options.UnitTimeoutSec / 2 + 1);
if (Options.HandleInt || Options.HandleTerm)
}
if (Options.HandleSegv || Options.HandleBus || Options.HandleIll ||
- Options.HandleFpe)
+ Options.HandleFpe || Options.HandleWinExcept)
SetUnhandledExceptionFilter(ExceptionHandler);
if (Options.HandleAbrt)
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#elif __APPLE__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_APPLE 1
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#elif __NetBSD__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_NETBSD 1
#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 0
#elif __FreeBSD__
#define LIBFUZZER_LINUX 0
#define LIBFUZZER_APPLE 0
#define LIBFUZZER_NETBSD 0
#define LIBFUZZER_FREEBSD 1
-#define LIBFUZZER_OPENBSD 0
-#elif __OpenBSD__
-#define LIBFUZZER_LINUX 0
-#define LIBFUZZER_APPLE 0
-#define LIBFUZZER_NETBSD 0
-#define LIBFUZZER_FREEBSD 0
-#define LIBFUZZER_OPENBSD 1
#else
#error "Support for your platform has not been implemented"
#endif
// Run:
// # Collect data flow and coverage for INPUT_FILE
// # write to OUTPUT_FILE (default: stdout)
-// export DFSAN_OPTIONS=fast16labels=1:warn_unimplemented=0
+// export DFSAN_OPTIONS=warn_unimplemented=0
// ./a.out INPUT_FILE [OUTPUT_FILE]
//
// # Print all instrumented functions. llvm-symbolizer must be present in PATH
return __dft.PCsBeg[BlockIdx * 2 + 1] & PCFLAG_FUNC_ENTRY;
}
-const int kNumLabels = 16;
+const int kNumLabels = 8;
// Prints all instrumented functions.
static int PrintFunctions() {
// We'll need to make a proper in-process symbolizer work with DFSan.
FILE *Pipe = popen("sed 's/(+/ /g; s/).*//g' "
"| llvm-symbolizer "
- "| grep 'dfs\\$' "
- "| sed 's/dfs\\$//g' "
+ "| grep '\\.dfsan' "
+ "| sed 's/\\.dfsan//g' "
"| c++filt",
"w");
for (size_t I = 0; I < __dft.NumGuards; I++) {
if(WIN32)
list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -Wl,-defaultlib:libcmt,-defaultlib:oldnames)
else()
+ if (APPLE)
+ list(APPEND LIBFUZZER_UNITTEST_CFLAGS -isysroot ${DARWIN_osx_SYSROOT})
+ list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -isysroot ${DARWIN_osx_SYSROOT})
+ endif()
list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -lpthread)
endif()
EXPECT_EQ(false, DataProv.ConsumeBool());
}
+TEST(FuzzedDataProvider, PickValueInStdArray) {
+ FuzzedDataProvider DataProv(Data, sizeof(Data));
+ const std::array<int, 5> Array = {1, 2, 3, 4, 5};
+ EXPECT_EQ(5, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(2, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(2, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(3, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(3, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(3, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(1, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(3, DataProv.PickValueInArray(Array));
+ EXPECT_EQ(2, DataProv.PickValueInArray(Array));
+}
+
TEST(FuzzedDataProvider, PickValueInArray) {
FuzzedDataProvider DataProv(Data, sizeof(Data));
const int Array[] = {1, 2, 3, 4, 5};
TEST(Corpus, Distribution) {
DataFlowTrace DFT;
Random Rand(0);
- struct EntropicOptions Entropic = {false, 0xFF, 100};
+ struct EntropicOptions Entropic = {false, 0xFF, 100, false};
std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
size_t N = 10;
size_t TriesPerUnit = 1<<16;
for (size_t i = 0; i < N; i++)
- C->AddToCorpus(Unit{static_cast<uint8_t>(i)}, 1, false, false, {}, DFT,
- nullptr);
+ C->AddToCorpus(Unit{static_cast<uint8_t>(i)}, /*NumFeatures*/ 1,
+ /*MayDeleteFile*/ false, /*HasFocusFunction*/ false,
+ /*ForceAddToCorpus*/ false,
+ /*TimeOfUnit*/ std::chrono::microseconds(0),
+ /*FeatureSet*/ {}, DFT,
+ /*BaseII*/ nullptr);
Vector<size_t> Hist(N);
for (size_t i = 0; i < N * TriesPerUnit; i++) {
}
}
-TEST(Merge, Bad) {
- const char *kInvalidInputs[] = {
- "",
- "x",
- "3\nx",
- "2\n3",
- "2\n2",
- "2\n2\nA\n",
- "2\n2\nA\nB\nC\n",
- "0\n0\n",
- "1\n1\nA\nFT 0",
- "1\n1\nA\nSTARTED 1",
- };
- Merger M;
- for (auto S : kInvalidInputs) {
- // fprintf(stderr, "TESTING:\n%s\n", S);
- EXPECT_FALSE(M.Parse(S, false));
- }
+template <typename T> void EQ(const Vector<T> &A, const Vector<T> &B) {
+ EXPECT_EQ(A, B);
}
-void EQ(const Vector<uint32_t> &A, const Vector<uint32_t> &B) {
- EXPECT_EQ(A, B);
+template <typename T> void EQ(const Set<T> &A, const Vector<T> &B) {
+ EXPECT_EQ(A, Set<T>(B.begin(), B.end()));
}
-void EQ(const Vector<std::string> &A, const Vector<std::string> &B) {
- Set<std::string> a(A.begin(), A.end());
+void EQ(const Vector<MergeFileInfo> &A, const Vector<std::string> &B) {
+ Set<std::string> a;
+ for (const auto &File : A)
+ a.insert(File.Name);
Set<std::string> b(B.begin(), B.end());
EXPECT_EQ(a, b);
}
-static void Merge(const std::string &Input,
- const Vector<std::string> Result,
- size_t NumNewFeatures) {
- Merger M;
- Vector<std::string> NewFiles;
- Set<uint32_t> NewFeatures, NewCov;
- EXPECT_TRUE(M.Parse(Input, true));
- EXPECT_EQ(NumNewFeatures, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles));
- EQ(NewFiles, Result);
-}
+#define TRACED_EQ(A, ...) \
+ { \
+ SCOPED_TRACE(#A); \
+ EQ(A, __VA_ARGS__); \
+ }
-TEST(Merge, Good) {
+TEST(Merger, Parse) {
Merger M;
+ const char *kInvalidInputs[] = {
+ // Bad file numbers
+ "",
+ "x",
+ "0\n0",
+ "3\nx",
+ "2\n3",
+ "2\n2",
+ // Bad file names
+ "2\n2\nA\n",
+ "2\n2\nA\nB\nC\n",
+ // Unknown markers
+ "2\n1\nA\nSTARTED 0\nBAD 0 0x0",
+ // Bad file IDs
+ "1\n1\nA\nSTARTED 1",
+ "2\n1\nA\nSTARTED 0\nFT 1 0x0",
+ };
+ for (auto S : kInvalidInputs) {
+ SCOPED_TRACE(S);
+ EXPECT_FALSE(M.Parse(S, false));
+ }
+
+ // Parse initial control file
EXPECT_TRUE(M.Parse("1\n0\nAA\n", false));
- EXPECT_EQ(M.Files.size(), 1U);
+ ASSERT_EQ(M.Files.size(), 1U);
EXPECT_EQ(M.NumFilesInFirstCorpus, 0U);
EXPECT_EQ(M.Files[0].Name, "AA");
EXPECT_TRUE(M.LastFailure.empty());
EXPECT_EQ(M.FirstNotProcessedFile, 0U);
+ // Parse control file that failed on first attempt
EXPECT_TRUE(M.Parse("2\n1\nAA\nBB\nSTARTED 0 42\n", false));
- EXPECT_EQ(M.Files.size(), 2U);
+ ASSERT_EQ(M.Files.size(), 2U);
EXPECT_EQ(M.NumFilesInFirstCorpus, 1U);
EXPECT_EQ(M.Files[0].Name, "AA");
EXPECT_EQ(M.Files[1].Name, "BB");
EXPECT_EQ(M.LastFailure, "AA");
EXPECT_EQ(M.FirstNotProcessedFile, 1U);
+ // Parse control file that failed on later attempt
EXPECT_TRUE(M.Parse("3\n1\nAA\nBB\nC\n"
- "STARTED 0 1000\n"
- "FT 0 1 2 3\n"
- "STARTED 1 1001\n"
- "FT 1 4 5 6 \n"
- "STARTED 2 1002\n"
- "", true));
- EXPECT_EQ(M.Files.size(), 3U);
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "",
+ true));
+ ASSERT_EQ(M.Files.size(), 3U);
EXPECT_EQ(M.NumFilesInFirstCorpus, 1U);
EXPECT_EQ(M.Files[0].Name, "AA");
EXPECT_EQ(M.Files[0].Size, 1000U);
EXPECT_EQ(M.Files[2].Size, 1002U);
EXPECT_EQ(M.LastFailure, "C");
EXPECT_EQ(M.FirstNotProcessedFile, 3U);
- EQ(M.Files[0].Features, {1, 2, 3});
- EQ(M.Files[1].Features, {4, 5, 6});
-
-
- Vector<std::string> NewFiles;
- Set<uint32_t> NewFeatures, NewCov;
+ TRACED_EQ(M.Files[0].Features, {1, 2, 3});
+ TRACED_EQ(M.Files[1].Features, {4, 5, 6});
+
+ // Parse control file without features or PCs
+ EXPECT_TRUE(M.Parse("2\n0\nAA\nBB\n"
+ "STARTED 0 1000\n"
+ "FT 0\n"
+ "COV 0\n"
+ "STARTED 1 1001\n"
+ "FT 1\n"
+ "COV 1\n"
+ "",
+ true));
+ ASSERT_EQ(M.Files.size(), 2U);
+ EXPECT_EQ(M.NumFilesInFirstCorpus, 0U);
+ EXPECT_TRUE(M.LastFailure.empty());
+ EXPECT_EQ(M.FirstNotProcessedFile, 2U);
+ EXPECT_TRUE(M.Files[0].Features.empty());
+ EXPECT_TRUE(M.Files[0].Cov.empty());
+ EXPECT_TRUE(M.Files[1].Features.empty());
+ EXPECT_TRUE(M.Files[1].Cov.empty());
+ // Parse features and PCs
EXPECT_TRUE(M.Parse("3\n2\nAA\nBB\nC\n"
- "STARTED 0 1000\nFT 0 1 2 3\n"
- "STARTED 1 1001\nFT 1 4 5 6 \n"
- "STARTED 2 1002\nFT 2 6 1 3 \n"
- "", true));
- EXPECT_EQ(M.Files.size(), 3U);
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "COV 0 11 12 13\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6\n"
+ "COV 1 7 8 9\n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "COV 2 16 11 13\n"
+ "",
+ true));
+ ASSERT_EQ(M.Files.size(), 3U);
EXPECT_EQ(M.NumFilesInFirstCorpus, 2U);
EXPECT_TRUE(M.LastFailure.empty());
EXPECT_EQ(M.FirstNotProcessedFile, 3U);
- EQ(M.Files[0].Features, {1, 2, 3});
- EQ(M.Files[1].Features, {4, 5, 6});
- EQ(M.Files[2].Features, {1, 3, 6});
- EXPECT_EQ(0U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles));
- EQ(NewFiles, {});
+ TRACED_EQ(M.Files[0].Features, {1, 2, 3});
+ TRACED_EQ(M.Files[0].Cov, {11, 12, 13});
+ TRACED_EQ(M.Files[1].Features, {4, 5, 6});
+ TRACED_EQ(M.Files[1].Cov, {7, 8, 9});
+ TRACED_EQ(M.Files[2].Features, {1, 3, 6});
+ TRACED_EQ(M.Files[2].Cov, {16});
+}
+
+TEST(Merger, Merge) {
+ Merger M;
+ Set<uint32_t> Features, NewFeatures;
+ Set<uint32_t> Cov, NewCov;
+ Vector<std::string> NewFiles;
+ // Adds new files and features
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 6U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"A", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ // Doesn't return features or files in the initial corpus.
EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n"
- "STARTED 0 1000\nFT 0 1 2 3\n"
- "STARTED 1 1001\nFT 1 4 5 6 \n"
- "STARTED 2 1002\nFT 2 6 1 3\n"
- "", true));
- EQ(M.Files[0].Features, {1, 2, 3});
- EQ(M.Files[1].Features, {4, 5, 6});
- EQ(M.Files[2].Features, {1, 3, 6});
- EXPECT_EQ(3U, M.Merge({}, &NewFeatures, {}, &NewCov, &NewFiles));
- EQ(NewFiles, {"B"});
-
- // Same as the above, but with InitialFeatures.
- EXPECT_TRUE(M.Parse("2\n0\nB\nC\n"
- "STARTED 0 1001\nFT 0 4 5 6 \n"
- "STARTED 1 1002\nFT 1 6 1 3\n"
- "", true));
- EQ(M.Files[0].Features, {4, 5, 6});
- EQ(M.Files[1].Features, {1, 3, 6});
- Set<uint32_t> InitialFeatures;
- InitialFeatures.insert(1);
- InitialFeatures.insert(2);
- InitialFeatures.insert(3);
- EXPECT_EQ(3U, M.Merge(InitialFeatures, &NewFeatures, {}, &NewCov, &NewFiles));
- EQ(NewFiles, {"B"});
-}
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+
+ // No new features, so no new files
+ EXPECT_TRUE(M.Parse("3\n2\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 0U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {});
+ TRACED_EQ(NewFeatures, {});
+
+ // Can pass initial features and coverage.
+ Features = {1, 2, 3};
+ Cov = {};
+ EXPECT_TRUE(M.Parse("2\n0\nA\nB\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6\n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U);
+ TRACED_EQ(M.Files, {"A", "B"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+ Features.clear();
+ Cov.clear();
-TEST(Merge, Merge) {
-
- Merge("3\n1\nA\nB\nC\n"
- "STARTED 0 1000\nFT 0 1 2 3\n"
- "STARTED 1 1001\nFT 1 4 5 6 \n"
- "STARTED 2 1002\nFT 2 6 1 3 \n",
- {"B"}, 3);
-
- Merge("3\n0\nA\nB\nC\n"
- "STARTED 0 2000\nFT 0 1 2 3\n"
- "STARTED 1 1001\nFT 1 4 5 6 \n"
- "STARTED 2 1002\nFT 2 6 1 3 \n",
- {"A", "B", "C"}, 6);
-
- Merge("4\n0\nA\nB\nC\nD\n"
- "STARTED 0 2000\nFT 0 1 2 3\n"
- "STARTED 1 1101\nFT 1 4 5 6 \n"
- "STARTED 2 1102\nFT 2 6 1 3 100 \n"
- "STARTED 3 1000\nFT 3 1 \n",
- {"A", "B", "C", "D"}, 7);
-
- Merge("4\n1\nA\nB\nC\nD\n"
- "STARTED 0 2000\nFT 0 4 5 6 7 8\n"
- "STARTED 1 1100\nFT 1 1 2 3 \n"
- "STARTED 2 1100\nFT 2 2 3 \n"
- "STARTED 3 1000\nFT 3 1 \n",
- {"B", "D"}, 3);
+ // Parse smaller files first
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 2000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3 \n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 6U);
+ TRACED_EQ(M.Files, {"B", "C", "A"});
+ TRACED_EQ(NewFiles, {"B", "C", "A"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ EXPECT_TRUE(M.Parse("4\n0\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1101\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1102\n"
+ "FT 2 6 1 3 100 \n"
+ "STARTED 3 1000\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 7U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"D", "B", "C", "A"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6, 100});
+
+ // For same sized file, parse more features first
+ EXPECT_TRUE(M.Parse("4\n1\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 4 5 6 7 8\n"
+ "STARTED 1 1100\n"
+ "FT 1 1 2 3 \n"
+ "STARTED 2 1100\n"
+ "FT 2 2 3 \n"
+ "STARTED 3 1000\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.Merge(Features, &NewFeatures, Cov, &NewCov, &NewFiles), 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"D", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3});
}
+#undef TRACED_EQ
+
TEST(DFT, BlockCoverage) {
BlockCoverage Cov;
// Assuming C0 has 5 instrumented blocks,
const size_t FeatIdx1 = 0, FeatIdx2 = 42, FeatIdx3 = 12, FeatIdx4 = 26;
size_t Index;
// Create input corpus with default entropic configuration
- struct EntropicOptions Entropic = {true, 0xFF, 100};
+ struct EntropicOptions Entropic = {true, 0xFF, 100, false};
std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
std::unique_ptr<InputInfo> II(new InputInfo());
TEST(Entropic, ComputeEnergy) {
const double Precision = 0.01;
- struct EntropicOptions Entropic = {true, 0xFF, 100};
+ struct EntropicOptions Entropic = {true, 0xFF, 100, false};
std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
std::unique_ptr<InputInfo> II(new InputInfo());
Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {{1, 3}, {2, 3}, {3, 3}};
II->FeatureFreqs = FeatureFreqs;
II->NumExecutedMutations = 0;
- II->UpdateEnergy(4);
+ II->UpdateEnergy(4, false, std::chrono::microseconds(0));
EXPECT_LT(SubAndSquare(II->Energy, 1.450805), Precision);
II->NumExecutedMutations = 9;
- II->UpdateEnergy(5);
+ II->UpdateEnergy(5, false, std::chrono::microseconds(0));
EXPECT_LT(SubAndSquare(II->Energy, 1.525496), Precision);
II->FeatureFreqs[0].second++;
II->FeatureFreqs.push_back(std::pair<uint32_t, uint16_t>(42, 6));
II->NumExecutedMutations = 20;
- II->UpdateEnergy(10);
+ II->UpdateEnergy(10, false, std::chrono::microseconds(0));
EXPECT_LT(SubAndSquare(II->Energy, 1.792831), Precision);
}
platform_specific/mutex_posix.cpp
platform_specific/utilities_posix.cpp
guarded_pool_allocator.cpp
- random.cpp
stack_trace_compressor.cpp
)
mutex.h
options.h
options.inc
- random.h
+ platform_specific/guarded_pool_allocator_fuchsia.h
+ platform_specific/guarded_pool_allocator_posix.h
+ platform_specific/guarded_pool_allocator_tls.h
+ platform_specific/mutex_fuchsia.h
+ platform_specific/mutex_posix.h
stack_trace_compressor.h
utilities.h
)
# allocators. Some supporting allocators (e.g. scudo standalone) cannot use any
# parts of the C++ standard library.
set(GWP_ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS} -fno-rtti -fno-exceptions
- -nostdinc++ -pthread)
+ -nostdinc++ -pthread -fno-omit-frame-pointer)
append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC GWP_ASAN_CFLAGS)
-append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
- GWP_ASAN_CFLAGS)
# Remove -stdlib= which is unused when passing -nostdinc++.
string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-# Options parsing support is optional. GwpAsan is totally independent of
-# sanitizer_common, the options parser is not. This is an optional library
-# that can be used by an allocator to automatically parse GwpAsan options from
-# the environment variable GWP_ASAN_FLAGS, but the allocator can choose to
-# implement its own options parsing and populate the Options struct itself.
+# Options parsing support is optional. This is an optional library that can be
+# used by an allocator to automatically parse GwpAsan options from the
+# environment variable GWP_ASAN_FLAGS, but the allocator can choose to implement
+# its own options parsing and populate the Options struct itself.
set(GWP_ASAN_OPTIONS_PARSER_SOURCES
optional/options_parser.cpp
)
options.h)
set(GWP_ASAN_OPTIONS_PARSER_CFLAGS
- ${GWP_ASAN_CFLAGS}
- ${SANITIZER_COMMON_CFLAGS})
-set(GWP_ASAN_OPTIONS_PARSER_OBJECT_LIBS
- RTSanitizerCommon
- RTSanitizerCommonNoLibc)
+ ${GWP_ASAN_CFLAGS})
if (COMPILER_RT_HAS_GWP_ASAN)
foreach(arch ${GWP_ASAN_SUPPORTED_ARCH})
ADDITIONAL_HEADERS ${GWP_ASAN_HEADERS}
CFLAGS ${GWP_ASAN_CFLAGS})
- # Note: If you choose to add this as an object library, ensure you also
- # include the sanitizer_common flag parsing object lib (generally
- # 'RTSanitizerCommonNoTermination'). Also, you'll need to either implement
- # your own backtrace support (see optional/backtrace.h), or choose between one
- # of the pre-implemented backtrace support options (see below).
add_compiler_rt_object_libraries(RTGwpAsanOptionsParser
ARCHS ${GWP_ASAN_SUPPORTED_ARCH}
SOURCES ${GWP_ASAN_OPTIONS_PARSER_SOURCES}
__builtin_trap();
}
+constexpr size_t AllocationMetadata::kStackFrameStorageBytes;
+constexpr size_t AllocationMetadata::kMaxTraceLengthToCollect;
+
void AllocationMetadata::RecordAllocation(uintptr_t AllocAddr,
size_t AllocSize) {
Addr = AllocAddr;
- Size = AllocSize;
+ RequestedSize = AllocSize;
IsDeallocated = false;
AllocationTrace.ThreadID = getThreadID();
static constexpr size_t kMaxTraceLengthToCollect = 128;
// Records the given allocation metadata into this struct.
- void RecordAllocation(uintptr_t Addr, size_t Size);
+ void RecordAllocation(uintptr_t Addr, size_t RequestedSize);
// Record that this allocation is now deallocated.
void RecordDeallocation();
// valid, as the allocation has never occurred.
uintptr_t Addr = 0;
// Represents the actual size of the allocation.
- size_t Size = 0;
+ size_t RequestedSize = 0;
CallSiteInfo AllocationTrace;
CallSiteInfo DeallocationTrace;
// crash handler. This, in conjunction with the Metadata array, forms the entire
// set of information required for understanding a GWP-ASan crash.
struct AllocatorState {
+ constexpr AllocatorState() {}
+
// Returns whether the provided pointer is a current sampled allocation that
// is owned by this pool.
GWP_ASAN_ALWAYS_INLINE bool pointerIsMine(const void *Ptr) const {
-//===-- crash_handler_interface.cpp -----------------------------*- C++ -*-===//
+//===-- crash_handler.cpp ---------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "gwp_asan/stack_trace_compressor.h"
#include <assert.h>
+#include <stdint.h>
+#include <string.h>
using AllocationMetadata = gwp_asan::AllocationMetadata;
using Error = gwp_asan::Error;
size_t __gwp_asan_get_allocation_size(
const gwp_asan::AllocationMetadata *AllocationMeta) {
- return AllocationMeta->Size;
+ return AllocationMeta->RequestedSize;
}
uint64_t __gwp_asan_get_allocation_thread_id(
size_t __gwp_asan_get_allocation_trace(
const gwp_asan::AllocationMetadata *AllocationMeta, uintptr_t *Buffer,
size_t BufferLen) {
- return gwp_asan::compression::unpack(
+ uintptr_t UncompressedBuffer[AllocationMetadata::kMaxTraceLengthToCollect];
+ size_t UnpackedLength = gwp_asan::compression::unpack(
AllocationMeta->AllocationTrace.CompressedTrace,
- AllocationMeta->AllocationTrace.TraceSize, Buffer, BufferLen);
+ AllocationMeta->AllocationTrace.TraceSize, UncompressedBuffer,
+ AllocationMetadata::kMaxTraceLengthToCollect);
+ if (UnpackedLength < BufferLen)
+ BufferLen = UnpackedLength;
+ memcpy(Buffer, UncompressedBuffer, BufferLen * sizeof(*Buffer));
+ return UnpackedLength;
}
bool __gwp_asan_is_deallocated(
size_t __gwp_asan_get_deallocation_trace(
const gwp_asan::AllocationMetadata *AllocationMeta, uintptr_t *Buffer,
size_t BufferLen) {
- return gwp_asan::compression::unpack(
+ uintptr_t UncompressedBuffer[AllocationMetadata::kMaxTraceLengthToCollect];
+ size_t UnpackedLength = gwp_asan::compression::unpack(
AllocationMeta->DeallocationTrace.CompressedTrace,
- AllocationMeta->DeallocationTrace.TraceSize, Buffer, BufferLen);
+ AllocationMeta->DeallocationTrace.TraceSize, UncompressedBuffer,
+ AllocationMetadata::kMaxTraceLengthToCollect);
+ if (UnpackedLength < BufferLen)
+ BufferLen = UnpackedLength;
+ memcpy(Buffer, UncompressedBuffer, BufferLen * sizeof(*Buffer));
+ return UnpackedLength;
}
#ifdef __cplusplus
-//===-- crash_handler_interface.h -------------------------------*- C++ -*-===//
+//===-- crash_handler.h -----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
-//===-- gwp_asan_definitions.h ----------------------------------*- C++ -*-===//
+//===-- definitions.h -------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "gwp_asan/guarded_pool_allocator.h"
-#include "gwp_asan/optional/segv_handler.h"
#include "gwp_asan/options.h"
-#include "gwp_asan/random.h"
#include "gwp_asan/utilities.h"
-// RHEL creates the PRIu64 format macro (for printing uint64_t's) only when this
-// macro is defined before including <inttypes.h>.
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS 1
-#endif
-
#include <assert.h>
-#include <inttypes.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
+#include <stddef.h>
using AllocationMetadata = gwp_asan::AllocationMetadata;
using Error = gwp_asan::Error;
// init-order-fiasco.
GuardedPoolAllocator *SingletonPtr = nullptr;
-class ScopedBoolean {
-public:
- ScopedBoolean(bool &B) : Bool(B) { Bool = true; }
- ~ScopedBoolean() { Bool = false; }
+size_t roundUpTo(size_t Size, size_t Boundary) {
+ return (Size + Boundary - 1) & ~(Boundary - 1);
+}
-private:
- bool &Bool;
-};
+uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) {
+ return Ptr & ~(PageSize - 1);
+}
+
+bool isPowerOfTwo(uintptr_t X) { return (X & (X - 1)) == 0; }
} // anonymous namespace
// Gets the singleton implementation of this class. Thread-compatible until
return;
Check(Opts.SampleRate >= 0, "GWP-ASan Error: SampleRate is < 0.");
- Check(Opts.SampleRate <= INT32_MAX, "GWP-ASan Error: SampleRate is > 2^31.");
+ Check(Opts.SampleRate < (1 << 30), "GWP-ASan Error: SampleRate is >= 2^30.");
Check(Opts.MaxSimultaneousAllocations >= 0,
"GWP-ASan Error: MaxSimultaneousAllocations is < 0.");
State.MaxSimultaneousAllocations = Opts.MaxSimultaneousAllocations;
- State.PageSize = getPlatformPageSize();
-
- PerfectlyRightAlign = Opts.PerfectlyRightAlign;
+ const size_t PageSize = getPlatformPageSize();
+ // getPageAddr() and roundUpTo() assume the page size to be a power of 2.
+ assert((PageSize & (PageSize - 1)) == 0);
+ State.PageSize = PageSize;
size_t PoolBytesRequired =
- State.PageSize * (1 + State.MaxSimultaneousAllocations) +
+ PageSize * (1 + State.MaxSimultaneousAllocations) +
State.MaxSimultaneousAllocations * State.maximumAllocationSize();
- void *GuardedPoolMemory = mapMemory(PoolBytesRequired, kGwpAsanGuardPageName);
+ assert(PoolBytesRequired % PageSize == 0);
+ void *GuardedPoolMemory = reserveGuardedPool(PoolBytesRequired);
- size_t BytesRequired = State.MaxSimultaneousAllocations * sizeof(*Metadata);
+ size_t BytesRequired =
+ roundUpTo(State.MaxSimultaneousAllocations * sizeof(*Metadata), PageSize);
Metadata = reinterpret_cast<AllocationMetadata *>(
- mapMemory(BytesRequired, kGwpAsanMetadataName));
- markReadWrite(Metadata, BytesRequired, kGwpAsanMetadataName);
+ map(BytesRequired, kGwpAsanMetadataName));
// Allocate memory and set up the free pages queue.
- BytesRequired = State.MaxSimultaneousAllocations * sizeof(*FreeSlots);
- FreeSlots = reinterpret_cast<size_t *>(
- mapMemory(BytesRequired, kGwpAsanFreeSlotsName));
- markReadWrite(FreeSlots, BytesRequired, kGwpAsanFreeSlotsName);
+ BytesRequired = roundUpTo(
+ State.MaxSimultaneousAllocations * sizeof(*FreeSlots), PageSize);
+ FreeSlots =
+ reinterpret_cast<size_t *>(map(BytesRequired, kGwpAsanFreeSlotsName));
// Multiply the sample rate by 2 to give a good, fast approximation for (1 /
// SampleRate) chance of sampling.
AdjustedSampleRatePlusOne = 2;
initPRNG();
- ThreadLocals.NextSampleCounter =
- (getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1;
+ getThreadLocals()->NextSampleCounter =
+ ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) &
+ ThreadLocalPackedVariables::NextSampleCounterMask;
State.GuardedPagePool = reinterpret_cast<uintptr_t>(GuardedPoolMemory);
State.GuardedPagePoolEnd =
installAtFork();
}
-void GuardedPoolAllocator::disable() { PoolMutex.lock(); }
+void GuardedPoolAllocator::disable() {
+ PoolMutex.lock();
+ BacktraceMutex.lock();
+}
-void GuardedPoolAllocator::enable() { PoolMutex.unlock(); }
+void GuardedPoolAllocator::enable() {
+ PoolMutex.unlock();
+ BacktraceMutex.unlock();
+}
void GuardedPoolAllocator::iterate(void *Base, size_t Size, iterate_callback Cb,
void *Arg) {
const AllocationMetadata &Meta = Metadata[i];
if (Meta.Addr && !Meta.IsDeallocated && Meta.Addr >= Start &&
Meta.Addr < Start + Size)
- Cb(Meta.Addr, Meta.Size, Arg);
+ Cb(Meta.Addr, Meta.RequestedSize, Arg);
}
}
void GuardedPoolAllocator::uninitTestOnly() {
if (State.GuardedPagePool) {
- unmapMemory(reinterpret_cast<void *>(State.GuardedPagePool),
- State.GuardedPagePoolEnd - State.GuardedPagePool,
- kGwpAsanGuardPageName);
+ unreserveGuardedPool();
State.GuardedPagePool = 0;
State.GuardedPagePoolEnd = 0;
}
if (Metadata) {
- unmapMemory(Metadata, State.MaxSimultaneousAllocations * sizeof(*Metadata),
- kGwpAsanMetadataName);
+ unmap(Metadata,
+ roundUpTo(State.MaxSimultaneousAllocations * sizeof(*Metadata),
+ State.PageSize));
Metadata = nullptr;
}
if (FreeSlots) {
- unmapMemory(FreeSlots,
- State.MaxSimultaneousAllocations * sizeof(*FreeSlots),
- kGwpAsanFreeSlotsName);
+ unmap(FreeSlots,
+ roundUpTo(State.MaxSimultaneousAllocations * sizeof(*FreeSlots),
+ State.PageSize));
FreeSlots = nullptr;
}
+ *getThreadLocals() = ThreadLocalPackedVariables();
}
-static uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) {
- return Ptr & ~(PageSize - 1);
+// Note, minimum backing allocation size in GWP-ASan is always one page, and
+// each slot could potentially be multiple pages (but always in
+// page-increments). Thus, for anything that requires less than page size
+// alignment, we don't need to allocate extra padding to ensure the alignment
+// can be met.
+size_t GuardedPoolAllocator::getRequiredBackingSize(size_t Size,
+ size_t Alignment,
+ size_t PageSize) {
+ assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
+ assert(Alignment != 0 && "Alignment should be non-zero");
+ assert(Size != 0 && "Size should be non-zero");
+
+ if (Alignment <= PageSize)
+ return Size;
+
+ return Size + Alignment - PageSize;
}
-void *GuardedPoolAllocator::allocate(size_t Size) {
+uintptr_t GuardedPoolAllocator::alignUp(uintptr_t Ptr, size_t Alignment) {
+ assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
+ assert(Alignment != 0 && "Alignment should be non-zero");
+ if ((Ptr & (Alignment - 1)) == 0)
+ return Ptr;
+
+ Ptr += Alignment - (Ptr & (Alignment - 1));
+ return Ptr;
+}
+
+uintptr_t GuardedPoolAllocator::alignDown(uintptr_t Ptr, size_t Alignment) {
+ assert(isPowerOfTwo(Alignment) && "Alignment must be a power of two!");
+ assert(Alignment != 0 && "Alignment should be non-zero");
+ if ((Ptr & (Alignment - 1)) == 0)
+ return Ptr;
+
+ Ptr -= Ptr & (Alignment - 1);
+ return Ptr;
+}
+
+void *GuardedPoolAllocator::allocate(size_t Size, size_t Alignment) {
// GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall
// back to the supporting allocator.
- if (State.GuardedPagePoolEnd == 0)
+ if (State.GuardedPagePoolEnd == 0) {
+ getThreadLocals()->NextSampleCounter =
+ (AdjustedSampleRatePlusOne - 1) &
+ ThreadLocalPackedVariables::NextSampleCounterMask;
return nullptr;
+ }
- // Protect against recursivity.
- if (ThreadLocals.RecursiveGuard)
+ if (Size == 0)
+ Size = 1;
+ if (Alignment == 0)
+ Alignment = alignof(max_align_t);
+
+ if (!isPowerOfTwo(Alignment) || Alignment > State.maximumAllocationSize() ||
+ Size > State.maximumAllocationSize())
return nullptr;
- ScopedBoolean SB(ThreadLocals.RecursiveGuard);
- if (Size == 0 || Size > State.maximumAllocationSize())
+ size_t BackingSize = getRequiredBackingSize(Size, Alignment, State.PageSize);
+ if (BackingSize > State.maximumAllocationSize())
+ return nullptr;
+
+ // Protect against recursivity.
+ if (getThreadLocals()->RecursiveGuard)
return nullptr;
+ ScopedRecursiveGuard SRG;
size_t Index;
{
if (Index == kInvalidSlotID)
return nullptr;
- uintptr_t Ptr = State.slotToAddr(Index);
- // Should we right-align this allocation?
- if (getRandomUnsigned32() % 2 == 0) {
- AlignmentStrategy Align = AlignmentStrategy::DEFAULT;
- if (PerfectlyRightAlign)
- Align = AlignmentStrategy::PERFECT;
- Ptr +=
- State.maximumAllocationSize() - rightAlignedAllocationSize(Size, Align);
- }
- AllocationMetadata *Meta = addrToMetadata(Ptr);
+ uintptr_t SlotStart = State.slotToAddr(Index);
+ AllocationMetadata *Meta = addrToMetadata(SlotStart);
+ uintptr_t SlotEnd = State.slotToAddr(Index) + State.maximumAllocationSize();
+ uintptr_t UserPtr;
+ // Randomly choose whether to left-align or right-align the allocation, and
+ // then apply the necessary adjustments to get an aligned pointer.
+ if (getRandomUnsigned32() % 2 == 0)
+ UserPtr = alignUp(SlotStart, Alignment);
+ else
+ UserPtr = alignDown(SlotEnd - Size, Alignment);
+
+ assert(UserPtr >= SlotStart);
+ assert(UserPtr + Size <= SlotEnd);
// If a slot is multiple pages in size, and the allocation takes up a single
// page, we can improve overflow detection by leaving the unused pages as
// unmapped.
- markReadWrite(reinterpret_cast<void *>(getPageAddr(Ptr, State.PageSize)),
- Size, kGwpAsanAliveSlotName);
+ const size_t PageSize = State.PageSize;
+ allocateInGuardedPool(
+ reinterpret_cast<void *>(getPageAddr(UserPtr, PageSize)),
+ roundUpTo(Size, PageSize));
- Meta->RecordAllocation(Ptr, Size);
- Meta->AllocationTrace.RecordBacktrace(Backtrace);
+ Meta->RecordAllocation(UserPtr, Size);
+ {
+ ScopedLock UL(BacktraceMutex);
+ Meta->AllocationTrace.RecordBacktrace(Backtrace);
+ }
- return reinterpret_cast<void *>(Ptr);
+ return reinterpret_cast<void *>(UserPtr);
}
void GuardedPoolAllocator::trapOnAddress(uintptr_t Address, Error E) {
// Raise a SEGV by touching first guard page.
volatile char *p = reinterpret_cast<char *>(State.GuardedPagePool);
*p = 0;
- __builtin_unreachable();
+ // Normally, would be __builtin_unreachable(), but because of
+ // https://bugs.llvm.org/show_bug.cgi?id=47480, unreachable will DCE the
+ // volatile store above, even though it has side effects.
+ __builtin_trap();
}
void GuardedPoolAllocator::stop() {
- ThreadLocals.RecursiveGuard = true;
+ getThreadLocals()->RecursiveGuard = true;
PoolMutex.tryLock();
}
// Ensure that the unwinder is not called if the recursive flag is set,
// otherwise non-reentrant unwinders may deadlock.
- if (!ThreadLocals.RecursiveGuard) {
- ScopedBoolean B(ThreadLocals.RecursiveGuard);
+ if (!getThreadLocals()->RecursiveGuard) {
+ ScopedRecursiveGuard SRG;
+ ScopedLock UL(BacktraceMutex);
Meta->DeallocationTrace.RecordBacktrace(Backtrace);
}
}
- markInaccessible(reinterpret_cast<void *>(SlotStart),
- State.maximumAllocationSize(), kGwpAsanGuardPageName);
+ deallocateInGuardedPool(reinterpret_cast<void *>(SlotStart),
+ State.maximumAllocationSize());
// And finally, lock again to release the slot back into the pool.
ScopedLock L(PoolMutex);
ScopedLock L(PoolMutex);
AllocationMetadata *Meta = addrToMetadata(reinterpret_cast<uintptr_t>(Ptr));
assert(Meta->Addr == reinterpret_cast<uintptr_t>(Ptr));
- return Meta->Size;
+ return Meta->RequestedSize;
}
AllocationMetadata *GuardedPoolAllocator::addrToMetadata(uintptr_t Ptr) const {
FreeSlots[FreeSlotsLength++] = SlotIndex;
}
-GWP_ASAN_TLS_INITIAL_EXEC
-GuardedPoolAllocator::ThreadLocalPackedVariables
- GuardedPoolAllocator::ThreadLocals;
+uint32_t GuardedPoolAllocator::getRandomUnsigned32() {
+ uint32_t RandomState = getThreadLocals()->RandomState;
+ RandomState ^= RandomState << 13;
+ RandomState ^= RandomState >> 17;
+ RandomState ^= RandomState << 5;
+ getThreadLocals()->RandomState = RandomState;
+ return RandomState;
+}
} // namespace gwp_asan
#include "gwp_asan/definitions.h"
#include "gwp_asan/mutex.h"
#include "gwp_asan/options.h"
-#include "gwp_asan/random.h"
-#include "gwp_asan/stack_trace_compressor.h"
+#include "gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h" // IWYU pragma: keep
+#include "gwp_asan/platform_specific/guarded_pool_allocator_posix.h" // IWYU pragma: keep
+#include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
#include <stddef.h>
#include <stdint.h>
+// IWYU pragma: no_include <__stddef_max_align_t.h>
namespace gwp_asan {
// This class is the primary implementation of the allocator portion of GWP-
// GWP-ASan. The constructor value-initialises the class such that if no
// further initialisation takes place, calls to shouldSample() and
// pointerIsMine() will return false.
- constexpr GuardedPoolAllocator(){};
+ constexpr GuardedPoolAllocator() {}
GuardedPoolAllocator(const GuardedPoolAllocator &) = delete;
GuardedPoolAllocator &operator=(const GuardedPoolAllocator &) = delete;
// class must be valid when zero-initialised, and we wish to sample as
// infrequently as possible when this is the case, hence we underflow to
// UINT32_MAX.
- if (GWP_ASAN_UNLIKELY(ThreadLocals.NextSampleCounter == 0))
- ThreadLocals.NextSampleCounter =
- (getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1;
+ if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0))
+ getThreadLocals()->NextSampleCounter =
+ ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) &
+ ThreadLocalPackedVariables::NextSampleCounterMask;
- return GWP_ASAN_UNLIKELY(--ThreadLocals.NextSampleCounter == 0);
+ return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0);
}
// Returns whether the provided pointer is a current sampled allocation that
return State.pointerIsMine(Ptr);
}
- // Allocate memory in a guarded slot, and return a pointer to the new
- // allocation. Returns nullptr if the pool is empty, the requested size is too
- // large for this pool to handle, or the requested size is zero.
- void *allocate(size_t Size);
+ // Allocate memory in a guarded slot, with the specified `Alignment`. Returns
+ // nullptr if the pool is empty, if the alignnment is not a power of two, or
+ // if the size/alignment makes the allocation too large for this pool to
+ // handle. By default, uses strong alignment (i.e. `max_align_t`), see
+ // http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2293.htm for discussion of
+ // alignment issues in the standard.
+ void *allocate(size_t Size, size_t Alignment = alignof(max_align_t));
// Deallocate memory in a guarded slot. The provided pointer must have been
// allocated using this pool. This will set the guarded slot as inaccessible.
// Returns a pointer to the AllocatorState region.
const AllocatorState *getAllocatorState() const { return &State; }
+ // Exposed as protected for testing.
+protected:
+ // Returns the actual allocation size required to service an allocation with
+ // the provided Size and Alignment.
+ static size_t getRequiredBackingSize(size_t Size, size_t Alignment,
+ size_t PageSize);
+
+ // Returns the provided pointer that meets the specified alignment, depending
+ // on whether it's left or right aligned.
+ static uintptr_t alignUp(uintptr_t Ptr, size_t Alignment);
+ static uintptr_t alignDown(uintptr_t Ptr, size_t Alignment);
+
private:
// Name of actively-occupied slot mappings.
static constexpr const char *kGwpAsanAliveSlotName = "GWP-ASan Alive Slot";
// memory into this process in a platform-specific way. Pointer and size
// arguments are expected to be page-aligned. These functions will never
// return on error, instead electing to kill the calling process on failure.
- // Note that memory is initially mapped inaccessible. In order for RW
- // mappings, call mapMemory() followed by markReadWrite() on the returned
- // pointer. Each mapping is named on platforms that support it, primarily
- // Android. This name must be a statically allocated string, as the Android
- // kernel uses the string pointer directly.
- void *mapMemory(size_t Size, const char *Name) const;
- void unmapMemory(void *Ptr, size_t Size, const char *Name) const;
- void markReadWrite(void *Ptr, size_t Size, const char *Name) const;
- void markInaccessible(void *Ptr, size_t Size, const char *Name) const;
+ // The pool memory is initially reserved and inaccessible, and RW mappings are
+ // subsequently created and destroyed via allocateInGuardedPool() and
+ // deallocateInGuardedPool(). Each mapping is named on platforms that support
+ // it, primarily Android. This name must be a statically allocated string, as
+ // the Android kernel uses the string pointer directly.
+ void *map(size_t Size, const char *Name) const;
+ void unmap(void *Ptr, size_t Size) const;
+
+ // The pool is managed separately, as some platforms (particularly Fuchsia)
+ // manage virtual memory regions as a chunk where individual pages can still
+ // have separate permissions. These platforms maintain metadata about the
+ // region in order to perform operations. The pool is unique as it's the only
+ // thing in GWP-ASan that treats pages in a single VM region on an individual
+ // basis for page protection.
+ // The pointer returned by reserveGuardedPool() is the reserved address range
+ // of (at least) Size bytes.
+ void *reserveGuardedPool(size_t Size);
+ // allocateInGuardedPool() Ptr and Size must be a subrange of the previously
+ // reserved pool range.
+ void allocateInGuardedPool(void *Ptr, size_t Size) const;
+ // deallocateInGuardedPool() Ptr and Size must be an exact pair previously
+ // passed to allocateInGuardedPool().
+ void deallocateInGuardedPool(void *Ptr, size_t Size) const;
+ void unreserveGuardedPool();
// Get the page size from the platform-specific implementation. Only needs to
// be called once, and the result should be cached in PageSize in this class.
// A mutex to protect the guarded slot and metadata pool for this class.
Mutex PoolMutex;
+ // Some unwinders can grab the libdl lock. In order to provide atfork
+ // protection, we need to ensure that we allow an unwinding thread to release
+ // the libdl lock before forking.
+ Mutex BacktraceMutex;
// Record the number allocations that we've sampled. We store this amount so
// that we don't randomly choose to recycle a slot that previously had an
// allocation before all the slots have been utilised.
// the sample rate.
uint32_t AdjustedSampleRatePlusOne = 0;
- // Pack the thread local variables into a struct to ensure that they're in
- // the same cache line for performance reasons. These are the most touched
- // variables in GWP-ASan.
- struct alignas(8) ThreadLocalPackedVariables {
- constexpr ThreadLocalPackedVariables() {}
- // Thread-local decrementing counter that indicates that a given allocation
- // should be sampled when it reaches zero.
- uint32_t NextSampleCounter = 0;
- // Guard against recursivity. Unwinders often contain complex behaviour that
- // may not be safe for the allocator (i.e. the unwinder calls dlopen(),
- // which calls malloc()). When recursive behaviour is detected, we will
- // automatically fall back to the supporting allocator to supply the
- // allocation.
- bool RecursiveGuard = false;
+ // Additional platform specific data structure for the guarded pool mapping.
+ PlatformSpecificMapData GuardedPagePoolPlatformData = {};
+
+ class ScopedRecursiveGuard {
+ public:
+ ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = true; }
+ ~ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = false; }
};
- static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables ThreadLocals;
+
+ // Initialise the PRNG, platform-specific.
+ void initPRNG();
+
+ // xorshift (32-bit output), extremely fast PRNG that uses arithmetic
+ // operations only. Seeded using platform-specific mechanisms by initPRNG().
+ uint32_t getRandomUnsigned32();
};
} // namespace gwp_asan
#ifndef GWP_ASAN_MUTEX_H_
#define GWP_ASAN_MUTEX_H_
-#ifdef __unix__
-#include <pthread.h>
-#else
-#error "GWP-ASan is not supported on this platform."
-#endif
+#include "gwp_asan/platform_specific/mutex_fuchsia.h" // IWYU pragma: keep
+#include "gwp_asan/platform_specific/mutex_posix.h" // IWYU pragma: keep
namespace gwp_asan {
-class Mutex {
+class Mutex final : PlatformMutex {
public:
constexpr Mutex() = default;
~Mutex() = default;
bool tryLock();
// Unlock the mutex.
void unlock();
-
-private:
-#ifdef __unix__
- pthread_mutex_t Mu = PTHREAD_MUTEX_INITIALIZER;
-#endif // defined(__unix__)
};
class ScopedLock {
#ifndef GWP_ASAN_OPTIONAL_BACKTRACE_H_
#define GWP_ASAN_OPTIONAL_BACKTRACE_H_
-#include "gwp_asan/optional/segv_handler.h"
+#include "gwp_asan/optional/printf.h"
#include "gwp_asan/options.h"
namespace gwp_asan {
-namespace options {
-// Functions to get the platform-specific and implementation-specific backtrace
-// and backtrace printing functions when RTGwpAsanBacktraceLibc or
-// RTGwpAsanBacktraceSanitizerCommon are linked. Use these functions to get the
-// backtrace function for populating the Options::Backtrace and
-// Options::PrintBacktrace when initialising the GuardedPoolAllocator. Please
-// note any thread-safety descriptions for the implementation of these functions
-// that you use.
-Backtrace_t getBacktraceFunction();
-crash_handler::PrintBacktrace_t getPrintBacktraceFunction();
-} // namespace options
+namespace backtrace {
+// ================================ Description ================================
+// This function shall take the backtrace provided in `TraceBuffer`, and print
+// it in a human-readable format using `Print`. Generally, this function shall
+// resolve raw pointers to section offsets and print them with the following
+// sanitizer-common format:
+// " #{frame_number} {pointer} in {function name} ({binary name}+{offset}"
+// e.g. " #5 0x420459 in _start (/tmp/uaf+0x420459)"
+// This format allows the backtrace to be symbolized offline successfully using
+// llvm-symbolizer.
+// =================================== Notes ===================================
+// This function may directly or indirectly call malloc(), as the
+// GuardedPoolAllocator contains a reentrancy barrier to prevent infinite
+// recursion. Any allocation made inside this function will be served by the
+// supporting allocator, and will not have GWP-ASan protections.
+typedef void (*PrintBacktrace_t)(uintptr_t *TraceBuffer, size_t TraceLength,
+ Printf_t Print);
+
+// Returns a function pointer to a backtrace function that's suitable for
+// unwinding through a signal handler. This is important primarily for frame-
+// pointer based unwinders, DWARF or other unwinders can simply provide the
+// normal backtrace function as the implementation here. On POSIX, SignalContext
+// should be the `ucontext_t` from the signal handler.
+typedef size_t (*SegvBacktrace_t)(uintptr_t *TraceBuffer, size_t Size,
+ void *SignalContext);
+
+// Returns platform-specific provided implementations of Backtrace_t for use
+// inside the GWP-ASan core allocator.
+options::Backtrace_t getBacktraceFunction();
+
+// Returns platform-specific provided implementations of PrintBacktrace_t and
+// SegvBacktrace_t for use in the optional SEGV handler.
+PrintBacktrace_t getPrintBacktraceFunction();
+SegvBacktrace_t getSegvBacktraceFunction();
+} // namespace backtrace
} // namespace gwp_asan
#endif // GWP_ASAN_OPTIONAL_BACKTRACE_H_
--- /dev/null
+//===-- backtrace_fuchsia.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/optional/backtrace.h"
+
+#include <zircon/sanitizer.h>
+
+namespace gwp_asan {
+namespace backtrace {
+
+// Fuchsia's C library provides safe, fast, best-effort backtraces itself.
+options::Backtrace_t getBacktraceFunction() {
+ return __sanitizer_fast_backtrace;
+}
+
+// These are only used in fatal signal handling, which is not used on Fuchsia.
+
+PrintBacktrace_t getPrintBacktraceFunction() { return nullptr; }
+SegvBacktrace_t getSegvBacktraceFunction() { return nullptr; }
+
+} // namespace backtrace
+} // namespace gwp_asan
#include <stdlib.h>
#include <string.h>
+#include "gwp_asan/definitions.h"
#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/optional/printf.h"
#include "gwp_asan/options.h"
namespace {
return backtrace(reinterpret_cast<void **>(TraceBuffer), Size);
}
+// We don't need any custom handling for the Segv backtrace - the libc unwinder
+// has no problems with unwinding through a signal handler. Force inlining here
+// to avoid the additional frame.
+GWP_ASAN_ALWAYS_INLINE size_t SegvBacktrace(uintptr_t *TraceBuffer, size_t Size,
+ void * /*Context*/) {
+ return Backtrace(TraceBuffer, Size);
+}
+
static void PrintBacktrace(uintptr_t *Trace, size_t TraceLength,
- gwp_asan::crash_handler::Printf_t Printf) {
+ gwp_asan::Printf_t Printf) {
if (TraceLength == 0) {
Printf(" <not found (does your allocator support backtracing?)>\n\n");
return;
} // anonymous namespace
namespace gwp_asan {
-namespace options {
-Backtrace_t getBacktraceFunction() { return Backtrace; }
-crash_handler::PrintBacktrace_t getPrintBacktraceFunction() {
- return PrintBacktrace;
-}
-} // namespace options
+namespace backtrace {
+
+options::Backtrace_t getBacktraceFunction() { return Backtrace; }
+PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; }
+SegvBacktrace_t getSegvBacktraceFunction() { return SegvBacktrace; }
+
+} // namespace backtrace
} // namespace gwp_asan
void *context,
bool request_fast,
u32 max_depth) {
- if (!StackTrace::WillUseFastUnwind(request_fast)) {
- return Unwind(max_depth, pc, bp, context, 0, 0, request_fast);
- }
- Unwind(max_depth, pc, 0, context, 0, 0, false);
+ if (!StackTrace::WillUseFastUnwind(request_fast))
+ return Unwind(max_depth, pc, 0, context, 0, 0, false);
+
+ uptr top = 0;
+ uptr bottom = 0;
+ GetThreadStackTopAndBottom(/*at_initialization*/ false, &top, &bottom);
+
+ return Unwind(max_depth, pc, bp, context, top, bottom, request_fast);
}
namespace {
-size_t Backtrace(uintptr_t *TraceBuffer, size_t Size) {
+size_t BacktraceCommon(uintptr_t *TraceBuffer, size_t Size, void *Context) {
+ // Use the slow sanitizer unwinder in the segv handler. Fast frame pointer
+ // unwinders can end up dropping frames because the kernel sigreturn() frame's
+ // return address is the return address at time of fault. This has the result
+ // of never actually capturing the PC where the signal was raised.
+ bool UseFastUnwind = (Context == nullptr);
+
__sanitizer::BufferedStackTrace Trace;
Trace.Reset();
if (Size > __sanitizer::kStackTraceMax)
Size = __sanitizer::kStackTraceMax;
Trace.Unwind((__sanitizer::uptr)__builtin_return_address(0),
- (__sanitizer::uptr)__builtin_frame_address(0),
- /* ucontext */ nullptr,
- /* fast unwind */ true, Size - 1);
+ (__sanitizer::uptr)__builtin_frame_address(0), Context,
+ UseFastUnwind, Size - 1);
memcpy(TraceBuffer, Trace.trace, Trace.size * sizeof(uintptr_t));
return Trace.size;
}
+size_t Backtrace(uintptr_t *TraceBuffer, size_t Size) {
+ return BacktraceCommon(TraceBuffer, Size, nullptr);
+}
+
+size_t SegvBacktrace(uintptr_t *TraceBuffer, size_t Size, void *Context) {
+ return BacktraceCommon(TraceBuffer, Size, Context);
+}
+
static void PrintBacktrace(uintptr_t *Trace, size_t TraceLength,
- gwp_asan::crash_handler::Printf_t Printf) {
+ gwp_asan::Printf_t Printf) {
__sanitizer::StackTrace StackTrace;
StackTrace.trace = reinterpret_cast<__sanitizer::uptr *>(Trace);
StackTrace.size = TraceLength;
} // anonymous namespace
namespace gwp_asan {
-namespace options {
+namespace backtrace {
+
// This function is thread-compatible. It must be synchronised in respect to any
// other calls to getBacktraceFunction(), calls to getPrintBacktraceFunction(),
// and calls to either of the functions that they return. Furthermore, this may
// require synchronisation with any calls to sanitizer_common that use flags.
// Generally, this function will be called during the initialisation of the
// allocator, which is done in a thread-compatible manner.
-Backtrace_t getBacktraceFunction() {
+options::Backtrace_t getBacktraceFunction() {
// The unwinder requires the default flags to be set.
__sanitizer::SetCommonFlagsDefaults();
__sanitizer::InitializeCommonFlags();
return Backtrace;
}
-crash_handler::PrintBacktrace_t getPrintBacktraceFunction() {
- return PrintBacktrace;
-}
-} // namespace options
+
+PrintBacktrace_t getPrintBacktraceFunction() { return PrintBacktrace; }
+SegvBacktrace_t getSegvBacktraceFunction() { return SegvBacktrace; }
+
+} // namespace backtrace
} // namespace gwp_asan
//===----------------------------------------------------------------------===//
#include "gwp_asan/optional/options_parser.h"
+#include "gwp_asan/optional/printf.h"
+#include "gwp_asan/utilities.h"
+#include <assert.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
-#include "gwp_asan/options.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_flag_parser.h"
-#include "sanitizer_common/sanitizer_flags.h"
-
-namespace gwp_asan {
-namespace options {
namespace {
-void registerGwpAsanFlags(__sanitizer::FlagParser *parser, Options *o) {
-#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \
- RegisterFlag(parser, #Name, Description, &o->Name);
+enum class OptionType : uint8_t {
+ OT_bool,
+ OT_int,
+};
+
+#define InvokeIfNonNull(Printf, ...) \
+ do { \
+ if (Printf) \
+ Printf(__VA_ARGS__); \
+ } while (0);
+
+class OptionParser {
+public:
+ explicit OptionParser(gwp_asan::Printf_t PrintfForWarnings)
+ : Printf(PrintfForWarnings) {}
+ void registerOption(const char *Name, const char *Desc, OptionType Type,
+ void *Var);
+ void parseString(const char *S);
+ void printOptionDescriptions();
+
+private:
+ // Calculate at compile-time how many options are available.
+#define GWP_ASAN_OPTION(...) +1
+ static constexpr size_t MaxOptions = 0
#include "gwp_asan/options.inc"
+ ;
#undef GWP_ASAN_OPTION
+
+ struct Option {
+ const char *Name;
+ const char *Desc;
+ OptionType Type;
+ void *Var;
+ } Options[MaxOptions];
+
+ size_t NumberOfOptions = 0;
+ const char *Buffer = nullptr;
+ uintptr_t Pos = 0;
+ gwp_asan::Printf_t Printf = nullptr;
+
+ void skipWhitespace();
+ void parseOptions();
+ bool parseOption();
+ bool setOptionToValue(const char *Name, const char *Value);
+};
+
+void OptionParser::printOptionDescriptions() {
+ InvokeIfNonNull(Printf, "GWP-ASan: Available options:\n");
+ for (size_t I = 0; I < NumberOfOptions; ++I)
+ InvokeIfNonNull(Printf, "\t%s\n\t\t- %s\n", Options[I].Name,
+ Options[I].Desc);
+}
+
+bool isSeparator(char C) {
+ return C == ' ' || C == ',' || C == ':' || C == '\n' || C == '\t' ||
+ C == '\r';
+}
+
+bool isSeparatorOrNull(char C) { return !C || isSeparator(C); }
+
+void OptionParser::skipWhitespace() {
+ while (isSeparator(Buffer[Pos]))
+ ++Pos;
+}
+
+bool OptionParser::parseOption() {
+ const uintptr_t NameStart = Pos;
+ while (Buffer[Pos] != '=' && !isSeparatorOrNull(Buffer[Pos]))
+ ++Pos;
+
+ const char *Name = Buffer + NameStart;
+ if (Buffer[Pos] != '=') {
+ InvokeIfNonNull(Printf, "GWP-ASan: Expected '=' when parsing option '%s'.",
+ Name);
+ return false;
+ }
+ const uintptr_t ValueStart = ++Pos;
+ const char *Value;
+ if (Buffer[Pos] == '\'' || Buffer[Pos] == '"') {
+ const char Quote = Buffer[Pos++];
+ while (Buffer[Pos] != 0 && Buffer[Pos] != Quote)
+ ++Pos;
+ if (Buffer[Pos] == 0) {
+ InvokeIfNonNull(Printf, "GWP-ASan: Unterminated string in option '%s'.",
+ Name);
+ return false;
+ }
+ Value = Buffer + ValueStart + 1;
+ ++Pos; // consume the closing quote
+ } else {
+ while (!isSeparatorOrNull(Buffer[Pos]))
+ ++Pos;
+ Value = Buffer + ValueStart;
+ }
+
+ return setOptionToValue(Name, Value);
}
-const char *getCompileDefinitionGwpAsanDefaultOptions() {
-#ifdef GWP_ASAN_DEFAULT_OPTIONS
- return SANITIZER_STRINGIFY(GWP_ASAN_DEFAULT_OPTIONS);
-#else
- return "";
-#endif
+void OptionParser::parseOptions() {
+ while (true) {
+ skipWhitespace();
+ if (Buffer[Pos] == 0)
+ break;
+ if (!parseOption()) {
+ InvokeIfNonNull(Printf, "GWP-ASan: Options parsing failed.\n");
+ return;
+ }
+ }
+}
+
+void OptionParser::parseString(const char *S) {
+ if (!S)
+ return;
+ Buffer = S;
+ Pos = 0;
+ parseOptions();
+}
+
+bool parseBool(const char *Value, bool *b) {
+ if (strncmp(Value, "0", 1) == 0 || strncmp(Value, "no", 2) == 0 ||
+ strncmp(Value, "false", 5) == 0) {
+ *b = false;
+ return true;
+ }
+ if (strncmp(Value, "1", 1) == 0 || strncmp(Value, "yes", 3) == 0 ||
+ strncmp(Value, "true", 4) == 0) {
+ *b = true;
+ return true;
+ }
+ return false;
+}
+
+bool OptionParser::setOptionToValue(const char *Name, const char *Value) {
+ for (size_t I = 0; I < NumberOfOptions; ++I) {
+ const uintptr_t Len = strlen(Options[I].Name);
+ if (strncmp(Name, Options[I].Name, Len) != 0 || Name[Len] != '=')
+ continue;
+ bool Ok = false;
+ switch (Options[I].Type) {
+ case OptionType::OT_bool:
+ Ok = parseBool(Value, reinterpret_cast<bool *>(Options[I].Var));
+ if (!Ok)
+ InvokeIfNonNull(
+ Printf, "GWP-ASan: Invalid boolean value '%s' for option '%s'.\n",
+ Value, Options[I].Name);
+ break;
+ case OptionType::OT_int:
+ char *ValueEnd;
+ *reinterpret_cast<int *>(Options[I].Var) =
+ static_cast<int>(strtol(Value, &ValueEnd, 10));
+ Ok =
+ *ValueEnd == '"' || *ValueEnd == '\'' || isSeparatorOrNull(*ValueEnd);
+ if (!Ok)
+ InvokeIfNonNull(
+ Printf, "GWP-ASan: Invalid integer value '%s' for option '%s'.\n",
+ Value, Options[I].Name);
+ break;
+ }
+ return Ok;
+ }
+
+ InvokeIfNonNull(Printf, "GWP-ASan: Unknown option '%s'.", Name);
+ return true;
+}
+
+void OptionParser::registerOption(const char *Name, const char *Desc,
+ OptionType Type, void *Var) {
+ assert(NumberOfOptions < MaxOptions &&
+ "GWP-ASan Error: Ran out of space for options.\n");
+ Options[NumberOfOptions].Name = Name;
+ Options[NumberOfOptions].Desc = Desc;
+ Options[NumberOfOptions].Type = Type;
+ Options[NumberOfOptions].Var = Var;
+ ++NumberOfOptions;
+}
+
+void registerGwpAsanOptions(OptionParser *parser,
+ gwp_asan::options::Options *o) {
+#define GWP_ASAN_OPTION(Type, Name, DefaultValue, Description) \
+ parser->registerOption(#Name, Description, OptionType::OT_##Type, &o->Name);
+#include "gwp_asan/options.inc"
+#undef GWP_ASAN_OPTION
}
const char *getGwpAsanDefaultOptions() {
return (__gwp_asan_default_options) ? __gwp_asan_default_options() : "";
}
-Options *getOptionsInternal() {
- static Options GwpAsanFlags;
- return &GwpAsanFlags;
+gwp_asan::options::Options *getOptionsInternal() {
+ static gwp_asan::options::Options GwpAsanOptions;
+ return &GwpAsanOptions;
}
} // anonymous namespace
-void initOptions() {
- __sanitizer::SetCommonFlagsDefaults();
+namespace gwp_asan {
+namespace options {
+void initOptions(const char *OptionsStr, Printf_t PrintfForWarnings) {
Options *o = getOptionsInternal();
o->setDefaults();
- __sanitizer::FlagParser Parser;
- registerGwpAsanFlags(&Parser, o);
-
- // Override from compile definition.
- Parser.ParseString(getCompileDefinitionGwpAsanDefaultOptions());
+ OptionParser Parser(PrintfForWarnings);
+ registerGwpAsanOptions(&Parser, o);
- // Override from user-specified string.
- Parser.ParseString(getGwpAsanDefaultOptions());
+ // Override from the weak function definition in this executable.
+ Parser.parseString(getGwpAsanDefaultOptions());
- // Override from environment.
- Parser.ParseString(__sanitizer::GetEnv("GWP_ASAN_OPTIONS"));
+ // Override from the provided options string.
+ Parser.parseString(OptionsStr);
- __sanitizer::InitializeCommonFlags();
- if (__sanitizer::Verbosity())
- __sanitizer::ReportUnrecognizedFlags();
+ if (o->help)
+ Parser.printOptionDescriptions();
if (!o->Enabled)
return;
- // Sanity checks for the parameters.
if (o->MaxSimultaneousAllocations <= 0) {
- __sanitizer::Printf("GWP-ASan ERROR: MaxSimultaneousAllocations must be > "
- "0 when GWP-ASan is enabled.\n");
- exit(EXIT_FAILURE);
+ InvokeIfNonNull(
+ PrintfForWarnings,
+ "GWP-ASan ERROR: MaxSimultaneousAllocations must be > 0 when GWP-ASan "
+ "is enabled.\n");
+ o->Enabled = false;
}
-
- if (o->SampleRate < 1) {
- __sanitizer::Printf(
+ if (o->SampleRate <= 0) {
+ InvokeIfNonNull(
+ PrintfForWarnings,
"GWP-ASan ERROR: SampleRate must be > 0 when GWP-ASan is enabled.\n");
- exit(EXIT_FAILURE);
+ o->Enabled = false;
}
}
+void initOptions(Printf_t PrintfForWarnings) {
+ initOptions(getenv("GWP_ASAN_OPTIONS"), PrintfForWarnings);
+}
+
Options &getOptions() { return *getOptionsInternal(); }
} // namespace options
#ifndef GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_
#define GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_
-#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/optional/printf.h"
#include "gwp_asan/options.h"
-#include "sanitizer_common/sanitizer_common.h"
namespace gwp_asan {
namespace options {
-// Parse the options from the GWP_ASAN_FLAGS environment variable.
-void initOptions();
+// Parse the options from the GWP_ASAN_OPTIONS environment variable.
+void initOptions(Printf_t PrintfForWarnings = nullptr);
+// Parse the options from the provided string.
+void initOptions(const char *OptionsStr, Printf_t PrintfForWarnings = nullptr);
// Returns the initialised options. Call initOptions() prior to calling this
// function.
Options &getOptions();
} // namespace gwp_asan
extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
-__gwp_asan_default_options();
+__attribute__((weak)) const char *__gwp_asan_default_options();
}
#endif // GWP_ASAN_OPTIONAL_OPTIONS_PARSER_H_
--- /dev/null
+//===-- printf.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef GWP_ASAN_OPTIONAL_PRINTF_H_
+#define GWP_ASAN_OPTIONAL_PRINTF_H_
+
+namespace gwp_asan {
+
+// ================================ Requirements ===============================
+// This function is required to be provided by the supporting allocator iff the
+// allocator wants to use any of the optional components.
+// ================================ Description ================================
+// This function shall produce output according to a strict subset of the C
+// standard library's printf() family. This function must support printing the
+// following formats:
+// 1. integers: "%([0-9]*)?(z|ll)?{d,u,x,X}"
+// 2. pointers: "%p"
+// 3. strings: "%[-]([0-9]*)?(\\.\\*)?s"
+// 4. chars: "%c"
+// This function must be implemented in a signal-safe manner, and thus must not
+// malloc().
+// =================================== Notes ===================================
+// This function has a slightly different signature than the C standard
+// library's printf(). Notably, it returns 'void' rather than 'int'.
+typedef void (*Printf_t)(const char *Format, ...);
+
+} // namespace gwp_asan
+#endif // GWP_ASAN_OPTIONAL_PRINTF_H_
-//===-- crash_handler.h -----------------------------------------*- C++ -*-===//
+//===-- segv_handler.h ------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
-#ifndef GWP_ASAN_OPTIONAL_CRASH_HANDLER_H_
-#define GWP_ASAN_OPTIONAL_CRASH_HANDLER_H_
+#ifndef GWP_ASAN_OPTIONAL_SEGV_HANDLER_H_
+#define GWP_ASAN_OPTIONAL_SEGV_HANDLER_H_
#include "gwp_asan/guarded_pool_allocator.h"
-#include "gwp_asan/options.h"
+#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/optional/printf.h"
namespace gwp_asan {
-namespace crash_handler {
-// ================================ Requirements ===============================
-// This function must be provided by the supporting allocator only when this
-// provided crash handler is used to dump the generic report.
-// sanitizer::Printf() function can be simply used here.
-// ================================ Description ================================
-// This function shall produce output according to a strict subset of the C
-// standard library's printf() family. This function must support printing the
-// following formats:
-// 1. integers: "%([0-9]*)?(z|ll)?{d,u,x,X}"
-// 2. pointers: "%p"
-// 3. strings: "%[-]([0-9]*)?(\\.\\*)?s"
-// 4. chars: "%c"
-// This function must be implemented in a signal-safe manner, and thus must not
-// malloc().
-// =================================== Notes ===================================
-// This function has a slightly different signature than the C standard
-// library's printf(). Notably, it returns 'void' rather than 'int'.
-typedef void (*Printf_t)(const char *Format, ...);
-
-// ================================ Requirements ===============================
-// This function is required for the supporting allocator, but one of the three
-// provided implementations may be used (RTGwpAsanBacktraceLibc,
-// RTGwpAsanBacktraceSanitizerCommon, or BasicPrintBacktraceFunction).
-// ================================ Description ================================
-// This function shall take the backtrace provided in `TraceBuffer`, and print
-// it in a human-readable format using `Print`. Generally, this function shall
-// resolve raw pointers to section offsets and print them with the following
-// sanitizer-common format:
-// " #{frame_number} {pointer} in {function name} ({binary name}+{offset}"
-// e.g. " #5 0x420459 in _start (/tmp/uaf+0x420459)"
-// This format allows the backtrace to be symbolized offline successfully using
-// llvm-symbolizer.
-// =================================== Notes ===================================
-// This function may directly or indirectly call malloc(), as the
-// GuardedPoolAllocator contains a reentrancy barrier to prevent infinite
-// recursion. Any allocation made inside this function will be served by the
-// supporting allocator, and will not have GWP-ASan protections.
-typedef void (*PrintBacktrace_t)(uintptr_t *TraceBuffer, size_t TraceLength,
- Printf_t Print);
-
-// Returns a function pointer to a basic PrintBacktrace implementation. This
-// implementation simply prints the stack trace in a human readable fashion
-// without any symbolization.
-PrintBacktrace_t getBasicPrintBacktraceFunction();
-
+namespace segv_handler {
// Install the SIGSEGV crash handler for printing use-after-free and heap-
// buffer-{under|over}flow exceptions if the user asked for it. This is platform
// specific as even though POSIX and Windows both support registering handlers
// the address that caused the SIGSEGV exception. GPA->init() must be called
// before this function.
void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
- PrintBacktrace_t PrintBacktrace,
- options::Backtrace_t Backtrace);
+ gwp_asan::backtrace::PrintBacktrace_t PrintBacktrace,
+ gwp_asan::backtrace::SegvBacktrace_t SegvBacktrace);
+// Uninistall the signal handlers, test-only.
void uninstallSignalHandlers();
-
-void dumpReport(uintptr_t ErrorPtr, const gwp_asan::AllocatorState *State,
- const gwp_asan::AllocationMetadata *Metadata,
- options::Backtrace_t Backtrace, Printf_t Printf,
- PrintBacktrace_t PrintBacktrace);
-} // namespace crash_handler
+} // namespace segv_handler
} // namespace gwp_asan
-#endif // GWP_ASAN_OPTIONAL_CRASH_HANDLER_H_
+#endif // GWP_ASAN_OPTIONAL_SEGV_HANDLER_H_
--- /dev/null
+//===-- segv_handler_fuchsia.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/optional/segv_handler.h"
+
+// GWP-ASan on Fuchsia doesn't currently support signal handlers.
+
+namespace gwp_asan {
+namespace segv_handler {
+void installSignalHandlers(gwp_asan::GuardedPoolAllocator * /* GPA */,
+ Printf_t /* Printf */,
+ backtrace::PrintBacktrace_t /* PrintBacktrace */,
+ backtrace::SegvBacktrace_t /* SegvBacktrace */) {}
+
+void uninstallSignalHandlers() {}
+} // namespace segv_handler
+} // namespace gwp_asan
-//===-- crash_handler_posix.cpp ---------------------------------*- C++ -*-===//
+//===-- segv_handler_posix.cpp ----------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "gwp_asan/optional/segv_handler.h"
#include "gwp_asan/options.h"
+// RHEL creates the PRIu64 format macro (for printing uint64_t's) only when this
+// macro is defined before including <inttypes.h>.
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS 1
+#endif
+
#include <assert.h>
#include <inttypes.h>
#include <signal.h>
#include <stdio.h>
-namespace {
using gwp_asan::AllocationMetadata;
using gwp_asan::Error;
using gwp_asan::GuardedPoolAllocator;
-using gwp_asan::crash_handler::PrintBacktrace_t;
-using gwp_asan::crash_handler::Printf_t;
-using gwp_asan::options::Backtrace_t;
-
-struct sigaction PreviousHandler;
-bool SignalHandlerInstalled;
-gwp_asan::GuardedPoolAllocator *GPAForSignalHandler;
-Printf_t PrintfForSignalHandler;
-PrintBacktrace_t PrintBacktraceForSignalHandler;
-Backtrace_t BacktraceForSignalHandler;
+using gwp_asan::Printf_t;
+using gwp_asan::backtrace::PrintBacktrace_t;
+using gwp_asan::backtrace::SegvBacktrace_t;
-static void sigSegvHandler(int sig, siginfo_t *info, void *ucontext) {
- if (GPAForSignalHandler) {
- GPAForSignalHandler->stop();
-
- gwp_asan::crash_handler::dumpReport(
- reinterpret_cast<uintptr_t>(info->si_addr),
- GPAForSignalHandler->getAllocatorState(),
- GPAForSignalHandler->getMetadataRegion(), BacktraceForSignalHandler,
- PrintfForSignalHandler, PrintBacktraceForSignalHandler);
- }
-
- // Process any previous handlers.
- if (PreviousHandler.sa_flags & SA_SIGINFO) {
- PreviousHandler.sa_sigaction(sig, info, ucontext);
- } else if (PreviousHandler.sa_handler == SIG_DFL) {
- // If the previous handler was the default handler, cause a core dump.
- signal(SIGSEGV, SIG_DFL);
- raise(SIGSEGV);
- } else if (PreviousHandler.sa_handler == SIG_IGN) {
- // If the previous segv handler was SIGIGN, crash iff we were responsible
- // for the crash.
- if (__gwp_asan_error_is_mine(GPAForSignalHandler->getAllocatorState(),
- reinterpret_cast<uintptr_t>(info->si_addr))) {
- signal(SIGSEGV, SIG_DFL);
- raise(SIGSEGV);
- }
- } else {
- PreviousHandler.sa_handler(sig);
- }
-}
+namespace {
struct ScopedEndOfReportDecorator {
- ScopedEndOfReportDecorator(gwp_asan::crash_handler::Printf_t Printf)
- : Printf(Printf) {}
+ ScopedEndOfReportDecorator(gwp_asan::Printf_t Printf) : Printf(Printf) {}
~ScopedEndOfReportDecorator() { Printf("*** End GWP-ASan report ***\n"); }
- gwp_asan::crash_handler::Printf_t Printf;
+ gwp_asan::Printf_t Printf;
};
// Prints the provided error and metadata information.
AccessPtr, DescriptionBuffer, ThreadBuffer);
}
-void defaultPrintStackTrace(uintptr_t *Trace, size_t TraceLength,
- gwp_asan::crash_handler::Printf_t Printf) {
- if (TraceLength == 0)
- Printf(" <unknown (does your allocator support backtracing?)>\n");
-
- for (size_t i = 0; i < TraceLength; ++i) {
- Printf(" #%zu 0x%zx in <unknown>\n", i, Trace[i]);
- }
- Printf("\n");
-}
-
-} // anonymous namespace
-
-namespace gwp_asan {
-namespace crash_handler {
-PrintBacktrace_t getBasicPrintBacktraceFunction() {
- return defaultPrintStackTrace;
-}
-
-void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
- PrintBacktrace_t PrintBacktrace,
- options::Backtrace_t Backtrace) {
- GPAForSignalHandler = GPA;
- PrintfForSignalHandler = Printf;
- PrintBacktraceForSignalHandler = PrintBacktrace;
- BacktraceForSignalHandler = Backtrace;
-
- struct sigaction Action;
- Action.sa_sigaction = sigSegvHandler;
- Action.sa_flags = SA_SIGINFO;
- sigaction(SIGSEGV, &Action, &PreviousHandler);
- SignalHandlerInstalled = true;
-}
-
-void uninstallSignalHandlers() {
- if (SignalHandlerInstalled) {
- sigaction(SIGSEGV, &PreviousHandler, nullptr);
- SignalHandlerInstalled = false;
- }
-}
-
void dumpReport(uintptr_t ErrorPtr, const gwp_asan::AllocatorState *State,
const gwp_asan::AllocationMetadata *Metadata,
- options::Backtrace_t Backtrace, Printf_t Printf,
- PrintBacktrace_t PrintBacktrace) {
+ SegvBacktrace_t SegvBacktrace, Printf_t Printf,
+ PrintBacktrace_t PrintBacktrace, void *Context) {
assert(State && "dumpReport missing Allocator State.");
assert(Metadata && "dumpReport missing Metadata.");
assert(Printf && "dumpReport missing Printf.");
// Print the fault backtrace.
static constexpr unsigned kMaximumStackFramesForCrashTrace = 512;
uintptr_t Trace[kMaximumStackFramesForCrashTrace];
- size_t TraceLength = Backtrace(Trace, kMaximumStackFramesForCrashTrace);
+ size_t TraceLength =
+ SegvBacktrace(Trace, kMaximumStackFramesForCrashTrace, Context);
PrintBacktrace(Trace, TraceLength, Printf);
// Maybe print the deallocation trace.
if (__gwp_asan_is_deallocated(AllocMeta)) {
uint64_t ThreadID = __gwp_asan_get_deallocation_thread_id(AllocMeta);
- if (ThreadID == kInvalidThreadID)
+ if (ThreadID == gwp_asan::kInvalidThreadID)
Printf("0x%zx was deallocated by thread <unknown> here:\n", ErrorPtr);
else
Printf("0x%zx was deallocated by thread %zu here:\n", ErrorPtr, ThreadID);
// Print the allocation trace.
uint64_t ThreadID = __gwp_asan_get_allocation_thread_id(AllocMeta);
- if (ThreadID == kInvalidThreadID)
+ if (ThreadID == gwp_asan::kInvalidThreadID)
Printf("0x%zx was allocated by thread <unknown> here:\n", ErrorPtr);
else
Printf("0x%zx was allocated by thread %zu here:\n", ErrorPtr, ThreadID);
AllocMeta, Trace, kMaximumStackFramesForCrashTrace);
PrintBacktrace(Trace, TraceLength, Printf);
}
-} // namespace crash_handler
+
+struct sigaction PreviousHandler;
+bool SignalHandlerInstalled;
+gwp_asan::GuardedPoolAllocator *GPAForSignalHandler;
+Printf_t PrintfForSignalHandler;
+PrintBacktrace_t PrintBacktraceForSignalHandler;
+SegvBacktrace_t BacktraceForSignalHandler;
+
+static void sigSegvHandler(int sig, siginfo_t *info, void *ucontext) {
+ if (GPAForSignalHandler) {
+ GPAForSignalHandler->stop();
+
+ dumpReport(reinterpret_cast<uintptr_t>(info->si_addr),
+ GPAForSignalHandler->getAllocatorState(),
+ GPAForSignalHandler->getMetadataRegion(),
+ BacktraceForSignalHandler, PrintfForSignalHandler,
+ PrintBacktraceForSignalHandler, ucontext);
+ }
+
+ // Process any previous handlers.
+ if (PreviousHandler.sa_flags & SA_SIGINFO) {
+ PreviousHandler.sa_sigaction(sig, info, ucontext);
+ } else if (PreviousHandler.sa_handler == SIG_DFL) {
+ // If the previous handler was the default handler, cause a core dump.
+ signal(SIGSEGV, SIG_DFL);
+ raise(SIGSEGV);
+ } else if (PreviousHandler.sa_handler == SIG_IGN) {
+ // If the previous segv handler was SIGIGN, crash iff we were responsible
+ // for the crash.
+ if (__gwp_asan_error_is_mine(GPAForSignalHandler->getAllocatorState(),
+ reinterpret_cast<uintptr_t>(info->si_addr))) {
+ signal(SIGSEGV, SIG_DFL);
+ raise(SIGSEGV);
+ }
+ } else {
+ PreviousHandler.sa_handler(sig);
+ }
+}
+} // anonymous namespace
+
+namespace gwp_asan {
+namespace segv_handler {
+
+void installSignalHandlers(gwp_asan::GuardedPoolAllocator *GPA, Printf_t Printf,
+ PrintBacktrace_t PrintBacktrace,
+ SegvBacktrace_t SegvBacktrace) {
+ assert(GPA && "GPA wasn't provided to installSignalHandlers.");
+ assert(Printf && "Printf wasn't provided to installSignalHandlers.");
+ assert(PrintBacktrace &&
+ "PrintBacktrace wasn't provided to installSignalHandlers.");
+ assert(SegvBacktrace &&
+ "SegvBacktrace wasn't provided to installSignalHandlers.");
+ GPAForSignalHandler = GPA;
+ PrintfForSignalHandler = Printf;
+ PrintBacktraceForSignalHandler = PrintBacktrace;
+ BacktraceForSignalHandler = SegvBacktrace;
+
+ struct sigaction Action = {};
+ Action.sa_sigaction = sigSegvHandler;
+ Action.sa_flags = SA_SIGINFO;
+ sigaction(SIGSEGV, &Action, &PreviousHandler);
+ SignalHandlerInstalled = true;
+}
+
+void uninstallSignalHandlers() {
+ if (SignalHandlerInstalled) {
+ sigaction(SIGSEGV, &PreviousHandler, nullptr);
+ SignalHandlerInstalled = false;
+ }
+}
+} // namespace segv_handler
} // namespace gwp_asan
#error "Define GWP_ASAN_OPTION prior to including this file!"
#endif
-GWP_ASAN_OPTION(bool, Enabled, true, "Is GWP-ASan enabled? Defaults to true.")
+#ifndef GWP_ASAN_DEFAULT_ENABLED
+#define GWP_ASAN_DEFAULT_ENABLED true
+#endif
-GWP_ASAN_OPTION(
- bool, PerfectlyRightAlign, false,
- "When allocations are right-aligned, should we perfectly align them up to "
- "the page boundary? By default (false), we round up allocation size to the "
- "nearest power of two (1, 2, 4, 8, 16) up to a maximum of 16-byte "
- "alignment for performance reasons. For Bionic, we use 8-byte alignment by "
- "default. Setting this to true can find single byte buffer-overflows for "
- "multibyte allocations at the cost of performance, and may be incompatible "
- "with some architectures.")
+#ifndef GWP_ASAN_STRINGIFY
+#define GWP_ASAN_STRINGIFY(S) GWP_ASAN_STRINGIFY_(S)
+#define GWP_ASAN_STRINGIFY_(S) #S
+#endif
+
+GWP_ASAN_OPTION(bool, Enabled, GWP_ASAN_DEFAULT_ENABLED,
+ "Is GWP-ASan enabled? Defaults to " GWP_ASAN_STRINGIFY(
+ GWP_ASAN_DEFAULT_ENABLED) ".")
GWP_ASAN_OPTION(int, MaxSimultaneousAllocations, 16,
"Number of simultaneously-guarded allocations available in the "
GWP_ASAN_OPTION(int, SampleRate, 5000,
"The probability (1 / SampleRate) that an allocation is "
"selected for GWP-ASan sampling. Default is 5000. Sample rates "
- "up to (2^31 - 1) are supported.")
+ "up to (2^30 - 1) are supported.")
// Developer note - This option is not actually processed by GWP-ASan itself. It
// is included here so that a user can specify whether they want signal handlers
GWP_ASAN_OPTION(bool, InstallForkHandlers, true,
"Install GWP-ASan atfork handlers to acquire internal locks "
"before fork and release them after.")
+
+GWP_ASAN_OPTION(bool, help, false, "Print a summary of the available options.")
+
+// =============================================================================
+// ==== WARNING
+// =============================================================================
+// If you are adding flags to GWP-ASan, please note that GWP-ASan flag strings
+// may be parsed by trusted system components (on Android, GWP-ASan flag strings
+// are parsed by libc during the dynamic loader). This means that GWP-ASan
+// should never feature flags like log paths on disk, because this can lead to
+// arbitrary file write and thus privilege escalation. For an example, see the
+// setuid ASan log_path exploits: https://www.exploit-db.com/exploits/46241.
+//
+// Please place all new flags above this warning, so that the warning always
+// stays at the bottom.
--- /dev/null
+//===-- common_fuchsia.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/common.h"
+
+namespace gwp_asan {
+// This is only used for AllocationTrace.ThreadID and allocation traces are not
+// yet supported on Fuchsia.
+uint64_t getThreadID() { return kInvalidThreadID; }
+} // namespace gwp_asan
-//===-- common_posix.cpp ---------------------------------*- C++ -*-===//
+//===-- common_posix.cpp ----------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
#include "gwp_asan/common.h"
-#include <sys/syscall.h>
+#include <stdint.h>
+#include <sys/syscall.h> // IWYU pragma: keep
+// IWYU pragma: no_include <syscall.h>
#include <unistd.h>
namespace gwp_asan {
--- /dev/null
+//===-- guarded_pool_allocator_fuchsia.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/guarded_pool_allocator.h"
+#include "gwp_asan/utilities.h"
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+
+namespace gwp_asan {
+void GuardedPoolAllocator::initPRNG() {
+ _zx_cprng_draw(&getThreadLocals()->RandomState, sizeof(uint32_t));
+}
+
+void *GuardedPoolAllocator::map(size_t Size, const char *Name) const {
+ assert((Size % State.PageSize) == 0);
+ zx_handle_t Vmo;
+ zx_status_t Status = _zx_vmo_create(Size, 0, &Vmo);
+ Check(Status == ZX_OK, "Failed to create Vmo");
+ _zx_object_set_property(Vmo, ZX_PROP_NAME, Name, strlen(Name));
+ zx_vaddr_t Addr;
+ Status = _zx_vmar_map(_zx_vmar_root_self(),
+ ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_ALLOW_FAULTS,
+ 0, Vmo, 0, Size, &Addr);
+ Check(Status == ZX_OK, "Vmo mapping failed");
+ _zx_handle_close(Vmo);
+ return reinterpret_cast<void *>(Addr);
+}
+
+void GuardedPoolAllocator::unmap(void *Ptr, size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
+ zx_status_t Status = _zx_vmar_unmap(_zx_vmar_root_self(),
+ reinterpret_cast<zx_vaddr_t>(Ptr), Size);
+ Check(Status == ZX_OK, "Vmo unmapping failed");
+}
+
+void *GuardedPoolAllocator::reserveGuardedPool(size_t Size) {
+ assert((Size % State.PageSize) == 0);
+ zx_vaddr_t Addr;
+ const zx_status_t Status = _zx_vmar_allocate(
+ _zx_vmar_root_self(),
+ ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0,
+ Size, &GuardedPagePoolPlatformData.Vmar, &Addr);
+ Check(Status == ZX_OK, "Failed to reserve guarded pool allocator memory");
+ _zx_object_set_property(GuardedPagePoolPlatformData.Vmar, ZX_PROP_NAME,
+ kGwpAsanGuardPageName, strlen(kGwpAsanGuardPageName));
+ return reinterpret_cast<void *>(Addr);
+}
+
+void GuardedPoolAllocator::unreserveGuardedPool() {
+ const zx_handle_t Vmar = GuardedPagePoolPlatformData.Vmar;
+ assert(Vmar != ZX_HANDLE_INVALID && Vmar != _zx_vmar_root_self());
+ Check(_zx_vmar_destroy(Vmar) == ZX_OK, "Failed to destroy a vmar");
+ Check(_zx_handle_close(Vmar) == ZX_OK, "Failed to close a vmar");
+ GuardedPagePoolPlatformData.Vmar = ZX_HANDLE_INVALID;
+}
+
+void GuardedPoolAllocator::allocateInGuardedPool(void *Ptr, size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
+ zx_handle_t Vmo;
+ zx_status_t Status = _zx_vmo_create(Size, 0, &Vmo);
+ Check(Status == ZX_OK, "Failed to create vmo");
+ _zx_object_set_property(Vmo, ZX_PROP_NAME, kGwpAsanAliveSlotName,
+ strlen(kGwpAsanAliveSlotName));
+ const zx_handle_t Vmar = GuardedPagePoolPlatformData.Vmar;
+ assert(Vmar != ZX_HANDLE_INVALID && Vmar != _zx_vmar_root_self());
+ const size_t Offset =
+ reinterpret_cast<uintptr_t>(Ptr) - State.GuardedPagePool;
+ zx_vaddr_t P;
+ Status = _zx_vmar_map(Vmar,
+ ZX_VM_PERM_READ | ZX_VM_PERM_WRITE |
+ ZX_VM_ALLOW_FAULTS | ZX_VM_SPECIFIC,
+ Offset, Vmo, 0, Size, &P);
+ Check(Status == ZX_OK, "Vmo mapping failed");
+ _zx_handle_close(Vmo);
+}
+
+void GuardedPoolAllocator::deallocateInGuardedPool(void *Ptr,
+ size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
+ const zx_handle_t Vmar = GuardedPagePoolPlatformData.Vmar;
+ assert(Vmar != ZX_HANDLE_INVALID && Vmar != _zx_vmar_root_self());
+ const zx_status_t Status =
+ _zx_vmar_unmap(Vmar, reinterpret_cast<zx_vaddr_t>(Ptr), Size);
+ Check(Status == ZX_OK, "Vmar unmapping failed");
+}
+
+size_t GuardedPoolAllocator::getPlatformPageSize() {
+ return _zx_system_get_page_size();
+}
+
+void GuardedPoolAllocator::installAtFork() {}
+} // namespace gwp_asan
--- /dev/null
+//===-- guarded_pool_allocator_fuchsia.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__Fuchsia__)
+#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_FUCHSIA_H_
+#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_FUCHSIA_H_
+
+#include <zircon/types.h>
+
+namespace gwp_asan {
+struct PlatformSpecificMapData {
+ zx_handle_t Vmar;
+};
+} // namespace gwp_asan
+
+#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_FUCHSIA_H_
+#endif // defined(__Fuchsia__)
//
//===----------------------------------------------------------------------===//
+#include "gwp_asan/common.h"
#include "gwp_asan/guarded_pool_allocator.h"
+#include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
#include "gwp_asan/utilities.h"
#include <assert.h>
-#include <errno.h>
-#include <signal.h>
+#include <pthread.h>
+#include <stdint.h>
#include <stdlib.h>
-#include <string.h>
#include <sys/mman.h>
-#include <sys/types.h>
+#include <time.h>
#include <unistd.h>
#ifdef ANDROID
#define PR_SET_VMA_ANON_NAME 0
#endif // ANDROID
+namespace {
void MaybeSetMappingName(void *Mapping, size_t Size, const char *Name) {
#ifdef ANDROID
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, Mapping, Size, Name);
// Anonymous mapping names are only supported on Android.
return;
}
+} // anonymous namespace
namespace gwp_asan {
-void *GuardedPoolAllocator::mapMemory(size_t Size, const char *Name) const {
- void *Ptr =
- mmap(nullptr, Size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+
+void GuardedPoolAllocator::initPRNG() {
+ getThreadLocals()->RandomState =
+ static_cast<uint32_t>(time(nullptr) + getThreadID());
+}
+
+void *GuardedPoolAllocator::map(size_t Size, const char *Name) const {
+ assert((Size % State.PageSize) == 0);
+ void *Ptr = mmap(nullptr, Size, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
Check(Ptr != MAP_FAILED, "Failed to map guarded pool allocator memory");
MaybeSetMappingName(Ptr, Size, Name);
return Ptr;
}
-void GuardedPoolAllocator::unmapMemory(void *Ptr, size_t Size,
- const char *Name) const {
+void GuardedPoolAllocator::unmap(void *Ptr, size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
Check(munmap(Ptr, Size) == 0,
"Failed to unmap guarded pool allocator memory.");
- MaybeSetMappingName(Ptr, Size, Name);
}
-void GuardedPoolAllocator::markReadWrite(void *Ptr, size_t Size,
- const char *Name) const {
+void *GuardedPoolAllocator::reserveGuardedPool(size_t Size) {
+ assert((Size % State.PageSize) == 0);
+ void *Ptr =
+ mmap(nullptr, Size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ Check(Ptr != MAP_FAILED, "Failed to reserve guarded pool allocator memory");
+ MaybeSetMappingName(Ptr, Size, kGwpAsanGuardPageName);
+ return Ptr;
+}
+
+void GuardedPoolAllocator::unreserveGuardedPool() {
+ unmap(reinterpret_cast<void *>(State.GuardedPagePool),
+ State.GuardedPagePoolEnd - State.GuardedPagePool);
+}
+
+void GuardedPoolAllocator::allocateInGuardedPool(void *Ptr, size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
Check(mprotect(Ptr, Size, PROT_READ | PROT_WRITE) == 0,
- "Failed to set guarded pool allocator memory at as RW.");
- MaybeSetMappingName(Ptr, Size, Name);
+ "Failed to allocate in guarded pool allocator memory");
+ MaybeSetMappingName(Ptr, Size, kGwpAsanAliveSlotName);
}
-void GuardedPoolAllocator::markInaccessible(void *Ptr, size_t Size,
- const char *Name) const {
+void GuardedPoolAllocator::deallocateInGuardedPool(void *Ptr,
+ size_t Size) const {
+ assert((reinterpret_cast<uintptr_t>(Ptr) % State.PageSize) == 0);
+ assert((Size % State.PageSize) == 0);
// mmap() a PROT_NONE page over the address to release it to the system, if
// we used mprotect() here the system would count pages in the quarantine
// against the RSS.
Check(mmap(Ptr, Size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1,
0) != MAP_FAILED,
- "Failed to set guarded pool allocator memory as inaccessible.");
- MaybeSetMappingName(Ptr, Size, Name);
+ "Failed to deallocate in guarded pool allocator memory");
+ MaybeSetMappingName(Ptr, Size, kGwpAsanGuardPageName);
}
size_t GuardedPoolAllocator::getPlatformPageSize() {
};
pthread_atfork(Disable, Enable, Enable);
}
-
} // namespace gwp_asan
--- /dev/null
+//===-- guarded_pool_allocator_posix.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__unix__)
+#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_POSIX_H_
+#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_POSIX_H_
+
+namespace gwp_asan {
+struct PlatformSpecificMapData {};
+} // namespace gwp_asan
+
+#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_POSIX_H_
+#endif // defined(__unix__)
--- /dev/null
+//===-- guarded_pool_allocator_tls.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_
+#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_
+
+#include "gwp_asan/definitions.h"
+
+#include <stdint.h>
+
+namespace gwp_asan {
+// Pack the thread local variables into a struct to ensure that they're in
+// the same cache line for performance reasons. These are the most touched
+// variables in GWP-ASan.
+struct ThreadLocalPackedVariables {
+ constexpr ThreadLocalPackedVariables()
+ : RandomState(0xacd979ce), NextSampleCounter(0), RecursiveGuard(false) {}
+ // Initialised to a magic constant so that an uninitialised GWP-ASan won't
+ // regenerate its sample counter for as long as possible. The xorshift32()
+ // algorithm used below results in getRandomUnsigned32(0xacd979ce) ==
+ // 0xfffffffe.
+ uint32_t RandomState;
+ // Thread-local decrementing counter that indicates that a given allocation
+ // should be sampled when it reaches zero.
+ uint32_t NextSampleCounter : 31;
+ // The mask is needed to silence conversion errors.
+ static const uint32_t NextSampleCounterMask = (1U << 31) - 1;
+ // Guard against recursivity. Unwinders often contain complex behaviour that
+ // may not be safe for the allocator (i.e. the unwinder calls dlopen(),
+ // which calls malloc()). When recursive behaviour is detected, we will
+ // automatically fall back to the supporting allocator to supply the
+ // allocation.
+ bool RecursiveGuard : 1;
+};
+static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t),
+ "thread local data does not fit in a uint64_t");
+} // namespace gwp_asan
+
+#ifdef GWP_ASAN_PLATFORM_TLS_HEADER
+#include GWP_ASAN_PLATFORM_TLS_HEADER
+#else
+namespace gwp_asan {
+inline ThreadLocalPackedVariables *getThreadLocals() {
+ alignas(8) static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables Locals;
+ return &Locals;
+}
+} // namespace gwp_asan
+#endif // GWP_ASAN_PLATFORM_TLS_HEADER
+
+#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_
--- /dev/null
+//===-- mutex_fuchsia.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/mutex.h"
+
+#include <lib/sync/mutex.h>
+
+namespace gwp_asan {
+void Mutex::lock() __TA_NO_THREAD_SAFETY_ANALYSIS { sync_mutex_lock(&Mu); }
+
+bool Mutex::tryLock() __TA_NO_THREAD_SAFETY_ANALYSIS {
+ return sync_mutex_trylock(&Mu) == ZX_OK;
+}
+
+void Mutex::unlock() __TA_NO_THREAD_SAFETY_ANALYSIS { sync_mutex_unlock(&Mu); }
+} // namespace gwp_asan
--- /dev/null
+//===-- mutex_fuchsia.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__Fuchsia__)
+#ifndef GWP_ASAN_MUTEX_FUCHSIA_H_
+#define GWP_ASAN_MUTEX_FUCHSIA_H_
+
+#include <lib/sync/mutex.h>
+
+namespace gwp_asan {
+class PlatformMutex {
+protected:
+ sync_mutex_t Mu = {};
+};
+} // namespace gwp_asan
+
+#endif // GWP_ASAN_MUTEX_FUCHSIA_H_
+#endif // defined(__Fuchsia__)
--- /dev/null
+//===-- mutex_posix.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(__unix__)
+#ifndef GWP_ASAN_MUTEX_POSIX_H_
+#define GWP_ASAN_MUTEX_POSIX_H_
+
+#include <pthread.h>
+
+namespace gwp_asan {
+class PlatformMutex {
+protected:
+ pthread_mutex_t Mu = PTHREAD_MUTEX_INITIALIZER;
+};
+} // namespace gwp_asan
+
+#endif // GWP_ASAN_MUTEX_POSIX_H_
+#endif // defined(__unix__)
--- /dev/null
+//===-- utilities_fuchsia.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/utilities.h"
+
+#include <string.h>
+#include <zircon/sanitizer.h>
+
+namespace gwp_asan {
+void die(const char *Message) {
+ __sanitizer_log_write(Message, strlen(Message));
+ __builtin_trap();
+}
+} // namespace gwp_asan
//
//===----------------------------------------------------------------------===//
-#include "gwp_asan/definitions.h"
-#include "gwp_asan/utilities.h"
-
-#include <assert.h>
+#include <features.h> // IWYU pragma: keep (for __BIONIC__ macro)
#ifdef __BIONIC__
+#include "gwp_asan/definitions.h"
#include <stdlib.h>
extern "C" GWP_ASAN_WEAK void android_set_abort_message(const char *);
#else // __BIONIC__
#endif
namespace gwp_asan {
-
+void die(const char *Message) {
#ifdef __BIONIC__
-void Check(bool Condition, const char *Message) {
- if (Condition)
- return;
if (&android_set_abort_message != nullptr)
android_set_abort_message(Message);
abort();
-}
#else // __BIONIC__
-void Check(bool Condition, const char *Message) {
- if (Condition)
- return;
fprintf(stderr, "%s", Message);
__builtin_trap();
-}
#endif // __BIONIC__
-
-// See `bionic/tests/malloc_test.cpp` in the Android source for documentation
-// regarding their alignment guarantees. We always round up to the closest
-// 8-byte window. As GWP-ASan's malloc(X) can always get exactly an X-sized
-// allocation, an allocation that rounds up to 16-bytes will always be given a
-// 16-byte aligned allocation.
-static size_t alignBionic(size_t RealAllocationSize) {
- if (RealAllocationSize % 8 == 0)
- return RealAllocationSize;
- return RealAllocationSize + 8 - (RealAllocationSize % 8);
-}
-
-static size_t alignPowerOfTwo(size_t RealAllocationSize) {
- if (RealAllocationSize <= 2)
- return RealAllocationSize;
- if (RealAllocationSize <= 4)
- return 4;
- if (RealAllocationSize <= 8)
- return 8;
- if (RealAllocationSize % 16 == 0)
- return RealAllocationSize;
- return RealAllocationSize + 16 - (RealAllocationSize % 16);
}
-
-#ifdef __BIONIC__
-static constexpr AlignmentStrategy PlatformDefaultAlignment =
- AlignmentStrategy::BIONIC;
-#else // __BIONIC__
-static constexpr AlignmentStrategy PlatformDefaultAlignment =
- AlignmentStrategy::POWER_OF_TWO;
-#endif // __BIONIC__
-
-size_t rightAlignedAllocationSize(size_t RealAllocationSize,
- AlignmentStrategy Align) {
- assert(RealAllocationSize > 0);
- if (Align == AlignmentStrategy::DEFAULT)
- Align = PlatformDefaultAlignment;
-
- switch (Align) {
- case AlignmentStrategy::BIONIC:
- return alignBionic(RealAllocationSize);
- case AlignmentStrategy::POWER_OF_TWO:
- return alignPowerOfTwo(RealAllocationSize);
- case AlignmentStrategy::PERFECT:
- return RealAllocationSize;
- case AlignmentStrategy::DEFAULT:
- __builtin_unreachable();
- }
- __builtin_unreachable();
-}
-
} // namespace gwp_asan
if [ -z "$function_name" ]; then
# If the offset is binary-relative, just resolve that.
- symbolized="$(echo $function_offset | addr2line -e $binary_name)"
+ symbolized="$(echo $function_offset | addr2line -ie $binary_name)"
else
# Otherwise, the offset is function-relative. Get the address of the
# function, and add it to the offset, then symbolize.
# Add the function address and offset to get the offset into the binary.
binary_offset="$(printf "0x%X" "$((function_addr+function_offset))")"
- symbolized="$(echo $binary_offset | addr2line -e $binary_name)"
+ symbolized="$(echo $binary_offset | addr2line -ie $binary_name)"
fi
# Check that it symbolized properly. If it didn't, output the old line.
else
echo "${frame_number}${symbolized}"
fi
-done
+done 2> >(grep -v "addr2line: DWARF error: could not find variable specification")
${COMPILER_RT_GTEST_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/lib/
-O2
- -g)
+ -g
+ -fno-omit-frame-pointer)
file(GLOB GWP_ASAN_HEADERS ../*.h)
set(GWP_ASAN_UNITTESTS
- optional/printf_sanitizer_common.cpp
+ platform_specific/printf_sanitizer_common.cpp
alignment.cpp
backtrace.cpp
basic.cpp
thread_contention.cpp
harness.cpp
enable_disable.cpp
- late_init.cpp)
+ late_init.cpp
+ options.cpp)
set(GWP_ASAN_UNIT_TEST_HEADERS
${GWP_ASAN_HEADERS}
$<TARGET_OBJECTS:RTGwpAsan.${arch}>
$<TARGET_OBJECTS:RTGwpAsanBacktraceSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTGwpAsanSegvHandler.${arch}>
+ $<TARGET_OBJECTS:RTGwpAsanOptionsParser.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>)
//
//===----------------------------------------------------------------------===//
+#include "gwp_asan/guarded_pool_allocator.h"
#include "gwp_asan/tests/harness.h"
-#include "gwp_asan/utilities.h"
-
-TEST(AlignmentTest, PowerOfTwo) {
- std::vector<std::pair<size_t, size_t>> AskedSizeToAlignedSize = {
- {1, 1}, {2, 2}, {3, 4}, {4, 4}, {5, 8}, {7, 8},
- {8, 8}, {9, 16}, {15, 16}, {16, 16}, {17, 32}, {31, 32},
- {32, 32}, {33, 48}, {4095, 4096}, {4096, 4096},
- };
-
- for (const auto &KV : AskedSizeToAlignedSize) {
- EXPECT_EQ(KV.second,
- gwp_asan::rightAlignedAllocationSize(
- KV.first, gwp_asan::AlignmentStrategy::POWER_OF_TWO));
+
+class AlignmentTestGPA : public gwp_asan::GuardedPoolAllocator {
+public:
+ static size_t getRequiredBackingSize(size_t Size, size_t Alignment,
+ size_t PageSize) {
+ return GuardedPoolAllocator::getRequiredBackingSize(Size, Alignment,
+ PageSize);
+ }
+ static uintptr_t alignUp(uintptr_t Ptr, size_t Alignment) {
+ return GuardedPoolAllocator::alignUp(Ptr, Alignment);
}
+ static uintptr_t alignDown(uintptr_t Ptr, size_t Alignment) {
+ return GuardedPoolAllocator::alignDown(Ptr, Alignment);
+ }
+};
+
+// Global assumptions for these tests:
+// 1. Page size is 0x1000.
+// 2. All tests assume a slot is multipage, between 0x4000 - 0x8000. While we
+// don't use multipage slots right now, this tests more boundary conditions
+// and allows us to add this feature at a later date without rewriting the
+// alignment functionality.
+// These aren't actual requirements of the allocator - but just simplifies the
+// numerics of the testing.
+TEST(AlignmentTest, LeftAlignedAllocs) {
+ // Alignment < Page Size.
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x1));
+ // Alignment == Page Size.
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x1000));
+ // Alignment > Page Size.
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignUp(
+ /* Ptr */ 0x4000, /* Alignment */ 0x4000));
}
-TEST(AlignmentTest, AlignBionic) {
- std::vector<std::pair<size_t, size_t>> AskedSizeToAlignedSize = {
- {1, 8}, {2, 8}, {3, 8}, {4, 8}, {5, 8}, {7, 8},
- {8, 8}, {9, 16}, {15, 16}, {16, 16}, {17, 24}, {31, 32},
- {32, 32}, {33, 40}, {4095, 4096}, {4096, 4096},
- };
+TEST(AlignmentTest, SingleByteAllocs) {
+ // Alignment < Page Size.
+ EXPECT_EQ(0x1,
+ AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1, /* Alignment */ 0x1, /* PageSize */ 0x1000));
+ EXPECT_EQ(0x7fff, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1));
- for (const auto &KV : AskedSizeToAlignedSize) {
- EXPECT_EQ(KV.second, gwp_asan::rightAlignedAllocationSize(
- KV.first, gwp_asan::AlignmentStrategy::BIONIC));
- }
+ // Alignment == Page Size.
+ EXPECT_EQ(0x1,
+ AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1, /* Alignment */ 0x1000, /* PageSize */ 0x1000));
+ EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x1000));
+
+ // Alignment > Page Size.
+ EXPECT_EQ(0x3001,
+ AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1, /* Alignment */ 0x4000, /* PageSize */ 0x1000));
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1, /* Alignment */ 0x4000));
}
-TEST(AlignmentTest, PerfectAlignment) {
- for (size_t i = 1; i <= 4096; ++i) {
- EXPECT_EQ(i, gwp_asan::rightAlignedAllocationSize(
- i, gwp_asan::AlignmentStrategy::PERFECT));
- }
+TEST(AlignmentTest, PageSizedAllocs) {
+ // Alignment < Page Size.
+ EXPECT_EQ(0x1000,
+ AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1000, /* Alignment */ 0x1, /* PageSize */ 0x1000));
+ EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1));
+
+ // Alignment == Page Size.
+ EXPECT_EQ(0x1000, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1000, /* Alignment */ 0x1000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x7000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x1000));
+
+ // Alignment > Page Size.
+ EXPECT_EQ(0x4000, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x1000, /* Alignment */ 0x4000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x1000, /* Alignment */ 0x4000));
+}
+
+TEST(AlignmentTest, MoreThanPageAllocs) {
+ // Alignment < Page Size.
+ EXPECT_EQ(0x2fff,
+ AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x2fff, /* Alignment */ 0x1, /* PageSize */ 0x1000));
+ EXPECT_EQ(0x5001, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1));
+
+ // Alignment == Page Size.
+ EXPECT_EQ(0x2fff, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x2fff, /* Alignment */ 0x1000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x5000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x1000));
+
+ // Alignment > Page Size.
+ EXPECT_EQ(0x5fff, AlignmentTestGPA::getRequiredBackingSize(
+ /* Size */ 0x2fff, /* Alignment */ 0x4000,
+ /* PageSize */ 0x1000));
+ EXPECT_EQ(0x4000, AlignmentTestGPA::alignDown(
+ /* Ptr */ 0x8000 - 0x2fff, /* Alignment */ 0x4000));
}
#include <string>
+#include "gwp_asan/common.h"
#include "gwp_asan/crash_handler.h"
#include "gwp_asan/tests/harness.h"
*(reinterpret_cast<volatile char *>(Ptr)) = 7;
}
-TEST_F(BacktraceGuardedPoolAllocator, DoubleFree) {
+TEST_F(BacktraceGuardedPoolAllocatorDeathTest, DoubleFree) {
void *Ptr = AllocateMemory(GPA);
DeallocateMemory(GPA, Ptr);
ASSERT_DEATH(DeallocateMemory2(GPA, Ptr), DeathRegex);
}
-TEST_F(BacktraceGuardedPoolAllocator, UseAfterFree) {
+TEST_F(BacktraceGuardedPoolAllocatorDeathTest, UseAfterFree) {
+#if defined(__linux__) && __ARM_ARCH == 7
+ // Incomplete backtrace on Armv7 Linux
+ GTEST_SKIP();
+#endif
+
void *Ptr = AllocateMemory(GPA);
DeallocateMemory(GPA, Ptr);
TEST(Backtrace, ExceedsStorableLength) {
gwp_asan::AllocationMetadata Meta;
Meta.AllocationTrace.RecordBacktrace(
- [](uintptr_t * /* TraceBuffer */, size_t /* Size */) -> size_t {
- return SIZE_MAX; // Wow, that's big!
+ [](uintptr_t *TraceBuffer, size_t Size) -> size_t {
+ // Need to inintialise the elements that will be packed.
+ memset(TraceBuffer, 0u, Size * sizeof(*TraceBuffer));
+
+ // Indicate that there were more frames, and we just didn't have enough
+ // room to store them.
+ return Size * 2;
+ });
+ // Retrieve a frame from the collected backtrace, make sure it works E2E.
+ uintptr_t TraceOutput;
+ EXPECT_EQ(gwp_asan::AllocationMetadata::kMaxTraceLengthToCollect,
+ __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+}
+
+TEST(Backtrace, ExceedsRetrievableAllocLength) {
+ gwp_asan::AllocationMetadata Meta;
+ constexpr size_t kNumFramesToStore = 3u;
+ Meta.AllocationTrace.RecordBacktrace(
+ [](uintptr_t *TraceBuffer, size_t /* Size */) -> size_t {
+ memset(TraceBuffer, kNumFramesToStore,
+ kNumFramesToStore * sizeof(*TraceBuffer));
+ return kNumFramesToStore;
+ });
+ uintptr_t TraceOutput;
+ // Ask for one element, get told that there's `kNumFramesToStore` available.
+ EXPECT_EQ(kNumFramesToStore,
+ __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+}
+
+TEST(Backtrace, ExceedsRetrievableDeallocLength) {
+ gwp_asan::AllocationMetadata Meta;
+ constexpr size_t kNumFramesToStore = 3u;
+ Meta.DeallocationTrace.RecordBacktrace(
+ [](uintptr_t *TraceBuffer, size_t /* Size */) -> size_t {
+ memset(TraceBuffer, kNumFramesToStore,
+ kNumFramesToStore * sizeof(*TraceBuffer));
+ return kNumFramesToStore;
});
uintptr_t TraceOutput;
- EXPECT_EQ(1u, __gwp_asan_get_allocation_trace(&Meta, &TraceOutput, 1));
+ // Ask for one element, get told that there's `kNumFramesToStore` available.
+ EXPECT_EQ(kNumFramesToStore,
+ __gwp_asan_get_deallocation_trace(&Meta, &TraceOutput, 1));
}
TEST_F(DefaultGuardedPoolAllocator, TooLargeAllocation) {
EXPECT_EQ(nullptr,
GPA.allocate(GPA.getAllocatorState()->maximumAllocationSize() + 1));
+ EXPECT_EQ(nullptr, GPA.allocate(SIZE_MAX, 0));
+ EXPECT_EQ(nullptr, GPA.allocate(SIZE_MAX, 1));
+ EXPECT_EQ(nullptr, GPA.allocate(0, SIZE_MAX / 2));
+ EXPECT_EQ(nullptr, GPA.allocate(1, SIZE_MAX / 2));
+ EXPECT_EQ(nullptr, GPA.allocate(SIZE_MAX, SIZE_MAX / 2));
+}
+
+TEST_F(DefaultGuardedPoolAllocator, ZeroSizeAndAlignmentAllocations) {
+ void *P;
+ EXPECT_NE(nullptr, (P = GPA.allocate(0, 0)));
+ GPA.deallocate(P);
+ EXPECT_NE(nullptr, (P = GPA.allocate(1, 0)));
+ GPA.deallocate(P);
+ EXPECT_NE(nullptr, (P = GPA.allocate(0, 1)));
+ GPA.deallocate(P);
+}
+
+TEST_F(DefaultGuardedPoolAllocator, NonPowerOfTwoAlignment) {
+ EXPECT_EQ(nullptr, GPA.allocate(0, 3));
+ EXPECT_EQ(nullptr, GPA.allocate(1, 3));
+ EXPECT_EQ(nullptr, GPA.allocate(0, SIZE_MAX));
+ EXPECT_EQ(nullptr, GPA.allocate(1, SIZE_MAX));
+}
+
+// Added multi-page slots? You'll need to expand this test.
+TEST_F(DefaultGuardedPoolAllocator, TooBigForSinglePageSlots) {
+ EXPECT_EQ(nullptr, GPA.allocate(0x1001, 0));
+ EXPECT_EQ(nullptr, GPA.allocate(0x1001, 1));
+ EXPECT_EQ(nullptr, GPA.allocate(0x1001, 0x1000));
+ EXPECT_EQ(nullptr, GPA.allocate(1, 0x2000));
+ EXPECT_EQ(nullptr, GPA.allocate(0, 0x2000));
}
TEST_F(CustomGuardedPoolAllocator, AllocAllSlots) {
//===----------------------------------------------------------------------===//
#include "gwp_asan/stack_trace_compressor.h"
-#include "gtest/gtest.h"
+#include "gwp_asan/tests/harness.h"
namespace gwp_asan {
namespace compression {
using AllocationMetadata = gwp_asan::AllocationMetadata;
using AllocatorState = gwp_asan::AllocatorState;
-class CrashHandlerAPITest : public ::testing::Test {
+class CrashHandlerAPITest : public Test {
public:
void SetUp() override { setupState(); }
size_t Slot = State.getNearestSlot(Addr);
Metadata[Slot].Addr = Addr;
- Metadata[Slot].Size = Size;
+ Metadata[Slot].RequestedSize = Size;
Metadata[Slot].IsDeallocated = IsDeallocated;
Metadata[Slot].AllocationTrace.ThreadID = 123;
Metadata[Slot].DeallocationTrace.ThreadID = 321;
__gwp_asan_get_metadata(&State, Metadata, ErrorPtr);
EXPECT_NE(nullptr, Meta);
EXPECT_EQ(Metadata[Index].Addr, __gwp_asan_get_allocation_address(Meta));
- EXPECT_EQ(Metadata[Index].Size, __gwp_asan_get_allocation_size(Meta));
+ EXPECT_EQ(Metadata[Index].RequestedSize,
+ __gwp_asan_get_allocation_size(Meta));
EXPECT_EQ(Metadata[Index].AllocationTrace.ThreadID,
__gwp_asan_get_allocation_thread_id(Meta));
//
//===----------------------------------------------------------------------===//
-#include "gtest/gtest.h"
+#include "gwp_asan/tests/harness.h"
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
constexpr size_t Size = 100;
-TEST_F(DefaultGuardedPoolAllocator, Fork) {
+TEST_F(DefaultGuardedPoolAllocatorDeathTest, Fork) {
void *P;
pid_t Pid = fork();
EXPECT_GE(Pid, 0);
-#include "harness.h"
+//===-- harness.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/tests/harness.h"
namespace gwp_asan {
namespace test {
#include <stdarg.h>
+#if defined(__Fuchsia__)
+#include <zxtest/zxtest.h>
+using Test = ::zxtest::Test;
+#else
#include "gtest/gtest.h"
+using Test = ::testing::Test;
+#endif
#include "gwp_asan/guarded_pool_allocator.h"
#include "gwp_asan/optional/backtrace.h"
+#include "gwp_asan/optional/printf.h"
#include "gwp_asan/optional/segv_handler.h"
#include "gwp_asan/options.h"
// their own signal-safe Printf function. In LLVM, we use
// `optional/printf_sanitizer_common.cpp` which supplies the __sanitizer::Printf
// for this purpose.
-crash_handler::Printf_t getPrintfFunction();
+Printf_t getPrintfFunction();
// First call returns true, all the following calls return false.
bool OnlyOnce();
}; // namespace test
}; // namespace gwp_asan
-class DefaultGuardedPoolAllocator : public ::testing::Test {
+class DefaultGuardedPoolAllocator : public Test {
public:
void SetUp() override {
gwp_asan::options::Options Opts;
MaxSimultaneousAllocations;
};
-class CustomGuardedPoolAllocator : public ::testing::Test {
+class CustomGuardedPoolAllocator : public Test {
public:
void
InitNumSlots(decltype(gwp_asan::options::Options::MaxSimultaneousAllocations)
MaxSimultaneousAllocations;
};
-class BacktraceGuardedPoolAllocator : public ::testing::Test {
+class BacktraceGuardedPoolAllocator : public Test {
public:
void SetUp() override {
gwp_asan::options::Options Opts;
Opts.setDefaults();
- Opts.Backtrace = gwp_asan::options::getBacktraceFunction();
+ Opts.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
Opts.InstallForkHandlers = gwp_asan::test::OnlyOnce();
GPA.init(Opts);
- gwp_asan::crash_handler::installSignalHandlers(
+ gwp_asan::segv_handler::installSignalHandlers(
&GPA, gwp_asan::test::getPrintfFunction(),
- gwp_asan::options::getPrintBacktraceFunction(), Opts.Backtrace);
+ gwp_asan::backtrace::getPrintBacktraceFunction(),
+ gwp_asan::backtrace::getSegvBacktraceFunction());
}
void TearDown() override {
GPA.uninitTestOnly();
- gwp_asan::crash_handler::uninstallSignalHandlers();
+ gwp_asan::segv_handler::uninstallSignalHandlers();
}
protected:
gwp_asan::GuardedPoolAllocator GPA;
};
+// https://github.com/google/googletest/blob/master/docs/advanced.md#death-tests-and-threads
+using DefaultGuardedPoolAllocatorDeathTest = DefaultGuardedPoolAllocator;
+using CustomGuardedPoolAllocatorDeathTest = CustomGuardedPoolAllocator;
+using BacktraceGuardedPoolAllocatorDeathTest = BacktraceGuardedPoolAllocator;
+
#endif // GWP_ASAN_TESTS_HARNESS_H_
#include "gwp_asan/tests/harness.h"
+#include <set>
+#include <vector>
+
TEST_F(CustomGuardedPoolAllocator, Iterate) {
InitNumSlots(7);
std::vector<std::pair<void *, size_t>> Allocated;
#include "gwp_asan/guarded_pool_allocator.h"
#include "gwp_asan/options.h"
-#include "gtest/gtest.h"
+#include "gwp_asan/tests/harness.h"
TEST(LateInit, CheckLateInitIsOK) {
gwp_asan::GuardedPoolAllocator GPA;
//===----------------------------------------------------------------------===//
#include "gwp_asan/mutex.h"
-#include "gtest/gtest.h"
+#include "gwp_asan/tests/harness.h"
#include <atomic>
#include <mutex>
--- /dev/null
+//===-- options.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/tests/harness.h"
+
+#include "gwp_asan/optional/options_parser.h"
+#include "gwp_asan/options.h"
+
+#include <stdarg.h>
+
+static char Message[1024];
+void MessageRecorder(const char *Format, ...) {
+ va_list Args;
+ va_start(Args, Format);
+ vsprintf(Message + strlen(Message), Format, Args);
+ va_end(Args);
+}
+
+TEST(GwpAsanOptionsTest, Basic) {
+ Message[0] = '\0';
+ gwp_asan::options::initOptions("Enabled=0:SampleRate=4:"
+ "InstallSignalHandlers=false",
+ MessageRecorder);
+ gwp_asan::options::Options Opts = gwp_asan::options::getOptions();
+ EXPECT_EQ('\0', Message[0]);
+ EXPECT_FALSE(Opts.Enabled);
+ EXPECT_FALSE(Opts.InstallSignalHandlers);
+ EXPECT_EQ(4, Opts.SampleRate);
+}
+
+void RunErrorTest(const char *OptionsStr, const char *ErrorNeedle) {
+ Message[0] = '\0';
+ gwp_asan::options::initOptions(OptionsStr, MessageRecorder);
+ EXPECT_NE('\0', Message[0])
+ << "Options string \"" << OptionsStr << "\" didn't generate an error.";
+ EXPECT_NE(nullptr, strstr(Message, ErrorNeedle))
+ << "Couldn't find error needle \"" << ErrorNeedle
+ << "\" in haystack created by options string \"" << OptionsStr
+ << "\". Error was: \"" << Message << "\".";
+}
+
+TEST(GwpAsanOptionsTest, FailureModes) {
+ RunErrorTest("Enabled=2", "Invalid boolean value '2' for option 'Enabled'");
+ RunErrorTest("Enabled=1:MaxSimultaneousAllocations=0",
+ "MaxSimultaneousAllocations must be > 0");
+ RunErrorTest("Enabled=1:MaxSimultaneousAllocations=-1",
+ "MaxSimultaneousAllocations must be > 0");
+ RunErrorTest("Enabled=1:SampleRate=0", "SampleRate must be > 0");
+ RunErrorTest("Enabled=1:SampleRate=-1", "SampleRate must be > 0");
+ RunErrorTest("Enabled=", "Invalid boolean value '' for option 'Enabled'");
+ RunErrorTest("==", "Unknown option '=='");
+ RunErrorTest("Enabled==0", "Invalid boolean value '=0' for option 'Enabled'");
+ RunErrorTest("Enabled:", "Expected '=' when parsing option 'Enabled:'");
+ RunErrorTest("Enabled:=", "Expected '=' when parsing option 'Enabled:='");
+ RunErrorTest("SampleRate=NOT_A_NUMBER",
+ "Invalid integer value 'NOT_A_NUMBER' for option 'SampleRate'");
+ RunErrorTest("NOT_A_VALUE=0", "Unknown option 'NOT_A_VALUE");
+}
--- /dev/null
+//===-- printf_sanitizer_common.cpp -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gwp_asan/optional/printf.h"
+
+#include "sanitizer_common/sanitizer_common.h"
+
+namespace gwp_asan {
+namespace test {
+
+Printf_t getPrintfFunction() { return __sanitizer::Printf; }
+
+} // namespace test
+} // namespace gwp_asan
#include "gwp_asan/tests/harness.h"
+#include <set>
+
void singleByteGoodAllocDealloc(gwp_asan::GuardedPoolAllocator *GPA) {
void *Ptr = GPA->allocate(1);
EXPECT_NE(nullptr, Ptr);
//
//===----------------------------------------------------------------------===//
+#ifndef GWP_ASAN_UTILITIES_H_
+#define GWP_ASAN_UTILITIES_H_
+
#include "gwp_asan/definitions.h"
#include <stddef.h>
-#include <stdint.h>
namespace gwp_asan {
-// Checks that `Condition` is true, otherwise fails in a platform-specific way
-// with `Message`.
-void Check(bool Condition, const char *Message);
-
-enum class AlignmentStrategy {
- // Default => POWER_OF_TWO on most platforms, BIONIC for Android Bionic.
- DEFAULT,
- POWER_OF_TWO,
- BIONIC,
- PERFECT,
-};
+// Terminates in a platform-specific way with `Message`.
+void die(const char *Message);
-// Returns the real size of a right-aligned allocation.
-size_t rightAlignedAllocationSize(
- size_t RealAllocationSize,
- AlignmentStrategy Align = AlignmentStrategy::DEFAULT);
+// Checks that `Condition` is true, otherwise dies with `Message`.
+GWP_ASAN_ALWAYS_INLINE void Check(bool Condition, const char *Message) {
+ if (Condition)
+ return;
+ die(Message);
+}
} // namespace gwp_asan
+
+#endif // GWP_ASAN_UTILITIES_H_
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
set(HWASAN_RTL_SOURCES
hwasan.cpp
hwasan_allocator.cpp
+ hwasan_allocation_functions.cpp
hwasan_dynamic_shadow.cpp
hwasan_exceptions.cpp
+ hwasan_fuchsia.cpp
hwasan_globals.cpp
hwasan_interceptors.cpp
hwasan_interceptors_vfork.S
set(HWASAN_DEFINITIONS)
append_list_if(COMPILER_RT_HWASAN_WITH_INTERCEPTORS HWASAN_WITH_INTERCEPTORS=1 HWASAN_DEFINITIONS)
+if(FUCHSIA)
+ # Set this explicitly on Fuchsia, otherwise the default value is set to HWASAN_WITH_INTERCEPTORS.
+ list(APPEND HWASAN_DEFINITIONS HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE=1)
+endif()
+
set(HWASAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF HWASAN_RTL_CFLAGS)
append_list_if(COMPILER_RT_HAS_FPIC_FLAG -fPIC HWASAN_RTL_CFLAGS)
append_list_if(COMPILER_RT_HAS_LIBM m HWASAN_DYNAMIC_LIBS)
append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread HWASAN_DYNAMIC_LIBS)
-if (TARGET cxx-headers OR HAVE_LIBCXX)
- set(HWASAN_DEPS cxx-headers)
-endif()
-
# Static runtime library.
add_compiler_rt_component(hwasan)
SOURCES ${HWASAN_RTL_SOURCES}
ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
CFLAGS ${HWASAN_RTL_CFLAGS}
- DEFS ${HWASAN_DEFINITIONS}
- DEPS ${HWASAN_DEPS})
+ DEFS ${HWASAN_DEFINITIONS})
add_compiler_rt_object_libraries(RTHwasan_cxx
ARCHS ${HWASAN_SUPPORTED_ARCH}
SOURCES ${HWASAN_RTL_CXX_SOURCES}
ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
CFLAGS ${HWASAN_RTL_CFLAGS}
- DEFS ${HWASAN_DEFINITIONS}
- DEPS ${HWASAN_DEPS})
+ DEFS ${HWASAN_DEFINITIONS})
add_compiler_rt_object_libraries(RTHwasan_dynamic
ARCHS ${HWASAN_SUPPORTED_ARCH}
SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES}
ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
- DEFS ${HWASAN_DEFINITIONS}
- DEPS ${HWASAN_DEPS})
+ DEFS ${HWASAN_DEFINITIONS})
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
ARCHS ${HWASAN_SUPPORTED_ARCH}
SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
- DEFS ${HWASAN_DEFINITIONS}
- DEPS ${HWASAN_DEPS})
-
-foreach(arch ${HWASAN_SUPPORTED_ARCH})
- add_compiler_rt_runtime(clang_rt.hwasan
+ DEFS ${HWASAN_DEFINITIONS})
+
+# If use_aliases is TRUE, adds the HWASan runtime built with alias support.
+# Otherwise adds the runtime without alias support.
+function(add_hwasan_runtimes arch use_aliases)
+ set(hwasan_object_lib RTHwasan)
+ set(hwasan_object_dyn_lib RTHwasan_dynamic)
+ set(hwasan_runtime clang_rt.hwasan)
+ set(hwasan_rtl_flags ${HWASAN_RTL_CFLAGS})
+ set(hwasan_dyn_flags ${HWASAN_DYNAMIC_CFLAGS})
+ if(use_aliases)
+ list(APPEND hwasan_rtl_flags -DHWASAN_ALIASING_MODE)
+ list(APPEND hwasan_dyn_flags -DHWASAN_ALIASING_MODE)
+ add_compiler_rt_object_libraries(RTHwasanAliases
+ ARCHS ${arch}
+ SOURCES ${HWASAN_RTL_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${hwasan_rtl_flags}
+ DEFS ${HWASAN_DEFINITIONS})
+ add_compiler_rt_object_libraries(RTHwasanAliases_dynamic
+ ARCHS ${arch}
+ SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+ CFLAGS ${hwasan_dyn_flags}
+ DEFS ${HWASAN_DEFINITIONS})
+
+ set(hwasan_object_lib RTHwasanAliases)
+ set(hwasan_object_dyn_lib RTHwasanAliases_dynamic)
+ set(hwasan_runtime clang_rt.hwasan_aliases)
+ endif()
+ add_compiler_rt_runtime(${hwasan_runtime}
STATIC
ARCHS ${arch}
- OBJECT_LIBS RTHwasan
+ OBJECT_LIBS ${hwasan_object_lib}
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
RTSanitizerCommonCoverage
RTSanitizerCommonSymbolizer
RTUbsan
- CFLAGS ${HWASAN_RTL_CFLAGS}
+ CFLAGS ${hwasan_rtl_flags}
PARENT_TARGET hwasan)
- add_compiler_rt_runtime(clang_rt.hwasan_cxx
+ add_compiler_rt_runtime(${hwasan_runtime}_cxx
STATIC
ARCHS ${arch}
OBJECT_LIBS RTHwasan_cxx
RTUbsan_cxx
- CFLAGS ${HWASAN_RTL_CFLAGS}
+ CFLAGS ${hwasan_rtl_flags}
PARENT_TARGET hwasan)
if (UNIX)
- add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch}
- LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch}
+ add_sanitizer_rt_version_list(${hwasan_runtime}-dynamic-${arch}
+ LIBS ${hwasan_runtime}-${arch} ${hwasan_runtime}_cxx-${arch}
EXTRA hwasan.syms.extra)
set(VERSION_SCRIPT_FLAG
- -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
set_property(SOURCE
${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
APPEND PROPERTY
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+ OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
else()
set(VERSION_SCRIPT_FLAG)
endif()
- add_compiler_rt_runtime(clang_rt.hwasan
+ add_compiler_rt_runtime(${hwasan_runtime}
SHARED
ARCHS ${arch}
OBJECT_LIBS
- RTHwasan_dynamic
+ ${hwasan_object_dyn_lib}
RTInterception
RTSanitizerCommon
RTSanitizerCommonLibc
# add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
# generates an order-only dependency in ninja.
RTHwasan_dynamic_version_script_dummy
- CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
+ CFLAGS ${hwasan_dyn_flags}
LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS}
${VERSION_SCRIPT_FLAG}
LINK_LIBS ${HWASAN_DYNAMIC_LIBS}
PARENT_TARGET hwasan)
if(SANITIZER_USE_SYMBOLS)
- add_sanitizer_rt_symbols(clang_rt.hwasan
+ add_sanitizer_rt_symbols(${hwasan_runtime}
ARCHS ${arch}
EXTRA hwasan.syms.extra)
- add_sanitizer_rt_symbols(clang_rt.hwasan_cxx
+ add_sanitizer_rt_symbols(${hwasan_runtime}_cxx
ARCHS ${arch}
EXTRA hwasan.syms.extra)
- add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols
- clang_rt.hwasan_cxx-${arch}-symbols)
+ add_dependencies(hwasan ${hwasan_runtime}-${arch}-symbols
+ ${hwasan_runtime}_cxx-${arch}-symbols)
+ endif()
+endfunction()
+
+foreach(arch ${HWASAN_SUPPORTED_ARCH})
+ add_hwasan_runtimes(${arch} FALSE)
+ if(${arch} MATCHES "x86_64")
+ add_hwasan_runtimes(${arch} TRUE)
endif()
endforeach()
-add_compiler_rt_resource_file(hwasan_blacklist hwasan_blacklist.txt hwasan)
+add_compiler_rt_resource_file(hwasan_ignorelist hwasan_ignorelist.txt hwasan)
add_subdirectory("scripts")
int hwasan_report_count = 0;
+uptr kLowShadowStart;
+uptr kLowShadowEnd;
+uptr kHighShadowStart;
+uptr kHighShadowEnd;
+
void Flags::SetDefaults() {
#define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
#include "hwasan_flags.inc"
if (__hwasan_default_options)
parser.ParseString(__hwasan_default_options());
#if HWASAN_CONTAINS_UBSAN
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
#endif
if (common_flags()->help) parser.PrintFlagDescriptions();
}
-static void HWAsanCheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2) {
- Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
- line, cond, (uptr)v1, (uptr)v2);
- PRINT_CURRENT_STACK_CHECK();
- Die();
+static void CheckUnwind() {
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+ stack.Print();
}
-static constexpr uptr kMemoryUsageBufferSize = 4096;
-
static void HwasanFormatMemoryUsage(InternalScopedString &s) {
HwasanThreadList &thread_list = hwasanThreadList();
auto thread_stats = thread_list.GetThreadStats();
}
#if SANITIZER_ANDROID
+static constexpr uptr kMemoryUsageBufferSize = 4096;
+
static char *memory_usage_buffer = nullptr;
static void InitMemoryUsage() {
return;
if (!memory_usage_buffer)
InitMemoryUsage();
- InternalScopedString s(kMemoryUsageBufferSize);
+ InternalScopedString s;
HwasanFormatMemoryUsage(s);
internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
void UpdateMemoryUsage() {}
#endif
+void HwasanAtExit() {
+ if (common_flags()->print_module_map)
+ DumpProcessMap();
+ if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+ ReportStats();
+ if (hwasan_report_count > 0) {
+ // ReportAtExitStatistics();
+ if (common_flags()->exitcode)
+ internal__exit(common_flags()->exitcode);
+ }
+}
+
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+ uptr *registers_frame) {
+ InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+ BufferedStackTrace *stack = stack_buffer.data();
+ stack->Reset();
+ stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
+
+ // The second stack frame contains the failure __hwasan_check function, as
+ // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
+ // we wish to ignore. This (currently) only occurs on AArch64, as x64
+ // implementations use SIGTRAP to implement the failure, and thus do not go
+ // through the stack saver.
+ if (registers_frame && stack->trace && stack->size > 0) {
+ stack->trace++;
+ stack->size--;
+ }
+
+ bool fatal = flags()->halt_on_error || !ai.recover;
+ ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
+ registers_frame);
+}
+
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize) {
+ __hwasan::AccessInfo ai;
+ ai.is_store = access_info & 0x10;
+ ai.is_load = !ai.is_store;
+ ai.recover = access_info & 0x20;
+ ai.addr = addr;
+ if ((access_info & 0xf) == 0xf)
+ ai.size = outsize;
+ else
+ ai.size = 1 << (access_info & 0xf);
+
+ HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
+ (uptr)__builtin_frame_address(0), nullptr, registers_frame);
+ __builtin_unreachable();
+}
+
+Thread *GetCurrentThread() {
+ uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
+ if (UNLIKELY(*ThreadLongPtr == 0))
+ return nullptr;
+ auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
+ return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
+}
+
} // namespace __hwasan
using namespace __hwasan;
static void InitInstrumentation() {
if (hwasan_instrumentation_inited) return;
- InitPrctl();
+ InitializeOsSupport();
if (!InitShadow()) {
Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
}
InitThreads();
- hwasanThreadList().CreateCurrentThread();
hwasan_instrumentation_inited = 1;
}
InitializeFlags();
// Install tool-specific callbacks in sanitizer_common.
- SetCheckFailedCallback(HWAsanCheckFailed);
+ SetCheckUnwindCallback(CheckUnwind);
__sanitizer_set_report_path(common_flags()->log_path);
// initialized when InitInstrumentation() was called.
GetCurrentThread()->InitRandomState();
- MadviseShadow();
-
SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
// This may call libc -> needs initialized shadow.
AndroidLogInit();
}
void __hwasan_print_memory_usage() {
- InternalScopedString s(kMemoryUsageBufferSize);
+ InternalScopedString s;
HwasanFormatMemoryUsage(s);
Printf("%s\n", s.data());
}
-static const u8 kFallbackTag = 0xBB;
+static const u8 kFallbackTag = 0xBB & kTagMask;
u8 __hwasan_generate_tag() {
Thread *t = GetCurrentThread();
GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
stack.Print();
}
+
+// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
+// rest of the mismatch handling code (C++).
+void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize) {
+ __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize);
+}
+
} // extern "C"
#ifndef HWASAN_H
#define HWASAN_H
+#include "hwasan_flags.h"
+#include "hwasan_interface_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
-#include "hwasan_interface_internal.h"
-#include "hwasan_flags.h"
#include "ubsan/ubsan_platform.h"
-#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
-# define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE 1
-#endif
-
#ifndef HWASAN_CONTAINS_UBSAN
# define HWASAN_CONTAINS_UBSAN CAN_SANITIZE_UB
#endif
#define HWASAN_WITH_INTERCEPTORS 0
#endif
+#ifndef HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+#define HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE HWASAN_WITH_INTERCEPTORS
+#endif
+
typedef u8 tag_t;
+#if defined(HWASAN_ALIASING_MODE)
+# if !defined(__x86_64__)
+# error Aliasing mode is only supported on x86_64
+# endif
+// Tags are done in middle bits using userspace aliasing.
+constexpr unsigned kAddressTagShift = 39;
+constexpr unsigned kTagBits = 3;
+
+// The alias region is placed next to the shadow so the upper bits of all
+// taggable addresses matches the upper bits of the shadow base. This shift
+// value determines which upper bits must match. It has a floor of 44 since the
+// shadow is always 8TB.
+// TODO(morehouse): In alias mode we can shrink the shadow and use a
+// simpler/faster shadow calculation.
+constexpr unsigned kTaggableRegionCheckShift =
+ __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#elif defined(__x86_64__)
+// Tags are done in upper bits using Intel LAM.
+constexpr unsigned kAddressTagShift = 57;
+constexpr unsigned kTagBits = 6;
+#else
// TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
// translation and can be used to store a tag.
-const unsigned kAddressTagShift = 56;
-const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+constexpr unsigned kAddressTagShift = 56;
+constexpr unsigned kTagBits = 8;
+#endif // defined(HWASAN_ALIASING_MODE)
+
+// Mask for extracting tag bits from the lower 8 bits.
+constexpr uptr kTagMask = (1UL << kTagBits) - 1;
+
+// Mask for extracting tag bits from full pointers.
+constexpr uptr kAddressTagMask = kTagMask << kAddressTagShift;
// Minimal alignment of the shadow base address. Determines the space available
// for threads and stack histories. This is an ABI constant.
const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
static inline tag_t GetTagFromPointer(uptr p) {
- return p >> kAddressTagShift;
+ return (p >> kAddressTagShift) & kTagMask;
}
static inline uptr UntagAddr(uptr tagged_addr) {
extern int hwasan_report_count;
bool InitShadow();
-void InitPrctl();
+void InitializeOsSupport();
void InitThreads();
-void MadviseShadow();
void InitializeInterceptors();
void HwasanAllocatorInit();
if (hwasan_inited) \
stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal)
-#define GET_FATAL_STACK_TRACE_HERE \
- GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
-
-#define PRINT_CURRENT_STACK_CHECK() \
- { \
- GET_FATAL_STACK_TRACE_HERE; \
- stack.Print(); \
- }
-
void HwasanTSDInit();
void HwasanTSDThreadInit();
+void HwasanAtExit();
void HwasanOnDeadlySignal(int signo, void *info, void *context);
void AndroidTestTlsSlot();
+// This is a compiler-generated struct that can be shared between hwasan
+// implementations.
+struct AccessInfo {
+ uptr addr;
+ uptr size;
+ bool is_store;
+ bool is_load;
+ bool recover;
+};
+
+// Given access info and frame information, unwind the stack and report the tag
+// mismatch.
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+ uptr *registers_frame = nullptr);
+
+// This dispatches to HandleTagMismatch but sets up the AccessInfo, program
+// counter, and frame pointer.
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+ size_t outsize);
+
} // namespace __hwasan
#define HWASAN_MALLOC_HOOK(ptr, size) \
typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1];
#endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
+#define ENSURE_HWASAN_INITED() \
+ do { \
+ CHECK(!hwasan_init_is_running); \
+ if (!hwasan_inited) { \
+ __hwasan_init(); \
+ } \
+ } while (0)
+
#endif // HWASAN_H
--- /dev/null
+//===-- hwasan_allocation_functions.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Definitions for __sanitizer allocation functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+using namespace __hwasan;
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ CHECK_NE(memptr, 0);
+ int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+ return res;
+}
+
+void *__sanitizer_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_memalign(alignment, size, &stack);
+}
+
+void *__sanitizer_aligned_alloc(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+void *__sanitizer___libc_memalign(uptr alignment, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ void *ptr = hwasan_memalign(alignment, size, &stack);
+ if (ptr)
+ DTLS_on_libc_memalign(ptr, size);
+ return ptr;
+}
+
+void *__sanitizer_valloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_valloc(size, &stack);
+}
+
+void *__sanitizer_pvalloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_pvalloc(size, &stack);
+}
+
+void __sanitizer_free(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ hwasan_free(ptr, &stack);
+}
+
+void __sanitizer_cfree(void *ptr) {
+ GET_MALLOC_STACK_TRACE;
+ if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ hwasan_free(ptr, &stack);
+}
+
+uptr __sanitizer_malloc_usable_size(const void *ptr) {
+ return __sanitizer_get_allocated_size(ptr);
+}
+
+struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
+ __sanitizer_struct_mallinfo sret;
+ internal_memset(&sret, 0, sizeof(sret));
+ return sret;
+}
+
+int __sanitizer_mallopt(int cmd, int value) { return 0; }
+
+void __sanitizer_malloc_stats(void) {
+ // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+
+void *__sanitizer_calloc(uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ return hwasan_calloc(nmemb, size, &stack);
+}
+
+void *__sanitizer_realloc(void *ptr, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(!hwasan_inited)) {
+ new_ptr = AllocateFromLocalPool(copy_size);
+ } else {
+ copy_size = size;
+ new_ptr = hwasan_malloc(copy_size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+ }
+ return hwasan_realloc(ptr, size, &stack);
+}
+
+void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ return hwasan_reallocarray(ptr, nmemb, size, &stack);
+}
+
+void *__sanitizer_malloc(uptr size) {
+ GET_MALLOC_STACK_TRACE;
+ if (UNLIKELY(!hwasan_init_is_running))
+ ENSURE_HWASAN_INITED();
+ if (UNLIKELY(!hwasan_inited))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ return hwasan_malloc(size, &stack);
+}
+
+#if HWASAN_WITH_INTERCEPTORS
+# define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
+ ALIAS("__sanitizer_" #FN); \
+ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
+ ARGS) ALIAS("__sanitizer_" #FN)
+
+INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
+ SIZE_T size);
+INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, free, void *ptr);
+INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
+INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
+
+# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, cfree, void *ptr);
+INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
+INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
+INTERCEPTOR_ALIAS(void, malloc_stats, void);
+# endif
+#endif // #if HWASAN_WITH_INTERCEPTORS
static SpinMutex fallback_mutex;
static atomic_uint8_t hwasan_allocator_tagging_enabled;
-static const tag_t kFallbackAllocTag = 0xBB;
-static const tag_t kFallbackFreeTag = 0xBC;
+static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
+static constexpr tag_t kFallbackFreeTag = 0xBC;
enum RightAlignMode {
kRightAlignNever,
static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1];
bool HwasanChunkView::IsAllocated() const {
- return metadata_ && metadata_->alloc_context_id && metadata_->requested_size;
+ return metadata_ && metadata_->alloc_context_id &&
+ metadata_->get_requested_size();
}
// Aligns the 'addr' right to the granule boundary.
uptr HwasanChunkView::Beg() const {
if (metadata_ && metadata_->right_aligned)
- return AlignRight(block_, metadata_->requested_size);
+ return AlignRight(block_, metadata_->get_requested_size());
return block_;
}
uptr HwasanChunkView::End() const {
return Beg() + UsedSize();
}
uptr HwasanChunkView::UsedSize() const {
- return metadata_->requested_size;
+ return metadata_->get_requested_size();
}
u32 HwasanChunkView::GetAllocStackId() const {
return metadata_->alloc_context_id;
allocator.GetStats(s);
}
+uptr GetAliasRegionStart() {
+#if defined(HWASAN_ALIASING_MODE)
+ constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
+ uptr AliasRegionStart =
+ __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
+
+ CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift,
+ __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+ CHECK_EQ(
+ (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
+ __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+ return AliasRegionStart;
+#else
+ return 0;
+#endif
+}
+
void HwasanAllocatorInit() {
atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
!flags()->disable_allocator_tagging);
SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
- allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+ allocator.Init(common_flags()->allocator_release_to_os_interval_ms,
+ GetAliasRegionStart());
for (uptr i = 0; i < sizeof(tail_magic); i++)
tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
}
}
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
- meta->requested_size = static_cast<u32>(orig_size);
+ meta->set_requested_size(orig_size);
meta->alloc_context_id = StackDepotPut(*stack);
meta->right_aligned = false;
if (zeroise) {
// Tagging can only be skipped when both tag_in_malloc and tag_in_free are
// false. When tag_in_malloc = false and tag_in_free = true malloc needs to
// retag to 0.
- if ((flags()->tag_in_malloc || flags()->tag_in_free) &&
+ if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) &&
+ (flags()->tag_in_malloc || flags()->tag_in_free) &&
atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
CHECK(tagged_ptr);
uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
+ if (!InTaggableRegion(tagged_uptr))
+ return true;
tag_t mem_tag = *reinterpret_cast<tag_t *>(
MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
if (!PointerAndMemoryTagsMatch(tagged_ptr))
ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
- void *untagged_ptr = UntagPtr(tagged_ptr);
+ void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))
+ ? UntagPtr(tagged_ptr)
+ : tagged_ptr;
void *aligned_ptr = reinterpret_cast<void *>(
RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
+ tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
- uptr orig_size = meta->requested_size;
+ uptr orig_size = meta->get_requested_size();
u32 free_context_id = StackDepotPut(*stack);
u32 alloc_context_id = meta->alloc_context_id;
orig_size, tail_magic);
}
- meta->requested_size = 0;
+ meta->set_requested_size(0);
meta->alloc_context_id = 0;
// This memory will not be reused by anyone else, so we are free to keep it
// poisoned.
Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
}
- if (flags()->tag_in_free && malloc_bisect(stack, 0) &&
- atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+ if (InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) &&
+ flags()->tag_in_free && malloc_bisect(stack, 0) &&
+ atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
+ // Always store full 8-bit tags on free to maximize UAF detection.
+ tag_t tag;
+ if (t) {
+ // Make sure we are not using a short granule tag as a poison tag. This
+ // would make us attempt to read the memory on a UaF.
+ // The tag can be zero if tagging is disabled on this thread.
+ do {
+ tag = t->GenerateRandomTag(/*num_bits=*/8);
+ } while (
+ UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0));
+ } else {
+ static_assert(kFallbackFreeTag >= kShadowAlignment,
+ "fallback tag must not be a short granule tag.");
+ tag = kFallbackFreeTag;
+ }
TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
- t ? t->GenerateRandomTag() : kFallbackFreeTag);
+ tag);
+ }
if (t) {
allocator.Deallocate(t->allocator_cache(), aligned_ptr);
if (auto *ha = t->heap_allocations())
void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
Metadata *meta =
reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
- internal_memcpy(UntagPtr(tagged_ptr_new), untagged_ptr_old,
- Min(new_size, static_cast<uptr>(meta->requested_size)));
+ internal_memcpy(
+ UntagPtr(tagged_ptr_new), untagged_ptr_old,
+ Min(new_size, static_cast<uptr>(meta->get_requested_size())));
HwasanDeallocate(stack, tagged_ptr_old);
}
return tagged_ptr_new;
} else {
if (beg != untagged_ptr) return 0;
}
- return b->requested_size;
+ return b->get_requested_size();
}
void *hwasan_malloc(uptr size, StackTrace *stack) {
// OOM error is already taken care of by HwasanAllocate.
return errno_ENOMEM;
CHECK(IsAligned((uptr)ptr, alignment));
- *(void **)UntagPtr(memptr) = ptr;
+ *memptr = ptr;
return 0;
}
#ifndef HWASAN_ALLOCATOR_H
#define HWASAN_ALLOCATOR_H
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
+#include "hwasan_poisoning.h"
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_ring_buffer.h"
-#include "hwasan_poisoning.h"
#if !defined(__aarch64__) && !defined(__x86_64__)
#error Unsupported platform
namespace __hwasan {
struct Metadata {
- u32 requested_size : 31; // sizes are < 2G.
- u32 right_aligned : 1;
+ u32 requested_size_low;
+ u32 requested_size_high : 31;
+ u32 right_aligned : 1;
u32 alloc_context_id;
+ u64 get_requested_size() {
+ return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
+ }
+ void set_requested_size(u64 size) {
+ requested_size_low = size & ((1ul << 32) - 1);
+ requested_size_high = size >> 32;
+ }
};
struct HwasanMapUnmapCallback {
}
};
-static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G
+static const uptr kMaxAllowedMallocSize = 1UL << 40; // 1T
struct AP64 {
static const uptr kSpaceBeg = ~0ULL;
+
+#if defined(HWASAN_ALIASING_MODE)
+ static const uptr kSpaceSize = 1ULL << kAddressTagShift;
+#else
static const uptr kSpaceSize = 0x2000000000ULL;
+#endif
static const uptr kMetadataSize = sizeof(Metadata);
typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap;
using AddressSpaceView = LocalAddressSpaceView;
void GetAllocatorStats(AllocatorStatCounters s);
+inline bool InTaggableRegion(uptr addr) {
+#if defined(HWASAN_ALIASING_MODE)
+ // Aliases are mapped next to shadow so that the upper bits match the shadow
+ // base.
+ return (addr >> kTaggableRegionCheckShift) ==
+ (GetShadowOffset() >> kTaggableRegionCheckShift);
+#endif
+ return true;
+}
+
} // namespace __hwasan
#endif // HWASAN_ALLOCATOR_H
#ifndef HWASAN_CHECKS_H
#define HWASAN_CHECKS_H
+#include "hwasan_allocator.h"
#include "hwasan_mapping.h"
#include "sanitizer_common/sanitizer_common.h"
template <ErrorAction EA, AccessType AT, unsigned LogSize>
__attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
+ if (!InTaggableRegion(p))
+ return;
uptr ptr_raw = p & ~kAddressTagMask;
tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw);
if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) {
template <ErrorAction EA, AccessType AT>
__attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
uptr sz) {
- if (sz == 0)
+ if (sz == 0 || !InTaggableRegion(p))
return;
tag_t ptr_tag = GetTagFromPointer(p);
uptr ptr_raw = p & ~kAddressTagMask;
///
//===----------------------------------------------------------------------===//
-#include "hwasan.h"
#include "hwasan_dynamic_shadow.h"
-#include "hwasan_mapping.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_posix.h"
#include <elf.h>
#include <link.h>
+#include "hwasan.h"
+#include "hwasan_mapping.h"
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
// The code in this file needs to run in an unrelocated binary. It should not
// access any external symbol, including its own non-hidden globals.
-namespace __hwasan {
-
-static void UnmapFromTo(uptr from, uptr to) {
- if (to == from)
- return;
- CHECK(to >= from);
- uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
- if (UNLIKELY(internal_iserror(res))) {
- Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
- SanitizerToolName, to - from, to - from, from);
- CHECK("unable to unmap" && 0);
- }
-}
-
-// Returns an address aligned to kShadowBaseAlignment, such that
-// 2**kShadowBaseAlingment on the left and shadow_size_bytes bytes on the right
-// of it are mapped no access.
-static uptr MapDynamicShadow(uptr shadow_size_bytes) {
- const uptr granularity = GetMmapGranularity();
- const uptr min_alignment = granularity << kShadowScale;
- const uptr alignment = 1ULL << kShadowBaseAlignment;
- CHECK_GE(alignment, min_alignment);
-
- const uptr left_padding = 1ULL << kShadowBaseAlignment;
- const uptr shadow_size =
- RoundUpTo(shadow_size_bytes, granularity);
- const uptr map_size = shadow_size + left_padding + alignment;
-
- const uptr map_start = (uptr)MmapNoAccess(map_size);
- CHECK_NE(map_start, ~(uptr)0);
-
- const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
-
- UnmapFromTo(map_start, shadow_start - left_padding);
- UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
-
- return shadow_start;
-}
-
-} // namespace __hwasan
-
#if SANITIZER_ANDROID
extern "C" {
}
static uptr PremapShadow() {
- return MapDynamicShadow(PremapShadowSize());
+ return MapDynamicShadow(PremapShadowSize(), kShadowScale,
+ kShadowBaseAlignment, kHighMemEnd);
}
static bool IsPremapShadowAvailable() {
uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
if (IsPremapShadowAvailable())
return FindPremappedShadowStart(shadow_size_bytes);
- return MapDynamicShadow(shadow_size_bytes);
+ return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
+ kHighMemEnd);
}
} // namespace __hwasan
+
+#elif SANITIZER_FUCHSIA
+
+namespace __hwasan {
+
+void InitShadowGOT() {}
+
+} // namespace __hwasan
+
#else
namespace __hwasan {
void InitShadowGOT() {}
uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
- return MapDynamicShadow(shadow_size_bytes);
+# if defined(HWASAN_ALIASING_MODE)
+ constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
+ constexpr uptr kNumAliases = 1ULL << kTagBits;
+ return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
+ RingBufferSize());
+# endif
+ return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
+ kHighMemEnd);
}
} // namespace __hwasan
#ifndef HWASAN_FLAGS_H
#define HWASAN_FLAGS_H
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
namespace __hwasan {
struct Flags {
HWASAN_FLAG(bool, malloc_bisect_dump, false,
"Print all allocations within [malloc_bisect_left, "
"malloc_bisect_right] range ")
+
+
+// Exit if we fail to enable the AArch64 kernel ABI relaxation which allows
+// tagged pointers in syscalls. This is the default, but being able to disable
+// that behaviour is useful for running the testsuite on more platforms (the
+// testsuite can run since we manually ensure any pointer arguments to syscalls
+// are untagged before the call.
+HWASAN_FLAG(bool, fail_without_syscall_abi, true,
+ "Exit if fail to request relaxed syscall ABI.")
--- /dev/null
+//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific
+/// code.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+
+// This TLS variable contains the location of the stack ring buffer and can be
+// used to always find the hwasan thread object associated with the current
+// running thread.
+[[gnu::tls_model("initial-exec")]]
+SANITIZER_INTERFACE_ATTRIBUTE
+THREADLOCAL uptr __hwasan_tls;
+
+namespace __hwasan {
+
+bool InitShadow() {
+ __sanitizer::InitShadowBounds();
+ CHECK_NE(__sanitizer::ShadowBounds.shadow_limit, 0);
+
+ // These variables are used by MemIsShadow for asserting we have a correct
+ // shadow address. On Fuchsia, we only have one region of shadow, so the
+ // bounds of Low shadow can be zero while High shadow represents the true
+ // bounds. Note that these are inclusive ranges.
+ kLowShadowStart = 0;
+ kLowShadowEnd = 0;
+ kHighShadowStart = __sanitizer::ShadowBounds.shadow_base;
+ kHighShadowEnd = __sanitizer::ShadowBounds.shadow_limit - 1;
+
+ return true;
+}
+
+bool MemIsApp(uptr p) {
+ CHECK(GetTagFromPointer(p) == 0);
+ return __sanitizer::ShadowBounds.shadow_limit <= p &&
+ p <= (__sanitizer::ShadowBounds.memory_limit - 1);
+}
+
+// These are known parameters passed to the hwasan runtime on thread creation.
+struct Thread::InitState {
+ uptr stack_bottom, stack_top;
+};
+
+static void FinishThreadInitialization(Thread *thread);
+
+void InitThreads() {
+ // This is the minimal alignment needed for the storage where hwasan threads
+ // and their stack ring buffers are placed. This alignment is necessary so the
+ // stack ring buffer can perform a simple calculation to get the next element
+ // in the RB. The instructions for this calculation are emitted by the
+ // compiler. (Full explanation in hwasan_thread_list.h.)
+ uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment;
+ uptr thread_start = reinterpret_cast<uptr>(
+ MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__));
+
+ InitThreadList(thread_start, alloc_size);
+
+ // Create the hwasan thread object for the current (main) thread. Stack info
+ // for this thread is known from information passed via
+ // __sanitizer_startup_hook.
+ const Thread::InitState state = {
+ .stack_bottom = __sanitizer::MainThreadStackBase,
+ .stack_top =
+ __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize,
+ };
+ FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&state));
+}
+
+uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
+
+// This is called from the parent thread before the new thread is created. Here
+// we can propagate known info like the stack bounds to Thread::Init before
+// jumping into the thread. We cannot initialize the stack ring buffer yet since
+// we have not entered the new thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+ const char *name, uptr stack_bottom,
+ uptr stack_size) {
+ const Thread::InitState state = {
+ .stack_bottom = stack_bottom,
+ .stack_top = stack_bottom + stack_size,
+ };
+ return hwasanThreadList().CreateCurrentThread(&state);
+}
+
+// This sets the stack top and bottom according to the InitState passed to
+// CreateCurrentThread above.
+void Thread::InitStackAndTls(const InitState *state) {
+ CHECK_NE(state->stack_bottom, 0);
+ CHECK_NE(state->stack_top, 0);
+ stack_bottom_ = state->stack_bottom;
+ stack_top_ = state->stack_top;
+ tls_end_ = tls_begin_ = 0;
+}
+
+// This is called after creating a new thread with the pointer returned by
+// BeforeThreadCreateHook. We are still in the creating thread and should check
+// if it was actually created correctly.
+static void ThreadCreateHook(void *hook, bool aborted) {
+ Thread *thread = static_cast<Thread *>(hook);
+ if (!aborted) {
+ // The thread was created successfully.
+ // ThreadStartHook can already be running in the new thread.
+ } else {
+ // The thread wasn't created after all.
+ // Clean up everything we set up in BeforeThreadCreateHook.
+ atomic_signal_fence(memory_order_seq_cst);
+ hwasanThreadList().ReleaseThread(thread);
+ }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above). Here we can
+// setup the stack ring buffer.
+static void ThreadStartHook(void *hook, thrd_t self) {
+ Thread *thread = static_cast<Thread *>(hook);
+ FinishThreadInitialization(thread);
+ thread->InitRandomState();
+}
+
+// This is the function that sets up the stack ring buffer and enables us to use
+// GetCurrentThread. This function should only be called while IN the thread
+// that we want to create the hwasan thread object for so __hwasan_tls can be
+// properly referenced.
+static void FinishThreadInitialization(Thread *thread) {
+ CHECK_NE(thread, nullptr);
+
+ // The ring buffer is located immediately before the thread object.
+ uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize();
+ uptr stack_buffer_start = reinterpret_cast<uptr>(thread) - stack_buffer_size;
+ thread->InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+}
+
+static void ThreadExitHook(void *hook, thrd_t self) {
+ Thread *thread = static_cast<Thread *>(hook);
+ atomic_signal_fence(memory_order_seq_cst);
+ hwasanThreadList().ReleaseThread(thread);
+}
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ __sanitizer_fill_shadow(p, size, tag,
+ common_flags()->clear_shadow_mmap_threshold);
+ return AddTagToPointer(p, tag);
+}
+
+// Not implemented because Fuchsia does not use signal handlers.
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {}
+
+// Not implemented because Fuchsia does not use interceptors.
+void InitializeInterceptors() {}
+
+// Not implemented because this is only relevant for Android.
+void AndroidTestTlsSlot() {}
+
+// TSD was normally used on linux as a means of calling the hwasan thread exit
+// handler passed to pthread_key_create. This is not needed on Fuchsia because
+// we will be using __sanitizer_thread_exit_hook.
+void HwasanTSDInit() {}
+void HwasanTSDThreadInit() {}
+
+// On linux, this just would call `atexit(HwasanAtExit)`. The functions in
+// HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this
+// function is unneeded.
+void InstallAtExitHandler() {}
+
+// TODO(fxbug.dev/81499): Once we finalize the tagged pointer ABI in zircon, we should come back
+// here and implement the appropriate check that TBI is enabled.
+void InitializeOsSupport() {}
+
+} // namespace __hwasan
+
+extern "C" {
+
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+ const char *name, void *stack_base,
+ size_t stack_size) {
+ return __hwasan::BeforeThreadCreateHook(
+ reinterpret_cast<uptr>(thread), detached, name,
+ reinterpret_cast<uptr>(stack_base), stack_size);
+}
+
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+ __hwasan::ThreadCreateHook(hook, error != thrd_success);
+}
+
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+ __hwasan::ThreadStartHook(hook, reinterpret_cast<uptr>(self));
+}
+
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) {
+ __hwasan::ThreadExitHook(hook, self);
+}
+
+} // extern "C"
+
+#endif // SANITIZER_FUCHSIA
--- /dev/null
+# Ignorelist for HWAddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of ignorelist
+# at compile-time using -fsanitize-ignorelist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
#include "interception/interception.h"
#include "hwasan.h"
-#include "hwasan_allocator.h"
-#include "hwasan_mapping.h"
#include "hwasan_thread.h"
-#include "hwasan_poisoning.h"
-#include "hwasan_report.h"
-#include "sanitizer_common/sanitizer_platform_limits_posix.h"
-#include "sanitizer_common/sanitizer_allocator.h"
-#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
-#include "sanitizer_common/sanitizer_atomic.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include <stdarg.h>
-// ACHTUNG! No other system header includes in this file.
-// Ideally, we should get rid of stdarg.h as well.
+#if !SANITIZER_FUCHSIA
using namespace __hwasan;
-using __sanitizer::memory_order;
-using __sanitizer::atomic_load;
-using __sanitizer::atomic_store;
-using __sanitizer::atomic_uintptr_t;
-
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
- uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
- uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
- void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
- allocated_for_dlsym += size_in_words;
- CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
- return mem;
-}
-
-#define ENSURE_HWASAN_INITED() do { \
- CHECK(!hwasan_init_is_running); \
- if (!hwasan_inited) { \
- __hwasan_init(); \
- } \
-} while (0)
-
-
-int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- CHECK_NE(memptr, 0);
- int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
- return res;
-}
-
-void * __sanitizer_memalign(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_memalign(alignment, size, &stack);
-}
-
-void * __sanitizer_aligned_alloc(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_aligned_alloc(alignment, size, &stack);
-}
-
-void * __sanitizer___libc_memalign(uptr alignment, uptr size) {
- GET_MALLOC_STACK_TRACE;
- void *ptr = hwasan_memalign(alignment, size, &stack);
- if (ptr)
- DTLS_on_libc_memalign(ptr, size);
- return ptr;
-}
-
-void * __sanitizer_valloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_valloc(size, &stack);
-}
-
-void * __sanitizer_pvalloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_pvalloc(size, &stack);
-}
-
-void __sanitizer_free(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
- hwasan_free(ptr, &stack);
-}
-
-void __sanitizer_cfree(void *ptr) {
- GET_MALLOC_STACK_TRACE;
- if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
- hwasan_free(ptr, &stack);
-}
-
-uptr __sanitizer_malloc_usable_size(const void *ptr) {
- return __sanitizer_get_allocated_size(ptr);
-}
-
-struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
- __sanitizer_struct_mallinfo sret;
- internal_memset(&sret, 0, sizeof(sret));
- return sret;
-}
-
-int __sanitizer_mallopt(int cmd, int value) {
- return 0;
-}
-
-void __sanitizer_malloc_stats(void) {
- // FIXME: implement, but don't call REAL(malloc_stats)!
-}
-
-void * __sanitizer_calloc(uptr nmemb, uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
- return AllocateFromLocalPool(nmemb * size);
- return hwasan_calloc(nmemb, size, &stack);
-}
-
-void * __sanitizer_realloc(void *ptr, uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
- uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
- uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
- void *new_ptr;
- if (UNLIKELY(!hwasan_inited)) {
- new_ptr = AllocateFromLocalPool(copy_size);
- } else {
- copy_size = size;
- new_ptr = hwasan_malloc(copy_size, &stack);
- }
- internal_memcpy(new_ptr, ptr, copy_size);
- return new_ptr;
- }
- return hwasan_realloc(ptr, size, &stack);
-}
-
-void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
- GET_MALLOC_STACK_TRACE;
- return hwasan_reallocarray(ptr, nmemb, size, &stack);
-}
-
-void * __sanitizer_malloc(uptr size) {
- GET_MALLOC_STACK_TRACE;
- if (UNLIKELY(!hwasan_init_is_running))
- ENSURE_HWASAN_INITED();
- if (UNLIKELY(!hwasan_inited))
- // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
- return AllocateFromLocalPool(size);
- return hwasan_malloc(size, &stack);
-}
-
#if HWASAN_WITH_INTERCEPTORS
-#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \
- ALIAS("__sanitizer_" #FN); \
- extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
- ARGS) ALIAS("__sanitizer_" #FN)
-
-INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
- SIZE_T size);
-INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, free, void *ptr);
-INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
-INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
-
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
-INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, cfree, void *ptr);
-INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
-INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
-INTERCEPTOR_ALIAS(void, malloc_stats, void);
-#endif
struct ThreadStartArg {
thread_callback_t callback;
ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
GetPageSizeCached(), "pthread_create"));
*A = {callback, param};
- int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr),
- &HwasanThreadStartFunc, A);
+ int res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
return res;
}
inited = 1;
}
} // namespace __hwasan
+
+#endif // #if !SANITIZER_FUCHSIA
#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
#if defined(__linux__) && HWASAN_WITH_INTERCEPTORS
#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area
#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork
#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S"
#include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S"
#endif
NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
void *__hwasan_memset(void *s, int c, uptr n);
SANITIZER_INTERFACE_ATTRIBUTE
void *__hwasan_memmove(void *dest, const void *src, uptr n);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_set_error_report_callback(void (*callback)(const char *));
} // extern "C"
#endif // HWASAN_INTERFACE_INTERNAL_H
namespace __hwasan {
-static void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name) {
- CHECK_EQ((beg % GetMmapGranularity()), 0);
- CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
- uptr size = end - beg + 1;
- DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
- if (!MmapFixedNoReserve(beg, size, name)) {
- Report(
- "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
- "Perhaps you're using ulimit -v\n",
- size);
- Abort();
- }
-}
+// With the zero shadow base we can not actually map pages starting from 0.
+// This constant is somewhat arbitrary.
+constexpr uptr kZeroBaseShadowStart = 0;
+constexpr uptr kZeroBaseMaxShadowStart = 1 << 18;
static void ProtectGap(uptr addr, uptr size) {
- if (!size)
- return;
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res)
- return;
- // A few pages at the start of the address space can not be protected.
- // But we really want to protect as much as possible, to prevent this memory
- // being returned as a result of a non-FIXED mmap().
- if (addr == 0) {
- uptr step = GetMmapGranularity();
- while (size > step) {
- addr += step;
- size -= step;
- void *res = MmapFixedNoAccess(addr, size, "shadow gap");
- if (addr == (uptr)res)
- return;
- }
- }
-
- Report(
- "ERROR: Failed to protect shadow gap [%p, %p]. "
- "HWASan cannot proceed correctly. ABORTING.\n", (void *)addr,
- (void *)(addr + size));
- DumpProcessMap();
- Die();
+ __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
+ kZeroBaseMaxShadowStart);
}
-static uptr kLowMemStart;
-static uptr kLowMemEnd;
-static uptr kLowShadowEnd;
-static uptr kLowShadowStart;
-static uptr kHighShadowStart;
-static uptr kHighShadowEnd;
-static uptr kHighMemStart;
-static uptr kHighMemEnd;
+uptr kLowMemStart;
+uptr kLowMemEnd;
+uptr kHighMemStart;
+uptr kHighMemEnd;
static void PrintRange(uptr start, uptr end, const char *name) {
Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
FindDynamicShadowStart(shadow_size_bytes);
}
-void InitPrctl() {
+void InitializeOsSupport() {
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_GET_TAGGED_ADDR_CTRL 56
#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
// Check we're running on a kernel that can use the tagged address ABI.
- if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) == (uptr)-1 &&
- errno == EINVAL) {
-#if SANITIZER_ANDROID
+ int local_errno = 0;
+ if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
+ &local_errno) &&
+ local_errno == EINVAL) {
+# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
// Some older Android kernels have the tagged pointer ABI on
// unconditionally, and hence don't have the tagged-addr prctl while still
// allow the ABI.
// If targeting Android and the prctl is not around we assume this is the
// case.
return;
-#else
- Printf(
- "FATAL: "
- "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
- Die();
-#endif
+# else
+ if (flags()->fail_without_syscall_abi) {
+ Printf(
+ "FATAL: "
+ "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
+ Die();
+ }
+# endif
}
// Turn on the tagged address ABI.
- if (internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) ==
- (uptr)-1 ||
- !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) {
- Printf(
- "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
- "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
- "configuration.\n");
- Die();
+ if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
+ !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
+# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ // Try the new prctl API for Intel LAM. The API is based on a currently
+ // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
+ // subject to change. Patch is here:
+ // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
+ int tag_bits = kTagBits;
+ int tag_shift = kAddressTagShift;
+ if (!internal_iserror(
+ internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
+ reinterpret_cast<unsigned long>(&tag_bits),
+ reinterpret_cast<unsigned long>(&tag_shift), 0))) {
+ CHECK_EQ(tag_bits, kTagBits);
+ CHECK_EQ(tag_shift, kAddressTagShift);
+ return;
+ }
+# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+ if (flags()->fail_without_syscall_abi) {
+ Printf(
+ "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+ "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+ "configuration.\n");
+ Die();
+ }
}
#undef PR_SET_TAGGED_ADDR_CTRL
#undef PR_GET_TAGGED_ADDR_CTRL
uptr thread_space_end =
__hwasan_shadow_memory_dynamic_address - guard_page_size;
ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
- "hwasan threads");
+ "hwasan threads", /*madvise_shadow*/ false);
ProtectGap(thread_space_end,
__hwasan_shadow_memory_dynamic_address - thread_space_end);
InitThreadList(thread_space_start, thread_space_end - thread_space_start);
-}
-
-static void MadviseShadowRegion(uptr beg, uptr end) {
- uptr size = end - beg + 1;
- SetShadowRegionHugePageMode(beg, size);
- if (common_flags()->use_madv_dontdump)
- DontDumpShadowMemory(beg, size);
-}
-
-void MadviseShadow() {
- MadviseShadowRegion(kLowShadowStart, kLowShadowEnd);
- MadviseShadowRegion(kHighShadowStart, kHighShadowEnd);
+ hwasanThreadList().CreateCurrentThread();
}
bool MemIsApp(uptr p) {
+// Memory outside the alias range has non-zero tags.
+# if !defined(HWASAN_ALIASING_MODE)
CHECK(GetTagFromPointer(p) == 0);
- return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
-}
+# endif
-static void HwasanAtExit(void) {
- if (common_flags()->print_module_map)
- DumpProcessMap();
- if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
- ReportStats();
- if (hwasan_report_count > 0) {
- // ReportAtExitStatistics();
- if (common_flags()->exitcode)
- internal__exit(common_flags()->exitcode);
- }
+ return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
}
void InstallAtExitHandler() {
void AndroidTestTlsSlot() {}
#endif
-Thread *GetCurrentThread() {
- uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
- if (UNLIKELY(*ThreadLongPtr == 0))
- return nullptr;
- auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
- return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
-}
-
-struct AccessInfo {
- uptr addr;
- uptr size;
- bool is_store;
- bool is_load;
- bool recover;
-};
-
static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
// Access type is passed in a platform dependent way (see below) and encoded
// as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
return AccessInfo{addr, size, is_store, !is_store, recover};
}
-static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame,
- ucontext_t *uc, uptr *registers_frame = nullptr) {
- InternalMmapVector<BufferedStackTrace> stack_buffer(1);
- BufferedStackTrace *stack = stack_buffer.data();
- stack->Reset();
- stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
-
- // The second stack frame contains the failure __hwasan_check function, as
- // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
- // we wish to ignore. This (currently) only occurs on AArch64, as x64
- // implementations use SIGTRAP to implement the failure, and thus do not go
- // through the stack saver.
- if (registers_frame && stack->trace && stack->size > 0) {
- stack->trace++;
- stack->size--;
- }
-
- bool fatal = flags()->halt_on_error || !ai.recover;
- ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
- registers_frame);
-}
-
static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
AccessInfo ai = GetAccessInfo(info, uc);
if (!ai.is_store && !ai.is_load)
HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
}
+void Thread::InitStackAndTls(const InitState *) {
+ uptr tls_size;
+ uptr stack_size;
+ GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
+ &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+}
-} // namespace __hwasan
-
-// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
-// rest of the mismatch handling code (C++).
-void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
- size_t outsize) {
- __hwasan::AccessInfo ai;
- ai.is_store = access_info & 0x10;
- ai.is_load = !ai.is_store;
- ai.recover = access_info & 0x20;
- ai.addr = addr;
- if ((access_info & 0xf) == 0xf)
- ai.size = outsize;
- else
- ai.size = 1 << (access_info & 0xf);
-
- __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
- (uptr)__builtin_frame_address(0), nullptr,
- registers_frame);
- __builtin_unreachable();
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+ CHECK(IsAligned(p, kShadowAlignment));
+ CHECK(IsAligned(size, kShadowAlignment));
+ uptr shadow_start = MemToShadow(p);
+ uptr shadow_size = MemToShadowSize(size);
+
+ uptr page_size = GetPageSizeCached();
+ uptr page_start = RoundUpTo(shadow_start, page_size);
+ uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+ uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+ if (SANITIZER_LINUX &&
+ UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+ internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+ internal_memset((void *)page_end, tag,
+ shadow_start + shadow_size - page_end);
+ // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+ // Linux.
+ ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+ } else {
+ internal_memset((void *)shadow_start, tag, shadow_size);
+ }
+ return AddTagToPointer(p, tag);
}
+} // namespace __hwasan
+
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
return H.get();
}
-static INLINE bool malloc_bisect(StackTrace *stack, uptr orig_size) {
+static inline bool malloc_bisect(StackTrace *stack, uptr orig_size) {
uptr left = flags()->malloc_bisect_left;
uptr right = flags()->malloc_bisect_right;
if (LIKELY(left == 0 && right == 0))
namespace __hwasan {
+extern uptr kLowMemStart;
+extern uptr kLowMemEnd;
+extern uptr kLowShadowEnd;
+extern uptr kLowShadowStart;
+extern uptr kHighShadowStart;
+extern uptr kHighShadowEnd;
+extern uptr kHighMemStart;
+extern uptr kHighMemEnd;
+
+inline uptr GetShadowOffset() {
+ return SANITIZER_FUCHSIA ? 0 : __hwasan_shadow_memory_dynamic_address;
+}
inline uptr MemToShadow(uptr untagged_addr) {
- return (untagged_addr >> kShadowScale) +
- __hwasan_shadow_memory_dynamic_address;
+ return (untagged_addr >> kShadowScale) + GetShadowOffset();
}
inline uptr ShadowToMem(uptr shadow_addr) {
- return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale;
+ return (shadow_addr - GetShadowOffset()) << kShadowScale;
}
inline uptr MemToShadowSize(uptr size) {
return size >> kShadowScale;
bool MemIsApp(uptr p);
+inline bool MemIsShadow(uptr p) {
+ return (kLowShadowStart <= p && p <= kLowShadowEnd) ||
+ (kHighShadowStart <= p && p <= kHighShadowEnd);
+}
+
+uptr GetAliasRegionStart();
+
} // namespace __hwasan
#endif // HWASAN_MAPPING_H
void *__hwasan_memset(void *block, int c, uptr size) {
CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
reinterpret_cast<uptr>(block), size);
- return memset(UntagPtr(block), c, size);
+ return memset(block, c, size);
}
void *__hwasan_memcpy(void *to, const void *from, uptr size) {
reinterpret_cast<uptr>(to), size);
CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
reinterpret_cast<uptr>(from), size);
- return memcpy(UntagPtr(to), UntagPtr(from), size);
+ return memcpy(to, from, size);
}
void *__hwasan_memmove(void *to, const void *from, uptr size) {
#include "sanitizer_common/sanitizer_allocator.h"
#include "sanitizer_common/sanitizer_allocator_report.h"
+#include <stddef.h>
+#include <stdlib.h>
+
#if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
-#include <stddef.h>
+// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
+#define OPERATOR_NEW_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_malloc(size, &stack);\
+ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
+ return res
+#define OPERATOR_NEW_ALIGN_BODY(nothrow) \
+ GET_MALLOC_STACK_TRACE; \
+ void *res = hwasan_aligned_alloc(static_cast<uptr>(align), size, &stack); \
+ if (!nothrow && UNLIKELY(!res)) \
+ ReportOutOfMemory(size, &stack); \
+ return res
+
+#define OPERATOR_DELETE_BODY \
+ GET_MALLOC_STACK_TRACE; \
+ if (ptr) hwasan_free(ptr, &stack)
+
+#elif defined(__ANDROID__)
+
+// We don't actually want to intercept operator new and delete on Android, but
+// since we previously released a runtime that intercepted these functions,
+// removing the interceptors would break ABI. Therefore we simply forward to
+// malloc and free.
+#define OPERATOR_NEW_BODY(nothrow) return malloc(size)
+#define OPERATOR_DELETE_BODY free(ptr)
+
+#endif
+
+#ifdef OPERATOR_NEW_BODY
using namespace __hwasan;
} // namespace std
-// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
-#define OPERATOR_NEW_BODY(nothrow) \
- GET_MALLOC_STACK_TRACE; \
- void *res = hwasan_malloc(size, &stack);\
- if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
- return res
INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
OPERATOR_NEW_BODY(true /*nothrow*/);
}
-#define OPERATOR_DELETE_BODY \
- GET_MALLOC_STACK_TRACE; \
- if (ptr) hwasan_free(ptr, &stack)
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr)
+ NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+ void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY;
+}
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&) {
+#endif // OPERATOR_NEW_BODY
+
+#ifdef OPERATOR_NEW_ALIGN_BODY
+
+namespace std {
+enum class align_val_t : size_t {};
+} // namespace std
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
+ size_t size, std::align_val_t align) {
+ OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+ size_t size, std::align_val_t align) {
+ OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
+ size_t size, std::align_val_t align, std::nothrow_t const &) {
+ OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+ size_t size, std::align_val_t align, std::nothrow_t const &) {
+ OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+}
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+ void *ptr, std::align_val_t align) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr, std::align_val_t) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+ void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
+ OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+ void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
OPERATOR_DELETE_BODY;
}
-#endif // HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE
+#endif // OPERATOR_NEW_ALIGN_BODY
namespace __hwasan {
-uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
- CHECK(IsAligned(p, kShadowAlignment));
- CHECK(IsAligned(size, kShadowAlignment));
- uptr shadow_start = MemToShadow(p);
- uptr shadow_size = MemToShadowSize(size);
-
- uptr page_size = GetPageSizeCached();
- uptr page_start = RoundUpTo(shadow_start, page_size);
- uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
- uptr threshold = common_flags()->clear_shadow_mmap_threshold;
- if (SANITIZER_LINUX &&
- UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
- internal_memset((void *)shadow_start, tag, page_start - shadow_start);
- internal_memset((void *)page_end, tag,
- shadow_start + shadow_size - page_end);
- // For an anonymous private mapping MADV_DONTNEED will return a zero page on
- // Linux.
- ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
- } else {
- internal_memset((void *)shadow_start, tag, shadow_size);
- }
- return AddTagToPointer(p, tag);
-}
-
uptr TagMemory(uptr p, uptr size, tag_t tag) {
uptr start = RoundDownTo(p, kShadowAlignment);
uptr end = RoundUpTo(p + size, kShadowAlignment);
}
~ScopedReport() {
+ void (*report_cb)(const char *);
{
BlockingMutexLock lock(&error_message_lock_);
- if (fatal)
- SetAbortMessage(error_message_.data());
+ report_cb = error_report_callback_;
error_message_ptr_ = nullptr;
}
+ if (report_cb)
+ report_cb(error_message_.data());
+ if (fatal)
+ SetAbortMessage(error_message_.data());
if (common_flags()->print_module_map >= 2 ||
(fatal && common_flags()->print_module_map))
DumpProcessMap();
// overwrite old trailing '\0', keep new trailing '\0' untouched.
internal_memcpy(&(*error_message_ptr_)[old_size - 1], msg, len);
}
+
+ static void SetErrorReportCallback(void (*callback)(const char *)) {
+ BlockingMutexLock lock(&error_message_lock_);
+ error_report_callback_ = callback;
+ }
+
private:
ScopedErrorReportLock error_report_lock_;
InternalMmapVector<char> error_message_;
static InternalMmapVector<char> *error_message_ptr_;
static BlockingMutex error_message_lock_;
+ static void (*error_report_callback_)(const char *);
};
InternalMmapVector<char> *ScopedReport::error_message_ptr_;
BlockingMutex ScopedReport::error_message_lock_;
+void (*ScopedReport::error_report_callback_)(const char *);
// If there is an active ScopedReport, append to its error message.
void AppendToErrorMessageBuffer(const char *buffer) {
// We didn't find any locals. Most likely we don't have symbols, so dump
// the information that we have for offline analysis.
- InternalScopedString frame_desc(GetPageSizeCached() * 2);
+ InternalScopedString frame_desc;
Printf("Previously allocated frames:\n");
for (uptr i = 0; i < frames; i++) {
const uptr *record_addr = &(*sa)[i];
frame_desc.append(" record_addr:0x%zx record:0x%zx",
reinterpret_cast<uptr>(record_addr), record);
if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) {
- RenderFrame(&frame_desc, " %F %L\n", 0, frame->info,
+ RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info,
common_flags()->symbolize_vs_style,
common_flags()->strip_path_prefix);
frame->ClearAll();
}
- Printf("%s", frame_desc.data());
+ Printf("%s\n", frame_desc.data());
frame_desc.clear();
}
}
static uptr GetGlobalSizeFromDescriptor(uptr ptr) {
// Find the ELF object that this global resides in.
Dl_info info;
- dladdr(reinterpret_cast<void *>(ptr), &info);
+ if (dladdr(reinterpret_cast<void *>(ptr), &info) == 0)
+ return 0;
auto *ehdr = reinterpret_cast<const ElfW(Ehdr) *>(info.dli_fbase);
auto *phdr_begin = reinterpret_cast<const ElfW(Phdr) *>(
reinterpret_cast<const u8 *>(ehdr) + ehdr->e_phoff);
return 0;
}
+static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate,
+ tag_t *left, tag_t *right) {
+ Decorator d;
+ uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
+ HwasanChunkView chunk = FindHeapChunkByAddress(mem);
+ if (chunk.IsAllocated()) {
+ uptr offset;
+ const char *whence;
+ if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) {
+ offset = untagged_addr - chunk.Beg();
+ whence = "inside";
+ } else if (candidate == left) {
+ offset = untagged_addr - chunk.End();
+ whence = "to the right of";
+ } else {
+ offset = chunk.Beg() - untagged_addr;
+ whence = "to the left of";
+ }
+ Printf("%s", d.Error());
+ Printf("\nCause: heap-buffer-overflow\n");
+ Printf("%s", d.Default());
+ Printf("%s", d.Location());
+ Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n",
+ untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(),
+ chunk.End());
+ Printf("%s", d.Allocation());
+ Printf("allocated here:\n");
+ Printf("%s", d.Default());
+ GetStackTraceFromId(chunk.GetAllocStackId()).Print();
+ return;
+ }
+ // Check whether the address points into a loaded library. If so, this is
+ // most likely a global variable.
+ const char *module_name;
+ uptr module_address;
+ Symbolizer *sym = Symbolizer::GetOrInit();
+ if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) {
+ Printf("%s", d.Error());
+ Printf("\nCause: global-overflow\n");
+ Printf("%s", d.Default());
+ DataInfo info;
+ Printf("%s", d.Location());
+ if (sym->SymbolizeData(mem, &info) && info.start) {
+ Printf(
+ "%p is located %zd bytes to the %s of %zd-byte global variable "
+ "%s [%p,%p) in %s\n",
+ untagged_addr,
+ candidate == left ? untagged_addr - (info.start + info.size)
+ : info.start - untagged_addr,
+ candidate == left ? "right" : "left", info.size, info.name,
+ info.start, info.start + info.size, module_name);
+ } else {
+ uptr size = GetGlobalSizeFromDescriptor(mem);
+ if (size == 0)
+ // We couldn't find the size of the global from the descriptors.
+ Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+ untagged_addr, candidate == left ? "right" : "left", module_name,
+ module_address);
+ else
+ Printf(
+ "%p is located to the %s of a %zd-byte global variable in "
+ "(%s+0x%x)\n",
+ untagged_addr, candidate == left ? "right" : "left", size,
+ module_name, module_address);
+ }
+ Printf("%s", d.Default());
+ }
+}
+
void PrintAddressDescription(
uptr tagged_addr, uptr access_size,
StackAllocationsRingBuffer *current_stack_allocations) {
d.Default());
}
+ tag_t addr_tag = GetTagFromPointer(tagged_addr);
+
+ bool on_stack = false;
+ // Check stack first. If the address is on the stack of a live thread, we
+ // know it cannot be a heap / global overflow.
+ hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
+ if (t->AddrIsInStack(untagged_addr)) {
+ on_stack = true;
+ // TODO(fmayer): figure out how to distinguish use-after-return and
+ // stack-buffer-overflow.
+ Printf("%s", d.Error());
+ Printf("\nCause: stack tag-mismatch\n");
+ Printf("%s", d.Location());
+ Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
+ t->unique_id());
+ Printf("%s", d.Default());
+ t->Announce();
+
+ auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+ ? current_stack_allocations
+ : t->stack_allocations();
+ PrintStackAllocations(sa, addr_tag, untagged_addr);
+ num_descriptions_printed++;
+ }
+ });
+
// Check if this looks like a heap buffer overflow by scanning
// the shadow left and right and looking for the first adjacent
// object with a different memory tag. If that tag matches addr_tag,
// check the allocator if it has a live chunk there.
- tag_t addr_tag = GetTagFromPointer(tagged_addr);
tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
- for (int i = 0; i < 1000; i++) {
- if (TagsEqual(addr_tag, left)) {
+ uptr candidate_distance = 0;
+ for (; candidate_distance < 1000; candidate_distance++) {
+ if (MemIsShadow(reinterpret_cast<uptr>(left)) &&
+ TagsEqual(addr_tag, left)) {
candidate = left;
break;
}
--left;
- if (TagsEqual(addr_tag, right)) {
+ if (MemIsShadow(reinterpret_cast<uptr>(right)) &&
+ TagsEqual(addr_tag, right)) {
candidate = right;
break;
}
++right;
}
- if (candidate) {
- uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
- HwasanChunkView chunk = FindHeapChunkByAddress(mem);
- if (chunk.IsAllocated()) {
- Printf("%s", d.Location());
- Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n",
- untagged_addr,
- candidate == left ? untagged_addr - chunk.End()
- : chunk.Beg() - untagged_addr,
- candidate == left ? "right" : "left", chunk.UsedSize(),
- chunk.Beg(), chunk.End());
- Printf("%s", d.Allocation());
- Printf("allocated here:\n");
- Printf("%s", d.Default());
- GetStackTraceFromId(chunk.GetAllocStackId()).Print();
- num_descriptions_printed++;
- } else {
- // Check whether the address points into a loaded library. If so, this is
- // most likely a global variable.
- const char *module_name;
- uptr module_address;
- Symbolizer *sym = Symbolizer::GetOrInit();
- if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
- &module_address)) {
- DataInfo info;
- if (sym->SymbolizeData(mem, &info) && info.start) {
- Printf(
- "%p is located %zd bytes to the %s of %zd-byte global variable "
- "%s [%p,%p) in %s\n",
- untagged_addr,
- candidate == left ? untagged_addr - (info.start + info.size)
- : info.start - untagged_addr,
- candidate == left ? "right" : "left", info.size, info.name,
- info.start, info.start + info.size, module_name);
- } else {
- uptr size = GetGlobalSizeFromDescriptor(mem);
- if (size == 0)
- // We couldn't find the size of the global from the descriptors.
- Printf(
- "%p is located to the %s of a global variable in (%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left",
- module_name, module_address);
- else
- Printf(
- "%p is located to the %s of a %zd-byte global variable in "
- "(%s+0x%x)\n",
- untagged_addr, candidate == left ? "right" : "left", size,
- module_name, module_address);
- }
- num_descriptions_printed++;
- }
- }
+ constexpr auto kCloseCandidateDistance = 1;
+
+ if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) {
+ ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+ num_descriptions_printed++;
}
hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har,
&ring_index, &num_matching_addrs,
&num_matching_addrs_4b)) {
+ Printf("%s", d.Error());
+ Printf("\nCause: use-after-free\n");
Printf("%s", d.Location());
Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
t->Announce();
num_descriptions_printed++;
}
-
- // Very basic check for stack memory.
- if (t->AddrIsInStack(untagged_addr)) {
- Printf("%s", d.Location());
- Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
- t->unique_id());
- Printf("%s", d.Default());
- t->Announce();
-
- auto *sa = (t == GetCurrentThread() && current_stack_allocations)
- ? current_stack_allocations
- : t->stack_allocations();
- PrintStackAllocations(sa, addr_tag, untagged_addr);
- num_descriptions_printed++;
- }
});
+ if (candidate && num_descriptions_printed == 0) {
+ ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+ num_descriptions_printed++;
+ }
+
// Print the remaining threads, as an extra information, 1 line per thread.
hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); });
if (!num_descriptions_printed)
// We exhausted our possibilities. Bail out.
Printf("HWAddressSanitizer can not describe address in more detail.\n");
+ if (num_descriptions_printed > 1) {
+ Printf(
+ "There are %d potential causes, printed above in order "
+ "of likeliness.\n",
+ num_descriptions_printed);
+ }
}
void ReportStats() {}
RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len));
tag_t *beg_row = center_row_beg - row_len * (num_rows / 2);
tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2);
- InternalScopedString s(GetPageSizeCached() * 8);
+ InternalScopedString s;
for (tag_t *row = beg_row; row < end_row; row += row_len) {
s.append("%s", row == center_row_beg ? "=>" : " ");
s.append("%p:", row);
Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName,
bug_type, untagged_addr, untagged_addr + orig_size, orig_size);
Printf("\n%s", d.Default());
+ Printf(
+ "Stack of invalid access unknown. Issue detected at deallocation "
+ "time.\n");
+ Printf("%s", d.Allocation());
+ Printf("deallocated here:\n");
+ Printf("%s", d.Default());
stack->Print();
HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
if (chunk.Beg()) {
GetStackTraceFromId(chunk.GetAllocStackId()).Print();
}
- InternalScopedString s(GetPageSizeCached() * 8);
+ InternalScopedString s;
CHECK_GT(tail_size, 0U);
CHECK_LT(tail_size, kShadowAlignment);
u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size);
frame[20], frame[21], frame[22], frame[23]);
Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n",
frame[24], frame[25], frame[26], frame[27]);
- Printf(" x28 %016llx x29 %016llx x30 %016llx\n",
- frame[28], frame[29], frame[30]);
+ // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch
+ // passes it to this function.
+ Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28],
+ frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256);
}
} // namespace __hwasan
+
+void __hwasan_set_error_report_callback(void (*callback)(const char *)) {
+ __hwasan::ScopedReport::SetErrorReportCallback(callback);
+}
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
#if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
#include "sanitizer_common/sanitizer_platform.h"
ASM_TYPE_FUNCTION(__interceptor_setjmp)
__interceptor_setjmp:
CFI_STARTPROC
+ BTI_C
mov x1, #0
b __interceptor_sigsetjmp
CFI_ENDPROC
ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic)
__interceptor_setjmp_bionic:
CFI_STARTPROC
+ BTI_C
mov x1, #1
b __interceptor_sigsetjmp
CFI_ENDPROC
ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
__interceptor_sigsetjmp:
CFI_STARTPROC
+ BTI_C
stp x19, x20, [x0, #0<<3]
stp x21, x22, [x0, #2<<3]
stp x23, x24, [x0, #4<<3]
// We do not need executable stack.
NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
// The content of this file is AArch64-only:
#if defined(__aarch64__)
.global __hwasan_tag_mismatch
.type __hwasan_tag_mismatch, %function
__hwasan_tag_mismatch:
+ BTI_J
+
// Compute the granule position one past the end of the access.
mov x16, #1
and x17, x1, #0xf
.type __hwasan_tag_mismatch_v2, %function
__hwasan_tag_mismatch_v2:
CFI_STARTPROC
+ BTI_J
// Set the CFA to be the return address for caller of __hwasan_check_*. Note
// that we do not emit CFI predicates to describe the contents of this stack
// We do not need executable stack.
NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
stack_allocations_->push(0);
}
-void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
+void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size,
+ const InitState *state) {
+ CHECK_EQ(0, unique_id_); // try to catch bad stack reuse
+ CHECK_EQ(0, stack_top_);
+ CHECK_EQ(0, stack_bottom_);
+
static u64 unique_id;
unique_id_ = unique_id++;
if (auto sz = flags()->heap_history_size)
heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
+ InitStackAndTls(state);
+#if !SANITIZER_FUCHSIA
+ // Do not initialize the stack ring buffer just yet on Fuchsia. Threads will
+ // be initialized before we enter the thread itself, so we will instead call
+ // this later.
+ InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+#endif
+}
+
+void Thread::InitStackRingBuffer(uptr stack_buffer_start,
+ uptr stack_buffer_size) {
HwasanTSDThreadInit(); // Only needed with interceptors.
uptr *ThreadLong = GetCurrentThreadLongPtr();
// The following implicitly sets (this) as the current thread.
// ScopedTaggingDisable needs GetCurrentThread to be set up.
ScopedTaggingDisabler disabler;
- uptr tls_size;
- uptr stack_size;
- GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
- &tls_size);
- stack_top_ = stack_bottom_ + stack_size;
- tls_end_ = tls_begin_ + tls_size;
-
if (stack_bottom_) {
int local;
CHECK(AddrIsInStack((uptr)&local));
}
// Generate a (pseudo-)random non-zero tag.
-tag_t Thread::GenerateRandomTag() {
+tag_t Thread::GenerateRandomTag(uptr num_bits) {
+ DCHECK_GT(num_bits, 0);
if (tagging_disabled_) return 0;
tag_t tag;
+ const uptr tag_mask = (1ULL << num_bits) - 1;
do {
if (flags()->random_tags) {
if (!random_buffer_)
random_buffer_ = random_state_ = xorshift(random_state_);
CHECK(random_buffer_);
- tag = random_buffer_ & 0xFF;
- random_buffer_ >>= 8;
+ tag = random_buffer_ & tag_mask;
+ random_buffer_ >>= num_bits;
} else {
- tag = random_state_ = (random_state_ + 1) & 0xFF;
+ random_state_ += 1;
+ tag = random_state_ & tag_mask;
}
} while (!tag);
return tag;
class Thread {
public:
- void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself.
+ // These are optional parameters that can be passed to Init.
+ struct InitState;
+
+ void Init(uptr stack_buffer_start, uptr stack_buffer_size,
+ const InitState *state = nullptr);
void InitRandomState();
+ void InitStackAndTls(const InitState *state = nullptr);
+
+ // Must be called from the thread itself.
+ void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size);
+
void Destroy();
uptr stack_top() { return stack_top_; }
HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; }
StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; }
- tag_t GenerateRandomTag();
+ tag_t GenerateRandomTag(uptr num_bits = kTagBits);
void DisableTagging() { tagging_disabled_++; }
void EnableTagging() { tagging_disabled_--; }
HeapAllocationsRingBuffer *heap_allocations_;
StackAllocationsRingBuffer *stack_allocations_;
- Thread *next_; // All live threads form a linked list.
-
u64 unique_id_; // counting from zero.
u32 tagging_disabled_; // if non-zero, malloc uses zero tag in this thread.
new (thread_list_placeholder) HwasanThreadList(storage, size);
}
-} // namespace
+} // namespace __hwasan
return 0;
}
-struct ThreadListHead {
- Thread *list_;
-
- ThreadListHead() : list_(nullptr) {}
-
- void Push(Thread *t) {
- t->next_ = list_;
- list_ = t;
- }
-
- Thread *Pop() {
- Thread *t = list_;
- if (t)
- list_ = t->next_;
- return t;
- }
-
- void Remove(Thread *t) {
- Thread **cur = &list_;
- while (*cur != t) cur = &(*cur)->next_;
- CHECK(*cur && "thread not found");
- *cur = (*cur)->next_;
- }
-
- template <class CB>
- void ForEach(CB cb) {
- Thread *t = list_;
- while (t) {
- cb(t);
- t = t->next_;
- }
- }
-};
-
struct ThreadStats {
uptr n_live_threads;
uptr total_stack_size;
RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
}
- Thread *CreateCurrentThread() {
- Thread *t;
+ Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) {
+ Thread *t = nullptr;
{
- SpinMutexLock l(&list_mutex_);
- t = free_list_.Pop();
- if (t) {
- uptr start = (uptr)t - ring_buffer_size_;
- internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
- } else {
- t = AllocThread();
+ SpinMutexLock l(&free_list_mutex_);
+ if (!free_list_.empty()) {
+ t = free_list_.back();
+ free_list_.pop_back();
}
- live_list_.Push(t);
}
- t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
+ if (t) {
+ uptr start = (uptr)t - ring_buffer_size_;
+ internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
+ } else {
+ t = AllocThread();
+ }
+ {
+ SpinMutexLock l(&live_list_mutex_);
+ live_list_.push_back(t);
+ }
+ t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state);
AddThreadStats(t);
return t;
}
ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);
}
+ void RemoveThreadFromLiveList(Thread *t) {
+ SpinMutexLock l(&live_list_mutex_);
+ for (Thread *&t2 : live_list_)
+ if (t2 == t) {
+ // To remove t2, copy the last element of the list in t2's position, and
+ // pop_back(). This works even if t2 is itself the last element.
+ t2 = live_list_.back();
+ live_list_.pop_back();
+ return;
+ }
+ CHECK(0 && "thread not found in live list");
+ }
+
void ReleaseThread(Thread *t) {
RemoveThreadStats(t);
t->Destroy();
- SpinMutexLock l(&list_mutex_);
- live_list_.Remove(t);
- free_list_.Push(t);
DontNeedThread(t);
+ RemoveThreadFromLiveList(t);
+ SpinMutexLock l(&free_list_mutex_);
+ free_list_.push_back(t);
}
Thread *GetThreadByBufferAddress(uptr p) {
template <class CB>
void VisitAllLiveThreads(CB cb) {
- SpinMutexLock l(&list_mutex_);
- live_list_.ForEach(cb);
+ SpinMutexLock l(&live_list_mutex_);
+ for (Thread *t : live_list_) cb(t);
}
void AddThreadStats(Thread *t) {
return stats_;
}
+ uptr GetRingBufferSize() const { return ring_buffer_size_; }
+
private:
Thread *AllocThread() {
+ SpinMutexLock l(&free_space_mutex_);
uptr align = ring_buffer_size_ * 2;
CHECK(IsAligned(free_space_, align));
Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
return t;
}
+ SpinMutex free_space_mutex_;
uptr free_space_;
uptr free_space_end_;
uptr ring_buffer_size_;
uptr thread_alloc_size_;
- ThreadListHead free_list_;
- ThreadListHead live_list_;
- SpinMutex list_mutex_;
+ SpinMutex free_list_mutex_;
+ InternalMmapVector<Thread *> free_list_;
+ SpinMutex live_list_mutex_;
+ InternalMmapVector<Thread *> live_list_;
ThreadStats stats_;
SpinMutex stats_mutex_;
void InitThreadList(uptr storage, uptr size);
HwasanThreadList &hwasanThreadList();
-} // namespace
+} // namespace __hwasan
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
#include "sanitizer_common/sanitizer_internal_defs.h"
-#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
- !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_WINDOWS && \
- !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS
-# error "Interception doesn't work on this operating system."
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
+ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
+ !SANITIZER_SOLARIS
+# error "Interception doesn't work on this operating system."
#endif
// These typedefs should be used only in the interceptor definitions to replace
extern "C" ret_type func(__VA_ARGS__);
# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
-#elif SANITIZER_RTEMS
-# define WRAP(x) x
-# define WRAPPER_NAME(x) #x
-# define INTERCEPTOR_ATTRIBUTE
-# define DECLARE_WRAPPER(ret_type, func, ...)
#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
# define WRAP(x) __interceptor_ ## x
# define WRAPPER_NAME(x) "__interceptor_" #x
# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
# define REAL(x) __unsanitized_##x
# define DECLARE_REAL(ret_type, func, ...)
-#elif SANITIZER_RTEMS
-# define REAL(x) __real_ ## x
-# define DECLARE_REAL(ret_type, func, ...) \
- extern "C" ret_type REAL(func)(__VA_ARGS__);
#elif !SANITIZER_MAC
# define PTR_TO_REAL(x) real_##x
# define REAL(x) __interception::PTR_TO_REAL(x)
# define ASSIGN_REAL(x, y)
#endif // SANITIZER_MAC
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
- DECLARE_REAL(ret_type, func, __VA_ARGS__) \
- extern "C" ret_type WRAP(func)(__VA_ARGS__);
+#if !SANITIZER_FUCHSIA
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+ DECLARE_REAL(ret_type, func, __VA_ARGS__) \
+ extern "C" ret_type WRAP(func)(__VA_ARGS__);
// Declare an interceptor and its wrapper defined in a different translation
// unit (ex. asm).
# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
// macros does its job. In exceptional cases you may need to call REAL(foo)
// without defining INTERCEPTOR(..., foo, ...). For example, if you override
// foo with an interceptor for other function.
-#if !SANITIZER_MAC && !SANITIZER_FUCHSIA && !SANITIZER_RTEMS
-# define DEFINE_REAL(ret_type, func, ...) \
+#if !SANITIZER_MAC && !SANITIZER_FUCHSIA
+# define DEFINE_REAL(ret_type, func, ...) \
typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
- namespace __interception { \
- FUNC_TYPE(func) PTR_TO_REAL(func); \
+ namespace __interception { \
+ FUNC_TYPE(func) PTR_TO_REAL(func); \
}
#else
# define DEFINE_REAL(ret_type, func, ...)
#define INCLUDED_FROM_INTERCEPTION_LIB
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
# include "interception_linux.h"
# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
#include "interception.h"
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#include <dlfcn.h> // for dlsym() and dlvsym()
return addr && (func == wrapper);
}
-// Android and Solaris do not have dlvsym
-#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
+// dlvsym is a GNU extension supported by some other platforms.
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
static void *GetFuncAddr(const char *name, const char *ver) {
return dlvsym(RTLD_NEXT, name, ver);
}
*ptr_to_real = (uptr)addr;
return addr && (func == wrapper);
}
-#endif // !SANITIZER_ANDROID
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
} // namespace __interception
#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ // SANITIZER_SOLARIS
//===----------------------------------------------------------------------===//
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
# error "interception_linux.h should be included from interception library only"
(::__interception::uptr) & (func), \
(::__interception::uptr) & WRAP(func))
-// Android, Solaris and OpenBSD do not have dlvsym
-#if !SANITIZER_ANDROID && !SANITIZER_SOLARIS && !SANITIZER_OPENBSD
+// dlvsym is a GNU extension supported by some other platforms.
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
::__interception::InterceptFunction( \
#func, symver, \
#else
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
-#endif // !SANITIZER_ANDROID && !SANITIZER_SOLARIS
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
#endif // INTERCEPTION_LINUX_H
#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ // SANITIZER_SOLARIS
static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
static const int kJumpInstructionLength = 5;
static const int kShortJumpInstructionLength = 2;
-static const int kIndirectJumpInstructionLength = 6;
+UNUSED static const int kIndirectJumpInstructionLength = 6;
static const int kBranchLength =
FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
static const int kDirectBranchLength = kBranchLength + kAddressLength;
return si.dwAllocationGranularity;
}
-static uptr RoundUpTo(uptr size, uptr boundary) {
+UNUSED static uptr RoundUpTo(uptr size, uptr boundary) {
return (size + boundary - 1) & ~(boundary - 1);
}
uptr max_size;
};
-static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
+UNUSED static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig
static const int kMaxTrampolineRegion = 1024;
static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
-I${COMPILER_RT_SOURCE_DIR}/lib/interception
-fno-rtti
-O2
- -Werror=sign-compare
- -Wno-non-virtual-dtor)
+ -Werror=sign-compare)
set(INTERCEPTION_TEST_LINK_FLAGS_COMMON
${COMPILER_RT_UNITTEST_LINK_FLAGS})
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
using namespace __lsan;
uptr stack_top = 0, stack_bottom = 0;
- ThreadContext *t;
- if (StackTrace::WillUseFastUnwind(request_fast) &&
- (t = CurrentThreadContext())) {
+ if (ThreadContext *t = CurrentThreadContext()) {
stack_top = t->stack_end();
stack_bottom = t->stack_begin();
}
- if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) {
- if (StackTrace::WillUseFastUnwind(request_fast))
- Unwind(max_depth, pc, bp, nullptr, stack_top, stack_bottom, true);
- else
- Unwind(max_depth, pc, 0, context, 0, 0, false);
- }
+ if (SANITIZER_MIPS && !IsValidFrame(bp, stack_top, stack_bottom))
+ return;
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast);
}
using namespace __lsan;
RegisterCommonFlags(&parser);
// Override from user-specified string.
- const char *lsan_default_options = MaybeCallLsanDefaultOptions();
+ const char *lsan_default_options = __lsan_default_options();
parser.ParseString(lsan_default_options);
parser.ParseStringFromEnv("LSAN_OPTIONS");
- SetVerbosity(common_flags()->verbosity);
+ InitializeCommonFlags();
if (Verbosity()) ReportUnrecognizedFlags();
void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
uptr alignment) {
- RegisterDeallocation(p);
if (new_size > max_malloc_size) {
- allocator.Deallocate(GetAllocatorCache(), p);
- return ReportAllocationSizeTooBig(new_size, stack);
+ ReportAllocationSizeTooBig(new_size, stack);
+ return nullptr;
}
- p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
- RegisterAllocation(stack, p, new_size);
- return p;
+ RegisterDeallocation(p);
+ void *new_p =
+ allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
+ if (new_p)
+ RegisterAllocation(stack, new_p, new_size);
+ else if (new_size != 0)
+ RegisterAllocation(stack, p, new_size);
+ return new_p;
}
void GetAllocatorCacheRange(uptr *begin, uptr *end) {
return kIgnoreObjectInvalid;
}
}
+
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
+ // This function can be used to treat memory reachable from `tctx` as live.
+ // This is useful for threads that have been created but not yet started.
+
+ // This is currently a no-op because the LSan `pthread_create()` interceptor
+ // blocks until the child thread starts which keeps the thread's `arg` pointer
+ // live.
+}
+
} // namespace __lsan
using namespace __lsan;
};
#if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \
- defined(__arm__)
+ defined(__arm__) || SANITIZER_RISCV64
template <typename AddressSpaceViewTy>
struct AP32 {
static const uptr kSpaceBeg = 0;
#include "sanitizer_common/sanitizer_thread_registry.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
-extern "C" const char *__lsan_current_stage = "unknown";
-
#if CAN_SANITIZE_LEAKS
namespace __lsan {
if (flags()->log_threads) Report(__VA_ARGS__); \
} while (0)
-ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)];
-static SuppressionContext *suppression_ctx = nullptr;
+class LeakSuppressionContext {
+ bool parsed = false;
+ SuppressionContext context;
+ bool suppressed_stacks_sorted = true;
+ InternalMmapVector<u32> suppressed_stacks;
+
+ Suppression *GetSuppressionForAddr(uptr addr);
+ void LazyInit();
+
+ public:
+ LeakSuppressionContext(const char *supprression_types[],
+ int suppression_types_num)
+ : context(supprression_types, suppression_types_num) {}
+
+ Suppression *GetSuppressionForStack(u32 stack_trace_id);
+
+ const InternalMmapVector<u32> &GetSortedSuppressedStacks() {
+ if (!suppressed_stacks_sorted) {
+ suppressed_stacks_sorted = true;
+ SortAndDedup(suppressed_stacks);
+ }
+ return suppressed_stacks;
+ }
+ void PrintMatchedSuppressions();
+};
+
+ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)];
+static LeakSuppressionContext *suppression_ctx = nullptr;
static const char kSuppressionLeak[] = "leak";
static const char *kSuppressionTypes[] = { kSuppressionLeak };
static const char kStdSuppressions[] =
#if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
- // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
- // definition.
- "leak:*pthread_exit*\n"
+ // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+ // definition.
+ "leak:*pthread_exit*\n"
#endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
#if SANITIZER_MAC
- // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
- "leak:*_os_trace*\n"
+ // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
+ "leak:*_os_trace*\n"
#endif
- // TLS leak in some glibc versions, described in
- // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
- "leak:*tls_get_addr*\n";
+ // TLS leak in some glibc versions, described in
+ // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
+ "leak:*tls_get_addr*\n";
void InitializeSuppressions() {
CHECK_EQ(nullptr, suppression_ctx);
suppression_ctx = new (suppression_placeholder)
- SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
- suppression_ctx->ParseFromFile(flags()->suppressions);
- if (&__lsan_default_suppressions)
- suppression_ctx->Parse(__lsan_default_suppressions());
- suppression_ctx->Parse(kStdSuppressions);
+ LeakSuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
}
-static SuppressionContext *GetSuppressionContext() {
+void LeakSuppressionContext::LazyInit() {
+ if (!parsed) {
+ parsed = true;
+ context.ParseFromFile(flags()->suppressions);
+ if (&__lsan_default_suppressions)
+ context.Parse(__lsan_default_suppressions());
+ context.Parse(kStdSuppressions);
+ }
+}
+
+static LeakSuppressionContext *GetSuppressionContext() {
CHECK(suppression_ctx);
return suppression_ctx;
}
root_regions = new (placeholder) InternalMmapVector<RootRegion>();
}
-const char *MaybeCallLsanDefaultOptions() {
- return (&__lsan_default_options) ? __lsan_default_options() : "";
-}
-
void InitCommonLsan() {
InitializeRootRegions();
if (common_flags()->detect_leaks) {
#else
+#if SANITIZER_ANDROID
+// FIXME: Move this out into *libcdep.cpp
+extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls(
+ pid_t, void (*cb)(void *, void *, uptr, void *), void *);
+#endif
+
+static void ProcessThreadRegistry(Frontier *frontier) {
+ InternalMmapVector<uptr> ptrs;
+ GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
+ GetAdditionalThreadContextPtrs, &ptrs);
+
+ for (uptr i = 0; i < ptrs.size(); ++i) {
+ void *ptr = reinterpret_cast<void *>(ptrs[i]);
+ uptr chunk = PointsIntoChunk(ptr);
+ if (!chunk)
+ continue;
+ LsanMetadata m(chunk);
+ if (!m.allocated())
+ continue;
+
+ // Mark as reachable and add to frontier.
+ LOG_POINTERS("Treating pointer %p from ThreadContext as reachable\n", ptr);
+ m.set_tag(kReachable);
+ frontier->push_back(chunk);
+ }
+}
+
// Scans thread data (stacks and TLS) for heap pointers.
static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
Frontier *frontier) {
- InternalMmapVector<uptr> registers(suspended_threads.RegisterCount());
- uptr registers_begin = reinterpret_cast<uptr>(registers.data());
- uptr registers_end =
- reinterpret_cast<uptr>(registers.data() + registers.size());
+ InternalMmapVector<uptr> registers;
for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
tid_t os_id = static_cast<tid_t>(suspended_threads.GetThreadID(i));
LOG_THREADS("Processing thread %d.\n", os_id);
}
uptr sp;
PtraceRegistersStatus have_registers =
- suspended_threads.GetRegistersAndSP(i, registers.data(), &sp);
+ suspended_threads.GetRegistersAndSP(i, ®isters, &sp);
if (have_registers != REGISTERS_AVAILABLE) {
Report("Unable to get registers from thread %d.\n", os_id);
// If unable to get SP, consider the entire stack to be reachable unless
sp = stack_begin;
}
- if (flags()->use_registers && have_registers)
+ if (flags()->use_registers && have_registers) {
+ uptr registers_begin = reinterpret_cast<uptr>(registers.data());
+ uptr registers_end =
+ reinterpret_cast<uptr>(registers.data() + registers.size());
ScanRangeForPointers(registers_begin, registers_end, frontier,
"REGISTERS", kReachable);
+ }
if (flags()->use_stacks) {
LOG_THREADS("Stack at %p-%p (SP = %p).\n", stack_begin, stack_end, sp);
kReachable);
}
}
+#if SANITIZER_ANDROID
+ auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/,
+ void *arg) -> void {
+ ScanRangeForPointers(reinterpret_cast<uptr>(dtls_begin),
+ reinterpret_cast<uptr>(dtls_end),
+ reinterpret_cast<Frontier *>(arg), "DTLS",
+ kReachable);
+ };
+
+ // FIXME: There might be a race-condition here (and in Bionic) if the
+ // thread is suspended in the middle of updating its DTLS. IOWs, we
+ // could scan already freed memory. (probably fine for now)
+ __libc_iterate_dynamic_tls(os_id, cb, frontier);
+#else
if (dtls && !DTLSInDestruction(dtls)) {
- for (uptr j = 0; j < dtls->dtv_size; ++j) {
- uptr dtls_beg = dtls->dtv[j].beg;
- uptr dtls_end = dtls_beg + dtls->dtv[j].size;
+ ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
+ uptr dtls_beg = dtv.beg;
+ uptr dtls_end = dtls_beg + dtv.size;
if (dtls_beg < dtls_end) {
- LOG_THREADS("DTLS %zu at %p-%p.\n", j, dtls_beg, dtls_end);
+ LOG_THREADS("DTLS %zu at %p-%p.\n", id, dtls_beg, dtls_end);
ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
kReachable);
}
- }
+ });
} else {
// We are handling a thread with DTLS under destruction. Log about
// this and continue.
LOG_THREADS("Thread %d has DTLS under destruction.\n", os_id);
}
+#endif
}
}
+
+ // Add pointers reachable from ThreadContexts
+ ProcessThreadRegistry(frontier);
}
#endif // SANITIZER_FUCHSIA
// ForEachChunk callback. If the chunk is marked as leaked, marks all chunks
// which are reachable from it as indirectly leaked.
static void MarkIndirectlyLeakedCb(uptr chunk, void *arg) {
- __lsan_current_stage = "MarkIndirectlyLeakedCb";
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (m.allocated() && m.tag() != kReachable) {
}
}
+static void IgnoredSuppressedCb(uptr chunk, void *arg) {
+ CHECK(arg);
+ chunk = GetUserBegin(chunk);
+ LsanMetadata m(chunk);
+ if (!m.allocated() || m.tag() == kIgnored)
+ return;
+
+ const InternalMmapVector<u32> &suppressed =
+ *static_cast<const InternalMmapVector<u32> *>(arg);
+ uptr idx = InternalLowerBound(suppressed, m.stack_trace_id());
+ if (idx >= suppressed.size() || m.stack_trace_id() != suppressed[idx])
+ return;
+
+ LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", chunk,
+ chunk + m.requested_size(), m.requested_size());
+ m.set_tag(kIgnored);
+}
+
// ForEachChunk callback. If chunk is marked as ignored, adds its address to
// frontier.
static void CollectIgnoredCb(uptr chunk, void *arg) {
CHECK(arg);
- __lsan_current_stage = "CollectIgnoredCb";
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (m.allocated() && m.tag() == kIgnored) {
static void MarkInvalidPCCb(uptr chunk, void *arg) {
CHECK(arg);
InvalidPCParam *param = reinterpret_cast<InvalidPCParam *>(arg);
- __lsan_current_stage = "MarkInvalidPCCb";
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (m.allocated() && m.tag() != kReachable && m.tag() != kIgnored) {
// Sets the appropriate tag on each chunk.
static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
Frontier *frontier) {
+ const InternalMmapVector<u32> &suppressed_stacks =
+ GetSuppressionContext()->GetSortedSuppressedStacks();
+ if (!suppressed_stacks.empty()) {
+ ForEachChunk(IgnoredSuppressedCb,
+ const_cast<InternalMmapVector<u32> *>(&suppressed_stacks));
+ }
ForEachChunk(CollectIgnoredCb, frontier);
ProcessGlobalRegions(frontier);
ProcessThreads(suspended_threads, frontier);
// ForEachChunk callback. Resets the tags to pre-leak-check state.
static void ResetTagsCb(uptr chunk, void *arg) {
(void)arg;
- __lsan_current_stage = "ResetTagsCb";
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (m.allocated() && m.tag() != kIgnored)
static void CollectLeaksCb(uptr chunk, void *arg) {
CHECK(arg);
LeakReport *leak_report = reinterpret_cast<LeakReport *>(arg);
- __lsan_current_stage = "CollectLeaksCb";
chunk = GetUserBegin(chunk);
LsanMetadata m(chunk);
if (!m.allocated()) return;
}
}
-static void PrintMatchedSuppressions() {
+void LeakSuppressionContext::PrintMatchedSuppressions() {
InternalMmapVector<Suppression *> matched;
- GetSuppressionContext()->GetMatched(&matched);
+ context.GetMatched(&matched);
if (!matched.size())
return;
const char *line = "-----------------------------------------------------";
Printf("%s\n", line);
Printf("Suppressions used:\n");
Printf(" count bytes template\n");
- for (uptr i = 0; i < matched.size(); i++)
- Printf("%7zu %10zu %s\n", static_cast<uptr>(atomic_load_relaxed(
- &matched[i]->hit_count)), matched[i]->weight, matched[i]->templ);
+ for (uptr i = 0; i < matched.size(); i++) {
+ Printf("%7zu %10zu %s\n",
+ static_cast<uptr>(atomic_load_relaxed(&matched[i]->hit_count)),
+ matched[i]->weight, matched[i]->templ);
+ }
Printf("%s\n\n", line);
}
const InternalMmapVector<tid_t> &suspended_threads =
*(const InternalMmapVector<tid_t> *)arg;
if (tctx->status == ThreadStatusRunning) {
- uptr i = InternalLowerBound(suspended_threads, 0, suspended_threads.size(),
- tctx->os_id, CompareLess<int>());
+ uptr i = InternalLowerBound(suspended_threads, tctx->os_id);
if (i >= suspended_threads.size() || suspended_threads[i] != tctx->os_id)
Report("Running thread %d was not suspended. False leaks are possible.\n",
tctx->os_id);
param->success = true;
}
-static bool CheckForLeaks() {
- if (&__lsan_is_turned_off && __lsan_is_turned_off())
- return false;
- EnsureMainThreadIDIsCorrect();
- CheckForLeaksParam param;
- LockStuffAndStopTheWorld(CheckForLeaksCallback, ¶m);
-
- if (!param.success) {
- Report("LeakSanitizer has encountered a fatal error.\n");
- Report(
- "HINT: For debugging, try setting environment variable "
- "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
- Report(
- "HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)\n");
- Die();
- }
- param.leak_report.ApplySuppressions();
- uptr unsuppressed_count = param.leak_report.UnsuppressedLeakCount();
- if (unsuppressed_count > 0) {
+static bool PrintResults(LeakReport &report) {
+ uptr unsuppressed_count = report.UnsuppressedLeakCount();
+ if (unsuppressed_count) {
Decorator d;
- Printf("\n"
- "================================================================="
- "\n");
+ Printf(
+ "\n"
+ "================================================================="
+ "\n");
Printf("%s", d.Error());
Report("ERROR: LeakSanitizer: detected memory leaks\n");
Printf("%s", d.Default());
- param.leak_report.ReportTopLeaks(flags()->max_leaks);
+ report.ReportTopLeaks(flags()->max_leaks);
}
if (common_flags()->print_suppressions)
- PrintMatchedSuppressions();
+ GetSuppressionContext()->PrintMatchedSuppressions();
if (unsuppressed_count > 0) {
- param.leak_report.PrintSummary();
+ report.PrintSummary();
return true;
}
return false;
}
+static bool CheckForLeaks() {
+ if (&__lsan_is_turned_off && __lsan_is_turned_off())
+ return false;
+ // Inside LockStuffAndStopTheWorld we can't run symbolizer, so we can't match
+ // suppressions. However if a stack id was previously suppressed, it should be
+ // suppressed in future checks as well.
+ for (int i = 0;; ++i) {
+ EnsureMainThreadIDIsCorrect();
+ CheckForLeaksParam param;
+ LockStuffAndStopTheWorld(CheckForLeaksCallback, ¶m);
+ if (!param.success) {
+ Report("LeakSanitizer has encountered a fatal error.\n");
+ Report(
+ "HINT: For debugging, try setting environment variable "
+ "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
+ Report(
+ "HINT: LeakSanitizer does not work under ptrace (strace, gdb, "
+ "etc)\n");
+ Die();
+ }
+ // No new suppressions stacks, so rerun will not help and we can report.
+ if (!param.leak_report.ApplySuppressions())
+ return PrintResults(param.leak_report);
+
+ // No indirect leaks to report, so we are done here.
+ if (!param.leak_report.IndirectUnsuppressedLeakCount())
+ return PrintResults(param.leak_report);
+
+ if (i >= 8) {
+ Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n");
+ return PrintResults(param.leak_report);
+ }
+
+ // We found a new previously unseen suppressed call stack. Rerun to make
+ // sure it does not hold indirect leaks.
+ VReport(1, "Rerun with %zu suppressed stacks.",
+ GetSuppressionContext()->GetSortedSuppressedStacks().size());
+ }
+}
+
static bool has_reported_leaks = false;
bool HasReportedLeaks() { return has_reported_leaks; }
void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
-static Suppression *GetSuppressionForAddr(uptr addr) {
+Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
Suppression *s = nullptr;
// Suppress by module name.
- SuppressionContext *suppressions = GetSuppressionContext();
if (const char *module_name =
Symbolizer::GetOrInit()->GetModuleNameForPc(addr))
- if (suppressions->Match(module_name, kSuppressionLeak, &s))
+ if (context.Match(module_name, kSuppressionLeak, &s))
return s;
// Suppress by file or function name.
SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
- if (suppressions->Match(cur->info.function, kSuppressionLeak, &s) ||
- suppressions->Match(cur->info.file, kSuppressionLeak, &s)) {
+ if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
+ context.Match(cur->info.file, kSuppressionLeak, &s)) {
break;
}
}
return s;
}
-static Suppression *GetSuppressionForStack(u32 stack_trace_id) {
+Suppression *LeakSuppressionContext::GetSuppressionForStack(
+ u32 stack_trace_id) {
+ LazyInit();
StackTrace stack = StackDepotGet(stack_trace_id);
for (uptr i = 0; i < stack.size; i++) {
Suppression *s = GetSuppressionForAddr(
StackTrace::GetPreviousInstructionPc(stack.trace[i]));
- if (s) return s;
+ if (s) {
+ suppressed_stacks_sorted = false;
+ suppressed_stacks.push_back(stack_trace_id);
+ return s;
+ }
}
return nullptr;
}
bytes += leaks_[i].total_size;
allocations += leaks_[i].hit_count;
}
- InternalScopedString summary(kMaxSummaryLength);
+ InternalScopedString summary;
summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes,
allocations);
ReportErrorSummary(summary.data());
}
-void LeakReport::ApplySuppressions() {
+uptr LeakReport::ApplySuppressions() {
+ LeakSuppressionContext *suppressions = GetSuppressionContext();
+ uptr new_suppressions = false;
for (uptr i = 0; i < leaks_.size(); i++) {
- Suppression *s = GetSuppressionForStack(leaks_[i].stack_trace_id);
+ Suppression *s =
+ suppressions->GetSuppressionForStack(leaks_[i].stack_trace_id);
if (s) {
s->weight += leaks_[i].total_size;
atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) +
leaks_[i].hit_count);
leaks_[i].is_suppressed = true;
+ ++new_suppressions;
}
}
+ return new_suppressions;
}
uptr LeakReport::UnsuppressedLeakCount() {
return result;
}
+uptr LeakReport::IndirectUnsuppressedLeakCount() {
+ uptr result = 0;
+ for (uptr i = 0; i < leaks_.size(); i++)
+ if (!leaks_[i].is_suppressed && !leaks_[i].is_directly_leaked)
+ result++;
+ return result;
+}
+
} // namespace __lsan
#else // CAN_SANITIZE_LEAKS
namespace __lsan {
return 0;
}
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char * __lsan_default_options() {
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) {
return "";
}
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
int __lsan_is_turned_off() {
return 0;
// To enable LeakSanitizer on a new architecture, one needs to implement the
// internal_clone function as well as (probably) adjust the TLS machinery for
// the new architecture inside the sanitizer library.
-#if (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC) && \
- (SANITIZER_WORDSIZE == 64) && \
- (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
+// Exclude leak-detection on arm32 for Android because `__aeabi_read_tp`
+// is missing. This caused a link error.
+#if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__))
+#define CAN_SANITIZE_LEAKS 0
+#elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \
+ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \
defined(__powerpc64__) || defined(__s390x__))
#define CAN_SANITIZE_LEAKS 1
-#elif defined(__i386__) && \
- (SANITIZER_LINUX && !SANITIZER_ANDROID || SANITIZER_MAC)
+#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC)
+#define CAN_SANITIZE_LEAKS 1
+#elif defined(__arm__) && SANITIZER_LINUX
#define CAN_SANITIZE_LEAKS 1
-#elif defined(__arm__) && \
- SANITIZER_LINUX && !SANITIZER_ANDROID
+#elif SANITIZER_RISCV64 && SANITIZER_LINUX
#define CAN_SANITIZE_LEAKS 1
#elif SANITIZER_NETBSD || SANITIZER_FUCHSIA
#define CAN_SANITIZE_LEAKS 1
namespace __sanitizer {
class FlagParser;
class ThreadRegistry;
+class ThreadContextBase;
struct DTLS;
}
kIgnored = 3
};
-const u32 kInvalidTid = (u32) -1;
-
struct Flags {
#define LSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
#include "lsan_flags.inc"
ChunkTag tag);
void ReportTopLeaks(uptr max_leaks);
void PrintSummary();
- void ApplySuppressions();
+ uptr ApplySuppressions();
uptr UnsuppressedLeakCount();
+ uptr IndirectUnsuppressedLeakCount();
private:
void PrintReportForLeak(uptr index);
void ScanRootRegion(Frontier *frontier, RootRegion const ®ion,
uptr region_begin, uptr region_end, bool is_readable);
void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg);
+void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs);
// Run stoptheworld while holding any platform-specific locks, as well as the
// allocator and thread registry locks.
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
// Returns true if [addr, addr + sizeof(void *)) is poisoned.
bool WordIsPoisoned(uptr addr);
// Wrappers for ThreadRegistry access.
-void LockThreadRegistry();
-void UnlockThreadRegistry();
+void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
+void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS;
ThreadRegistry *GetThreadRegistryLocked();
bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
#include "lsan_allocator.h"
#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_stoptheworld_fuchsia.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
// Ensure that the Zircon system ABI is linked in.
auto params = static_cast<const Params *>(data);
uptr begin = reinterpret_cast<uptr>(chunk);
uptr end = begin + size;
- auto i = __sanitizer::InternalLowerBound(params->allocator_caches, 0,
- params->allocator_caches.size(),
- begin, CompareLess<uptr>());
+ auto i = __sanitizer::InternalLowerBound(params->allocator_caches, begin);
if (i < params->allocator_caches.size() &&
params->allocator_caches[i] >= begin &&
end - params->allocator_caches[i] <= sizeof(AllocatorCache)) {
¶ms->argument->frontier);
}
- params->callback({}, params->argument);
+ params->callback(SuspendedThreadsListFuchsia(), params->argument);
},
¶ms);
return 0;
}
+#if SANITIZER_ANDROID && __ANDROID_API__ < 21
+extern "C" __attribute__((weak)) int dl_iterate_phdr(
+ int (*)(struct dl_phdr_info *, size_t, void *), void *);
+#endif
+
// Scans global variables for heap pointers.
void ProcessGlobalRegions(Frontier *frontier) {
if (!flags()->use_globals) return;
namespace __lsan {
-class ThreadContext : public ThreadContextLsanBase {
+class ThreadContext final : public ThreadContextLsanBase {
public:
explicit ThreadContext(int tid);
void OnCreated(void *arg) override;
return lsan_memalign(alignment, size, stack);
}
#define LSAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign)
+#else
+#define LSAN_MAYBE_INTERCEPT_MEMALIGN
+#endif // SANITIZER_INTERCEPT_MEMALIGN
+#if SANITIZER_INTERCEPT___LIBC_MEMALIGN
INTERCEPTOR(void *, __libc_memalign, uptr alignment, uptr size) {
ENSURE_LSAN_INITED;
GET_STACK_TRACE_MALLOC;
}
#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN INTERCEPT_FUNCTION(__libc_memalign)
#else
-#define LSAN_MAYBE_INTERCEPT_MEMALIGN
#define LSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN
-#endif // SANITIZER_INTERCEPT_MEMALIGN
+#endif // SANITIZER_INTERCEPT___LIBC_MEMALIGN
#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
INTERCEPTOR(void*, aligned_alloc, uptr alignment, uptr size) {
if (res == 0) {
int tid = ThreadCreate(GetCurrentThread(), *(uptr *)th,
IsStateDetached(detached));
- CHECK_NE(tid, 0);
+ CHECK_NE(tid, kMainTid);
atomic_store(&p.tid, tid, memory_order_release);
while (atomic_load(&p.tid, memory_order_acquire) != 0)
internal_sched_yield();
return res;
}
+INTERCEPTOR(int, pthread_detach, void *th) {
+ ENSURE_LSAN_INITED;
+ int tid = ThreadTid((uptr)th);
+ int res = REAL(pthread_detach)(th);
+ if (res == 0)
+ ThreadDetach(tid);
+ return res;
+}
+
INTERCEPTOR(void, _exit, int status) {
if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
REAL(_exit)(status);
LSAN_MAYBE_INTERCEPT_MALLINFO;
LSAN_MAYBE_INTERCEPT_MALLOPT;
INTERCEPT_FUNCTION(pthread_create);
+ INTERCEPT_FUNCTION(pthread_detach);
INTERCEPT_FUNCTION(pthread_join);
INTERCEPT_FUNCTION(_exit);
OnStartedArgs args;
uptr stack_size = 0;
uptr tls_size = 0;
- GetThreadStackAndTls(tid == 0, &args.stack_begin, &stack_size,
+ GetThreadStackAndTls(tid == kMainTid, &args.stack_begin, &stack_size,
&args.tls_begin, &tls_size);
args.stack_end = args.stack_begin + stack_size;
args.tls_end = args.tls_begin + tls_size;
}
void InitializeMainThread() {
- u32 tid = ThreadCreate(0, 0, true);
- CHECK_EQ(tid, 0);
+ u32 tid = ThreadCreate(kMainTid, 0, true);
+ CHECK_EQ(tid, kMainTid);
ThreadStart(tid, GetTid());
}
namespace __lsan {
-class ThreadContext : public ThreadContextLsanBase {
+class ThreadContext final : public ThreadContextLsanBase {
public:
explicit ThreadContext(int tid);
void OnStarted(void *arg) override;
return new (mem) ThreadContext(tid);
}
-static const uptr kMaxThreads = 1 << 13;
-static const uptr kThreadQuarantineSize = 64;
-
void InitializeThreadRegistry() {
static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
- thread_registry = new (thread_registry_placeholder)
- ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize);
+ thread_registry =
+ new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext);
}
ThreadContextLsanBase::ThreadContextLsanBase(int tid)
return thread_registry->FindThread(FindThreadByUid, (void *)uid);
}
+void ThreadDetach(u32 tid) {
+ CHECK_NE(tid, kInvalidTid);
+ thread_registry->DetachThread(tid, /* arg */ nullptr);
+}
+
void ThreadJoin(u32 tid) {
CHECK_NE(tid, kInvalidTid);
thread_registry->JoinThread(tid, /* arg */ nullptr);
}
void EnsureMainThreadIDIsCorrect() {
- if (GetCurrentThread() == 0)
+ if (GetCurrentThread() == kMainTid)
CurrentThreadContext()->os_id = GetTid();
}
void *onstarted_arg);
protected:
+ ~ThreadContextLsanBase() {}
uptr stack_begin_ = 0;
uptr stack_end_ = 0;
uptr cache_begin_ = 0;
u32 ThreadCreate(u32 tid, uptr uid, bool detached, void *arg = nullptr);
void ThreadFinish();
+void ThreadDetach(u32 tid);
void ThreadJoin(u32 tid);
u32 ThreadTid(uptr uid);
--- /dev/null
+# Build for the Memory Profiler runtime support library.
+
+set(MEMPROF_SOURCES
+ memprof_allocator.cpp
+ memprof_descriptions.cpp
+ memprof_flags.cpp
+ memprof_interceptors.cpp
+ memprof_interceptors_memintrinsics.cpp
+ memprof_linux.cpp
+ memprof_malloc_linux.cpp
+ memprof_posix.cpp
+ memprof_rtl.cpp
+ memprof_shadow_setup.cpp
+ memprof_stack.cpp
+ memprof_stats.cpp
+ memprof_thread.cpp
+ )
+
+set(MEMPROF_CXX_SOURCES
+ memprof_new_delete.cpp
+ )
+
+set(MEMPROF_PREINIT_SOURCES
+ memprof_preinit.cpp
+ )
+
+SET(MEMPROF_HEADERS
+ memprof_allocator.h
+ memprof_descriptions.h
+ memprof_flags.h
+ memprof_flags.inc
+ memprof_init_version.h
+ memprof_interceptors.h
+ memprof_interceptors_memintrinsics.h
+ memprof_interface_internal.h
+ memprof_internal.h
+ memprof_mapping.h
+ memprof_stack.h
+ memprof_stats.h
+ memprof_thread.h
+ )
+
+include_directories(..)
+
+set(MEMPROF_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+set(MEMPROF_COMMON_DEFINITIONS "")
+
+append_rtti_flag(OFF MEMPROF_CFLAGS)
+
+set(MEMPROF_DYNAMIC_LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS})
+
+set(MEMPROF_DYNAMIC_DEFINITIONS
+ ${MEMPROF_COMMON_DEFINITIONS} MEMPROF_DYNAMIC=1)
+
+set(MEMPROF_DYNAMIC_CFLAGS ${MEMPROF_CFLAGS})
+append_list_if(COMPILER_RT_HAS_FTLS_MODEL_INITIAL_EXEC
+ -ftls-model=initial-exec MEMPROF_DYNAMIC_CFLAGS)
+
+set(MEMPROF_DYNAMIC_LIBS ${SANITIZER_CXX_ABI_LIBRARIES} ${SANITIZER_COMMON_LINK_LIBS})
+
+append_list_if(COMPILER_RT_HAS_LIBDL dl MEMPROF_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBRT rt MEMPROF_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBM m MEMPROF_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread MEMPROF_DYNAMIC_LIBS)
+append_list_if(COMPILER_RT_HAS_LIBLOG log MEMPROF_DYNAMIC_LIBS)
+
+# Compile MemProf sources into an object library.
+
+add_compiler_rt_object_libraries(RTMemprof_dynamic
+ OS ${SANITIZER_COMMON_SUPPORTED_OS}
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ SOURCES ${MEMPROF_SOURCES} ${MEMPROF_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${MEMPROF_HEADERS}
+ CFLAGS ${MEMPROF_DYNAMIC_CFLAGS}
+ DEFS ${MEMPROF_DYNAMIC_DEFINITIONS}
+ DEPS ${MEMPROF_DEPS})
+
+add_compiler_rt_object_libraries(RTMemprof
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ SOURCES ${MEMPROF_SOURCES}
+ ADDITIONAL_HEADERS ${MEMPROF_HEADERS}
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ DEPS ${MEMPROF_DEPS})
+add_compiler_rt_object_libraries(RTMemprof_cxx
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ SOURCES ${MEMPROF_CXX_SOURCES}
+ ADDITIONAL_HEADERS ${MEMPROF_HEADERS}
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ DEPS ${MEMPROF_DEPS})
+add_compiler_rt_object_libraries(RTMemprof_preinit
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ SOURCES ${MEMPROF_PREINIT_SOURCES}
+ ADDITIONAL_HEADERS ${MEMPROF_HEADERS}
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ DEPS ${MEMPROF_DEPS})
+
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
+add_compiler_rt_object_libraries(RTMemprof_dynamic_version_script_dummy
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
+ CFLAGS ${MEMPROF_DYNAMIC_CFLAGS}
+ DEFS ${MEMPROF_DYNAMIC_DEFINITIONS}
+ DEPS ${MEMPROF_DEPS})
+
+# Build MemProf runtimes shipped with Clang.
+add_compiler_rt_component(memprof)
+
+# Build separate libraries for each target.
+
+set(MEMPROF_COMMON_RUNTIME_OBJECT_LIBS
+ RTInterception
+ RTSanitizerCommon
+ RTSanitizerCommonLibc
+ RTSanitizerCommonCoverage
+ RTSanitizerCommonSymbolizer)
+
+add_compiler_rt_runtime(clang_rt.memprof
+ STATIC
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ OBJECT_LIBS RTMemprof_preinit
+ RTMemprof
+ ${MEMPROF_COMMON_RUNTIME_OBJECT_LIBS}
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ PARENT_TARGET memprof)
+
+add_compiler_rt_runtime(clang_rt.memprof_cxx
+ STATIC
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ OBJECT_LIBS RTMemprof_cxx
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ PARENT_TARGET memprof)
+
+add_compiler_rt_runtime(clang_rt.memprof-preinit
+ STATIC
+ ARCHS ${MEMPROF_SUPPORTED_ARCH}
+ OBJECT_LIBS RTMemprof_preinit
+ CFLAGS ${MEMPROF_CFLAGS}
+ DEFS ${MEMPROF_COMMON_DEFINITIONS}
+ PARENT_TARGET memprof)
+
+foreach(arch ${MEMPROF_SUPPORTED_ARCH})
+ if (UNIX)
+ add_sanitizer_rt_version_list(clang_rt.memprof-dynamic-${arch}
+ LIBS clang_rt.memprof-${arch} clang_rt.memprof_cxx-${arch}
+ EXTRA memprof.syms.extra)
+ set(VERSION_SCRIPT_FLAG
+ -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.memprof-dynamic-${arch}.vers)
+ set_property(SOURCE
+ ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
+ APPEND PROPERTY
+ OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.memprof-dynamic-${arch}.vers)
+ else()
+ set(VERSION_SCRIPT_FLAG)
+ endif()
+
+ set(MEMPROF_DYNAMIC_WEAK_INTERCEPTION)
+
+ add_compiler_rt_runtime(clang_rt.memprof
+ SHARED
+ ARCHS ${arch}
+ OBJECT_LIBS ${MEMPROF_COMMON_RUNTIME_OBJECT_LIBS}
+ RTMemprof_dynamic
+ # The only purpose of RTMemprof_dynamic_version_script_dummy is to
+ # carry a dependency of the shared runtime on the version script.
+ # Replacing it with a straightforward
+ # add_dependencies(clang_rt.memprof-dynamic-${arch} clang_rt.memprof-dynamic-${arch}-version-list)
+ # generates an order-only dependency in ninja.
+ RTMemprof_dynamic_version_script_dummy
+ ${MEMPROF_DYNAMIC_WEAK_INTERCEPTION}
+ CFLAGS ${MEMPROF_DYNAMIC_CFLAGS}
+ LINK_FLAGS ${MEMPROF_DYNAMIC_LINK_FLAGS}
+ ${VERSION_SCRIPT_FLAG}
+ LINK_LIBS ${MEMPROF_DYNAMIC_LIBS}
+ DEFS ${MEMPROF_DYNAMIC_DEFINITIONS}
+ PARENT_TARGET memprof)
+
+ if (SANITIZER_USE_SYMBOLS)
+ add_sanitizer_rt_symbols(clang_rt.memprof_cxx
+ ARCHS ${arch})
+ add_dependencies(memprof clang_rt.memprof_cxx-${arch}-symbols)
+ add_sanitizer_rt_symbols(clang_rt.memprof
+ ARCHS ${arch}
+ EXTRA memprof.syms.extra)
+ add_dependencies(memprof clang_rt.memprof-${arch}-symbols)
+ endif()
+endforeach()
--- /dev/null
+MemProfiling RT
+================================
+This directory contains sources of the MemProfiling (MemProf) runtime library.
+
+Directory structure:
+README.txt : This file.
+CMakeLists.txt : File for cmake-based build.
+memprof_*.{cc,h} : Sources of the memprof runtime library.
+
+Also MemProf runtime needs the following libraries:
+lib/interception/ : Machinery used to intercept function calls.
+lib/sanitizer_common/ : Code shared between various sanitizers.
+
+MemProf runtime can only be built by CMake. You can run MemProf tests
+from the root of your CMake build tree:
+
+make check-memprof
--- /dev/null
+__memprof_*
--- /dev/null
+//===-- memprof_allocator.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Implementation of MemProf's memory allocator, which uses the allocator
+// from sanitizer_common.
+//
+//===----------------------------------------------------------------------===//
+
+#include "memprof_allocator.h"
+#include "memprof_mapping.h"
+#include "memprof_stack.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_file.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_list.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+#include <sched.h>
+#include <stdlib.h>
+#include <time.h>
+
+namespace __memprof {
+
+static int GetCpuId(void) {
+ // _memprof_preinit is called via the preinit_array, which subsequently calls
+ // malloc. Since this is before _dl_init calls VDSO_SETUP, sched_getcpu
+ // will seg fault as the address of __vdso_getcpu will be null.
+ if (!memprof_init_done)
+ return -1;
+ return sched_getcpu();
+}
+
+// Compute the timestamp in ms.
+static int GetTimestamp(void) {
+ // timespec_get will segfault if called from dl_init
+ if (!memprof_timestamp_inited) {
+ // By returning 0, this will be effectively treated as being
+ // timestamped at memprof init time (when memprof_init_timestamp_s
+ // is initialized).
+ return 0;
+ }
+ timespec ts;
+ clock_gettime(CLOCK_REALTIME, &ts);
+ return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000;
+}
+
+static MemprofAllocator &get_allocator();
+
+// The memory chunk allocated from the underlying allocator looks like this:
+// H H U U U U U U
+// H -- ChunkHeader (32 bytes)
+// U -- user memory.
+
+// If there is left padding before the ChunkHeader (due to use of memalign),
+// we store a magic value in the first uptr word of the memory block and
+// store the address of ChunkHeader in the next uptr.
+// M B L L L L L L L L L H H U U U U U U
+// | ^
+// ---------------------|
+// M -- magic value kAllocBegMagic
+// B -- address of ChunkHeader pointing to the first 'H'
+
+constexpr uptr kMaxAllowedMallocBits = 40;
+
+// Should be no more than 32-bytes
+struct ChunkHeader {
+ // 1-st 4 bytes.
+ u32 alloc_context_id;
+ // 2-nd 4 bytes
+ u32 cpu_id;
+ // 3-rd 4 bytes
+ u32 timestamp_ms;
+ // 4-th 4 bytes
+ // Note only 1 bit is needed for this flag if we need space in the future for
+ // more fields.
+ u32 from_memalign;
+ // 5-th and 6-th 4 bytes
+ // The max size of an allocation is 2^40 (kMaxAllowedMallocSize), so this
+ // could be shrunk to kMaxAllowedMallocBits if we need space in the future for
+ // more fields.
+ atomic_uint64_t user_requested_size;
+ // 23 bits available
+ // 7-th and 8-th 4 bytes
+ u64 data_type_id; // TODO: hash of type name
+};
+
+static const uptr kChunkHeaderSize = sizeof(ChunkHeader);
+COMPILER_CHECK(kChunkHeaderSize == 32);
+
+struct MemprofChunk : ChunkHeader {
+ uptr Beg() { return reinterpret_cast<uptr>(this) + kChunkHeaderSize; }
+ uptr UsedSize() {
+ return atomic_load(&user_requested_size, memory_order_relaxed);
+ }
+ void *AllocBeg() {
+ if (from_memalign)
+ return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this));
+ return reinterpret_cast<void *>(this);
+ }
+};
+
+class LargeChunkHeader {
+ static constexpr uptr kAllocBegMagic =
+ FIRST_32_SECOND_64(0xCC6E96B9, 0xCC6E96B9CC6E96B9ULL);
+ atomic_uintptr_t magic;
+ MemprofChunk *chunk_header;
+
+public:
+ MemprofChunk *Get() const {
+ return atomic_load(&magic, memory_order_acquire) == kAllocBegMagic
+ ? chunk_header
+ : nullptr;
+ }
+
+ void Set(MemprofChunk *p) {
+ if (p) {
+ chunk_header = p;
+ atomic_store(&magic, kAllocBegMagic, memory_order_release);
+ return;
+ }
+
+ uptr old = kAllocBegMagic;
+ if (!atomic_compare_exchange_strong(&magic, &old, 0,
+ memory_order_release)) {
+ CHECK_EQ(old, kAllocBegMagic);
+ }
+ }
+};
+
+void FlushUnneededMemProfShadowMemory(uptr p, uptr size) {
+ // Since memprof's mapping is compacting, the shadow chunk may be
+ // not page-aligned, so we only flush the page-aligned portion.
+ ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+}
+
+void MemprofMapUnmapCallback::OnMap(uptr p, uptr size) const {
+ // Statistics.
+ MemprofStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.mmaps++;
+ thread_stats.mmaped += size;
+}
+void MemprofMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
+ // We are about to unmap a chunk of user memory.
+ // Mark the corresponding shadow memory as not needed.
+ FlushUnneededMemProfShadowMemory(p, size);
+ // Statistics.
+ MemprofStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.munmaps++;
+ thread_stats.munmaped += size;
+}
+
+AllocatorCache *GetAllocatorCache(MemprofThreadLocalMallocStorage *ms) {
+ CHECK(ms);
+ return &ms->allocator_cache;
+}
+
+struct MemInfoBlock {
+ u32 alloc_count;
+ u64 total_access_count, min_access_count, max_access_count;
+ u64 total_size;
+ u32 min_size, max_size;
+ u32 alloc_timestamp, dealloc_timestamp;
+ u64 total_lifetime;
+ u32 min_lifetime, max_lifetime;
+ u32 alloc_cpu_id, dealloc_cpu_id;
+ u32 num_migrated_cpu;
+
+ // Only compared to prior deallocated object currently.
+ u32 num_lifetime_overlaps;
+ u32 num_same_alloc_cpu;
+ u32 num_same_dealloc_cpu;
+
+ u64 data_type_id; // TODO: hash of type name
+
+ MemInfoBlock() : alloc_count(0) {}
+
+ MemInfoBlock(u32 size, u64 access_count, u32 alloc_timestamp,
+ u32 dealloc_timestamp, u32 alloc_cpu, u32 dealloc_cpu)
+ : alloc_count(1), total_access_count(access_count),
+ min_access_count(access_count), max_access_count(access_count),
+ total_size(size), min_size(size), max_size(size),
+ alloc_timestamp(alloc_timestamp), dealloc_timestamp(dealloc_timestamp),
+ total_lifetime(dealloc_timestamp - alloc_timestamp),
+ min_lifetime(total_lifetime), max_lifetime(total_lifetime),
+ alloc_cpu_id(alloc_cpu), dealloc_cpu_id(dealloc_cpu),
+ num_lifetime_overlaps(0), num_same_alloc_cpu(0),
+ num_same_dealloc_cpu(0) {
+ num_migrated_cpu = alloc_cpu_id != dealloc_cpu_id;
+ }
+
+ void Print(u64 id) {
+ u64 p;
+ if (flags()->print_terse) {
+ p = total_size * 100 / alloc_count;
+ Printf("MIB:%llu/%u/%d.%02d/%u/%u/", id, alloc_count, p / 100, p % 100,
+ min_size, max_size);
+ p = total_access_count * 100 / alloc_count;
+ Printf("%d.%02d/%u/%u/", p / 100, p % 100, min_access_count,
+ max_access_count);
+ p = total_lifetime * 100 / alloc_count;
+ Printf("%d.%02d/%u/%u/", p / 100, p % 100, min_lifetime, max_lifetime);
+ Printf("%u/%u/%u/%u\n", num_migrated_cpu, num_lifetime_overlaps,
+ num_same_alloc_cpu, num_same_dealloc_cpu);
+ } else {
+ p = total_size * 100 / alloc_count;
+ Printf("Memory allocation stack id = %llu\n", id);
+ Printf("\talloc_count %u, size (ave/min/max) %d.%02d / %u / %u\n",
+ alloc_count, p / 100, p % 100, min_size, max_size);
+ p = total_access_count * 100 / alloc_count;
+ Printf("\taccess_count (ave/min/max): %d.%02d / %u / %u\n", p / 100,
+ p % 100, min_access_count, max_access_count);
+ p = total_lifetime * 100 / alloc_count;
+ Printf("\tlifetime (ave/min/max): %d.%02d / %u / %u\n", p / 100, p % 100,
+ min_lifetime, max_lifetime);
+ Printf("\tnum migrated: %u, num lifetime overlaps: %u, num same alloc "
+ "cpu: %u, num same dealloc_cpu: %u\n",
+ num_migrated_cpu, num_lifetime_overlaps, num_same_alloc_cpu,
+ num_same_dealloc_cpu);
+ }
+ }
+
+ static void printHeader() {
+ CHECK(flags()->print_terse);
+ Printf("MIB:StackID/AllocCount/AveSize/MinSize/MaxSize/AveAccessCount/"
+ "MinAccessCount/MaxAccessCount/AveLifetime/MinLifetime/MaxLifetime/"
+ "NumMigratedCpu/NumLifetimeOverlaps/NumSameAllocCpu/"
+ "NumSameDeallocCpu\n");
+ }
+
+ void Merge(MemInfoBlock &newMIB) {
+ alloc_count += newMIB.alloc_count;
+
+ total_access_count += newMIB.total_access_count;
+ min_access_count = Min(min_access_count, newMIB.min_access_count);
+ max_access_count = Max(max_access_count, newMIB.max_access_count);
+
+ total_size += newMIB.total_size;
+ min_size = Min(min_size, newMIB.min_size);
+ max_size = Max(max_size, newMIB.max_size);
+
+ total_lifetime += newMIB.total_lifetime;
+ min_lifetime = Min(min_lifetime, newMIB.min_lifetime);
+ max_lifetime = Max(max_lifetime, newMIB.max_lifetime);
+
+ // We know newMIB was deallocated later, so just need to check if it was
+ // allocated before last one deallocated.
+ num_lifetime_overlaps += newMIB.alloc_timestamp < dealloc_timestamp;
+ alloc_timestamp = newMIB.alloc_timestamp;
+ dealloc_timestamp = newMIB.dealloc_timestamp;
+
+ num_same_alloc_cpu += alloc_cpu_id == newMIB.alloc_cpu_id;
+ num_same_dealloc_cpu += dealloc_cpu_id == newMIB.dealloc_cpu_id;
+ alloc_cpu_id = newMIB.alloc_cpu_id;
+ dealloc_cpu_id = newMIB.dealloc_cpu_id;
+ }
+};
+
+static u32 AccessCount = 0;
+static u32 MissCount = 0;
+
+struct SetEntry {
+ SetEntry() : id(0), MIB() {}
+ bool Empty() { return id == 0; }
+ void Print() {
+ CHECK(!Empty());
+ MIB.Print(id);
+ }
+ // The stack id
+ u64 id;
+ MemInfoBlock MIB;
+};
+
+struct CacheSet {
+ enum { kSetSize = 4 };
+
+ void PrintAll() {
+ for (int i = 0; i < kSetSize; i++) {
+ if (Entries[i].Empty())
+ continue;
+ Entries[i].Print();
+ }
+ }
+ void insertOrMerge(u64 new_id, MemInfoBlock &newMIB) {
+ AccessCount++;
+ SetAccessCount++;
+
+ for (int i = 0; i < kSetSize; i++) {
+ auto id = Entries[i].id;
+ // Check if this is a hit or an empty entry. Since we always move any
+ // filled locations to the front of the array (see below), we don't need
+ // to look after finding the first empty entry.
+ if (id == new_id || !id) {
+ if (id == 0) {
+ Entries[i].id = new_id;
+ Entries[i].MIB = newMIB;
+ } else {
+ Entries[i].MIB.Merge(newMIB);
+ }
+ // Assuming some id locality, we try to swap the matching entry
+ // into the first set position.
+ if (i != 0) {
+ auto tmp = Entries[0];
+ Entries[0] = Entries[i];
+ Entries[i] = tmp;
+ }
+ return;
+ }
+ }
+
+ // Miss
+ MissCount++;
+ SetMissCount++;
+
+ // We try to find the entries with the lowest alloc count to be evicted:
+ int min_idx = 0;
+ u64 min_count = Entries[0].MIB.alloc_count;
+ for (int i = 1; i < kSetSize; i++) {
+ CHECK(!Entries[i].Empty());
+ if (Entries[i].MIB.alloc_count < min_count) {
+ min_idx = i;
+ min_count = Entries[i].MIB.alloc_count;
+ }
+ }
+
+ // Print the evicted entry profile information
+ if (!flags()->print_terse)
+ Printf("Evicted:\n");
+ Entries[min_idx].Print();
+
+ // Similar to the hit case, put new MIB in first set position.
+ if (min_idx != 0)
+ Entries[min_idx] = Entries[0];
+ Entries[0].id = new_id;
+ Entries[0].MIB = newMIB;
+ }
+
+ void PrintMissRate(int i) {
+ u64 p = SetAccessCount ? SetMissCount * 10000ULL / SetAccessCount : 0;
+ Printf("Set %d miss rate: %d / %d = %5d.%02d%%\n", i, SetMissCount,
+ SetAccessCount, p / 100, p % 100);
+ }
+
+ SetEntry Entries[kSetSize];
+ u32 SetAccessCount = 0;
+ u32 SetMissCount = 0;
+};
+
+struct MemInfoBlockCache {
+ MemInfoBlockCache() {
+ if (common_flags()->print_module_map)
+ DumpProcessMap();
+ if (flags()->print_terse)
+ MemInfoBlock::printHeader();
+ Sets =
+ (CacheSet *)malloc(sizeof(CacheSet) * flags()->mem_info_cache_entries);
+ Constructed = true;
+ }
+
+ ~MemInfoBlockCache() { free(Sets); }
+
+ void insertOrMerge(u64 new_id, MemInfoBlock &newMIB) {
+ u64 hv = new_id;
+
+ // Use mod method where number of entries should be a prime close to power
+ // of 2.
+ hv %= flags()->mem_info_cache_entries;
+
+ return Sets[hv].insertOrMerge(new_id, newMIB);
+ }
+
+ void PrintAll() {
+ for (int i = 0; i < flags()->mem_info_cache_entries; i++) {
+ Sets[i].PrintAll();
+ }
+ }
+
+ void PrintMissRate() {
+ if (!flags()->print_mem_info_cache_miss_rate)
+ return;
+ u64 p = AccessCount ? MissCount * 10000ULL / AccessCount : 0;
+ Printf("Overall miss rate: %d / %d = %5d.%02d%%\n", MissCount, AccessCount,
+ p / 100, p % 100);
+ if (flags()->print_mem_info_cache_miss_rate_details)
+ for (int i = 0; i < flags()->mem_info_cache_entries; i++)
+ Sets[i].PrintMissRate(i);
+ }
+
+ CacheSet *Sets;
+ // Flag when the Sets have been allocated, in case a deallocation is called
+ // very early before the static init of the Allocator and therefore this table
+ // have completed.
+ bool Constructed = false;
+};
+
+// Accumulates the access count from the shadow for the given pointer and size.
+u64 GetShadowCount(uptr p, u32 size) {
+ u64 *shadow = (u64 *)MEM_TO_SHADOW(p);
+ u64 *shadow_end = (u64 *)MEM_TO_SHADOW(p + size);
+ u64 count = 0;
+ for (; shadow <= shadow_end; shadow++)
+ count += *shadow;
+ return count;
+}
+
+// Clears the shadow counters (when memory is allocated).
+void ClearShadow(uptr addr, uptr size) {
+ CHECK(AddrIsAlignedByGranularity(addr));
+ CHECK(AddrIsInMem(addr));
+ CHECK(AddrIsAlignedByGranularity(addr + size));
+ CHECK(AddrIsInMem(addr + size - SHADOW_GRANULARITY));
+ CHECK(REAL(memset));
+ uptr shadow_beg = MEM_TO_SHADOW(addr);
+ uptr shadow_end = MEM_TO_SHADOW(addr + size - SHADOW_GRANULARITY) + 1;
+ if (shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
+ REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
+ } else {
+ uptr page_size = GetPageSizeCached();
+ uptr page_beg = RoundUpTo(shadow_beg, page_size);
+ uptr page_end = RoundDownTo(shadow_end, page_size);
+
+ if (page_beg >= page_end) {
+ REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
+ } else {
+ if (page_beg != shadow_beg) {
+ REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
+ }
+ if (page_end != shadow_end) {
+ REAL(memset)((void *)page_end, 0, shadow_end - page_end);
+ }
+ ReserveShadowMemoryRange(page_beg, page_end - 1, nullptr);
+ }
+ }
+}
+
+struct Allocator {
+ static const uptr kMaxAllowedMallocSize = 1ULL << kMaxAllowedMallocBits;
+
+ MemprofAllocator allocator;
+ StaticSpinMutex fallback_mutex;
+ AllocatorCache fallback_allocator_cache;
+
+ uptr max_user_defined_malloc_size;
+ atomic_uint8_t rss_limit_exceeded;
+
+ MemInfoBlockCache MemInfoBlockTable;
+ bool destructing;
+
+ // ------------------- Initialization ------------------------
+ explicit Allocator(LinkerInitialized) : destructing(false) {}
+
+ ~Allocator() { FinishAndPrint(); }
+
+ void FinishAndPrint() {
+ if (!flags()->print_terse)
+ Printf("Live on exit:\n");
+ allocator.ForceLock();
+ allocator.ForEachChunk(
+ [](uptr chunk, void *alloc) {
+ u64 user_requested_size;
+ MemprofChunk *m =
+ ((Allocator *)alloc)
+ ->GetMemprofChunk((void *)chunk, user_requested_size);
+ if (!m)
+ return;
+ uptr user_beg = ((uptr)m) + kChunkHeaderSize;
+ u64 c = GetShadowCount(user_beg, user_requested_size);
+ long curtime = GetTimestamp();
+ MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
+ m->cpu_id, GetCpuId());
+ ((Allocator *)alloc)
+ ->MemInfoBlockTable.insertOrMerge(m->alloc_context_id, newMIB);
+ },
+ this);
+ allocator.ForceUnlock();
+
+ destructing = true;
+ MemInfoBlockTable.PrintMissRate();
+ MemInfoBlockTable.PrintAll();
+ StackDepotPrintAll();
+ }
+
+ void InitLinkerInitialized() {
+ SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+ allocator.InitLinkerInitialized(
+ common_flags()->allocator_release_to_os_interval_ms);
+ max_user_defined_malloc_size = common_flags()->max_allocation_size_mb
+ ? common_flags()->max_allocation_size_mb
+ << 20
+ : kMaxAllowedMallocSize;
+ }
+
+ bool RssLimitExceeded() {
+ return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
+ }
+
+ void SetRssLimitExceeded(bool limit_exceeded) {
+ atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
+ }
+
+ // -------------------- Allocation/Deallocation routines ---------------
+ void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
+ AllocType alloc_type) {
+ if (UNLIKELY(!memprof_inited))
+ MemprofInitFromRtl();
+ if (RssLimitExceeded()) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportRssLimitExceeded(stack);
+ }
+ CHECK(stack);
+ const uptr min_alignment = MEMPROF_ALIGNMENT;
+ if (alignment < min_alignment)
+ alignment = min_alignment;
+ if (size == 0) {
+ // We'd be happy to avoid allocating memory for zero-size requests, but
+ // some programs/tests depend on this behavior and assume that malloc
+ // would not return NULL even for zero-size allocations. Moreover, it
+ // looks like operator new should never return NULL, and results of
+ // consecutive "new" calls must be different even if the allocated size
+ // is zero.
+ size = 1;
+ }
+ CHECK(IsPowerOfTwo(alignment));
+ uptr rounded_size = RoundUpTo(size, alignment);
+ uptr needed_size = rounded_size + kChunkHeaderSize;
+ if (alignment > min_alignment)
+ needed_size += alignment;
+ CHECK(IsAligned(needed_size, min_alignment));
+ if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
+ size > max_user_defined_malloc_size) {
+ if (AllocatorMayReturnNull()) {
+ Report("WARNING: MemProfiler failed to allocate 0x%zx bytes\n",
+ (void *)size);
+ return nullptr;
+ }
+ uptr malloc_limit =
+ Min(kMaxAllowedMallocSize, max_user_defined_malloc_size);
+ ReportAllocationSizeTooBig(size, malloc_limit, stack);
+ }
+
+ MemprofThread *t = GetCurrentThread();
+ void *allocated;
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocated = allocator.Allocate(cache, needed_size, 8);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocated = allocator.Allocate(cache, needed_size, 8);
+ }
+ if (UNLIKELY(!allocated)) {
+ SetAllocatorOutOfMemory();
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportOutOfMemory(size, stack);
+ }
+
+ uptr alloc_beg = reinterpret_cast<uptr>(allocated);
+ uptr alloc_end = alloc_beg + needed_size;
+ uptr beg_plus_header = alloc_beg + kChunkHeaderSize;
+ uptr user_beg = beg_plus_header;
+ if (!IsAligned(user_beg, alignment))
+ user_beg = RoundUpTo(user_beg, alignment);
+ uptr user_end = user_beg + size;
+ CHECK_LE(user_end, alloc_end);
+ uptr chunk_beg = user_beg - kChunkHeaderSize;
+ MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
+ m->from_memalign = alloc_beg != chunk_beg;
+ CHECK(size);
+
+ m->cpu_id = GetCpuId();
+ m->timestamp_ms = GetTimestamp();
+ m->alloc_context_id = StackDepotPut(*stack);
+
+ uptr size_rounded_down_to_granularity =
+ RoundDownTo(size, SHADOW_GRANULARITY);
+ if (size_rounded_down_to_granularity)
+ ClearShadow(user_beg, size_rounded_down_to_granularity);
+
+ MemprofStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.mallocs++;
+ thread_stats.malloced += size;
+ thread_stats.malloced_overhead += needed_size - size;
+ if (needed_size > SizeClassMap::kMaxSize)
+ thread_stats.malloc_large++;
+ else
+ thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++;
+
+ void *res = reinterpret_cast<void *>(user_beg);
+ atomic_store(&m->user_requested_size, size, memory_order_release);
+ if (alloc_beg != chunk_beg) {
+ CHECK_LE(alloc_beg + sizeof(LargeChunkHeader), chunk_beg);
+ reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(m);
+ }
+ MEMPROF_MALLOC_HOOK(res, size);
+ return res;
+ }
+
+ void Deallocate(void *ptr, uptr delete_size, uptr delete_alignment,
+ BufferedStackTrace *stack, AllocType alloc_type) {
+ uptr p = reinterpret_cast<uptr>(ptr);
+ if (p == 0)
+ return;
+
+ MEMPROF_FREE_HOOK(ptr);
+
+ uptr chunk_beg = p - kChunkHeaderSize;
+ MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
+
+ u64 user_requested_size =
+ atomic_exchange(&m->user_requested_size, 0, memory_order_acquire);
+ if (memprof_inited && memprof_init_done && !destructing &&
+ MemInfoBlockTable.Constructed) {
+ u64 c = GetShadowCount(p, user_requested_size);
+ long curtime = GetTimestamp();
+
+ MemInfoBlock newMIB(user_requested_size, c, m->timestamp_ms, curtime,
+ m->cpu_id, GetCpuId());
+ {
+ SpinMutexLock l(&fallback_mutex);
+ MemInfoBlockTable.insertOrMerge(m->alloc_context_id, newMIB);
+ }
+ }
+
+ MemprofStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.frees++;
+ thread_stats.freed += user_requested_size;
+
+ void *alloc_beg = m->AllocBeg();
+ if (alloc_beg != m) {
+ // Clear the magic value, as allocator internals may overwrite the
+ // contents of deallocated chunk, confusing GetMemprofChunk lookup.
+ reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Set(nullptr);
+ }
+
+ MemprofThread *t = GetCurrentThread();
+ if (t) {
+ AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+ allocator.Deallocate(cache, alloc_beg);
+ } else {
+ SpinMutexLock l(&fallback_mutex);
+ AllocatorCache *cache = &fallback_allocator_cache;
+ allocator.Deallocate(cache, alloc_beg);
+ }
+ }
+
+ void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) {
+ CHECK(old_ptr && new_size);
+ uptr p = reinterpret_cast<uptr>(old_ptr);
+ uptr chunk_beg = p - kChunkHeaderSize;
+ MemprofChunk *m = reinterpret_cast<MemprofChunk *>(chunk_beg);
+
+ MemprofStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.reallocs++;
+ thread_stats.realloced += new_size;
+
+ void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC);
+ if (new_ptr) {
+ CHECK_NE(REAL(memcpy), nullptr);
+ uptr memcpy_size = Min(new_size, m->UsedSize());
+ REAL(memcpy)(new_ptr, old_ptr, memcpy_size);
+ Deallocate(old_ptr, 0, 0, stack, FROM_MALLOC);
+ }
+ return new_ptr;
+ }
+
+ void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportCallocOverflow(nmemb, size, stack);
+ }
+ void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC);
+ // If the memory comes from the secondary allocator no need to clear it
+ // as it comes directly from mmap.
+ if (ptr && allocator.FromPrimary(ptr))
+ REAL(memset)(ptr, 0, nmemb * size);
+ return ptr;
+ }
+
+ void CommitBack(MemprofThreadLocalMallocStorage *ms,
+ BufferedStackTrace *stack) {
+ AllocatorCache *ac = GetAllocatorCache(ms);
+ allocator.SwallowCache(ac);
+ }
+
+ // -------------------------- Chunk lookup ----------------------
+
+ // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
+ MemprofChunk *GetMemprofChunk(void *alloc_beg, u64 &user_requested_size) {
+ if (!alloc_beg)
+ return nullptr;
+ MemprofChunk *p = reinterpret_cast<LargeChunkHeader *>(alloc_beg)->Get();
+ if (!p) {
+ if (!allocator.FromPrimary(alloc_beg))
+ return nullptr;
+ p = reinterpret_cast<MemprofChunk *>(alloc_beg);
+ }
+ // The size is reset to 0 on deallocation (and a min of 1 on
+ // allocation).
+ user_requested_size =
+ atomic_load(&p->user_requested_size, memory_order_acquire);
+ if (user_requested_size)
+ return p;
+ return nullptr;
+ }
+
+ MemprofChunk *GetMemprofChunkByAddr(uptr p, u64 &user_requested_size) {
+ void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast<void *>(p));
+ return GetMemprofChunk(alloc_beg, user_requested_size);
+ }
+
+ uptr AllocationSize(uptr p) {
+ u64 user_requested_size;
+ MemprofChunk *m = GetMemprofChunkByAddr(p, user_requested_size);
+ if (!m)
+ return 0;
+ if (m->Beg() != p)
+ return 0;
+ return user_requested_size;
+ }
+
+ void Purge(BufferedStackTrace *stack) { allocator.ForceReleaseToOS(); }
+
+ void PrintStats() { allocator.PrintStats(); }
+
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+ allocator.ForceLock();
+ fallback_mutex.Lock();
+ }
+
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+ fallback_mutex.Unlock();
+ allocator.ForceUnlock();
+ }
+};
+
+static Allocator instance(LINKER_INITIALIZED);
+
+static MemprofAllocator &get_allocator() { return instance.allocator; }
+
+void InitializeAllocator() { instance.InitLinkerInitialized(); }
+
+void MemprofThreadLocalMallocStorage::CommitBack() {
+ GET_STACK_TRACE_MALLOC;
+ instance.CommitBack(this, &stack);
+}
+
+void PrintInternalAllocatorStats() { instance.PrintStats(); }
+
+void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
+ instance.Deallocate(ptr, 0, 0, stack, alloc_type);
+}
+
+void memprof_delete(void *ptr, uptr size, uptr alignment,
+ BufferedStackTrace *stack, AllocType alloc_type) {
+ instance.Deallocate(ptr, size, alignment, stack, alloc_type);
+}
+
+void *memprof_malloc(uptr size, BufferedStackTrace *stack) {
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC));
+}
+
+void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
+ return SetErrnoOnNull(instance.Calloc(nmemb, size, stack));
+}
+
+void *memprof_reallocarray(void *p, uptr nmemb, uptr size,
+ BufferedStackTrace *stack) {
+ if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportReallocArrayOverflow(nmemb, size, stack);
+ }
+ return memprof_realloc(p, nmemb * size, stack);
+}
+
+void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack) {
+ if (!p)
+ return SetErrnoOnNull(instance.Allocate(size, 8, stack, FROM_MALLOC));
+ if (size == 0) {
+ if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
+ instance.Deallocate(p, 0, 0, stack, FROM_MALLOC);
+ return nullptr;
+ }
+ // Allocate a size of 1 if we shouldn't free() on Realloc to 0
+ size = 1;
+ }
+ return SetErrnoOnNull(instance.Reallocate(p, size, stack));
+}
+
+void *memprof_valloc(uptr size, BufferedStackTrace *stack) {
+ return SetErrnoOnNull(
+ instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC));
+}
+
+void *memprof_pvalloc(uptr size, BufferedStackTrace *stack) {
+ uptr PageSize = GetPageSizeCached();
+ if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
+ errno = errno_ENOMEM;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportPvallocOverflow(size, stack);
+ }
+ // pvalloc(0) should allocate one page.
+ size = size ? RoundUpTo(size, PageSize) : PageSize;
+ return SetErrnoOnNull(instance.Allocate(size, PageSize, stack, FROM_MALLOC));
+}
+
+void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
+ AllocType alloc_type) {
+ if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAllocationAlignment(alignment, stack);
+ }
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, alloc_type));
+}
+
+void *memprof_aligned_alloc(uptr alignment, uptr size,
+ BufferedStackTrace *stack) {
+ if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
+ errno = errno_EINVAL;
+ if (AllocatorMayReturnNull())
+ return nullptr;
+ ReportInvalidAlignedAllocAlignment(size, alignment, stack);
+ }
+ return SetErrnoOnNull(instance.Allocate(size, alignment, stack, FROM_MALLOC));
+}
+
+int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
+ BufferedStackTrace *stack) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
+ if (AllocatorMayReturnNull())
+ return errno_EINVAL;
+ ReportInvalidPosixMemalignAlignment(alignment, stack);
+ }
+ void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC);
+ if (UNLIKELY(!ptr))
+ // OOM error is already taken care of by Allocate.
+ return errno_ENOMEM;
+ CHECK(IsAligned((uptr)ptr, alignment));
+ *memptr = ptr;
+ return 0;
+}
+
+uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
+ if (!ptr)
+ return 0;
+ uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr));
+ return usable_size;
+}
+
+void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) {
+ instance.SetRssLimitExceeded(limit_exceeded);
+}
+
+} // namespace __memprof
+
+// ---------------------- Interface ---------------- {{{1
+using namespace __memprof;
+
+#if !SANITIZER_SUPPORTS_WEAK_HOOKS
+// Provide default (no-op) implementation of malloc hooks.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr,
+ uptr size) {
+ (void)ptr;
+ (void)size;
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
+ (void)ptr;
+}
+#endif
+
+uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
+
+int __sanitizer_get_ownership(const void *p) {
+ return memprof_malloc_usable_size(p, 0, 0) != 0;
+}
+
+uptr __sanitizer_get_allocated_size(const void *p) {
+ return memprof_malloc_usable_size(p, 0, 0);
+}
+
+int __memprof_profile_dump() {
+ instance.FinishAndPrint();
+ // In the future we may want to return non-zero if there are any errors
+ // detected during the dumping process.
+ return 0;
+}
--- /dev/null
+//===-- memprof_allocator.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_allocator.cpp.
+//===----------------------------------------------------------------------===//
+
+#ifndef MEMPROF_ALLOCATOR_H
+#define MEMPROF_ALLOCATOR_H
+
+#include "memprof_flags.h"
+#include "memprof_interceptors.h"
+#include "memprof_internal.h"
+#include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_list.h"
+
+#if !defined(__x86_64__)
+#error Unsupported platform
+#endif
+#if !SANITIZER_CAN_USE_ALLOCATOR64
+#error Only 64-bit allocator supported
+#endif
+
+namespace __memprof {
+
+enum AllocType {
+ FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc.
+ FROM_NEW = 2, // Memory block came from operator new.
+ FROM_NEW_BR = 3 // Memory block came from operator new [ ]
+};
+
+void InitializeAllocator();
+
+struct MemprofMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const;
+ void OnUnmap(uptr p, uptr size) const;
+};
+
+constexpr uptr kAllocatorSpace = 0x600000000000ULL;
+constexpr uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+typedef DefaultSizeClassMap SizeClassMap;
+template <typename AddressSpaceViewTy>
+struct AP64 { // Allocator64 parameters. Deliberately using a short name.
+ static const uptr kSpaceBeg = kAllocatorSpace;
+ static const uptr kSpaceSize = kAllocatorSize;
+ static const uptr kMetadataSize = 0;
+ typedef __memprof::SizeClassMap SizeClassMap;
+ typedef MemprofMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags = 0;
+ using AddressSpaceView = AddressSpaceViewTy;
+};
+
+template <typename AddressSpaceView>
+using PrimaryAllocatorASVT = SizeClassAllocator64<AP64<AddressSpaceView>>;
+using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>;
+
+static const uptr kNumberOfSizeClasses = SizeClassMap::kNumClasses;
+
+template <typename AddressSpaceView>
+using MemprofAllocatorASVT =
+ CombinedAllocator<PrimaryAllocatorASVT<AddressSpaceView>>;
+using MemprofAllocator = MemprofAllocatorASVT<LocalAddressSpaceView>;
+using AllocatorCache = MemprofAllocator::AllocatorCache;
+
+struct MemprofThreadLocalMallocStorage {
+ AllocatorCache allocator_cache;
+ void CommitBack();
+
+private:
+ // These objects are allocated via mmap() and are zero-initialized.
+ MemprofThreadLocalMallocStorage() {}
+};
+
+void *memprof_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
+ AllocType alloc_type);
+void memprof_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type);
+void memprof_delete(void *ptr, uptr size, uptr alignment,
+ BufferedStackTrace *stack, AllocType alloc_type);
+
+void *memprof_malloc(uptr size, BufferedStackTrace *stack);
+void *memprof_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack);
+void *memprof_realloc(void *p, uptr size, BufferedStackTrace *stack);
+void *memprof_reallocarray(void *p, uptr nmemb, uptr size,
+ BufferedStackTrace *stack);
+void *memprof_valloc(uptr size, BufferedStackTrace *stack);
+void *memprof_pvalloc(uptr size, BufferedStackTrace *stack);
+
+void *memprof_aligned_alloc(uptr alignment, uptr size,
+ BufferedStackTrace *stack);
+int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
+ BufferedStackTrace *stack);
+uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp);
+
+void PrintInternalAllocatorStats();
+void MemprofSoftRssLimitExceededCallback(bool exceeded);
+
+} // namespace __memprof
+#endif // MEMPROF_ALLOCATOR_H
--- /dev/null
+//===-- memprof_descriptions.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf functions for getting information about an address and/or printing
+// it.
+//===----------------------------------------------------------------------===//
+
+#include "memprof_descriptions.h"
+#include "memprof_mapping.h"
+#include "memprof_stack.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+namespace __memprof {
+
+MemprofThreadIdAndName::MemprofThreadIdAndName(MemprofThreadContext *t) {
+ Init(t->tid, t->name);
+}
+
+MemprofThreadIdAndName::MemprofThreadIdAndName(u32 tid) {
+ if (tid == kInvalidTid) {
+ Init(tid, "");
+ } else {
+ memprofThreadRegistry().CheckLocked();
+ MemprofThreadContext *t = GetThreadContextByTidLocked(tid);
+ Init(tid, t->name);
+ }
+}
+
+void MemprofThreadIdAndName::Init(u32 tid, const char *tname) {
+ int len = internal_snprintf(name, sizeof(name), "T%d", tid);
+ CHECK(((unsigned int)len) < sizeof(name));
+ if (tname[0] != '\0')
+ internal_snprintf(&name[len], sizeof(name) - len, " (%s)", tname);
+}
+
+void DescribeThread(MemprofThreadContext *context) {
+ CHECK(context);
+ memprofThreadRegistry().CheckLocked();
+ // No need to announce the main thread.
+ if (context->tid == kMainTid || context->announced) {
+ return;
+ }
+ context->announced = true;
+ InternalScopedString str;
+ str.append("Thread %s", MemprofThreadIdAndName(context).c_str());
+ if (context->parent_tid == kInvalidTid) {
+ str.append(" created by unknown thread\n");
+ Printf("%s", str.data());
+ return;
+ }
+ str.append(" created by %s here:\n",
+ MemprofThreadIdAndName(context->parent_tid).c_str());
+ Printf("%s", str.data());
+ StackDepotGet(context->stack_id).Print();
+ // Recursively described parent thread if needed.
+ if (flags()->print_full_thread_history) {
+ MemprofThreadContext *parent_context =
+ GetThreadContextByTidLocked(context->parent_tid);
+ DescribeThread(parent_context);
+ }
+}
+
+} // namespace __memprof
--- /dev/null
+//===-- memprof_descriptions.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_descriptions.cpp.
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_DESCRIPTIONS_H
+#define MEMPROF_DESCRIPTIONS_H
+
+#include "memprof_allocator.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_report_decorator.h"
+
+namespace __memprof {
+
+void DescribeThread(MemprofThreadContext *context);
+inline void DescribeThread(MemprofThread *t) {
+ if (t)
+ DescribeThread(t->context());
+}
+
+class MemprofThreadIdAndName {
+public:
+ explicit MemprofThreadIdAndName(MemprofThreadContext *t);
+ explicit MemprofThreadIdAndName(u32 tid);
+
+ // Contains "T%tid (%name)" or "T%tid" if the name is empty.
+ const char *c_str() const { return &name[0]; }
+
+private:
+ void Init(u32 tid, const char *tname);
+
+ char name[128];
+};
+
+} // namespace __memprof
+
+#endif // MEMPROF_DESCRIPTIONS_H
--- /dev/null
+//===-- memprof_flags.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf flag parsing logic.
+//===----------------------------------------------------------------------===//
+
+#include "memprof_flags.h"
+#include "memprof_interface_internal.h"
+#include "memprof_stack.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
+
+namespace __memprof {
+
+Flags memprof_flags_dont_use_directly; // use via flags().
+
+static const char *MaybeUseMemprofDefaultOptionsCompileDefinition() {
+#ifdef MEMPROF_DEFAULT_OPTIONS
+ return SANITIZER_STRINGIFY(MEMPROF_DEFAULT_OPTIONS);
+#else
+ return "";
+#endif
+}
+
+void Flags::SetDefaults() {
+#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "memprof_flags.inc"
+#undef MEMPROF_FLAG
+}
+
+static void RegisterMemprofFlags(FlagParser *parser, Flags *f) {
+#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) \
+ RegisterFlag(parser, #Name, Description, &f->Name);
+#include "memprof_flags.inc"
+#undef MEMPROF_FLAG
+}
+
+void InitializeFlags() {
+ // Set the default values and prepare for parsing MemProf and common flags.
+ SetCommonFlagsDefaults();
+ {
+ CommonFlags cf;
+ cf.CopyFrom(*common_flags());
+ cf.external_symbolizer_path = GetEnv("MEMPROF_SYMBOLIZER_PATH");
+ cf.malloc_context_size = kDefaultMallocContextSize;
+ cf.intercept_tls_get_addr = true;
+ cf.exitcode = 1;
+ OverrideCommonFlags(cf);
+ }
+ Flags *f = flags();
+ f->SetDefaults();
+
+ FlagParser memprof_parser;
+ RegisterMemprofFlags(&memprof_parser, f);
+ RegisterCommonFlags(&memprof_parser);
+
+ // Override from MemProf compile definition.
+ const char *memprof_compile_def =
+ MaybeUseMemprofDefaultOptionsCompileDefinition();
+ memprof_parser.ParseString(memprof_compile_def);
+
+ // Override from user-specified string.
+ const char *memprof_default_options = __memprof_default_options();
+ memprof_parser.ParseString(memprof_default_options);
+
+ // Override from command line.
+ memprof_parser.ParseStringFromEnv("MEMPROF_OPTIONS");
+
+ InitializeCommonFlags();
+
+ if (Verbosity())
+ ReportUnrecognizedFlags();
+
+ if (common_flags()->help) {
+ memprof_parser.PrintFlagDescriptions();
+ }
+
+ CHECK_LE((uptr)common_flags()->malloc_context_size, kStackTraceMax);
+}
+
+} // namespace __memprof
+
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __memprof_default_options, void) {
+ return "";
+}
--- /dev/null
+//===-- memprof_flags.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf runtime flags.
+//===----------------------------------------------------------------------===//
+
+#ifndef MEMPROF_FLAGS_H
+#define MEMPROF_FLAGS_H
+
+#include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+// MemProf flag values can be defined in four ways:
+// 1) initialized with default values at startup.
+// 2) overriden during compilation of MemProf runtime by providing
+// compile definition MEMPROF_DEFAULT_OPTIONS.
+// 3) overriden from string returned by user-specified function
+// __memprof_default_options().
+// 4) overriden from env variable MEMPROF_OPTIONS.
+
+namespace __memprof {
+
+struct Flags {
+#define MEMPROF_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "memprof_flags.inc"
+#undef MEMPROF_FLAG
+
+ void SetDefaults();
+};
+
+extern Flags memprof_flags_dont_use_directly;
+inline Flags *flags() { return &memprof_flags_dont_use_directly; }
+
+void InitializeFlags();
+
+} // namespace __memprof
+
+#endif // MEMPROF_FLAGS_H
--- /dev/null
+//===-- memprof_flags.inc --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// MemProf runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_FLAG
+#error "Define MEMPROF_FLAG prior to including this file!"
+#endif
+
+// MEMPROF_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+MEMPROF_FLAG(bool, unmap_shadow_on_exit, false,
+ "If set, explicitly unmaps the (huge) shadow at exit.")
+MEMPROF_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap")
+MEMPROF_FLAG(bool, print_legend, true, "Print the legend for the shadow bytes.")
+MEMPROF_FLAG(bool, atexit, false,
+ "If set, prints MemProf exit stats even after program terminates "
+ "successfully.")
+MEMPROF_FLAG(
+ bool, print_full_thread_history, true,
+ "If set, prints thread creation stacks for the threads involved in the "
+ "report and their ancestors up to the main thread.")
+
+MEMPROF_FLAG(bool, halt_on_error, true,
+ "Crash the program after printing the first error report "
+ "(WARNING: USE AT YOUR OWN RISK!)")
+MEMPROF_FLAG(bool, allocator_frees_and_returns_null_on_realloc_zero, true,
+ "realloc(p, 0) is equivalent to free(p) by default (Same as the "
+ "POSIX standard). If set to false, realloc(p, 0) will return a "
+ "pointer to an allocated space which can not be used.")
+MEMPROF_FLAG(bool, print_terse, false,
+ "If set, prints memory profile in a terse format.")
+
+MEMPROF_FLAG(
+ int, mem_info_cache_entries, 16381,
+ "Size in entries of the mem info block cache, should be closest prime"
+ " number to a power of two for best hashing.")
+MEMPROF_FLAG(bool, print_mem_info_cache_miss_rate, false,
+ "If set, prints the miss rate of the mem info block cache.")
+MEMPROF_FLAG(
+ bool, print_mem_info_cache_miss_rate_details, false,
+ "If set, prints detailed miss rates of the mem info block cache sets.")
--- /dev/null
+//===-- memprof_init_version.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// This header defines a versioned __memprof_init function to be called at the
+// startup of the instrumented program.
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_INIT_VERSION_H
+#define MEMPROF_INIT_VERSION_H
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+extern "C" {
+// Every time the Memprof ABI changes we also change the version number in the
+// __memprof_init function name. Objects built with incompatible Memprof ABI
+// versions will not link with run-time.
+#define __memprof_version_mismatch_check __memprof_version_mismatch_check_v1
+}
+
+#endif // MEMPROF_INIT_VERSION_H
--- /dev/null
+//===-- memprof_interceptors.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Intercept various libc functions.
+//===----------------------------------------------------------------------===//
+
+#include "memprof_interceptors.h"
+#include "memprof_allocator.h"
+#include "memprof_internal.h"
+#include "memprof_mapping.h"
+#include "memprof_stack.h"
+#include "memprof_stats.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
+namespace __memprof {
+
+#define MEMPROF_READ_STRING(s, n) MEMPROF_READ_RANGE((s), (n))
+
+static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) {
+#if SANITIZER_INTERCEPT_STRNLEN
+ if (REAL(strnlen)) {
+ return REAL(strnlen)(s, maxlen);
+ }
+#endif
+ return internal_strnlen(s, maxlen);
+}
+
+void SetThreadName(const char *name) {
+ MemprofThread *t = GetCurrentThread();
+ if (t)
+ memprofThreadRegistry().SetThreadName(t->tid(), name);
+}
+
+int OnExit() {
+ // FIXME: ask frontend whether we need to return failure.
+ return 0;
+}
+
+} // namespace __memprof
+
+// ---------------------- Wrappers ---------------- {{{1
+using namespace __memprof;
+
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
+
+#define MEMPROF_INTERCEPTOR_ENTER(ctx, func) \
+ ctx = 0; \
+ (void)ctx;
+
+#define COMMON_INTERCEPT_FUNCTION(name) MEMPROF_INTERCEPT_FUNC(name)
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
+ MEMPROF_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
+ MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)
+#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
+ MEMPROF_WRITE_RANGE(ptr, size)
+#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
+ MEMPROF_READ_RANGE(ptr, size)
+#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+ MEMPROF_INTERCEPTOR_ENTER(ctx, func); \
+ do { \
+ if (memprof_init_is_running) \
+ return REAL(func)(__VA_ARGS__); \
+ ENSURE_MEMPROF_INITED(); \
+ } while (false)
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) SetThreadName(name)
+// Should be memprofThreadRegistry().SetThreadNameByUserId(thread, name)
+// But memprof does not remember UserId's for threads (pthread_t);
+// and remembers all ever existed threads, so the linear search by UserId
+// can be slow.
+#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
+ do { \
+ } while (false)
+#define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
+#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
+ do { \
+ CheckNoDeepBind(filename, flag); \
+ } while (false)
+#define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle)
+#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED()
+#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!memprof_inited)
+#define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
+ if (MemprofThread *t = GetCurrentThread()) { \
+ *begin = t->tls_begin(); \
+ *end = t->tls_end(); \
+ } else { \
+ *begin = *end = 0; \
+ }
+
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
+ do { \
+ MEMPROF_INTERCEPTOR_ENTER(ctx, memmove); \
+ MEMPROF_MEMMOVE_IMPL(to, from, size); \
+ } while (false)
+
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
+ do { \
+ MEMPROF_INTERCEPTOR_ENTER(ctx, memcpy); \
+ MEMPROF_MEMCPY_IMPL(to, from, size); \
+ } while (false)
+
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
+ do { \
+ MEMPROF_INTERCEPTOR_ENTER(ctx, memset); \
+ MEMPROF_MEMSET_IMPL(block, c, size); \
+ } while (false)
+
+#include "sanitizer_common/sanitizer_common_interceptors.inc"
+
+#define COMMON_SYSCALL_PRE_READ_RANGE(p, s) MEMPROF_READ_RANGE(p, s)
+#define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) MEMPROF_WRITE_RANGE(p, s)
+#define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+ do { \
+ (void)(p); \
+ (void)(s); \
+ } while (false)
+#include "sanitizer_common/sanitizer_common_syscalls.inc"
+
+struct ThreadStartParam {
+ atomic_uintptr_t t;
+ atomic_uintptr_t is_registered;
+};
+
+static thread_return_t THREAD_CALLING_CONV memprof_thread_start(void *arg) {
+ ThreadStartParam *param = reinterpret_cast<ThreadStartParam *>(arg);
+ MemprofThread *t = nullptr;
+ while ((t = reinterpret_cast<MemprofThread *>(
+ atomic_load(¶m->t, memory_order_acquire))) == nullptr)
+ internal_sched_yield();
+ SetCurrentThread(t);
+ return t->ThreadStart(GetTid(), ¶m->is_registered);
+}
+
+INTERCEPTOR(int, pthread_create, void *thread, void *attr,
+ void *(*start_routine)(void *), void *arg) {
+ EnsureMainThreadIDIsCorrect();
+ GET_STACK_TRACE_THREAD;
+ int detached = 0;
+ if (attr)
+ REAL(pthread_attr_getdetachstate)(attr, &detached);
+ ThreadStartParam param;
+ atomic_store(¶m.t, 0, memory_order_relaxed);
+ atomic_store(¶m.is_registered, 0, memory_order_relaxed);
+ int result;
+ {
+ // Ignore all allocations made by pthread_create: thread stack/TLS may be
+ // stored by pthread for future reuse even after thread destruction, and
+ // the linked list it's stored in doesn't even hold valid pointers to the
+ // objects, the latter are calculated by obscure pointer arithmetic.
+ result = REAL(pthread_create)(thread, attr, memprof_thread_start, ¶m);
+ }
+ if (result == 0) {
+ u32 current_tid = GetCurrentTidOrInvalid();
+ MemprofThread *t = MemprofThread::Create(start_routine, arg, current_tid,
+ &stack, detached);
+ atomic_store(¶m.t, reinterpret_cast<uptr>(t), memory_order_release);
+ // Wait until the MemprofThread object is initialized and the
+ // ThreadRegistry entry is in "started" state.
+ while (atomic_load(¶m.is_registered, memory_order_acquire) == 0)
+ internal_sched_yield();
+ }
+ return result;
+}
+
+INTERCEPTOR(int, pthread_join, void *t, void **arg) {
+ return real_pthread_join(t, arg);
+}
+
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
+INTERCEPTOR(char *, index, const char *string, int c)
+ALIAS(WRAPPER_NAME(strchr));
+
+// For both strcat() and strncat() we need to check the validity of |to|
+// argument irrespective of the |from| length.
+INTERCEPTOR(char *, strcat, char *to, const char *from) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strcat);
+ ENSURE_MEMPROF_INITED();
+ uptr from_length = REAL(strlen)(from);
+ MEMPROF_READ_RANGE(from, from_length + 1);
+ uptr to_length = REAL(strlen)(to);
+ MEMPROF_READ_STRING(to, to_length);
+ MEMPROF_WRITE_RANGE(to + to_length, from_length + 1);
+ return REAL(strcat)(to, from);
+}
+
+INTERCEPTOR(char *, strncat, char *to, const char *from, uptr size) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strncat);
+ ENSURE_MEMPROF_INITED();
+ uptr from_length = MaybeRealStrnlen(from, size);
+ uptr copy_length = Min(size, from_length + 1);
+ MEMPROF_READ_RANGE(from, copy_length);
+ uptr to_length = REAL(strlen)(to);
+ MEMPROF_READ_STRING(to, to_length);
+ MEMPROF_WRITE_RANGE(to + to_length, from_length + 1);
+ return REAL(strncat)(to, from, size);
+}
+
+INTERCEPTOR(char *, strcpy, char *to, const char *from) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strcpy);
+ if (memprof_init_is_running) {
+ return REAL(strcpy)(to, from);
+ }
+ ENSURE_MEMPROF_INITED();
+ uptr from_size = REAL(strlen)(from) + 1;
+ MEMPROF_READ_RANGE(from, from_size);
+ MEMPROF_WRITE_RANGE(to, from_size);
+ return REAL(strcpy)(to, from);
+}
+
+INTERCEPTOR(char *, strdup, const char *s) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strdup);
+ if (UNLIKELY(!memprof_inited))
+ return internal_strdup(s);
+ ENSURE_MEMPROF_INITED();
+ uptr length = REAL(strlen)(s);
+ MEMPROF_READ_RANGE(s, length + 1);
+ GET_STACK_TRACE_MALLOC;
+ void *new_mem = memprof_malloc(length + 1, &stack);
+ REAL(memcpy)(new_mem, s, length + 1);
+ return reinterpret_cast<char *>(new_mem);
+}
+
+INTERCEPTOR(char *, __strdup, const char *s) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strdup);
+ if (UNLIKELY(!memprof_inited))
+ return internal_strdup(s);
+ ENSURE_MEMPROF_INITED();
+ uptr length = REAL(strlen)(s);
+ MEMPROF_READ_RANGE(s, length + 1);
+ GET_STACK_TRACE_MALLOC;
+ void *new_mem = memprof_malloc(length + 1, &stack);
+ REAL(memcpy)(new_mem, s, length + 1);
+ return reinterpret_cast<char *>(new_mem);
+}
+
+INTERCEPTOR(char *, strncpy, char *to, const char *from, uptr size) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strncpy);
+ ENSURE_MEMPROF_INITED();
+ uptr from_size = Min(size, MaybeRealStrnlen(from, size) + 1);
+ MEMPROF_READ_RANGE(from, from_size);
+ MEMPROF_WRITE_RANGE(to, size);
+ return REAL(strncpy)(to, from, size);
+}
+
+INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strtol);
+ ENSURE_MEMPROF_INITED();
+ char *real_endptr;
+ long result = REAL(strtol)(nptr, &real_endptr, base);
+ StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+ return result;
+}
+
+INTERCEPTOR(int, atoi, const char *nptr) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, atoi);
+ ENSURE_MEMPROF_INITED();
+ char *real_endptr;
+ // "man atoi" tells that behavior of atoi(nptr) is the same as
+ // strtol(nptr, 0, 10), i.e. it sets errno to ERANGE if the
+ // parsed integer can't be stored in *long* type (even if it's
+ // different from int). So, we just imitate this behavior.
+ int result = REAL(strtol)(nptr, &real_endptr, 10);
+ FixRealStrtolEndptr(nptr, &real_endptr);
+ MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1);
+ return result;
+}
+
+INTERCEPTOR(long, atol, const char *nptr) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, atol);
+ ENSURE_MEMPROF_INITED();
+ char *real_endptr;
+ long result = REAL(strtol)(nptr, &real_endptr, 10);
+ FixRealStrtolEndptr(nptr, &real_endptr);
+ MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1);
+ return result;
+}
+
+INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, strtoll);
+ ENSURE_MEMPROF_INITED();
+ char *real_endptr;
+ long long result = REAL(strtoll)(nptr, &real_endptr, base);
+ StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
+ return result;
+}
+
+INTERCEPTOR(long long, atoll, const char *nptr) {
+ void *ctx;
+ MEMPROF_INTERCEPTOR_ENTER(ctx, atoll);
+ ENSURE_MEMPROF_INITED();
+ char *real_endptr;
+ long long result = REAL(strtoll)(nptr, &real_endptr, 10);
+ FixRealStrtolEndptr(nptr, &real_endptr);
+ MEMPROF_READ_STRING(nptr, (real_endptr - nptr) + 1);
+ return result;
+}
+
+// ---------------------- InitializeMemprofInterceptors ---------------- {{{1
+namespace __memprof {
+void InitializeMemprofInterceptors() {
+ static bool was_called_once;
+ CHECK(!was_called_once);
+ was_called_once = true;
+ InitializeCommonInterceptors();
+
+ // Intercept str* functions.
+ MEMPROF_INTERCEPT_FUNC(strcat);
+ MEMPROF_INTERCEPT_FUNC(strcpy);
+ MEMPROF_INTERCEPT_FUNC(strncat);
+ MEMPROF_INTERCEPT_FUNC(strncpy);
+ MEMPROF_INTERCEPT_FUNC(strdup);
+ MEMPROF_INTERCEPT_FUNC(__strdup);
+ MEMPROF_INTERCEPT_FUNC(index);
+
+ MEMPROF_INTERCEPT_FUNC(atoi);
+ MEMPROF_INTERCEPT_FUNC(atol);
+ MEMPROF_INTERCEPT_FUNC(strtol);
+ MEMPROF_INTERCEPT_FUNC(atoll);
+ MEMPROF_INTERCEPT_FUNC(strtoll);
+
+ // Intercept threading-related functions
+ MEMPROF_INTERCEPT_FUNC(pthread_create);
+ MEMPROF_INTERCEPT_FUNC(pthread_join);
+
+ InitializePlatformInterceptors();
+
+ VReport(1, "MemProfiler: libc interceptors initialized\n");
+}
+
+} // namespace __memprof
--- /dev/null
+//===-- memprof_interceptors.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_interceptors.cpp
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_INTERCEPTORS_H
+#define MEMPROF_INTERCEPTORS_H
+
+#include "interception/interception.h"
+#include "memprof_interceptors_memintrinsics.h"
+#include "memprof_internal.h"
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
+
+namespace __memprof {
+
+void InitializeMemprofInterceptors();
+void InitializePlatformInterceptors();
+
+#define ENSURE_MEMPROF_INITED() \
+ do { \
+ CHECK(!memprof_init_is_running); \
+ if (UNLIKELY(!memprof_inited)) { \
+ MemprofInitFromRtl(); \
+ } \
+ } while (0)
+
+} // namespace __memprof
+
+DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
+DECLARE_REAL(char *, strchr, const char *str, int c)
+DECLARE_REAL(SIZE_T, strlen, const char *s)
+DECLARE_REAL(char *, strncpy, char *to, const char *from, uptr size)
+DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen)
+DECLARE_REAL(char *, strstr, const char *s1, const char *s2)
+
+#define MEMPROF_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "MemProfiler: failed to intercept '%s'\n'", #name); \
+ } while (0)
+#define MEMPROF_INTERCEPT_FUNC_VER(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver)) \
+ VReport(1, "MemProfiler: failed to intercept '%s@@%s'\n", #name, #ver); \
+ } while (0)
+#define MEMPROF_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
+ VReport(1, "MemProfiler: failed to intercept '%s@@%s' or '%s'\n", #name, \
+ #ver, #name); \
+ } while (0)
+
+#endif // MEMPROF_INTERCEPTORS_H
--- /dev/null
+//===-- memprof_interceptors_memintrinsics.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf versions of memcpy, memmove, and memset.
+//===---------------------------------------------------------------------===//
+
+#include "memprof_interceptors_memintrinsics.h"
+#include "memprof_stack.h"
+
+using namespace __memprof;
+
+void *__memprof_memcpy(void *to, const void *from, uptr size) {
+ MEMPROF_MEMCPY_IMPL(to, from, size);
+}
+
+void *__memprof_memset(void *block, int c, uptr size) {
+ MEMPROF_MEMSET_IMPL(block, c, size);
+}
+
+void *__memprof_memmove(void *to, const void *from, uptr size) {
+ MEMPROF_MEMMOVE_IMPL(to, from, size);
+}
--- /dev/null
+//===-- memprof_interceptors_memintrinsics.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_interceptors_memintrinsics.cpp
+//===---------------------------------------------------------------------===//
+#ifndef MEMPROF_MEMINTRIN_H
+#define MEMPROF_MEMINTRIN_H
+
+#include "interception/interception.h"
+#include "memprof_interface_internal.h"
+#include "memprof_internal.h"
+#include "memprof_mapping.h"
+
+DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size)
+DECLARE_REAL(void *, memset, void *block, int c, uptr size)
+
+namespace __memprof {
+
+// We implement ACCESS_MEMORY_RANGE, MEMPROF_READ_RANGE,
+// and MEMPROF_WRITE_RANGE as macro instead of function so
+// that no extra frames are created, and stack trace contains
+// relevant information only.
+#define ACCESS_MEMORY_RANGE(offset, size) \
+ do { \
+ __memprof_record_access_range(offset, size); \
+ } while (0)
+
+// memcpy is called during __memprof_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define MEMPROF_MEMCPY_IMPL(to, from, size) \
+ do { \
+ if (UNLIKELY(!memprof_inited)) \
+ return internal_memcpy(to, from, size); \
+ if (memprof_init_is_running) { \
+ return REAL(memcpy)(to, from, size); \
+ } \
+ ENSURE_MEMPROF_INITED(); \
+ MEMPROF_READ_RANGE(from, size); \
+ MEMPROF_WRITE_RANGE(to, size); \
+ return REAL(memcpy)(to, from, size); \
+ } while (0)
+
+// memset is called inside Printf.
+#define MEMPROF_MEMSET_IMPL(block, c, size) \
+ do { \
+ if (UNLIKELY(!memprof_inited)) \
+ return internal_memset(block, c, size); \
+ if (memprof_init_is_running) { \
+ return REAL(memset)(block, c, size); \
+ } \
+ ENSURE_MEMPROF_INITED(); \
+ MEMPROF_WRITE_RANGE(block, size); \
+ return REAL(memset)(block, c, size); \
+ } while (0)
+
+#define MEMPROF_MEMMOVE_IMPL(to, from, size) \
+ do { \
+ if (UNLIKELY(!memprof_inited)) \
+ return internal_memmove(to, from, size); \
+ ENSURE_MEMPROF_INITED(); \
+ MEMPROF_READ_RANGE(from, size); \
+ MEMPROF_WRITE_RANGE(to, size); \
+ return internal_memmove(to, from, size); \
+ } while (0)
+
+#define MEMPROF_READ_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size)
+#define MEMPROF_WRITE_RANGE(offset, size) ACCESS_MEMORY_RANGE(offset, size)
+
+} // namespace __memprof
+
+#endif // MEMPROF_MEMINTRIN_H
--- /dev/null
+//===-- memprof_interface_internal.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// This header declares the MemProfiler runtime interface functions.
+// The runtime library has to define these functions so the instrumented program
+// could call them.
+//
+// See also include/sanitizer/memprof_interface.h
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_INTERFACE_INTERNAL_H
+#define MEMPROF_INTERFACE_INTERNAL_H
+
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#include "memprof_init_version.h"
+
+using __sanitizer::u32;
+using __sanitizer::u64;
+using __sanitizer::uptr;
+
+extern "C" {
+// This function should be called at the very beginning of the process,
+// before any instrumented code is executed and before any call to malloc.
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_init();
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_preinit();
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_version_mismatch_check_v1();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __memprof_record_access(void const volatile *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __memprof_record_access_range(void const volatile *addr, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_print_accumulated_stats();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__memprof_default_options();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+extern uptr __memprof_shadow_memory_dynamic_address;
+
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE extern char
+ __memprof_profile_filename[1];
+SANITIZER_INTERFACE_ATTRIBUTE int __memprof_profile_dump();
+
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_load(uptr p);
+SANITIZER_INTERFACE_ATTRIBUTE void __memprof_store(uptr p);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__memprof_memcpy(void *dst, const void *src, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__memprof_memset(void *s, int c, uptr n);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__memprof_memmove(void *dest, const void *src, uptr n);
+} // extern "C"
+
+#endif // MEMPROF_INTERFACE_INTERNAL_H
--- /dev/null
+//===-- memprof_internal.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header which defines various general utilities.
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_INTERNAL_H
+#define MEMPROF_INTERNAL_H
+
+#include "memprof_flags.h"
+#include "memprof_interface_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
+#error "The MemProfiler run-time should not be instrumented by MemProfiler"
+#endif
+
+// Build-time configuration options.
+
+// If set, memprof will intercept C++ exception api call(s).
+#ifndef MEMPROF_HAS_EXCEPTIONS
+#define MEMPROF_HAS_EXCEPTIONS 1
+#endif
+
+#ifndef MEMPROF_DYNAMIC
+#ifdef PIC
+#define MEMPROF_DYNAMIC 1
+#else
+#define MEMPROF_DYNAMIC 0
+#endif
+#endif
+
+// All internal functions in memprof reside inside the __memprof namespace
+// to avoid namespace collisions with the user programs.
+// Separate namespace also makes it simpler to distinguish the memprof
+// run-time functions from the instrumented user code in a profile.
+namespace __memprof {
+
+class MemprofThread;
+using __sanitizer::StackTrace;
+
+void MemprofInitFromRtl();
+
+// memprof_rtl.cpp
+void PrintAddressSpaceLayout();
+
+// memprof_shadow_setup.cpp
+void InitializeShadowMemory();
+
+// memprof_malloc_linux.cpp
+void ReplaceSystemMalloc();
+
+// memprof_linux.cpp
+uptr FindDynamicShadowStart();
+void *MemprofDoesNotSupportStaticLinkage();
+
+// memprof_thread.cpp
+MemprofThread *CreateMainThread();
+
+void ReadContextStack(void *context, uptr *stack, uptr *ssize);
+
+// Wrapper for TLS/TSD.
+void TSDInit(void (*destructor)(void *tsd));
+void *TSDGet();
+void TSDSet(void *tsd);
+void PlatformTSDDtor(void *tsd);
+
+void *MemprofDlSymNext(const char *sym);
+
+// Add convenient macro for interface functions that may be represented as
+// weak hooks.
+#define MEMPROF_MALLOC_HOOK(ptr, size) \
+ do { \
+ if (&__sanitizer_malloc_hook) \
+ __sanitizer_malloc_hook(ptr, size); \
+ RunMallocHooks(ptr, size); \
+ } while (false)
+#define MEMPROF_FREE_HOOK(ptr) \
+ do { \
+ if (&__sanitizer_free_hook) \
+ __sanitizer_free_hook(ptr); \
+ RunFreeHooks(ptr); \
+ } while (false)
+
+extern int memprof_inited;
+extern int memprof_timestamp_inited;
+extern int memprof_init_done;
+// Used to avoid infinite recursion in __memprof_init().
+extern bool memprof_init_is_running;
+extern void (*death_callback)(void);
+extern long memprof_init_timestamp_s;
+
+} // namespace __memprof
+
+#endif // MEMPROF_INTERNAL_H
--- /dev/null
+//===-- memprof_linux.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Linux-specific details.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if !SANITIZER_LINUX
+#error Unsupported OS
+#endif
+
+#include "memprof_interceptors.h"
+#include "memprof_internal.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_freebsd.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_procmaps.h"
+
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <link.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/ucontext.h>
+#include <unistd.h>
+#include <unwind.h>
+
+extern ElfW(Dyn) _DYNAMIC[];
+
+typedef enum {
+ MEMPROF_RT_VERSION_UNDEFINED = 0,
+ MEMPROF_RT_VERSION_DYNAMIC,
+ MEMPROF_RT_VERSION_STATIC,
+} memprof_rt_version_t;
+
+// FIXME: perhaps also store abi version here?
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+memprof_rt_version_t __memprof_rt_version;
+}
+
+namespace __memprof {
+
+void InitializePlatformInterceptors() {}
+void InitializePlatformExceptionHandlers() {}
+
+void *MemprofDoesNotSupportStaticLinkage() {
+ // This will fail to link with -static.
+ return &_DYNAMIC;
+}
+
+uptr FindDynamicShadowStart() {
+ uptr shadow_size_bytes = MemToShadowSize(kHighMemEnd);
+ return MapDynamicShadow(shadow_size_bytes, SHADOW_SCALE,
+ /*min_shadow_base_alignment*/ 0, kHighMemEnd);
+}
+
+void ReadContextStack(void *context, uptr *stack, uptr *ssize) {
+ ucontext_t *ucp = (ucontext_t *)context;
+ *stack = (uptr)ucp->uc_stack.ss_sp;
+ *ssize = ucp->uc_stack.ss_size;
+}
+
+void *MemprofDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); }
+
+} // namespace __memprof
--- /dev/null
+//===-- memprof_malloc_linux.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Linux-specific malloc interception.
+// We simply define functions like malloc, free, realloc, etc.
+// They will replace the corresponding libc functions automagically.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if !SANITIZER_LINUX
+#error Unsupported OS
+#endif
+
+#include "memprof_allocator.h"
+#include "memprof_interceptors.h"
+#include "memprof_internal.h"
+#include "memprof_stack.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+// ---------------------- Replacement functions ---------------- {{{1
+using namespace __memprof;
+
+static uptr allocated_for_dlsym;
+static uptr last_dlsym_alloc_size_in_words;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static inline bool IsInDlsymAllocPool(const void *ptr) {
+ uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+ uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+ void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ last_dlsym_alloc_size_in_words = size_in_words;
+ allocated_for_dlsym += size_in_words;
+ CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+ return mem;
+}
+
+static void DeallocateFromLocalPool(const void *ptr) {
+ // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
+ // error messages and instead uses malloc followed by free. To avoid pool
+ // exhaustion due to long object filenames, handle that special case here.
+ uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
+ void *prev_mem = (void *)&alloc_memory_for_dlsym[prev_offset];
+ if (prev_mem == ptr) {
+ REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
+ allocated_for_dlsym = prev_offset;
+ last_dlsym_alloc_size_in_words = 0;
+ }
+}
+
+static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
+ uptr size_in_bytes) {
+ if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
+ return errno_EINVAL;
+
+ CHECK(alignment >= kWordSize);
+
+ uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
+ uptr aligned_addr = RoundUpTo(addr, alignment);
+ uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
+
+ uptr *end_mem = (uptr *)(aligned_addr + aligned_size);
+ uptr allocated = end_mem - alloc_memory_for_dlsym;
+ if (allocated >= kDlsymAllocPoolSize)
+ return errno_ENOMEM;
+
+ allocated_for_dlsym = allocated;
+ *memptr = (void *)aligned_addr;
+ return 0;
+}
+
+static inline bool MaybeInDlsym() { return memprof_init_is_running; }
+
+static inline bool UseLocalPool() { return MaybeInDlsym(); }
+
+static void *ReallocFromLocalPool(void *ptr, uptr size) {
+ const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+ const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+ void *new_ptr;
+ if (UNLIKELY(UseLocalPool())) {
+ new_ptr = AllocateFromLocalPool(size);
+ } else {
+ ENSURE_MEMPROF_INITED();
+ GET_STACK_TRACE_MALLOC;
+ new_ptr = memprof_malloc(size, &stack);
+ }
+ internal_memcpy(new_ptr, ptr, copy_size);
+ return new_ptr;
+}
+
+INTERCEPTOR(void, free, void *ptr) {
+ GET_STACK_TRACE_FREE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+ DeallocateFromLocalPool(ptr);
+ return;
+ }
+ memprof_free(ptr, &stack, FROM_MALLOC);
+}
+
+#if SANITIZER_INTERCEPT_CFREE
+INTERCEPTOR(void, cfree, void *ptr) {
+ GET_STACK_TRACE_FREE;
+ if (UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return;
+ memprof_free(ptr, &stack, FROM_MALLOC);
+}
+#endif // SANITIZER_INTERCEPT_CFREE
+
+INTERCEPTOR(void *, malloc, uptr size) {
+ if (UNLIKELY(UseLocalPool()))
+ // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(size);
+ ENSURE_MEMPROF_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return memprof_malloc(size, &stack);
+}
+
+INTERCEPTOR(void *, calloc, uptr nmemb, uptr size) {
+ if (UNLIKELY(UseLocalPool()))
+ // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+ return AllocateFromLocalPool(nmemb * size);
+ ENSURE_MEMPROF_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return memprof_calloc(nmemb, size, &stack);
+}
+
+INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
+ if (UNLIKELY(IsInDlsymAllocPool(ptr)))
+ return ReallocFromLocalPool(ptr, size);
+ if (UNLIKELY(UseLocalPool()))
+ return AllocateFromLocalPool(size);
+ ENSURE_MEMPROF_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return memprof_realloc(ptr, size, &stack);
+}
+
+#if SANITIZER_INTERCEPT_REALLOCARRAY
+INTERCEPTOR(void *, reallocarray, void *ptr, uptr nmemb, uptr size) {
+ ENSURE_MEMPROF_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return memprof_reallocarray(ptr, nmemb, size, &stack);
+}
+#endif // SANITIZER_INTERCEPT_REALLOCARRAY
+
+#if SANITIZER_INTERCEPT_MEMALIGN
+INTERCEPTOR(void *, memalign, uptr boundary, uptr size) {
+ GET_STACK_TRACE_MALLOC;
+ return memprof_memalign(boundary, size, &stack, FROM_MALLOC);
+}
+
+INTERCEPTOR(void *, __libc_memalign, uptr boundary, uptr size) {
+ GET_STACK_TRACE_MALLOC;
+ void *res = memprof_memalign(boundary, size, &stack, FROM_MALLOC);
+ DTLS_on_libc_memalign(res, size);
+ return res;
+}
+#endif // SANITIZER_INTERCEPT_MEMALIGN
+
+#if SANITIZER_INTERCEPT_ALIGNED_ALLOC
+INTERCEPTOR(void *, aligned_alloc, uptr boundary, uptr size) {
+ GET_STACK_TRACE_MALLOC;
+ return memprof_aligned_alloc(boundary, size, &stack);
+}
+#endif // SANITIZER_INTERCEPT_ALIGNED_ALLOC
+
+INTERCEPTOR(uptr, malloc_usable_size, void *ptr) {
+ GET_CURRENT_PC_BP_SP;
+ (void)sp;
+ return memprof_malloc_usable_size(ptr, pc, bp);
+}
+
+#if SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
+// We avoid including malloc.h for portability reasons.
+// man mallinfo says the fields are "long", but the implementation uses int.
+// It doesn't matter much -- we just need to make sure that the libc's mallinfo
+// is not called.
+struct fake_mallinfo {
+ int x[10];
+};
+
+INTERCEPTOR(struct fake_mallinfo, mallinfo, void) {
+ struct fake_mallinfo res;
+ REAL(memset)(&res, 0, sizeof(res));
+ return res;
+}
+
+INTERCEPTOR(int, mallopt, int cmd, int value) { return 0; }
+#endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
+
+INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
+ if (UNLIKELY(UseLocalPool()))
+ return PosixMemalignFromLocalPool(memptr, alignment, size);
+ GET_STACK_TRACE_MALLOC;
+ return memprof_posix_memalign(memptr, alignment, size, &stack);
+}
+
+INTERCEPTOR(void *, valloc, uptr size) {
+ GET_STACK_TRACE_MALLOC;
+ return memprof_valloc(size, &stack);
+}
+
+#if SANITIZER_INTERCEPT_PVALLOC
+INTERCEPTOR(void *, pvalloc, uptr size) {
+ GET_STACK_TRACE_MALLOC;
+ return memprof_pvalloc(size, &stack);
+}
+#endif // SANITIZER_INTERCEPT_PVALLOC
+
+INTERCEPTOR(void, malloc_stats, void) { __memprof_print_accumulated_stats(); }
+
+namespace __memprof {
+void ReplaceSystemMalloc() {}
+} // namespace __memprof
--- /dev/null
+//===-- memprof_mapping.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Defines MemProf memory mapping.
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_MAPPING_H
+#define MEMPROF_MAPPING_H
+
+#include "memprof_internal.h"
+
+static const u64 kDefaultShadowScale = 3;
+#define SHADOW_SCALE kDefaultShadowScale
+
+#define SHADOW_OFFSET __memprof_shadow_memory_dynamic_address
+
+#define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE)
+#define MEMPROF_ALIGNMENT 32
+
+namespace __memprof {
+
+extern uptr kHighMemEnd; // Initialized in __memprof_init.
+
+} // namespace __memprof
+
+#define SHADOW_ENTRY_SIZE 8
+
+// Size of memory block mapped to a single shadow location
+#define MEM_GRANULARITY 64ULL
+
+#define SHADOW_MASK ~(MEM_GRANULARITY - 1)
+
+#define MEM_TO_SHADOW(mem) \
+ ((((mem) & SHADOW_MASK) >> SHADOW_SCALE) + (SHADOW_OFFSET))
+
+#define kLowMemBeg 0
+#define kLowMemEnd (SHADOW_OFFSET ? SHADOW_OFFSET - 1 : 0)
+
+#define kLowShadowBeg SHADOW_OFFSET
+#define kLowShadowEnd (MEM_TO_SHADOW(kLowMemEnd) + SHADOW_ENTRY_SIZE - 1)
+
+#define kHighMemBeg (MEM_TO_SHADOW(kHighMemEnd) + 1 + SHADOW_ENTRY_SIZE - 1)
+
+#define kHighShadowBeg MEM_TO_SHADOW(kHighMemBeg)
+#define kHighShadowEnd (MEM_TO_SHADOW(kHighMemEnd) + SHADOW_ENTRY_SIZE - 1)
+
+// With the zero shadow base we can not actually map pages starting from 0.
+// This constant is somewhat arbitrary.
+#define kZeroBaseShadowStart 0
+#define kZeroBaseMaxShadowStart (1 << 18)
+
+#define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 : kZeroBaseShadowStart)
+#define kShadowGapEnd (kHighShadowBeg - 1)
+
+namespace __memprof {
+
+inline uptr MemToShadowSize(uptr size) { return size >> SHADOW_SCALE; }
+inline bool AddrIsInLowMem(uptr a) { return a <= kLowMemEnd; }
+
+inline bool AddrIsInLowShadow(uptr a) {
+ return a >= kLowShadowBeg && a <= kLowShadowEnd;
+}
+
+inline bool AddrIsInHighMem(uptr a) {
+ return kHighMemBeg && a >= kHighMemBeg && a <= kHighMemEnd;
+}
+
+inline bool AddrIsInHighShadow(uptr a) {
+ return kHighMemBeg && a >= kHighShadowBeg && a <= kHighShadowEnd;
+}
+
+inline bool AddrIsInShadowGap(uptr a) {
+ // In zero-based shadow mode we treat addresses near zero as addresses
+ // in shadow gap as well.
+ if (SHADOW_OFFSET == 0)
+ return a <= kShadowGapEnd;
+ return a >= kShadowGapBeg && a <= kShadowGapEnd;
+}
+
+inline bool AddrIsInMem(uptr a) {
+ return AddrIsInLowMem(a) || AddrIsInHighMem(a) ||
+ (flags()->protect_shadow_gap == 0 && AddrIsInShadowGap(a));
+}
+
+inline uptr MemToShadow(uptr p) {
+ CHECK(AddrIsInMem(p));
+ return MEM_TO_SHADOW(p);
+}
+
+inline bool AddrIsInShadow(uptr a) {
+ return AddrIsInLowShadow(a) || AddrIsInHighShadow(a);
+}
+
+inline bool AddrIsAlignedByGranularity(uptr a) {
+ return (a & (SHADOW_GRANULARITY - 1)) == 0;
+}
+
+inline void RecordAccess(uptr a) {
+ // If we use a different shadow size then the type below needs adjustment.
+ CHECK_EQ(SHADOW_ENTRY_SIZE, 8);
+ u64 *shadow_address = (u64 *)MEM_TO_SHADOW(a);
+ (*shadow_address)++;
+}
+
+} // namespace __memprof
+
+#endif // MEMPROF_MAPPING_H
--- /dev/null
+//===-- memprof_interceptors.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Interceptors for operators new and delete.
+//===----------------------------------------------------------------------===//
+
+#include "memprof_allocator.h"
+#include "memprof_internal.h"
+#include "memprof_stack.h"
+#include "sanitizer_common/sanitizer_allocator_report.h"
+
+#include "interception/interception.h"
+
+#include <stddef.h>
+
+#define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE
+
+using namespace __memprof;
+
+// Fake std::nothrow_t and std::align_val_t to avoid including <new>.
+namespace std {
+struct nothrow_t {};
+enum class align_val_t : size_t {};
+} // namespace std
+
+#define OPERATOR_NEW_BODY(type, nothrow) \
+ GET_STACK_TRACE_MALLOC; \
+ void *res = memprof_memalign(0, size, &stack, type); \
+ if (!nothrow && UNLIKELY(!res)) \
+ ReportOutOfMemory(size, &stack); \
+ return res;
+#define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \
+ GET_STACK_TRACE_MALLOC; \
+ void *res = memprof_memalign((uptr)align, size, &stack, type); \
+ if (!nothrow && UNLIKELY(!res)) \
+ ReportOutOfMemory(size, &stack); \
+ return res;
+
+CXX_OPERATOR_ATTRIBUTE
+void *operator new(size_t size) {
+ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new[](size_t size) {
+ OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new(size_t size, std::nothrow_t const &) {
+ OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new[](size_t size, std::nothrow_t const &) {
+ OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align) {
+ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align) {
+ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new(size_t size, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/);
+}
+CXX_OPERATOR_ATTRIBUTE
+void *operator new[](size_t size, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/);
+}
+
+#define OPERATOR_DELETE_BODY(type) \
+ GET_STACK_TRACE_FREE; \
+ memprof_delete(ptr, 0, 0, &stack, type);
+
+#define OPERATOR_DELETE_BODY_SIZE(type) \
+ GET_STACK_TRACE_FREE; \
+ memprof_delete(ptr, size, 0, &stack, type);
+
+#define OPERATOR_DELETE_BODY_ALIGN(type) \
+ GET_STACK_TRACE_FREE; \
+ memprof_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
+
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
+ GET_STACK_TRACE_FREE; \
+ memprof_delete(ptr, size, static_cast<uptr>(align), &stack, type);
+
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr)NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW); }
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr) NOEXCEPT {
+ OPERATOR_DELETE_BODY(FROM_NEW_BR);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY(FROM_NEW);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr, std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY(FROM_NEW_BR);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size)NOEXCEPT {
+ OPERATOR_DELETE_BODY_SIZE(FROM_NEW);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size) NOEXCEPT {
+ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align)NOEXCEPT {
+ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr, std::align_val_t align,
+ std::nothrow_t const &) {
+ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete(void *ptr, size_t size, std::align_val_t align)NOEXCEPT {
+ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW);
+}
+CXX_OPERATOR_ATTRIBUTE
+void operator delete[](void *ptr, size_t size,
+ std::align_val_t align) NOEXCEPT {
+ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR);
+}
--- /dev/null
+//===-- memprof_posix.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Posix-specific details.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+#if !SANITIZER_POSIX
+#error Only Posix supported
+#endif
+
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
+#include <pthread.h>
+
+namespace __memprof {
+
+// ---------------------- TSD ---------------- {{{1
+
+static pthread_key_t tsd_key;
+static bool tsd_key_inited = false;
+void TSDInit(void (*destructor)(void *tsd)) {
+ CHECK(!tsd_key_inited);
+ tsd_key_inited = true;
+ CHECK_EQ(0, pthread_key_create(&tsd_key, destructor));
+}
+
+void *TSDGet() {
+ CHECK(tsd_key_inited);
+ return pthread_getspecific(tsd_key);
+}
+
+void TSDSet(void *tsd) {
+ CHECK(tsd_key_inited);
+ pthread_setspecific(tsd_key, tsd);
+}
+
+void PlatformTSDDtor(void *tsd) {
+ MemprofThreadContext *context = (MemprofThreadContext *)tsd;
+ if (context->destructor_iterations > 1) {
+ context->destructor_iterations--;
+ CHECK_EQ(0, pthread_setspecific(tsd_key, tsd));
+ return;
+ }
+ MemprofThread::TSDDtor(tsd);
+}
+} // namespace __memprof
--- /dev/null
+//===-- memprof_preinit.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Call __memprof_init at the very early stage of process startup.
+//===----------------------------------------------------------------------===//
+#include "memprof_internal.h"
+
+using namespace __memprof;
+
+#if SANITIZER_CAN_USE_PREINIT_ARRAY
+// The symbol is called __local_memprof_preinit, because it's not intended to
+// be exported. This code linked into the main executable when -fmemory-profile
+// is in the link flags. It can only use exported interface functions.
+__attribute__((section(".preinit_array"),
+ used)) void (*__local_memprof_preinit)(void) = __memprof_preinit;
+#endif
--- /dev/null
+//===-- memprof_rtl.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Main file of the MemProf run-time library.
+//===----------------------------------------------------------------------===//
+
+#include "memprof_allocator.h"
+#include "memprof_interceptors.h"
+#include "memprof_interface_internal.h"
+#include "memprof_internal.h"
+#include "memprof_mapping.h"
+#include "memprof_stack.h"
+#include "memprof_stats.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_symbolizer.h"
+
+#include <time.h>
+
+uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
+
+// Allow the user to specify a profile output file via the binary.
+SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1];
+
+namespace __memprof {
+
+static void MemprofDie() {
+ static atomic_uint32_t num_calls;
+ if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
+ // Don't die twice - run a busy loop.
+ while (1) {
+ }
+ }
+ if (common_flags()->print_module_map >= 1)
+ DumpProcessMap();
+ if (flags()->unmap_shadow_on_exit) {
+ if (kHighShadowEnd)
+ UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+ }
+}
+
+static void CheckUnwind() {
+ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
+ stack.Print();
+}
+
+// -------------------------- Globals --------------------- {{{1
+int memprof_inited;
+int memprof_init_done;
+bool memprof_init_is_running;
+int memprof_timestamp_inited;
+long memprof_init_timestamp_s;
+
+uptr kHighMemEnd;
+
+// -------------------------- Run-time entry ------------------- {{{1
+// exported functions
+
+#define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
+
+#define MEMPROF_MEMORY_ACCESS_CALLBACK(type) \
+ extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) { \
+ MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() \
+ }
+
+MEMPROF_MEMORY_ACCESS_CALLBACK(load)
+MEMPROF_MEMORY_ACCESS_CALLBACK(store)
+
+// Force the linker to keep the symbols for various MemProf interface
+// functions. We want to keep those in the executable in order to let the
+// instrumented dynamic libraries access the symbol even if it is not used by
+// the executable itself. This should help if the build system is removing dead
+// code at link time.
+static NOINLINE void force_interface_symbols() {
+ volatile int fake_condition = 0; // prevent dead condition elimination.
+ // clang-format off
+ switch (fake_condition) {
+ case 1: __memprof_record_access(nullptr); break;
+ case 2: __memprof_record_access_range(nullptr, 0); break;
+ }
+ // clang-format on
+}
+
+static void memprof_atexit() {
+ Printf("MemProfiler exit stats:\n");
+ __memprof_print_accumulated_stats();
+}
+
+static void InitializeHighMemEnd() {
+ kHighMemEnd = GetMaxUserVirtualAddress();
+ // Increase kHighMemEnd to make sure it's properly
+ // aligned together with kHighMemBeg:
+ kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
+}
+
+void PrintAddressSpaceLayout() {
+ if (kHighMemBeg) {
+ Printf("|| `[%p, %p]` || HighMem ||\n", (void *)kHighMemBeg,
+ (void *)kHighMemEnd);
+ Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
+ (void *)kHighShadowEnd);
+ }
+ Printf("|| `[%p, %p]` || ShadowGap ||\n", (void *)kShadowGapBeg,
+ (void *)kShadowGapEnd);
+ if (kLowShadowBeg) {
+ Printf("|| `[%p, %p]` || LowShadow ||\n", (void *)kLowShadowBeg,
+ (void *)kLowShadowEnd);
+ Printf("|| `[%p, %p]` || LowMem ||\n", (void *)kLowMemBeg,
+ (void *)kLowMemEnd);
+ }
+ Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
+ (void *)MEM_TO_SHADOW(kLowShadowEnd));
+ if (kHighMemBeg) {
+ Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
+ (void *)MEM_TO_SHADOW(kHighShadowEnd));
+ }
+ Printf("\n");
+ Printf("malloc_context_size=%zu\n",
+ (uptr)common_flags()->malloc_context_size);
+
+ Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
+ Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
+ Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
+ CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+}
+
+static bool UNUSED __local_memprof_dyninit = [] {
+ MaybeStartBackgroudThread();
+ SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
+
+ return false;
+}();
+
+static void MemprofInitInternal() {
+ if (LIKELY(memprof_inited))
+ return;
+ SanitizerToolName = "MemProfiler";
+ CHECK(!memprof_init_is_running && "MemProf init calls itself!");
+ memprof_init_is_running = true;
+
+ CacheBinaryName();
+
+ // Initialize flags. This must be done early, because most of the
+ // initialization steps look at flags().
+ InitializeFlags();
+
+ AvoidCVE_2016_2143();
+
+ SetMallocContextSize(common_flags()->malloc_context_size);
+
+ InitializeHighMemEnd();
+
+ // Make sure we are not statically linked.
+ MemprofDoesNotSupportStaticLinkage();
+
+ // Install tool-specific callbacks in sanitizer_common.
+ AddDieCallback(MemprofDie);
+ SetCheckUnwindCallback(CheckUnwind);
+
+ // Use profile name specified via the binary itself if it exists, and hasn't
+ // been overrriden by a flag at runtime.
+ if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path)
+ __sanitizer_set_report_path(__memprof_profile_filename);
+ else
+ __sanitizer_set_report_path(common_flags()->log_path);
+
+ __sanitizer::InitializePlatformEarly();
+
+ // Re-exec ourselves if we need to set additional env or command line args.
+ MaybeReexec();
+
+ // Setup internal allocator callback.
+ SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
+
+ InitializeMemprofInterceptors();
+ CheckASLR();
+
+ ReplaceSystemMalloc();
+
+ DisableCoreDumperIfNecessary();
+
+ InitializeShadowMemory();
+
+ TSDInit(PlatformTSDDtor);
+
+ InitializeAllocator();
+
+ // On Linux MemprofThread::ThreadStart() calls malloc() that's why
+ // memprof_inited should be set to 1 prior to initializing the threads.
+ memprof_inited = 1;
+ memprof_init_is_running = false;
+
+ if (flags()->atexit)
+ Atexit(memprof_atexit);
+
+ InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
+
+ // interceptors
+ InitTlsSize();
+
+ // Create main thread.
+ MemprofThread *main_thread = CreateMainThread();
+ CHECK_EQ(0, main_thread->tid());
+ force_interface_symbols(); // no-op.
+ SanitizerInitializeUnwinder();
+
+ Symbolizer::LateInitialize();
+
+ VReport(1, "MemProfiler Init done\n");
+
+ memprof_init_done = 1;
+}
+
+void MemprofInitTime() {
+ if (LIKELY(memprof_timestamp_inited))
+ return;
+ timespec ts;
+ clock_gettime(CLOCK_REALTIME, &ts);
+ memprof_init_timestamp_s = ts.tv_sec;
+ memprof_timestamp_inited = 1;
+}
+
+// Initialize as requested from some part of MemProf runtime library
+// (interceptors, allocator, etc).
+void MemprofInitFromRtl() { MemprofInitInternal(); }
+
+#if MEMPROF_DYNAMIC
+// Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
+// (and thus normal initializers from .preinit_array or modules haven't run).
+
+class MemprofInitializer {
+public:
+ MemprofInitializer() { MemprofInitFromRtl(); }
+};
+
+static MemprofInitializer memprof_initializer;
+#endif // MEMPROF_DYNAMIC
+
+} // namespace __memprof
+
+// ---------------------- Interface ---------------- {{{1
+using namespace __memprof;
+
+// Initialize as requested from instrumented application code.
+void __memprof_init() {
+ MemprofInitTime();
+ MemprofInitInternal();
+}
+
+void __memprof_preinit() { MemprofInitInternal(); }
+
+void __memprof_version_mismatch_check_v1() {}
+
+void __memprof_record_access(void const volatile *addr) {
+ __memprof::RecordAccess((uptr)addr);
+}
+
+// We only record the access on the first location in the range,
+// since we will later accumulate the access counts across the
+// full allocation, and we don't want to inflate the hotness from
+// a memory intrinsic on a large range of memory.
+// TODO: Should we do something else so we can better track utilization?
+void __memprof_record_access_range(void const volatile *addr,
+ UNUSED uptr size) {
+ __memprof::RecordAccess((uptr)addr);
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
+__sanitizer_unaligned_load16(const uu16 *p) {
+ __memprof_record_access(p);
+ return *p;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
+__sanitizer_unaligned_load32(const uu32 *p) {
+ __memprof_record_access(p);
+ return *p;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
+__sanitizer_unaligned_load64(const uu64 *p) {
+ __memprof_record_access(p);
+ return *p;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__sanitizer_unaligned_store16(uu16 *p, u16 x) {
+ __memprof_record_access(p);
+ *p = x;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__sanitizer_unaligned_store32(uu32 *p, u32 x) {
+ __memprof_record_access(p);
+ *p = x;
+}
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__sanitizer_unaligned_store64(uu64 *p, u64 x) {
+ __memprof_record_access(p);
+ *p = x;
+}
--- /dev/null
+//===-- memprof_shadow_setup.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Set up the shadow memory.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_platform.h"
+
+#include "memprof_internal.h"
+#include "memprof_mapping.h"
+
+namespace __memprof {
+
+static void ProtectGap(uptr addr, uptr size) {
+ if (!flags()->protect_shadow_gap) {
+ // The shadow gap is unprotected, so there is a chance that someone
+ // is actually using this memory. Which means it needs a shadow...
+ uptr GapShadowBeg = RoundDownTo(MEM_TO_SHADOW(addr), GetPageSizeCached());
+ uptr GapShadowEnd =
+ RoundUpTo(MEM_TO_SHADOW(addr + size), GetPageSizeCached()) - 1;
+ if (Verbosity())
+ Printf("protect_shadow_gap=0:"
+ " not protecting shadow gap, allocating gap's shadow\n"
+ "|| `[%p, %p]` || ShadowGap's shadow ||\n",
+ GapShadowBeg, GapShadowEnd);
+ ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
+ "unprotected gap shadow");
+ return;
+ }
+ __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
+ kZeroBaseMaxShadowStart);
+}
+
+void InitializeShadowMemory() {
+ uptr shadow_start = FindDynamicShadowStart();
+ // Update the shadow memory address (potentially) used by instrumentation.
+ __memprof_shadow_memory_dynamic_address = shadow_start;
+
+ if (kLowShadowBeg)
+ shadow_start -= GetMmapGranularity();
+
+ if (Verbosity())
+ PrintAddressSpaceLayout();
+
+ // mmap the low shadow plus at least one page at the left.
+ if (kLowShadowBeg)
+ ReserveShadowMemoryRange(shadow_start, kLowShadowEnd, "low shadow");
+ // mmap the high shadow.
+ ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd, "high shadow");
+ // protect the gap.
+ ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+ CHECK_EQ(kShadowGapEnd, kHighShadowBeg - 1);
+}
+
+} // namespace __memprof
--- /dev/null
+//===-- memprof_stack.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Code for MemProf stack trace.
+//===----------------------------------------------------------------------===//
+#include "memprof_stack.h"
+#include "memprof_internal.h"
+#include "sanitizer_common/sanitizer_atomic.h"
+
+namespace __memprof {
+
+static atomic_uint32_t malloc_context_size;
+
+void SetMallocContextSize(u32 size) {
+ atomic_store(&malloc_context_size, size, memory_order_release);
+}
+
+u32 GetMallocContextSize() {
+ return atomic_load(&malloc_context_size, memory_order_acquire);
+}
+
+} // namespace __memprof
+
+void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
+ void *context,
+ bool request_fast,
+ u32 max_depth) {
+ using namespace __memprof;
+ size = 0;
+ if (UNLIKELY(!memprof_inited))
+ return;
+ request_fast = StackTrace::WillUseFastUnwind(request_fast);
+ MemprofThread *t = GetCurrentThread();
+ if (request_fast) {
+ if (t) {
+ Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(),
+ true);
+ }
+ return;
+ }
+ Unwind(max_depth, pc, bp, context, 0, 0, false);
+}
+
+// ------------------ Interface -------------- {{{1
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_print_stack_trace() {
+ using namespace __memprof;
+ PRINT_CURRENT_STACK();
+}
+} // extern "C"
--- /dev/null
+//===-- memprof_stack.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_stack.cpp.
+//===----------------------------------------------------------------------===//
+
+#ifndef MEMPROF_STACK_H
+#define MEMPROF_STACK_H
+
+#include "memprof_flags.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_stacktrace.h"
+
+namespace __memprof {
+
+static const u32 kDefaultMallocContextSize = 30;
+
+void SetMallocContextSize(u32 size);
+u32 GetMallocContextSize();
+
+} // namespace __memprof
+
+// NOTE: A Rule of thumb is to retrieve stack trace in the interceptors
+// as early as possible (in functions exposed to the user), as we generally
+// don't want stack trace to contain functions from MemProf internals.
+
+#define GET_STACK_TRACE(max_size, fast) \
+ BufferedStackTrace stack; \
+ if (max_size <= 2) { \
+ stack.size = max_size; \
+ if (max_size > 0) { \
+ stack.top_frame_bp = GET_CURRENT_FRAME(); \
+ stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \
+ if (max_size > 1) \
+ stack.trace_buffer[1] = GET_CALLER_PC(); \
+ } \
+ } else { \
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
+ fast, max_size); \
+ }
+
+#define GET_STACK_TRACE_FATAL_HERE \
+ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
+
+#define GET_STACK_TRACE_THREAD GET_STACK_TRACE(kStackTraceMax, true)
+
+#define GET_STACK_TRACE_MALLOC \
+ GET_STACK_TRACE(GetMallocContextSize(), common_flags()->fast_unwind_on_malloc)
+
+#define GET_STACK_TRACE_FREE GET_STACK_TRACE_MALLOC
+
+#define PRINT_CURRENT_STACK() \
+ { \
+ GET_STACK_TRACE_FATAL_HERE; \
+ stack.Print(); \
+ }
+
+#endif // MEMPROF_STACK_H
--- /dev/null
+//===-- memprof_stats.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Code related to statistics collected by MemProfiler.
+//===----------------------------------------------------------------------===//
+#include "memprof_stats.h"
+#include "memprof_interceptors.h"
+#include "memprof_internal.h"
+#include "memprof_thread.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+
+namespace __memprof {
+
+MemprofStats::MemprofStats() { Clear(); }
+
+void MemprofStats::Clear() {
+ if (REAL(memset))
+ return (void)REAL(memset)(this, 0, sizeof(MemprofStats));
+ internal_memset(this, 0, sizeof(MemprofStats));
+}
+
+static void PrintMallocStatsArray(const char *prefix,
+ uptr (&array)[kNumberOfSizeClasses]) {
+ Printf("%s", prefix);
+ for (uptr i = 0; i < kNumberOfSizeClasses; i++) {
+ if (!array[i])
+ continue;
+ Printf("%zu:%zu; ", i, array[i]);
+ }
+ Printf("\n");
+}
+
+void MemprofStats::Print() {
+ Printf("Stats: %zuM malloced (%zuM for overhead) by %zu calls\n",
+ malloced >> 20, malloced_overhead >> 20, mallocs);
+ Printf("Stats: %zuM realloced by %zu calls\n", realloced >> 20, reallocs);
+ Printf("Stats: %zuM freed by %zu calls\n", freed >> 20, frees);
+ Printf("Stats: %zuM really freed by %zu calls\n", really_freed >> 20,
+ real_frees);
+ Printf("Stats: %zuM (%zuM-%zuM) mmaped; %zu maps, %zu unmaps\n",
+ (mmaped - munmaped) >> 20, mmaped >> 20, munmaped >> 20, mmaps,
+ munmaps);
+
+ PrintMallocStatsArray(" mallocs by size class: ", malloced_by_size);
+ Printf("Stats: malloc large: %zu\n", malloc_large);
+}
+
+void MemprofStats::MergeFrom(const MemprofStats *stats) {
+ uptr *dst_ptr = reinterpret_cast<uptr *>(this);
+ const uptr *src_ptr = reinterpret_cast<const uptr *>(stats);
+ uptr num_fields = sizeof(*this) / sizeof(uptr);
+ for (uptr i = 0; i < num_fields; i++)
+ dst_ptr[i] += src_ptr[i];
+}
+
+static BlockingMutex print_lock(LINKER_INITIALIZED);
+
+static MemprofStats unknown_thread_stats(LINKER_INITIALIZED);
+static MemprofStats dead_threads_stats(LINKER_INITIALIZED);
+static BlockingMutex dead_threads_stats_lock(LINKER_INITIALIZED);
+// Required for malloc_zone_statistics() on OS X. This can't be stored in
+// per-thread MemprofStats.
+static uptr max_malloced_memory;
+
+static void MergeThreadStats(ThreadContextBase *tctx_base, void *arg) {
+ MemprofStats *accumulated_stats = reinterpret_cast<MemprofStats *>(arg);
+ MemprofThreadContext *tctx = static_cast<MemprofThreadContext *>(tctx_base);
+ if (MemprofThread *t = tctx->thread)
+ accumulated_stats->MergeFrom(&t->stats());
+}
+
+static void GetAccumulatedStats(MemprofStats *stats) {
+ stats->Clear();
+ {
+ ThreadRegistryLock l(&memprofThreadRegistry());
+ memprofThreadRegistry().RunCallbackForEachThreadLocked(MergeThreadStats,
+ stats);
+ }
+ stats->MergeFrom(&unknown_thread_stats);
+ {
+ BlockingMutexLock lock(&dead_threads_stats_lock);
+ stats->MergeFrom(&dead_threads_stats);
+ }
+ // This is not very accurate: we may miss allocation peaks that happen
+ // between two updates of accumulated_stats_. For more accurate bookkeeping
+ // the maximum should be updated on every malloc(), which is unacceptable.
+ if (max_malloced_memory < stats->malloced) {
+ max_malloced_memory = stats->malloced;
+ }
+}
+
+void FlushToDeadThreadStats(MemprofStats *stats) {
+ BlockingMutexLock lock(&dead_threads_stats_lock);
+ dead_threads_stats.MergeFrom(stats);
+ stats->Clear();
+}
+
+MemprofStats &GetCurrentThreadStats() {
+ MemprofThread *t = GetCurrentThread();
+ return (t) ? t->stats() : unknown_thread_stats;
+}
+
+static void PrintAccumulatedStats() {
+ MemprofStats stats;
+ GetAccumulatedStats(&stats);
+ // Use lock to keep reports from mixing up.
+ BlockingMutexLock lock(&print_lock);
+ stats.Print();
+ StackDepotStats *stack_depot_stats = StackDepotGetStats();
+ Printf("Stats: StackDepot: %zd ids; %zdM allocated\n",
+ stack_depot_stats->n_uniq_ids, stack_depot_stats->allocated >> 20);
+ PrintInternalAllocatorStats();
+}
+
+} // namespace __memprof
+
+// ---------------------- Interface ---------------- {{{1
+using namespace __memprof;
+
+uptr __sanitizer_get_current_allocated_bytes() {
+ MemprofStats stats;
+ GetAccumulatedStats(&stats);
+ uptr malloced = stats.malloced;
+ uptr freed = stats.freed;
+ // Return sane value if malloced < freed due to racy
+ // way we update accumulated stats.
+ return (malloced > freed) ? malloced - freed : 1;
+}
+
+uptr __sanitizer_get_heap_size() {
+ MemprofStats stats;
+ GetAccumulatedStats(&stats);
+ return stats.mmaped - stats.munmaped;
+}
+
+uptr __sanitizer_get_free_bytes() {
+ MemprofStats stats;
+ GetAccumulatedStats(&stats);
+ uptr total_free = stats.mmaped - stats.munmaped + stats.really_freed;
+ uptr total_used = stats.malloced;
+ // Return sane value if total_free < total_used due to racy
+ // way we update accumulated stats.
+ return (total_free > total_used) ? total_free - total_used : 1;
+}
+
+uptr __sanitizer_get_unmapped_bytes() { return 0; }
+
+void __memprof_print_accumulated_stats() { PrintAccumulatedStats(); }
--- /dev/null
+//===-- memprof_stats.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for statistics.
+//===----------------------------------------------------------------------===//
+#ifndef MEMPROF_STATS_H
+#define MEMPROF_STATS_H
+
+#include "memprof_allocator.h"
+#include "memprof_internal.h"
+
+namespace __memprof {
+
+// MemprofStats struct is NOT thread-safe.
+// Each MemprofThread has its own MemprofStats, which are sometimes flushed
+// to the accumulated MemprofStats.
+struct MemprofStats {
+ // MemprofStats must be a struct consisting of uptr fields only.
+ // When merging two MemprofStats structs, we treat them as arrays of uptr.
+ uptr mallocs;
+ uptr malloced;
+ uptr malloced_overhead;
+ uptr frees;
+ uptr freed;
+ uptr real_frees;
+ uptr really_freed;
+ uptr reallocs;
+ uptr realloced;
+ uptr mmaps;
+ uptr mmaped;
+ uptr munmaps;
+ uptr munmaped;
+ uptr malloc_large;
+ uptr malloced_by_size[kNumberOfSizeClasses];
+
+ // Ctor for global MemprofStats (accumulated stats for dead threads).
+ explicit MemprofStats(LinkerInitialized) {}
+ // Creates empty stats.
+ MemprofStats();
+
+ void Print(); // Prints formatted stats to stderr.
+ void Clear();
+ void MergeFrom(const MemprofStats *stats);
+};
+
+// Returns stats for GetCurrentThread(), or stats for fake "unknown thread"
+// if GetCurrentThread() returns 0.
+MemprofStats &GetCurrentThreadStats();
+// Flushes a given stats into accumulated stats of dead threads.
+void FlushToDeadThreadStats(MemprofStats *stats);
+
+} // namespace __memprof
+
+#endif // MEMPROF_STATS_H
--- /dev/null
+//===-- memprof_thread.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// Thread-related code.
+//===----------------------------------------------------------------------===//
+#include "memprof_thread.h"
+#include "memprof_allocator.h"
+#include "memprof_interceptors.h"
+#include "memprof_mapping.h"
+#include "memprof_stack.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+namespace __memprof {
+
+// MemprofThreadContext implementation.
+
+void MemprofThreadContext::OnCreated(void *arg) {
+ CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs *>(arg);
+ if (args->stack)
+ stack_id = StackDepotPut(*args->stack);
+ thread = args->thread;
+ thread->set_context(this);
+}
+
+void MemprofThreadContext::OnFinished() {
+ // Drop the link to the MemprofThread object.
+ thread = nullptr;
+}
+
+static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)];
+static ThreadRegistry *memprof_thread_registry;
+
+static BlockingMutex mu_for_thread_context(LINKER_INITIALIZED);
+static LowLevelAllocator allocator_for_thread_context;
+
+static ThreadContextBase *GetMemprofThreadContext(u32 tid) {
+ BlockingMutexLock lock(&mu_for_thread_context);
+ return new (allocator_for_thread_context) MemprofThreadContext(tid);
+}
+
+ThreadRegistry &memprofThreadRegistry() {
+ static bool initialized;
+ // Don't worry about thread_safety - this should be called when there is
+ // a single thread.
+ if (!initialized) {
+ // Never reuse MemProf threads: we store pointer to MemprofThreadContext
+ // in TSD and can't reliably tell when no more TSD destructors will
+ // be called. It would be wrong to reuse MemprofThreadContext for another
+ // thread before all TSD destructors will be called for it.
+ memprof_thread_registry = new (thread_registry_placeholder)
+ ThreadRegistry(GetMemprofThreadContext);
+ initialized = true;
+ }
+ return *memprof_thread_registry;
+}
+
+MemprofThreadContext *GetThreadContextByTidLocked(u32 tid) {
+ return static_cast<MemprofThreadContext *>(
+ memprofThreadRegistry().GetThreadLocked(tid));
+}
+
+// MemprofThread implementation.
+
+MemprofThread *MemprofThread::Create(thread_callback_t start_routine, void *arg,
+ u32 parent_tid, StackTrace *stack,
+ bool detached) {
+ uptr PageSize = GetPageSizeCached();
+ uptr size = RoundUpTo(sizeof(MemprofThread), PageSize);
+ MemprofThread *thread = (MemprofThread *)MmapOrDie(size, __func__);
+ thread->start_routine_ = start_routine;
+ thread->arg_ = arg;
+ MemprofThreadContext::CreateThreadContextArgs args = {thread, stack};
+ memprofThreadRegistry().CreateThread(*reinterpret_cast<uptr *>(thread),
+ detached, parent_tid, &args);
+
+ return thread;
+}
+
+void MemprofThread::TSDDtor(void *tsd) {
+ MemprofThreadContext *context = (MemprofThreadContext *)tsd;
+ VReport(1, "T%d TSDDtor\n", context->tid);
+ if (context->thread)
+ context->thread->Destroy();
+}
+
+void MemprofThread::Destroy() {
+ int tid = this->tid();
+ VReport(1, "T%d exited\n", tid);
+
+ malloc_storage().CommitBack();
+ memprofThreadRegistry().FinishThread(tid);
+ FlushToDeadThreadStats(&stats_);
+ uptr size = RoundUpTo(sizeof(MemprofThread), GetPageSizeCached());
+ UnmapOrDie(this, size);
+ DTLS_Destroy();
+}
+
+inline MemprofThread::StackBounds MemprofThread::GetStackBounds() const {
+ if (stack_bottom_ >= stack_top_)
+ return {0, 0};
+ return {stack_bottom_, stack_top_};
+}
+
+uptr MemprofThread::stack_top() { return GetStackBounds().top; }
+
+uptr MemprofThread::stack_bottom() { return GetStackBounds().bottom; }
+
+uptr MemprofThread::stack_size() {
+ const auto bounds = GetStackBounds();
+ return bounds.top - bounds.bottom;
+}
+
+void MemprofThread::Init(const InitOptions *options) {
+ CHECK_EQ(this->stack_size(), 0U);
+ SetThreadStackAndTls(options);
+ if (stack_top_ != stack_bottom_) {
+ CHECK_GT(this->stack_size(), 0U);
+ CHECK(AddrIsInMem(stack_bottom_));
+ CHECK(AddrIsInMem(stack_top_ - 1));
+ }
+ int local = 0;
+ VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
+ (void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
+ &local);
+}
+
+thread_return_t
+MemprofThread::ThreadStart(tid_t os_id,
+ atomic_uintptr_t *signal_thread_is_registered) {
+ Init();
+ memprofThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular,
+ nullptr);
+ if (signal_thread_is_registered)
+ atomic_store(signal_thread_is_registered, 1, memory_order_release);
+
+ if (!start_routine_) {
+ // start_routine_ == 0 if we're on the main thread or on one of the
+ // OS X libdispatch worker threads. But nobody is supposed to call
+ // ThreadStart() for the worker threads.
+ CHECK_EQ(tid(), 0);
+ return 0;
+ }
+
+ return start_routine_(arg_);
+}
+
+MemprofThread *CreateMainThread() {
+ MemprofThread *main_thread = MemprofThread::Create(
+ /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ kMainTid,
+ /* stack */ nullptr, /* detached */ true);
+ SetCurrentThread(main_thread);
+ main_thread->ThreadStart(internal_getpid(),
+ /* signal_thread_is_registered */ nullptr);
+ return main_thread;
+}
+
+// This implementation doesn't use the argument, which is just passed down
+// from the caller of Init (which see, above). It's only there to support
+// OS-specific implementations that need more information passed through.
+void MemprofThread::SetThreadStackAndTls(const InitOptions *options) {
+ DCHECK_EQ(options, nullptr);
+ uptr tls_size = 0;
+ uptr stack_size = 0;
+ GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size,
+ &tls_begin_, &tls_size);
+ stack_top_ = stack_bottom_ + stack_size;
+ tls_end_ = tls_begin_ + tls_size;
+ dtls_ = DTLS_Get();
+
+ if (stack_top_ != stack_bottom_) {
+ int local;
+ CHECK(AddrIsInStack((uptr)&local));
+ }
+}
+
+bool MemprofThread::AddrIsInStack(uptr addr) {
+ const auto bounds = GetStackBounds();
+ return addr >= bounds.bottom && addr < bounds.top;
+}
+
+MemprofThread *GetCurrentThread() {
+ MemprofThreadContext *context =
+ reinterpret_cast<MemprofThreadContext *>(TSDGet());
+ if (!context)
+ return nullptr;
+ return context->thread;
+}
+
+void SetCurrentThread(MemprofThread *t) {
+ CHECK(t->context());
+ VReport(2, "SetCurrentThread: %p for thread %p\n", t->context(),
+ (void *)GetThreadSelf());
+ // Make sure we do not reset the current MemprofThread.
+ CHECK_EQ(0, TSDGet());
+ TSDSet(t->context());
+ CHECK_EQ(t->context(), TSDGet());
+}
+
+u32 GetCurrentTidOrInvalid() {
+ MemprofThread *t = GetCurrentThread();
+ return t ? t->tid() : kInvalidTid;
+}
+
+void EnsureMainThreadIDIsCorrect() {
+ MemprofThreadContext *context =
+ reinterpret_cast<MemprofThreadContext *>(TSDGet());
+ if (context && (context->tid == kMainTid))
+ context->os_id = GetTid();
+}
+} // namespace __memprof
--- /dev/null
+//===-- memprof_thread.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler, a memory profiler.
+//
+// MemProf-private header for memprof_thread.cpp.
+//===----------------------------------------------------------------------===//
+
+#ifndef MEMPROF_THREAD_H
+#define MEMPROF_THREAD_H
+
+#include "memprof_allocator.h"
+#include "memprof_internal.h"
+#include "memprof_stats.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_thread_registry.h"
+
+namespace __sanitizer {
+struct DTLS;
+} // namespace __sanitizer
+
+namespace __memprof {
+
+class MemprofThread;
+
+// These objects are created for every thread and are never deleted,
+// so we can find them by tid even if the thread is long dead.
+struct MemprofThreadContext final : public ThreadContextBase {
+ explicit MemprofThreadContext(int tid)
+ : ThreadContextBase(tid), announced(false),
+ destructor_iterations(GetPthreadDestructorIterations()), stack_id(0),
+ thread(nullptr) {}
+ bool announced;
+ u8 destructor_iterations;
+ u32 stack_id;
+ MemprofThread *thread;
+
+ void OnCreated(void *arg) override;
+ void OnFinished() override;
+
+ struct CreateThreadContextArgs {
+ MemprofThread *thread;
+ StackTrace *stack;
+ };
+};
+
+// MemprofThreadContext objects are never freed, so we need many of them.
+COMPILER_CHECK(sizeof(MemprofThreadContext) <= 256);
+
+// MemprofThread are stored in TSD and destroyed when the thread dies.
+class MemprofThread {
+public:
+ static MemprofThread *Create(thread_callback_t start_routine, void *arg,
+ u32 parent_tid, StackTrace *stack,
+ bool detached);
+ static void TSDDtor(void *tsd);
+ void Destroy();
+
+ struct InitOptions;
+ void Init(const InitOptions *options = nullptr);
+
+ thread_return_t ThreadStart(tid_t os_id,
+ atomic_uintptr_t *signal_thread_is_registered);
+
+ uptr stack_top();
+ uptr stack_bottom();
+ uptr stack_size();
+ uptr tls_begin() { return tls_begin_; }
+ uptr tls_end() { return tls_end_; }
+ DTLS *dtls() { return dtls_; }
+ u32 tid() { return context_->tid; }
+ MemprofThreadContext *context() { return context_; }
+ void set_context(MemprofThreadContext *context) { context_ = context; }
+
+ bool AddrIsInStack(uptr addr);
+
+ // True is this thread is currently unwinding stack (i.e. collecting a stack
+ // trace). Used to prevent deadlocks on platforms where libc unwinder calls
+ // malloc internally. See PR17116 for more details.
+ bool isUnwinding() const { return unwinding_; }
+ void setUnwinding(bool b) { unwinding_ = b; }
+
+ MemprofThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
+ MemprofStats &stats() { return stats_; }
+
+private:
+ // NOTE: There is no MemprofThread constructor. It is allocated
+ // via mmap() and *must* be valid in zero-initialized state.
+
+ void SetThreadStackAndTls(const InitOptions *options);
+
+ struct StackBounds {
+ uptr bottom;
+ uptr top;
+ };
+ StackBounds GetStackBounds() const;
+
+ MemprofThreadContext *context_;
+ thread_callback_t start_routine_;
+ void *arg_;
+
+ uptr stack_top_;
+ uptr stack_bottom_;
+
+ uptr tls_begin_;
+ uptr tls_end_;
+ DTLS *dtls_;
+
+ MemprofThreadLocalMallocStorage malloc_storage_;
+ MemprofStats stats_;
+ bool unwinding_;
+};
+
+// Returns a single instance of registry.
+ThreadRegistry &memprofThreadRegistry();
+
+// Must be called under ThreadRegistryLock.
+MemprofThreadContext *GetThreadContextByTidLocked(u32 tid);
+
+// Get the current thread. May return 0.
+MemprofThread *GetCurrentThread();
+void SetCurrentThread(MemprofThread *t);
+u32 GetCurrentTidOrInvalid();
+
+// Used to handle fork().
+void EnsureMainThreadIDIsCorrect();
+} // namespace __memprof
+
+#endif // MEMPROF_THREAD_H
--- /dev/null
+___memprof_default_options __memprof_profile_filename
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
msan_origin.h
msan_poisoning.h
msan_report.h
- msan_thread.h)
+ msan_thread.h
+ )
set(MSAN_RTL_CFLAGS ${SANITIZER_COMMON_CFLAGS})
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
endif()
endforeach()
-add_compiler_rt_resource_file(msan_blacklist msan_blacklist.txt msan)
-list(APPEND MSAN_RUNTIME_LIBRARIES msan_blacklist)
+add_compiler_rt_resource_file(msan_ignorelist msan_ignorelist.txt msan)
+list(APPEND MSAN_RUNTIME_LIBRARIES msan_ignorelist)
if(COMPILER_RT_INCLUDE_TESTS)
add_subdirectory(tests)
// keep_going is an old name for halt_on_error,
// and it has inverse meaning.
-class FlagHandlerKeepGoing : public FlagHandlerBase {
+class FlagHandlerKeepGoing final : public FlagHandlerBase {
bool *halt_on_error_;
public:
// FIXME: test and enable.
cf.check_printf = false;
cf.intercept_tls_get_addr = true;
- cf.exitcode = 77;
OverrideCommonFlags(cf);
}
#endif
// Override from user-specified string.
- if (__msan_default_options)
- parser.ParseString(__msan_default_options());
+ parser.ParseString(__msan_default_options());
#if MSAN_CONTAINS_UBSAN
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
#endif
if (!t || !StackTrace::WillUseFastUnwind(request_fast)) {
// Block reports from our interceptors during _Unwind_Backtrace.
SymbolizerScope sym_scope;
- return Unwind(max_depth, pc, bp, context, 0, 0, false);
+ return Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0,
+ t ? t->stack_bottom() : 0, false);
}
if (StackTrace::WillUseFastUnwind(request_fast))
Unwind(max_depth, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), true);
HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
}
-static void MsanCheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2) {
- Report("MemorySanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
- line, cond, (uptr)v1, (uptr)v2);
- PRINT_CURRENT_STACK_CHECK();
- Die();
+static void CheckUnwind() {
+ GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+ stack.Print();
}
void __msan_init() {
InitializeFlags();
// Install tool-specific callbacks in sanitizer_common.
- SetCheckFailedCallback(MsanCheckFailed);
+ SetCheckUnwindCallback(CheckUnwind);
__sanitizer_set_report_path(common_flags()->log_path);
sptr __msan_test_shadow(const void *x, uptr size) {
if (!MEM_IS_APP(x)) return -1;
unsigned char *s = (unsigned char *)MEM_TO_SHADOW((uptr)x);
+ if (__sanitizer::mem_is_zero((const char *)s, size))
+ return -1;
+ // Slow path: loop through again to find the location.
for (uptr i = 0; i < size; ++i)
if (s[i])
return i;
SetUserDieCallback(callback);
}
-#if !SANITIZER_SUPPORTS_WEAK_HOOKS
-extern "C" {
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-const char* __msan_default_options() { return ""; }
-} // extern "C"
-#endif
+void __msan_start_switch_fiber(const void *bottom, uptr size) {
+ MsanThread *t = GetCurrentThread();
+ if (!t) {
+ VReport(1, "__msan_start_switch_fiber called from unknown thread\n");
+ return;
+ }
+ t->StartSwitchFiber((uptr)bottom, size);
+}
+
+void __msan_finish_switch_fiber(const void **bottom_old, uptr *size_old) {
+ MsanThread *t = GetCurrentThread();
+ if (!t) {
+ VReport(1, "__msan_finish_switch_fiber called from unknown thread\n");
+ return;
+ }
+ t->FinishSwitchFiber((uptr *)bottom_old, (uptr *)size_old);
+
+ internal_memset(__msan_param_tls, 0, sizeof(__msan_param_tls));
+ internal_memset(__msan_retval_tls, 0, sizeof(__msan_retval_tls));
+ internal_memset(__msan_va_arg_tls, 0, sizeof(__msan_va_arg_tls));
+
+ if (__msan_get_track_origins()) {
+ internal_memset(__msan_param_origin_tls, 0,
+ sizeof(__msan_param_origin_tls));
+ internal_memset(&__msan_retval_origin_tls, 0,
+ sizeof(__msan_retval_origin_tls));
+ internal_memset(__msan_va_arg_origin_tls, 0,
+ sizeof(__msan_va_arg_origin_tls));
+ }
+}
+
+SANITIZER_INTERFACE_WEAK_DEF(const char *, __msan_default_options, void) {
+ return "";
+}
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
void InitializeInterceptors();
void MsanAllocatorInit();
-void MsanAllocatorThreadFinish();
void MsanDeallocate(StackTrace *stack, void *ptr);
void *msan_malloc(uptr size, StackTrace *stack);
stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal); \
}
-#define GET_FATAL_STACK_TRACE_HERE \
- GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
-
-#define PRINT_CURRENT_STACK_CHECK() \
- { \
- GET_FATAL_STACK_TRACE_HERE; \
- stack.Print(); \
- }
-
class ScopedThreadLocalStateBackup {
public:
ScopedThreadLocalStateBackup() { Backup(); }
}
}
-void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size,
- uptr alignment) {
+static void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size,
+ uptr alignment) {
Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));
uptr old_size = meta->requested_size;
uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
return new_p;
}
-void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
+static void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
if (AllocatorMayReturnNull())
return nullptr;
namespace __msan {
struct MsanThreadLocalMallocStorage {
- uptr quarantine_cache[16];
// Allocator cache contains atomic_uint64_t which must be 8-byte aligned.
ALIGNED(8) uptr allocator_cache[96 * (512 * 8 + 16)]; // Opaque.
void CommitBack();
-//===-- msan_chained_origin_depot.cpp ----------------------------------===//
+//===-- msan_chained_origin_depot.cpp -------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
//
+// This file is a part of MemorySanitizer.
+//
// A storage for chained origins.
//===----------------------------------------------------------------------===//
#include "msan_chained_origin_depot.h"
-#include "sanitizer_common/sanitizer_stackdepotbase.h"
+#include "sanitizer_common/sanitizer_chained_origin_depot.h"
namespace __msan {
-struct ChainedOriginDepotDesc {
- u32 here_id;
- u32 prev_id;
-};
-
-struct ChainedOriginDepotNode {
- ChainedOriginDepotNode *link;
- u32 id;
- u32 here_id;
- u32 prev_id;
-
- typedef ChainedOriginDepotDesc args_type;
-
- bool eq(u32 hash, const args_type &args) const {
- return here_id == args.here_id && prev_id == args.prev_id;
- }
-
- static uptr storage_size(const args_type &args) {
- return sizeof(ChainedOriginDepotNode);
- }
-
- /* This is murmur2 hash for the 64->32 bit case.
- It does not behave all that well because the keys have a very biased
- distribution (I've seen 7-element buckets with the table only 14% full).
-
- here_id is built of
- * (1 bits) Reserved, zero.
- * (8 bits) Part id = bits 13..20 of the hash value of here_id's key.
- * (23 bits) Sequential number (each part has each own sequence).
-
- prev_id has either the same distribution as here_id (but with 3:8:21)
- split, or one of two reserved values (-1) or (-2). Either case can
- dominate depending on the workload.
- */
- static u32 hash(const args_type &args) {
- const u32 m = 0x5bd1e995;
- const u32 seed = 0x9747b28c;
- const u32 r = 24;
- u32 h = seed;
- u32 k = args.here_id;
- k *= m;
- k ^= k >> r;
- k *= m;
- h *= m;
- h ^= k;
-
- k = args.prev_id;
- k *= m;
- k ^= k >> r;
- k *= m;
- h *= m;
- h ^= k;
-
- h ^= h >> 13;
- h *= m;
- h ^= h >> 15;
- return h;
- }
- static bool is_valid(const args_type &args) { return true; }
- void store(const args_type &args, u32 other_hash) {
- here_id = args.here_id;
- prev_id = args.prev_id;
- }
-
- args_type load() const {
- args_type ret = {here_id, prev_id};
- return ret;
- }
-
- struct Handle {
- ChainedOriginDepotNode *node_;
- Handle() : node_(nullptr) {}
- explicit Handle(ChainedOriginDepotNode *node) : node_(node) {}
- bool valid() { return node_; }
- u32 id() { return node_->id; }
- int here_id() { return node_->here_id; }
- int prev_id() { return node_->prev_id; }
- };
-
- Handle get_handle() { return Handle(this); }
-
- typedef Handle handle_type;
-};
-
-static StackDepotBase<ChainedOriginDepotNode, 4, 20> chainedOriginDepot;
+static ChainedOriginDepot chainedOriginDepot;
StackDepotStats *ChainedOriginDepotGetStats() {
return chainedOriginDepot.GetStats();
}
bool ChainedOriginDepotPut(u32 here_id, u32 prev_id, u32 *new_id) {
- ChainedOriginDepotDesc desc = {here_id, prev_id};
- bool inserted;
- ChainedOriginDepotNode::Handle h = chainedOriginDepot.Put(desc, &inserted);
- *new_id = h.valid() ? h.id() : 0;
- return inserted;
+ return chainedOriginDepot.Put(here_id, prev_id, new_id);
}
-// Retrieves a stored stack trace by the id.
u32 ChainedOriginDepotGet(u32 id, u32 *other) {
- ChainedOriginDepotDesc desc = chainedOriginDepot.Get(id);
- *other = desc.prev_id;
- return desc.here_id;
+ return chainedOriginDepot.Get(id, other);
}
void ChainedOriginDepotLockAll() {
-//===-- msan_chained_origin_depot.h --------------------------*- C++ -*-===//
+//===-- msan_chained_origin_depot.h -----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
//
//===----------------------------------------------------------------------===//
//
+// This file is a part of MemorySanitizer.
+//
// A storage for chained origins.
//===----------------------------------------------------------------------===//
+
#ifndef MSAN_CHAINED_ORIGIN_DEPOT_H
#define MSAN_CHAINED_ORIGIN_DEPOT_H
namespace __msan {
+// Gets the statistic of the origin chain storage.
StackDepotStats *ChainedOriginDepotGetStats();
+
+// Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
+// If successful, returns true and the new chain id new_id.
+// If the same element already exists, returns false and sets new_id to the
+// existing ID.
bool ChainedOriginDepotPut(u32 here_id, u32 prev_id, u32 *new_id);
-// Retrieves a stored stack trace by the id.
+
+// Retrieves the stored StackDepot ID for the given origin ID.
u32 ChainedOriginDepotGet(u32 id, u32 *other);
void ChainedOriginDepotLockAll();
--- /dev/null
+# Ignorelist for MemorySanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of ignorelist
+# at compile-time using -fsanitize-ignorelist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
+
+# https://bugs.llvm.org/show_bug.cgi?id=31877
+fun:__gxx_personality_*
#include "msan_report.h"
#include "msan_thread.h"
#include "msan_poisoning.h"
+#include "sanitizer_common/sanitizer_errno_codes.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
#include "sanitizer_common/sanitizer_allocator.h"
INTERCEPTOR(int, gethostname, char *name, SIZE_T len) {
ENSURE_MSAN_INITED();
int res = REAL(gethostname)(name, len);
- if (!res) {
+ if (!res || (res == -1 && errno == errno_ENAMETOOLONG)) {
SIZE_T real_len = REAL(strnlen)(name, len);
if (real_len < len)
++real_len;
CHECK_UNPOISONED_0(x, n); \
} while (0)
-#define MSAN_INTERCEPT_FUNC(name) \
- do { \
- if (!INTERCEPT_FUNCTION(name)) \
- VReport(1, "MemorySanitizer: failed to intercept '%s'\n'", #name); \
+#define MSAN_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "MemorySanitizer: failed to intercept '%s'\n", #name); \
} while (0)
#define MSAN_INTERCEPT_FUNC_VER(name, ver) \
VReport(1, "MemorySanitizer: failed to intercept '%s@@%s'\n", #name, \
#ver); \
} while (0)
+#define MSAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \
+ do { \
+ if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \
+ VReport(1, "MemorySanitizer: failed to intercept '%s@@%s' or '%s'\n", \
+ #name, #ver, #name); \
+ } while (0)
#define COMMON_INTERCEPT_FUNCTION(name) MSAN_INTERCEPT_FUNC(name)
-#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
+#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
MSAN_INTERCEPT_FUNC_VER(name, ver)
+#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
+ MSAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver)
#define COMMON_INTERCEPTOR_UNPOISON_PARAM(count) \
UnpoisonParam(count)
#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
static int sigaction_impl(int signo, const __sanitizer_sigaction *act,
__sanitizer_sigaction *oldact) {
ENSURE_MSAN_INITED();
+ if (signo <= 0 || signo >= kMaxSignals) {
+ errno = errno_EINVAL;
+ return -1;
+ }
if (act) read_sigaction(act);
int res;
if (flags()->wrap_signals) {
SpinMutexLock lock(&sigactions_mu);
- CHECK_LT(signo, kMaxSignals);
uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed);
__sanitizer_sigaction new_act;
__sanitizer_sigaction *pnew_act = act ? &new_act : nullptr;
static uptr signal_impl(int signo, uptr cb) {
ENSURE_MSAN_INITED();
+ if (signo <= 0 || signo >= kMaxSignals) {
+ errno = errno_EINVAL;
+ return -1;
+ }
if (flags()->wrap_signals) {
- CHECK_LT(signo, kMaxSignals);
SpinMutexLock lock(&sigactions_mu);
if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
atomic_store(&sigactions[signo], cb, memory_order_relaxed);
SANITIZER_INTERFACE_ATTRIBUTE
int __msan_set_poison_in_malloc(int do_poison);
-SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-/* OPTIONAL */ const char* __msan_default_options();
+SANITIZER_INTERFACE_ATTRIBUTE
+const char *__msan_default_options();
// For testing.
SANITIZER_INTERFACE_ATTRIBUTE
SANITIZER_INTERFACE_ATTRIBUTE
void __msan_scoped_enable_interceptor_checks();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __msan_start_switch_fiber(const void *bottom, uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __msan_finish_switch_fiber(const void **bottom_old, uptr *size_old);
} // extern "C"
#endif // MSAN_INTERFACE_INTERNAL_H
#include <signal.h>
#include <unistd.h>
#include <unwind.h>
-#include <execinfo.h>
#include <sys/time.h>
#include <sys/resource.h>
if (map) {
if (!CheckMemoryRangeAvailability(start, size))
return false;
- if (!MmapFixedNoReserve(start, size, kMemoryLayout[i].name))
+ if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name))
return false;
if (common_flags()->use_madv_dontdump)
DontDumpShadowMemory(start, size);
uptr beg = d & ~3UL;
// Copy left unaligned origin if that memory is poisoned.
if (beg < d) {
- u32 o = GetOriginIfPoisoned((uptr)src, d - beg);
+ u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);
if (o) {
if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack);
*(u32 *)MEM_TO_ORIGIN(beg) = o;
}
}
+void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
+ StackTrace *stack) {
+ if (!MEM_IS_APP(dst) || !MEM_IS_APP(src))
+ return;
+
+ uptr d = (uptr)dst;
+ uptr end = (d + size) & ~3UL;
+
+ // Copy right unaligned origin if that memory is poisoned.
+ if (end < d + size) {
+ u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);
+ if (o) {
+ if (__msan_get_track_origins() > 1)
+ o = ChainOrigin(o, stack);
+ *(u32 *)MEM_TO_ORIGIN(end) = o;
+ }
+ }
+
+ uptr beg = d & ~3UL;
+
+ if (beg + 4 < end) {
+ // Align src up.
+ uptr s = ((uptr)src + 3) & ~3UL;
+ if (__msan_get_track_origins() > 1) {
+ u32 *src = (u32 *)MEM_TO_ORIGIN(s + end - beg - 4);
+ u32 *src_s = (u32 *)MEM_TO_SHADOW(s + end - beg - 4);
+ u32 *src_begin = (u32 *)MEM_TO_ORIGIN(s);
+ u32 *dst = (u32 *)MEM_TO_ORIGIN(end - 4);
+ u32 src_o = 0;
+ u32 dst_o = 0;
+ for (; src >= src_begin; --src, --src_s, --dst) {
+ if (!*src_s)
+ continue;
+ if (*src != src_o) {
+ src_o = *src;
+ dst_o = ChainOrigin(src_o, stack);
+ }
+ *dst = dst_o;
+ }
+ } else {
+ REAL(memmove)
+ ((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s), end - beg - 4);
+ }
+ }
+
+ // Copy left unaligned origin if that memory is poisoned.
+ if (beg < d) {
+ u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);
+ if (o) {
+ if (__msan_get_track_origins() > 1)
+ o = ChainOrigin(o, stack);
+ *(u32 *)MEM_TO_ORIGIN(beg) = o;
+ }
+ }
+}
+
+void MoveOrigin(const void *dst, const void *src, uptr size,
+ StackTrace *stack) {
+ // If destination origin range overlaps with source origin range, move
+ // origins by coping origins in a reverse order; otherwise, copy origins in
+ // a normal order.
+ uptr src_aligned_beg = reinterpret_cast<uptr>(src) & ~3UL;
+ uptr src_aligned_end = (reinterpret_cast<uptr>(src) + size) & ~3UL;
+ uptr dst_aligned_beg = reinterpret_cast<uptr>(dst) & ~3UL;
+ if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
+ return ReverseCopyOrigin(dst, src, size, stack);
+ return CopyOrigin(dst, src, size, stack);
+}
+
void MoveShadowAndOrigin(const void *dst, const void *src, uptr size,
StackTrace *stack) {
if (!MEM_IS_APP(dst)) return;
if (!MEM_IS_APP(src)) return;
if (src == dst) return;
+ // MoveOrigin transfers origins by refering to their shadows. So we
+ // need to move origins before moving shadows.
+ if (__msan_get_track_origins())
+ MoveOrigin(dst, src, size, stack);
REAL(memmove)((void *)MEM_TO_SHADOW((uptr)dst),
(void *)MEM_TO_SHADOW((uptr)src), size);
- if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack);
}
void CopyShadowAndOrigin(const void *dst, const void *src, uptr size,
StackTrace *stack) {
if (!MEM_IS_APP(dst)) return;
if (!MEM_IS_APP(src)) return;
+ // Because origin's range is slightly larger than app range, memcpy may also
+ // cause overlapped origin ranges.
REAL(memcpy)((void *)MEM_TO_SHADOW((uptr)dst),
(void *)MEM_TO_SHADOW((uptr)src), size);
- if (__msan_get_track_origins()) CopyOrigin(dst, src, size, stack);
+ if (__msan_get_track_origins())
+ MoveOrigin(dst, src, size, stack);
}
void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack) {
if (page_end != shadow_end) {
REAL(memset)((void *)page_end, 0, shadow_end - page_end);
}
- if (!MmapFixedNoReserve(page_beg, page_end - page_beg))
+ if (!MmapFixedSuperNoReserve(page_beg, page_end - page_beg))
Die();
}
}
void MsanThread::SetThreadStackAndTls() {
uptr tls_size = 0;
uptr stack_size = 0;
- GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size,
- &tls_begin_, &tls_size);
- stack_top_ = stack_bottom_ + stack_size;
+ GetThreadStackAndTls(IsMainThread(), &stack_.bottom, &stack_size, &tls_begin_,
+ &tls_size);
+ stack_.top = stack_.bottom + stack_size;
tls_end_ = tls_begin_ + tls_size;
int local;
}
void MsanThread::ClearShadowForThreadStackAndTLS() {
- __msan_unpoison((void *)stack_bottom_, stack_top_ - stack_bottom_);
+ __msan_unpoison((void *)stack_.bottom, stack_.top - stack_.bottom);
if (tls_begin_ != tls_end_)
__msan_unpoison((void *)tls_begin_, tls_end_ - tls_begin_);
DTLS *dtls = DTLS_Get();
CHECK_NE(dtls, 0);
- for (uptr i = 0; i < dtls->dtv_size; ++i)
- __msan_unpoison((void *)(dtls->dtv[i].beg), dtls->dtv[i].size);
+ ForEachDVT(dtls, [](const DTLS::DTV &dtv, int id) {
+ __msan_unpoison((void *)(dtv.beg), dtv.size);
+ });
}
void MsanThread::Init() {
SetThreadStackAndTls();
- CHECK(MEM_IS_APP(stack_bottom_));
- CHECK(MEM_IS_APP(stack_top_ - 1));
+ CHECK(MEM_IS_APP(stack_.bottom));
+ CHECK(MEM_IS_APP(stack_.top - 1));
ClearShadowForThreadStackAndTLS();
}
return res;
}
+MsanThread::StackBounds MsanThread::GetStackBounds() const {
+ if (!stack_switching_)
+ return {stack_.bottom, stack_.top};
+ const uptr cur_stack = GET_CURRENT_FRAME();
+ // Note: need to check next stack first, because FinishSwitchFiber
+ // may be in process of overwriting stack_.top/bottom_. But in such case
+ // we are already on the next stack.
+ if (cur_stack >= next_stack_.bottom && cur_stack < next_stack_.top)
+ return {next_stack_.bottom, next_stack_.top};
+ return {stack_.bottom, stack_.top};
+}
+
+uptr MsanThread::stack_top() { return GetStackBounds().top; }
+
+uptr MsanThread::stack_bottom() { return GetStackBounds().bottom; }
+
+bool MsanThread::AddrIsInStack(uptr addr) {
+ const auto bounds = GetStackBounds();
+ return addr >= bounds.bottom && addr < bounds.top;
+}
+
+void MsanThread::StartSwitchFiber(uptr bottom, uptr size) {
+ CHECK(!stack_switching_);
+ next_stack_.bottom = bottom;
+ next_stack_.top = bottom + size;
+ stack_switching_ = true;
+}
+
+void MsanThread::FinishSwitchFiber(uptr *bottom_old, uptr *size_old) {
+ CHECK(stack_switching_);
+ if (bottom_old)
+ *bottom_old = stack_.bottom;
+ if (size_old)
+ *size_old = stack_.top - stack_.bottom;
+ stack_.bottom = next_stack_.bottom;
+ stack_.top = next_stack_.top;
+ stack_switching_ = false;
+ next_stack_.top = 0;
+ next_stack_.bottom = 0;
+}
+
} // namespace __msan
void Init(); // Should be called from the thread itself.
thread_return_t ThreadStart();
- uptr stack_top() { return stack_top_; }
- uptr stack_bottom() { return stack_bottom_; }
+ uptr stack_top();
+ uptr stack_bottom();
uptr tls_begin() { return tls_begin_; }
uptr tls_end() { return tls_end_; }
bool IsMainThread() { return start_routine_ == nullptr; }
- bool AddrIsInStack(uptr addr) {
- return addr >= stack_bottom_ && addr < stack_top_;
- }
+ bool AddrIsInStack(uptr addr);
bool InSignalHandler() { return in_signal_handler_; }
void EnterSignalHandler() { in_signal_handler_++; }
void LeaveSignalHandler() { in_signal_handler_--; }
+ void StartSwitchFiber(uptr bottom, uptr size);
+ void FinishSwitchFiber(uptr *bottom_old, uptr *size_old);
+
MsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
int destructor_iterations_;
// via mmap() and *must* be valid in zero-initialized state.
void SetThreadStackAndTls();
void ClearShadowForThreadStackAndTLS();
+ struct StackBounds {
+ uptr bottom;
+ uptr top;
+ };
+ StackBounds GetStackBounds() const;
thread_callback_t start_routine_;
void *arg_;
- uptr stack_top_;
- uptr stack_bottom_;
+
+ bool stack_switching_;
+
+ StackBounds stack_;
+ StackBounds next_stack_;
+
uptr tls_begin_;
uptr tls_end_;
-fsanitize=memory
-fsanitize-memory-track-origins
-Wno-pedantic
- -Xclang -fdepfile-entry=${COMPILER_RT_OUTPUT_DIR}/share/msan_blacklist.txt
+ -Xclang -fdepfile-entry=${COMPILER_RT_OUTPUT_DIR}/share/msan_ignorelist.txt
)
# Unittest sources and build flags.
)
set(MSAN_UNITTEST_COMMON_CFLAGS
-nostdinc++
- -isystem ${COMPILER_RT_LIBCXX_PATH}/include
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
KIND ${kind}
COMPILE_DEPS ${MSAN_UNITTEST_HEADERS}
DEPS gtest msan
- CFLAGS ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags}
+ CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1
+ ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags}
)
endmacro()
}
#endif
+#if defined(__linux__) && !defined(__GLIBC__) && !defined(__ANDROID__)
+#define MUSL 1
+#endif
+
#include <inttypes.h>
#include <stdlib.h>
#include <stdarg.h>
typedef signed int S4;
typedef signed long long S8;
#define NOINLINE __attribute__((noinline))
-#define INLINE __attribute__((always_inline))
+#define ALWAYS_INLINE __attribute__((always_inline))
static bool TrackingOrigins() {
S8 x;
return res;
}
-TEST(MemorySanitizer, strcmp) {
- char s1[10];
- char s2[10];
- strncpy(s1, "foo", 10);
- s2[0] = 'f';
- s2[1] = 'n';
- EXPECT_GT(strcmp(s1, s2), 0);
- s2[1] = 'o';
- int res;
- EXPECT_UMR(res = strcmp(s1, s2));
- EXPECT_NOT_POISONED(res);
- EXPECT_EQ(strncmp(s1, s2, 1), 0);
-}
-
TEST(MemorySanitizer, LargeRet) {
LargeStruct a = LargeRetTest();
EXPECT_POISONED(a.x[0]);
return result;
}
-INSTANTIATE_TEST_CASE_P(IpTests, MemorySanitizerIpTest,
- ::testing::ValuesIn(GetAvailableIpSocketFamilies()));
+INSTANTIATE_TEST_SUITE_P(IpTests, MemorySanitizerIpTest,
+ ::testing::ValuesIn(GetAvailableIpSocketFamilies()));
TEST_P(MemorySanitizerIpTest, accept) {
int listen_socket = CreateSocket(SOCK_STREAM);
} while (0)
TEST(MemorySanitizer, gethostent) {
+ sethostent(0);
struct hostent *he = gethostent();
ASSERT_NE((void *)NULL, he);
EXPECT_HOSTENT_NOT_POISONED(he);
EXPECT_HOSTENT_NOT_POISONED(he);
}
-#if !defined(__NetBSD__)
+#if defined(__GLIBC__) || defined(__FreeBSD__)
TEST(MemorySanitizer, gethostent_r) {
+ sethostent(0);
char buf[2000];
struct hostent he;
struct hostent *result;
ASSERT_GT(res, -1);
}
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, random_r) {
int32_t x;
char z[64];
free(res);
}
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, canonicalize_file_name) {
const char* relpath = ".";
char* res = canonicalize_file_name(relpath);
TEST_STRTO_INT(strtoll, char, )
TEST_STRTO_INT(strtoul, char, )
TEST_STRTO_INT(strtoull, char, )
+#ifndef MUSL
TEST_STRTO_INT(strtouq, char, )
+#endif
TEST_STRTO_FLOAT(strtof, char, )
TEST_STRTO_FLOAT(strtod, char, )
TEST_STRTO_FLOAT(strtold, char, )
+#ifndef MUSL
TEST_STRTO_FLOAT_LOC(strtof_l, char, )
TEST_STRTO_FLOAT_LOC(strtod_l, char, )
TEST_STRTO_FLOAT_LOC(strtold_l, char, )
TEST_STRTO_INT_LOC(strtoll_l, char, )
TEST_STRTO_INT_LOC(strtoul_l, char, )
TEST_STRTO_INT_LOC(strtoull_l, char, )
+#endif
TEST_STRTO_INT(wcstol, wchar_t, L)
TEST_STRTO_INT(wcstoll, wchar_t, L)
TEST_STRTO_FLOAT(wcstod, wchar_t, L)
TEST_STRTO_FLOAT(wcstold, wchar_t, L)
+#ifndef MUSL
TEST_STRTO_FLOAT_LOC(wcstof_l, wchar_t, L)
TEST_STRTO_FLOAT_LOC(wcstod_l, wchar_t, L)
TEST_STRTO_FLOAT_LOC(wcstold_l, wchar_t, L)
TEST_STRTO_INT_LOC(wcstoll_l, wchar_t, L)
TEST_STRTO_INT_LOC(wcstoul_l, wchar_t, L)
TEST_STRTO_INT_LOC(wcstoull_l, wchar_t, L)
+#endif
TEST(MemorySanitizer, strtoimax) {
#endif // __GLIBC__
TEST(MemorySanitizer, modf) {
- double x, y;
- x = modf(2.1, &y);
+ double y;
+ modf(2.1, &y);
EXPECT_NOT_POISONED(y);
}
TEST(MemorySanitizer, modff) {
- float x, y;
- x = modff(2.1, &y);
+ float y;
+ modff(2.1, &y);
EXPECT_NOT_POISONED(y);
}
TEST(MemorySanitizer, modfl) {
- long double x, y;
- x = modfl(2.1, &y);
+ long double y;
+ modfl(2.1, &y);
EXPECT_NOT_POISONED(y);
}
}
#endif
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, drand48_r) {
struct drand48_data buf;
srand48_r(0, &buf);
drand48_r(&buf, &d);
EXPECT_NOT_POISONED(d);
}
-#endif
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
TEST(MemorySanitizer, lrand48_r) {
struct drand48_data buf;
srand48_r(0, &buf);
}
#endif
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, getmntent_r) {
TempFstabFile fstabtmp;
ASSERT_TRUE(fstabtmp.Create());
int res = sysctl(mib, 4, buf, &sz, NULL, 0);
ASSERT_EQ(0, res);
}
-#elif defined(__GLIBC__)
+#elif defined(__GLIBC__) || defined(MUSL)
static void GetProgramPath(char *buf, size_t sz) {
extern char *program_invocation_name;
int res = snprintf(buf, sz, "%s", program_invocation_name);
#if !defined(__FreeBSD__) && !defined(__NetBSD__)
TEST(MemorySanitizer, sched_getaffinity) {
cpu_set_t mask;
- int res = sched_getaffinity(getpid(), sizeof(mask), &mask);
- ASSERT_EQ(0, res);
- EXPECT_NOT_POISONED(mask);
+ if (sched_getaffinity(getpid(), sizeof(mask), &mask) == 0)
+ EXPECT_NOT_POISONED(mask);
+ else {
+ // The call to sched_getaffinity() may have failed because the Affinity
+ // mask is too small for the number of CPUs on the system (i.e. the
+ // system has more than 1024 CPUs). Allocate a mask large enough for
+ // twice as many CPUs.
+ cpu_set_t *DynAffinity;
+ DynAffinity = CPU_ALLOC(2048);
+ int res = sched_getaffinity(getpid(), CPU_ALLOC_SIZE(2048), DynAffinity);
+ ASSERT_EQ(0, res);
+ EXPECT_NOT_POISONED(*DynAffinity);
+ }
}
#endif
EXPECT_NOT_POISONED(v);
EXPECT_NOT_POISONED(w);
}
-#if !defined(__NetBSD__)
+#ifdef __GLIBC__
{
cpu_set_t v;
res = pthread_attr_getaffinity_np(&attr, sizeof(v), &v);
free(a);
}
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, pvalloc) {
uintptr_t PageSize = GetPageSize();
void *p = pvalloc(PageSize + 100);
}
TEST(MemorySanitizer, gethostname) {
- char buf[100];
- int res = gethostname(buf, 100);
- ASSERT_EQ(0, res);
+ char buf[1000];
+ EXPECT_EQ(-1, gethostname(buf, 1));
+ EXPECT_EQ(ENAMETOOLONG, errno);
+ EXPECT_NOT_POISONED(buf[0]);
+ EXPECT_POISONED(buf[1]);
+
+ __msan_poison(buf, sizeof(buf));
+ EXPECT_EQ(0, gethostname(buf, sizeof(buf)));
EXPECT_NOT_POISONED(strlen(buf));
}
strncpy(s, "abcd", 5);
__msan_poison(s, 5);
char buf[10000];
- int res;
- EXPECT_UMR(res = getpwnam_r(s, &pwd, buf, sizeof(buf), &pwdres));
+ EXPECT_UMR(getpwnam_r(s, &pwd, buf, sizeof(buf), &pwdres));
}
TEST(MemorySanitizer, getgrnam_r) {
EXPECT_NOT_POISONED(p->pw_uid);
}
+#ifndef MUSL
TEST(MemorySanitizer, getpwent_r) {
struct passwd pwd;
struct passwd *pwdres;
EXPECT_NOT_POISONED(pwd.pw_uid);
EXPECT_NOT_POISONED(pwdres);
}
+#endif
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, fgetpwent) {
FILE *fp = fopen("/etc/passwd", "r");
struct passwd *p = fgetpwent(fp);
EXPECT_NOT_POISONED(p->gr_gid);
}
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, fgetgrent) {
FILE *fp = fopen("/etc/group", "r");
struct group *grp = fgetgrent(fp);
}
#endif
+#if defined(__GLIBC__) || defined(__FreeBSD__)
TEST(MemorySanitizer, getgrent_r) {
struct group grp;
struct group *grpres;
EXPECT_NOT_POISONED(grp.gr_gid);
EXPECT_NOT_POISONED(grpres);
}
+#endif
-#if !defined(__FreeBSD__) && !defined(__NetBSD__)
+#ifdef __GLIBC__
TEST(MemorySanitizer, fgetgrent_r) {
FILE *fp = fopen("/etc/group", "r");
struct group grp;
} // namespace
template<class T, class BinaryOp>
-INLINE
+ALWAYS_INLINE
void BinaryOpOriginTest(BinaryOp op) {
U4 ox = rand(); //NOLINT
U4 oy = rand(); //NOLINT
EXPECT_ORIGIN(ox, __msan_get_origin(z));
}
-template<class T> INLINE T XOR(const T &a, const T&b) { return a ^ b; }
-template<class T> INLINE T ADD(const T &a, const T&b) { return a + b; }
-template<class T> INLINE T SUB(const T &a, const T&b) { return a - b; }
-template<class T> INLINE T MUL(const T &a, const T&b) { return a * b; }
-template<class T> INLINE T AND(const T &a, const T&b) { return a & b; }
-template<class T> INLINE T OR (const T &a, const T&b) { return a | b; }
+template<class T> ALWAYS_INLINE T XOR(const T &a, const T&b) { return a ^ b; }
+template<class T> ALWAYS_INLINE T ADD(const T &a, const T&b) { return a + b; }
+template<class T> ALWAYS_INLINE T SUB(const T &a, const T&b) { return a - b; }
+template<class T> ALWAYS_INLINE T MUL(const T &a, const T&b) { return a * b; }
+template<class T> ALWAYS_INLINE T AND(const T &a, const T&b) { return a & b; }
+template<class T> ALWAYS_INLINE T OR (const T &a, const T&b) { return a | b; }
TEST(MemorySanitizerOrigins, BinaryOp) {
if (!TrackingOrigins()) return;
__builtin_ia32_bzhi_di(0xABCDABCDABCDABCD, Poisoned<U8>(1, 0xFFFFFFFF00000000ULL)));
}
-inline U4 bextr_imm(U4 start, U4 len) {
+ALWAYS_INLINE U4 bextr_imm(U4 start, U4 len) {
start &= 0xFF;
len &= 0xFF;
return (len << 8) | start;
// __gxx_personality_v0 is instrumented, libgcc_s is not; as a result,
// __msan_param_tls is not updated and __gxx_personality_v0 can find
// leftover poison from the previous call.
- // A suppression in msan_blacklist.txt makes it work.
+ // A suppression in msan_ignorelist.txt makes it work.
throw_stuff();
} catch (const int &e) {
// pass
--- /dev/null
+# Build for all components of the ORC runtime support library.
+
+# ORC runtime library implementation files.
+set(ORC_SOURCES
+ extensible_rtti.cpp
+ log_error_to_stderr.cpp
+ macho_platform.cpp
+ run_program_wrapper.cpp
+ )
+
+# Implementation files for all ORC architectures.
+set(x86_64_SOURCES
+# x86-64 specific assembly files will go here.
+ macho_tlv.x86-64.S
+)
+
+set(ORC_IMPL_HEADERS
+# Implementation headers will go here.
+ adt.h
+ c_api.h
+ common.h
+ compiler.h
+ endianness.h
+ error.h
+ executor_address.h
+ extensible_rtti.h
+ macho_platform.h
+ simple_packed_serialization.h
+ stl_extras.h
+ wrapper_function_utils.h
+)
+
+# Create list of all source files for
+# consumption by tests.
+set(ORC_ALL_SOURCE_FILES
+ ${ORC_SOURCES}
+ ${x86_64_SOURCES}
+ ${ORC_IMPL_HEADERS}
+ )
+
+list(REMOVE_DUPLICATES ORC_ALL_SOURCE_FILES)
+
+# Now put it all together...
+include_directories(..)
+include_directories(../../include)
+
+set(ORC_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
+
+# Allow the ORC runtime to reference LLVM headers.
+foreach (DIR ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
+ list(APPEND ORC_CFLAGS -I${DIR})
+endforeach()
+
+add_compiler_rt_component(orc)
+
+# ORC uses C++ standard library headers.
+if (TARGET cxx-headers OR HAVE_LIBCXX)
+ set(ORC_DEPS cxx-headers)
+endif()
+
+if (APPLE)
+ add_compiler_rt_object_libraries(RTOrc
+ OS ${ORC_SUPPORTED_OS}
+ ARCHS ${ORC_SUPPORTED_ARCH}
+ SOURCES ${ORC_SOURCES} ${x86_64_SOURCES}
+ ADDITIONAL_HEADERS ${ORC_IMPL_HEADERS}
+ CFLAGS ${ORC_CFLAGS}
+ DEPS ${ORC_DEPS})
+
+ # We only support running on osx for now.
+ add_compiler_rt_runtime(clang_rt.orc
+ STATIC
+ OS ${ORC_SUPPORTED_OS}
+ ARCHS ${ORC_SUPPORTED_ARCH}
+ OBJECT_LIBS RTOrc
+ CFLAGS ${ORC_CFLAGS}
+ LINK_FLAGS ${SANITIZER_COMMON_LINK_FLAGS} ${WEAK_SYMBOL_LINK_FLAGS}
+ LINK_LIBS ${ORC_LINK_LIBS}
+ PARENT_TARGET orc)
+else() # not Apple
+ foreach(arch ${ORC_SUPPORTED_ARCH})
+ if(NOT CAN_TARGET_${arch})
+ continue()
+ endif()
+ add_compiler_rt_object_libraries(RTOrc
+ ARCHS ${arch}
+ SOURCES ${ORC_SOURCES} ${${arch}_SOURCES}
+ ADDITIONAL_HEADERS ${ORC_IMPL_HEADERS}
+ CFLAGS ${ORC_CFLAGS}
+ DEPS ${ORC_DEPS})
+
+ # Common ORC archive for instrumented binaries.
+ add_compiler_rt_runtime(clang_rt.orc
+ STATIC
+ ARCHS ${arch}
+ CFLAGS ${ORC_CFLAGS}
+ OBJECT_LIBS ${ORC_COMMON_RUNTIME_OBJECT_LIBS} RTOrc
+ PARENT_TARGET orc)
+ endforeach()
+endif() # not Apple
+
+if(COMPILER_RT_INCLUDE_TESTS)
+ add_subdirectory(unittests)
+endif()
--- /dev/null
+//===----------------------- adt.h - Handy ADTs -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ADT_H
+#define ORC_RT_ADT_H
+
+#include <cstring>
+#include <limits>
+#include <string>
+
+namespace __orc_rt {
+
+constexpr std::size_t dynamic_extent = std::numeric_limits<std::size_t>::max();
+
+/// A substitute for std::span (and llvm::ArrayRef).
+/// FIXME: Remove in favor of std::span once we can use c++20.
+template <typename T, std::size_t Extent = dynamic_extent> class span {
+public:
+ typedef T element_type;
+ typedef std::remove_cv<T> value_type;
+ typedef std::size_t size_type;
+ typedef std::ptrdiff_t difference_type;
+ typedef T *pointer;
+ typedef const T *const_pointer;
+ typedef T &reference;
+ typedef const T &const_reference;
+
+ typedef pointer iterator;
+
+ static constexpr std::size_t extent = Extent;
+
+ constexpr span() noexcept = default;
+ constexpr span(T *first, size_type count) noexcept
+ : Data(first), Size(count) {}
+
+ template <std::size_t N>
+ constexpr span(T (&arr)[N]) noexcept : Data(&arr[0]), Size(N) {}
+
+ constexpr iterator begin() const noexcept { return Data; }
+ constexpr iterator end() const noexcept { return Data + Size; }
+ constexpr pointer data() const noexcept { return Data; }
+ constexpr reference operator[](size_type idx) const { return Data[idx]; }
+ constexpr size_type size() const noexcept { return Size; }
+ constexpr bool empty() const noexcept { return Size == 0; }
+
+private:
+ T *Data = nullptr;
+ size_type Size = 0;
+};
+
+/// A substitue for std::string_view (and llvm::StringRef).
+/// FIXME: Remove in favor of std::string_view once we have c++17.
+class string_view {
+public:
+ typedef char value_type;
+ typedef char *pointer;
+ typedef const char *const_pointer;
+ typedef char &reference;
+ typedef const char &const_reference;
+ typedef std::size_t size_type;
+ typedef std::ptrdiff_t difference_type;
+
+ typedef const_pointer const_iterator;
+ typedef const_iterator iterator;
+
+ constexpr string_view() noexcept = default;
+ constexpr string_view(const char *S, size_type Count)
+ : Data(S), Size(Count) {}
+ string_view(const char *S) : Data(S), Size(strlen(S)) {}
+
+ constexpr const_iterator begin() const noexcept { return Data; }
+ constexpr const_iterator end() const noexcept { return Data + Size; }
+ constexpr const_pointer data() const noexcept { return Data; }
+ constexpr const_reference operator[](size_type idx) { return Data[idx]; }
+ constexpr size_type size() const noexcept { return Size; }
+ constexpr bool empty() const noexcept { return Size == 0; }
+
+ friend bool operator==(const string_view &LHS, const string_view &RHS) {
+ if (LHS.Size != RHS.Size)
+ return false;
+ if (LHS.Data == RHS.Data)
+ return true;
+ for (size_t I = 0; I != LHS.Size; ++I)
+ if (LHS.Data[I] != RHS.Data[I])
+ return false;
+ return true;
+ }
+
+ friend bool operator!=(const string_view &LHS, const string_view &RHS) {
+ return !(LHS == RHS);
+ }
+
+private:
+ const char *Data = nullptr;
+ size_type Size = 0;
+};
+
+inline std::string to_string(string_view SV) {
+ return std::string(SV.data(), SV.size());
+}
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_COMMON_H
--- /dev/null
+/*===- c_api.h - C API for the ORC runtime ------------------------*- C -*-===*\
+|* *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *|
+|* Exceptions. *|
+|* See https://llvm.org/LICENSE.txt for license information. *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *|
+|* *|
+|*===----------------------------------------------------------------------===*|
+|* *|
+|* This file defines the C API for the ORC runtime *|
+|* *|
+|* NOTE: The OrtRTWrapperFunctionResult type must be kept in sync with the *|
+|* definition in llvm/include/llvm-c/OrcShared.h. *|
+|* *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef ORC_RT_C_API_H
+#define ORC_RT_C_API_H
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Helper to suppress strict prototype warnings. */
+#ifdef __clang__
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN \
+ _Pragma("clang diagnostic push") \
+ _Pragma("clang diagnostic error \"-Wstrict-prototypes\"")
+#define ORC_RT_C_STRICT_PROTOTYPES_END _Pragma("clang diagnostic pop")
+#else
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+/* Helper to wrap C code for C++ */
+#ifdef __cplusplus
+#define ORC_RT_C_EXTERN_C_BEGIN \
+ extern "C" { \
+ ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END \
+ ORC_RT_C_STRICT_PROTOTYPES_END \
+ }
+#else
+#define ORC_RT_C_EXTERN_C_BEGIN ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+ORC_RT_C_EXTERN_C_BEGIN
+
+typedef union {
+ char *ValuePtr;
+ char Value[sizeof(ValuePtr)];
+} __orc_rt_CWrapperFunctionResultDataUnion;
+
+/**
+ * __orc_rt_CWrapperFunctionResult is a kind of C-SmallVector with an
+ * out-of-band error state.
+ *
+ * If Size == 0 and Data.ValuePtr is non-zero then the value is in the
+ * 'out-of-band error' state, and Data.ValuePtr points at a malloc-allocated,
+ * null-terminated string error message.
+ *
+ * If Size <= sizeof(__orc_rt_CWrapperFunctionResultData) then the value is in
+ * the 'small' state and the content is held in the first Size bytes of
+ * Data.Value.
+ *
+ * If Size > sizeof(OrtRTCWrapperFunctionResultData) then the value is in the
+ * 'large' state and the content is held in the first Size bytes of the
+ * memory pointed to by Data.ValuePtr. This memory must have been allocated by
+ * malloc, and will be freed with free when this value is destroyed.
+ */
+typedef struct {
+ __orc_rt_CWrapperFunctionResultDataUnion Data;
+ size_t Size;
+} __orc_rt_CWrapperFunctionResult;
+
+typedef struct __orc_rt_CSharedOpaqueJITProcessControl
+ *__orc_rt_SharedJITProcessControlRef;
+
+/**
+ * Zero-initialize an __orc_rt_CWrapperFunctionResult.
+ */
+static inline void
+__orc_rt_CWrapperFunctionResultInit(__orc_rt_CWrapperFunctionResult *R) {
+ R->Size = 0;
+ R->Data.ValuePtr = 0;
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult with an uninitialized buffer of
+ * size Size. The buffer is returned via the DataPtr argument.
+ */
+static inline char *
+__orc_rt_CWrapperFunctionResultAllocate(__orc_rt_CWrapperFunctionResult *R,
+ size_t Size) {
+ R->Size = Size;
+ if (Size <= sizeof(R->Data.Value))
+ return R->Data.Value;
+
+ R->Data.ValuePtr = (char *)malloc(Size);
+ return R->Data.ValuePtr;
+}
+
+/**
+ * Create an __orc_rt_WrapperFunctionResult from the given data range.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromRange(const char *Data, size_t Size) {
+ __orc_rt_CWrapperFunctionResult R;
+ R.Size = Size;
+ if (R.Size > sizeof(R.Data.Value)) {
+ char *Tmp = (char *)malloc(Size);
+ memcpy(Tmp, Data, Size);
+ R.Data.ValuePtr = Tmp;
+ } else
+ memcpy(R.Data.Value, Data, Size);
+ return R;
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult by copying the given string,
+ * including the null-terminator.
+ *
+ * This function copies the input string. The client is responsible for freeing
+ * the ErrMsg arg.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromString(const char *Source) {
+ return __orc_rt_CreateCWrapperFunctionResultFromRange(Source,
+ strlen(Source) + 1);
+}
+
+/**
+ * Create an __orc_rt_CWrapperFunctionResult representing an out-of-band
+ * error.
+ *
+ * This function takes ownership of the string argument which must have been
+ * allocated with malloc.
+ */
+static inline __orc_rt_CWrapperFunctionResult
+__orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
+ __orc_rt_CWrapperFunctionResult R;
+ R.Size = 0;
+ char *Tmp = (char *)malloc(strlen(ErrMsg) + 1);
+ strcpy(Tmp, ErrMsg);
+ R.Data.ValuePtr = Tmp;
+ return R;
+}
+
+/**
+ * This should be called to destroy __orc_rt_CWrapperFunctionResult values
+ * regardless of their state.
+ */
+static inline void
+__orc_rt_DisposeCWrapperFunctionResult(__orc_rt_CWrapperFunctionResult *R) {
+ if (R->Size > sizeof(R->Data.Value) ||
+ (R->Size == 0 && R->Data.ValuePtr))
+ free(R->Data.ValuePtr);
+}
+
+/**
+ * Get a pointer to the data contained in the given
+ * __orc_rt_CWrapperFunctionResult.
+ */
+static inline const char *
+__orc_rt_CWrapperFunctionResultData(const __orc_rt_CWrapperFunctionResult *R) {
+ assert((R->Size != 0 || R->Data.ValuePtr == nullptr) &&
+ "Cannot get data for out-of-band error value");
+ return R->Size > sizeof(R->Data.Value) ? R->Data.ValuePtr : R->Data.Value;
+}
+
+/**
+ * Safely get the size of the given __orc_rt_CWrapperFunctionResult.
+ *
+ * Asserts that we're not trying to access the size of an error value.
+ */
+static inline size_t
+__orc_rt_CWrapperFunctionResultSize(const __orc_rt_CWrapperFunctionResult *R) {
+ assert((R->Size != 0 || R->Data.ValuePtr == nullptr) &&
+ "Cannot get size for out-of-band error value");
+ return R->Size;
+}
+
+/**
+ * Returns 1 if this value is equivalent to a value just initialized by
+ * __orc_rt_CWrapperFunctionResultInit, 0 otherwise.
+ */
+static inline size_t
+__orc_rt_CWrapperFunctionResultEmpty(const __orc_rt_CWrapperFunctionResult *R) {
+ return R->Size == 0 && R->Data.ValuePtr == 0;
+}
+
+/**
+ * Returns a pointer to the out-of-band error string for this
+ * __orc_rt_CWrapperFunctionResult, or null if there is no error.
+ *
+ * The __orc_rt_CWrapperFunctionResult retains ownership of the error
+ * string, so it should be copied if the caller wishes to preserve it.
+ */
+static inline const char *__orc_rt_CWrapperFunctionResultGetOutOfBandError(
+ const __orc_rt_CWrapperFunctionResult *R) {
+ return R->Size == 0 ? R->Data.ValuePtr : 0;
+}
+
+ORC_RT_C_EXTERN_C_END
+
+#endif /* ORC_RT_C_API_H */
--- /dev/null
+//===- common.h - Common utilities for the ORC runtime ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_COMMON_H
+#define ORC_RT_COMMON_H
+
+#include "c_api.h"
+#include "compiler.h"
+#include <type_traits>
+
+/// This macro should be used to define tags that will be associated with
+/// handlers in the JIT process, and call can be used to define tags f
+#define ORC_RT_JIT_DISPATCH_TAG(X) \
+extern "C" char X; \
+char X = 0;
+
+/// Opaque struct for external symbols.
+struct __orc_rt_Opaque {};
+
+/// Error reporting function.
+extern "C" void __orc_rt_log_error(const char *ErrMsg);
+
+/// Context object for dispatching calls to the JIT object.
+///
+/// This is declared for use by the runtime, but should be implemented in the
+/// executor or provided by a definition added to the JIT before the runtime
+/// is loaded.
+extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT;
+
+/// For dispatching calls to the JIT object.
+///
+/// This is declared for use by the runtime, but should be implemented in the
+/// executor or provided by a definition added to the JIT before the runtime
+/// is loaded.
+extern "C" __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag,
+ const char *Data, size_t Size) ORC_RT_WEAK_IMPORT;
+
+#endif // ORC_RT_COMMON_H
--- /dev/null
+//===--------- compiler.h - Compiler abstraction support --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+// Most functionality in this file was swiped from llvm/Support/Compiler.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_COMPILER_H
+#define ORC_RT_COMPILER_H
+
+#define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default")))
+#define ORC_RT_HIDDEN __attribute__((visibility("hidden")))
+
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+// Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in
+// C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid.
+#ifndef ORC_RT_HAS_CPP_ATTRIBUTE
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+#define ORC_RT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+#define ORC_RT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+#endif
+
+// Use the 'nodiscard' attribute in C++17 or newer mode.
+#if defined(__cplusplus) && __cplusplus > 201402L && \
+ ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard)
+#define ORC_RT_NODISCARD [[nodiscard]]
+#elif ORC_RT_HAS_CPP_ATTRIBUTE(clang::warn_unused_result)
+#define ORC_RT_NODISCARD [[clang::warn_unused_result]]
+// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also
+// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518).
+// Use the 'nodiscard' attribute in C++14 mode only with GCC.
+// TODO: remove this workaround when PR33518 is resolved.
+#elif defined(__GNUC__) && ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard)
+#define ORC_RT_NODISCARD [[nodiscard]]
+#else
+#define ORC_RT_NODISCARD
+#endif
+
+#if __has_builtin(__builtin_expect)
+#define ORC_RT_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true)
+#define ORC_RT_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false)
+#else
+#define ORC_RT_LIKELY(EXPR) (EXPR)
+#define ORC_RT_UNLIKELY(EXPR) (EXPR)
+#endif
+
+#ifdef __APPLE__
+#define ORC_RT_WEAK_IMPORT __attribute__((weak_import))
+#else
+#define ORC_RT_WEAK_IMPORT __attribute__((weak))
+#endif
+
+#endif // ORC_RT_COMPILER_H
--- /dev/null
+//===- endian.h - Endianness support ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares generic and optimized functions to swap the byte order of
+// an integral type.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ENDIAN_H
+#define ORC_RT_ENDIAN_H
+
+#include <cstddef>
+#include <cstdint>
+#include <type_traits>
+#if defined(_MSC_VER) && !defined(_DEBUG)
+#include <stdlib.h>
+#endif
+
+#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) || \
+ defined(__Fuchsia__) || defined(__EMSCRIPTEN__)
+#include <endian.h>
+#elif defined(_AIX)
+#include <sys/machine.h>
+#elif defined(__sun)
+/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */
+#include <sys/types.h>
+#define BIG_ENDIAN 4321
+#define LITTLE_ENDIAN 1234
+#if defined(_BIG_ENDIAN)
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#define BYTE_ORDER LITTLE_ENDIAN
+#endif
+#elif defined(__MVS__)
+#define BIG_ENDIAN 4321
+#define LITTLE_ENDIAN 1234
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#if !defined(BYTE_ORDER) && !defined(_WIN32)
+#include <machine/endian.h>
+#endif
+#endif
+
+namespace __orc_rt {
+
+/// ByteSwap_16 - This function returns a byte-swapped representation of
+/// the 16-bit argument.
+inline uint16_t ByteSwap_16(uint16_t value) {
+#if defined(_MSC_VER) && !defined(_DEBUG)
+ // The DLL version of the runtime lacks these functions (bug!?), but in a
+ // release build they're replaced with BSWAP instructions anyway.
+ return _byteswap_ushort(value);
+#else
+ uint16_t Hi = value << 8;
+ uint16_t Lo = value >> 8;
+ return Hi | Lo;
+#endif
+}
+
+/// This function returns a byte-swapped representation of the 32-bit argument.
+inline uint32_t ByteSwap_32(uint32_t value) {
+#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC))
+ return __builtin_bswap32(value);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+ return _byteswap_ulong(value);
+#else
+ uint32_t Byte0 = value & 0x000000FF;
+ uint32_t Byte1 = value & 0x0000FF00;
+ uint32_t Byte2 = value & 0x00FF0000;
+ uint32_t Byte3 = value & 0xFF000000;
+ return (Byte0 << 24) | (Byte1 << 8) | (Byte2 >> 8) | (Byte3 >> 24);
+#endif
+}
+
+/// This function returns a byte-swapped representation of the 64-bit argument.
+inline uint64_t ByteSwap_64(uint64_t value) {
+#if defined(__llvm__) || (defined(__GNUC__) && !defined(__ICC))
+ return __builtin_bswap64(value);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+ return _byteswap_uint64(value);
+#else
+ uint64_t Hi = ByteSwap_32(uint32_t(value));
+ uint32_t Lo = ByteSwap_32(uint32_t(value >> 32));
+ return (Hi << 32) | Lo;
+#endif
+}
+
+#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN
+constexpr bool IsBigEndianHost = true;
+#else
+constexpr bool IsBigEndianHost = false;
+#endif
+
+static const bool IsLittleEndianHost = !IsBigEndianHost;
+
+inline unsigned char getSwappedBytes(unsigned char C) { return C; }
+inline signed char getSwappedBytes(signed char C) { return C; }
+inline char getSwappedBytes(char C) { return C; }
+
+inline unsigned short getSwappedBytes(unsigned short C) {
+ return ByteSwap_16(C);
+}
+inline signed short getSwappedBytes(signed short C) { return ByteSwap_16(C); }
+
+inline unsigned int getSwappedBytes(unsigned int C) { return ByteSwap_32(C); }
+inline signed int getSwappedBytes(signed int C) { return ByteSwap_32(C); }
+
+inline unsigned long getSwappedBytes(unsigned long C) {
+ // Handle LLP64 and LP64 platforms.
+ return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C)
+ : ByteSwap_64((uint64_t)C);
+}
+inline signed long getSwappedBytes(signed long C) {
+ // Handle LLP64 and LP64 platforms.
+ return sizeof(long) == sizeof(int) ? ByteSwap_32((uint32_t)C)
+ : ByteSwap_64((uint64_t)C);
+}
+
+inline unsigned long long getSwappedBytes(unsigned long long C) {
+ return ByteSwap_64(C);
+}
+inline signed long long getSwappedBytes(signed long long C) {
+ return ByteSwap_64(C);
+}
+
+template <typename T>
+inline std::enable_if_t<std::is_enum<T>::value, T> getSwappedBytes(T C) {
+ return static_cast<T>(
+ getSwappedBytes(static_cast<std::underlying_type_t<T>>(C)));
+}
+
+template <typename T> inline void swapByteOrder(T &Value) {
+ Value = getSwappedBytes(Value);
+}
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_ENDIAN_H
--- /dev/null
+//===-------- Error.h - Enforced error checking for ORC RT ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ERROR_H
+#define ORC_RT_ERROR_H
+
+#include "compiler.h"
+#include "extensible_rtti.h"
+#include "stl_extras.h"
+
+#include <cassert>
+#include <memory>
+#include <string>
+#include <type_traits>
+
+namespace __orc_rt {
+
+/// Base class for all errors.
+class ErrorInfoBase : public RTTIExtends<ErrorInfoBase, RTTIRoot> {
+public:
+ virtual std::string toString() const = 0;
+};
+
+/// Represents an environmental error.
+class ORC_RT_NODISCARD Error {
+
+ template <typename ErrT, typename... ArgTs>
+ friend Error make_error(ArgTs &&...Args);
+
+ friend Error repackage_error(std::unique_ptr<ErrorInfoBase>);
+
+ template <typename ErrT> friend std::unique_ptr<ErrT> error_cast(Error &);
+
+ template <typename T> friend class Expected;
+
+public:
+ /// Destroy this error. Aborts if error was not checked, or was checked but
+ /// not handled.
+ ~Error() { assertIsChecked(); }
+
+ Error(const Error &) = delete;
+ Error &operator=(const Error &) = delete;
+
+ /// Move-construct an error. The newly constructed error is considered
+ /// unchecked, even if the source error had been checked. The original error
+ /// becomes a checked success value.
+ Error(Error &&Other) {
+ setChecked(true);
+ *this = std::move(Other);
+ }
+
+ /// Move-assign an error value. The current error must represent success, you
+ /// you cannot overwrite an unhandled error. The current error is then
+ /// considered unchecked. The source error becomes a checked success value,
+ /// regardless of its original state.
+ Error &operator=(Error &&Other) {
+ // Don't allow overwriting of unchecked values.
+ assertIsChecked();
+ setPtr(Other.getPtr());
+
+ // This Error is unchecked, even if the source error was checked.
+ setChecked(false);
+
+ // Null out Other's payload and set its checked bit.
+ Other.setPtr(nullptr);
+ Other.setChecked(true);
+
+ return *this;
+ }
+
+ /// Create a success value.
+ static Error success() { return Error(); }
+
+ /// Error values convert to true for failure values, false otherwise.
+ explicit operator bool() {
+ setChecked(getPtr() == nullptr);
+ return getPtr() != nullptr;
+ }
+
+ /// Return true if this Error contains a failure value of the given type.
+ template <typename ErrT> bool isA() const {
+ return getPtr() && getPtr()->isA<ErrT>();
+ }
+
+private:
+ Error() = default;
+
+ Error(std::unique_ptr<ErrorInfoBase> ErrInfo) {
+ auto RawErrPtr = reinterpret_cast<uintptr_t>(ErrInfo.release());
+ assert((RawErrPtr & 0x1) == 0 && "ErrorInfo is insufficiently aligned");
+ ErrPtr = RawErrPtr | 0x1;
+ }
+
+ void assertIsChecked() {
+ if (ORC_RT_UNLIKELY(!isChecked() || getPtr())) {
+ fprintf(stderr, "Error must be checked prior to destruction.\n");
+ abort(); // Some sort of JIT program abort?
+ }
+ }
+
+ template <typename ErrT = ErrorInfoBase> ErrT *getPtr() const {
+ return reinterpret_cast<ErrT *>(ErrPtr & ~uintptr_t(1));
+ }
+
+ void setPtr(ErrorInfoBase *Ptr) {
+ ErrPtr = (reinterpret_cast<uintptr_t>(Ptr) & ~uintptr_t(1)) | (ErrPtr & 1);
+ }
+
+ bool isChecked() const { return ErrPtr & 0x1; }
+
+ void setChecked(bool Checked) {
+ ErrPtr = (reinterpret_cast<uintptr_t>(ErrPtr) & ~uintptr_t(1)) | Checked;
+ }
+
+ template <typename ErrT = ErrorInfoBase> std::unique_ptr<ErrT> takePayload() {
+ static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
+ "ErrT is not an ErrorInfoBase subclass");
+ std::unique_ptr<ErrT> Tmp(getPtr<ErrT>());
+ setPtr(nullptr);
+ setChecked(true);
+ return Tmp;
+ }
+
+ uintptr_t ErrPtr = 0;
+};
+
+/// Construct an error of ErrT with the given arguments.
+template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&...Args) {
+ static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
+ "ErrT is not an ErrorInfoBase subclass");
+ return Error(std::make_unique<ErrT>(std::forward<ArgTs>(Args)...));
+}
+
+/// Construct an error of ErrT using a std::unique_ptr<ErrorInfoBase>. The
+/// primary use-case for this is 're-packaging' errors after inspecting them
+/// using error_cast, hence the name.
+inline Error repackage_error(std::unique_ptr<ErrorInfoBase> EIB) {
+ return Error(std::move(EIB));
+}
+
+/// If the argument is an error of type ErrT then this function unpacks it
+/// and returns a std::unique_ptr<ErrT>. Otherwise returns a nullptr and
+/// leaves the error untouched. Common usage looks like:
+///
+/// \code{.cpp}
+/// if (Error E = foo()) {
+/// if (auto EV1 = error_cast<ErrorType1>(E)) {
+/// // use unwrapped EV1 value.
+/// } else if (EV2 = error_cast<ErrorType2>(E)) {
+/// // use unwrapped EV2 value.
+/// } ...
+/// }
+/// \endcode
+template <typename ErrT> std::unique_ptr<ErrT> error_cast(Error &Err) {
+ static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
+ "ErrT is not an ErrorInfoBase subclass");
+ if (Err.isA<ErrT>())
+ return Err.takePayload<ErrT>();
+ return nullptr;
+}
+
+/// Helper for Errors used as out-parameters.
+/// Sets the 'checked' flag on construction, resets it on destruction.
+class ErrorAsOutParameter {
+public:
+ ErrorAsOutParameter(Error *Err) : Err(Err) {
+ // Raise the checked bit if Err is success.
+ if (Err)
+ (void)!!*Err;
+ }
+
+ ~ErrorAsOutParameter() {
+ // Clear the checked bit.
+ if (Err && !*Err)
+ *Err = Error::success();
+ }
+
+private:
+ Error *Err;
+};
+
+template <typename T> class ORC_RT_NODISCARD Expected {
+
+ template <class OtherT> friend class Expected;
+
+ static constexpr bool IsRef = std::is_reference<T>::value;
+ using wrap = std::reference_wrapper<std::remove_reference_t<T>>;
+ using error_type = std::unique_ptr<ErrorInfoBase>;
+ using storage_type = std::conditional_t<IsRef, wrap, T>;
+ using value_type = T;
+
+ using reference = std::remove_reference_t<T> &;
+ using const_reference = const std::remove_reference_t<T> &;
+ using pointer = std::remove_reference_t<T> *;
+ using const_pointer = const std::remove_reference_t<T> *;
+
+public:
+ /// Create an Expected from a failure value.
+ Expected(Error Err) : HasError(true), Unchecked(true) {
+ assert(Err && "Cannot create Expected<T> from Error success value");
+ new (getErrorStorage()) error_type(Err.takePayload());
+ }
+
+ /// Create an Expected from a T value.
+ template <typename OtherT>
+ Expected(OtherT &&Val,
+ std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
+ : HasError(false), Unchecked(true) {
+ new (getStorage()) storage_type(std::forward<OtherT>(Val));
+ }
+
+ /// Move-construct an Expected<T> from an Expected<OtherT>.
+ Expected(Expected &&Other) { moveConstruct(std::move(Other)); }
+
+ /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
+ /// must be convertible to T.
+ template <class OtherT>
+ Expected(
+ Expected<OtherT> &&Other,
+ std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
+ moveConstruct(std::move(Other));
+ }
+
+ /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
+ /// isn't convertible to T.
+ template <class OtherT>
+ explicit Expected(
+ Expected<OtherT> &&Other,
+ std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) {
+ moveConstruct(std::move(Other));
+ }
+
+ /// Move-assign from another Expected<T>.
+ Expected &operator=(Expected &&Other) {
+ moveAssign(std::move(Other));
+ return *this;
+ }
+
+ /// Destroy an Expected<T>.
+ ~Expected() {
+ assertIsChecked();
+ if (!HasError)
+ getStorage()->~storage_type();
+ else
+ getErrorStorage()->~error_type();
+ }
+
+ /// Returns true if this Expected value is in a success state (holding a T),
+ /// and false if this Expected value is in a failure state.
+ explicit operator bool() {
+ Unchecked = HasError;
+ return !HasError;
+ }
+
+ /// Returns true if this Expected value holds an Error of type error_type.
+ template <typename ErrT> bool isFailureOfType() const {
+ return HasError && (*getErrorStorage())->template isFailureOfType<ErrT>();
+ }
+
+ /// Take ownership of the stored error.
+ ///
+ /// If this Expected value is in a success state (holding a T) then this
+ /// method is a no-op and returns Error::success.
+ ///
+ /// If thsi Expected value is in a failure state (holding an Error) then this
+ /// method returns the contained error and leaves this Expected in an
+ /// 'empty' state from which it may be safely destructed but not otherwise
+ /// accessed.
+ Error takeError() {
+ Unchecked = false;
+ return HasError ? Error(std::move(*getErrorStorage())) : Error::success();
+ }
+
+ /// Returns a pointer to the stored T value.
+ pointer operator->() {
+ assertIsChecked();
+ return toPointer(getStorage());
+ }
+
+ /// Returns a pointer to the stored T value.
+ const_pointer operator->() const {
+ assertIsChecked();
+ return toPointer(getStorage());
+ }
+
+ /// Returns a reference to the stored T value.
+ reference operator*() {
+ assertIsChecked();
+ return *getStorage();
+ }
+
+ /// Returns a reference to the stored T value.
+ const_reference operator*() const {
+ assertIsChecked();
+ return *getStorage();
+ }
+
+private:
+ template <class T1>
+ static bool compareThisIfSameType(const T1 &a, const T1 &b) {
+ return &a == &b;
+ }
+
+ template <class T1, class T2>
+ static bool compareThisIfSameType(const T1 &a, const T2 &b) {
+ return false;
+ }
+
+ template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) {
+ HasError = Other.HasError;
+ Unchecked = true;
+ Other.Unchecked = false;
+
+ if (!HasError)
+ new (getStorage()) storage_type(std::move(*Other.getStorage()));
+ else
+ new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage()));
+ }
+
+ template <class OtherT> void moveAssign(Expected<OtherT> &&Other) {
+ assertIsChecked();
+
+ if (compareThisIfSameType(*this, Other))
+ return;
+
+ this->~Expected();
+ new (this) Expected(std::move(Other));
+ }
+
+ pointer toPointer(pointer Val) { return Val; }
+
+ const_pointer toPointer(const_pointer Val) const { return Val; }
+
+ pointer toPointer(wrap *Val) { return &Val->get(); }
+
+ const_pointer toPointer(const wrap *Val) const { return &Val->get(); }
+
+ storage_type *getStorage() {
+ assert(!HasError && "Cannot get value when an error exists!");
+ return reinterpret_cast<storage_type *>(&TStorage);
+ }
+
+ const storage_type *getStorage() const {
+ assert(!HasError && "Cannot get value when an error exists!");
+ return reinterpret_cast<const storage_type *>(&TStorage);
+ }
+
+ error_type *getErrorStorage() {
+ assert(HasError && "Cannot get error when a value exists!");
+ return reinterpret_cast<error_type *>(&ErrorStorage);
+ }
+
+ const error_type *getErrorStorage() const {
+ assert(HasError && "Cannot get error when a value exists!");
+ return reinterpret_cast<const error_type *>(&ErrorStorage);
+ }
+
+ void assertIsChecked() {
+ if (ORC_RT_UNLIKELY(Unchecked)) {
+ fprintf(stderr,
+ "Expected<T> must be checked before access or destruction.\n");
+ abort();
+ }
+ }
+
+ union {
+ std::aligned_union_t<1, storage_type> TStorage;
+ std::aligned_union_t<1, error_type> ErrorStorage;
+ };
+
+ bool HasError : 1;
+ bool Unchecked : 1;
+};
+
+/// Consume an error without doing anything.
+inline void consumeError(Error Err) {
+ if (Err)
+ (void)error_cast<ErrorInfoBase>(Err);
+}
+
+/// Consumes success values. It is a programmatic error to call this function
+/// on a failure value.
+inline void cantFail(Error Err) {
+ assert(!Err && "cantFail called on failure value");
+ consumeError(std::move(Err));
+}
+
+/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic
+/// error to call this function on a failure value.
+template <typename T> T cantFail(Expected<T> E) {
+ assert(E && "cantFail called on failure value");
+ consumeError(E.takeError());
+ return std::move(*E);
+}
+
+/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic
+/// error to call this function on a failure value.
+template <typename T> T &cantFail(Expected<T &> E) {
+ assert(E && "cantFail called on failure value");
+ consumeError(E.takeError());
+ return *E;
+}
+
+/// Convert the given error to a string. The error value is consumed in the
+/// process.
+inline std::string toString(Error Err) {
+ if (auto EIB = error_cast<ErrorInfoBase>(Err))
+ return EIB->toString();
+ return {};
+}
+
+class StringError : public RTTIExtends<StringError, ErrorInfoBase> {
+public:
+ StringError(std::string ErrMsg) : ErrMsg(std::move(ErrMsg)) {}
+ std::string toString() const override { return ErrMsg; }
+
+private:
+ std::string ErrMsg;
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_ERROR_H
--- /dev/null
+//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Represents an address in the executing program.
+//
+// This file was derived from
+// llvm/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_EXECUTOR_ADDRESS_H
+#define ORC_RT_EXECUTOR_ADDRESS_H
+
+#include "adt.h"
+#include "simple_packed_serialization.h"
+
+#include <cassert>
+#include <type_traits>
+
+namespace __orc_rt {
+
+/// Represents the difference between two addresses in the executor process.
+class ExecutorAddrDiff {
+public:
+ ExecutorAddrDiff() = default;
+ explicit ExecutorAddrDiff(uint64_t Value) : Value(Value) {}
+
+ uint64_t getValue() const { return Value; }
+
+private:
+ int64_t Value = 0;
+};
+
+/// Represents an address in the executor process.
+class ExecutorAddress {
+public:
+ ExecutorAddress() = default;
+ explicit ExecutorAddress(uint64_t Addr) : Addr(Addr) {}
+
+ /// Create an ExecutorAddress from the given pointer.
+ /// Warning: This should only be used when JITing in-process.
+ template <typename T> static ExecutorAddress fromPtr(T *Value) {
+ return ExecutorAddress(
+ static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Value)));
+ }
+
+ /// Cast this ExecutorAddress to a pointer of the given type.
+ /// Warning: This should only be esude when JITing in-process.
+ template <typename T> T toPtr() const {
+ static_assert(std::is_pointer<T>::value, "T must be a pointer type");
+ uintptr_t IntPtr = static_cast<uintptr_t>(Addr);
+ assert(IntPtr == Addr &&
+ "JITTargetAddress value out of range for uintptr_t");
+ return reinterpret_cast<T>(IntPtr);
+ }
+
+ uint64_t getValue() const { return Addr; }
+ void setValue(uint64_t Addr) { this->Addr = Addr; }
+ bool isNull() const { return Addr == 0; }
+
+ explicit operator bool() const { return Addr != 0; }
+
+ friend bool operator==(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr == RHS.Addr;
+ }
+
+ friend bool operator!=(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr != RHS.Addr;
+ }
+
+ friend bool operator<(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr < RHS.Addr;
+ }
+
+ friend bool operator<=(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr <= RHS.Addr;
+ }
+
+ friend bool operator>(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr > RHS.Addr;
+ }
+
+ friend bool operator>=(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return LHS.Addr >= RHS.Addr;
+ }
+
+ ExecutorAddress &operator++() {
+ ++Addr;
+ return *this;
+ }
+ ExecutorAddress &operator--() {
+ --Addr;
+ return *this;
+ }
+ ExecutorAddress operator++(int) { return ExecutorAddress(Addr++); }
+ ExecutorAddress operator--(int) { return ExecutorAddress(Addr++); }
+
+ ExecutorAddress &operator+=(const ExecutorAddrDiff Delta) {
+ Addr += Delta.getValue();
+ return *this;
+ }
+
+ ExecutorAddress &operator-=(const ExecutorAddrDiff Delta) {
+ Addr -= Delta.getValue();
+ return *this;
+ }
+
+private:
+ uint64_t Addr = 0;
+};
+
+/// Subtracting two addresses yields an offset.
+inline ExecutorAddrDiff operator-(const ExecutorAddress &LHS,
+ const ExecutorAddress &RHS) {
+ return ExecutorAddrDiff(LHS.getValue() - RHS.getValue());
+}
+
+/// Adding an offset and an address yields an address.
+inline ExecutorAddress operator+(const ExecutorAddress &LHS,
+ const ExecutorAddrDiff &RHS) {
+ return ExecutorAddress(LHS.getValue() + RHS.getValue());
+}
+
+/// Adding an address and an offset yields an address.
+inline ExecutorAddress operator+(const ExecutorAddrDiff &LHS,
+ const ExecutorAddress &RHS) {
+ return ExecutorAddress(LHS.getValue() + RHS.getValue());
+}
+
+/// Represents an address range in the exceutor process.
+struct ExecutorAddressRange {
+ ExecutorAddressRange() = default;
+ ExecutorAddressRange(ExecutorAddress StartAddress, ExecutorAddress EndAddress)
+ : StartAddress(StartAddress), EndAddress(EndAddress) {}
+
+ bool empty() const { return StartAddress == EndAddress; }
+ ExecutorAddrDiff size() const { return EndAddress - StartAddress; }
+
+ template <typename T> span<T> toSpan() const {
+ assert(size().getValue() % sizeof(T) == 0 &&
+ "AddressRange is not a multiple of sizeof(T)");
+ return span<T>(StartAddress.toPtr<T *>(), size().getValue() / sizeof(T));
+ }
+
+ ExecutorAddress StartAddress;
+ ExecutorAddress EndAddress;
+};
+
+/// SPS serializatior for ExecutorAddress.
+template <> class SPSSerializationTraits<SPSExecutorAddress, ExecutorAddress> {
+public:
+ static size_t size(const ExecutorAddress &EA) {
+ return SPSArgList<uint64_t>::size(EA.getValue());
+ }
+
+ static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddress &EA) {
+ return SPSArgList<uint64_t>::serialize(BOB, EA.getValue());
+ }
+
+ static bool deserialize(SPSInputBuffer &BIB, ExecutorAddress &EA) {
+ uint64_t Tmp;
+ if (!SPSArgList<uint64_t>::deserialize(BIB, Tmp))
+ return false;
+ EA = ExecutorAddress(Tmp);
+ return true;
+ }
+};
+
+using SPSExecutorAddressRange =
+ SPSTuple<SPSExecutorAddress, SPSExecutorAddress>;
+
+/// Serialization traits for address ranges.
+template <>
+class SPSSerializationTraits<SPSExecutorAddressRange, ExecutorAddressRange> {
+public:
+ static size_t size(const ExecutorAddressRange &Value) {
+ return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::size(
+ Value.StartAddress, Value.EndAddress);
+ }
+
+ static bool serialize(SPSOutputBuffer &BOB,
+ const ExecutorAddressRange &Value) {
+ return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::serialize(
+ BOB, Value.StartAddress, Value.EndAddress);
+ }
+
+ static bool deserialize(SPSInputBuffer &BIB, ExecutorAddressRange &Value) {
+ return SPSArgList<SPSExecutorAddress, SPSExecutorAddress>::deserialize(
+ BIB, Value.StartAddress, Value.EndAddress);
+ }
+};
+
+using SPSExecutorAddressRangeSequence = SPSSequence<SPSExecutorAddressRange>;
+
+} // End namespace __orc_rt
+
+#endif // ORC_RT_EXECUTOR_ADDRESS_H
--- /dev/null
+//===- extensible_rtti.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+// Note:
+// This source file was adapted from lib/Support/ExtensibleRTTI.cpp, however
+// the data structures are not shared and the code need not be kept in sync.
+//
+//===----------------------------------------------------------------------===//
+
+#include "extensible_rtti.h"
+
+namespace __orc_rt {
+
+char RTTIRoot::ID = 0;
+void RTTIRoot::anchor() {}
+
+} // end namespace __orc_rt
--- /dev/null
+//===------ extensible_rtti.h - Extensible RTTI for ORC RT ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file
+//
+// Provides an extensible RTTI mechanism, that can be used regardless of whether
+// the runtime is built with -frtti or not. This is predominantly used to
+// support error handling.
+//
+// The RTTIRoot class defines methods for comparing type ids. Implementations
+// of these methods can be injected into new classes using the RTTIExtends
+// class template.
+//
+// E.g.
+//
+// @code{.cpp}
+// class MyBaseClass : public RTTIExtends<MyBaseClass, RTTIRoot> {
+// public:
+// static char ID;
+// virtual void foo() = 0;
+// };
+//
+// class MyDerivedClass1 : public RTTIExtends<MyDerivedClass1, MyBaseClass> {
+// public:
+// static char ID;
+// void foo() override {}
+// };
+//
+// class MyDerivedClass2 : public RTTIExtends<MyDerivedClass2, MyBaseClass> {
+// public:
+// static char ID;
+// void foo() override {}
+// };
+//
+// char MyBaseClass::ID = 0;
+// char MyDerivedClass1::ID = 0;
+// char MyDerivedClass2:: ID = 0;
+//
+// void fn() {
+// std::unique_ptr<MyBaseClass> B = std::make_unique<MyDerivedClass1>();
+// outs() << isa<MyBaseClass>(B) << "\n"; // Outputs "1".
+// outs() << isa<MyDerivedClass1>(B) << "\n"; // Outputs "1".
+// outs() << isa<MyDerivedClass2>(B) << "\n"; // Outputs "0'.
+// }
+//
+// @endcode
+//
+// Note:
+// This header was adapted from llvm/Support/ExtensibleRTTI.h, however the
+// data structures are not shared and the code need not be kept in sync.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_EXTENSIBLE_RTTI_H
+#define ORC_RT_EXTENSIBLE_RTTI_H
+
+namespace __orc_rt {
+
+template <typename ThisT, typename ParentT> class RTTIExtends;
+
+/// Base class for the extensible RTTI hierarchy.
+///
+/// This class defines virtual methods, dynamicClassID and isA, that enable
+/// type comparisons.
+class RTTIRoot {
+public:
+ virtual ~RTTIRoot() = default;
+
+ /// Returns the class ID for this type.
+ static const void *classID() { return &ID; }
+
+ /// Returns the class ID for the dynamic type of this RTTIRoot instance.
+ virtual const void *dynamicClassID() const = 0;
+
+ /// Returns true if this class's ID matches the given class ID.
+ virtual bool isA(const void *const ClassID) const {
+ return ClassID == classID();
+ }
+
+ /// Check whether this instance is a subclass of QueryT.
+ template <typename QueryT> bool isA() const { return isA(QueryT::classID()); }
+
+ static bool classof(const RTTIRoot *R) { return R->isA<RTTIRoot>(); }
+
+private:
+ virtual void anchor();
+
+ static char ID;
+};
+
+/// Inheritance utility for extensible RTTI.
+///
+/// Supports single inheritance only: A class can only have one
+/// ExtensibleRTTI-parent (i.e. a parent for which the isa<> test will work),
+/// though it can have many non-ExtensibleRTTI parents.
+///
+/// RTTIExtents uses CRTP so the first template argument to RTTIExtends is the
+/// newly introduced type, and the *second* argument is the parent class.
+///
+/// class MyType : public RTTIExtends<MyType, RTTIRoot> {
+/// public:
+/// static char ID;
+/// };
+///
+/// class MyDerivedType : public RTTIExtends<MyDerivedType, MyType> {
+/// public:
+/// static char ID;
+/// };
+///
+template <typename ThisT, typename ParentT> class RTTIExtends : public ParentT {
+public:
+ // Inherit constructors and isA methods from ParentT.
+ using ParentT::isA;
+ using ParentT::ParentT;
+
+ static char ID;
+
+ static const void *classID() { return &ThisT::ID; }
+
+ const void *dynamicClassID() const override { return &ThisT::ID; }
+
+ bool isA(const void *const ClassID) const override {
+ return ClassID == classID() || ParentT::isA(ClassID);
+ }
+
+ static bool classof(const RTTIRoot *R) { return R->isA<ThisT>(); }
+};
+
+template <typename ThisT, typename ParentT>
+char RTTIExtends<ThisT, ParentT>::ID = 0;
+
+/// Returns true if the given value is an instance of the template type
+/// parameter.
+template <typename To, typename From> bool isa(const From &Value) {
+ return To::classof(&Value);
+}
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_EXTENSIBLE_RTTI_H
--- /dev/null
+//===-- log_error_to_stderr.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "compiler.h"
+
+#include <stdio.h>
+
+ORC_RT_INTERFACE void __orc_rt_log_error_to_stderr(const char *ErrMsg) {
+ fprintf(stderr, "orc runtime error: %s\n", ErrMsg);
+}
--- /dev/null
+//===- macho_platform.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code required to load the rest of the MachO runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "macho_platform.h"
+#include "common.h"
+#include "error.h"
+#include "wrapper_function_utils.h"
+
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <unordered_map>
+#include <vector>
+
+using namespace __orc_rt;
+using namespace __orc_rt::macho;
+
+// Declare function tags for functions in the JIT process.
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_initializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_deinitializers_tag)
+ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag)
+
+// eh-frame registration functions.
+// We expect these to be available for all processes.
+extern "C" void __register_frame(const void *);
+extern "C" void __deregister_frame(const void *);
+
+// Objective-C types.
+struct objc_class;
+struct objc_image_info;
+struct objc_object;
+struct objc_selector;
+
+using Class = objc_class *;
+using id = objc_object *;
+using SEL = objc_selector *;
+
+// Objective-C registration functions.
+// These are weakly imported. If the Objective-C runtime has not been loaded
+// then code containing Objective-C sections will generate an error.
+extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT;
+extern "C" Class objc_readClassPair(Class,
+ const objc_image_info *) ORC_RT_WEAK_IMPORT;
+extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT;
+
+// Swift types.
+class ProtocolRecord;
+class ProtocolConformanceRecord;
+
+extern "C" void
+swift_registerProtocols(const ProtocolRecord *begin,
+ const ProtocolRecord *end) ORC_RT_WEAK_IMPORT;
+
+extern "C" void swift_registerProtocolConformances(
+ const ProtocolConformanceRecord *begin,
+ const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT;
+
+namespace {
+
+template <typename HandleFDEFn>
+void walkEHFrameSection(span<const char> EHFrameSection,
+ HandleFDEFn HandleFDE) {
+ const char *CurCFIRecord = EHFrameSection.data();
+ uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
+
+ while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
+ const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
+ if (Size == 0xffffffff)
+ Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
+ else
+ Size += 4;
+ uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
+
+ if (Offset != 0)
+ HandleFDE(CurCFIRecord);
+
+ CurCFIRecord += Size;
+ Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
+ }
+}
+
+Error validatePointerSectionExtent(const char *SectionName,
+ const ExecutorAddressRange &SE) {
+ if (SE.size().getValue() % sizeof(uintptr_t)) {
+ std::ostringstream ErrMsg;
+ ErrMsg << std::hex << "Size of " << SectionName << " 0x"
+ << SE.StartAddress.getValue() << " -- 0x" << SE.EndAddress.getValue()
+ << " is not a pointer multiple";
+ return make_error<StringError>(ErrMsg.str());
+ }
+ return Error::success();
+}
+
+Error registerObjCSelectors(
+ const std::vector<ExecutorAddressRange> &ObjCSelRefsSections,
+ const MachOJITDylibInitializers &MOJDIs) {
+
+ if (ORC_RT_UNLIKELY(!sel_registerName))
+ return make_error<StringError>("sel_registerName is not available");
+
+ for (const auto &ObjCSelRefs : ObjCSelRefsSections) {
+
+ if (auto Err = validatePointerSectionExtent("__objc_selrefs", ObjCSelRefs))
+ return Err;
+
+ fprintf(stderr, "Processing selrefs section at 0x%llx\n",
+ ObjCSelRefs.StartAddress.getValue());
+ for (uintptr_t SelEntry : ObjCSelRefs.toSpan<uintptr_t>()) {
+ const char *SelName = reinterpret_cast<const char *>(SelEntry);
+ fprintf(stderr, "Registering selector \"%s\"\n", SelName);
+ auto Sel = sel_registerName(SelName);
+ *reinterpret_cast<SEL *>(SelEntry) = Sel;
+ }
+ }
+
+ return Error::success();
+}
+
+Error registerObjCClasses(
+ const std::vector<ExecutorAddressRange> &ObjCClassListSections,
+ const MachOJITDylibInitializers &MOJDIs) {
+
+ if (ObjCClassListSections.empty())
+ return Error::success();
+
+ if (ORC_RT_UNLIKELY(!objc_msgSend))
+ return make_error<StringError>("objc_msgSend is not available");
+ if (ORC_RT_UNLIKELY(!objc_readClassPair))
+ return make_error<StringError>("objc_readClassPair is not available");
+
+ struct ObjCClassCompiled {
+ void *Metaclass;
+ void *Parent;
+ void *Cache1;
+ void *Cache2;
+ void *Data;
+ };
+
+ auto *ImageInfo =
+ MOJDIs.ObjCImageInfoAddress.toPtr<const objc_image_info *>();
+ auto ClassSelector = sel_registerName("class");
+
+ for (const auto &ObjCClassList : ObjCClassListSections) {
+
+ if (auto Err =
+ validatePointerSectionExtent("__objc_classlist", ObjCClassList))
+ return Err;
+
+ for (uintptr_t ClassPtr : ObjCClassList.toSpan<uintptr_t>()) {
+ auto *Cls = reinterpret_cast<Class>(ClassPtr);
+ auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr);
+ objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector);
+ auto Registered = objc_readClassPair(Cls, ImageInfo);
+
+ // FIXME: Improve diagnostic by reporting the failed class's name.
+ if (Registered != Cls)
+ return make_error<StringError>("Unable to register Objective-C class");
+ }
+ }
+ return Error::success();
+}
+
+Error registerSwift5Protocols(
+ const std::vector<ExecutorAddressRange> &Swift5ProtocolSections,
+ const MachOJITDylibInitializers &MOJDIs) {
+
+ if (ORC_RT_UNLIKELY(!Swift5ProtocolSections.empty() &&
+ !swift_registerProtocols))
+ return make_error<StringError>("swift_registerProtocols is not available");
+
+ for (const auto &Swift5Protocols : Swift5ProtocolSections)
+ swift_registerProtocols(
+ Swift5Protocols.StartAddress.toPtr<const ProtocolRecord *>(),
+ Swift5Protocols.EndAddress.toPtr<const ProtocolRecord *>());
+
+ return Error::success();
+}
+
+Error registerSwift5ProtocolConformances(
+ const std::vector<ExecutorAddressRange> &Swift5ProtocolConformanceSections,
+ const MachOJITDylibInitializers &MOJDIs) {
+
+ if (ORC_RT_UNLIKELY(!Swift5ProtocolConformanceSections.empty() &&
+ !swift_registerProtocolConformances))
+ return make_error<StringError>(
+ "swift_registerProtocolConformances is not available");
+
+ for (const auto &ProtoConfSec : Swift5ProtocolConformanceSections)
+ swift_registerProtocolConformances(
+ ProtoConfSec.StartAddress.toPtr<const ProtocolConformanceRecord *>(),
+ ProtoConfSec.EndAddress.toPtr<const ProtocolConformanceRecord *>());
+
+ return Error::success();
+}
+
+Error runModInits(const std::vector<ExecutorAddressRange> &ModInitsSections,
+ const MachOJITDylibInitializers &MOJDIs) {
+
+ for (const auto &ModInits : ModInitsSections) {
+ if (auto Err = validatePointerSectionExtent("__mod_inits", ModInits))
+ return Err;
+
+ using InitFunc = void (*)();
+ for (auto *Init : ModInits.toSpan<InitFunc>())
+ (*Init)();
+ }
+
+ return Error::success();
+}
+
+struct TLVDescriptor {
+ void *(*Thunk)(TLVDescriptor *) = nullptr;
+ unsigned long Key = 0;
+ unsigned long DataAddress = 0;
+};
+
+class MachOPlatformRuntimeState {
+private:
+ struct AtExitEntry {
+ void (*Func)(void *);
+ void *Arg;
+ };
+
+ using AtExitsVector = std::vector<AtExitEntry>;
+
+ struct PerJITDylibState {
+ void *Header = nullptr;
+ size_t RefCount = 0;
+ bool AllowReinitialization = false;
+ AtExitsVector AtExits;
+ };
+
+public:
+ static void initialize();
+ static MachOPlatformRuntimeState &get();
+ static void destroy();
+
+ MachOPlatformRuntimeState() = default;
+
+ // Delete copy and move constructors.
+ MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
+ MachOPlatformRuntimeState &
+ operator=(const MachOPlatformRuntimeState &) = delete;
+ MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
+ MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
+
+ Error registerObjectSections(MachOPerObjectSectionsToRegister POSR);
+ Error deregisterObjectSections(MachOPerObjectSectionsToRegister POSR);
+
+ const char *dlerror();
+ void *dlopen(string_view Name, int Mode);
+ int dlclose(void *DSOHandle);
+ void *dlsym(void *DSOHandle, string_view Symbol);
+
+ int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
+ void runAtExits(void *DSOHandle);
+
+ /// Returns the base address of the section containing ThreadData.
+ Expected<std::pair<const char *, size_t>>
+ getThreadDataSectionFor(const char *ThreadData);
+
+private:
+ PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
+ PerJITDylibState *getJITDylibStateByName(string_view Path);
+ PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs);
+
+ Error registerThreadDataSection(span<const char> ThreadDataSec);
+
+ Expected<ExecutorAddress> lookupSymbolInJITDylib(void *DSOHandle,
+ string_view Symbol);
+
+ Expected<MachOJITDylibInitializerSequence>
+ getJITDylibInitializersByName(string_view Path);
+ Expected<void *> dlopenInitialize(string_view Path, int Mode);
+ Error initializeJITDylib(MachOJITDylibInitializers &MOJDIs);
+
+ static MachOPlatformRuntimeState *MOPS;
+
+ using InitSectionHandler =
+ Error (*)(const std::vector<ExecutorAddressRange> &Sections,
+ const MachOJITDylibInitializers &MOJDIs);
+ const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
+ {{"__DATA,__objc_selrefs", registerObjCSelectors},
+ {"__DATA,__objc_classlist", registerObjCClasses},
+ {"__TEXT,__swift5_protos", registerSwift5Protocols},
+ {"__TEXT,__swift5_proto", registerSwift5ProtocolConformances},
+ {"__DATA,__mod_init_func", runModInits}};
+
+ // FIXME: Move to thread-state.
+ std::string DLFcnError;
+
+ std::recursive_mutex JDStatesMutex;
+ std::unordered_map<void *, PerJITDylibState> JDStates;
+ std::unordered_map<std::string, void *> JDNameToHeader;
+
+ std::mutex ThreadDataSectionsMutex;
+ std::map<const char *, size_t> ThreadDataSections;
+};
+
+MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
+
+void MachOPlatformRuntimeState::initialize() {
+ assert(!MOPS && "MachOPlatformRuntimeState should be null");
+ MOPS = new MachOPlatformRuntimeState();
+}
+
+MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
+ assert(MOPS && "MachOPlatformRuntimeState not initialized");
+ return *MOPS;
+}
+
+void MachOPlatformRuntimeState::destroy() {
+ assert(MOPS && "MachOPlatformRuntimeState not initialized");
+ delete MOPS;
+}
+
+Error MachOPlatformRuntimeState::registerObjectSections(
+ MachOPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.StartAddress)
+ walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(),
+ __register_frame);
+
+ if (POSR.ThreadDataSection.StartAddress) {
+ if (auto Err = registerThreadDataSection(
+ POSR.ThreadDataSection.toSpan<const char>()))
+ return Err;
+ }
+
+ return Error::success();
+}
+
+Error MachOPlatformRuntimeState::deregisterObjectSections(
+ MachOPerObjectSectionsToRegister POSR) {
+ if (POSR.EHFrameSection.StartAddress)
+ walkEHFrameSection(POSR.EHFrameSection.toSpan<const char>(),
+ __deregister_frame);
+
+ return Error::success();
+}
+
+const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
+
+void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+
+ // Use fast path if all JITDylibs are already loaded and don't require
+ // re-running initializers.
+ if (auto *JDS = getJITDylibStateByName(Path)) {
+ if (!JDS->AllowReinitialization) {
+ ++JDS->RefCount;
+ return JDS->Header;
+ }
+ }
+
+ auto H = dlopenInitialize(Path, Mode);
+ if (!H) {
+ DLFcnError = toString(H.takeError());
+ return nullptr;
+ }
+
+ return *H;
+}
+
+int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
+ runAtExits(DSOHandle);
+ return 0;
+}
+
+void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
+ auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
+ if (!Addr) {
+ DLFcnError = toString(Addr.takeError());
+ return 0;
+ }
+
+ return Addr->toPtr<void *>();
+}
+
+int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
+ void *DSOHandle) {
+ // FIXME: Handle out-of-memory errors, returning -1 if OOM.
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDylib state not initialized");
+ JDS->AtExits.push_back({F, Arg});
+ return 0;
+}
+
+void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
+ // FIXME: Should atexits be allowed to run concurrently with access to
+ // JDState?
+ AtExitsVector V;
+ {
+ std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+ auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
+ assert(JDS && "JITDlybi state not initialized");
+ std::swap(V, JDS->AtExits);
+ }
+
+ while (!V.empty()) {
+ auto &AE = V.back();
+ AE.Func(AE.Arg);
+ V.pop_back();
+ }
+}
+
+Expected<std::pair<const char *, size_t>>
+MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadData);
+ // Check that we have a valid entry covering this address.
+ if (I == ThreadDataSections.begin())
+ return make_error<StringError>("No thread local data section for key");
+ I = std::prev(I);
+ if (ThreadData >= I->first + I->second)
+ return make_error<StringError>("No thread local data section for key");
+ return *I;
+}
+
+MachOPlatformRuntimeState::PerJITDylibState *
+MachOPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
+ auto I = JDStates.find(DSOHandle);
+ if (I == JDStates.end())
+ return nullptr;
+ return &I->second;
+}
+
+MachOPlatformRuntimeState::PerJITDylibState *
+MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
+ // FIXME: Avoid creating string copy here.
+ auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
+ if (I == JDNameToHeader.end())
+ return nullptr;
+ void *H = I->second;
+ auto J = JDStates.find(H);
+ assert(J != JDStates.end() &&
+ "JITDylib has name map entry but no header map entry");
+ return &J->second;
+}
+
+MachOPlatformRuntimeState::PerJITDylibState &
+MachOPlatformRuntimeState::getOrCreateJITDylibState(
+ MachOJITDylibInitializers &MOJDIs) {
+ void *Header = MOJDIs.MachOHeaderAddress.toPtr<void *>();
+
+ auto &JDS = JDStates[Header];
+
+ // If this entry hasn't been created yet.
+ if (!JDS.Header) {
+ assert(!JDNameToHeader.count(MOJDIs.Name) &&
+ "JITDylib has header map entry but no name map entry");
+ JDNameToHeader[MOJDIs.Name] = Header;
+ JDS.Header = Header;
+ }
+
+ return JDS;
+}
+
+Error MachOPlatformRuntimeState::registerThreadDataSection(
+ span<const char> ThreadDataSection) {
+ std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
+ auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
+ if (I != ThreadDataSections.begin()) {
+ auto J = std::prev(I);
+ if (J->first + J->second > ThreadDataSection.data())
+ return make_error<StringError>("Overlapping __thread_data sections");
+ }
+ ThreadDataSections.insert(
+ I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
+ return Error::success();
+}
+
+Expected<ExecutorAddress>
+MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
+ string_view Sym) {
+ Expected<ExecutorAddress> Result((ExecutorAddress()));
+ if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddress>(
+ SPSExecutorAddress,
+ SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, Result,
+ ExecutorAddress::fromPtr(DSOHandle), Sym))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<MachOJITDylibInitializerSequence>
+MachOPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
+ Expected<MachOJITDylibInitializerSequence> Result(
+ (MachOJITDylibInitializerSequence()));
+ std::string PathStr(Path.data(), Path.size());
+ if (auto Err =
+ WrapperFunction<SPSExpected<SPSMachOJITDylibInitializerSequence>(
+ SPSString)>::call(&__orc_rt_macho_get_initializers_tag, Result,
+ Path))
+ return std::move(Err);
+ return Result;
+}
+
+Expected<void *> MachOPlatformRuntimeState::dlopenInitialize(string_view Path,
+ int Mode) {
+ // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
+ // reinitialization. We need to call in to the JIT to see if there's any new
+ // work pending.
+ auto InitSeq = getJITDylibInitializersByName(Path);
+ if (!InitSeq)
+ return InitSeq.takeError();
+
+ // Init sequences should be non-empty.
+ if (InitSeq->empty())
+ return make_error<StringError>(
+ "__orc_rt_macho_get_initializers returned an "
+ "empty init sequence");
+
+ // Otherwise register and run initializers for each JITDylib.
+ for (auto &MOJDIs : *InitSeq)
+ if (auto Err = initializeJITDylib(MOJDIs))
+ return std::move(Err);
+
+ // Return the header for the last item in the list.
+ auto *JDS = getJITDylibStateByHeaderAddr(
+ InitSeq->back().MachOHeaderAddress.toPtr<void *>());
+ assert(JDS && "Missing state entry for JD");
+ return JDS->Header;
+}
+
+Error MachOPlatformRuntimeState::initializeJITDylib(
+ MachOJITDylibInitializers &MOJDIs) {
+
+ auto &JDS = getOrCreateJITDylibState(MOJDIs);
+ ++JDS.RefCount;
+
+ for (auto &KV : InitSections) {
+ const auto &Name = KV.first;
+ const auto &Handler = KV.second;
+ auto I = MOJDIs.InitSections.find(Name);
+ if (I != MOJDIs.InitSections.end()) {
+ if (auto Err = Handler(I->second, MOJDIs))
+ return Err;
+ }
+ }
+
+ return Error::success();
+}
+
+class MachOPlatformRuntimeTLVManager {
+public:
+ void *getInstance(const char *ThreadData);
+
+private:
+ std::unordered_map<const char *, char *> Instances;
+ std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
+};
+
+void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
+ auto I = Instances.find(ThreadData);
+ if (I != Instances.end())
+ return I->second;
+
+ auto TDS =
+ MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
+ if (!TDS) {
+ __orc_rt_log_error(toString(TDS.takeError()).c_str());
+ return nullptr;
+ }
+
+ auto &Allocated = AllocatedSections[TDS->first];
+ if (!Allocated) {
+ Allocated = std::make_unique<char[]>(TDS->second);
+ memcpy(Allocated.get(), TDS->first, TDS->second);
+ }
+
+ size_t ThreadDataDelta = ThreadData - TDS->first;
+ assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
+
+ char *Instance = Allocated.get() + ThreadDataDelta;
+ Instances[ThreadData] = Instance;
+ return Instance;
+}
+
+void destroyMachOTLVMgr(void *MachOTLVMgr) {
+ delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
+}
+
+} // end anonymous namespace
+
+//------------------------------------------------------------------------------
+// JIT entry points
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
+ MachOPlatformRuntimeState::initialize();
+ return WrapperFunctionResult().release();
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
+ MachOPlatformRuntimeState::destroy();
+ return WrapperFunctionResult().release();
+}
+
+/// Wrapper function for registering metadata on a per-object basis.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_register_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle(
+ ArgData, ArgSize,
+ [](MachOPerObjectSectionsToRegister &POSR) {
+ return MachOPlatformRuntimeState::get().registerObjectSections(
+ std::move(POSR));
+ })
+ .release();
+}
+
+/// Wrapper for releasing per-object metadat.
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_deregister_object_sections(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSError(SPSMachOPerObjectSectionsToRegister)>::handle(
+ ArgData, ArgSize,
+ [](MachOPerObjectSectionsToRegister &POSR) {
+ return MachOPlatformRuntimeState::get().deregisterObjectSections(
+ std::move(POSR));
+ })
+ .release();
+}
+
+//------------------------------------------------------------------------------
+// TLV support
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
+ auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
+ pthread_getspecific(D->Key));
+ if (!TLVMgr) {
+ TLVMgr = new MachOPlatformRuntimeTLVManager();
+ if (pthread_setspecific(D->Key, TLVMgr)) {
+ __orc_rt_log_error("Call to pthread_setspecific failed");
+ return nullptr;
+ }
+ }
+
+ return TLVMgr->getInstance(
+ reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
+}
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
+ return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
+ ArgData, ArgSize,
+ []() -> Expected<uint64_t> {
+ pthread_key_t Key;
+ if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
+ __orc_rt_log_error("Call to pthread_key_create failed");
+ return make_error<StringError>(strerror(Err));
+ }
+ return static_cast<uint64_t>(Key);
+ })
+ .release();
+}
+
+//------------------------------------------------------------------------------
+// cxa_atexit support
+//------------------------------------------------------------------------------
+
+int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle) {
+ return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
+}
+
+void __orc_rt_macho_cxa_finalize(void *dso_handle) {
+ MachOPlatformRuntimeState::get().runAtExits(dso_handle);
+}
+
+//------------------------------------------------------------------------------
+// JIT'd dlfcn alternatives.
+//------------------------------------------------------------------------------
+
+const char *__orc_rt_macho_jit_dlerror() {
+ return MachOPlatformRuntimeState::get().dlerror();
+}
+
+void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
+ return MachOPlatformRuntimeState::get().dlopen(path, mode);
+}
+
+int __orc_rt_macho_jit_dlclose(void *dso_handle) {
+ return MachOPlatformRuntimeState::get().dlclose(dso_handle);
+}
+
+void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
+ return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
+}
+
+//------------------------------------------------------------------------------
+// MachO Run Program
+//------------------------------------------------------------------------------
+
+ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
+ const char *EntrySymbolName,
+ int argc, char *argv[]) {
+ using MainTy = int (*)(int, char *[]);
+
+ void *H = __orc_rt_macho_jit_dlopen(JITDylibName,
+ __orc_rt::macho::ORC_RT_RTLD_LAZY);
+ if (!H) {
+ __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
+ return -1;
+ }
+
+ auto *Main =
+ reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
+
+ if (!Main) {
+ __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
+ return -1;
+ }
+
+ int Result = Main(argc, argv);
+
+ if (__orc_rt_macho_jit_dlclose(H) == -1)
+ __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
+
+ return Result;
+}
--- /dev/null
+//===- macho_platform.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ORC Runtime support for Darwin dynamic loading features.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_MACHO_PLATFORM_H
+#define ORC_RT_MACHO_PLATFORM_H
+
+#include "common.h"
+#include "executor_address.h"
+
+// Atexit functions.
+ORC_RT_INTERFACE int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
+ void *dso_handle);
+ORC_RT_INTERFACE void __orc_rt_macho_cxa_finalize(void *dso_handle);
+
+// dlfcn functions.
+ORC_RT_INTERFACE const char *__orc_rt_macho_jit_dlerror();
+ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlopen(const char *path, int mode);
+ORC_RT_INTERFACE int __orc_rt_macho_jit_dlclose(void *dso_handle);
+ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle,
+ const char *symbol);
+
+namespace __orc_rt {
+namespace macho {
+
+struct MachOPerObjectSectionsToRegister {
+ ExecutorAddressRange EHFrameSection;
+ ExecutorAddressRange ThreadDataSection;
+};
+
+struct MachOJITDylibInitializers {
+ using SectionList = std::vector<ExecutorAddressRange>;
+
+ MachOJITDylibInitializers() = default;
+ MachOJITDylibInitializers(std::string Name,
+ ExecutorAddress MachOHeaderAddress)
+ : Name(std::move(Name)),
+ MachOHeaderAddress(std::move(MachOHeaderAddress)) {}
+
+ std::string Name;
+ ExecutorAddress MachOHeaderAddress;
+ ExecutorAddress ObjCImageInfoAddress;
+
+ std::unordered_map<std::string, SectionList> InitSections;
+};
+
+class MachOJITDylibDeinitializers {};
+
+using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>;
+
+using MachOJITDylibDeinitializerSequence =
+ std::vector<MachOJITDylibDeinitializers>;
+
+enum dlopen_mode : int {
+ ORC_RT_RTLD_LAZY = 0x1,
+ ORC_RT_RTLD_NOW = 0x2,
+ ORC_RT_RTLD_LOCAL = 0x4,
+ ORC_RT_RTLD_GLOBAL = 0x8
+};
+
+} // end namespace macho
+
+using SPSMachOPerObjectSectionsToRegister =
+ SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>;
+
+template <>
+class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
+ macho::MachOPerObjectSectionsToRegister> {
+
+public:
+ static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
+ return SPSMachOPerObjectSectionsToRegister::AsArgList::size(
+ MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const macho::MachOPerObjectSectionsToRegister &MOPOSR) {
+ return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize(
+ OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ macho::MachOPerObjectSectionsToRegister &MOPOSR) {
+ return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize(
+ IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
+ }
+};
+
+using SPSNamedExecutorAddressRangeSequenceMap =
+ SPSSequence<SPSTuple<SPSString, SPSExecutorAddressRangeSequence>>;
+
+using SPSMachOJITDylibInitializers =
+ SPSTuple<SPSString, SPSExecutorAddress, SPSExecutorAddress,
+ SPSNamedExecutorAddressRangeSequenceMap>;
+
+using SPSMachOJITDylibInitializerSequence =
+ SPSSequence<SPSMachOJITDylibInitializers>;
+
+/// Serialization traits for MachOJITDylibInitializers.
+template <>
+class SPSSerializationTraits<SPSMachOJITDylibInitializers,
+ macho::MachOJITDylibInitializers> {
+public:
+ static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) {
+ return SPSMachOJITDylibInitializers::AsArgList::size(
+ MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
+ MOJDIs.InitSections);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const macho::MachOJITDylibInitializers &MOJDIs) {
+ return SPSMachOJITDylibInitializers::AsArgList::serialize(
+ OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
+ MOJDIs.InitSections);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ macho::MachOJITDylibInitializers &MOJDIs) {
+ return SPSMachOJITDylibInitializers::AsArgList::deserialize(
+ IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress,
+ MOJDIs.InitSections);
+ }
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_MACHO_PLATFORM_H
--- /dev/null
+//===-- orc_rt_macho_tlv.x86-64.s -------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#define REGISTER_SAVE_SPACE_SIZE 512
+
+ .text
+
+ // returns address of TLV in %rax, all other registers preserved
+ .globl ___orc_rt_macho_tlv_get_addr
+___orc_rt_macho_tlv_get_addr:
+ pushq %rbp
+ movq %rsp, %rbp
+ subq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ movq %rbx, -8(%rbp)
+ movq %rcx, -16(%rbp)
+ movq %rdx, -24(%rbp)
+ movq %rsi, -32(%rbp)
+ movq %rdi, -40(%rbp)
+ movq %r8, -48(%rbp)
+ movq %r9, -56(%rbp)
+ movq %r10, -64(%rbp)
+ movq %r11, -72(%rbp)
+ movq %r12, -80(%rbp)
+ movq %r13, -88(%rbp)
+ movq %r14, -96(%rbp)
+ movq %r15, -104(%rbp)
+ movdqa %xmm0, -128(%rbp)
+ movdqa %xmm1, -144(%rbp)
+ movdqa %xmm2, -160(%rbp)
+ movdqa %xmm3, -176(%rbp)
+ movdqa %xmm4, -192(%rbp)
+ movdqa %xmm5, -208(%rbp)
+ movdqa %xmm6, -224(%rbp)
+ movdqa %xmm7, -240(%rbp)
+ call ___orc_rt_macho_tlv_get_addr_impl
+ movq -8(%rbp), %rbx
+ movq -16(%rbp), %rcx
+ movq -24(%rbp), %rdx
+ movq -32(%rbp), %rsi
+ movq -40(%rbp), %rdi
+ movq -48(%rbp), %r8
+ movq -56(%rbp), %r9
+ movq -64(%rbp), %r10
+ movq -72(%rbp), %r11
+ movq -80(%rbp), %r12
+ movq -88(%rbp), %r13
+ movq -96(%rbp), %r14
+ movq -104(%rbp), %r15
+ movdqa -128(%rbp), %xmm0
+ movdqa -144(%rbp), %xmm1
+ movdqa -160(%rbp), %xmm2
+ movdqa -176(%rbp), %xmm3
+ movdqa -192(%rbp), %xmm4
+ movdqa -208(%rbp), %xmm5
+ movdqa -224(%rbp), %xmm6
+ movdqa -240(%rbp), %xmm7
+ addq $REGISTER_SAVE_SPACE_SIZE, %rsp
+ popq %rbp
+ ret
--- /dev/null
+//===- run_program_wrapper.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#include "adt.h"
+#include "common.h"
+#include "wrapper_function_utils.h"
+
+#include <vector>
+
+using namespace __orc_rt;
+
+extern "C" int64_t __orc_rt_run_program(const char *JITDylibName,
+ const char *EntrySymbolName, int argc,
+ char *argv[]);
+
+ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
+__orc_rt_run_program_wrapper(const char *ArgData, size_t ArgSize) {
+ return WrapperFunction<int64_t(SPSString, SPSString,
+ SPSSequence<SPSString>)>::
+ handle(ArgData, ArgSize,
+ [](const std::string &JITDylibName,
+ const std::string &EntrySymbolName,
+ const std::vector<string_view> &Args) {
+ std::vector<std::unique_ptr<char[]>> ArgVStorage;
+ ArgVStorage.reserve(Args.size());
+ for (auto &Arg : Args) {
+ ArgVStorage.push_back(
+ std::make_unique<char[]>(Arg.size() + 1));
+ memcpy(ArgVStorage.back().get(), Arg.data(), Arg.size());
+ ArgVStorage.back()[Arg.size()] = '\0';
+ }
+ std::vector<char *> ArgV;
+ ArgV.reserve(ArgVStorage.size() + 1);
+ for (auto &ArgStorage : ArgVStorage)
+ ArgV.push_back(ArgStorage.get());
+ ArgV.push_back(nullptr);
+ return __orc_rt_run_program(JITDylibName.c_str(),
+ EntrySymbolName.c_str(),
+ ArgV.size() - 1, ArgV.data());
+ })
+ .release();
+}
--- /dev/null
+//===--- simple_packed_serialization.h - simple serialization ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+// The behavior of the utilities in this header must be synchronized with the
+// behavior of the utilities in
+// llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h.
+//
+// The Simple Packed Serialization (SPS) utilities are used to generate
+// argument and return buffers for wrapper functions using the following
+// serialization scheme:
+//
+// Primitives:
+// bool, char, int8_t, uint8_t -- Two's complement 8-bit (0=false, 1=true)
+// int16_t, uint16_t -- Two's complement 16-bit little endian
+// int32_t, uint32_t -- Two's complement 32-bit little endian
+// int64_t, int64_t -- Two's complement 64-bit little endian
+//
+// Sequence<T>:
+// Serialized as the sequence length (as a uint64_t) followed by the
+// serialization of each of the elements without padding.
+//
+// Tuple<T1, ..., TN>:
+// Serialized as each of the element types from T1 to TN without padding.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_SIMPLE_PACKED_SERIALIZATION_H
+#define ORC_RT_SIMPLE_PACKED_SERIALIZATION_H
+
+#include "adt.h"
+#include "endianness.h"
+#include "error.h"
+#include "stl_extras.h"
+
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace __orc_rt {
+
+/// Output char buffer with overflow check.
+class SPSOutputBuffer {
+public:
+ SPSOutputBuffer(char *Buffer, size_t Remaining)
+ : Buffer(Buffer), Remaining(Remaining) {}
+ bool write(const char *Data, size_t Size) {
+ if (Size > Remaining)
+ return false;
+ memcpy(Buffer, Data, Size);
+ Buffer += Size;
+ Remaining -= Size;
+ return true;
+ }
+
+private:
+ char *Buffer = nullptr;
+ size_t Remaining = 0;
+};
+
+/// Input char buffer with underflow check.
+class SPSInputBuffer {
+public:
+ SPSInputBuffer() = default;
+ SPSInputBuffer(const char *Buffer, size_t Remaining)
+ : Buffer(Buffer), Remaining(Remaining) {}
+ bool read(char *Data, size_t Size) {
+ if (Size > Remaining)
+ return false;
+ memcpy(Data, Buffer, Size);
+ Buffer += Size;
+ Remaining -= Size;
+ return true;
+ }
+
+ const char *data() const { return Buffer; }
+ bool skip(size_t Size) {
+ if (Size > Remaining)
+ return false;
+ Buffer += Size;
+ Remaining -= Size;
+ return true;
+ }
+
+private:
+ const char *Buffer = nullptr;
+ size_t Remaining = 0;
+};
+
+/// Specialize to describe how to serialize/deserialize to/from the given
+/// concrete type.
+template <typename SPSTagT, typename ConcreteT, typename _ = void>
+class SPSSerializationTraits;
+
+/// A utility class for serializing to a blob from a variadic list.
+template <typename... ArgTs> class SPSArgList;
+
+// Empty list specialization for SPSArgList.
+template <> class SPSArgList<> {
+public:
+ static size_t size() { return 0; }
+
+ static bool serialize(SPSOutputBuffer &OB) { return true; }
+ static bool deserialize(SPSInputBuffer &IB) { return true; }
+};
+
+// Non-empty list specialization for SPSArgList.
+template <typename SPSTagT, typename... SPSTagTs>
+class SPSArgList<SPSTagT, SPSTagTs...> {
+public:
+ template <typename ArgT, typename... ArgTs>
+ static size_t size(const ArgT &Arg, const ArgTs &...Args) {
+ return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) +
+ SPSArgList<SPSTagTs...>::size(Args...);
+ }
+
+ template <typename ArgT, typename... ArgTs>
+ static bool serialize(SPSOutputBuffer &OB, const ArgT &Arg,
+ const ArgTs &...Args) {
+ return SPSSerializationTraits<SPSTagT, ArgT>::serialize(OB, Arg) &&
+ SPSArgList<SPSTagTs...>::serialize(OB, Args...);
+ }
+
+ template <typename ArgT, typename... ArgTs>
+ static bool deserialize(SPSInputBuffer &IB, ArgT &Arg, ArgTs &...Args) {
+ return SPSSerializationTraits<SPSTagT, ArgT>::deserialize(IB, Arg) &&
+ SPSArgList<SPSTagTs...>::deserialize(IB, Args...);
+ }
+};
+
+/// SPS serialization for integral types, bool, and char.
+template <typename SPSTagT>
+class SPSSerializationTraits<
+ SPSTagT, SPSTagT,
+ std::enable_if_t<std::is_same<SPSTagT, bool>::value ||
+ std::is_same<SPSTagT, char>::value ||
+ std::is_same<SPSTagT, int8_t>::value ||
+ std::is_same<SPSTagT, int16_t>::value ||
+ std::is_same<SPSTagT, int32_t>::value ||
+ std::is_same<SPSTagT, int64_t>::value ||
+ std::is_same<SPSTagT, uint8_t>::value ||
+ std::is_same<SPSTagT, uint16_t>::value ||
+ std::is_same<SPSTagT, uint32_t>::value ||
+ std::is_same<SPSTagT, uint64_t>::value>> {
+public:
+ static size_t size(const SPSTagT &Value) { return sizeof(SPSTagT); }
+
+ static bool serialize(SPSOutputBuffer &OB, const SPSTagT &Value) {
+ SPSTagT Tmp = Value;
+ if (IsBigEndianHost)
+ swapByteOrder(Tmp);
+ return OB.write(reinterpret_cast<const char *>(&Tmp), sizeof(Tmp));
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, SPSTagT &Value) {
+ SPSTagT Tmp;
+ if (!IB.read(reinterpret_cast<char *>(&Tmp), sizeof(Tmp)))
+ return false;
+ if (IsBigEndianHost)
+ swapByteOrder(Tmp);
+ Value = Tmp;
+ return true;
+ }
+};
+
+/// Any empty placeholder suitable as a substitute for void when deserializing
+class SPSEmpty {};
+
+/// Represents an address in the executor.
+class SPSExecutorAddress {};
+
+/// SPS tag type for tuples.
+///
+/// A blob tuple should be serialized by serializing each of the elements in
+/// sequence.
+template <typename... SPSTagTs> class SPSTuple {
+public:
+ /// Convenience typedef of the corresponding arg list.
+ typedef SPSArgList<SPSTagTs...> AsArgList;
+};
+
+/// SPS tag type for sequences.
+///
+/// SPSSequences should be serialized as a uint64_t sequence length,
+/// followed by the serialization of each of the elements.
+template <typename SPSElementTagT> class SPSSequence;
+
+/// SPS tag type for strings, which are equivalent to sequences of chars.
+using SPSString = SPSSequence<char>;
+
+/// SPS tag type for maps.
+///
+/// SPS maps are just sequences of (Key, Value) tuples.
+template <typename SPSTagT1, typename SPSTagT2>
+using SPSMap = SPSSequence<SPSTuple<SPSTagT1, SPSTagT2>>;
+
+/// Serialization for SPSEmpty type.
+template <> class SPSSerializationTraits<SPSEmpty, SPSEmpty> {
+public:
+ static size_t size(const SPSEmpty &EP) { return 0; }
+ static bool serialize(SPSOutputBuffer &OB, const SPSEmpty &BE) {
+ return true;
+ }
+ static bool deserialize(SPSInputBuffer &IB, SPSEmpty &BE) { return true; }
+};
+
+/// Specialize this to implement 'trivial' sequence serialization for
+/// a concrete sequence type.
+///
+/// Trivial sequence serialization uses the sequence's 'size' member to get the
+/// length of the sequence, and uses a range-based for loop to iterate over the
+/// elements.
+///
+/// Specializing this template class means that you do not need to provide a
+/// specialization of SPSSerializationTraits for your type.
+template <typename SPSElementTagT, typename ConcreteSequenceT>
+class TrivialSPSSequenceSerialization {
+public:
+ static constexpr bool available = false;
+};
+
+/// Specialize this to implement 'trivial' sequence deserialization for
+/// a concrete sequence type.
+///
+/// Trivial deserialization calls a static 'reserve(SequenceT&)' method on your
+/// specialization (you must implement this) to reserve space, and then calls
+/// a static 'append(SequenceT&, ElementT&) method to append each of the
+/// deserialized elements.
+///
+/// Specializing this template class means that you do not need to provide a
+/// specialization of SPSSerializationTraits for your type.
+template <typename SPSElementTagT, typename ConcreteSequenceT>
+class TrivialSPSSequenceDeserialization {
+public:
+ static constexpr bool available = false;
+};
+
+/// Trivial std::string -> SPSSequence<char> serialization.
+template <> class TrivialSPSSequenceSerialization<char, std::string> {
+public:
+ static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<char> -> std::string deserialization.
+template <> class TrivialSPSSequenceDeserialization<char, std::string> {
+public:
+ static constexpr bool available = true;
+
+ using element_type = char;
+
+ static void reserve(std::string &S, uint64_t Size) { S.reserve(Size); }
+ static bool append(std::string &S, char C) {
+ S.push_back(C);
+ return true;
+ }
+};
+
+/// Trivial std::vector<T> -> SPSSequence<SPSElementTagT> serialization.
+template <typename SPSElementTagT, typename T>
+class TrivialSPSSequenceSerialization<SPSElementTagT, std::vector<T>> {
+public:
+ static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<SPSElementTagT> -> std::vector<T> deserialization.
+template <typename SPSElementTagT, typename T>
+class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> {
+public:
+ static constexpr bool available = true;
+
+ using element_type = typename std::vector<T>::value_type;
+
+ static void reserve(std::vector<T> &V, uint64_t Size) { V.reserve(Size); }
+ static bool append(std::vector<T> &V, T E) {
+ V.push_back(std::move(E));
+ return true;
+ }
+};
+
+/// Trivial std::unordered_map<K, V> -> SPSSequence<SPSTuple<SPSKey, SPSValue>>
+/// serialization.
+template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
+class TrivialSPSSequenceSerialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
+ std::unordered_map<K, V>> {
+public:
+ static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<SPSTuple<SPSKey, SPSValue>> -> std::unordered_map<K, V>
+/// deserialization.
+template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
+class TrivialSPSSequenceDeserialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
+ std::unordered_map<K, V>> {
+public:
+ static constexpr bool available = true;
+
+ using element_type = std::pair<K, V>;
+
+ static void reserve(std::unordered_map<K, V> &M, uint64_t Size) {
+ M.reserve(Size);
+ }
+ static bool append(std::unordered_map<K, V> &M, element_type E) {
+ return M.insert(std::move(E)).second;
+ }
+};
+
+/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size
+/// followed by a for-earch loop over the elements of the sequence to serialize
+/// each of them.
+template <typename SPSElementTagT, typename SequenceT>
+class SPSSerializationTraits<SPSSequence<SPSElementTagT>, SequenceT,
+ std::enable_if_t<TrivialSPSSequenceSerialization<
+ SPSElementTagT, SequenceT>::available>> {
+public:
+ static size_t size(const SequenceT &S) {
+ size_t Size = SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size()));
+ for (const auto &E : S)
+ Size += SPSArgList<SPSElementTagT>::size(E);
+ return Size;
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const SequenceT &S) {
+ if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+ return false;
+ for (const auto &E : S)
+ if (!SPSArgList<SPSElementTagT>::serialize(OB, E))
+ return false;
+ return true;
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, SequenceT &S) {
+ using TBSD = TrivialSPSSequenceDeserialization<SPSElementTagT, SequenceT>;
+ uint64_t Size;
+ if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+ return false;
+ TBSD::reserve(S, Size);
+ for (size_t I = 0; I != Size; ++I) {
+ typename TBSD::element_type E;
+ if (!SPSArgList<SPSElementTagT>::deserialize(IB, E))
+ return false;
+ if (!TBSD::append(S, std::move(E)))
+ return false;
+ }
+ return true;
+ }
+};
+
+/// SPSTuple serialization for std::pair.
+template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2>
+class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> {
+public:
+ static size_t size(const std::pair<T1, T2> &P) {
+ return SPSArgList<SPSTagT1>::size(P.first) +
+ SPSArgList<SPSTagT2>::size(P.second);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const std::pair<T1, T2> &P) {
+ return SPSArgList<SPSTagT1>::serialize(OB, P.first) &&
+ SPSArgList<SPSTagT2>::serialize(OB, P.second);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, std::pair<T1, T2> &P) {
+ return SPSArgList<SPSTagT1>::deserialize(IB, P.first) &&
+ SPSArgList<SPSTagT2>::deserialize(IB, P.second);
+ }
+};
+
+/// Serialization for string_views.
+///
+/// Serialization is as for regular strings. Deserialization points directly
+/// into the blob.
+template <> class SPSSerializationTraits<SPSString, __orc_rt::string_view> {
+public:
+ static size_t size(const __orc_rt::string_view &S) {
+ return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) +
+ S.size();
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const __orc_rt::string_view &S) {
+ if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+ return false;
+ return OB.write(S.data(), S.size());
+ }
+
+ static bool deserialize(SPSInputBuffer &IB, __orc_rt::string_view &S) {
+ const char *Data = nullptr;
+ uint64_t Size;
+ if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+ return false;
+ Data = IB.data();
+ if (!IB.skip(Size))
+ return false;
+ S = {Data, Size};
+ return true;
+ }
+};
+
+/// SPS tag type for errors.
+class SPSError;
+
+/// SPS tag type for expecteds, which are either a T or a string representing
+/// an error.
+template <typename SPSTagT> class SPSExpected;
+
+namespace detail {
+
+/// Helper type for serializing Errors.
+///
+/// llvm::Errors are move-only, and not inspectable except by consuming them.
+/// This makes them unsuitable for direct serialization via
+/// SPSSerializationTraits, which needs to inspect values twice (once to
+/// determine the amount of space to reserve, and then again to serialize).
+///
+/// The SPSSerializableError type is a helper that can be
+/// constructed from an llvm::Error, but inspected more than once.
+struct SPSSerializableError {
+ bool HasError = false;
+ std::string ErrMsg;
+};
+
+/// Helper type for serializing Expected<T>s.
+///
+/// See SPSSerializableError for more details.
+///
+// FIXME: Use std::variant for storage once we have c++17.
+template <typename T> struct SPSSerializableExpected {
+ bool HasValue = false;
+ T Value{};
+ std::string ErrMsg;
+};
+
+inline SPSSerializableError toSPSSerializable(Error Err) {
+ if (Err)
+ return {true, toString(std::move(Err))};
+ return {false, {}};
+}
+
+inline Error fromSPSSerializable(SPSSerializableError BSE) {
+ if (BSE.HasError)
+ return make_error<StringError>(BSE.ErrMsg);
+ return Error::success();
+}
+
+template <typename T>
+SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) {
+ if (E)
+ return {true, std::move(*E), {}};
+ else
+ return {false, {}, toString(E.takeError())};
+}
+
+template <typename T>
+Expected<T> fromSPSSerializable(SPSSerializableExpected<T> BSE) {
+ if (BSE.HasValue)
+ return std::move(BSE.Value);
+ else
+ return make_error<StringError>(BSE.ErrMsg);
+}
+
+} // end namespace detail
+
+/// Serialize to a SPSError from a detail::SPSSerializableError.
+template <>
+class SPSSerializationTraits<SPSError, detail::SPSSerializableError> {
+public:
+ static size_t size(const detail::SPSSerializableError &BSE) {
+ size_t Size = SPSArgList<bool>::size(BSE.HasError);
+ if (BSE.HasError)
+ Size += SPSArgList<SPSString>::size(BSE.ErrMsg);
+ return Size;
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const detail::SPSSerializableError &BSE) {
+ if (!SPSArgList<bool>::serialize(OB, BSE.HasError))
+ return false;
+ if (BSE.HasError)
+ if (!SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg))
+ return false;
+ return true;
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ detail::SPSSerializableError &BSE) {
+ if (!SPSArgList<bool>::deserialize(IB, BSE.HasError))
+ return false;
+
+ if (!BSE.HasError)
+ return true;
+
+ return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg);
+ }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a
+/// detail::SPSSerializableExpected<T>.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSExpected<SPSTagT>,
+ detail::SPSSerializableExpected<T>> {
+public:
+ static size_t size(const detail::SPSSerializableExpected<T> &BSE) {
+ size_t Size = SPSArgList<bool>::size(BSE.HasValue);
+ if (BSE.HasValue)
+ Size += SPSArgList<SPSTagT>::size(BSE.Value);
+ else
+ Size += SPSArgList<SPSString>::size(BSE.ErrMsg);
+ return Size;
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const detail::SPSSerializableExpected<T> &BSE) {
+ if (!SPSArgList<bool>::serialize(OB, BSE.HasValue))
+ return false;
+
+ if (BSE.HasValue)
+ return SPSArgList<SPSTagT>::serialize(OB, BSE.Value);
+
+ return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg);
+ }
+
+ static bool deserialize(SPSInputBuffer &IB,
+ detail::SPSSerializableExpected<T> &BSE) {
+ if (!SPSArgList<bool>::deserialize(IB, BSE.HasValue))
+ return false;
+
+ if (BSE.HasValue)
+ return SPSArgList<SPSTagT>::deserialize(IB, BSE.Value);
+
+ return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg);
+ }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a detail::SPSSerializableError.
+template <typename SPSTagT>
+class SPSSerializationTraits<SPSExpected<SPSTagT>,
+ detail::SPSSerializableError> {
+public:
+ static size_t size(const detail::SPSSerializableError &BSE) {
+ assert(BSE.HasError && "Cannot serialize expected from a success value");
+ return SPSArgList<bool>::size(false) +
+ SPSArgList<SPSString>::size(BSE.ErrMsg);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB,
+ const detail::SPSSerializableError &BSE) {
+ assert(BSE.HasError && "Cannot serialize expected from a success value");
+ if (!SPSArgList<bool>::serialize(OB, false))
+ return false;
+ return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg);
+ }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a T.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSExpected<SPSTagT>, T> {
+public:
+ static size_t size(const T &Value) {
+ return SPSArgList<bool>::size(true) + SPSArgList<SPSTagT>::size(Value);
+ }
+
+ static bool serialize(SPSOutputBuffer &OB, const T &Value) {
+ if (!SPSArgList<bool>::serialize(OB, true))
+ return false;
+ return SPSArgList<SPSTagT>::serialize(Value);
+ }
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_SIMPLE_PACKED_SERIALIZATION_H
--- /dev/null
+//===-------- stl_extras.h - Useful STL related functions-------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_STL_EXTRAS_H
+#define ORC_RT_STL_EXTRAS_H
+
+#include <utility>
+#include <tuple>
+
+namespace __orc_rt {
+
+namespace detail {
+
+template <typename F, typename Tuple, std::size_t... I>
+decltype(auto) apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) {
+ return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...);
+}
+
+} // end namespace detail
+
+/// Given an input tuple (a1, a2, ..., an), pass the arguments of the
+/// tuple variadically to f as if by calling f(a1, a2, ..., an) and
+/// return the result.
+///
+/// FIXME: Switch to std::apply once we can use c++17.
+template <typename F, typename Tuple>
+decltype(auto) apply_tuple(F &&f, Tuple &&t) {
+ using Indices = std::make_index_sequence<
+ std::tuple_size<typename std::decay<Tuple>::type>::value>;
+
+ return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t),
+ Indices{});
+}
+
+} // namespace __orc_rt
+
+#endif // ORC_RT_STL_EXTRAS
--- /dev/null
+include(CompilerRTCompile)
+
+include_directories(..)
+
+add_custom_target(OrcRTUnitTests)
+set_target_properties(OrcRTUnitTests PROPERTIES FOLDER "OrcRT unittests")
+
+set(ORC_UNITTEST_CFLAGS
+ ${ORC_CFLAGS}
+ ${COMPILER_RT_UNITTEST_CFLAGS}
+ ${COMPILER_RT_GTEST_CFLAGS}
+ -I${COMPILER_RT_SOURCE_DIR}/lib/orc
+ )
+
+# We add the include directories one at a time in our CFLAGS.
+foreach (DIR ${LLVM_INCLUDE_DIR} ${LLVM_MAIN_INCLUDE_DIR})
+ list(APPEND ORC_UNITTEST_CFLAGS -I${DIR})
+endforeach()
+
+function(add_orc_lib library)
+ add_library(${library} STATIC ${ARGN})
+ set_target_properties(${library} PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ FOLDER "Compiler-RT Runtime tests")
+endfunction()
+
+function(get_orc_lib_for_arch arch lib)
+ if(APPLE)
+ set(tgt_name "RTOrc.test.osx")
+ else()
+ set(tgt_name "RTOrc.test.${arch}")
+ endif()
+ set(${lib} "${tgt_name}" PARENT_SCOPE)
+endfunction()
+
+set(ORC_TEST_ARCH ${ORC_SUPPORTED_ARCH})
+set(ORC_UNITTEST_LINK_FLAGS
+ ${COMPILER_RT_UNITTEST_LINK_FLAGS}
+ ${CMAKE_THREAD_LIBS_INIT}
+ )
+
+if(APPLE)
+ darwin_filter_host_archs(ORC_SUPPORTED_ARCH ORC_TEST_ARCH)
+ list(APPEND ORC_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
+ list(APPEND ORC_UNITTEST_LINK_FLAGS ${DARWIN_osx_LINK_FLAGS})
+else()
+ append_list_if(COMPILER_RT_HAS_LIBM -lm ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBRT -lrt ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBDL -ldl ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ORC_UNITTEST_LINK_FLAGS)
+ append_list_if(COMPILER_RT_HAS_LIBEXECINFO -lexecinfo ORC_UNITTEST_LINK_FLAGS)
+endif()
+
+foreach(lib ${SANITIZER_TEST_CXX_LIBRARIES})
+ list(APPEND ORC_UNITTEST_LINK_FLAGS -l${lib})
+endforeach()
+
+set(ORC_DEPS gtest orc)
+# ORC uses C++ standard library headers.
+if (TARGET cxx-headers OR HAVE_LIBCXX)
+ set(ORC_DEPS cxx-headers)
+endif()
+
+macro(add_orc_unittest testname)
+ cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN})
+ if(UNIX)
+ foreach(arch ${ORC_TEST_ARCH})
+ set(TEST_OBJECTS)
+ get_orc_lib_for_arch(${arch} ORC_RUNTIME_LIBS)
+ generate_compiler_rt_tests(TEST_OBJECTS
+ OrcRTUnitTests "${testname}-${arch}-Test" "${arch}"
+ SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}
+ RUNTIME "${ORC_RUNTIME_LIBS}"
+ COMPILE_DEPS ${TEST_HEADERS}
+ DEPS ${ORC_DEPS}
+ CFLAGS ${ORC_UNITTEST_CFLAGS}
+ LINK_FLAGS ${ORC_UNITTEST_LINK_FLAGS})
+ endforeach()
+ endif()
+endmacro()
+
+set(UNITTEST_SOURCES
+ adt_test.cpp
+ c_api_test.cpp
+ endian_test.cpp
+ error_test.cpp
+ extensible_rtti_test.cpp
+ orc_unit_test_main.cpp
+ stl_extras_test.cpp
+ wrapper_function_utils_test.cpp
+ simple_packed_serialization_test.cpp
+ )
+
+if (COMPILER_RT_CAN_EXECUTE_TESTS)
+
+ if (APPLE)
+ add_orc_lib("RTOrc.test.osx"
+ $<TARGET_OBJECTS:RTOrc.osx>)
+ else()
+ foreach(arch ${ORC_SUPPORTED_ARCH})
+ add_orc_lib("RTOrc.test.${arch}"
+ $<TARGET_OBJECTS:RTOrc.${arch}>)
+ endforeach()
+ endif()
+
+ add_orc_unittest(OrcUnitTest SOURCES ${UNITTEST_SOURCES})
+
+endif()
--- /dev/null
+//===-- adt_test.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "adt.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(ADTTest, SpanDefaultConstruction) {
+ span<int> S;
+ EXPECT_TRUE(S.empty()) << "Default constructed span not empty";
+ EXPECT_EQ(S.size(), 0U) << "Default constructed span size not zero";
+ EXPECT_EQ(S.begin(), S.end()) << "Default constructed span begin != end";
+}
+
+TEST(ADTTest, SpanConstructFromFixedArray) {
+ int A[] = {1, 2, 3, 4, 5};
+ span<int> S(A);
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+ EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
+
+TEST(ADTTest, SpanConstructFromIteratorAndSize) {
+ int A[] = {1, 2, 3, 4, 5};
+ span<int> S(&A[0], 5);
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+ EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
+
+TEST(ADTTest, StringViewDefaultConstruction) {
+ string_view S;
+ EXPECT_TRUE(S.empty()) << "Default constructed span not empty";
+ EXPECT_EQ(S.size(), 0U) << "Default constructed span size not zero";
+ EXPECT_EQ(S.begin(), S.end()) << "Default constructed span begin != end";
+}
+
+TEST(ADTTest, StringViewConstructFromCharPtrAndSize) {
+ const char *Str = "abcdefg";
+ string_view S(Str, 5);
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+ EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &Str[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], Str[I]) << "Unexpected span element value";
+}
+
+TEST(ADTTest, StringViewConstructFromCharPtr) {
+ const char *Str = "abcdefg";
+ size_t StrLen = strlen(Str);
+ string_view S(Str);
+
+ EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+ EXPECT_EQ(S.size(), StrLen) << "Span has unexpected size";
+ EXPECT_EQ(static_cast<size_t>(std::distance(S.begin(), S.end())), StrLen)
+ << "Unexpected iterator range size";
+ EXPECT_EQ(S.data(), &Str[0]) << "Span data has unexpected value";
+ for (unsigned I = 0; I != S.size(); ++I)
+ EXPECT_EQ(S[I], Str[I]) << "Unexpected span element value";
+}
+
+TEST(ADTTest, StringViewEquality) {
+ EXPECT_EQ("", string_view());
+ EXPECT_FALSE(string_view("aab") == string_view("aac"));
+ EXPECT_FALSE(string_view("aab") != string_view("aab"));
+ EXPECT_NE(string_view("aab"), string_view("aac"));
+}
--- /dev/null
+//===-- c_api_test.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "c_api.h"
+#include "gtest/gtest.h"
+
+TEST(CAPITest, CWrapperFunctionResultInit) {
+ __orc_rt_CWrapperFunctionResult R;
+ __orc_rt_CWrapperFunctionResultInit(&R);
+
+ EXPECT_EQ(R.Size, 0U);
+ EXPECT_EQ(R.Data.ValuePtr, nullptr);
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultAllocSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ __orc_rt_CWrapperFunctionResult R;
+ char *DataPtr = __orc_rt_CWrapperFunctionResultAllocate(&R, SmallAllocSize);
+
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ DataPtr[I] = 0x55 + I;
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that __orc_rt_CWrapperFunctionResult(Data|Result|Size) and
+ // __orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultData(&R), R.Data.Value);
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultSize(&R), SmallAllocSize);
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultAllocLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ __orc_rt_CWrapperFunctionResult R;
+ char *DataPtr = __orc_rt_CWrapperFunctionResultAllocate(&R, LargeAllocSize);
+
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ DataPtr[I] = 0x55 + I;
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ EXPECT_EQ(R.Data.ValuePtr, DataPtr);
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that this value isn't treated as an out-of-band error.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that __orc_rt_CWrapperFunctionResult(Data|Result|Size) and
+ // __orc_rt_CWrapperFunctionResultGetOutOfBandError behave as expected.
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultData(&R), R.Data.ValuePtr);
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultSize(&R), LargeAllocSize);
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ EXPECT_EQ(__orc_rt_CWrapperFunctionResultGetOutOfBandError(&R), nullptr);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromRangeSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ char Source[SmallAllocSize];
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ Source[I] = 0x55 + I;
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromRange(Source, SmallAllocSize);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromRangeLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ char Source[LargeAllocSize];
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ Source[I] = 0x55 + I;
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromRange(Source, LargeAllocSize);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ for (size_t I = 0; I != LargeAllocSize; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)(0x55 + I))
+ << "Unexpected value at index " << I;
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromStringSmall) {
+ constexpr size_t SmallAllocSize = sizeof(const char *);
+
+ char Source[SmallAllocSize];
+ for (size_t I = 0; I != SmallAllocSize - 1; ++I)
+ Source[I] = 'a' + I;
+ Source[SmallAllocSize - 1] = '\0';
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromString(Source);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, SmallAllocSize);
+ for (size_t I = 0; I != SmallAllocSize - 1; ++I)
+ EXPECT_EQ(R.Data.Value[I], (char)('a' + I))
+ << "Unexpected value at index " << I;
+ EXPECT_EQ(R.Data.Value[SmallAllocSize - 1], '\0')
+ << "Unexpected value at index " << (SmallAllocSize - 1);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromStringLarge) {
+ constexpr size_t LargeAllocSize = sizeof(const char *) + 1;
+
+ char Source[LargeAllocSize];
+ for (size_t I = 0; I != LargeAllocSize - 1; ++I)
+ Source[I] = 'a' + I;
+ Source[LargeAllocSize - 1] = '\0';
+
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromString(Source);
+
+ // Check that the inline storage in R.Data.Value contains the expected
+ // sequence.
+ EXPECT_EQ(R.Size, LargeAllocSize);
+ for (size_t I = 0; I != LargeAllocSize - 1; ++I)
+ EXPECT_EQ(R.Data.ValuePtr[I], (char)('a' + I))
+ << "Unexpected value at index " << I;
+ EXPECT_EQ(R.Data.ValuePtr[LargeAllocSize - 1], '\0')
+ << "Unexpected value at index " << (LargeAllocSize - 1);
+
+ // Check that we can dispose of the value.
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
+
+TEST(CAPITest, CWrapperFunctionResultFromOutOfBandError) {
+ constexpr const char *ErrMsg = "test error message";
+ __orc_rt_CWrapperFunctionResult R =
+ __orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(ErrMsg);
+
+#ifndef NDEBUG
+ EXPECT_DEATH({ __orc_rt_CWrapperFunctionResultData(&R); },
+ "Cannot get data for out-of-band error value");
+ EXPECT_DEATH({ __orc_rt_CWrapperFunctionResultSize(&R); },
+ "Cannot get size for out-of-band error value");
+#endif
+
+ EXPECT_FALSE(__orc_rt_CWrapperFunctionResultEmpty(&R));
+ const char *OOBErrMsg = __orc_rt_CWrapperFunctionResultGetOutOfBandError(&R);
+ EXPECT_NE(OOBErrMsg, nullptr);
+ EXPECT_NE(OOBErrMsg, ErrMsg);
+ EXPECT_TRUE(strcmp(OOBErrMsg, ErrMsg) == 0);
+
+ __orc_rt_DisposeCWrapperFunctionResult(&R);
+}
--- /dev/null
+//===- endian_test.cpp ------------------------- swap byte order test -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Adapted from the llvm/unittests/Support/SwapByteOrderTest.cpp LLVM unit test.
+//
+//===----------------------------------------------------------------------===//
+
+#include "endianness.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(Endian, ByteSwap_32) {
+ EXPECT_EQ(0x44332211u, ByteSwap_32(0x11223344));
+ EXPECT_EQ(0xDDCCBBAAu, ByteSwap_32(0xAABBCCDD));
+}
+
+TEST(Endian, ByteSwap_64) {
+ EXPECT_EQ(0x8877665544332211ULL, ByteSwap_64(0x1122334455667788LL));
+ EXPECT_EQ(0x1100FFEEDDCCBBAAULL, ByteSwap_64(0xAABBCCDDEEFF0011LL));
+}
+
+// In these first two tests all of the original_uintx values are truncated
+// except for 64. We could avoid this, but there's really no point.
+TEST(Endian, getSwappedBytes_UnsignedRoundTrip) {
+ // The point of the bit twiddling of magic is to test with and without bits
+ // in every byte.
+ uint64_t value = 1;
+ for (std::size_t i = 0; i <= sizeof(value); ++i) {
+ uint8_t original_uint8 = static_cast<uint8_t>(value);
+ EXPECT_EQ(original_uint8, getSwappedBytes(getSwappedBytes(original_uint8)));
+
+ uint16_t original_uint16 = static_cast<uint16_t>(value);
+ EXPECT_EQ(original_uint16,
+ getSwappedBytes(getSwappedBytes(original_uint16)));
+
+ uint32_t original_uint32 = static_cast<uint32_t>(value);
+ EXPECT_EQ(original_uint32,
+ getSwappedBytes(getSwappedBytes(original_uint32)));
+
+ uint64_t original_uint64 = static_cast<uint64_t>(value);
+ EXPECT_EQ(original_uint64,
+ getSwappedBytes(getSwappedBytes(original_uint64)));
+
+ value = (value << 8) | 0x55; // binary 0101 0101.
+ }
+}
+
+TEST(Endian, getSwappedBytes_SignedRoundTrip) {
+ // The point of the bit twiddling of magic is to test with and without bits
+ // in every byte.
+ uint64_t value = 1;
+ for (std::size_t i = 0; i <= sizeof(value); ++i) {
+ int8_t original_int8 = static_cast<int8_t>(value);
+ EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8)));
+
+ int16_t original_int16 = static_cast<int16_t>(value);
+ EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16)));
+
+ int32_t original_int32 = static_cast<int32_t>(value);
+ EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32)));
+
+ int64_t original_int64 = static_cast<int64_t>(value);
+ EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64)));
+
+ // Test other sign.
+ value *= -1;
+
+ original_int8 = static_cast<int8_t>(value);
+ EXPECT_EQ(original_int8, getSwappedBytes(getSwappedBytes(original_int8)));
+
+ original_int16 = static_cast<int16_t>(value);
+ EXPECT_EQ(original_int16, getSwappedBytes(getSwappedBytes(original_int16)));
+
+ original_int32 = static_cast<int32_t>(value);
+ EXPECT_EQ(original_int32, getSwappedBytes(getSwappedBytes(original_int32)));
+
+ original_int64 = static_cast<int64_t>(value);
+ EXPECT_EQ(original_int64, getSwappedBytes(getSwappedBytes(original_int64)));
+
+ // Return to normal sign and twiddle.
+ value *= -1;
+ value = (value << 8) | 0x55; // binary 0101 0101.
+ }
+}
+
+TEST(Endian, getSwappedBytes_uint8_t) {
+ EXPECT_EQ(uint8_t(0x11), getSwappedBytes(uint8_t(0x11)));
+}
+
+TEST(Endian, getSwappedBytes_uint16_t) {
+ EXPECT_EQ(uint16_t(0x1122), getSwappedBytes(uint16_t(0x2211)));
+}
+
+TEST(Endian, getSwappedBytes_uint32_t) {
+ EXPECT_EQ(uint32_t(0x11223344), getSwappedBytes(uint32_t(0x44332211)));
+}
+
+TEST(Endian, getSwappedBytes_uint64_t) {
+ EXPECT_EQ(uint64_t(0x1122334455667788ULL),
+ getSwappedBytes(uint64_t(0x8877665544332211ULL)));
+}
+
+TEST(Endian, getSwappedBytes_int8_t) {
+ EXPECT_EQ(int8_t(0x11), getSwappedBytes(int8_t(0x11)));
+}
+
+TEST(Endian, getSwappedBytes_int16_t) {
+ EXPECT_EQ(int16_t(0x1122), getSwappedBytes(int16_t(0x2211)));
+}
+
+TEST(Endian, getSwappedBytes_int32_t) {
+ EXPECT_EQ(int32_t(0x11223344), getSwappedBytes(int32_t(0x44332211)));
+}
+
+TEST(Endian, getSwappedBytes_int64_t) {
+ EXPECT_EQ(int64_t(0x1122334455667788LL),
+ getSwappedBytes(int64_t(0x8877665544332211LL)));
+}
+
+TEST(Endian, swapByteOrder_uint8_t) {
+ uint8_t value = 0x11;
+ swapByteOrder(value);
+ EXPECT_EQ(uint8_t(0x11), value);
+}
+
+TEST(Endian, swapByteOrder_uint16_t) {
+ uint16_t value = 0x2211;
+ swapByteOrder(value);
+ EXPECT_EQ(uint16_t(0x1122), value);
+}
+
+TEST(Endian, swapByteOrder_uint32_t) {
+ uint32_t value = 0x44332211;
+ swapByteOrder(value);
+ EXPECT_EQ(uint32_t(0x11223344), value);
+}
+
+TEST(Endian, swapByteOrder_uint64_t) {
+ uint64_t value = 0x8877665544332211ULL;
+ swapByteOrder(value);
+ EXPECT_EQ(uint64_t(0x1122334455667788ULL), value);
+}
+
+TEST(Endian, swapByteOrder_int8_t) {
+ int8_t value = 0x11;
+ swapByteOrder(value);
+ EXPECT_EQ(int8_t(0x11), value);
+}
+
+TEST(Endian, swapByteOrder_int16_t) {
+ int16_t value = 0x2211;
+ swapByteOrder(value);
+ EXPECT_EQ(int16_t(0x1122), value);
+}
+
+TEST(Endian, swapByteOrder_int32_t) {
+ int32_t value = 0x44332211;
+ swapByteOrder(value);
+ EXPECT_EQ(int32_t(0x11223344), value);
+}
+
+TEST(Endian, swapByteOrder_int64_t) {
+ int64_t value = 0x8877665544332211LL;
+ swapByteOrder(value);
+ EXPECT_EQ(int64_t(0x1122334455667788LL), value);
+}
--- /dev/null
+//===-- error_test.cpp --sssssssss-----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from
+// llvm/unittests/Support/ExtensibleRTTITest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "error.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+
+class CustomError : public RTTIExtends<CustomError, ErrorInfoBase> {
+public:
+ CustomError(int V1) : V1(V1) {}
+ std::string toString() const override {
+ return "CustomError V1 = " + std::to_string(V1);
+ }
+ int getV1() const { return V1; }
+
+protected:
+ int V1;
+};
+
+class CustomSubError : public RTTIExtends<CustomSubError, CustomError> {
+public:
+ CustomSubError(int V1, std::string V2)
+ : RTTIExtends<CustomSubError, CustomError>(V1), V2(std::move(V2)) {}
+ std::string toString() const override {
+ return "CustomSubError V1 = " + std::to_string(V1) + ", " + V2;
+ }
+ const std::string &getV2() const { return V2; }
+
+protected:
+ std::string V2;
+};
+
+} // end anonymous namespace
+
+// Test that a checked success value doesn't cause any issues.
+TEST(Error, CheckedSuccess) {
+ Error E = Error::success();
+ EXPECT_FALSE(E) << "Unexpected error while testing Error 'Success'";
+}
+
+// Check that a consumed success value doesn't cause any issues.
+TEST(Error, ConsumeSuccess) { consumeError(Error::success()); }
+
+TEST(Error, ConsumeError) {
+ Error E = make_error<CustomError>(42);
+ if (E) {
+ consumeError(std::move(E));
+ } else
+ ADD_FAILURE() << "Error failure value should convert to true";
+}
+
+// Test that unchecked success values cause an abort.
+TEST(Error, UncheckedSuccess) {
+ EXPECT_DEATH({ Error E = Error::success(); },
+ "Error must be checked prior to destruction")
+ << "Unchecked Error Succes value did not cause abort()";
+}
+
+// Test that a checked but unhandled error causes an abort.
+TEST(Error, CheckedButUnhandledError) {
+ auto DropUnhandledError = []() {
+ Error E = make_error<CustomError>(42);
+ (void)!E;
+ };
+ EXPECT_DEATH(DropUnhandledError(),
+ "Error must be checked prior to destruction")
+ << "Unhandled Error failure value did not cause an abort()";
+}
+
+// Test that error_cast works as expected.
+TEST(Error, BasicErrorCast) {
+ {
+ // Check casting base error value to base error type.
+ auto E = make_error<CustomError>(42);
+ if (auto CSE = error_cast<CustomSubError>(E)) {
+ ADD_FAILURE() << "Derived cast incorrectly matched base error";
+ } else if (auto CE = error_cast<CustomError>(E)) {
+ EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+
+ {
+ // Check casting derived error value to base error type.
+ auto E = make_error<CustomSubError>(42, "foo");
+ if (auto CE = error_cast<CustomError>(E)) {
+ EXPECT_EQ(CE->getV1(), 42) << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+
+ {
+ // Check casting derived error value to derived error type.
+ auto E = make_error<CustomSubError>(42, "foo");
+ if (auto CSE = error_cast<CustomSubError>(E)) {
+ EXPECT_EQ(CSE->getV1(), 42) << "Unexpected wrapped value";
+ EXPECT_EQ(CSE->getV2(), "foo") << "Unexpected wrapped value";
+ } else
+ ADD_FAILURE() << "Unexpected error value";
+ }
+}
+
+// ErrorAsOutParameter tester.
+static void errAsOutParamHelper(Error &Err) {
+ ErrorAsOutParameter ErrAsOutParam(&Err);
+ // Verify that checked flag is raised - assignment should not crash.
+ Err = Error::success();
+ // Raise the checked bit manually - caller should still have to test the
+ // error.
+ (void)!!Err;
+}
+
+// Test that ErrorAsOutParameter sets the checked flag on construction.
+TEST(Error, ErrorAsOutParameterChecked) {
+ Error E = Error::success();
+ errAsOutParamHelper(E);
+ (void)!!E;
+}
+
+// Test that ErrorAsOutParameter clears the checked flag on destruction.
+TEST(Error, ErrorAsOutParameterUnchecked) {
+ EXPECT_DEATH(
+ {
+ Error E = Error::success();
+ errAsOutParamHelper(E);
+ },
+ "Error must be checked prior to destruction")
+ << "ErrorAsOutParameter did not clear the checked flag on destruction.";
+}
+
+// Check 'Error::isA<T>' method handling.
+TEST(Error, IsAHandling) {
+ // Check 'isA' handling.
+ Error E = make_error<CustomError>(42);
+ Error F = make_error<CustomSubError>(42, "foo");
+ Error G = Error::success();
+
+ EXPECT_TRUE(E.isA<CustomError>());
+ EXPECT_FALSE(E.isA<CustomSubError>());
+ EXPECT_TRUE(F.isA<CustomError>());
+ EXPECT_TRUE(F.isA<CustomSubError>());
+ EXPECT_FALSE(G.isA<CustomError>());
+
+ consumeError(std::move(E));
+ consumeError(std::move(F));
+ consumeError(std::move(G));
+}
+
+TEST(Error, StringError) {
+ auto E = make_error<StringError>("foo");
+ if (auto SE = error_cast<StringError>(E)) {
+ EXPECT_EQ(SE->toString(), "foo") << "Unexpected StringError value";
+ } else
+ ADD_FAILURE() << "Expected StringError value";
+}
+
+// Test Checked Expected<T> in success mode.
+TEST(Error, CheckedExpectedInSuccessMode) {
+ Expected<int> A = 7;
+ EXPECT_TRUE(!!A) << "Expected with non-error value doesn't convert to 'true'";
+ // Access is safe in second test, since we checked the error in the first.
+ EXPECT_EQ(*A, 7) << "Incorrect Expected non-error value";
+}
+
+// Test Expected with reference type.
+TEST(Error, ExpectedWithReferenceType) {
+ int A = 7;
+ Expected<int &> B = A;
+ // 'Check' B.
+ (void)!!B;
+ int &C = *B;
+ EXPECT_EQ(&A, &C) << "Expected failed to propagate reference";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeDestruction) {
+ EXPECT_DEATH({ Expected<int> A = 7; },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAccess) {
+ EXPECT_DEATH(
+ {
+ Expected<int> A = 7;
+ *A;
+ },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAssignment) {
+ EXPECT_DEATH(
+ {
+ Expected<int> A = 7;
+ A = 7;
+ },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Expected<T> in failure mode.
+TEST(Error, ExpectedInFailureMode) {
+ Expected<int> A = make_error<CustomError>(42);
+ EXPECT_FALSE(!!A) << "Expected with error value doesn't convert to 'false'";
+ Error E = A.takeError();
+ EXPECT_TRUE(E.isA<CustomError>()) << "Incorrect Expected error value";
+ consumeError(std::move(E));
+}
+
+// Check that an Expected instance with an error value doesn't allow access to
+// operator*.
+// Test runs in debug mode only.
+TEST(Error, AccessExpectedInFailureMode) {
+ Expected<int> A = make_error<CustomError>(42);
+ EXPECT_DEATH(*A, "Expected<T> must be checked before access or destruction.")
+ << "Incorrect Expected error value";
+ consumeError(A.takeError());
+}
+
+// Check that an Expected instance with an error triggers an abort if
+// unhandled.
+// Test runs in debug mode only.
+TEST(Error, UnhandledExpectedInFailureMode) {
+ EXPECT_DEATH({ Expected<int> A = make_error<CustomError>(42); },
+ "Expected<T> must be checked before access or destruction.")
+ << "Unchecked Expected<T> failure value did not cause an abort()";
+}
+
+// Test covariance of Expected.
+TEST(Error, ExpectedCovariance) {
+ class B {};
+ class D : public B {};
+
+ Expected<B *> A1(Expected<D *>(nullptr));
+ // Check A1 by converting to bool before assigning to it.
+ (void)!!A1;
+ A1 = Expected<D *>(nullptr);
+ // Check A1 again before destruction.
+ (void)!!A1;
+
+ Expected<std::unique_ptr<B>> A2(Expected<std::unique_ptr<D>>(nullptr));
+ // Check A2 by converting to bool before assigning to it.
+ (void)!!A2;
+ A2 = Expected<std::unique_ptr<D>>(nullptr);
+ // Check A2 again before destruction.
+ (void)!!A2;
+}
+
+// Test that the ExitOnError utility works as expected.
+TEST(Error, CantFailSuccess) {
+ cantFail(Error::success());
+
+ int X = cantFail(Expected<int>(42));
+ EXPECT_EQ(X, 42) << "Expected value modified by cantFail";
+
+ int Dummy = 42;
+ int &Y = cantFail(Expected<int &>(Dummy));
+ EXPECT_EQ(&Dummy, &Y) << "Reference mangled by cantFail";
+}
+
+// Test that cantFail results in a crash if you pass it a failure value.
+TEST(Error, CantFailDeath) {
+ EXPECT_DEATH(cantFail(make_error<StringError>("foo")),
+ "cantFail called on failure value")
+ << "cantFail(Error) did not cause an abort for failure value";
+
+ EXPECT_DEATH(cantFail(Expected<int>(make_error<StringError>("foo"))),
+ "cantFail called on failure value")
+ << "cantFail(Expected<int>) did not cause an abort for failure value";
+}
--- /dev/null
+//===-- extensible_rtti_test.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from
+// llvm/unittests/Support/ExtensibleRTTITest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "extensible_rtti.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+
+class MyBase : public RTTIExtends<MyBase, RTTIRoot> {};
+
+class MyDerivedA : public RTTIExtends<MyDerivedA, MyBase> {};
+
+class MyDerivedB : public RTTIExtends<MyDerivedB, MyBase> {};
+
+} // end anonymous namespace
+
+TEST(ExtensibleRTTITest, BaseCheck) {
+ MyBase MB;
+ MyDerivedA MDA;
+ MyDerivedB MDB;
+
+ // Check MB properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MB));
+ EXPECT_TRUE(isa<MyBase>(MB));
+ EXPECT_FALSE(isa<MyDerivedA>(MB));
+ EXPECT_FALSE(isa<MyDerivedB>(MB));
+
+ // Check MDA properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MDA));
+ EXPECT_TRUE(isa<MyBase>(MDA));
+ EXPECT_TRUE(isa<MyDerivedA>(MDA));
+ EXPECT_FALSE(isa<MyDerivedB>(MDA));
+
+ // Check MDB properties.
+ EXPECT_TRUE(isa<RTTIRoot>(MDB));
+ EXPECT_TRUE(isa<MyBase>(MDB));
+ EXPECT_FALSE(isa<MyDerivedA>(MDB));
+ EXPECT_TRUE(isa<MyDerivedB>(MDB));
+}
--- /dev/null
+//===-- orc_unit_test_main.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+#include "gtest/gtest.h"
+
+int main(int argc, char **argv) {
+ testing::GTEST_FLAG(death_test_style) = "threadsafe";
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
--- /dev/null
+//===-- simple_packed_serialization_test.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "simple_packed_serialization.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(SimplePackedSerializationTest, SPSOutputBuffer) {
+ constexpr unsigned NumBytes = 8;
+ char Buffer[NumBytes];
+ char Zero = 0;
+ SPSOutputBuffer OB(Buffer, NumBytes);
+
+ // Expect that we can write NumBytes of content.
+ for (unsigned I = 0; I != NumBytes; ++I) {
+ char C = I;
+ EXPECT_TRUE(OB.write(&C, 1));
+ }
+
+ // Expect an error when we attempt to write an extra byte.
+ EXPECT_FALSE(OB.write(&Zero, 1));
+
+ // Check that the buffer contains the expected content.
+ for (unsigned I = 0; I != NumBytes; ++I)
+ EXPECT_EQ(Buffer[I], (char)I);
+}
+
+TEST(SimplePackedSerializationTest, SPSInputBuffer) {
+ char Buffer[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07};
+ SPSInputBuffer IB(Buffer, sizeof(Buffer));
+
+ char C;
+ for (unsigned I = 0; I != sizeof(Buffer); ++I) {
+ EXPECT_TRUE(IB.read(&C, 1));
+ EXPECT_EQ(C, (char)I);
+ }
+
+ EXPECT_FALSE(IB.read(&C, 1));
+}
+
+template <typename SPSTagT, typename T>
+static void blobSerializationRoundTrip(const T &Value) {
+ using BST = SPSSerializationTraits<SPSTagT, T>;
+
+ size_t Size = BST::size(Value);
+ auto Buffer = std::make_unique<char[]>(Size);
+ SPSOutputBuffer OB(Buffer.get(), Size);
+
+ EXPECT_TRUE(BST::serialize(OB, Value));
+
+ SPSInputBuffer IB(Buffer.get(), Size);
+
+ T DSValue;
+ EXPECT_TRUE(BST::deserialize(IB, DSValue));
+
+ EXPECT_EQ(Value, DSValue)
+ << "Incorrect value after serialization/deserialization round-trip";
+}
+
+template <typename T> static void testFixedIntegralTypeSerialization() {
+ blobSerializationRoundTrip<T, T>(0);
+ blobSerializationRoundTrip<T, T>(static_cast<T>(1));
+ if (std::is_signed<T>::value) {
+ blobSerializationRoundTrip<T, T>(static_cast<T>(-1));
+ blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::min());
+ }
+ blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::max());
+}
+
+TEST(SimplePackedSerializationTest, BoolSerialization) {
+ blobSerializationRoundTrip<bool, bool>(true);
+ blobSerializationRoundTrip<bool, bool>(false);
+}
+
+TEST(SimplePackedSerializationTest, CharSerialization) {
+ blobSerializationRoundTrip<char, char>((char)0x00);
+ blobSerializationRoundTrip<char, char>((char)0xAA);
+ blobSerializationRoundTrip<char, char>((char)0xFF);
+}
+
+TEST(SimplePackedSerializationTest, Int8Serialization) {
+ testFixedIntegralTypeSerialization<int8_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt8Serialization) {
+ testFixedIntegralTypeSerialization<uint8_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int16Serialization) {
+ testFixedIntegralTypeSerialization<int16_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt16Serialization) {
+ testFixedIntegralTypeSerialization<uint16_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int32Serialization) {
+ testFixedIntegralTypeSerialization<int32_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt32Serialization) {
+ testFixedIntegralTypeSerialization<uint32_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int64Serialization) {
+ testFixedIntegralTypeSerialization<int64_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt64Serialization) {
+ testFixedIntegralTypeSerialization<uint64_t>();
+}
+
+TEST(SimplePackedSerializationTest, SequenceSerialization) {
+ std::vector<int32_t> V({1, 2, -47, 139});
+ blobSerializationRoundTrip<SPSSequence<int32_t>, std::vector<int32_t>>(V);
+}
+
+TEST(SimplePackedSerializationTest, StringViewCharSequenceSerialization) {
+ const char *HW = "Hello, world!";
+ blobSerializationRoundTrip<SPSString, string_view>(string_view(HW));
+}
+
+TEST(SimplePackedSerializationTest, StdPairSerialization) {
+ std::pair<int32_t, std::string> P(42, "foo");
+ blobSerializationRoundTrip<SPSTuple<int32_t, SPSString>,
+ std::pair<int32_t, std::string>>(P);
+}
+
+TEST(SimplePackedSerializationTest, ArgListSerialization) {
+ using BAL = SPSArgList<bool, int32_t, SPSString>;
+
+ bool Arg1 = true;
+ int32_t Arg2 = 42;
+ std::string Arg3 = "foo";
+
+ size_t Size = BAL::size(Arg1, Arg2, Arg3);
+ auto Buffer = std::make_unique<char[]>(Size);
+ SPSOutputBuffer OB(Buffer.get(), Size);
+
+ EXPECT_TRUE(BAL::serialize(OB, Arg1, Arg2, Arg3));
+
+ SPSInputBuffer IB(Buffer.get(), Size);
+
+ bool ArgOut1;
+ int32_t ArgOut2;
+ std::string ArgOut3;
+
+ EXPECT_TRUE(BAL::deserialize(IB, ArgOut1, ArgOut2, ArgOut3));
+
+ EXPECT_EQ(Arg1, ArgOut1);
+ EXPECT_EQ(Arg2, ArgOut2);
+ EXPECT_EQ(Arg3, ArgOut3);
+}
--- /dev/null
+//===-- stl_extras_test.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+// This unit test was adapted from tests in
+// llvm/unittests/ADT/STLExtrasTest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "stl_extras.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+TEST(STLExtrasTest, ApplyTuple) {
+ auto T = std::make_tuple(1, 3, 7);
+ auto U = __orc_rt::apply_tuple(
+ [](int A, int B, int C) { return std::make_tuple(A - B, B - C, C - A); },
+ T);
+
+ EXPECT_EQ(-2, std::get<0>(U));
+ EXPECT_EQ(-4, std::get<1>(U));
+ EXPECT_EQ(6, std::get<2>(U));
+
+ auto V = __orc_rt::apply_tuple(
+ [](int A, int B, int C) {
+ return std::make_tuple(std::make_pair(A, char('A' + A)),
+ std::make_pair(B, char('A' + B)),
+ std::make_pair(C, char('A' + C)));
+ },
+ T);
+
+ EXPECT_EQ(std::make_pair(1, 'B'), std::get<0>(V));
+ EXPECT_EQ(std::make_pair(3, 'D'), std::get<1>(V));
+ EXPECT_EQ(std::make_pair(7, 'H'), std::get<2>(V));
+}
+
+class apply_variadic {
+ static int apply_one(int X) { return X + 1; }
+ static char apply_one(char C) { return C + 1; }
+ static std::string apply_one(std::string S) {
+ return S.substr(0, S.size() - 1);
+ }
+
+public:
+ template <typename... Ts> auto operator()(Ts &&... Items) {
+ return std::make_tuple(apply_one(Items)...);
+ }
+};
+
+TEST(STLExtrasTest, ApplyTupleVariadic) {
+ auto Items = std::make_tuple(1, std::string("Test"), 'X');
+ auto Values = __orc_rt::apply_tuple(apply_variadic(), Items);
+
+ EXPECT_EQ(2, std::get<0>(Values));
+ EXPECT_EQ("Tes", std::get<1>(Values));
+ EXPECT_EQ('Y', std::get<2>(Values));
+}
--- /dev/null
+//===-- wrapper_function_utils_test.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "wrapper_function_utils.h"
+#include "gtest/gtest.h"
+
+using namespace __orc_rt;
+
+namespace {
+constexpr const char *TestString = "test string";
+} // end anonymous namespace
+
+TEST(WrapperFunctionUtilsTest, DefaultWrapperFunctionResult) {
+ WrapperFunctionResult R;
+ EXPECT_TRUE(R.empty());
+ EXPECT_EQ(R.size(), 0U);
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCStruct) {
+ __orc_rt_CWrapperFunctionResult CR =
+ __orc_rt_CreateCWrapperFunctionResultFromString(TestString);
+ WrapperFunctionResult R(CR);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromRange) {
+ auto R = WrapperFunctionResult::copyFrom(TestString, strlen(TestString) + 1);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCString) {
+ auto R = WrapperFunctionResult::copyFrom(TestString);
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromStdString) {
+ auto R = WrapperFunctionResult::copyFrom(std::string(TestString));
+ EXPECT_EQ(R.size(), strlen(TestString) + 1);
+ EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+ EXPECT_FALSE(R.empty());
+ EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromOutOfBandError) {
+ auto R = WrapperFunctionResult::createOutOfBandError(TestString);
+ EXPECT_FALSE(R.empty());
+ EXPECT_TRUE(strcmp(R.getOutOfBandError(), TestString) == 0);
+}
+
+static void voidNoop() {}
+
+static __orc_rt_CWrapperFunctionResult voidNoopWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<void()>::handle(ArgData, ArgSize, voidNoop).release();
+}
+
+static __orc_rt_CWrapperFunctionResult addWrapper(const char *ArgData,
+ size_t ArgSize) {
+ return WrapperFunction<int32_t(int32_t, int32_t)>::handle(
+ ArgData, ArgSize,
+ [](int32_t X, int32_t Y) -> int32_t { return X + Y; })
+ .release();
+}
+
+extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx{};
+
+extern "C" __orc_rt_CWrapperFunctionResult
+__orc_rt_jit_dispatch(__orc_rt_Opaque *Ctx, const void *FnTag,
+ const char *ArgData, size_t ArgSize) {
+ using WrapperFunctionType =
+ __orc_rt_CWrapperFunctionResult (*)(const char *, size_t);
+
+ return reinterpret_cast<WrapperFunctionType>(const_cast<void *>(FnTag))(
+ ArgData, ArgSize);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionCallVoidNoopAndHandle) {
+ EXPECT_FALSE(!!WrapperFunction<void()>::call((void *)&voidNoopWrapper));
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionCallAddWrapperAndHandle) {
+ int32_t Result;
+ EXPECT_FALSE(!!WrapperFunction<int32_t(int32_t, int32_t)>::call(
+ (void *)&addWrapper, Result, 1, 2));
+ EXPECT_EQ(Result, (int32_t)3);
+}
--- /dev/null
+//===-- wrapper_function_utils.h - Utilities for wrapper funcs --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_WRAPPER_FUNCTION_UTILS_H
+#define ORC_RT_WRAPPER_FUNCTION_UTILS_H
+
+#include "c_api.h"
+#include "common.h"
+#include "error.h"
+#include "simple_packed_serialization.h"
+#include <type_traits>
+
+namespace __orc_rt {
+
+/// C++ wrapper function result: Same as CWrapperFunctionResult but
+/// auto-releases memory.
+class WrapperFunctionResult {
+public:
+ /// Create a default WrapperFunctionResult.
+ WrapperFunctionResult() { __orc_rt_CWrapperFunctionResultInit(&R); }
+
+ /// Create a WrapperFunctionResult from a CWrapperFunctionResult. This
+ /// instance takes ownership of the result object and will automatically
+ /// call dispose on the result upon destruction.
+ WrapperFunctionResult(__orc_rt_CWrapperFunctionResult R) : R(R) {}
+
+ WrapperFunctionResult(const WrapperFunctionResult &) = delete;
+ WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete;
+
+ WrapperFunctionResult(WrapperFunctionResult &&Other) {
+ __orc_rt_CWrapperFunctionResultInit(&R);
+ std::swap(R, Other.R);
+ }
+
+ WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) {
+ __orc_rt_CWrapperFunctionResult Tmp;
+ __orc_rt_CWrapperFunctionResultInit(&Tmp);
+ std::swap(Tmp, Other.R);
+ std::swap(R, Tmp);
+ return *this;
+ }
+
+ ~WrapperFunctionResult() { __orc_rt_DisposeCWrapperFunctionResult(&R); }
+
+ /// Relinquish ownership of and return the
+ /// __orc_rt_CWrapperFunctionResult.
+ __orc_rt_CWrapperFunctionResult release() {
+ __orc_rt_CWrapperFunctionResult Tmp;
+ __orc_rt_CWrapperFunctionResultInit(&Tmp);
+ std::swap(R, Tmp);
+ return Tmp;
+ }
+
+ /// Get a pointer to the data contained in this instance.
+ const char *data() const { return __orc_rt_CWrapperFunctionResultData(&R); }
+
+ /// Returns the size of the data contained in this instance.
+ size_t size() const { return __orc_rt_CWrapperFunctionResultSize(&R); }
+
+ /// Returns true if this value is equivalent to a default-constructed
+ /// WrapperFunctionResult.
+ bool empty() const { return __orc_rt_CWrapperFunctionResultEmpty(&R); }
+
+ /// Create a WrapperFunctionResult with the given size and return a pointer
+ /// to the underlying memory.
+ static char *allocate(WrapperFunctionResult &R, size_t Size) {
+ __orc_rt_DisposeCWrapperFunctionResult(&R.R);
+ __orc_rt_CWrapperFunctionResultInit(&R.R);
+ return __orc_rt_CWrapperFunctionResultAllocate(&R.R, Size);
+ }
+
+ /// Copy from the given char range.
+ static WrapperFunctionResult copyFrom(const char *Source, size_t Size) {
+ return __orc_rt_CreateCWrapperFunctionResultFromRange(Source, Size);
+ }
+
+ /// Copy from the given null-terminated string (includes the null-terminator).
+ static WrapperFunctionResult copyFrom(const char *Source) {
+ return __orc_rt_CreateCWrapperFunctionResultFromString(Source);
+ }
+
+ /// Copy from the given std::string (includes the null terminator).
+ static WrapperFunctionResult copyFrom(const std::string &Source) {
+ return copyFrom(Source.c_str());
+ }
+
+ /// Create an out-of-band error by copying the given string.
+ static WrapperFunctionResult createOutOfBandError(const char *Msg) {
+ return __orc_rt_CreateCWrapperFunctionResultFromOutOfBandError(Msg);
+ }
+
+ /// Create an out-of-band error by copying the given string.
+ static WrapperFunctionResult createOutOfBandError(const std::string &Msg) {
+ return createOutOfBandError(Msg.c_str());
+ }
+
+ /// If this value is an out-of-band error then this returns the error message,
+ /// otherwise returns nullptr.
+ const char *getOutOfBandError() const {
+ return __orc_rt_CWrapperFunctionResultGetOutOfBandError(&R);
+ }
+
+private:
+ __orc_rt_CWrapperFunctionResult R;
+};
+
+namespace detail {
+
+template <typename SPSArgListT, typename... ArgTs>
+Expected<WrapperFunctionResult>
+serializeViaSPSToWrapperFunctionResult(const ArgTs &...Args) {
+ WrapperFunctionResult Result;
+ char *DataPtr =
+ WrapperFunctionResult::allocate(Result, SPSArgListT::size(Args...));
+ SPSOutputBuffer OB(DataPtr, Result.size());
+ if (!SPSArgListT::serialize(OB, Args...))
+ return make_error<StringError>(
+ "Error serializing arguments to blob in call");
+ return std::move(Result);
+}
+
+template <typename RetT> class WrapperFunctionHandlerCaller {
+public:
+ template <typename HandlerT, typename ArgTupleT, std::size_t... I>
+ static decltype(auto) call(HandlerT &&H, ArgTupleT &Args,
+ std::index_sequence<I...>) {
+ return std::forward<HandlerT>(H)(std::get<I>(Args)...);
+ }
+};
+
+template <> class WrapperFunctionHandlerCaller<void> {
+public:
+ template <typename HandlerT, typename ArgTupleT, std::size_t... I>
+ static SPSEmpty call(HandlerT &&H, ArgTupleT &Args,
+ std::index_sequence<I...>) {
+ std::forward<HandlerT>(H)(std::get<I>(Args)...);
+ return SPSEmpty();
+ }
+};
+
+template <typename WrapperFunctionImplT,
+ template <typename> class ResultSerializer, typename... SPSTagTs>
+class WrapperFunctionHandlerHelper
+ : public WrapperFunctionHandlerHelper<
+ decltype(&std::remove_reference_t<WrapperFunctionImplT>::operator()),
+ ResultSerializer, SPSTagTs...> {};
+
+template <typename RetT, typename... ArgTs,
+ template <typename> class ResultSerializer, typename... SPSTagTs>
+class WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer,
+ SPSTagTs...> {
+public:
+ using ArgTuple = std::tuple<std::decay_t<ArgTs>...>;
+ using ArgIndices = std::make_index_sequence<std::tuple_size<ArgTuple>::value>;
+
+ template <typename HandlerT>
+ static WrapperFunctionResult apply(HandlerT &&H, const char *ArgData,
+ size_t ArgSize) {
+ ArgTuple Args;
+ if (!deserialize(ArgData, ArgSize, Args, ArgIndices{}))
+ return WrapperFunctionResult::createOutOfBandError(
+ "Could not deserialize arguments for wrapper function call");
+
+ auto HandlerResult = WrapperFunctionHandlerCaller<RetT>::call(
+ std::forward<HandlerT>(H), Args, ArgIndices{});
+
+ if (auto Result = ResultSerializer<decltype(HandlerResult)>::serialize(
+ std::move(HandlerResult)))
+ return std::move(*Result);
+ else
+ return WrapperFunctionResult::createOutOfBandError(
+ toString(Result.takeError()));
+ }
+
+private:
+ template <std::size_t... I>
+ static bool deserialize(const char *ArgData, size_t ArgSize, ArgTuple &Args,
+ std::index_sequence<I...>) {
+ SPSInputBuffer IB(ArgData, ArgSize);
+ return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...);
+ }
+
+};
+
+// Map function references to function types.
+template <typename RetT, typename... ArgTs,
+ template <typename> class ResultSerializer, typename... SPSTagTs>
+class WrapperFunctionHandlerHelper<RetT (&)(ArgTs...), ResultSerializer,
+ SPSTagTs...>
+ : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer,
+ SPSTagTs...> {};
+
+// Map non-const member function types to function types.
+template <typename ClassT, typename RetT, typename... ArgTs,
+ template <typename> class ResultSerializer, typename... SPSTagTs>
+class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...), ResultSerializer,
+ SPSTagTs...>
+ : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer,
+ SPSTagTs...> {};
+
+// Map const member function types to function types.
+template <typename ClassT, typename RetT, typename... ArgTs,
+ template <typename> class ResultSerializer, typename... SPSTagTs>
+class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...) const,
+ ResultSerializer, SPSTagTs...>
+ : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer,
+ SPSTagTs...> {};
+
+template <typename SPSRetTagT, typename RetT> class ResultSerializer {
+public:
+ static Expected<WrapperFunctionResult> serialize(RetT Result) {
+ return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
+ Result);
+ }
+};
+
+template <typename SPSRetTagT> class ResultSerializer<SPSRetTagT, Error> {
+public:
+ static Expected<WrapperFunctionResult> serialize(Error Err) {
+ return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
+ toSPSSerializable(std::move(Err)));
+ }
+};
+
+template <typename SPSRetTagT, typename T>
+class ResultSerializer<SPSRetTagT, Expected<T>> {
+public:
+ static Expected<WrapperFunctionResult> serialize(Expected<T> E) {
+ return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>(
+ toSPSSerializable(std::move(E)));
+ }
+};
+
+template <typename SPSRetTagT, typename RetT> class ResultDeserializer {
+public:
+ static void makeSafe(RetT &Result) {}
+
+ static Error deserialize(RetT &Result, const char *ArgData, size_t ArgSize) {
+ SPSInputBuffer IB(ArgData, ArgSize);
+ if (!SPSArgList<SPSRetTagT>::deserialize(IB, Result))
+ return make_error<StringError>(
+ "Error deserializing return value from blob in call");
+ return Error::success();
+ }
+};
+
+template <> class ResultDeserializer<SPSError, Error> {
+public:
+ static void makeSafe(Error &Err) { cantFail(std::move(Err)); }
+
+ static Error deserialize(Error &Err, const char *ArgData, size_t ArgSize) {
+ SPSInputBuffer IB(ArgData, ArgSize);
+ SPSSerializableError BSE;
+ if (!SPSArgList<SPSError>::deserialize(IB, BSE))
+ return make_error<StringError>(
+ "Error deserializing return value from blob in call");
+ Err = fromSPSSerializable(std::move(BSE));
+ return Error::success();
+ }
+};
+
+template <typename SPSTagT, typename T>
+class ResultDeserializer<SPSExpected<SPSTagT>, Expected<T>> {
+public:
+ static void makeSafe(Expected<T> &E) { cantFail(E.takeError()); }
+
+ static Error deserialize(Expected<T> &E, const char *ArgData,
+ size_t ArgSize) {
+ SPSInputBuffer IB(ArgData, ArgSize);
+ SPSSerializableExpected<T> BSE;
+ if (!SPSArgList<SPSExpected<SPSTagT>>::deserialize(IB, BSE))
+ return make_error<StringError>(
+ "Error deserializing return value from blob in call");
+ E = fromSPSSerializable(std::move(BSE));
+ return Error::success();
+ }
+};
+
+} // end namespace detail
+
+template <typename SPSSignature> class WrapperFunction;
+
+template <typename SPSRetTagT, typename... SPSTagTs>
+class WrapperFunction<SPSRetTagT(SPSTagTs...)> {
+private:
+ template <typename RetT>
+ using ResultSerializer = detail::ResultSerializer<SPSRetTagT, RetT>;
+
+public:
+ template <typename RetT, typename... ArgTs>
+ static Error call(const void *FnTag, RetT &Result, const ArgTs &...Args) {
+
+ // RetT might be an Error or Expected value. Set the checked flag now:
+ // we don't want the user to have to check the unused result if this
+ // operation fails.
+ detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result);
+
+ if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx))
+ return make_error<StringError>("__orc_rt_jit_dispatch_ctx not set");
+ if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch))
+ return make_error<StringError>("__orc_rt_jit_dispatch not set");
+
+ auto ArgBuffer =
+ detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>(
+ Args...);
+ if (!ArgBuffer)
+ return ArgBuffer.takeError();
+
+ WrapperFunctionResult ResultBuffer =
+ __orc_rt_jit_dispatch(&__orc_rt_jit_dispatch_ctx, FnTag,
+ ArgBuffer->data(), ArgBuffer->size());
+ if (auto ErrMsg = ResultBuffer.getOutOfBandError())
+ return make_error<StringError>(ErrMsg);
+
+ return detail::ResultDeserializer<SPSRetTagT, RetT>::deserialize(
+ Result, ResultBuffer.data(), ResultBuffer.size());
+ }
+
+ template <typename HandlerT>
+ static WrapperFunctionResult handle(const char *ArgData, size_t ArgSize,
+ HandlerT &&Handler) {
+ using WFHH =
+ detail::WrapperFunctionHandlerHelper<HandlerT, ResultSerializer,
+ SPSTagTs...>;
+ return WFHH::apply(std::forward<HandlerT>(Handler), ArgData, ArgSize);
+ }
+
+private:
+ template <typename T> static const T &makeSerializable(const T &Value) {
+ return Value;
+ }
+
+ static detail::SPSSerializableError makeSerializable(Error Err) {
+ return detail::toSPSSerializable(std::move(Err));
+ }
+
+ template <typename T>
+ static detail::SPSSerializableExpected<T> makeSerializable(Expected<T> E) {
+ return detail::toSPSSerializable(std::move(E));
+ }
+};
+
+template <typename... SPSTagTs>
+class WrapperFunction<void(SPSTagTs...)>
+ : private WrapperFunction<SPSEmpty(SPSTagTs...)> {
+public:
+ template <typename... ArgTs>
+ static Error call(const void *FnTag, const ArgTs &...Args) {
+ SPSEmpty BE;
+ return WrapperFunction<SPSEmpty(SPSTagTs...)>::call(FnTag, BE, Args...);
+ }
+
+ using WrapperFunction<SPSEmpty(SPSTagTs...)>::handle;
+};
+
+} // end namespace __orc_rt
+
+#endif // ORC_RT_WRAPPER_FUNCTION_UTILS_H
InstrProfiling.c
InstrProfilingInternal.c
InstrProfilingValue.c
- InstrProfilingBiasVar.c
InstrProfilingBuffer.c
InstrProfilingFile.c
InstrProfilingMerge.c
InstrProfilingMergeFile.c
InstrProfilingNameVar.c
+ InstrProfilingVersionVar.c
InstrProfilingWriter.c
InstrProfilingPlatformDarwin.c
InstrProfilingPlatformFuchsia.c
set(EXTRA_FLAGS
${EXTRA_FLAGS}
-DCOMPILER_RT_HAS_ATOMICS=1)
-endif()
+endif()
if(COMPILER_RT_TARGET_HAS_FCNTL_LCK)
set(EXTRA_FLAGS
-DCOMPILER_RT_HAS_UNAME=1)
endif()
+# We don't use the C++ Standard Library here, so avoid including it by mistake.
+append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ EXTRA_FLAGS)
+# XRay uses C++ standard library headers.
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+
# This appears to be a C-only warning banning the use of locals in aggregate
# initializers. All other compilers accept this, though.
# nonstandard extension used : 'identifier' : cannot be initialized using address of automatic variable
#include <errno.h>
#include <fcntl.h>
+#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#endif
-#if defined(__FreeBSD__) && defined(__i386__)
-#define I386_FREEBSD 1
-#else
-#define I386_FREEBSD 0
-#endif
-
-#if !defined(_MSC_VER) && !I386_FREEBSD
-#include <stdint.h>
-#endif
-
-#if defined(_MSC_VER)
-typedef unsigned char uint8_t;
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-#elif I386_FREEBSD
-/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to
- * FreeBSD 10, r232261) when compiled in 32-bit mode.
- */
-typedef unsigned char uint8_t;
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-#endif
-
#include "InstrProfiling.h"
#include "InstrProfilingUtil.h"
*/
struct fn_list writeout_fn_list;
-/*
- * A list of flush functions that our __gcov_flush() function should call, shared between all dynamic objects.
- */
-struct fn_list flush_fn_list;
-
/*
* A list of reset functions, shared between all dynamic objects.
*/
mmap_handle = NULL;
#else
- if (msync(write_buffer, file_size, MS_SYNC) == -1) {
+ if (munmap(write_buffer, file_size) == -1) {
int errnum = errno;
- fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename,
+ fprintf(stderr, "profiling: %s: cannot munmap: %s\n", filename,
strerror(errnum));
}
-
- /* We explicitly ignore errors from unmapping because at this point the data
- * is written and we don't care.
- */
- (void)munmap(write_buffer, file_size);
#endif
write_buffer = NULL;
#endif
}
-/* Given an array of pointers to counters (counters), increment the n-th one,
- * where we're also given a pointer to n (predecessor).
- */
-COMPILER_RT_VISIBILITY
-void llvm_gcda_increment_indirect_counter(uint32_t *predecessor,
- uint64_t **counters) {
- uint64_t *counter;
- uint32_t pred;
-
- pred = *predecessor;
- if (pred == 0xffffffff)
- return;
- counter = counters[pred];
-
- /* Don't crash if the pred# is out of sync. This can happen due to threads,
- or because of a TODO in GCOVProfiling.cpp buildEdgeLookupTable(). */
- if (counter)
- ++*counter;
-#ifdef DEBUG_GCDAPROFILING
- else
- fprintf(stderr,
- "llvmgcda: increment_indirect_counter counters=%08llx, pred=%u\n",
- *counter, *predecessor);
-#endif
-}
-
COMPILER_RT_VISIBILITY
void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
uint32_t cfg_checksum) {
if (val != (uint32_t)-1) {
/* There are counters present in the file. Merge them. */
- if (val != (gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY
- : GCOV_TAG_PROGRAM_SUMMARY)) {
+ uint32_t gcov_tag =
+ gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY;
+ if (val != gcov_tag) {
fprintf(stderr,
"profiling: %s: cannot merge previous run count: "
"corrupt object tag (0x%08x)\n",
fn_list_remove(&writeout_fn_list);
}
-COMPILER_RT_VISIBILITY
-void llvm_register_flush_function(fn_ptr fn) {
- fn_list_insert(&flush_fn_list, fn);
-}
-
-void __gcov_flush() {
- struct fn_node* curr = flush_fn_list.head;
-
- while (curr) {
- curr->fn();
- curr = curr->next;
- }
-}
-
-COMPILER_RT_VISIBILITY
-void llvm_delete_flush_function_list(void) {
- fn_list_remove(&flush_fn_list);
-}
-
COMPILER_RT_VISIBILITY
void llvm_register_reset_function(fn_ptr fn) {
fn_list_insert(&reset_fn_list, fn);
#endif
COMPILER_RT_VISIBILITY
-void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn, fn_ptr rfn) {
+void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) {
static int atexit_ran = 0;
if (wfn)
llvm_register_writeout_function(wfn);
- if (ffn)
- llvm_register_flush_function(ffn);
-
if (rfn)
llvm_register_reset_function(rfn);
/* Make sure we write out the data and delete the data structures. */
atexit(llvm_delete_reset_function_list);
- atexit(llvm_delete_flush_function_list);
#ifdef _WIN32
atexit(llvm_writeout_and_clear);
#endif
}
}
+void __gcov_dump(void) {
+ for (struct fn_node *f = writeout_fn_list.head; f; f = f->next)
+ f->fn();
+}
+
+void __gcov_reset(void) {
+ for (struct fn_node *f = reset_fn_list.head; f; f = f->next)
+ f->fn();
+}
+
#endif
|*
\*===----------------------------------------------------------------------===*/
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#define INSTR_PROF_VALUE_PROF_DATA
#include "profile/InstrProfData.inc"
-
-COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION;
-
COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) {
return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64)
: (INSTR_PROF_RAW_MAGIC_32);
*/
void __llvm_profile_enable_continuous_mode(void);
+/*!
+ * \brief Set the page size.
+ *
+ * This is a pre-requisite for enabling continuous mode. The buffer size
+ * calculation code inside of libprofile cannot simply call getpagesize(), as
+ * it is not allowed to depend on libc.
+ */
+void __llvm_profile_set_page_size(unsigned PageSize);
+
/*!
* \brief Get number of bytes necessary to pad the argument to eight
* byte boundary.
/*!
* \brief Merge profile data from buffer.
*
- * Read profile data form buffer \p Profile and merge with
- * in-process profile counters. The client is expected to
- * have checked or already knows the profile data in the
- * buffer matches the in-process counter structure before
- * calling it.
+ * Read profile data form buffer \p Profile and merge with in-process profile
+ * counters. The client is expected to have checked or already knows the profile
+ * data in the buffer matches the in-process counter structure before calling
+ * it. Returns 0 (success) if the profile data is valid. Upon reading
+ * invalid/corrupted profile data, returns 1 (failure).
*/
-void __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
+int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size);
/*! \brief Check if profile in buffer matches the current binary.
*
*/
extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */
-/*!
- * This variable is a weak symbol defined in InstrProfilingBiasVar.c. It
- * allows compiler instrumentation to provide overriding definition with
- * value from compiler command line. This variable has hidden visibility.
- */
-COMPILER_RT_VISIBILITY extern intptr_t __llvm_profile_counter_bias;
-
#endif /* PROFILE_INSTRPROFILING_H_ */
|*
\*===----------------------------------------------------------------------===*/
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
#include "InstrProfilingPort.h"
* layering is violated. */
static int ContinuouslySyncProfile = 0;
+/* The system page size. Only valid when non-zero. If 0, the page size is
+ * unavailable. */
+static unsigned PageSize = 0;
+
COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
- return ContinuouslySyncProfile;
+ return ContinuouslySyncProfile && PageSize;
}
COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
ContinuouslySyncProfile = 1;
}
+COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
+ PageSize = PS;
+}
+
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_size_for_buffer(void) {
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
/// Calculate the number of padding bytes needed to add to \p Offset in order
/// for (\p Offset + Padding) to be page-aligned.
-static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset,
- unsigned PageSize) {
+static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
uint64_t OffsetModPage = Offset % PageSize;
if (OffsetModPage > 0)
return PageSize - OffsetModPage;
return 0;
}
+static int needsCounterPadding(void) {
+#if defined(__APPLE__)
+ return __llvm_profile_is_continuous_mode_enabled();
+#else
+ return 0;
+#endif
+}
+
COMPILER_RT_VISIBILITY
void __llvm_profile_get_padding_sizes_for_counters(
uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize,
uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
uint64_t *PaddingBytesAfterNames) {
- if (!__llvm_profile_is_continuous_mode_enabled() ||
- lprofRuntimeCounterRelocation()) {
+ if (!needsCounterPadding()) {
*PaddingBytesBeforeCounters = 0;
*PaddingBytesAfterCounters = 0;
*PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
// In continuous mode, the file offsets for headers and for the start of
// counter sections need to be page-aligned.
- unsigned PageSize = getpagesize();
uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data);
uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t);
*PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign(
- sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize);
+ sizeof(__llvm_profile_header) + DataSizeInBytes);
*PaddingBytesAfterCounters =
- calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize);
- *PaddingBytesAfterNames =
- calculateBytesNeededToPageAlign(NamesSize, PageSize);
+ calculateBytesNeededToPageAlign(CountersSizeInBytes);
+ *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
}
COMPILER_RT_VISIBILITY
DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
&PaddingBytesAfterCounters, &PaddingBytesAfterNames);
- return sizeof(__llvm_profile_header) +
+ return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
(DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters +
(CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters +
NamesSize + PaddingBytesAfterNames;
#if !defined(__Fuchsia__)
+#include <assert.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
unsigned OwnsFilenamePat;
const char *ProfilePathPrefix;
char PidChars[MAX_PID_SIZE];
+ char *TmpDir;
char Hostname[COMPILER_RT_MAX_HOSTLEN];
unsigned NumPids;
unsigned NumHosts;
ProfileNameSpecifier PNS;
} lprofFilename;
-static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0},
- 0, 0, 0, PNS_unknown};
+static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL,
+ {0}, 0, 0, 0, PNS_unknown};
static int ProfileMergeRequested = 0;
static int isProfileMergeRequested() { return ProfileMergeRequested; }
return -1;
/* Now start merging */
- __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
+ if (__llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize)) {
+ PROF_ERR("%s\n", "Invalid profile data to merge");
+ (void)munmap(ProfileBuffer, ProfileFileSize);
+ return -1;
+ }
- // Truncate the file in case merging of value profile did not happend to
+ // Truncate the file in case merging of value profile did not happen to
// prevent from leaving garbage data at the end of the profile file.
- COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer());
+ (void)COMPILER_RT_FTRUNCATE(ProfileFile,
+ __llvm_profile_get_size_for_buffer());
(void)munmap(ProfileBuffer, ProfileFileSize);
*MergeDone = 1;
fclose(File);
}
-#ifndef _MSC_VER
+// TODO: Move these functions into InstrProfilingPlatform* files.
+#if defined(__APPLE__)
static void assertIsZero(int *i) {
if (*i)
PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
}
-#endif
-#if !defined(__Fuchsia__) && !defined(_WIN32)
/* Write a partial profile to \p Filename, which is required to be backed by
* the open file object \p File. */
static int writeProfileWithFileObject(const char *Filename, FILE *File) {
if (!*ProfileRequiresUnlock) {
PROF_WARN("%s", "Expected to require profile unlock\n");
}
+
lprofUnlockFileHandle(File);
*ProfileRequiresUnlock = 0;
}
-#endif // !defined(__Fuchsia__) && !defined(_WIN32)
-
-static int writeMMappedFile(FILE *OutputFile, char **Profile) {
- if (!OutputFile)
- return -1;
-
- /* Write the data into a file. */
- setupIOBuffer();
- ProfDataWriter fileWriter;
- initFileWriter(&fileWriter, OutputFile);
- if (lprofWriteData(&fileWriter, NULL, 0)) {
- PROF_ERR("Failed to write profile: %s\n", strerror(errno));
- return -1;
- }
- fflush(OutputFile);
-
- /* Get the file size. */
- uint64_t FileSize = ftell(OutputFile);
-
- /* Map the profile. */
- *Profile = (char *)mmap(
- NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
- if (*Profile == MAP_FAILED) {
- PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
- return -1;
- }
-
- return 0;
-}
-
-static void relocateCounters(void) {
- if (!__llvm_profile_is_continuous_mode_enabled() ||
- !lprofRuntimeCounterRelocation())
- return;
-
- /* Get the sizes of various profile data sections. Taken from
- * __llvm_profile_get_size_for_buffer(). */
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
- (DataSize * sizeof(__llvm_profile_data));
-
- int Length = getCurFilenameLength();
- char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
- const char *Filename = getCurFilename(FilenameBuf, 0);
- if (!Filename)
- return;
-
- FILE *File = NULL;
- char *Profile = NULL;
-
- if (!doMerging()) {
- File = fopen(Filename, "w+b");
- if (!File)
- return;
-
- if (writeMMappedFile(File, &Profile) == -1) {
- fclose(File);
- return;
- }
- } else {
- File = lprofOpenFileEx(Filename);
- if (!File)
- return;
-
- uint64_t ProfileFileSize = 0;
- if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
- lprofUnlockFileHandle(File);
- fclose(File);
- return;
- }
-
- if (!ProfileFileSize) {
- if (writeMMappedFile(File, &Profile) == -1) {
- fclose(File);
- return;
- }
- } else {
- /* The merged profile has a non-zero length. Check that it is compatible
- * with the data in this process. */
- if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
- fclose(File);
- return;
- }
- }
-
- lprofUnlockFileHandle(File);
- }
-
- /* Update the profile fields based on the current mapping. */
- __llvm_profile_counter_bias = (intptr_t)Profile -
- (uintptr_t)__llvm_profile_begin_counters() + CountersOffset;
-}
static void initializeProfileForContinuousMode(void) {
if (!__llvm_profile_is_continuous_mode_enabled())
return;
-#if defined(__Fuchsia__) || defined(_WIN32)
- PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows.");
-#else // defined(__Fuchsia__) || defined(_WIN32)
/* Get the sizes of various profile data sections. Taken from
* __llvm_profile_get_size_for_buffer(). */
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
}
}
- int Fileno = fileno(File);
-
- /* Determine how much padding is needed before/after the counters and after
- * the names. */
- uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
- PaddingBytesAfterNames;
- __llvm_profile_get_padding_sizes_for_counters(
- DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
- &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
-
- uint64_t PageAlignedCountersLength =
- (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters;
- uint64_t FileOffsetToCounters =
- CurrentFileOffset + sizeof(__llvm_profile_header) +
- (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters;
-
- uint64_t *CounterMmap = (uint64_t *)mmap(
- (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters);
- if (CounterMmap != CountersBegin) {
- PROF_ERR(
- "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
- " - CountersBegin: %p\n"
- " - PageAlignedCountersLength: %" PRIu64 "\n"
- " - Fileno: %d\n"
- " - FileOffsetToCounters: %" PRIu64 "\n",
- strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
- FileOffsetToCounters);
+ /* mmap() the profile counters so long as there is at least one counter.
+ * If there aren't any counters, mmap() would fail with EINVAL. */
+ if (CountersSize > 0) {
+ int Fileno = fileno(File);
+
+ /* Determine how much padding is needed before/after the counters and after
+ * the names. */
+ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
+ PaddingBytesAfterNames;
+ __llvm_profile_get_padding_sizes_for_counters(
+ DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
+ &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
+
+ uint64_t PageAlignedCountersLength =
+ (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters;
+ uint64_t FileOffsetToCounters =
+ CurrentFileOffset + sizeof(__llvm_profile_header) +
+ (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters;
+
+ uint64_t *CounterMmap = (uint64_t *)mmap(
+ (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters);
+ if (CounterMmap != CountersBegin) {
+ PROF_ERR(
+ "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
+ " - CountersBegin: %p\n"
+ " - PageAlignedCountersLength: %" PRIu64 "\n"
+ " - Fileno: %d\n"
+ " - FileOffsetToCounters: %" PRIu64 "\n",
+ strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
+ FileOffsetToCounters);
+ }
}
- unlockProfile(&ProfileRequiresUnlock, File);
-#endif // defined(__Fuchsia__) || defined(_WIN32)
+ if (ProfileRequiresUnlock)
+ unlockProfile(&ProfileRequiresUnlock, File);
}
+#elif defined(__ELF__) || defined(_WIN32)
+
+#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \
+ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default)
+intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0;
+
+/* This variable is a weak external reference which could be used to detect
+ * whether or not the compiler defined this symbol. */
+#if defined(_MSC_VER)
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+#if defined(_M_IX86) || defined(__i386__)
+#define WIN_SYM_PREFIX "_"
+#else
+#define WIN_SYM_PREFIX
+#endif
+#pragma comment( \
+ linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \
+ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))
+#else
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
+ __attribute__((weak, alias(INSTR_PROF_QUOTE(
+ INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR))));
+#endif
+
+static int writeMMappedFile(FILE *OutputFile, char **Profile) {
+ if (!OutputFile)
+ return -1;
+
+ /* Write the data into a file. */
+ setupIOBuffer();
+ ProfDataWriter fileWriter;
+ initFileWriter(&fileWriter, OutputFile);
+ if (lprofWriteData(&fileWriter, NULL, 0)) {
+ PROF_ERR("Failed to write profile: %s\n", strerror(errno));
+ return -1;
+ }
+ fflush(OutputFile);
+
+ /* Get the file size. */
+ uint64_t FileSize = ftell(OutputFile);
+
+ /* Map the profile. */
+ *Profile = (char *)mmap(
+ NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
+ if (*Profile == MAP_FAILED) {
+ PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return 0;
+}
+
+static void initializeProfileForContinuousMode(void) {
+ if (!__llvm_profile_is_continuous_mode_enabled())
+ return;
+
+ /* This symbol is defined by the compiler when runtime counter relocation is
+ * used and runtime provides a weak alias so we can check if it's defined. */
+ void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+ void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR;
+ if (BiasAddr == BiasDefaultAddr) {
+ PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined");
+ return;
+ }
+
+ /* Get the sizes of various profile data sections. Taken from
+ * __llvm_profile_get_size_for_buffer(). */
+ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
+ const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const uint64_t *CountersBegin = __llvm_profile_begin_counters();
+ const uint64_t *CountersEnd = __llvm_profile_end_counters();
+ uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
+ const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
+ __llvm_write_binary_ids(NULL) +
+ (DataSize * sizeof(__llvm_profile_data));
+
+ int Length = getCurFilenameLength();
+ char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
+ const char *Filename = getCurFilename(FilenameBuf, 0);
+ if (!Filename)
+ return;
+
+ FILE *File = NULL;
+ char *Profile = NULL;
+
+ if (!doMerging()) {
+ File = fopen(Filename, "w+b");
+ if (!File)
+ return;
+
+ if (writeMMappedFile(File, &Profile) == -1) {
+ fclose(File);
+ return;
+ }
+ } else {
+ File = lprofOpenFileEx(Filename);
+ if (!File)
+ return;
+
+ uint64_t ProfileFileSize = 0;
+ if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
+ lprofUnlockFileHandle(File);
+ fclose(File);
+ return;
+ }
+
+ if (!ProfileFileSize) {
+ if (writeMMappedFile(File, &Profile) == -1) {
+ fclose(File);
+ return;
+ }
+ } else {
+ /* The merged profile has a non-zero length. Check that it is compatible
+ * with the data in this process. */
+ if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
+ fclose(File);
+ return;
+ }
+ }
+
+ lprofUnlockFileHandle(File);
+ }
+
+ /* Update the profile fields based on the current mapping. */
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
+ (intptr_t)Profile - (uintptr_t)CountersBegin +
+ CountersOffset;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
+}
+#else
+static void initializeProfileForContinuousMode(void) {
+ PROF_ERR("%s\n", "continuous mode is unsupported on this platform");
+}
+#endif
static const char *DefaultProfileName = "default.profraw";
static void resetFilenameToDefault(void) {
return 0;
}
+/* Assert that Idx does index past a string null terminator. Return the
+ * result of the check. */
+static int checkBounds(int Idx, int Strlen) {
+ assert(Idx <= Strlen && "Indexing past string null terminator");
+ return Idx <= Strlen;
+}
+
/* Parses the pattern string \p FilenamePat and stores the result to
* lprofcurFilename structure. */
static int parseFilenamePattern(const char *FilenamePat,
char *PidChars = &lprofCurFilename.PidChars[0];
char *Hostname = &lprofCurFilename.Hostname[0];
int MergingEnabled = 0;
+ int FilenamePatLen = strlen(FilenamePat);
/* Clean up cached prefix and filename. */
if (lprofCurFilename.ProfilePathPrefix)
lprofCurFilename.OwnsFilenamePat = 1;
}
/* Check the filename for "%p", which indicates a pid-substitution. */
- for (I = 0; FilenamePat[I]; ++I)
+ for (I = 0; checkBounds(I, FilenamePatLen) && FilenamePat[I]; ++I) {
if (FilenamePat[I] == '%') {
- if (FilenamePat[++I] == 'p') {
+ ++I; /* Advance to the next character. */
+ if (!checkBounds(I, FilenamePatLen))
+ break;
+ if (FilenamePat[I] == 'p') {
if (!NumPids++) {
if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
PROF_WARN("Unable to get pid for filename pattern %s. Using the "
FilenamePat);
return -1;
}
+ } else if (FilenamePat[I] == 't') {
+ lprofCurFilename.TmpDir = getenv("TMPDIR");
+ if (!lprofCurFilename.TmpDir) {
+ PROF_WARN("Unable to get the TMPDIR environment variable, referenced "
+ "in %s. Using the default path.",
+ FilenamePat);
+ return -1;
+ }
} else if (FilenamePat[I] == 'c') {
if (__llvm_profile_is_continuous_mode_enabled()) {
PROF_WARN("%%c specifier can only be specified once in %s.\n",
FilenamePat);
return -1;
}
-
+#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32)
+ __llvm_profile_set_page_size(getpagesize());
__llvm_profile_enable_continuous_mode();
- I++; /* advance to 'c' */
+#else
+ PROF_WARN("%s", "Continous mode is currently only supported for Mach-O,"
+ " ELF and COFF formats.");
+ return -1;
+#endif
} else {
unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
if (!MergePoolSize)
lprofCurFilename.MergePoolSize = MergePoolSize;
}
}
+ }
lprofCurFilename.NumPids = NumPids;
lprofCurFilename.NumHosts = NumHosts;
}
truncateCurrentFile();
- if (__llvm_profile_is_continuous_mode_enabled()) {
- if (lprofRuntimeCounterRelocation())
- relocateCounters();
- else
- initializeProfileForContinuousMode();
- }
+ if (__llvm_profile_is_continuous_mode_enabled())
+ initializeProfileForContinuousMode();
}
/* Return buffer length that is required to store the current profile
return 0;
if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
- lprofCurFilename.MergePoolSize))
+ lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize))
return strlen(lprofCurFilename.FilenamePat);
Len = strlen(lprofCurFilename.FilenamePat) +
lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
- lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2);
+ lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) +
+ (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0);
if (lprofCurFilename.MergePoolSize)
Len += SIGLEN;
return Len;
* current filename pattern string is directly returned, unless ForceUseBuf
* is enabled. */
static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
- int I, J, PidLength, HostNameLength, FilenamePatLength;
+ int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength;
const char *FilenamePat = lprofCurFilename.FilenamePat;
if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
return 0;
if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
- lprofCurFilename.MergePoolSize ||
+ lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize ||
__llvm_profile_is_continuous_mode_enabled())) {
if (!ForceUseBuf)
return lprofCurFilename.FilenamePat;
PidLength = strlen(lprofCurFilename.PidChars);
HostNameLength = strlen(lprofCurFilename.Hostname);
+ TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0;
/* Construct the new filename. */
for (I = 0, J = 0; FilenamePat[I]; ++I)
if (FilenamePat[I] == '%') {
} else if (FilenamePat[I] == 'h') {
memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
J += HostNameLength;
+ } else if (FilenamePat[I] == 't') {
+ memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength);
+ FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR;
+ J += TmpDirLength + 1;
} else {
if (!getMergePoolSize(FilenamePat, &I))
continue;
ProfileNameSpecifier PNS = PNS_unknown;
int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
- if (__llvm_profile_counter_bias != -1)
- lprofSetRuntimeCounterRelocation(1);
-
EnvFilenamePat = getFilenamePatFromEnv();
if (EnvFilenamePat) {
/* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
|*
\*===----------------------------------------------------------------------===*/
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
#if !defined(__Fuchsia__)
#include "InstrProfilingInternal.h"
ProfileDumped = Value;
}
-static unsigned RuntimeCounterRelocation = 0;
-
-COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) {
- return RuntimeCounterRelocation;
-}
-
-COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) {
- RuntimeCounterRelocation = Value;
-}
-
#endif
unsigned lprofProfileDumped(void);
void lprofSetProfileDumped(unsigned);
-/* Return non zero value if counters are being relocated at runtime. */
-unsigned lprofRuntimeCounterRelocation(void);
-void lprofSetRuntimeCounterRelocation(unsigned);
-
COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *);
COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer;
COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize;
COMPILER_RT_VISIBILITY extern ValueProfNode *EndVNode;
extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *);
+/*
+ * Write binary ids into profiles if writer is given.
+ * Return -1 if an error occurs, otherwise, return total size of binary ids.
+ */
+int __llvm_write_binary_ids(ProfDataWriter *Writer);
+
#endif
COMPILER_RT_VISIBILITY
uint64_t lprofGetLoadModuleSignature() {
/* A very fast way to compute a module signature. */
+ uint64_t Version = __llvm_profile_get_version();
uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() -
__llvm_profile_begin_counters());
uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(),
const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) +
- (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0);
+ (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0) + Version;
}
/* Returns 1 if profile is not structurally compatible. */
__llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
__llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
SrcDataStart =
- (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header));
+ (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
+ Header->BinaryIdsSize);
SrcDataEnd = SrcDataStart + Header->DataSize;
if (ProfileSize < sizeof(__llvm_profile_header))
Header->ValueKindLast != IPVK_Last)
return 1;
- if (ProfileSize < sizeof(__llvm_profile_header) +
+ if (ProfileSize < sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
Header->DataSize * sizeof(__llvm_profile_data) +
Header->NamesSize + Header->CountersSize)
return 1;
}
COMPILER_RT_VISIBILITY
-void __llvm_profile_merge_from_buffer(const char *ProfileData,
- uint64_t ProfileSize) {
+int __llvm_profile_merge_from_buffer(const char *ProfileData,
+ uint64_t ProfileSize) {
__llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
__llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
uint64_t *SrcCountersStart;
const char *SrcNameStart;
- ValueProfData *SrcValueProfDataStart, *SrcValueProfData;
+ const char *SrcValueProfDataStart, *SrcValueProfData;
SrcDataStart =
- (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header));
+ (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
+ Header->BinaryIdsSize);
SrcDataEnd = SrcDataStart + Header->DataSize;
SrcCountersStart = (uint64_t *)SrcDataEnd;
SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize);
SrcValueProfDataStart =
- (ValueProfData *)(SrcNameStart + Header->NamesSize +
- __llvm_profile_get_num_padding_bytes(
- Header->NamesSize));
+ SrcNameStart + Header->NamesSize +
+ __llvm_profile_get_num_padding_bytes(Header->NamesSize);
+ if (SrcNameStart < (const char *)SrcCountersStart)
+ return 1;
for (SrcData = SrcDataStart,
DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
SrcValueProfData = SrcValueProfDataStart;
SrcData < SrcDataEnd; ++SrcData, ++DstData) {
- uint64_t *SrcCounters;
uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr;
- unsigned I, NC, NVK = 0;
+ unsigned NVK = 0;
- NC = SrcData->NumCounters;
- SrcCounters = SrcCountersStart +
- ((size_t)SrcData->CounterPtr - Header->CountersDelta) /
- sizeof(uint64_t);
- for (I = 0; I < NC; I++)
+ unsigned NC = SrcData->NumCounters;
+ if (NC == 0)
+ return 1;
+ uint64_t *SrcCounters = SrcCountersStart + ((size_t)SrcData->CounterPtr -
+ Header->CountersDelta) /
+ sizeof(uint64_t);
+ if (SrcCounters < SrcCountersStart ||
+ (const char *)SrcCounters >= SrcNameStart ||
+ (const char *)(SrcCounters + NC) > SrcNameStart)
+ return 1;
+ for (unsigned I = 0; I < NC; I++)
DstCounters[I] += SrcCounters[I];
- /* Now merge value profile data. */
+ /* Now merge value profile data. */
if (!VPMergeHook)
continue;
- for (I = 0; I <= IPVK_Last; I++)
+ for (unsigned I = 0; I <= IPVK_Last; I++)
NVK += (SrcData->NumValueSites[I] != 0);
if (!NVK)
continue;
- VPMergeHook(SrcValueProfData, DstData);
- SrcValueProfData = (ValueProfData *)((char *)SrcValueProfData +
- SrcValueProfData->TotalSize);
+ if (SrcValueProfData >= ProfileData + ProfileSize)
+ return 1;
+ VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
+ SrcValueProfData =
+ SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
}
+
+ return 0;
}
|*
\*===----------------------------------------------------------------------===*/
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
#if defined(__APPLE__)
/* Use linker magic to find the bounds of the Data section. */
COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart;
COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd;
+
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+
#endif
#include "InstrProfilingInternal.h"
#include "InstrProfilingUtil.h"
+/* This variable is an external reference to symbol defined by the compiler. */
+COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR;
+
COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() {
return 1;
}
COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {}
-COMPILER_RT_VISIBILITY unsigned lprofRuntimeCounterRelocation(void) {
- return 1;
-}
-COMPILER_RT_VISIBILITY void lprofSetRuntimeCounterRelocation(unsigned Value) {}
-
static const char ProfileSinkName[] = "llvm-profile";
static inline void lprofWrite(const char *fmt, ...) {
return;
}
- /* This symbol is defined as weak and initialized to -1 by the runtimer, but
- * compiler will generate a strong definition initialized to 0 when runtime
- * counter relocation is used. */
- if (__llvm_profile_counter_bias == -1) {
- lprofWrite("LLVM Profile: counter relocation at runtime is required\n");
- return;
- }
-
const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
+ const uint64_t *CountersBegin = __llvm_profile_begin_counters();
+ const uint64_t *CountersEnd = __llvm_profile_end_counters();
const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
- const uint64_t CountersOffset =
- sizeof(__llvm_profile_header) + (DataSize * sizeof(__llvm_profile_data));
+ const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
+ __llvm_write_binary_ids(NULL) +
+ (DataSize * sizeof(__llvm_profile_data));
+ uint64_t CountersSize = CountersEnd - CountersBegin;
+
+ /* Don't publish a VMO if there are no counters. */
+ if (!CountersSize)
+ return;
zx_status_t Status;
- /* Create VMO to hold the profile data. */
+ /* Create a VMO to hold the profile data. */
zx_handle_t Vmo = ZX_HANDLE_INVALID;
Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo);
if (Status != ZX_OK) {
lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName);
/* Update the profile fields based on the current mapping. */
- __llvm_profile_counter_bias = (intptr_t)Mapping -
- (uintptr_t)__llvm_profile_begin_counters() +
- CountersOffset;
+ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR =
+ (intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset;
+
+ /* Return the memory allocated for counters to OS. */
+ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd);
}
#endif
#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
(defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__)
+#include <elf.h>
+#include <link.h>
#include <stdlib.h>
+#include <string.h>
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
+
+#if defined(__FreeBSD__) && !defined(ElfW)
+/*
+ * FreeBSD's elf.h and link.h headers do not define the ElfW(type) macro yet.
+ * If this is added to all supported FreeBSD versions in the future, this
+ * compatibility macro can be removed.
+ */
+#define ElfW(type) __ElfN(type)
+#endif
#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON)
#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON)
/* Declare section start and stop symbols for various sections
* generated by compiler instrumentation.
*/
-extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY;
-extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY;
-extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY;
-extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY;
-extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY;
-extern char PROF_NAME_START COMPILER_RT_VISIBILITY;
-extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY;
-extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY;
-extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY;
-
-/* Add dummy data to ensure the section is always created. */
-__llvm_profile_data
- __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME);
-uint64_t
- __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME);
-uint32_t
- __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME);
-char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME);
-ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME);
+extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY
+ COMPILER_RT_WEAK;
+extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
+extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK;
COMPILER_RT_VISIBILITY const __llvm_profile_data *
__llvm_profile_begin_data(void) {
COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START;
COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP;
+#ifdef NT_GNU_BUILD_ID
+static size_t RoundUp(size_t size, size_t align) {
+ return (size + align - 1) & ~(align - 1);
+}
+
+/*
+ * Write binary id length and then its data, because binary id does not
+ * have a fixed length.
+ */
+static int WriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
+ const uint8_t *BinaryIdData) {
+ ProfDataIOVec BinaryIdIOVec[] = {
+ {&BinaryIdLen, sizeof(uint64_t), 1, 0},
+ {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}};
+ if (Writer->Write(Writer, BinaryIdIOVec,
+ sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
+ return -1;
+
+ /* Successfully wrote binary id, report success. */
+ return 0;
+}
+
+/*
+ * Look for the note that has the name "GNU\0" and type NT_GNU_BUILD_ID
+ * that contains build id. If build id exists, write binary id.
+ *
+ * Each note in notes section starts with a struct which includes
+ * n_namesz, n_descsz, and n_type members. It is followed by the name
+ * (whose length is defined in n_namesz) and then by the descriptor
+ * (whose length is defined in n_descsz).
+ *
+ * Note sections like .note.ABI-tag and .note.gnu.build-id are aligned
+ * to 4 bytes, so round n_namesz and n_descsz to the nearest 4 bytes.
+ */
+static int WriteBinaryIdForNote(ProfDataWriter *Writer,
+ const ElfW(Nhdr) * Note) {
+ int BinaryIdSize = 0;
+
+ const char *NoteName = (const char *)Note + sizeof(ElfW(Nhdr));
+ if (Note->n_type == NT_GNU_BUILD_ID && Note->n_namesz == 4 &&
+ memcmp(NoteName, "GNU\0", 4) == 0) {
+
+ uint64_t BinaryIdLen = Note->n_descsz;
+ const uint8_t *BinaryIdData =
+ (const uint8_t *)(NoteName + RoundUp(Note->n_namesz, 4));
+ if (Writer != NULL &&
+ WriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData) == -1)
+ return -1;
+
+ BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen;
+ }
+
+ return BinaryIdSize;
+}
+
+/*
+ * Helper function that iterates through notes section and find build ids.
+ * If writer is given, write binary ids into profiles.
+ * If an error happens while writing, return -1.
+ */
+static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note,
+ const ElfW(Nhdr) * NotesEnd) {
+ int TotalBinaryIdsSize = 0;
+ while (Note < NotesEnd) {
+ int Result = WriteBinaryIdForNote(Writer, Note);
+ if (Result == -1)
+ return -1;
+ TotalBinaryIdsSize += Result;
+
+ /* Calculate the offset of the next note in notes section. */
+ size_t NoteOffset = sizeof(ElfW(Nhdr)) + RoundUp(Note->n_namesz, 4) +
+ RoundUp(Note->n_descsz, 4);
+ Note = (const ElfW(Nhdr) *)((const char *)(Note) + NoteOffset);
+ }
+
+ return TotalBinaryIdsSize;
+}
+
+/*
+ * Write binary ids into profiles if writer is given.
+ * Return the total size of binary ids.
+ * If an error happens while writing, return -1.
+ */
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ extern const ElfW(Ehdr) __ehdr_start __attribute__((visibility("hidden")));
+ const ElfW(Ehdr) *ElfHeader = &__ehdr_start;
+ const ElfW(Phdr) *ProgramHeader =
+ (const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff);
+
+ uint32_t I;
+ /* Iterate through entries in the program header. */
+ for (I = 0; I < ElfHeader->e_phnum; I++) {
+ /* Look for the notes section in program header entries. */
+ if (ProgramHeader[I].p_type != PT_NOTE)
+ continue;
+
+ const ElfW(Nhdr) *Note =
+ (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_offset);
+ const ElfW(Nhdr) *NotesEnd =
+ (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_filesz);
+ return WriteBinaryIds(Writer, Note, NotesEnd);
+ }
+
+ return 0;
+}
+#else /* !NT_GNU_BUILD_ID */
+/*
+ * Fallback implementation for targets that don't support the GNU
+ * extensions NT_GNU_BUILD_ID and __ehdr_start.
+ */
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+#endif
+
#endif
#include <stdio.h>
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
static const __llvm_profile_data *DataFirst = NULL;
static const __llvm_profile_data *DataLast = NULL;
COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = 0;
COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = 0;
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+
#endif
\*===----------------------------------------------------------------------===*/
#include "InstrProfiling.h"
+#include "InstrProfilingInternal.h"
#if defined(_WIN32)
ValueProfNode *CurrentVNode = &VNodesStart + 1;
ValueProfNode *EndVNode = &VNodesEnd;
+COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) {
+ return 0;
+}
+
#endif
#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l)
#define COMPILER_RT_ALWAYS_INLINE __forceinline
#define COMPILER_RT_CLEANUP(x)
+#define COMPILER_RT_USED
#elif __GNUC__
-#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x)))
+#ifdef _WIN32
+#define COMPILER_RT_FTRUNCATE(f, l) _chsize(fileno(f), l)
+#define COMPILER_RT_VISIBILITY
+#define COMPILER_RT_WEAK __attribute__((selectany))
+#else
+#define COMPILER_RT_FTRUNCATE(f, l) ftruncate(fileno(f), l)
#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden")))
#define COMPILER_RT_WEAK __attribute__((weak))
+#endif
+#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x)))
#define COMPILER_RT_ALLOCA __builtin_alloca
-#define COMPILER_RT_FTRUNCATE(f,l) ftruncate(fileno(f),l)
#define COMPILER_RT_ALWAYS_INLINE inline __attribute((always_inline))
#define COMPILER_RT_CLEANUP(x) __attribute__((cleanup(x)))
+#define COMPILER_RT_USED __attribute__((used))
#endif
#if defined(__APPLE__)
#include <windows.h>
#include "WindowsMMap.h"
#else
+#include <errno.h>
+#include <fcntl.h>
#include <sys/file.h>
+#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
-#include <fcntl.h>
-#include <errno.h>
#endif
#ifdef COMPILER_RT_HAS_UNAME
#include <sys/prctl.h>
#endif
+#if defined(__Fuchsia__)
+#include <zircon/syscalls.h>
+#endif
+
#include "InstrProfiling.h"
#include "InstrProfilingUtil.h"
prctl(PR_SET_PDEATHSIG, SIGKILL);
#endif
}
+
+COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin,
+ uintptr_t End) {
+ size_t PageSize = getpagesize();
+ uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize);
+ uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize);
+ if (BeginAligned < EndAligned) {
+#if defined(__Fuchsia__)
+ return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT,
+ (zx_vaddr_t)BeginAligned,
+ EndAligned - BeginAligned, NULL, 0);
+#else
+ return madvise((void *)BeginAligned, EndAligned - BeginAligned,
+ MADV_DONTNEED);
+#endif
+ }
+ return 0;
+}
#ifndef PROFILE_INSTRPROFILINGUTIL_H
#define PROFILE_INSTRPROFILINGUTIL_H
+#include <inttypes.h>
#include <stddef.h>
#include <stdio.h>
/* Restore previously suspended SIGKILL. */
void lprofRestoreSigKill();
+static inline size_t lprofRoundUpTo(size_t x, size_t boundary) {
+ return (x + boundary - 1) & ~(boundary - 1);
+}
+
+static inline size_t lprofRoundDownTo(size_t x, size_t boundary) {
+ return x & ~(boundary - 1);
+}
+
+int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End);
+
#endif /* PROFILE_INSTRPROFILINGUTIL_H */
|*
\*===----------------------------------------------------------------------===*/
+#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#define INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
#include "profile/InstrProfData.inc"
static int hasStaticCounters = 1;
static int OutOfNodesWarnings = 0;
static int hasNonDefaultValsPerSite = 0;
#define INSTR_PROF_MAX_VP_WARNS 10
-#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24
#define INSTR_PROF_VNODE_POOL_SIZE 1024
#ifndef _MSC_VER
for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
NumVSites += Data->NumValueSites[VKI];
+ // If NumVSites = 0, calloc is allowed to return a non-null pointer.
+ assert(NumVSites > 0 && "NumVSites can't be zero");
ValueProfNode **Mem =
(ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
if (!Mem)
}
/*
- * The target values are partitioned into multiple regions/ranges. There is one
- * contiguous region which is precise -- every value in the range is tracked
- * individually. A value outside the precise region will be collapsed into one
- * value depending on the region it falls in.
- *
- * There are three regions:
- * 1. (-inf, PreciseRangeStart) and (PreciseRangeLast, LargeRangeValue) belong
- * to one region -- all values here should be mapped to one value of
- * "PreciseRangeLast + 1".
- * 2. [PreciseRangeStart, PreciseRangeLast]
- * 3. Large values: [LargeValue, +inf) maps to one value of LargeValue.
- *
- * The range for large values is optional. The default value of INT64_MIN
- * indicates it is not specified.
+ * The target values are partitioned into multiple ranges. The range spec is
+ * defined in InstrProfData.inc.
*/
-COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
- uint64_t TargetValue, void *Data, uint32_t CounterIndex,
- int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) {
-
- if (LargeValue != INT64_MIN && (int64_t)TargetValue >= LargeValue)
- TargetValue = LargeValue;
- else if ((int64_t)TargetValue < PreciseRangeStart ||
- (int64_t)TargetValue > PreciseRangeLast)
- TargetValue = PreciseRangeLast + 1;
-
- __llvm_profile_instrument_target(TargetValue, Data, CounterIndex);
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex) {
+ // Map the target value to the representative value of its range.
+ uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue);
+ __llvm_profile_instrument_target(RepValue, Data, CounterIndex);
}
/*
--- /dev/null
+/*===- InstrProfilingVersionVar.c - profile version variable setup -------===*\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+
+#include "InstrProfiling.h"
+
+/* uint64 __llvm_profile_raw_version
+ *
+ * The runtime should only provide its own definition of this symbol when the
+ * user has not specified one. Set this up by moving the runtime's copy of this
+ * symbol to an object file within the archive.
+ */
+COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION;
|*
\*===----------------------------------------------------------------------===*/
+// Note: This is linked into the Darwin kernel, and must remain compatible
+// with freestanding compilation. See `darwin_add_builtin_libraries`.
+
#ifdef _MSC_VER
/* For _alloca */
#include <malloc.h>
#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
#include "profile/InstrProfData.inc"
- /* Write the data. */
- ProfDataIOVec IOVec[] = {
- {&Header, sizeof(__llvm_profile_header), 1, 0},
+ /* Write the profile header. */
+ ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
+ if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
+ return -1;
+
+ /* Write the binary id lengths and data. */
+ if (__llvm_write_binary_ids(Writer) == -1)
+ return -1;
+
+ /* Write the profile data. */
+ ProfDataIOVec IOVecData[] = {
{DataBegin, sizeof(__llvm_profile_data), DataSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
{CountersBegin, sizeof(uint64_t), CountersSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
{SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
- if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
+ if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;
/* Value profiling is not yet supported in continuous mode. */
return 0;
}
+COMPILER_RT_VISIBILITY
+int madvise(void *addr, size_t length, int advice)
+{
+ if (advice != MADV_DONTNEED)
+ return -1; /* Not supported. */
+
+ if (!VirtualUnlock(addr, length))
+ return -1;
+
+ return 0;
+}
+
COMPILER_RT_VISIBILITY
int lock(HANDLE handle, DWORD lockType, BOOL blocking) {
DWORD flags = lockType;
#define MS_INVALIDATE 0x0002 /* invalidate all cached data */
#define MS_SYNC 0x0010 /* msync synchronously */
+/*
+ * madvise() flags
+ */
+
+#define MADV_NORMAL 0 /* no special treatment */
+#define MADV_WILLNEED 3 /* expect access in the near future */
+#define MADV_DONTNEED 4 /* do not expect access in the near future */
+
/*
* flock() operations
*/
int msync(void *addr, size_t length, int flags);
+int madvise(void *addr, size_t length, int advice);
+
int flock(int fd, int operation);
#endif /* _WIN32 */
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
sanitizer_linux.cpp
sanitizer_linux_s390.cpp
sanitizer_mac.cpp
+ sanitizer_mutex.cpp
sanitizer_netbsd.cpp
- sanitizer_openbsd.cpp
sanitizer_persistent_allocator.cpp
sanitizer_platform_limits_freebsd.cpp
sanitizer_platform_limits_linux.cpp
sanitizer_platform_limits_netbsd.cpp
- sanitizer_platform_limits_openbsd.cpp
sanitizer_platform_limits_posix.cpp
sanitizer_platform_limits_solaris.cpp
sanitizer_posix.cpp
sanitizer_procmaps_linux.cpp
sanitizer_procmaps_mac.cpp
sanitizer_procmaps_solaris.cpp
- sanitizer_rtems.cpp
sanitizer_solaris.cpp
sanitizer_stoptheworld_fuchsia.cpp
sanitizer_stoptheworld_mac.cpp
set(SANITIZER_SYMBOLIZER_SOURCES
sanitizer_allocator_report.cpp
+ sanitizer_chained_origin_depot.cpp
sanitizer_stackdepot.cpp
sanitizer_stacktrace.cpp
sanitizer_stacktrace_libcdep.cpp
sanitizer_atomic_msvc.h
sanitizer_bitvector.h
sanitizer_bvgraph.h
+ sanitizer_chained_origin_depot.h
sanitizer_common.h
sanitizer_common_interceptors.inc
sanitizer_common_interceptors_format.inc
sanitizer_platform.h
sanitizer_platform_interceptors.h
sanitizer_platform_limits_netbsd.h
- sanitizer_platform_limits_openbsd.h
sanitizer_platform_limits_posix.h
sanitizer_platform_limits_solaris.h
sanitizer_posix.h
sanitizer_quarantine.h
sanitizer_report_decorator.h
sanitizer_ring_buffer.h
- sanitizer_rtems.h
sanitizer_signal_interceptors.inc
sanitizer_stackdepot.h
sanitizer_stackdepotbase.h
sanitizer_symbolizer_internal.h
sanitizer_symbolizer_libbacktrace.h
sanitizer_symbolizer_mac.h
- sanitizer_symbolizer_rtems.h
sanitizer_syscall_generic.inc
sanitizer_syscall_linux_aarch64.inc
sanitizer_syscall_linux_arm.inc
sanitizer_syscall_linux_x86_64.inc
+ sanitizer_syscall_linux_riscv64.inc
sanitizer_syscalls_netbsd.inc
sanitizer_thread_registry.h
+ sanitizer_thread_safety.h
sanitizer_tls_get_addr.h
sanitizer_vector.h
sanitizer_win.h
table_ = (Bucket*)MmapOrDie(kSize * sizeof(table_[0]), "AddrHashMap");
}
-template<typename T, uptr kSize>
-void AddrHashMap<T, kSize>::acquire(Handle *h) {
+template <typename T, uptr kSize>
+void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
uptr addr = h->addr_;
uptr hash = calcHash(addr);
Bucket *b = &table_[hash];
CHECK_EQ(atomic_load(&c->addr, memory_order_relaxed), 0);
h->addidx_ = i;
h->cell_ = c;
-}
-
-template<typename T, uptr kSize>
-void AddrHashMap<T, kSize>::release(Handle *h) {
- if (!h->cell_)
- return;
- Bucket *b = h->bucket_;
- Cell *c = h->cell_;
- uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
- if (h->created_) {
- // Denote completion of insertion.
- CHECK_EQ(addr1, 0);
- // After the following store, the element becomes available
- // for lock-free reads.
- atomic_store(&c->addr, h->addr_, memory_order_release);
- b->mtx.Unlock();
- } else if (h->remove_) {
- // Denote that the cell is empty now.
- CHECK_EQ(addr1, h->addr_);
- atomic_store(&c->addr, 0, memory_order_release);
- // See if we need to compact the bucket.
- AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed);
- if (h->addidx_ == -1U) {
- // Removed from embed array, move an add element into the freed cell.
- if (add && add->size != 0) {
- uptr last = --add->size;
- Cell *c1 = &add->cells[last];
- c->val = c1->val;
- uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed);
- atomic_store(&c->addr, addr1, memory_order_release);
- atomic_store(&c1->addr, 0, memory_order_release);
- }
- } else {
- // Removed from add array, compact it.
- uptr last = --add->size;
- Cell *c1 = &add->cells[last];
- if (c != c1) {
- *c = *c1;
- atomic_store(&c1->addr, 0, memory_order_relaxed);
- }
- }
- if (add && add->size == 0) {
- // FIXME(dvyukov): free add?
- }
- b->mtx.Unlock();
- } else {
- CHECK_EQ(addr1, h->addr_);
- if (h->addidx_ != -1U)
- b->mtx.ReadUnlock();
- }
-}
+ }
+
+ template <typename T, uptr kSize>
+ void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
+ if (!h->cell_)
+ return;
+ Bucket *b = h->bucket_;
+ Cell *c = h->cell_;
+ uptr addr1 = atomic_load(&c->addr, memory_order_relaxed);
+ if (h->created_) {
+ // Denote completion of insertion.
+ CHECK_EQ(addr1, 0);
+ // After the following store, the element becomes available
+ // for lock-free reads.
+ atomic_store(&c->addr, h->addr_, memory_order_release);
+ b->mtx.Unlock();
+ } else if (h->remove_) {
+ // Denote that the cell is empty now.
+ CHECK_EQ(addr1, h->addr_);
+ atomic_store(&c->addr, 0, memory_order_release);
+ // See if we need to compact the bucket.
+ AddBucket *add = (AddBucket *)atomic_load(&b->add, memory_order_relaxed);
+ if (h->addidx_ == -1U) {
+ // Removed from embed array, move an add element into the freed cell.
+ if (add && add->size != 0) {
+ uptr last = --add->size;
+ Cell *c1 = &add->cells[last];
+ c->val = c1->val;
+ uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed);
+ atomic_store(&c->addr, addr1, memory_order_release);
+ atomic_store(&c1->addr, 0, memory_order_release);
+ }
+ } else {
+ // Removed from add array, compact it.
+ uptr last = --add->size;
+ Cell *c1 = &add->cells[last];
+ if (c != c1) {
+ *c = *c1;
+ atomic_store(&c1->addr, 0, memory_order_relaxed);
+ }
+ }
+ if (add && add->size == 0) {
+ // FIXME(dvyukov): free add?
+ }
+ b->mtx.Unlock();
+ } else {
+ CHECK_EQ(addr1, h->addr_);
+ if (h->addidx_ != -1U)
+ b->mtx.ReadUnlock();
+ }
+ }
template<typename T, uptr kSize>
uptr AddrHashMap<T, kSize>::calcHash(uptr addr) {
#endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
-const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull;
-
static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
SetAllocatorOutOfMemory();
Report("FATAL: %s: internal allocator is out of memory trying to allocate "
}
void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {
- if (size + sizeof(u64) < size)
- return nullptr;
- void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment);
+ void *p = RawInternalAlloc(size, cache, alignment);
if (UNLIKELY(!p))
- ReportInternalAllocatorOutOfMemory(size + sizeof(u64));
- ((u64*)p)[0] = kBlockMagic;
- return (char*)p + sizeof(u64);
+ ReportInternalAllocatorOutOfMemory(size);
+ return p;
}
void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {
- if (!addr)
- return InternalAlloc(size, cache);
- if (size + sizeof(u64) < size)
- return nullptr;
- addr = (char*)addr - sizeof(u64);
- size = size + sizeof(u64);
- CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
void *p = RawInternalRealloc(addr, size, cache);
if (UNLIKELY(!p))
ReportInternalAllocatorOutOfMemory(size);
- return (char*)p + sizeof(u64);
+ return p;
}
void *InternalReallocArray(void *addr, uptr count, uptr size,
}
void InternalFree(void *addr, InternalAllocatorCache *cache) {
- if (!addr)
- return;
- addr = (char*)addr - sizeof(u64);
- CHECK_EQ(kBlockMagic, ((u64*)addr)[0]);
- ((u64*)addr)[0] = 0;
RawInternalFree(addr, cache);
}
// Callback type for iterating over chunks.
typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
-INLINE u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
+inline u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
return (*state = *state * 1103515245 + 12345) >> 16;
}
-INLINE u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
+inline u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
template<typename T>
-INLINE void RandomShuffle(T *a, u32 n, u32 *rand_state) {
+inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
if (n <= 1) return;
u32 state = *rand_state;
for (u32 i = n - 1; i > 0; i--)
void SetErrnoToENOMEM();
// A common errno setting logic shared by almost all sanitizer allocator APIs.
-INLINE void *SetErrnoOnNull(void *ptr) {
+inline void *SetErrnoOnNull(void *ptr) {
if (UNLIKELY(!ptr))
SetErrnoToENOMEM();
return ptr;
// two and that the size is a multiple of alignment for POSIX implementation,
// and a bit relaxed requirement for non-POSIX ones, that the size is a multiple
// of alignment.
-INLINE bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
+inline bool CheckAlignedAllocAlignmentAndSize(uptr alignment, uptr size) {
#if SANITIZER_POSIX
return alignment != 0 && IsPowerOfTwo(alignment) &&
(size & (alignment - 1)) == 0;
// Checks posix_memalign() parameters, verifies that alignment is a power of two
// and a multiple of sizeof(void *).
-INLINE bool CheckPosixMemalignAlignment(uptr alignment) {
+inline bool CheckPosixMemalignAlignment(uptr alignment) {
return alignment != 0 && IsPowerOfTwo(alignment) &&
(alignment % sizeof(void *)) == 0;
}
// Returns true if calloc(size, n) call overflows on size*n calculation.
-INLINE bool CheckForCallocOverflow(uptr size, uptr n) {
+inline bool CheckForCallocOverflow(uptr size, uptr n) {
if (!size)
return false;
uptr max = (uptr)-1L;
// Returns true if the size passed to pvalloc overflows when rounded to the next
// multiple of page_size.
-INLINE bool CheckForPvallocOverflow(uptr size, uptr page_size) {
+inline bool CheckForPvallocOverflow(uptr size, uptr page_size) {
return RoundUpTo(size, page_size) < size;
}
secondary_.InitLinkerInitialized();
}
- void Init(s32 release_to_os_interval_ms) {
+ void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
stats_.Init();
- primary_.Init(release_to_os_interval_ms);
+ primary_.Init(release_to_os_interval_ms, heap_start);
secondary_.Init();
}
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
primary_.ForceLock();
secondary_.ForceLock();
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
secondary_.ForceUnlock();
primary_.ForceUnlock();
}
template <class SizeClassAllocator>
struct SizeClassAllocator64LocalCache {
typedef SizeClassAllocator Allocator;
+ typedef MemoryMapper<Allocator> MemoryMapperT;
void Init(AllocatorGlobalStats *s) {
stats_.Init();
PerClass *c = &per_class_[class_id];
InitCache(c);
if (UNLIKELY(c->count == c->max_count))
- Drain(c, allocator, class_id, c->max_count / 2);
+ DrainHalfMax(c, allocator, class_id);
CompactPtrT chunk = allocator->PointerToCompactPtr(
allocator->GetRegionBeginBySizeClass(class_id),
reinterpret_cast<uptr>(p));
}
void Drain(SizeClassAllocator *allocator) {
+ MemoryMapperT memory_mapper(*allocator);
for (uptr i = 1; i < kNumClasses; i++) {
PerClass *c = &per_class_[i];
- while (c->count > 0)
- Drain(c, allocator, i, c->count);
+ while (c->count > 0) Drain(&memory_mapper, c, allocator, i, c->count);
}
}
return true;
}
- NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id,
- uptr count) {
+ NOINLINE void DrainHalfMax(PerClass *c, SizeClassAllocator *allocator,
+ uptr class_id) {
+ MemoryMapperT memory_mapper(*allocator);
+ Drain(&memory_mapper, c, allocator, class_id, c->max_count / 2);
+ }
+
+ void Drain(MemoryMapperT *memory_mapper, PerClass *c,
+ SizeClassAllocator *allocator, uptr class_id, uptr count) {
CHECK_GE(c->count, count);
const uptr first_idx_to_drain = c->count - count;
c->count -= count;
- allocator->ReturnToAllocator(&stats_, class_id,
+ allocator->ReturnToAllocator(memory_mapper, &stats_, class_id,
&c->chunks[first_idx_to_drain], count);
}
};
typedef SizeClassAllocator32<Params> ThisT;
typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache;
- void Init(s32 release_to_os_interval_ms) {
+ void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
+ CHECK(!heap_start);
possible_regions.Init();
internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
}
}
void *GetMetaData(const void *p) {
+ CHECK(kMetadataSize);
CHECK(PointerIsMine(p));
uptr mem = reinterpret_cast<uptr>(p);
uptr beg = ComputeRegionBeg(mem);
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetSizeClassInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
for (int i = kNumClasses - 1; i >= 0; i--) {
GetSizeClassInfo(i)->mutex.Unlock();
}
// The template parameter Params is a class containing the actual parameters.
//
// Space: a portion of address space of kSpaceSize bytes starting at SpaceBeg.
-// If kSpaceBeg is ~0 then SpaceBeg is chosen dynamically my mmap.
+// If kSpaceBeg is ~0 then SpaceBeg is chosen dynamically by mmap.
// Otherwise SpaceBeg=kSpaceBeg (fixed address).
// kSpaceSize is a power of two.
// At the beginning the entire space is mprotect-ed, then small parts of it
};
};
+template <typename Allocator>
+class MemoryMapper {
+ public:
+ typedef typename Allocator::CompactPtrT CompactPtrT;
+
+ explicit MemoryMapper(const Allocator &allocator) : allocator_(allocator) {}
+
+ bool GetAndResetStats(uptr &ranges, uptr &bytes) {
+ ranges = released_ranges_count_;
+ released_ranges_count_ = 0;
+ bytes = released_bytes_;
+ released_bytes_ = 0;
+ return ranges != 0;
+ }
+
+ u64 *MapPackedCounterArrayBuffer(uptr count) {
+ buffer_.clear();
+ buffer_.resize(count);
+ return buffer_.data();
+ }
+
+ // Releases [from, to) range of pages back to OS.
+ void ReleasePageRangeToOS(uptr class_id, CompactPtrT from, CompactPtrT to) {
+ const uptr region_base = allocator_.GetRegionBeginBySizeClass(class_id);
+ const uptr from_page = allocator_.CompactPtrToPointer(region_base, from);
+ const uptr to_page = allocator_.CompactPtrToPointer(region_base, to);
+ ReleaseMemoryPagesToOS(from_page, to_page);
+ released_ranges_count_++;
+ released_bytes_ += to_page - from_page;
+ }
+
+ private:
+ const Allocator &allocator_;
+ uptr released_ranges_count_ = 0;
+ uptr released_bytes_ = 0;
+ InternalMmapVector<u64> buffer_;
+};
+
template <class Params>
class SizeClassAllocator64 {
public:
typedef SizeClassAllocator64<Params> ThisT;
typedef SizeClassAllocator64LocalCache<ThisT> AllocatorCache;
+ typedef MemoryMapper<ThisT> MemoryMapperT;
// When we know the size class (the region base) we can represent a pointer
// as a 4-byte integer (offset from the region start shifted right by 4).
return base + (static_cast<uptr>(ptr32) << kCompactPtrScale);
}
- void Init(s32 release_to_os_interval_ms) {
+ // If heap_start is nonzero, assumes kSpaceSize bytes are already mapped R/W
+ // at heap_start and places the heap there. This mode requires kSpaceBeg ==
+ // ~(uptr)0.
+ void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
- if (kUsingConstantSpaceBeg) {
- CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
- CHECK_EQ(kSpaceBeg, address_range.Init(TotalSpaceSize,
- PrimaryAllocatorName, kSpaceBeg));
+ PremappedHeap = heap_start != 0;
+ if (PremappedHeap) {
+ CHECK(!kUsingConstantSpaceBeg);
+ NonConstSpaceBeg = heap_start;
+ uptr RegionInfoSize = AdditionalSize();
+ RegionInfoSpace =
+ address_range.Init(RegionInfoSize, PrimaryAllocatorName);
+ CHECK_NE(RegionInfoSpace, ~(uptr)0);
+ CHECK_EQ(RegionInfoSpace,
+ address_range.MapOrDie(RegionInfoSpace, RegionInfoSize,
+ "SizeClassAllocator: region info"));
+ MapUnmapCallback().OnMap(RegionInfoSpace, RegionInfoSize);
} else {
- // Combined allocator expects that an 2^N allocation is always aligned to
- // 2^N. For this to work, the start of the space needs to be aligned as
- // high as the largest size class (which also needs to be a power of 2).
- NonConstSpaceBeg = address_range.InitAligned(
- TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
- CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+ if (kUsingConstantSpaceBeg) {
+ CHECK(IsAligned(kSpaceBeg, SizeClassMap::kMaxSize));
+ CHECK_EQ(kSpaceBeg,
+ address_range.Init(TotalSpaceSize, PrimaryAllocatorName,
+ kSpaceBeg));
+ } else {
+ // Combined allocator expects that an 2^N allocation is always aligned
+ // to 2^N. For this to work, the start of the space needs to be aligned
+ // as high as the largest size class (which also needs to be a power of
+ // 2).
+ NonConstSpaceBeg = address_range.InitAligned(
+ TotalSpaceSize, SizeClassMap::kMaxSize, PrimaryAllocatorName);
+ CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+ }
+ RegionInfoSpace = SpaceEnd();
+ MapWithCallbackOrDie(RegionInfoSpace, AdditionalSize(),
+ "SizeClassAllocator: region info");
}
SetReleaseToOSIntervalMs(release_to_os_interval_ms);
- MapWithCallbackOrDie(SpaceEnd(), AdditionalSize(),
- "SizeClassAllocator: region info");
// Check that the RegionInfo array is aligned on the CacheLine size.
- DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0);
+ DCHECK_EQ(RegionInfoSpace % kCacheLineSize, 0);
}
s32 ReleaseToOSIntervalMs() const {
}
void ForceReleaseToOS() {
+ MemoryMapperT memory_mapper(*this);
for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
- MaybeReleaseToOS(class_id, true /*force*/);
+ MaybeReleaseToOS(&memory_mapper, class_id, true /*force*/);
}
}
alignment <= SizeClassMap::kMaxSize;
}
- NOINLINE void ReturnToAllocator(AllocatorStats *stat, uptr class_id,
+ NOINLINE void ReturnToAllocator(MemoryMapperT *memory_mapper,
+ AllocatorStats *stat, uptr class_id,
const CompactPtrT *chunks, uptr n_chunks) {
RegionInfo *region = GetRegionInfo(class_id);
uptr region_beg = GetRegionBeginBySizeClass(class_id);
region->num_freed_chunks = new_num_freed_chunks;
region->stats.n_freed += n_chunks;
- MaybeReleaseToOS(class_id, false /*force*/);
+ MaybeReleaseToOS(memory_mapper, class_id, false /*force*/);
}
NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id,
CompactPtrT *free_array = GetFreeArray(region_beg);
BlockingMutexLock l(®ion->mutex);
+#if SANITIZER_WINDOWS
+ /* On Windows unmapping of memory during __sanitizer_purge_allocator is
+ explicit and immediate, so unmapped regions must be explicitly mapped back
+ in when they are accessed again. */
+ if (region->rtoi.last_released_bytes > 0) {
+ MmapFixedOrDie(region_beg, region->mapped_user,
+ "SizeClassAllocator: region data");
+ region->rtoi.n_freed_at_last_release = 0;
+ region->rtoi.last_released_bytes = 0;
+ }
+#endif
if (UNLIKELY(region->num_freed_chunks < n_chunks)) {
if (UNLIKELY(!PopulateFreeArray(stat, class_id, region,
n_chunks - region->num_freed_chunks)))
void *GetBlockBegin(const void *p) {
uptr class_id = GetSizeClass(p);
+ if (class_id >= kNumClasses) return nullptr;
uptr size = ClassIdToSize(class_id);
if (!size) return nullptr;
uptr chunk_idx = GetChunkIdx((uptr)p, size);
uptr reg_beg = GetRegionBegin(p);
uptr beg = chunk_idx * size;
uptr next_beg = beg + size;
- if (class_id >= kNumClasses) return nullptr;
const RegionInfo *region = AddressSpaceView::Load(GetRegionInfo(class_id));
if (region->mapped_user >= next_beg)
return reinterpret_cast<void*>(reg_beg + beg);
static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); }
void *GetMetaData(const void *p) {
+ CHECK(kMetadataSize);
uptr class_id = GetSizeClass(p);
uptr size = ClassIdToSize(class_id);
uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
+ void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
for (uptr i = 0; i < kNumClasses; i++) {
GetRegionInfo(i)->mutex.Lock();
}
}
- void ForceUnlock() {
+ void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
for (int i = (int)kNumClasses - 1; i >= 0; i--) {
GetRegionInfo(i)->mutex.Unlock();
}
// For the performance sake, none of the accessors check the validity of the
// arguments, it is assumed that index is always in [0, n) range and the value
// is not incremented past max_value.
- template<class MemoryMapperT>
class PackedCounterArray {
public:
- PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper)
- : n(num_counters), memory_mapper(mapper) {
+ template <typename MemoryMapper>
+ PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapper *mapper)
+ : n(num_counters) {
CHECK_GT(num_counters, 0);
CHECK_GT(max_value, 0);
constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL;
packing_ratio_log = Log2(packing_ratio);
bit_offset_mask = packing_ratio - 1;
- buffer_size =
- (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) *
- sizeof(*buffer);
- buffer = reinterpret_cast<u64*>(
- memory_mapper->MapPackedCounterArrayBuffer(buffer_size));
- }
- ~PackedCounterArray() {
- if (buffer) {
- memory_mapper->UnmapPackedCounterArrayBuffer(
- reinterpret_cast<uptr>(buffer), buffer_size);
- }
+ buffer = mapper->MapPackedCounterArrayBuffer(
+ RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log);
}
bool IsAllocated() const {
u64 counter_mask;
u64 packing_ratio_log;
u64 bit_offset_mask;
-
- MemoryMapperT* const memory_mapper;
- u64 buffer_size;
u64* buffer;
};
- template<class MemoryMapperT>
+ template <class MemoryMapperT>
class FreePagesRangeTracker {
public:
- explicit FreePagesRangeTracker(MemoryMapperT* mapper)
+ FreePagesRangeTracker(MemoryMapperT *mapper, uptr class_id)
: memory_mapper(mapper),
- page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)),
- in_the_range(false), current_page(0), current_range_start_page(0) {}
+ class_id(class_id),
+ page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)) {}
void NextPage(bool freed) {
if (freed) {
void CloseOpenedRange() {
if (in_the_range) {
memory_mapper->ReleasePageRangeToOS(
- current_range_start_page << page_size_scaled_log,
+ class_id, current_range_start_page << page_size_scaled_log,
current_page << page_size_scaled_log);
in_the_range = false;
}
}
- MemoryMapperT* const memory_mapper;
- const uptr page_size_scaled_log;
- bool in_the_range;
- uptr current_page;
- uptr current_range_start_page;
+ MemoryMapperT *const memory_mapper = nullptr;
+ const uptr class_id = 0;
+ const uptr page_size_scaled_log = 0;
+ bool in_the_range = false;
+ uptr current_page = 0;
+ uptr current_range_start_page = 0;
};
// Iterates over the free_array to identify memory pages containing freed
// chunks only and returns these pages back to OS.
// allocated_pages_count is the total number of pages allocated for the
// current bucket.
- template<class MemoryMapperT>
+ template <typename MemoryMapper>
static void ReleaseFreeMemoryToOS(CompactPtrT *free_array,
uptr free_array_count, uptr chunk_size,
uptr allocated_pages_count,
- MemoryMapperT *memory_mapper) {
+ MemoryMapper *memory_mapper,
+ uptr class_id) {
const uptr page_size = GetPageSizeCached();
// Figure out the number of chunks per page and whether we can take a fast
UNREACHABLE("All chunk_size/page_size ratios must be handled.");
}
- PackedCounterArray<MemoryMapperT> counters(allocated_pages_count,
- full_pages_chunk_count_max,
- memory_mapper);
+ PackedCounterArray counters(allocated_pages_count,
+ full_pages_chunk_count_max, memory_mapper);
if (!counters.IsAllocated())
return;
// Iterate over pages detecting ranges of pages with chunk counters equal
// to the expected number of chunks for the particular page.
- FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper);
+ FreePagesRangeTracker<MemoryMapper> range_tracker(memory_mapper, class_id);
if (same_chunk_count_per_page) {
// Fast path, every page has the same number of chunks affecting it.
for (uptr i = 0; i < counters.GetCount(); i++)
}
private:
- friend class MemoryMapper;
+ friend class MemoryMapper<ThisT>;
ReservedAddressRange address_range;
atomic_sint32_t release_to_os_interval_ms_;
+ uptr RegionInfoSpace;
+
+ // True if the user has already mapped the entire heap R/W.
+ bool PremappedHeap;
+
struct Stats {
uptr n_allocated;
uptr n_freed;
RegionInfo *GetRegionInfo(uptr class_id) const {
DCHECK_LT(class_id, kNumClasses);
- RegionInfo *regions = reinterpret_cast<RegionInfo *>(SpaceEnd());
+ RegionInfo *regions = reinterpret_cast<RegionInfo *>(RegionInfoSpace);
return ®ions[class_id];
}
}
bool MapWithCallback(uptr beg, uptr size, const char *name) {
+ if (PremappedHeap)
+ return beg >= NonConstSpaceBeg &&
+ beg + size <= NonConstSpaceBeg + kSpaceSize;
uptr mapped = address_range.Map(beg, size, name);
if (UNLIKELY(!mapped))
return false;
}
void MapWithCallbackOrDie(uptr beg, uptr size, const char *name) {
+ if (PremappedHeap) {
+ CHECK_GE(beg, NonConstSpaceBeg);
+ CHECK_LE(beg + size, NonConstSpaceBeg + kSpaceSize);
+ return;
+ }
CHECK_EQ(beg, address_range.MapOrDie(beg, size, name));
MapUnmapCallback().OnMap(beg, size);
}
void UnmapWithCallbackOrDie(uptr beg, uptr size) {
+ if (PremappedHeap)
+ return;
MapUnmapCallback().OnUnmap(beg, size);
address_range.Unmap(beg, size);
}
return true;
}
- class MemoryMapper {
- public:
- MemoryMapper(const ThisT& base_allocator, uptr class_id)
- : allocator(base_allocator),
- region_base(base_allocator.GetRegionBeginBySizeClass(class_id)),
- released_ranges_count(0),
- released_bytes(0) {
- }
-
- uptr GetReleasedRangesCount() const {
- return released_ranges_count;
- }
-
- uptr GetReleasedBytes() const {
- return released_bytes;
- }
-
- uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
- // TODO(alekseyshl): The idea to explore is to check if we have enough
- // space between num_freed_chunks*sizeof(CompactPtrT) and
- // mapped_free_array to fit buffer_size bytes and use that space instead
- // of mapping a temporary one.
- return reinterpret_cast<uptr>(
- MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters"));
- }
-
- void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {
- UnmapOrDie(reinterpret_cast<void *>(buffer), buffer_size);
- }
-
- // Releases [from, to) range of pages back to OS.
- void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) {
- const uptr from_page = allocator.CompactPtrToPointer(region_base, from);
- const uptr to_page = allocator.CompactPtrToPointer(region_base, to);
- ReleaseMemoryPagesToOS(from_page, to_page);
- released_ranges_count++;
- released_bytes += to_page - from_page;
- }
-
- private:
- const ThisT& allocator;
- const uptr region_base;
- uptr released_ranges_count;
- uptr released_bytes;
- };
-
// Attempts to release RAM occupied by freed chunks back to OS. The region is
// expected to be locked.
- void MaybeReleaseToOS(uptr class_id, bool force) {
+ //
+ // TODO(morehouse): Support a callback on memory release so HWASan can release
+ // aliases as well.
+ void MaybeReleaseToOS(MemoryMapperT *memory_mapper, uptr class_id,
+ bool force) {
RegionInfo *region = GetRegionInfo(class_id);
const uptr chunk_size = ClassIdToSize(class_id);
const uptr page_size = GetPageSizeCached();
}
}
- MemoryMapper memory_mapper(*this, class_id);
-
- ReleaseFreeMemoryToOS<MemoryMapper>(
+ ReleaseFreeMemoryToOS(
GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size,
- RoundUpTo(region->allocated_user, page_size) / page_size,
- &memory_mapper);
+ RoundUpTo(region->allocated_user, page_size) / page_size, memory_mapper,
+ class_id);
- if (memory_mapper.GetReleasedRangesCount() > 0) {
+ uptr ranges, bytes;
+ if (memory_mapper->GetAndResetStats(ranges, bytes)) {
region->rtoi.n_freed_at_last_release = region->stats.n_freed;
- region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount();
- region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes();
+ region->rtoi.num_releases += ranges;
+ region->rtoi.last_released_bytes = bytes;
}
region->rtoi.last_release_at_ns = MonotonicNanoTime();
}
Die();
}
+void NORETURN ReportRssLimitExceeded(const StackTrace *stack) {
+ {
+ ScopedAllocatorErrorReport report("rss-limit-exceeded", stack);
+ Report("ERROR: %s: allocator exceeded the RSS limit\n", SanitizerToolName);
+ }
+ Die();
+}
+
} // namespace __sanitizer
void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
const StackTrace *stack);
void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack);
+void NORETURN ReportRssLimitExceeded(const StackTrace *stack);
} // namespace __sanitizer
// (currently, 32 bits and internal allocator).
class LargeMmapAllocatorPtrArrayStatic {
public:
- INLINE void *Init() { return &p_[0]; }
- INLINE void EnsureSpace(uptr n) { CHECK_LT(n, kMaxNumChunks); }
+ inline void *Init() { return &p_[0]; }
+ inline void EnsureSpace(uptr n) { CHECK_LT(n, kMaxNumChunks); }
private:
static const int kMaxNumChunks = 1 << 15;
uptr p_[kMaxNumChunks];
// same functionality in Fuchsia case, which does not support MAP_NORESERVE.
class LargeMmapAllocatorPtrArrayDynamic {
public:
- INLINE void *Init() {
+ inline void *Init() {
uptr p = address_range_.Init(kMaxNumChunks * sizeof(uptr),
SecondaryAllocatorName);
CHECK(p);
return reinterpret_cast<void*>(p);
}
- INLINE void EnsureSpace(uptr n) {
+ inline void EnsureSpace(uptr n) {
CHECK_LT(n, kMaxNumChunks);
DCHECK(n <= n_reserved_);
if (UNLIKELY(n == n_reserved_)) {
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
// introspection API.
- void ForceLock() {
- mutex_.Lock();
- }
+ void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); }
- void ForceUnlock() {
- mutex_.Unlock();
- }
+ void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); }
// Iterate over all existing chunks.
// The allocator must be locked when calling this function.
// E.g. with kNumBits==3 all size classes after 2^kMidSizeLog
// look like 0b1xx0..0, where x is either 0 or 1.
//
-// Example: kNumBits=3, kMidSizeLog=4, kMidSizeLog=8, kMaxSizeLog=17:
+// Example: kNumBits=3, kMinSizeLog=4, kMidSizeLog=8, kMaxSizeLog=17:
//
// Classes 1 - 16 correspond to sizes 16 to 256 (size = class_id * 16).
// Next 4 classes: 256 + i * 64 (i = 1 to 4).
// Clutter-reducing helpers.
template<typename T>
-INLINE typename T::Type atomic_load_relaxed(const volatile T *a) {
+inline typename T::Type atomic_load_relaxed(const volatile T *a) {
return atomic_load(a, memory_order_relaxed);
}
template<typename T>
-INLINE void atomic_store_relaxed(volatile T *a, typename T::Type v) {
+inline void atomic_store_relaxed(volatile T *a, typename T::Type v) {
atomic_store(a, v, memory_order_relaxed);
}
// See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
// for mappings of the memory model to different processors.
-INLINE void atomic_signal_fence(memory_order) {
+inline void atomic_signal_fence(memory_order) {
__asm__ __volatile__("" ::: "memory");
}
-INLINE void atomic_thread_fence(memory_order) {
+inline void atomic_thread_fence(memory_order) {
__sync_synchronize();
}
template<typename T>
-INLINE typename T::Type atomic_fetch_add(volatile T *a,
+inline typename T::Type atomic_fetch_add(volatile T *a,
typename T::Type v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
}
template<typename T>
-INLINE typename T::Type atomic_fetch_sub(volatile T *a,
+inline typename T::Type atomic_fetch_sub(volatile T *a,
typename T::Type v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
}
template<typename T>
-INLINE typename T::Type atomic_exchange(volatile T *a,
+inline typename T::Type atomic_exchange(volatile T *a,
typename T::Type v, memory_order mo) {
DCHECK(!((uptr)a % sizeof(*a)));
if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
}
template <typename T>
-INLINE bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
+inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
typedef typename T::Type Type;
}
template<typename T>
-INLINE bool atomic_compare_exchange_weak(volatile T *a,
+inline bool atomic_compare_exchange_weak(volatile T *a,
typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
} __attribute__((aligned(32))) lock = {0, {0}};
template <>
-INLINE atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type val,
memory_order mo) {
DCHECK(mo &
- (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+ (memory_order_relaxed | memory_order_release | memory_order_seq_cst));
DCHECK(!((uptr)ptr % sizeof(*ptr)));
atomic_uint64_t::Type ret;
}
template <>
-INLINE atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type val,
memory_order mo) {
return atomic_fetch_add(ptr, -val, mo);
}
template <>
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr,
atomic_uint64_t::Type *cmp,
atomic_uint64_t::Type xchg,
memory_order mo) {
DCHECK(mo &
- (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+ (memory_order_relaxed | memory_order_release | memory_order_seq_cst));
DCHECK(!((uptr)ptr % sizeof(*ptr)));
typedef atomic_uint64_t::Type Type;
}
template <>
-INLINE atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
+inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr,
memory_order mo) {
DCHECK(mo &
- (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+ (memory_order_relaxed | memory_order_release | memory_order_seq_cst));
DCHECK(!((uptr)ptr % sizeof(*ptr)));
atomic_uint64_t::Type zero = 0;
}
template <>
-INLINE void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
+inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v,
memory_order mo) {
DCHECK(mo &
- (memory_order_relaxed | memory_order_releasae | memory_order_seq_cst));
+ (memory_order_relaxed | memory_order_release | memory_order_seq_cst));
DCHECK(!((uptr)ptr % sizeof(*ptr)));
__spin_lock(&lock.lock);
namespace __sanitizer {
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
__asm__ __volatile__("" ::: "memory");
}
template<typename T>
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
__sync_synchronize();
}
} else {
- // 64-bit load on 32-bit platform.
- // Gross, but simple and reliable.
- // Assume that it is not in read-only memory.
- v = __sync_fetch_and_add(
- const_cast<typename T::Type volatile *>(&a->val_dont_use), 0);
+ __atomic_load(const_cast<typename T::Type volatile *>(&a->val_dont_use), &v,
+ __ATOMIC_SEQ_CST);
}
return v;
}
template<typename T>
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
__sync_synchronize();
}
} else {
- // 64-bit store on 32-bit platform.
- // Gross, but simple and reliable.
- typename T::Type cmp = a->val_dont_use;
- typename T::Type cur;
- for (;;) {
- cur = __sync_val_compare_and_swap(&a->val_dont_use, cmp, v);
- if (cur == cmp || cur == v)
- break;
- cmp = cur;
- }
+ __atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST);
}
}
namespace __sanitizer {
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
__asm__ __volatile__("" ::: "memory");
for (int i = 0; i < cnt; i++)
__asm__ __volatile__("pause");
}
template<typename T>
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
}
template<typename T>
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
namespace __sanitizer {
-INLINE void atomic_signal_fence(memory_order) {
+inline void atomic_signal_fence(memory_order) {
_ReadWriteBarrier();
}
-INLINE void atomic_thread_fence(memory_order) {
+inline void atomic_thread_fence(memory_order) {
_mm_mfence();
}
-INLINE void proc_yield(int cnt) {
+inline void proc_yield(int cnt) {
for (int i = 0; i < cnt; i++)
_mm_pause();
}
template<typename T>
-INLINE typename T::Type atomic_load(
+inline typename T::Type atomic_load(
const volatile T *a, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_consume
| memory_order_acquire | memory_order_seq_cst));
}
template<typename T>
-INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
+inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
DCHECK(mo & (memory_order_relaxed | memory_order_release
| memory_order_seq_cst));
DCHECK(!((uptr)a % sizeof(*a)));
atomic_thread_fence(memory_order_seq_cst);
}
-INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
+inline u32 atomic_fetch_add(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
(long)v);
}
-INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
+inline uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
#endif
}
-INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
+inline u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
-(long)v);
}
-INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
+inline uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
uptr v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
#endif
}
-INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
+inline u8 atomic_exchange(volatile atomic_uint8_t *a,
u8 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
}
-INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
+inline u16 atomic_exchange(volatile atomic_uint16_t *a,
u16 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
}
-INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
+inline u32 atomic_exchange(volatile atomic_uint32_t *a,
u32 v, memory_order mo) {
(void)mo;
DCHECK(!((uptr)a % sizeof(*a)));
return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
u8 *cmp,
u8 xchgv,
memory_order mo) {
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
uptr *cmp,
uptr xchg,
memory_order mo) {
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
u16 *cmp,
u16 xchg,
memory_order mo) {
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
u32 *cmp,
u32 xchg,
memory_order mo) {
return false;
}
-INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
+inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
u64 *cmp,
u64 xchg,
memory_order mo) {
}
template<typename T>
-INLINE bool atomic_compare_exchange_weak(volatile T *a,
+inline bool atomic_compare_exchange_weak(volatile T *a,
typename T::Type *cmp,
typename T::Type xchg,
memory_order mo) {
--- /dev/null
+//===-- sanitizer_chained_origin_depot.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A storage for chained origins.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_chained_origin_depot.h"
+
+namespace __sanitizer {
+
+bool ChainedOriginDepot::ChainedOriginDepotNode::eq(
+ u32 hash, const args_type &args) const {
+ return here_id == args.here_id && prev_id == args.prev_id;
+}
+
+uptr ChainedOriginDepot::ChainedOriginDepotNode::storage_size(
+ const args_type &args) {
+ return sizeof(ChainedOriginDepotNode);
+}
+
+/* This is murmur2 hash for the 64->32 bit case.
+ It does not behave all that well because the keys have a very biased
+ distribution (I've seen 7-element buckets with the table only 14% full).
+
+ here_id is built of
+ * (1 bits) Reserved, zero.
+ * (8 bits) Part id = bits 13..20 of the hash value of here_id's key.
+ * (23 bits) Sequential number (each part has each own sequence).
+
+ prev_id has either the same distribution as here_id (but with 3:8:21)
+ split, or one of two reserved values (-1) or (-2). Either case can
+ dominate depending on the workload.
+*/
+u32 ChainedOriginDepot::ChainedOriginDepotNode::hash(const args_type &args) {
+ const u32 m = 0x5bd1e995;
+ const u32 seed = 0x9747b28c;
+ const u32 r = 24;
+ u32 h = seed;
+ u32 k = args.here_id;
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+ h *= m;
+ h ^= k;
+
+ k = args.prev_id;
+ k *= m;
+ k ^= k >> r;
+ k *= m;
+ h *= m;
+ h ^= k;
+
+ h ^= h >> 13;
+ h *= m;
+ h ^= h >> 15;
+ return h;
+}
+
+bool ChainedOriginDepot::ChainedOriginDepotNode::is_valid(
+ const args_type &args) {
+ return true;
+}
+
+void ChainedOriginDepot::ChainedOriginDepotNode::store(const args_type &args,
+ u32 other_hash) {
+ here_id = args.here_id;
+ prev_id = args.prev_id;
+}
+
+ChainedOriginDepot::ChainedOriginDepotNode::args_type
+ChainedOriginDepot::ChainedOriginDepotNode::load() const {
+ args_type ret = {here_id, prev_id};
+ return ret;
+}
+
+ChainedOriginDepot::ChainedOriginDepotNode::Handle
+ChainedOriginDepot::ChainedOriginDepotNode::get_handle() {
+ return Handle(this);
+}
+
+ChainedOriginDepot::ChainedOriginDepot() {}
+
+StackDepotStats *ChainedOriginDepot::GetStats() { return depot.GetStats(); }
+
+bool ChainedOriginDepot::Put(u32 here_id, u32 prev_id, u32 *new_id) {
+ ChainedOriginDepotDesc desc = {here_id, prev_id};
+ bool inserted;
+ ChainedOriginDepotNode::Handle h = depot.Put(desc, &inserted);
+ *new_id = h.valid() ? h.id() : 0;
+ return inserted;
+}
+
+u32 ChainedOriginDepot::Get(u32 id, u32 *other) {
+ ChainedOriginDepotDesc desc = depot.Get(id);
+ *other = desc.prev_id;
+ return desc.here_id;
+}
+
+void ChainedOriginDepot::LockAll() { depot.LockAll(); }
+
+void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
+
+} // namespace __sanitizer
--- /dev/null
+//===-- sanitizer_chained_origin_depot.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A storage for chained origins.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_CHAINED_ORIGIN_DEPOT_H
+#define SANITIZER_CHAINED_ORIGIN_DEPOT_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_stackdepotbase.h"
+
+namespace __sanitizer {
+
+class ChainedOriginDepot {
+ public:
+ ChainedOriginDepot();
+
+ // Gets the statistic of the origin chain storage.
+ StackDepotStats *GetStats();
+
+ // Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
+ // If successful, returns true and the new chain id new_id.
+ // If the same element already exists, returns false and sets new_id to the
+ // existing ID.
+ bool Put(u32 here_id, u32 prev_id, u32 *new_id);
+
+ // Retrieves the stored StackDepot ID for the given origin ID.
+ u32 Get(u32 id, u32 *other);
+
+ void LockAll();
+ void UnlockAll();
+
+ private:
+ struct ChainedOriginDepotDesc {
+ u32 here_id;
+ u32 prev_id;
+ };
+
+ struct ChainedOriginDepotNode {
+ ChainedOriginDepotNode *link;
+ u32 id;
+ u32 here_id;
+ u32 prev_id;
+
+ typedef ChainedOriginDepotDesc args_type;
+
+ bool eq(u32 hash, const args_type &args) const;
+
+ static uptr storage_size(const args_type &args);
+
+ static u32 hash(const args_type &args);
+
+ static bool is_valid(const args_type &args);
+
+ void store(const args_type &args, u32 other_hash);
+
+ args_type load() const;
+
+ struct Handle {
+ ChainedOriginDepotNode *node_;
+ Handle() : node_(nullptr) {}
+ explicit Handle(ChainedOriginDepotNode *node) : node_(node) {}
+ bool valid() { return node_; }
+ u32 id() { return node_->id; }
+ int here_id() { return node_->here_id; }
+ int prev_id() { return node_->prev_id; }
+ };
+
+ Handle get_handle();
+
+ typedef Handle handle_type;
+ };
+
+ StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
+
+ ChainedOriginDepot(const ChainedOriginDepot &) = delete;
+ void operator=(const ChainedOriginDepot &) = delete;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_CHAINED_ORIGIN_DEPOT_H
const char *mmap_type, error_t err,
bool raw_report) {
static int recursion_count;
- if (SANITIZER_RTEMS || raw_report || recursion_count) {
- // If we are on RTEMS or raw report is requested or we went into recursion,
- // just die. The Report() and CHECK calls below may call mmap recursively
- // and fail.
+ if (raw_report || recursion_count) {
+ // If raw report is requested or we went into recursion just die. The
+ // Report() and CHECK calls below may call mmap recursively and fail.
RawWrite("ERROR: Failed to mmap\n");
Die();
}
void ReportErrorSummary(const char *error_message, const char *alt_tool_name) {
if (!common_flags()->print_summary)
return;
- InternalScopedString buff(kMaxSummaryLength);
+ InternalScopedString buff;
buff.append("SUMMARY: %s: %s",
alt_tool_name ? alt_tool_name : SanitizerToolName, error_message);
__sanitizer_report_error_summary(buff.data());
return name_len;
}
+uptr ReadBinaryDir(/*out*/ char *buf, uptr buf_len) {
+ ReadBinaryNameCached(buf, buf_len);
+ const char *exec_name_pos = StripModuleName(buf);
+ uptr name_len = exec_name_pos - buf;
+ buf[name_len] = '\0';
+ return name_len;
+}
+
#if !SANITIZER_GO
void PrintCmdline() {
char **argv = GetArgv();
return 0;
}
+void internal_sleep(unsigned seconds) {
+ internal_usleep((u64)seconds * 1000 * 1000);
+}
+void SleepForSeconds(unsigned seconds) {
+ internal_usleep((u64)seconds * 1000 * 1000);
+}
+void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); }
+
} // namespace __sanitizer
using namespace __sanitizer;
const uptr kMaxThreadStackSize = 1 << 30; // 1Gb
-static const uptr kErrorMessageBufferSize = 1 << 16;
+const uptr kErrorMessageBufferSize = 1 << 16;
// Denotes fake PC values that come from JIT/JAVA/etc.
// For such PC values __tsan_symbolize_external_ex() will be called.
extern const char *SanitizerToolName; // Can be changed by the tool.
extern atomic_uint32_t current_verbosity;
-INLINE void SetVerbosity(int verbosity) {
+inline void SetVerbosity(int verbosity) {
atomic_store(¤t_verbosity, verbosity, memory_order_relaxed);
}
-INLINE int Verbosity() {
+inline int Verbosity() {
return atomic_load(¤t_verbosity, memory_order_relaxed);
}
#if SANITIZER_ANDROID
-INLINE uptr GetPageSize() {
+inline uptr GetPageSize() {
// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
return 4096;
}
-INLINE uptr GetPageSizeCached() {
+inline uptr GetPageSizeCached() {
return 4096;
}
#else
uptr GetPageSize();
extern uptr PageSizeCached;
-INLINE uptr GetPageSizeCached() {
+inline uptr GetPageSizeCached() {
if (!PageSizeCached)
PageSizeCached = GetPageSize();
return PageSizeCached;
// Memory management
void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
-INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
+inline void *MmapOrDieQuietly(uptr size, const char *mem_type) {
return MmapOrDie(size, mem_type, /*raw_report*/ true);
}
void UnmapOrDie(void *addr, uptr size);
void MprotectMallocZones(void *addr, int prot);
+#if SANITIZER_LINUX
+// Unmap memory. Currently only used on Linux.
+void UnmapFromTo(uptr from, uptr to);
+#endif
+
+// Maps shadow_size_bytes of shadow memory and returns shadow address. It will
+// be aligned to the mmap granularity * 2^shadow_scale, or to
+// 2^min_shadow_base_alignment if that is larger. The returned address will
+// have max(2^min_shadow_base_alignment, mmap granularity) on the left, and
+// shadow_size_bytes bytes on the right, which on linux is mapped no access.
+// The high_mem_end may be updated if the original shadow size doesn't fit.
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment, uptr &high_mem_end);
+
+// Let S = max(shadow_size, num_aliases * alias_size, ring_buffer_size).
+// Reserves 2*S bytes of address space to the right of the returned address and
+// ring_buffer_size bytes to the left. The returned address is aligned to 2*S.
+// Also creates num_aliases regions of accessible memory starting at offset S
+// from the returned address. Each region has size alias_size and is backed by
+// the same physical memory.
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+ uptr num_aliases, uptr ring_buffer_size);
+
+// Reserve memory range [beg, end]. If madvise_shadow is true then apply
+// madvise (e.g. hugepages, core dumping) requested by options.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
+ bool madvise_shadow = true);
+
+// Protect size bytes of memory starting at addr. Also try to protect
+// several pages at the start of the address space as specified by
+// zero_base_shadow_start, at most up to the size or zero_base_max_shadow_start.
+void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
+ uptr zero_base_max_shadow_start);
+
// Find an available address space.
uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
uptr *largest_gap_found, uptr *max_occupied_addr);
// Lock sanitizer error reporting and protects against nested errors.
class ScopedErrorReportLock {
public:
- ScopedErrorReportLock();
- ~ScopedErrorReportLock();
+ ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); }
+ ~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); }
- static void CheckLocked();
+ static void Lock() ACQUIRE(mutex_);
+ static void Unlock() RELEASE(mutex_);
+ static void CheckLocked() CHECK_LOCKED(mutex_);
+
+ private:
+ static atomic_uintptr_t reporting_thread_;
+ static StaticSpinMutex mutex_;
};
extern uptr stoptheworld_tracer_pid;
// OS
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len);
uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len);
+uptr ReadBinaryDir(/*out*/ char *buf, uptr buf_len);
uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len);
const char *GetProcessName();
void UpdateProcessName();
void CacheBinaryName();
void DisableCoreDumperIfNecessary();
void DumpProcessMap();
-void PrintModuleMap();
const char *GetEnv(const char *name);
bool SetEnv(const char *name, const char *value);
uptr GetTlsSize();
// Other
-void SleepForSeconds(int seconds);
-void SleepForMillis(int millis);
+void SleepForSeconds(unsigned seconds);
+void SleepForMillis(unsigned millis);
u64 NanoTime();
u64 MonotonicNanoTime();
int Atexit(void (*function)(void));
const char *mmap_type, error_t err,
bool raw_report = false);
-// Specific tools may override behavior of "Die" and "CheckFailed" functions
-// to do tool-specific job.
+// Specific tools may override behavior of "Die" function to do tool-specific
+// job.
typedef void (*DieCallbackType)(void);
// It's possible to add several callbacks that would be run when "Die" is
void SetUserDieCallback(DieCallbackType callback);
-typedef void (*CheckFailedCallbackType)(const char *, int, const char *,
- u64, u64);
-void SetCheckFailedCallback(CheckFailedCallbackType callback);
+void SetCheckUnwindCallback(void (*callback)());
// Callback will be called if soft_rss_limit_mb is given and the limit is
// exceeded (exceeded==true) or if rss went down below the limit
void SetAlternateSignalStack();
void UnsetAlternateSignalStack();
-// We don't want a summary too long.
-const int kMaxSummaryLength = 1024;
// Construct a one-line string:
// SUMMARY: SanitizerToolName: error_message
// and pass it to __sanitizer_report_error_summary.
}
#endif
-INLINE uptr MostSignificantSetBitIndex(uptr x) {
+inline uptr MostSignificantSetBitIndex(uptr x) {
CHECK_NE(x, 0U);
unsigned long up;
#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
return up;
}
-INLINE uptr LeastSignificantSetBitIndex(uptr x) {
+inline uptr LeastSignificantSetBitIndex(uptr x) {
CHECK_NE(x, 0U);
unsigned long up;
#if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__)
return up;
}
-INLINE bool IsPowerOfTwo(uptr x) {
+inline bool IsPowerOfTwo(uptr x) {
return (x & (x - 1)) == 0;
}
-INLINE uptr RoundUpToPowerOfTwo(uptr size) {
+inline uptr RoundUpToPowerOfTwo(uptr size) {
CHECK(size);
if (IsPowerOfTwo(size)) return size;
return 1ULL << (up + 1);
}
-INLINE uptr RoundUpTo(uptr size, uptr boundary) {
+inline uptr RoundUpTo(uptr size, uptr boundary) {
RAW_CHECK(IsPowerOfTwo(boundary));
return (size + boundary - 1) & ~(boundary - 1);
}
-INLINE uptr RoundDownTo(uptr x, uptr boundary) {
+inline uptr RoundDownTo(uptr x, uptr boundary) {
return x & ~(boundary - 1);
}
-INLINE bool IsAligned(uptr a, uptr alignment) {
+inline bool IsAligned(uptr a, uptr alignment) {
return (a & (alignment - 1)) == 0;
}
-INLINE uptr Log2(uptr x) {
+inline uptr Log2(uptr x) {
CHECK(IsPowerOfTwo(x));
return LeastSignificantSetBitIndex(x);
}
// Don't use std::min, std::max or std::swap, to minimize dependency
// on libstdc++.
-template<class T> T Min(T a, T b) { return a < b ? a : b; }
-template<class T> T Max(T a, T b) { return a > b ? a : b; }
+template <class T>
+constexpr T Min(T a, T b) {
+ return a < b ? a : b;
+}
+template <class T>
+constexpr T Max(T a, T b) {
+ return a > b ? a : b;
+}
template<class T> void Swap(T& a, T& b) {
T tmp = a;
a = b;
}
// Char handling
-INLINE bool IsSpace(int c) {
+inline bool IsSpace(int c) {
return (c == ' ') || (c == '\n') || (c == '\t') ||
(c == '\f') || (c == '\r') || (c == '\v');
}
-INLINE bool IsDigit(int c) {
+inline bool IsDigit(int c) {
return (c >= '0') && (c <= '9');
}
-INLINE int ToLower(int c) {
+inline int ToLower(int c) {
return (c >= 'A' && c <= 'Z') ? (c + 'a' - 'A') : c;
}
template<typename T>
class InternalMmapVectorNoCtor {
public:
+ using value_type = T;
void Initialize(uptr initial_capacity) {
capacity_bytes_ = 0;
size_ = 0;
InternalMmapVector &operator=(InternalMmapVector &&) = delete;
};
-class InternalScopedString : public InternalMmapVector<char> {
+class InternalScopedString {
public:
- explicit InternalScopedString(uptr max_length)
- : InternalMmapVector<char>(max_length), length_(0) {
- (*this)[0] = '\0';
- }
- uptr length() { return length_; }
+ InternalScopedString() : buffer_(1) { buffer_[0] = '\0'; }
+
+ uptr length() const { return buffer_.size() - 1; }
void clear() {
- (*this)[0] = '\0';
- length_ = 0;
+ buffer_.resize(1);
+ buffer_[0] = '\0';
}
void append(const char *format, ...);
+ const char *data() const { return buffer_.data(); }
+ char *data() { return buffer_.data(); }
private:
- uptr length_;
+ InternalMmapVector<char> buffer_;
};
template <class T>
// Works like std::lower_bound: finds the first element that is not less
// than the val.
-template <class Container, class Value, class Compare>
-uptr InternalLowerBound(const Container &v, uptr first, uptr last,
- const Value &val, Compare comp) {
+template <class Container,
+ class Compare = CompareLess<typename Container::value_type>>
+uptr InternalLowerBound(const Container &v,
+ const typename Container::value_type &val,
+ Compare comp = {}) {
+ uptr first = 0;
+ uptr last = v.size();
while (last > first) {
uptr mid = (first + last) / 2;
if (comp(v[mid], val))
kModuleArchARMV7,
kModuleArchARMV7S,
kModuleArchARMV7K,
- kModuleArchARM64
+ kModuleArchARM64,
+ kModuleArchRISCV64
};
+// Sorts and removes duplicates from the container.
+template <class Container,
+ class Compare = CompareLess<typename Container::value_type>>
+void SortAndDedup(Container &v, Compare comp = {}) {
+ Sort(v.data(), v.size(), comp);
+ uptr size = v.size();
+ if (size < 2)
+ return;
+ uptr last = 0;
+ for (uptr i = 1; i < size; ++i) {
+ if (comp(v[last], v[i])) {
+ ++last;
+ if (last != i)
+ v[last] = v[i];
+ } else {
+ CHECK(!comp(v[i], v[last]));
+ }
+ }
+ v.resize(last + 1);
+}
+
// Opens the file 'file_name" and reads up to 'max_len' bytes.
// The resulting buffer is mmaped and stored in '*buff'.
// Returns true if file was successfully opened and read.
return "armv7k";
case kModuleArchARM64:
return "arm64";
+ case kModuleArchRISCV64:
+ return "riscv64";
}
CHECK(0 && "Invalid module arch");
return "";
#if SANITIZER_MAC || SANITIZER_WIN_TRACE
void LogFullErrorReport(const char *buffer);
#else
-INLINE void LogFullErrorReport(const char *buffer) {}
+inline void LogFullErrorReport(const char *buffer) {}
#endif
#if SANITIZER_LINUX || SANITIZER_MAC
void WriteOneLineToSyslog(const char *s);
void LogMessageOnPrintf(const char *str);
#else
-INLINE void WriteOneLineToSyslog(const char *s) {}
-INLINE void LogMessageOnPrintf(const char *str) {}
+inline void WriteOneLineToSyslog(const char *s) {}
+inline void LogMessageOnPrintf(const char *str) {}
#endif
#if SANITIZER_LINUX || SANITIZER_WIN_TRACE
void AndroidLogInit();
void SetAbortMessage(const char *);
#else
-INLINE void AndroidLogInit() {}
+inline void AndroidLogInit() {}
// FIXME: MacOS implementation could use CRSetCrashLogMessage.
-INLINE void SetAbortMessage(const char *) {}
+inline void SetAbortMessage(const char *) {}
#endif
#if SANITIZER_ANDROID
void SanitizerInitializeUnwinder();
AndroidApiLevel AndroidGetApiLevel();
#else
-INLINE void AndroidLogWrite(const char *buffer_unused) {}
-INLINE void SanitizerInitializeUnwinder() {}
-INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
+inline void AndroidLogWrite(const char *buffer_unused) {}
+inline void SanitizerInitializeUnwinder() {}
+inline AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; }
#endif
-INLINE uptr GetPthreadDestructorIterations() {
+inline uptr GetPthreadDestructorIterations() {
#if SANITIZER_ANDROID
return (AndroidGetApiLevel() == ANDROID_LOLLIPOP_MR1) ? 8 : 4;
#elif SANITIZER_POSIX
#if SANITIZER_LINUX && SANITIZER_S390_64
void AvoidCVE_2016_2143();
#else
-INLINE void AvoidCVE_2016_2143() {}
+inline void AvoidCVE_2016_2143() {}
#endif
struct StackDepotStats {
// Returns the number of logical processors on the system.
u32 GetNumberOfCPUs();
extern u32 NumberOfCPUsCached;
-INLINE u32 GetNumberOfCPUsCached() {
+inline u32 GetNumberOfCPUsCached() {
if (!NumberOfCPUsCached)
NumberOfCPUsCached = GetNumberOfCPUs();
return NumberOfCPUsCached;
T *end_ = nullptr;
};
+#define PRINTF_128(v) \
+ (*((u8 *)&v + 0)), (*((u8 *)&v + 1)), (*((u8 *)&v + 2)), (*((u8 *)&v + 3)), \
+ (*((u8 *)&v + 4)), (*((u8 *)&v + 5)), (*((u8 *)&v + 6)), \
+ (*((u8 *)&v + 7)), (*((u8 *)&v + 8)), (*((u8 *)&v + 9)), \
+ (*((u8 *)&v + 10)), (*((u8 *)&v + 11)), (*((u8 *)&v + 12)), \
+ (*((u8 *)&v + 13)), (*((u8 *)&v + 14)), (*((u8 *)&v + 15))
+
} // namespace __sanitizer
inline void *operator new(__sanitizer::operator_new_size_type size,
// Platform-specific options.
#if SANITIZER_MAC
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#elif SANITIZER_WINDOWS64
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
#else
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
#endif // SANITIZER_MAC
#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
COMMON_INTERCEPT_FUNCTION(fn)
#endif
+#if SANITIZER_GLIBC
+// If we could not find the versioned symbol, fall back to an unversioned
+// lookup. This is needed to work around a GLibc bug that causes dlsym
+// with RTLD_NEXT to return the oldest versioned symbol.
+// See https://sourceware.org/bugzilla/show_bug.cgi?id=14932.
+// For certain symbols (e.g. regexec) we have to perform a versioned lookup,
+// but that versioned symbol will only exist for architectures where the
+// oldest Glibc version pre-dates support for that architecture.
+// For example, regexec@GLIBC_2.3.4 exists on x86_64, but not RISC-V.
+// See also https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98920.
+#define COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(fn, ver) \
+ COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(fn, ver)
+#else
+#define COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(fn, ver) \
+ COMMON_INTERCEPT_FUNCTION(fn)
+#endif
+
#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
{ \
c2 = (unsigned char)s2[i];
if (c1 != c2 || c1 == '\0') break;
}
- COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
- COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+ if (common_flags()->intercept_strcmp) {
+ COMMON_INTERCEPTOR_READ_STRING(ctx, s1, i + 1);
+ COMMON_INTERCEPTOR_READ_STRING(ctx, s2, i + 1);
+ }
int result = CharCmpX(c1, c2);
CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1,
s2, result);
// N.B.: If we switch this to internal_ we'll have to use internal_memmove
// due to memcpy being an alias of memmove on OS X.
void *ctx;
- if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) {
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
- } else {
+#else
COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
- }
+#endif
}
#define INIT_MEMCPY \
// Assuming frexp() always writes to |exp|.
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
double res = REAL(frexp)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
INTERCEPTOR(float, frexpf, float x, int *exp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- float res = REAL(frexpf)(x, exp);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+ float res = REAL(frexpf)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
INTERCEPTOR(long double, frexpl, long double x, int *exp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- long double res = REAL(frexpl)(x, exp);
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp));
+ long double res = REAL(frexpl)(x, exp);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp));
return res;
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
REAL(strlen)(pwd->pw_gecos) + 1);
#endif
-#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD
if (pwd->pw_class)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
REAL(strlen)(pwd->pw_class) + 1);
}
return res;
}
+#if SANITIZER_GLIBC
namespace __sanitizer {
extern "C" {
int real_clock_gettime(u32 clk_id, void *tp) {
}
} // extern "C"
} // namespace __sanitizer
+#endif
INTERCEPTOR(int, clock_settime, u32 clk_id, const void *tp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, clock_settime, clk_id, tp);
COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, REAL(strlen)(locale) + 1);
char *res = REAL(setlocale)(category, locale);
if (res) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
unpoison_ctype_arrays(ctx);
}
return res;
// static storage.
#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \
SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD || \
- SANITIZER_FREEBSD || SANITIZER_OPENBSD
+ SANITIZER_FREEBSD
// POSIX version. Spec is not clear on whether buf is NULL-terminated.
// At least on OSX, buf contents are valid even when the call fails.
INTERCEPTOR(int, strerror_r, int errnum, char *buf, SIZE_T buflen) {
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
- int res = REAL(sigwait)(set, sig);
+ int res = COMMON_INTERCEPTOR_BLOCK_REAL(sigwait)(set, sig);
if (!res && sig) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sig, sizeof(*sig));
return res;
}
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
- int res = REAL(sigwaitinfo)(set, info);
+ int res = COMMON_INTERCEPTOR_BLOCK_REAL(sigwaitinfo)(set, info);
if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz);
return res;
}
// FIXME: under ASan the call below may write to freed memory and corrupt
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
- int res = REAL(sigtimedwait)(set, info, timeout);
+ int res = COMMON_INTERCEPTOR_BLOCK_REAL(sigtimedwait)(set, info, timeout);
if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz);
return res;
}
#define INIT_SIGSETOPS
#endif
+#if SANITIZER_INTERCEPT_SIGSET_LOGICOPS
+INTERCEPTOR(int, sigandset, __sanitizer_sigset_t *dst,
+ __sanitizer_sigset_t *src1, __sanitizer_sigset_t *src2) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sigandset, dst, src1, src2);
+ if (src1)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src1, sizeof(*src1));
+ if (src2)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src2, sizeof(*src2));
+ int res = REAL(sigandset)(dst, src1, src2);
+ if (!res && dst)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+ return res;
+}
+
+INTERCEPTOR(int, sigorset, __sanitizer_sigset_t *dst,
+ __sanitizer_sigset_t *src1, __sanitizer_sigset_t *src2) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, sigorset, dst, src1, src2);
+ if (src1)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src1, sizeof(*src1));
+ if (src2)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src2, sizeof(*src2));
+ int res = REAL(sigorset)(dst, src1, src2);
+ if (!res && dst)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(*dst));
+ return res;
+}
+#define INIT_SIGSET_LOGICOPS \
+ COMMON_INTERCEPT_FUNCTION(sigandset); \
+ COMMON_INTERCEPT_FUNCTION(sigorset);
+#else
+#define INIT_SIGSET_LOGICOPS
+#endif
+
#if SANITIZER_INTERCEPT_SIGPENDING
INTERCEPTOR(int, sigpending, __sanitizer_sigset_t *set) {
void *ctx;
#define INIT_TMPNAM_R
#endif
+#if SANITIZER_INTERCEPT_PTSNAME
+INTERCEPTOR(char *, ptsname, int fd) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);
+ char *res = REAL(ptsname)(fd);
+ if (res != nullptr)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
+ return res;
+}
+#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);
+#else
+#define INIT_PTSNAME
+#endif
+
+#if SANITIZER_INTERCEPT_PTSNAME_R
+INTERCEPTOR(int, ptsname_r, int fd, char *name, SIZE_T namesize) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);
+ int res = REAL(ptsname_r)(fd, name, namesize);
+ if (res == 0)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
+ return res;
+}
+#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);
+#else
+#define INIT_PTSNAME_R
+#endif
+
#if SANITIZER_INTERCEPT_TTYNAME
INTERCEPTOR(char *, ttyname, int fd) {
void *ctx;
#define INIT_TIMES
#endif
+#if SANITIZER_S390 && \
+ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET)
+extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg));
+DEFINE_REAL(uptr, __tls_get_offset, void *arg)
+#endif
+
#if SANITIZER_INTERCEPT_TLS_GET_ADDR
#if !SANITIZER_S390
#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr)
// descriptor offset as an argument instead of a pointer. GOT address
// is passed in r12, so it's necessary to write it in assembly. This is
// the function used by the compiler.
-extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg));
#define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_offset)
-DEFINE_REAL(uptr, __tls_get_offset, void *arg)
-extern "C" uptr __tls_get_offset(void *arg);
-extern "C" uptr __interceptor___tls_get_offset(void *arg);
INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr_internal, arg);
}
return res;
}
+#endif // SANITIZER_S390
+#else
+#define INIT_TLS_GET_ADDR
+#endif
+
+#if SANITIZER_S390 && \
+ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET)
+extern "C" uptr __tls_get_offset(void *arg);
+extern "C" uptr __interceptor___tls_get_offset(void *arg);
// We need a hidden symbol aliasing the above, so that we can jump
// directly to it from the assembly below.
extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),
"br %r3\n"
".size __tls_get_offset_wrapper, .-__tls_get_offset_wrapper\n"
);
-#endif // SANITIZER_S390
-#else
-#define INIT_TLS_GET_ADDR
#endif
#if SANITIZER_INTERCEPT_LISTXATTR
#define INIT_XDR
#endif // SANITIZER_INTERCEPT_XDR
+#if SANITIZER_INTERCEPT_XDRREC
+typedef int (*xdrrec_cb)(char*, char*, int);
+struct XdrRecWrapper {
+ char *handle;
+ xdrrec_cb rd, wr;
+};
+typedef AddrHashMap<XdrRecWrapper *, 11> XdrRecWrapMap;
+static XdrRecWrapMap *xdrrec_wrap_map;
+
+static int xdrrec_wr_wrap(char *handle, char *buf, int count) {
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(buf, count);
+ XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+ return wrap->wr(wrap->handle, buf, count);
+}
+
+static int xdrrec_rd_wrap(char *handle, char *buf, int count) {
+ COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
+ XdrRecWrapper *wrap = (XdrRecWrapper *)handle;
+ return wrap->rd(wrap->handle, buf, count);
+}
+
+// This doesn't apply to the solaris version as it has a different function
+// signature.
+INTERCEPTOR(void, xdrrec_create, __sanitizer_XDR *xdr, unsigned sndsize,
+ unsigned rcvsize, char *handle, int (*rd)(char*, char*, int),
+ int (*wr)(char*, char*, int)) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, xdrrec_create, xdr, sndsize, rcvsize,
+ handle, rd, wr);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, &xdr->x_op, sizeof xdr->x_op);
+
+ // We can't allocate a wrapper on the stack, as the handle is used outside
+ // this stack frame. So we put it on the heap, and keep track of it with
+ // the HashMap (keyed by x_private). When we later need to xdr_destroy,
+ // we can index the map, free the wrapper, and then clean the map entry.
+ XdrRecWrapper *wrap_data =
+ (XdrRecWrapper *)InternalAlloc(sizeof(XdrRecWrapper));
+ wrap_data->handle = handle;
+ wrap_data->rd = rd;
+ wrap_data->wr = wr;
+ if (wr)
+ wr = xdrrec_wr_wrap;
+ if (rd)
+ rd = xdrrec_rd_wrap;
+ handle = (char *)wrap_data;
+
+ REAL(xdrrec_create)(xdr, sndsize, rcvsize, handle, rd, wr);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdr, sizeof *xdr);
+
+ XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, false, true);
+ *wrap = wrap_data;
+}
+
+// We have to intercept this to be able to free wrapper memory;
+// otherwise it's not necessary.
+INTERCEPTOR(void, xdr_destroy, __sanitizer_XDR *xdr) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, xdr_destroy, xdr);
+
+ XdrRecWrapMap::Handle wrap(xdrrec_wrap_map, xdr->x_private, true);
+ InternalFree(*wrap);
+ REAL(xdr_destroy)(xdr);
+}
+#define INIT_XDRREC_LINUX \
+ static u64 xdrrec_wrap_mem[sizeof(XdrRecWrapMap) / sizeof(u64) + 1]; \
+ xdrrec_wrap_map = new ((void *)&xdrrec_wrap_mem) XdrRecWrapMap(); \
+ COMMON_INTERCEPT_FUNCTION(xdrrec_create); \
+ COMMON_INTERCEPT_FUNCTION(xdr_destroy);
+#else
+#define INIT_XDRREC_LINUX
+#endif
+
#if SANITIZER_INTERCEPT_TSEARCH
INTERCEPTOR(void *, tsearch, void *key, void **rootp,
int (*compar)(const void *, const void *)) {
if (fp->_IO_read_base && fp->_IO_read_base < fp->_IO_read_end)
COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_read_base,
fp->_IO_read_end - fp->_IO_read_base);
+ if (fp->_IO_write_base && fp->_IO_write_base < fp->_IO_write_end)
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(fp->_IO_write_base,
+ fp->_IO_write_end - fp->_IO_write_base);
#endif
#endif // SANITIZER_HAS_STRUCT_FILE
}
#define INIT_FOPEN
#endif
+#if SANITIZER_INTERCEPT_FLOPEN
+INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
+ void *ctx;
+ va_list ap;
+ va_start(ap, flags);
+ u16 mode = static_cast<u16>(va_arg(ap, u32));
+ va_end(ap);
+ COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
+ if (path) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ }
+ return REAL(flopen)(path, flags, mode);
+}
+
+INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
+ void *ctx;
+ va_list ap;
+ va_start(ap, flags);
+ u16 mode = static_cast<u16>(va_arg(ap, u32));
+ va_end(ap);
+ COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
+ if (path) {
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
+ }
+ return REAL(flopenat)(dirfd, path, flags, mode);
+}
+
+#define INIT_FLOPEN \
+ COMMON_INTERCEPT_FUNCTION(flopen); \
+ COMMON_INTERCEPT_FUNCTION(flopenat);
+#else
+#define INIT_FLOPEN
+#endif
+
#if SANITIZER_INTERCEPT_FOPEN64
INTERCEPTOR(__sanitizer_FILE *, fopen64, const char *path, const char *mode) {
void *ctx;
INTERCEPTOR(int, fflush, __sanitizer_FILE *fp) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, fflush, fp);
+ if (fp)
+ unpoison_file(fp);
int res = REAL(fflush)(fp);
// FIXME: handle fp == NULL
if (fp) {
COMMON_INTERCEPTOR_ENTER(ctx, fclose, fp);
COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
const FileMetadata *m = GetInterceptorMetadata(fp);
+ if (fp)
+ unpoison_file(fp);
int res = REAL(fclose)(fp);
if (m) {
COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size);
INTERCEPTOR(int, sem_trywait, __sanitizer_sem_t *s) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, sem_trywait, s);
- int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_trywait)(s);
+ int res = REAL(sem_trywait)(s);
if (res == 0) {
COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s);
}
}
#define INIT_REGEX \
COMMON_INTERCEPT_FUNCTION(regcomp); \
- COMMON_INTERCEPT_FUNCTION(regexec); \
+ COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(regexec, "GLIBC_2.3.4"); \
COMMON_INTERCEPT_FUNCTION(regerror); \
COMMON_INTERCEPT_FUNCTION(regfree);
#else
}
}
qsort_compar_f old_compar = qsort_compar;
- qsort_compar = compar;
SIZE_T old_size = qsort_size;
- qsort_size = size;
+ // Handle qsort() implementations that recurse using an
+ // interposable function call:
+ bool already_wrapped = compar == wrapped_qsort_compar;
+ if (already_wrapped) {
+ // This case should only happen if the qsort() implementation calls itself
+ // using a preemptible function call (e.g. the FreeBSD libc version).
+ // Check that the size and comparator arguments are as expected.
+ CHECK_NE(compar, qsort_compar);
+ CHECK_EQ(qsort_size, size);
+ } else {
+ qsort_compar = compar;
+ qsort_size = size;
+ }
REAL(qsort)(base, nmemb, size, wrapped_qsort_compar);
- qsort_compar = old_compar;
- qsort_size = old_size;
+ if (!already_wrapped) {
+ qsort_compar = old_compar;
+ qsort_size = old_size;
+ }
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
}
#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
}
}
qsort_r_compar_f old_compar = qsort_r_compar;
- qsort_r_compar = compar;
SIZE_T old_size = qsort_r_size;
- qsort_r_size = size;
+ // Handle qsort_r() implementations that recurse using an
+ // interposable function call:
+ bool already_wrapped = compar == wrapped_qsort_r_compar;
+ if (already_wrapped) {
+ // This case should only happen if the qsort() implementation calls itself
+ // using a preemptible function call (e.g. the FreeBSD libc version).
+ // Check that the size and comparator arguments are as expected.
+ CHECK_NE(compar, qsort_r_compar);
+ CHECK_EQ(qsort_r_size, size);
+ } else {
+ qsort_r_compar = compar;
+ qsort_r_size = size;
+ }
REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
- qsort_r_compar = old_compar;
- qsort_r_size = old_size;
+ if (!already_wrapped) {
+ qsort_r_compar = old_compar;
+ qsort_r_size = old_size;
+ }
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
}
#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
INIT_SIGWAITINFO;
INIT_SIGTIMEDWAIT;
INIT_SIGSETOPS;
+ INIT_SIGSET_LOGICOPS;
INIT_SIGPENDING;
INIT_SIGPROCMASK;
INIT_PTHREAD_SIGMASK;
INIT_PTHREAD_BARRIERATTR_GETPSHARED;
INIT_TMPNAM;
INIT_TMPNAM_R;
+ INIT_PTSNAME;
+ INIT_PTSNAME_R;
INIT_TTYNAME;
INIT_TTYNAME_R;
INIT_TEMPNAM;
INIT_BZERO;
INIT_FTIME;
INIT_XDR;
+ INIT_XDRREC_LINUX;
INIT_TSEARCH;
INIT_LIBIO_INTERNALS;
INIT_FOPEN;
INIT_FOPEN64;
+ INIT_FLOPEN;
INIT_OPEN_MEMSTREAM;
INIT_OBSTACK;
INIT_FFLUSH;
size = 0;
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
+ // For %ms/%mc, write the allocated output buffer as well.
+ if (dir.allocate) {
+ char *buf = *(char **)argp;
+ if (buf)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
+ }
}
}
_(SOUND_PCM_WRITE_CHANNELS, WRITE, sizeof(int));
_(SOUND_PCM_WRITE_FILTER, WRITE, sizeof(int));
_(TCFLSH, NONE, 0);
+#if SANITIZER_GLIBC
_(TCGETA, WRITE, struct_termio_sz);
+#endif
_(TCGETS, WRITE, struct_termios_sz);
_(TCSBRK, NONE, 0);
_(TCSBRKP, NONE, 0);
+#if SANITIZER_GLIBC
_(TCSETA, READ, struct_termio_sz);
_(TCSETAF, READ, struct_termio_sz);
_(TCSETAW, READ, struct_termio_sz);
+#endif
_(TCSETS, READ, struct_termios_sz);
_(TCSETSF, READ, struct_termios_sz);
_(TCSETSW, READ, struct_termios_sz);
_(VT_WAITACTIVE, NONE, 0);
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
// _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE
- _(CYGETDEFTHRESH, WRITE, sizeof(int));
- _(CYGETDEFTIMEOUT, WRITE, sizeof(int));
- _(CYGETMON, WRITE, struct_cyclades_monitor_sz);
- _(CYGETTHRESH, WRITE, sizeof(int));
- _(CYGETTIMEOUT, WRITE, sizeof(int));
- _(CYSETDEFTHRESH, NONE, 0);
- _(CYSETDEFTIMEOUT, NONE, 0);
- _(CYSETTHRESH, NONE, 0);
- _(CYSETTIMEOUT, NONE, 0);
_(EQL_EMANCIPATE, WRITE, struct_ifreq_sz);
_(EQL_ENSLAVE, WRITE, struct_ifreq_sz);
_(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz);
#if defined(__aarch64__) && defined(__linux__)
#include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
ASM_WRAPPER_NAME(vfork):
// Save x30 in the off-stack spill area.
+ hint #25 // paciasp
stp xzr, x30, [sp, #-16]!
bl COMMON_INTERCEPTOR_SPILL_AREA
ldp xzr, x30, [sp], 16
bl COMMON_INTERCEPTOR_SPILL_AREA
ldr x30, [x0]
ldp x0, xzr, [sp], 16
+ hint #29 // autiasp
ret
ASM_SIZE(vfork)
.weak vfork
.set vfork, ASM_WRAPPER_NAME(vfork)
+GNU_PROPERTY_BTI_PAC
+
#endif
--- /dev/null
+#if (defined(__riscv) && (__riscv_xlen == 64)) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
+
+.comm _ZN14__interception10real_vforkE,8,8
+.globl ASM_WRAPPER_NAME(vfork)
+ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
+ASM_WRAPPER_NAME(vfork):
+ // Save ra in the off-stack spill area.
+ // allocate space on stack
+ addi sp, sp, -16
+ // store ra value
+ sd ra, 8(sp)
+ call COMMON_INTERCEPTOR_SPILL_AREA
+ // restore previous values from stack
+ ld ra, 8(sp)
+ // adjust stack
+ addi sp, sp, 16
+ // store ra by x10
+ sd ra, 0(x10)
+
+ // Call real vfork. This may return twice. User code that runs between the first and the second return
+ // may clobber the stack frame of the interceptor; that's why it does not have a frame.
+ la x10, _ZN14__interception10real_vforkE
+ ld x10, 0(x10)
+ jalr x10
+
+ // adjust stack
+ addi sp, sp, -16
+ // store x10 by adjusted stack
+ sd x10, 8(sp)
+ // jump to exit label if x10 is 0
+ beqz x10, .L_exit
+
+ // x0 != 0 => parent process. Clear stack shadow.
+ // put old sp to x10
+ addi x10, sp, 16
+ call COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+ // Restore ra
+ call COMMON_INTERCEPTOR_SPILL_AREA
+ ld ra, 0(x10)
+ // load value by stack
+ ld x10, 8(sp)
+ // adjust stack
+ addi sp, sp, 16
+ ret
+ASM_SIZE(vfork)
+
+.weak vfork
+.set vfork, ASM_WRAPPER_NAME(vfork)
+
+#endif
INTERFACE_FUNCTION(__sanitizer_set_death_callback)
INTERFACE_FUNCTION(__sanitizer_set_report_path)
INTERFACE_FUNCTION(__sanitizer_set_report_fd)
+INTERFACE_FUNCTION(__sanitizer_get_report_path)
INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
#endif
void WriteToSyslog(const char *msg) {
- InternalScopedString msg_copy(kErrorMessageBufferSize);
+ InternalScopedString msg_copy;
msg_copy.append("%s", msg);
- char *p = msg_copy.data();
- char *q;
+ const char *p = msg_copy.data();
// Print one line at a time.
// syslog, at least on Android, has an implicit message length limit.
- while ((q = internal_strchr(p, '\n'))) {
+ while (char* q = internal_strchr(p, '\n')) {
*q = '\0';
WriteOneLineToSyslog(p);
p = q + 1;
return start;
}
+#if !SANITIZER_FUCHSIA
+
+// Reserve memory range [beg, end].
+// We need to use inclusive range because end+1 may not be representable.
+void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
+ bool madvise_shadow) {
+ CHECK_EQ((beg % GetMmapGranularity()), 0);
+ CHECK_EQ(((end + 1) % GetMmapGranularity()), 0);
+ uptr size = end - beg + 1;
+ DecreaseTotalMmap(size); // Don't count the shadow against mmap_limit_mb.
+ if (madvise_shadow ? !MmapFixedSuperNoReserve(beg, size, name)
+ : !MmapFixedNoReserve(beg, size, name)) {
+ Report(
+ "ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
+ "Perhaps you're using ulimit -v\n",
+ size);
+ Abort();
+ }
+ if (madvise_shadow && common_flags()->use_madv_dontdump)
+ DontDumpShadowMemory(beg, size);
+}
+
+void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
+ uptr zero_base_max_shadow_start) {
+ if (!size)
+ return;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ // A few pages at the start of the address space can not be protected.
+ // But we really want to protect as much as possible, to prevent this memory
+ // being returned as a result of a non-FIXED mmap().
+ if (addr == zero_base_shadow_start) {
+ uptr step = GetMmapGranularity();
+ while (size > step && addr < zero_base_max_shadow_start) {
+ addr += step;
+ size -= step;
+ void *res = MmapFixedNoAccess(addr, size, "shadow gap");
+ if (addr == (uptr)res)
+ return;
+ }
+ }
+
+ Report(
+ "ERROR: Failed to protect the shadow gap. "
+ "%s cannot proceed correctly. ABORTING.\n",
+ SanitizerToolName);
+ DumpProcessMap();
+ Die();
+}
+
+#endif // !SANITIZER_FUCHSIA
+
} // namespace __sanitizer
SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
// libc in no-libcdep sources.
//===----------------------------------------------------------------------===//
-#include "sanitizer_platform.h"
#include "sanitizer_common.h"
+#include "sanitizer_flags.h"
#include "sanitizer_libc.h"
+#include "sanitizer_platform.h"
namespace __sanitizer {
#endif
void WriteToSyslog(const char *buffer) {}
void Abort() { internal__exit(1); }
-void SleepForSeconds(int seconds) { internal_sleep(seconds); }
#endif // !SANITIZER_WINDOWS
#if !SANITIZER_WINDOWS && !SANITIZER_MAC
void ListOfModules::init() {}
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
#endif
} // namespace __sanitizer
POST_SYSCALL(ni_syscall)(long res) {}
PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+#if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ SANITIZER_RISCV64)
if (data) {
if (request == ptrace_setregs) {
PRE_READ((void *)data, struct_user_regs_struct_sz);
}
POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__))
+#if !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+ SANITIZER_RISCV64)
if (res >= 0 && data) {
// Note that this is different from the interceptor in
// sanitizer_common_interceptors.inc.
bool report_pending;
};
-struct DD : public DDetector {
+struct DD final : public DDetector {
SpinMutex mtx;
DeadlockDetector<DDBV> dd;
DDFlags flags;
DDMutex *m0 = (DDMutex*)dd.getData(from);
DDMutex *m1 = (DDMutex*)dd.getData(to);
- u32 stk_from = -1U, stk_to = -1U;
+ u32 stk_from = 0, stk_to = 0;
int unique_tid = 0;
dd.findEdge(from, to, &stk_from, &stk_to, &unique_tid);
// Printf("Edge: %zd=>%zd: %u/%u T%d\n", from, to, stk_from, stk_to,
int nlocked;
};
-struct Mutex {
+struct MutexState {
StaticSpinMutex mtx;
u32 seq;
int nlink;
Link link[kMaxLink];
};
-struct DD : public DDetector {
+struct DD final : public DDetector {
explicit DD(const DDFlags *flags);
DDPhysicalThread* CreatePhysicalThread();
void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx);
void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath);
u32 allocateId(DDCallback *cb);
- Mutex *getMutex(u32 id);
- u32 getMutexId(Mutex *m);
+ MutexState *getMutex(u32 id);
+ u32 getMutexId(MutexState *m);
DDFlags flags;
- Mutex* mutex[kL1Size];
+ MutexState *mutex[kL1Size];
SpinMutex mtx;
InternalMmapVector<u32> free_id;
atomic_store(&m->owner, 0, memory_order_relaxed);
}
-Mutex *DD::getMutex(u32 id) {
- return &mutex[id / kL2Size][id % kL2Size];
-}
+MutexState *DD::getMutex(u32 id) { return &mutex[id / kL2Size][id % kL2Size]; }
-u32 DD::getMutexId(Mutex *m) {
+u32 DD::getMutexId(MutexState *m) {
for (int i = 0; i < kL1Size; i++) {
- Mutex *tab = mutex[i];
+ MutexState *tab = mutex[i];
if (tab == 0)
break;
if (m >= tab && m < tab + kL2Size)
} else {
CHECK_LT(id_gen, kMaxMutex);
if ((id_gen % kL2Size) == 0) {
- mutex[id_gen / kL2Size] = (Mutex*)MmapOrDie(kL2Size * sizeof(Mutex),
- "deadlock detector (mutex table)");
+ mutex[id_gen / kL2Size] = (MutexState *)MmapOrDie(
+ kL2Size * sizeof(MutexState), "deadlock detector (mutex table)");
}
id = id_gen++;
}
}
bool added = false;
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
for (int i = 0; i < lt->nlocked - 1; i++) {
u32 id1 = lt->locked[i].id;
u32 stk1 = lt->locked[i].stk;
- Mutex *mtx1 = getMutex(id1);
+ MutexState *mtx1 = getMutex(id1);
SpinMutexLock l(&mtx1->mtx);
if (mtx1->nlink == kMaxLink) {
// FIXME(dvyukov): check stale links
// Clear and invalidate the mutex descriptor.
{
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
SpinMutexLock l(&mtx->mtx);
mtx->seq++;
mtx->nlink = 0;
int npath = 0;
int npending = 0;
{
- Mutex *mtx = getMutex(m->id);
+ MutexState *mtx = getMutex(m->id);
SpinMutexLock l(&mtx->mtx);
for (int li = 0; li < mtx->nlink; li++)
pt->pending[npending++] = mtx->link[li];
}
if (pt->visited[link.id])
continue;
- Mutex *mtx1 = getMutex(link.id);
+ MutexState *mtx1 = getMutex(link.id);
SpinMutexLock l(&mtx1->mtx);
if (mtx1->seq != link.seq)
continue;
return Report(pt, lt, npath); // Bingo!
for (int li = 0; li < mtx1->nlink; li++) {
Link *link1 = &mtx1->link[li];
- // Mutex *mtx2 = getMutex(link->id);
+ // MutexState *mtx2 = getMutex(link->id);
// FIXME(dvyukov): fast seq check
// FIXME(dvyukov): fast nlink != 0 check
// FIXME(dvyukov): fast pending check?
virtual u32 Unwind() { return 0; }
virtual int UniqueTid() { return 0; }
+
+ protected:
+ ~DDCallback() {}
};
struct DDetector {
virtual void MutexDestroy(DDCallback *cb, DDMutex *m) {}
virtual DDReport *GetReport(DDCallback *cb) { return nullptr; }
+
+ protected:
+ ~DDetector() {}
};
} // namespace __sanitizer
#if SANITIZER_FREEBSD || SANITIZER_MAC
# define __errno_location __error
-#elif SANITIZER_ANDROID || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
- SANITIZER_RTEMS
+#elif SANITIZER_ANDROID || SANITIZER_NETBSD
# define __errno_location __errno
#elif SANITIZER_SOLARIS
# define __errno_location ___errno
#define errno_ENOMEM 12
#define errno_EBUSY 16
#define errno_EINVAL 22
+#define errno_ENAMETOOLONG 36
// Those might not present or their value differ on different platforms.
extern const int errno_EOWNERDEAD;
} else {
internal_snprintf(full_path, kMaxPathLength, "%s.%zu", path_prefix, pid);
}
- fd = OpenFile(full_path, WrOnly);
+ if (common_flags()->log_suffix) {
+ internal_strlcat(full_path, common_flags()->log_suffix, kMaxPathLength);
+ }
+ error_t err;
+ fd = OpenFile(full_path, WrOnly, &err);
if (fd == kInvalidFd) {
const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
+ char errmsg[100];
+ internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)", err);
+ WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));
Die();
}
fd_pid = pid;
}
void ReportFile::SetReportPath(const char *path) {
- if (!path)
- return;
- uptr len = internal_strlen(path);
- if (len > sizeof(path_prefix) - 100) {
- Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n",
- path[0], path[1], path[2], path[3],
- path[4], path[5], path[6], path[7]);
- Die();
+ if (path) {
+ uptr len = internal_strlen(path);
+ if (len > sizeof(path_prefix) - 100) {
+ Report("ERROR: Path is too long: %c%c%c%c%c%c%c%c...\n", path[0], path[1],
+ path[2], path[3], path[4], path[5], path[6], path[7]);
+ Die();
+ }
}
SpinMutexLock l(mu);
if (fd != kStdoutFd && fd != kStderrFd && fd != kInvalidFd)
CloseFile(fd);
fd = kInvalidFd;
- if (internal_strcmp(path, "stdout") == 0) {
- fd = kStdoutFd;
- } else if (internal_strcmp(path, "stderr") == 0) {
+ if (!path || internal_strcmp(path, "stderr") == 0) {
fd = kStderrFd;
+ } else if (internal_strcmp(path, "stdout") == 0) {
+ fd = kStdoutFd;
} else {
internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
}
}
+const char *ReportFile::GetReportPath() {
+ SpinMutexLock l(mu);
+ ReopenIfNecessary();
+ return full_path;
+}
+
bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
uptr *read_len, uptr max_len, error_t *errno_p) {
*buff = nullptr;
report_file.fd = (fd_t)reinterpret_cast<uptr>(fd);
report_file.fd_pid = internal_getpid();
}
+
+const char *__sanitizer_get_report_path() {
+ return report_file.GetReportPath();
+}
} // extern "C"
#endif // !SANITIZER_FUCHSIA
void Write(const char *buffer, uptr length);
bool SupportsColors();
void SetReportPath(const char *path);
+ const char *GetReportPath();
// Don't use fields directly. They are only declared public to allow
// aggregate initialization.
};
template <typename T>
-class FlagHandler : public FlagHandlerBase {
+class FlagHandler final : public FlagHandlerBase {
T *t_;
public:
#include "sanitizer_flags.h"
#include "sanitizer_common.h"
+#include "sanitizer_flag_parser.h"
#include "sanitizer_libc.h"
+#include "sanitizer_linux.h"
#include "sanitizer_list.h"
-#include "sanitizer_flag_parser.h"
namespace __sanitizer {
// Copy the string from "s" to "out", making the following substitutions:
// %b = binary basename
// %p = pid
+// %d = binary directory
void SubstituteForFlagValue(const char *s, char *out, uptr out_size) {
char *out_end = out + out_size;
while (*s && out < out_end - 1) {
s += 2; // skip "%p"
break;
}
+ case 'd': {
+ uptr len = ReadBinaryDir(out, out_end - out);
+ out += len;
+ s += 2; // skip "%d"
+ break;
+ }
default:
*out++ = *s++;
break;
*out = '\0';
}
-class FlagHandlerInclude : public FlagHandlerBase {
+class FlagHandlerInclude final : public FlagHandlerBase {
FlagParser *parser_;
bool ignore_missing_;
const char *original_path_;
}
return parser_->ParseFile(value, ignore_missing_);
}
- bool Format(char *buffer, uptr size) {
+ bool Format(char *buffer, uptr size) override {
// Note `original_path_` isn't actually what's parsed due to `%`
// substitutions. Printing the substituted path would require holding onto
// mmap'ed memory.
// need to record coverage to generate coverage report.
cf->coverage |= cf->html_cov_report;
SetVerbosity(cf->verbosity);
+
+ InitializePlatformCommonFlags(cf);
}
} // namespace __sanitizer
// and perform initializations common to all sanitizers (e.g. setting
// verbosity).
void InitializeCommonFlags(CommonFlags *cf = &common_flags_dont_use);
+
+// Platform specific flags initialization.
+void InitializePlatformCommonFlags(CommonFlags *cf);
+
} // namespace __sanitizer
#endif // SANITIZER_FLAGS_H
COMMON_FLAG(bool, fast_unwind_on_fatal, false,
"If available, use the fast frame-pointer-based unwinder on fatal "
"errors.")
-COMMON_FLAG(bool, fast_unwind_on_malloc, true,
+// ARM thumb/thumb2 frame pointer is inconsistent on GCC and Clang [1]
+// and fast-unwider is also unreliable with mixing arm and thumb code [2].
+// [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92172
+// [2] https://bugs.llvm.org/show_bug.cgi?id=44158
+COMMON_FLAG(bool, fast_unwind_on_malloc,
+ !(SANITIZER_LINUX && !SANITIZER_ANDROID && SANITIZER_ARM),
"If available, use the fast frame-pointer-based unwinder on "
"malloc/free.")
COMMON_FLAG(bool, handle_ioctl, false, "Intercept and handle ioctl requests.")
COMMON_FLAG(int, malloc_context_size, 1,
"Max number of stack frames kept for each allocation/deallocation.")
COMMON_FLAG(
- const char *, log_path, "stderr",
+ const char *, log_path, nullptr,
"Write logs to \"log_path.pid\". The special values are \"stdout\" and "
- "\"stderr\". The default is \"stderr\".")
+ "\"stderr\". If unspecified, defaults to \"stderr\".")
COMMON_FLAG(
bool, log_exe_name, false,
"Mention name of executable when reporting error and "
"append executable name to logs (as in \"log_path.exe_name.pid\").")
+COMMON_FLAG(const char *, log_suffix, nullptr,
+ "String to append to log file name, e.g. \".txt\".")
COMMON_FLAG(
bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
"Write all sanitizer output to syslog in addition to other means of "
"If false, disable printing error summaries in addition to error "
"reports.")
COMMON_FLAG(int, print_module_map, 0,
- "OS X only (0 - don't print, 1 - print only once before process "
- "exits, 2 - print after each report).")
+ "Print the process module map where supported (0 - don't print, "
+ "1 - print only once before process exits, 2 - print after each "
+ "report).")
COMMON_FLAG(bool, check_printf, true, "Check printf arguments.")
#define COMMON_FLAG_HANDLE_SIGNAL_HELP(signal) \
"Controls custom tool's " #signal " handler (0 - do not registers the " \
COMMON_FLAG(bool, intercept_strpbrk, true,
"If set, uses custom wrappers for strpbrk function "
"to find more errors.")
+COMMON_FLAG(
+ bool, intercept_strcmp, true,
+ "If set, uses custom wrappers for strcmp functions to find more errors.")
COMMON_FLAG(bool, intercept_strlen, true,
"If set, uses custom wrappers for strlen and strnlen functions "
"to find more errors.")
#include "sanitizer_fuchsia.h"
#if SANITIZER_FUCHSIA
-#include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_mutex.h"
-
-#include <limits.h>
#include <pthread.h>
#include <stdlib.h>
#include <unistd.h>
#include <zircon/errors.h>
#include <zircon/process.h>
#include <zircon/syscalls.h>
+#include <zircon/utc.h>
+
+#include "sanitizer_common.h"
+#include "sanitizer_libc.h"
+#include "sanitizer_mutex.h"
namespace __sanitizer {
return 0; // Why doesn't this return void?
}
-static void internal_nanosleep(zx_time_t ns) {
- zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns));
+void internal_usleep(u64 useconds) {
+ zx_status_t status = _zx_nanosleep(_zx_deadline_after(ZX_USEC(useconds)));
CHECK_EQ(status, ZX_OK);
}
-unsigned int internal_sleep(unsigned int seconds) {
- internal_nanosleep(ZX_SEC(seconds));
- return 0;
-}
-
u64 NanoTime() {
+ zx_handle_t utc_clock = _zx_utc_reference_get();
+ CHECK_NE(utc_clock, ZX_HANDLE_INVALID);
zx_time_t time;
- zx_status_t status = _zx_clock_get(ZX_CLOCK_UTC, &time);
+ zx_status_t status = _zx_clock_read(utc_clock, &time);
CHECK_EQ(status, ZX_OK);
return time;
}
return pid;
}
-int internal_dlinfo(void *handle, int request, void *p) {
- UNIMPLEMENTED();
-}
+int internal_dlinfo(void *handle, int request, void *p) { UNIMPLEMENTED(); }
uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); }
int Atexit(void (*function)(void)) { return atexit(function); }
-void SleepForSeconds(int seconds) { internal_sleep(seconds); }
-
-void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); }
-
void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
pthread_attr_t attr;
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
void UnsetAlternateSignalStack() {}
void InitTlsSize() {}
-void PrintModuleMap() {}
-
bool SignalContext::IsStackOverflow() const { return false; }
void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
const char *SignalContext::Describe() const { UNIMPLEMENTED(); }
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(p), cmp,
+ ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
+ if (status != ZX_ERR_BAD_STATE) // Normal race.
+ CHECK_EQ(status, ZX_OK);
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+ zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(p), count);
+ CHECK_EQ(status, ZX_OK);
+}
+
enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
BlockingMutex::BlockingMutex() {
}
}
-void BlockingMutex::CheckLocked() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+void BlockingMutex::CheckLocked() const {
+ auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
}
-uptr GetPageSize() { return PAGE_SIZE; }
+uptr GetPageSize() { return _zx_system_get_page_size(); }
-uptr GetMmapGranularity() { return PAGE_SIZE; }
+uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
sanitizer_shadow_bounds_t ShadowBounds;
+void InitShadowBounds() { ShadowBounds = __sanitizer_shadow_bounds(); }
+
uptr GetMaxUserVirtualAddress() {
- ShadowBounds = __sanitizer_shadow_bounds();
+ InitShadowBounds();
return ShadowBounds.memory_limit - 1;
}
static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
bool raw_report, bool die_for_nomem) {
- size = RoundUpTo(size, PAGE_SIZE);
+ size = RoundUpTo(size, GetPageSize());
zx_handle_t vmo;
zx_status_t status = _zx_vmo_create(size, 0, &vmo);
uptr ReservedAddressRange::Init(uptr init_size, const char *name,
uptr fixed_addr) {
- init_size = RoundUpTo(init_size, PAGE_SIZE);
+ init_size = RoundUpTo(init_size, GetPageSize());
DCHECK_EQ(os_handle_, ZX_HANDLE_INVALID);
uintptr_t base;
zx_handle_t vmar;
- zx_status_t status =
- _zx_vmar_allocate(
- _zx_vmar_root_self(),
- ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC,
- 0, init_size, &vmar, &base);
+ zx_status_t status = _zx_vmar_allocate(
+ _zx_vmar_root_self(),
+ ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0,
+ init_size, &vmar, &base);
if (status != ZX_OK)
ReportMmapFailureAndDie(init_size, name, "zx_vmar_allocate", status);
base_ = reinterpret_cast<void *>(base);
static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size,
void *base, const char *name, bool die_for_nomem) {
uptr offset = fixed_addr - reinterpret_cast<uptr>(base);
- map_size = RoundUpTo(map_size, PAGE_SIZE);
+ map_size = RoundUpTo(map_size, GetPageSize());
zx_handle_t vmo;
zx_status_t status = _zx_vmo_create(map_size, 0, &vmo);
if (status != ZX_OK) {
uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size,
const char *name) {
- return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
- name_, false);
+ return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_,
+ false);
}
uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size,
const char *name) {
- return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_,
- name_, true);
+ return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true);
}
void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) {
- if (!addr || !size) return;
- size = RoundUpTo(size, PAGE_SIZE);
+ if (!addr || !size)
+ return;
+ size = RoundUpTo(size, GetPageSize());
zx_status_t status =
_zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size);
void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
const char *mem_type) {
- CHECK_GE(size, PAGE_SIZE);
+ CHECK_GE(size, GetPageSize());
CHECK(IsPowerOfTwo(size));
CHECK(IsPowerOfTwo(alignment));
_zx_vmar_root_self(),
ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE,
addr - info.base, vmo, 0, size, &new_addr);
- if (status == ZX_OK) CHECK_EQ(new_addr, addr);
+ if (status == ZX_OK)
+ CHECK_EQ(new_addr, addr);
}
}
if (status == ZX_OK && addr != map_addr)
UnmapOrDieVmar(addr, size, _zx_vmar_root_self());
}
-// This is used on the shadow mapping, which cannot be changed.
-// Zircon doesn't have anything like MADV_DONTNEED.
-void ReleaseMemoryPagesToOS(uptr beg, uptr end) {}
+void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
+ uptr beg_aligned = RoundUpTo(beg, GetPageSize());
+ uptr end_aligned = RoundDownTo(end, GetPageSize());
+ if (beg_aligned < end_aligned) {
+ zx_handle_t root_vmar = _zx_vmar_root_self();
+ CHECK_NE(root_vmar, ZX_HANDLE_INVALID);
+ zx_status_t status =
+ _zx_vmar_op_range(root_vmar, ZX_VMAR_OP_DECOMMIT, beg_aligned,
+ end_aligned - beg_aligned, nullptr, 0);
+ CHECK_EQ(status, ZX_OK);
+ }
+}
void DumpProcessMap() {
// TODO(mcgrathr): write it
uint64_t vmo_size;
status = _zx_vmo_get_size(vmo, &vmo_size);
if (status == ZX_OK) {
- if (vmo_size < max_len) max_len = vmo_size;
- size_t map_size = RoundUpTo(max_len, PAGE_SIZE);
+ if (vmo_size < max_len)
+ max_len = vmo_size;
+ size_t map_size = RoundUpTo(max_len, GetPageSize());
uintptr_t addr;
status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0,
map_size, &addr);
}
_zx_handle_close(vmo);
}
- if (status != ZX_OK && errno_p) *errno_p = status;
+ if (status != ZX_OK && errno_p)
+ *errno_p = status;
return status == ZX_OK;
}
return true;
}
-u32 GetNumberOfCPUs() {
- return zx_system_get_num_cpus();
-}
+u32 GetNumberOfCPUs() { return zx_system_get_num_cpus(); }
uptr GetRSS() { UNIMPLEMENTED(); }
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
using namespace __sanitizer;
void __sanitizer_set_report_fd(void *fd) {
UNREACHABLE("not available on Fuchsia");
}
+
+const char *__sanitizer_get_report_path() {
+ UNREACHABLE("not available on Fuchsia");
+}
} // extern "C"
#endif // SANITIZER_FUCHSIA
size_t current; // Current index into the vector.
};
+void InitShadowBounds();
+
} // namespace __sanitizer
#endif // SANITIZER_FUCHSIA
#if SANITIZER_LINUX || SANITIZER_FUCHSIA
-# if __GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
- SANITIZER_FUCHSIA
+# if (__GLIBC_PREREQ(2, 16) || (SANITIZER_ANDROID && __ANDROID_API__ >= 21) || \
+ SANITIZER_FUCHSIA) && \
+ !SANITIZER_GO
# define SANITIZER_USE_GETAUXVAL 1
# else
# define SANITIZER_USE_GETAUXVAL 0
// (casted to void *).
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_set_report_fd(void *fd);
+ // Get the current full report file path, if a path was specified by
+ // an earlier call to __sanitizer_set_report_path. Returns null otherwise.
+ SANITIZER_INTERFACE_ATTRIBUTE
+ const char *__sanitizer_get_report_path();
typedef struct {
int coverage_sandboxed;
// TLS is handled differently on different platforms
#if SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_FREEBSD || SANITIZER_OPENBSD
+ SANITIZER_FREEBSD
# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
__attribute__((tls_model("initial-exec"))) thread_local
#else
//
// FIXME: do we have anything like this on Mac?
#ifndef SANITIZER_CAN_USE_PREINIT_ARRAY
-#if ((SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_OPENBSD || \
- SANITIZER_FUCHSIA || SANITIZER_NETBSD) && !defined(PIC)
+#if (SANITIZER_LINUX || SANITIZER_FUCHSIA || SANITIZER_NETBSD) && !defined(PIC)
#define SANITIZER_CAN_USE_PREINIT_ARRAY 1
// Before Solaris 11.4, .preinit_array is fully supported only with GNU ld.
// FIXME: Check for those conditions.
#endif
#if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_MAC || \
+ SANITIZER_MAC || \
(SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
(SANITIZER_LINUX && defined(__x86_64__))
typedef u64 OFF_T;
#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC
typedef uptr operator_new_size_type;
#else
-# if SANITIZER_OPENBSD || defined(__s390__) && !defined(__s390x__)
+# if defined(__s390__) && !defined(__s390x__)
// Special case: 31-bit s390 has unsigned long as size_t.
typedef unsigned long operator_new_size_type;
# else
// This header should NOT include any other headers to avoid portability issues.
// Common defs.
-#ifndef INLINE
-#define INLINE inline
-#endif
#define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE
#define SANITIZER_WEAK_DEFAULT_IMPL \
extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE
#define UNIMPLEMENTED() UNREACHABLE("unimplemented")
-#define COMPILER_CHECK(pred) IMPL_COMPILER_ASSERT(pred, __LINE__)
+#define COMPILER_CHECK(pred) static_assert(pred, "")
#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
-#define IMPL_PASTE(a, b) a##b
-#define IMPL_COMPILER_ASSERT(pred, line) \
- typedef char IMPL_PASTE(assertion_failed_##_, line)[2*(int)(pred)-1]
-
// Limits for integral types. We have to redefine it in case we don't
// have stdint.h (like in Visual Studio 9).
#undef __INT64_C
(void)enable_fp; \
} while (0)
+constexpr u32 kInvalidTid = -1;
+constexpr u32 kMainTid = 0;
+
} // namespace __sanitizer
namespace __asan {
namespace __hwasan {
using namespace __sanitizer;
}
+namespace __memprof {
+using namespace __sanitizer;
+}
#endif // SANITIZER_DEFS_H
// OS
void NORETURN internal__exit(int exitcode);
-unsigned int internal_sleep(unsigned int seconds);
+void internal_sleep(unsigned seconds);
+void internal_usleep(u64 useconds);
uptr internal_getpid();
uptr internal_getppid();
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
- SANITIZER_NETBSD || SANITIZER_OPENBSD
+ SANITIZER_NETBSD
#include "sanitizer_libignore.h"
#include "sanitizer_flags.h"
void LibIgnore::OnLibraryLoaded(const char *name) {
BlockingMutexLock lock(&mutex_);
// Try to match suppressions with symlink target.
- InternalScopedString buf(kMaxPathLength);
+ InternalMmapVector<char> buf(kMaxPathLength);
if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
buf[0]) {
for (uptr i = 0; i < count_; i++) {
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
#include <asm/unistd.h>
#include <sys/types.h>
#define stat kernel_stat
+#if SANITIZER_GO
+#undef st_atime
+#undef st_mtime
+#undef st_ctime
+#define st_atime st_atim
+#define st_mtime st_mtim
+#define st_ctime st_ctim
+#endif
#include <asm/stat.h>
#undef stat
#endif
#include <sys/syscall.h>
#include <sys/time.h>
#include <sys/types.h>
-#if !SANITIZER_OPENBSD
#include <ucontext.h>
-#endif
-#if SANITIZER_OPENBSD
-#include <sys/futex.h>
-#include <sys/sysctl.h>
-#endif
#include <unistd.h>
#if SANITIZER_LINUX
#endif
// Note : FreeBSD had implemented both
-// Linux and OpenBSD apis, available from
+// Linux apis, available from
// future 12.x version most likely
#if SANITIZER_LINUX && defined(__NR_getrandom)
# if !defined(GRND_NONBLOCK)
# define SANITIZER_USE_GETRANDOM 0
#endif // SANITIZER_LINUX && defined(__NR_getrandom)
-#if SANITIZER_OPENBSD
-# define SANITIZER_USE_GETENTROPY 1
+#if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
+# define SANITIZER_USE_GETENTROPY 1
#else
-# if SANITIZER_FREEBSD && __FreeBSD_version >= 1200000
-# define SANITIZER_USE_GETENTROPY 1
-# else
-# define SANITIZER_USE_GETENTROPY 0
-# endif
-#endif // SANITIZER_USE_GETENTROPY
+# define SANITIZER_USE_GETENTROPY 0
+#endif
namespace __sanitizer {
#if SANITIZER_LINUX && defined(__x86_64__)
#include "sanitizer_syscall_linux_x86_64.inc"
+#elif SANITIZER_LINUX && SANITIZER_RISCV64
+#include "sanitizer_syscall_linux_riscv64.inc"
#elif SANITIZER_LINUX && defined(__aarch64__)
#include "sanitizer_syscall_linux_aarch64.inc"
#elif SANITIZER_LINUX && defined(__arm__)
// --------------- sanitizer_libc.h
#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
-#if !SANITIZER_S390 && !SANITIZER_OPENBSD
+#if !SANITIZER_S390
uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
u64 offset) {
#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
offset / 4096);
#endif
}
-#endif // !SANITIZER_S390 && !SANITIZER_OPENBSD
+#endif // !SANITIZER_S390
-#if !SANITIZER_OPENBSD
uptr internal_munmap(void *addr, uptr length) {
return internal_syscall(SYSCALL(munmap), (uptr)addr, length);
}
+#if SANITIZER_LINUX
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+ void *new_address) {
+ return internal_syscall(SYSCALL(mremap), (uptr)old_address, old_size,
+ new_size, flags, (uptr)new_address);
+}
+#endif
+
int internal_mprotect(void *addr, uptr length, int prot) {
return internal_syscall(SYSCALL(mprotect), (uptr)addr, length, prot);
}
-#endif
+
+int internal_madvise(uptr addr, uptr length, int advice) {
+ return internal_syscall(SYSCALL(madvise), addr, length, advice);
+}
uptr internal_close(fd_t fd) {
return internal_syscall(SYSCALL(close), fd);
// Undefine compatibility macros from <sys/stat.h>
// so that they would not clash with the kernel_stat
// st_[a|m|c]time fields
+#if !SANITIZER_GO
#undef st_atime
#undef st_mtime
#undef st_ctime
+#endif
#if defined(SANITIZER_ANDROID)
// Bionic sys/stat.h defines additional macros
// for compatibility with the old NDKs and
#endif
uptr internal_stat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
}
uptr internal_lstat(const char *path, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
AT_SYMLINK_NOFOLLOW);
#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
}
uptr internal_fstat(fd_t fd, void *buf) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD || \
- SANITIZER_LINUX_USES_64BIT_SYSCALLS
-#if SANITIZER_MIPS64 && !SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD || SANITIZER_LINUX_USES_64BIT_SYSCALLS
+#if SANITIZER_MIPS64
// For mips64, fstat syscall fills buffer in the format of kernel_stat
struct kernel_stat kbuf;
int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
bufsize);
-#elif SANITIZER_OPENBSD
- return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
- bufsize);
#else
return internal_syscall(SYSCALL(readlink), (uptr)path, (uptr)buf, bufsize);
#endif
}
uptr internal_unlink(const char *path) {
-#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
#else
return internal_syscall(SYSCALL(unlink), (uptr)path);
#if defined(__riscv)
return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
(uptr)newpath, 0);
-#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS || SANITIZER_OPENBSD
+#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
(uptr)newpath);
#else
return internal_syscall(SYSCALL(sched_yield));
}
-void internal__exit(int exitcode) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
- internal_syscall(SYSCALL(exit), exitcode);
-#else
- internal_syscall(SYSCALL(exit_group), exitcode);
-#endif
- Die(); // Unreachable.
-}
-
-unsigned int internal_sleep(unsigned int seconds) {
+void internal_usleep(u64 useconds) {
struct timespec ts;
- ts.tv_sec = seconds;
- ts.tv_nsec = 0;
- int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts);
- if (res) return ts.tv_sec;
- return 0;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
+ internal_syscall(SYSCALL(nanosleep), &ts, &ts);
}
uptr internal_execve(const char *filename, char *const argv[],
}
#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+#if !SANITIZER_NETBSD
+void internal__exit(int exitcode) {
+#if SANITIZER_FREEBSD || SANITIZER_SOLARIS
+ internal_syscall(SYSCALL(exit), exitcode);
+#else
+ internal_syscall(SYSCALL(exit_group), exitcode);
+#endif
+ Die(); // Unreachable.
+}
+#endif // !SANITIZER_NETBSD
+
// ----------------- sanitizer_common.h
bool FileExists(const char *filename) {
if (ShouldMockFailureToOpen(filename))
long Tid;
thr_self(&Tid);
return Tid;
-#elif SANITIZER_OPENBSD
- return internal_syscall(SYSCALL(getthrid));
#elif SANITIZER_SOLARIS
return thr_self();
#else
return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
#elif SANITIZER_FREEBSD
return internal_syscall(SYSCALL(thr_kill2), pid, tid, sig);
-#elif SANITIZER_OPENBSD
- (void)pid;
- return internal_syscall(SYSCALL(thrkill), tid, sig, nullptr);
#elif SANITIZER_SOLARIS
(void)pid;
return thr_kill(tid, sig);
}
#endif
-#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+#if SANITIZER_GLIBC
u64 NanoTime() {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
- timeval tv;
-#else
kernel_timeval tv;
-#endif
internal_memset(&tv, 0, sizeof(tv));
internal_syscall(SYSCALL(gettimeofday), &tv, 0);
- return (u64)tv.tv_sec * 1000*1000*1000 + tv.tv_usec * 1000;
+ return (u64)tv.tv_sec * 1000 * 1000 * 1000 + tv.tv_usec * 1000;
}
-
+// Used by real_clock_gettime.
uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) {
return internal_syscall(SYSCALL(clock_gettime), clk_id, tp);
}
-#endif // !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+#elif !SANITIZER_SOLARIS && !SANITIZER_NETBSD
+u64 NanoTime() {
+ struct timespec ts;
+ clock_gettime(CLOCK_REALTIME, &ts);
+ return (u64)ts.tv_sec * 1000 * 1000 * 1000 + ts.tv_nsec;
+}
+#endif
// Like getenv, but reads env directly from /proc (on Linux) or parses the
// 'environ' array (on some others) and does not use libc. This function
// should be called first inside __asan_init.
const char *GetEnv(const char *name) {
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || \
- SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_SOLARIS
if (::environ != 0) {
uptr NameLen = internal_strlen(name);
for (char **Env = ::environ; *Env != 0; Env++) {
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_OPENBSD && \
- !SANITIZER_GO
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && !SANITIZER_GO
extern "C" {
SANITIZER_WEAK_ATTRIBUTE extern void *__libc_stack_end;
}
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD && \
- !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
static void ReadNullSepFileToArray(const char *path, char ***arr,
int arr_size) {
char *buff;
}
#endif
-#if !SANITIZER_OPENBSD
static void GetArgsAndEnv(char ***argv, char ***envp) {
#if SANITIZER_FREEBSD
// On FreeBSD, retrieving the argument and environment arrays is done via the
return envp;
}
-#endif // !SANITIZER_OPENBSD
-
#if !SANITIZER_SOLARIS
-enum MutexState {
- MtxUnlocked = 0,
- MtxLocked = 1,
- MtxSleeping = 2
-};
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+# if SANITIZER_FREEBSD
+ _umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0);
+# elif SANITIZER_NETBSD
+ sched_yield(); /* No userspace futex-like synchronization */
+# else
+ internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0);
+# endif
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+# if SANITIZER_FREEBSD
+ _umtx_op(p, UMTX_OP_WAKE, count, 0, 0);
+# elif SANITIZER_NETBSD
+ /* No userspace futex-like synchronization */
+# else
+ internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0);
+# endif
+}
+
+enum { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
BlockingMutex::BlockingMutex() {
internal_memset(this, 0, sizeof(*this));
}
}
-void BlockingMutex::CheckLocked() {
- atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
+void BlockingMutex::CheckLocked() const {
+ auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
}
-#endif // !SANITIZER_SOLARIS
+# endif // !SANITIZER_SOLARIS
// ----------------- sanitizer_linux.h
// The actual size of this structure is specified by d_reclen.
// 32-bit syscall here.
#if SANITIZER_NETBSD
// Not used
-#elif SANITIZER_OPENBSD
-// struct dirent is different for Linux and us. At this moment, we use only
-// d_fileno (Linux call this d_ino), d_reclen, and d_name.
-struct linux_dirent {
- u64 d_ino; // d_fileno
- u16 d_reclen;
- u16 d_namlen; // not used
- u8 d_type; // not used
- char d_name[NAME_MAX + 1];
-};
#else
struct linux_dirent {
-#if SANITIZER_X32 || defined(__aarch64__)
+#if SANITIZER_X32 || defined(__aarch64__) || SANITIZER_RISCV64
u64 d_ino;
u64 d_off;
#else
unsigned long d_off;
#endif
unsigned short d_reclen;
-#ifdef __aarch64__
+#if defined(__aarch64__) || SANITIZER_RISCV64
unsigned char d_type;
#endif
char d_name[256];
#endif
}
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
int internal_sysctl(const int *name, unsigned int namelen, void *oldp,
uptr *oldlenp, const void *newp, uptr newlen) {
-#if SANITIZER_OPENBSD
- return sysctl(name, namelen, oldp, (size_t *)oldlenp, (void *)newp,
- (size_t)newlen);
-#else
return internal_syscall(SYSCALL(__sysctl), name, namelen, oldp,
(size_t *)oldlenp, newp, (size_t)newlen);
-#endif
}
-#if SANITIZER_FREEBSD
int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
const void *newp, uptr newlen) {
- static decltype(sysctlbyname) *real = nullptr;
- if (!real)
- real = (decltype(sysctlbyname) *)dlsym(RTLD_NEXT, "sysctlbyname");
- CHECK(real);
- return real(sname, oldp, (size_t *)oldlenp, newp, (size_t)newlen);
-}
+ // Note: this function can be called during startup, so we need to avoid
+ // calling any interceptable functions. On FreeBSD >= 1300045 sysctlbyname()
+ // is a real syscall, but for older versions it calls sysctlnametomib()
+ // followed by sysctl(). To avoid calling the intercepted version and
+ // asserting if this happens during startup, call the real sysctlnametomib()
+ // followed by internal_sysctl() if the syscall is not available.
+#ifdef SYS___sysctlbyname
+ return internal_syscall(SYSCALL(__sysctlbyname), sname,
+ internal_strlen(sname), oldp, (size_t *)oldlenp, newp,
+ (size_t)newlen);
+#else
+ static decltype(sysctlnametomib) *real_sysctlnametomib = nullptr;
+ if (!real_sysctlnametomib)
+ real_sysctlnametomib =
+ (decltype(sysctlnametomib) *)dlsym(RTLD_NEXT, "sysctlnametomib");
+ CHECK(real_sysctlnametomib);
+
+ int oid[CTL_MAXNAME];
+ size_t len = CTL_MAXNAME;
+ if (real_sysctlnametomib(sname, oid, &len) == -1)
+ return (-1);
+ return internal_sysctl(oid, len, oldp, oldlenp, newp, newlen);
#endif
+}
#endif
#if SANITIZER_LINUX
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset) {
-#if SANITIZER_FREEBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD
return internal_syscall(SYSCALL(sigprocmask), how, set, oldset);
#else
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
- k_set->sig[idx] &= ~(1 << bit);
+ k_set->sig[idx] &= ~((uptr)1 << bit);
}
bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
__sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set;
const uptr idx = signum / (sizeof(k_set->sig[0]) * 8);
const uptr bit = signum % (sizeof(k_set->sig[0]) * 8);
- return k_set->sig[idx] & (1 << bit);
+ return k_set->sig[idx] & ((uptr)1 << bit);
}
#elif SANITIZER_FREEBSD
void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
#endif // SANITIZER_WORDSIZE == 32
uptr GetMaxVirtualAddress() {
-#if (SANITIZER_NETBSD || SANITIZER_OPENBSD) && defined(__x86_64__)
+#if SANITIZER_NETBSD && defined(__x86_64__)
return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
#elif SANITIZER_WORDSIZE == 64
# if defined(__powerpc64__) || defined(__aarch64__)
// This should (does) work for both PowerPC64 Endian modes.
// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+#elif SANITIZER_RISCV64
+ return (1ULL << 38) - 1;
# elif defined(__mips64)
return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
# elif defined(__s390x__)
}
#endif // !SANITIZER_ANDROID
-#if !SANITIZER_OPENBSD
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
#if SANITIZER_SOLARIS
const char *default_module_name = getexecname();
return module_name_len;
#endif
}
-#endif // !SANITIZER_OPENBSD
uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) {
#if SANITIZER_LINUX
// Call cb for each region mapped by map.
void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr)) {
CHECK_NE(map, nullptr);
-#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD
typedef ElfW(Phdr) Elf_Phdr;
typedef ElfW(Ehdr) Elf_Ehdr;
-#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#endif // !SANITIZER_FREEBSD
char *base = (char *)map->l_addr;
Elf_Ehdr *ehdr = (Elf_Ehdr *)base;
char *phdrs = base + ehdr->e_phoff;
: "memory", "$29" );
return res;
}
+#elif SANITIZER_RISCV64
+uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
+ int *parent_tidptr, void *newtls, int *child_tidptr) {
+ if (!fn || !child_stack)
+ return -EINVAL;
+
+ CHECK_EQ(0, (uptr)child_stack % 16);
+
+ register int res __asm__("a0");
+ register int __flags __asm__("a0") = flags;
+ register void *__stack __asm__("a1") = child_stack;
+ register int *__ptid __asm__("a2") = parent_tidptr;
+ register void *__tls __asm__("a3") = newtls;
+ register int *__ctid __asm__("a4") = child_tidptr;
+ register int (*__fn)(void *) __asm__("a5") = fn;
+ register void *__arg __asm__("a6") = arg;
+ register int nr_clone __asm__("a7") = __NR_clone;
+
+ __asm__ __volatile__(
+ "ecall\n"
+
+ /* if (a0 != 0)
+ * return a0;
+ */
+ "bnez a0, 1f\n"
+
+ // In the child, now. Call "fn(arg)".
+ "mv a0, a6\n"
+ "jalr a5\n"
+
+ // Call _exit(a0).
+ "addi a7, zero, %9\n"
+ "ecall\n"
+ "1:\n"
+
+ : "=r"(res)
+ : "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__tls), "r"(__ctid),
+ "r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)
+ : "memory");
+ return res;
+}
#elif defined(__aarch64__)
uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
int *parent_tidptr, void *newtls, int *child_tidptr) {
}
#endif
-#if SANITIZER_OPENBSD
-using Context = sigcontext;
-#else
using Context = ucontext_t;
-#endif
SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
Context *ucontext = (Context *)context;
uptr err = ucontext->uc_mcontext.mc_err;
#elif SANITIZER_NETBSD
uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
-#elif SANITIZER_OPENBSD
- uptr err = ucontext->sc_err;
#elif SANITIZER_SOLARIS && defined(__i386__)
const int Err = 13;
uptr err = ucontext->uc_mcontext.gregs[Err];
*pc = ucontext->uc_mcontext.mc_rip;
*bp = ucontext->uc_mcontext.mc_rbp;
*sp = ucontext->uc_mcontext.mc_rsp;
-#elif SANITIZER_OPENBSD
- sigcontext *ucontext = (sigcontext *)context;
- *pc = ucontext->sc_rip;
- *bp = ucontext->sc_rbp;
- *sp = ucontext->sc_rsp;
# else
ucontext_t *ucontext = (ucontext_t*)context;
*pc = ucontext->uc_mcontext.gregs[REG_RIP];
*pc = ucontext->uc_mcontext.mc_eip;
*bp = ucontext->uc_mcontext.mc_ebp;
*sp = ucontext->uc_mcontext.mc_esp;
-#elif SANITIZER_OPENBSD
- sigcontext *ucontext = (sigcontext *)context;
- *pc = ucontext->sc_eip;
- *bp = ucontext->sc_ebp;
- *sp = ucontext->sc_esp;
# else
ucontext_t *ucontext = (ucontext_t*)context;
# if SANITIZER_SOLARIS
#endif
}
-void PrintModuleMap() { }
-
void CheckNoDeepBind(const char *filename, int flag) {
#ifdef RTLD_DEEPBIND
if (flag & RTLD_DEEPBIND) {
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform_limits_freebsd.h"
#include "sanitizer_platform_limits_netbsd.h"
-#include "sanitizer_platform_limits_openbsd.h"
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_platform_limits_solaris.h"
#include "sanitizer_posix.h"
uptr internal_sigaltstack(const void* ss, void* oss);
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset);
+#if SANITIZER_GLIBC
uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
+#endif
// Linux-only syscalls.
#if SANITIZER_LINUX
// internal_sigaction instead.
int internal_sigaction_norestorer(int signum, const void *act, void *oldact);
void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
-#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \
- || defined(__powerpc64__) || defined(__s390__) || defined(__i386__) \
- || defined(__arm__)
+#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \
+ defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \
+ defined(__arm__) || SANITIZER_RISCV64
uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
int *parent_tidptr, void *newtls, int *child_tidptr);
#endif
// Exposed for testing.
uptr ThreadDescriptorSize();
uptr ThreadSelf();
-uptr ThreadSelfOffset();
// Matches a library's file name against a base name (stripping path and version
// information).
// Releases memory pages entirely within the [beg, end] address range.
// The pages no longer count toward RSS; reads are guaranteed to return 0.
// Requires (but does not verify!) that pages are MAP_PRIVATE.
-INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
// man madvise on Linux promises zero-fill for anonymous private pages.
// Testing shows the same behaviour for private (but not anonymous) mappings
// of shm_open() files, as long as the underlying file is untouched.
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
#include "sanitizer_allocator_internal.h"
#include "sanitizer_atomic.h"
#include "sanitizer_placement_new.h"
#include "sanitizer_procmaps.h"
+#if SANITIZER_NETBSD
+#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
+#endif
+
#include <dlfcn.h> // for dlsym()
#include <link.h>
#include <pthread.h>
#include <signal.h>
+#include <sys/mman.h>
#include <sys/resource.h>
#include <syslog.h>
#include <osreldate.h>
#include <sys/sysctl.h>
#define pthread_getattr_np pthread_attr_get_np
-#endif
-
-#if SANITIZER_OPENBSD
-#include <pthread_np.h>
-#include <sys/sysctl.h>
+// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
+// that, it was never implemented. So just define it to zero.
+#undef MAP_NORESERVE
+#define MAP_NORESERVE 0
#endif
#if SANITIZER_NETBSD
CHECK_EQ(thr_stksegment(&ss), 0);
stacksize = ss.ss_size;
stackaddr = (char *)ss.ss_sp - stacksize;
-#elif SANITIZER_OPENBSD
- stack_t sattr;
- CHECK_EQ(pthread_stackseg_np(pthread_self(), &sattr), 0);
- stackaddr = sattr.ss_sp;
- stacksize = sattr.ss_size;
#else // !SANITIZER_SOLARIS
pthread_attr_t attr;
pthread_attr_init(&attr);
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
pthread_attr_destroy(&attr);
-#endif // SANITIZER_SOLARIS
+#endif // SANITIZER_SOLARIS
*stack_top = (uptr)stackaddr + stacksize;
*stack_bottom = (uptr)stackaddr;
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO && \
- !SANITIZER_NETBSD && !SANITIZER_OPENBSD && !SANITIZER_SOLARIS
-static uptr g_tls_size;
-
-#ifdef __i386__
-# define CHECK_GET_TLS_STATIC_INFO_VERSION (!__GLIBC_PREREQ(2, 27))
-#else
-# define CHECK_GET_TLS_STATIC_INFO_VERSION 0
-#endif
-
-#if CHECK_GET_TLS_STATIC_INFO_VERSION
-# define DL_INTERNAL_FUNCTION __attribute__((regparm(3), stdcall))
-#else
-# define DL_INTERNAL_FUNCTION
-#endif
-
-namespace {
-struct GetTlsStaticInfoCall {
- typedef void (*get_tls_func)(size_t*, size_t*);
-};
-struct GetTlsStaticInfoRegparmCall {
- typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION;
-};
-
-template <typename T>
-void CallGetTls(void* ptr, size_t* size, size_t* align) {
- typename T::get_tls_func get_tls;
- CHECK_EQ(sizeof(get_tls), sizeof(ptr));
- internal_memcpy(&get_tls, &ptr, sizeof(ptr));
- CHECK_NE(get_tls, 0);
- get_tls(size, align);
-}
-
-bool CmpLibcVersion(int major, int minor, int patch) {
- int ma;
- int mi;
- int pa;
- if (!GetLibcVersion(&ma, &mi, &pa))
- return false;
- if (ma > major)
- return true;
- if (ma < major)
- return false;
- if (mi > minor)
- return true;
- if (mi < minor)
- return false;
- return pa >= patch;
-}
-
-} // namespace
+// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
+// #19826) so dlpi_tls_data cannot be used.
+//
+// musl before 1.2.3 and FreeBSD as of 12.2 incorrectly set dlpi_tls_data to
+// the TLS initialization image
+// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=254774
+__attribute__((unused)) static int g_use_dlpi_tls_data;
+#if SANITIZER_GLIBC && !SANITIZER_GO
+__attribute__((unused)) static size_t g_tls_size;
void InitTlsSize() {
- // all current supported platforms have 16 bytes stack alignment
- const size_t kStackAlign = 16;
- void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
- size_t tls_size = 0;
- size_t tls_align = 0;
- // On i?86, _dl_get_tls_static_info used to be internal_function, i.e.
- // __attribute__((regparm(3), stdcall)) before glibc 2.27 and is normal
- // function in 2.27 and later.
- if (CHECK_GET_TLS_STATIC_INFO_VERSION && !CmpLibcVersion(2, 27, 0))
- CallGetTls<GetTlsStaticInfoRegparmCall>(get_tls_static_info_ptr,
- &tls_size, &tls_align);
- else
- CallGetTls<GetTlsStaticInfoCall>(get_tls_static_info_ptr,
- &tls_size, &tls_align);
- if (tls_align < kStackAlign)
- tls_align = kStackAlign;
- g_tls_size = RoundUpTo(tls_size, tls_align);
+ int major, minor, patch;
+ g_use_dlpi_tls_data =
+ GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
+
+#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__)
+ void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
+ size_t tls_align;
+ ((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
+#endif
}
#else
void InitTlsSize() { }
-#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO &&
- // !SANITIZER_NETBSD && !SANITIZER_SOLARIS
+#endif // SANITIZER_GLIBC && !SANITIZER_GO
-#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) || \
- defined(__aarch64__) || defined(__powerpc64__) || defined(__s390__) || \
- defined(__arm__)) && \
- SANITIZER_LINUX && !SANITIZER_ANDROID
+// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
+// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
+// to get the pointer to thread-specific data keys in the thread control block.
+#if (SANITIZER_FREEBSD || SANITIZER_LINUX) && !SANITIZER_ANDROID
// sizeof(struct pthread) from glibc.
static atomic_uintptr_t thread_descriptor_size;
val = FIRST_32_SECOND_64(1168, 2288);
else if (minor <= 14)
val = FIRST_32_SECOND_64(1168, 2304);
- else
+ else if (minor < 32) // Unknown version
val = FIRST_32_SECOND_64(1216, 2304);
+ else // minor == 32
+ val = FIRST_32_SECOND_64(1344, 2496);
}
+#elif defined(__s390__) || defined(__sparc__)
+ // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
+ // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
+ // changed since 2007-05. Technically this applies to i386/x86_64 as well but
+ // we call _dl_get_tls_static_info and need the precise size of struct
+ // pthread.
+ return FIRST_32_SECOND_64(524, 1552);
#elif defined(__mips__)
// TODO(sagarthakur): add more values as per different glibc versions.
val = FIRST_32_SECOND_64(1152, 1776);
+#elif SANITIZER_RISCV64
+ int major;
+ int minor;
+ int patch;
+ if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
+ // TODO: consider adding an optional runtime check for an unknown (untested)
+ // glibc version
+ if (minor <= 28) // WARNING: the highest tested version is 2.29
+ val = 1772; // no guarantees for this one
+ else if (minor <= 31)
+ val = 1772; // tested against glibc 2.29, 2.31
+ else
+ val = 1936; // tested against glibc 2.32
+ }
+
#elif defined(__aarch64__)
// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
val = 1776;
#elif defined(__powerpc64__)
val = 1776; // from glibc.ppc64le 2.20-8.fc21
-#elif defined(__s390__)
- val = FIRST_32_SECOND_64(1152, 1776); // valid for glibc 2.22
#endif
if (val)
atomic_store_relaxed(&thread_descriptor_size, val);
return val;
}
-// The offset at which pointer to self is located in the thread descriptor.
-const uptr kThreadSelfOffset = FIRST_32_SECOND_64(8, 16);
-
-uptr ThreadSelfOffset() {
- return kThreadSelfOffset;
-}
-
-#if defined(__mips__) || defined(__powerpc64__)
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
// head structure. It lies before the static tls blocks.
static uptr TlsPreTcbSize() {
-# if defined(__mips__)
+#if defined(__mips__)
const uptr kTcbHead = 16; // sizeof (tcbhead_t)
-# elif defined(__powerpc64__)
+#elif defined(__powerpc64__)
const uptr kTcbHead = 88; // sizeof (tcbhead_t)
-# endif
+#elif SANITIZER_RISCV64
+ const uptr kTcbHead = 16; // sizeof (tcbhead_t)
+#endif
const uptr kTlsAlign = 16;
const uptr kTlsPreTcbSize =
RoundUpTo(ThreadDescriptorSize() + kTcbHead, kTlsAlign);
}
#endif
-uptr ThreadSelf() {
- uptr descr_addr;
-# if defined(__i386__)
- asm("mov %%gs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-# elif defined(__x86_64__)
- asm("mov %%fs:%c1,%0" : "=r"(descr_addr) : "i"(kThreadSelfOffset));
-# elif defined(__mips__)
- // MIPS uses TLS variant I. The thread pointer (in hardware register $29)
- // points to the end of the TCB + 0x7000. The pthread_descr structure is
- // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
- // TCB and the size of pthread_descr.
- const uptr kTlsTcbOffset = 0x7000;
- uptr thread_pointer;
- asm volatile(".set push;\
- .set mips64r2;\
- rdhwr %0,$29;\
- .set pop" : "=r" (thread_pointer));
- descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize();
-# elif defined(__aarch64__) || defined(__arm__)
- descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
- ThreadDescriptorSize();
-# elif defined(__s390__)
- descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer());
-# elif defined(__powerpc64__)
- // PPC64LE uses TLS variant I. The thread pointer (in GPR 13)
- // points to the end of the TCB + 0x7000. The pthread_descr structure is
- // immediately in front of the TCB. TlsPreTcbSize() includes the size of the
- // TCB and the size of pthread_descr.
- const uptr kTlsTcbOffset = 0x7000;
- uptr thread_pointer;
- asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset));
- descr_addr = thread_pointer - TlsPreTcbSize();
-# else
-# error "unsupported CPU arch"
-# endif
- return descr_addr;
-}
-#endif // (x86_64 || i386 || MIPS) && SANITIZER_LINUX
+#if !SANITIZER_GO
+namespace {
+struct TlsBlock {
+ uptr begin, end, align;
+ size_t tls_modid;
+ bool operator<(const TlsBlock &rhs) const { return begin < rhs.begin; }
+};
+} // namespace
-#if SANITIZER_FREEBSD
-static void **ThreadSelfSegbase() {
- void **segbase = 0;
-# if defined(__i386__)
- // sysarch(I386_GET_GSBASE, segbase);
- __asm __volatile("mov %%gs:0, %0" : "=r" (segbase));
-# elif defined(__x86_64__)
- // sysarch(AMD64_GET_FSBASE, segbase);
- __asm __volatile("movq %%fs:0, %0" : "=r" (segbase));
-# else
-# error "unsupported CPU arch"
-# endif
- return segbase;
+#ifdef __s390__
+extern "C" uptr __tls_get_offset(void *arg);
+
+static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
+ // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an
+ // offset of a struct tls_index inside GOT. We don't possess either of the
+ // two, so violate the letter of the "ELF Handling For Thread-Local
+ // Storage" document and assume that the implementation just dereferences
+ // %r2 + %r12.
+ uptr tls_index[2] = {ti_module, ti_offset};
+ register uptr r2 asm("2") = 0;
+ register void *r12 asm("12") = tls_index;
+ asm("basr %%r14, %[__tls_get_offset]"
+ : "+r"(r2)
+ : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12)
+ : "memory", "cc", "0", "1", "3", "4", "5", "14");
+ return r2;
}
+#else
+extern "C" void *__tls_get_addr(size_t *);
+#endif
-uptr ThreadSelf() {
- return (uptr)ThreadSelfSegbase()[2];
+static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
+ void *data) {
+ if (!info->dlpi_tls_modid)
+ return 0;
+ uptr begin = (uptr)info->dlpi_tls_data;
+ if (!g_use_dlpi_tls_data) {
+ // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
+ // and FreeBSD.
+#ifdef __s390__
+ begin = (uptr)__builtin_thread_pointer() +
+ TlsGetOffset(info->dlpi_tls_modid, 0);
+#else
+ size_t mod_and_off[2] = {info->dlpi_tls_modid, 0};
+ begin = (uptr)__tls_get_addr(mod_and_off);
+#endif
+ }
+ for (unsigned i = 0; i != info->dlpi_phnum; ++i)
+ if (info->dlpi_phdr[i].p_type == PT_TLS) {
+ static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
+ TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz,
+ info->dlpi_phdr[i].p_align, info->dlpi_tls_modid});
+ break;
+ }
+ return 0;
}
-#endif // SANITIZER_FREEBSD
+
+__attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,
+ uptr *align) {
+ InternalMmapVector<TlsBlock> ranges;
+ dl_iterate_phdr(CollectStaticTlsBlocks, &ranges);
+ uptr len = ranges.size();
+ Sort(ranges.begin(), len);
+ // Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS,
+ // this module is guaranteed to exist and is one of the initially loaded
+ // modules.
+ uptr one = 0;
+ while (one != len && ranges[one].tls_modid != 1) ++one;
+ if (one == len) {
+ // This may happen with musl if no module uses PT_TLS.
+ *addr = 0;
+ *size = 0;
+ *align = 1;
+ return;
+ }
+ // Find the maximum consecutive ranges. We consider two modules consecutive if
+ // the gap is smaller than the alignment. The dynamic loader places static TLS
+ // blocks this way not to waste space.
+ uptr l = one;
+ *align = ranges[l].align;
+ while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align)
+ *align = Max(*align, ranges[--l].align);
+ uptr r = one + 1;
+ while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align)
+ *align = Max(*align, ranges[r++].align);
+ *addr = ranges[l].begin;
+ *size = ranges[r - 1].end - ranges[l].begin;
+}
+#endif // !SANITIZER_GO
+#endif // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||
+ // SANITIZER_LINUX) && !SANITIZER_ANDROID
#if SANITIZER_NETBSD
static struct tls_tcb * ThreadSelfTlsTcb() {
- return (struct tls_tcb *)_lwp_getprivate();
+ struct tls_tcb *tcb = nullptr;
+#ifdef __HAVE___LWP_GETTCB_FAST
+ tcb = (struct tls_tcb *)__lwp_gettcb_fast();
+#elif defined(__HAVE___LWP_GETPRIVATE_FAST)
+ tcb = (struct tls_tcb *)__lwp_getprivate_fast();
+#endif
+ return tcb;
}
uptr ThreadSelf() {
}
#endif // SANITIZER_NETBSD
+#if SANITIZER_ANDROID
+// Bionic provides this API since S.
+extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_get_static_tls_bounds(void **,
+ void **);
+#endif
+
#if !SANITIZER_GO
static void GetTls(uptr *addr, uptr *size) {
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-# if defined(__x86_64__) || defined(__i386__) || defined(__s390__)
- *addr = ThreadSelf();
- *size = GetTlsSize();
+#if SANITIZER_ANDROID
+ if (&__libc_get_static_tls_bounds) {
+ void *start_addr;
+ void *end_addr;
+ __libc_get_static_tls_bounds(&start_addr, &end_addr);
+ *addr = reinterpret_cast<uptr>(start_addr);
+ *size =
+ reinterpret_cast<uptr>(end_addr) - reinterpret_cast<uptr>(start_addr);
+ } else {
+ *addr = 0;
+ *size = 0;
+ }
+#elif SANITIZER_GLIBC && defined(__x86_64__)
+ // For aarch64 and x86-64, use an O(1) approach which requires relatively
+ // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
+ asm("mov %%fs:16,%0" : "=r"(*addr));
+ *size = g_tls_size;
*addr -= *size;
*addr += ThreadDescriptorSize();
-# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) \
- || defined(__arm__)
- *addr = ThreadSelf();
- *size = GetTlsSize();
-# else
- *addr = 0;
- *size = 0;
-# endif
-#elif SANITIZER_FREEBSD
- void** segbase = ThreadSelfSegbase();
- *addr = 0;
- *size = 0;
- if (segbase != 0) {
- // tcbalign = 16
- // tls_size = round(tls_static_space, tcbalign);
- // dtv = segbase[1];
- // dtv[2] = segbase - tls_static_space;
- void **dtv = (void**) segbase[1];
- *addr = (uptr) dtv[2];
- *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]);
+#elif SANITIZER_GLIBC && defined(__aarch64__)
+ *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
+ ThreadDescriptorSize();
+ *size = g_tls_size + ThreadDescriptorSize();
+#elif SANITIZER_GLIBC && defined(__powerpc64__)
+ // Workaround for glibc<2.25(?). 2.27 is known to not need this.
+ uptr tp;
+ asm("addi %0,13,-0x7000" : "=r"(tp));
+ const uptr pre_tcb_size = TlsPreTcbSize();
+ *addr = tp - pre_tcb_size;
+ *size = g_tls_size + pre_tcb_size;
+#elif SANITIZER_FREEBSD || SANITIZER_LINUX
+ uptr align;
+ GetStaticTlsBoundary(addr, size, &align);
+#if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \
+ defined(__sparc__)
+ if (SANITIZER_GLIBC) {
+#if defined(__x86_64__) || defined(__i386__)
+ align = Max<uptr>(align, 64);
+#else
+ align = Max<uptr>(align, 16);
+#endif
}
+ const uptr tp = RoundUpTo(*addr + *size, align);
+
+ // lsan requires the range to additionally cover the static TLS surplus
+ // (elf/dl-tls.c defines 1664). Otherwise there may be false positives for
+ // allocations only referenced by tls in dynamically loaded modules.
+ if (SANITIZER_GLIBC)
+ *size += 1644;
+ else if (SANITIZER_FREEBSD)
+ *size += 128; // RTLD_STATIC_TLS_EXTRA
+
+ // Extend the range to include the thread control block. On glibc, lsan needs
+ // the range to include pthread::{specific_1stblock,specific} so that
+ // allocations only referenced by pthread_setspecific can be scanned. This may
+ // underestimate by at most TLS_TCB_ALIGN-1 bytes but it should be fine
+ // because the number of bytes after pthread::specific is larger.
+ *addr = tp - RoundUpTo(*size, align);
+ *size = tp - *addr + ThreadDescriptorSize();
+#else
+ if (SANITIZER_GLIBC)
+ *size += 1664;
+ else if (SANITIZER_FREEBSD)
+ *size += 128; // RTLD_STATIC_TLS_EXTRA
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
+ const uptr pre_tcb_size = TlsPreTcbSize();
+ *addr -= pre_tcb_size;
+ *size += pre_tcb_size;
+#else
+ // arm and aarch64 reserve two words at TP, so this underestimates the range.
+ // However, this is sufficient for the purpose of finding the pointers to
+ // thread-specific data keys.
+ const uptr tcb_size = ThreadDescriptorSize();
+ *addr -= tcb_size;
+ *size += tcb_size;
+#endif
+#endif
#elif SANITIZER_NETBSD
struct tls_tcb * const tcb = ThreadSelfTlsTcb();
*addr = 0;
*addr = (uptr)tcb->tcb_dtv[1];
}
}
-#elif SANITIZER_OPENBSD
- *addr = 0;
- *size = 0;
-#elif SANITIZER_ANDROID
- *addr = 0;
- *size = 0;
#elif SANITIZER_SOLARIS
// FIXME
*addr = 0;
*size = 0;
#else
-# error "Unknown OS"
+#error "Unknown OS"
#endif
}
#endif
#if !SANITIZER_GO
uptr GetTlsSize() {
-#if SANITIZER_FREEBSD || SANITIZER_ANDROID || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
+ SANITIZER_SOLARIS
uptr addr, size;
GetTls(&addr, &size);
return size;
-#elif defined(__mips__) || defined(__powerpc64__)
- return RoundUpTo(g_tls_size + TlsPreTcbSize(), 16);
#else
- return g_tls_size;
+ return 0;
#endif
}
#endif
if (!main) {
// If stack and tls intersect, make them non-intersecting.
if (*tls_addr > *stk_addr && *tls_addr < *stk_addr + *stk_size) {
- CHECK_GT(*tls_addr + *tls_size, *stk_addr);
- CHECK_LE(*tls_addr + *tls_size, *stk_addr + *stk_size);
- *stk_size -= *tls_size;
- *tls_addr = *stk_addr + *stk_size;
+ if (*stk_addr + *stk_size < *tls_addr + *tls_size)
+ *tls_size = *stk_addr + *stk_size - *tls_addr;
+ *stk_size = *tls_addr - *stk_addr;
}
}
#endif
}
-#if !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#if !SANITIZER_FREEBSD
typedef ElfW(Phdr) Elf_Phdr;
-#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
+#elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2
#define Elf_Phdr XElf32_Phdr
#define dl_phdr_info xdl_phdr_info
#define dl_iterate_phdr(c, b) xdl_iterate_phdr((c), (b))
-#endif // !SANITIZER_FREEBSD && !SANITIZER_OPENBSD
+#endif // !SANITIZER_FREEBSD
struct DlIteratePhdrData {
InternalMmapVectorNoCtor<LoadedModule> *modules;
bool first;
};
-static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
- DlIteratePhdrData *data = (DlIteratePhdrData*)arg;
- InternalScopedString module_name(kMaxPathLength);
- if (data->first) {
- data->first = false;
- // First module is the binary itself.
- ReadBinaryNameCached(module_name.data(), module_name.size());
- } else if (info->dlpi_name) {
- module_name.append("%s", info->dlpi_name);
- }
+static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
+ InternalMmapVectorNoCtor<LoadedModule> *modules) {
if (module_name[0] == '\0')
return 0;
LoadedModule cur_module;
- cur_module.set(module_name.data(), info->dlpi_addr);
+ cur_module.set(module_name, info->dlpi_addr);
for (int i = 0; i < (int)info->dlpi_phnum; i++) {
const Elf_Phdr *phdr = &info->dlpi_phdr[i];
if (phdr->p_type == PT_LOAD) {
writable);
}
}
- data->modules->push_back(cur_module);
+ modules->push_back(cur_module);
+ return 0;
+}
+
+static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
+ DlIteratePhdrData *data = (DlIteratePhdrData *)arg;
+ if (data->first) {
+ InternalMmapVector<char> module_name(kMaxPathLength);
+ data->first = false;
+ // First module is the binary itself.
+ ReadBinaryNameCached(module_name.data(), module_name.size());
+ return AddModuleSegments(module_name.data(), info, data->modules);
+ }
+
+ if (info->dlpi_name) {
+ InternalScopedString module_name;
+ module_name.append("%s", info->dlpi_name);
+ return AddModuleSegments(module_name.data(), info, data->modules);
+ }
+
return 0;
}
// sysconf(_SC_NPROCESSORS_{CONF,ONLN}) cannot be used on most platforms as
// they allocate memory.
u32 GetNumberOfCPUs() {
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
u32 ncpu;
int req[2];
uptr len = sizeof(ncpu);
#if SANITIZER_LINUX
-# if SANITIZER_ANDROID
+#if SANITIZER_ANDROID
static atomic_uint8_t android_log_initialized;
void AndroidLogInit() {
if (&android_set_abort_message)
android_set_abort_message(str);
}
-# else
+#else
void AndroidLogInit() {}
static bool ShouldLogAfterPrintf() { return true; }
void WriteOneLineToSyslog(const char *s) { syslog(LOG_INFO, "%s", s); }
void SetAbortMessage(const char *str) {}
-# endif // SANITIZER_ANDROID
+#endif // SANITIZER_ANDROID
void LogMessageOnPrintf(const char *str) {
if (common_flags()->log_to_syslog && ShouldLogAfterPrintf())
#endif // SANITIZER_LINUX
-#if SANITIZER_LINUX && !SANITIZER_GO
+#if SANITIZER_GLIBC && !SANITIZER_GO
// glibc crashes when using clock_gettime from a preinit_array function as the
// vDSO function pointers haven't been initialized yet. __progname is
// initialized after the vDSO function pointers, so if it exists, is not null
// and is not empty, we can use clock_gettime.
extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
-INLINE bool CanUseVDSO() {
- // Bionic is safe, it checks for the vDSO function pointers to be initialized.
- if (SANITIZER_ANDROID)
- return true;
- if (&__progname && __progname && *__progname)
- return true;
- return false;
-}
+inline bool CanUseVDSO() { return &__progname && __progname && *__progname; }
// MonotonicNanoTime is a timing function that can leverage the vDSO by calling
// clock_gettime. real_clock_gettime only exists if clock_gettime is
return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
}
#else
-// Non-Linux & Go always use the syscall.
+// Non-glibc & Go always use the regular function.
u64 MonotonicNanoTime() {
timespec ts;
- internal_clock_gettime(CLOCK_MONOTONIC, &ts);
+ clock_gettime(CLOCK_MONOTONIC, &ts);
return (u64)ts.tv_sec * (1000ULL * 1000 * 1000) + ts.tv_nsec;
}
-#endif // SANITIZER_LINUX && !SANITIZER_GO
+#endif // SANITIZER_GLIBC && !SANITIZER_GO
-#if !SANITIZER_OPENBSD
void ReExec() {
const char *pathname = "/proc/self/exe";
Printf("execve failed, errno %d\n", rverrno);
Die();
}
-#endif // !SANITIZER_OPENBSD
+
+void UnmapFromTo(uptr from, uptr to) {
+ if (to == from)
+ return;
+ CHECK(to >= from);
+ uptr res = internal_munmap(reinterpret_cast<void *>(from), to - from);
+ if (UNLIKELY(internal_iserror(res))) {
+ Report("ERROR: %s failed to unmap 0x%zx (%zd) bytes at address %p\n",
+ SanitizerToolName, to - from, to - from, (void *)from);
+ CHECK("unable to unmap" && 0);
+ }
+}
+
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment,
+ UNUSED uptr &high_mem_end) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment =
+ Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
+ const uptr left_padding =
+ Max<uptr>(granularity, 1ULL << min_shadow_base_alignment);
+
+ const uptr shadow_size = RoundUpTo(shadow_size_bytes, granularity);
+ const uptr map_size = shadow_size + left_padding + alignment;
+
+ const uptr map_start = (uptr)MmapNoAccess(map_size);
+ CHECK_NE(map_start, ~(uptr)0);
+
+ const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
+
+ UnmapFromTo(map_start, shadow_start - left_padding);
+ UnmapFromTo(shadow_start + shadow_size, map_start + map_size);
+
+ return shadow_start;
+}
+
+static uptr MmapSharedNoReserve(uptr addr, uptr size) {
+ return internal_mmap(
+ reinterpret_cast<void *>(addr), size, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+}
+
+static uptr MremapCreateAlias(uptr base_addr, uptr alias_addr,
+ uptr alias_size) {
+#if SANITIZER_LINUX
+ return internal_mremap(reinterpret_cast<void *>(base_addr), 0, alias_size,
+ MREMAP_MAYMOVE | MREMAP_FIXED,
+ reinterpret_cast<void *>(alias_addr));
+#else
+ CHECK(false && "mremap is not supported outside of Linux");
+ return 0;
+#endif
+}
+
+static void CreateAliases(uptr start_addr, uptr alias_size, uptr num_aliases) {
+ uptr total_size = alias_size * num_aliases;
+ uptr mapped = MmapSharedNoReserve(start_addr, total_size);
+ CHECK_EQ(mapped, start_addr);
+
+ for (uptr i = 1; i < num_aliases; ++i) {
+ uptr alias_addr = start_addr + i * alias_size;
+ CHECK_EQ(MremapCreateAlias(start_addr, alias_addr, alias_size), alias_addr);
+ }
+}
+
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+ uptr num_aliases, uptr ring_buffer_size) {
+ CHECK_EQ(alias_size & (alias_size - 1), 0);
+ CHECK_EQ(num_aliases & (num_aliases - 1), 0);
+ CHECK_EQ(ring_buffer_size & (ring_buffer_size - 1), 0);
+
+ const uptr granularity = GetMmapGranularity();
+ shadow_size = RoundUpTo(shadow_size, granularity);
+ CHECK_EQ(shadow_size & (shadow_size - 1), 0);
+
+ const uptr alias_region_size = alias_size * num_aliases;
+ const uptr alignment =
+ 2 * Max(Max(shadow_size, alias_region_size), ring_buffer_size);
+ const uptr left_padding = ring_buffer_size;
+
+ const uptr right_size = alignment;
+ const uptr map_size = left_padding + 2 * alignment;
+
+ const uptr map_start = reinterpret_cast<uptr>(MmapNoAccess(map_size));
+ CHECK_NE(map_start, static_cast<uptr>(-1));
+ const uptr right_start = RoundUpTo(map_start + left_padding, alignment);
+
+ UnmapFromTo(map_start, right_start - left_padding);
+ UnmapFromTo(right_start + right_size, map_start + map_size);
+
+ CreateAliases(right_start + right_size / 2, alias_size, num_aliases);
+
+ return right_start;
+}
+
+void InitializePlatformCommonFlags(CommonFlags *cf) {
+#if SANITIZER_ANDROID
+ if (&__libc_get_static_tls_bounds == nullptr)
+ cf->detect_leaks = false;
+#endif
+}
} // namespace __sanitizer
//===----------------------------------------------------------------------===//
//
// `LocalAddressSpaceView` provides the local (i.e. target and current address
-// space are the same) implementation of the `AddressSpaveView` interface which
+// space are the same) implementation of the `AddressSpaceView` interface which
// provides a simple interface to load memory from another process (i.e.
// out-of-process)
//
#define SANITIZER_OS_TRACE 0
#endif
+// import new crash reporting api
+#if defined(__has_include) && __has_include(<CrashReporterClient.h>)
+#define HAVE_CRASHREPORTERCLIENT_H 1
+#include <CrashReporterClient.h>
+#else
+#define HAVE_CRASHREPORTERCLIENT_H 0
+#endif
+
#if !SANITIZER_IOS
#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron
#else
#include <mach/mach_time.h>
#include <mach/vm_statistics.h>
#include <malloc/malloc.h>
+#include <os/log.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
return munmap(addr, length);
}
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+ void *new_address) {
+ CHECK(false && "internal_mremap is unimplemented on Mac");
+ return 0;
+}
+
int internal_mprotect(void *addr, uptr length, int prot) {
return mprotect(addr, length, prot);
}
+int internal_madvise(uptr addr, uptr length, int advice) {
+ return madvise((void *)addr, length, advice);
+}
+
uptr internal_close(fd_t fd) {
return close(fd);
}
_exit(exitcode);
}
-unsigned int internal_sleep(unsigned int seconds) {
- return sleep(seconds);
-}
+void internal_usleep(u64 useconds) { usleep(useconds); }
uptr internal_getpid() {
return getpid();
// On OS X the executable path is saved to the stack by dyld. Reading it
// from there is much faster than calling dladdr, especially for large
// binaries with symbols.
- InternalScopedString exe_path(kMaxPathLength);
+ InternalMmapVector<char> exe_path(kMaxPathLength);
uint32_t size = exe_path.size();
if (_NSGetExecutablePath(exe_path.data(), &size) == 0 &&
realpath(exe_path.data(), buf) != 0) {
}
}
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ // FIXME: implement actual blocking.
+ sched_yield();
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {}
+
BlockingMutex::BlockingMutex() {
internal_memset(this, 0, sizeof(*this));
}
OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
}
-void BlockingMutex::CheckLocked() {
- CHECK_NE(*(OSSpinLock*)&opaque_storage_, 0);
+void BlockingMutex::CheckLocked() const {
+ CHECK_NE(*(const OSSpinLock*)&opaque_storage_, 0);
}
u64 NanoTime() {
return result;
}
-// This corresponds to Triple::getMacOSXVersion() in the Clang driver.
-static MacosVersion GetMacosAlignedVersionInternal() {
+// Offset example:
+// XNU 17 -- macOS 10.13 -- iOS 11 -- tvOS 11 -- watchOS 4
+constexpr u16 GetOSMajorKernelOffset() {
+ if (TARGET_OS_OSX) return 4;
+ if (TARGET_OS_IOS || TARGET_OS_TV) return 6;
+ if (TARGET_OS_WATCH) return 13;
+}
+
+using VersStr = char[64];
+
+static uptr ApproximateOSVersionViaKernelVersion(VersStr vers) {
u16 kernel_major = GetDarwinKernelVersion().major;
- // Darwin 0-3 -> unsupported
- // Darwin 4-19 -> macOS 10.x
- // Darwin 20+ -> macOS 11+
- CHECK_GE(kernel_major, 4);
- u16 major, minor;
- if (kernel_major < 20) {
- major = 10;
- minor = kernel_major - 4;
+ u16 offset = GetOSMajorKernelOffset();
+ CHECK_GE(kernel_major, offset);
+ u16 os_major = kernel_major - offset;
+
+ const char *format = "%d.0";
+ if (TARGET_OS_OSX) {
+ if (os_major >= 16) { // macOS 11+
+ os_major -= 5;
+ } else { // macOS 10.15 and below
+ format = "10.%d";
+ }
+ }
+ return internal_snprintf(vers, sizeof(VersStr), format, os_major);
+}
+
+static void GetOSVersion(VersStr vers) {
+ uptr len = sizeof(VersStr);
+ if (SANITIZER_IOSSIM) {
+ const char *vers_env = GetEnv("SIMULATOR_RUNTIME_VERSION");
+ if (!vers_env) {
+ Report("ERROR: Running in simulator but SIMULATOR_RUNTIME_VERSION env "
+ "var is not set.\n");
+ Die();
+ }
+ len = internal_strlcpy(vers, vers_env, len);
} else {
- major = 11 + kernel_major - 20;
- minor = 0;
+ int res =
+ internal_sysctlbyname("kern.osproductversion", vers, &len, nullptr, 0);
+
+ // XNU 17 (macOS 10.13) and below do not provide the sysctl
+ // `kern.osproductversion` entry (res != 0).
+ bool no_os_version = res != 0;
+
+ // For launchd, sanitizer initialization runs before sysctl is setup
+ // (res == 0 && len != strlen(vers), vers is not a valid version). However,
+ // the kernel version `kern.osrelease` is available.
+ bool launchd = (res == 0 && internal_strlen(vers) < 3);
+ if (launchd) CHECK_EQ(internal_getpid(), 1);
+
+ if (no_os_version || launchd) {
+ len = ApproximateOSVersionViaKernelVersion(vers);
+ }
+ }
+ CHECK_LT(len, sizeof(VersStr));
+}
+
+void ParseVersion(const char *vers, u16 *major, u16 *minor) {
+ // Format: <major>.<minor>[.<patch>]\0
+ CHECK_GE(internal_strlen(vers), 3);
+ const char *p = vers;
+ *major = internal_simple_strtoll(p, &p, /*base=*/10);
+ CHECK_EQ(*p, '.');
+ p += 1;
+ *minor = internal_simple_strtoll(p, &p, /*base=*/10);
+}
+
+// Aligned versions example:
+// macOS 10.15 -- iOS 13 -- tvOS 13 -- watchOS 6
+static void MapToMacos(u16 *major, u16 *minor) {
+ if (TARGET_OS_OSX)
+ return;
+
+ if (TARGET_OS_IOS || TARGET_OS_TV)
+ *major += 2;
+ else if (TARGET_OS_WATCH)
+ *major += 9;
+ else
+ UNREACHABLE("unsupported platform");
+
+ if (*major >= 16) { // macOS 11+
+ *major -= 5;
+ } else { // macOS 10.15 and below
+ *minor = *major;
+ *major = 10;
}
+}
+
+static MacosVersion GetMacosAlignedVersionInternal() {
+ VersStr vers = {};
+ GetOSVersion(vers);
+
+ u16 major, minor;
+ ParseVersion(vers, &major, &minor);
+ MapToMacos(&major, &minor);
+
return MacosVersion(major, minor);
}
return *reinterpret_cast<MacosVersion *>(&result);
}
-void ParseVersion(const char *vers, u16 *major, u16 *minor) {
- // Format: <major>.<minor>.<patch>\0
- CHECK_GE(internal_strlen(vers), 5);
- const char *p = vers;
- *major = internal_simple_strtoll(p, &p, /*base=*/10);
- CHECK_EQ(*p, '.');
- p += 1;
- *minor = internal_simple_strtoll(p, &p, /*base=*/10);
-}
-
DarwinKernelVersion GetDarwinKernelVersion() {
- char buf[100];
- size_t len = sizeof(buf);
- int res = internal_sysctlbyname("kern.osrelease", buf, &len, nullptr, 0);
+ VersStr vers = {};
+ uptr len = sizeof(VersStr);
+ int res = internal_sysctlbyname("kern.osrelease", vers, &len, nullptr, 0);
CHECK_EQ(res, 0);
+ CHECK_LT(len, sizeof(VersStr));
u16 major, minor;
- ParseVersion(buf, &major, &minor);
+ ParseVersion(vers, &major, &minor);
return DarwinKernelVersion(major, minor);
}
void WriteOneLineToSyslog(const char *s) {
#if !SANITIZER_GO
syslog_lock.CheckLocked();
- asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);
+ if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {
+ os_log_error(OS_LOG_DEFAULT, "%{public}s", s);
+ } else {
+ asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);
+ }
+#endif
+}
+
+// buffer to store crash report application information
+static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
+static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
+
+extern "C" {
+// Integrate with crash reporter libraries.
+#if HAVE_CRASHREPORTERCLIENT_H
+CRASH_REPORTER_CLIENT_HIDDEN
+struct crashreporter_annotations_t gCRAnnotations
+ __attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = {
+ CRASHREPORTER_ANNOTATIONS_VERSION,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+#if CRASHREPORTER_ANNOTATIONS_VERSION > 4
+ 0,
+#endif
+};
+
+#else
+// fall back to old crashreporter api
+static const char *__crashreporter_info__ __attribute__((__used__)) =
+ &crashreporter_info_buff[0];
+asm(".desc ___crashreporter_info__, 0x10");
+#endif
+
+} // extern "C"
+
+static void CRAppendCrashLogMessage(const char *msg) {
+ BlockingMutexLock l(&crashreporter_info_mutex);
+ internal_strlcat(crashreporter_info_buff, msg,
+ sizeof(crashreporter_info_buff));
+#if HAVE_CRASHREPORTERCLIENT_H
+ (void)CRSetCrashLogMessage(crashreporter_info_buff);
#endif
}
GetPcSpBp(context, &pc, &sp, &bp);
}
+// ASan/TSan use mmap in a way that creates “deallocation gaps” which triggers
+// EXC_GUARD exceptions on macOS 10.15+ (XNU 19.0+).
+static void DisableMmapExcGuardExceptions() {
+ using task_exc_guard_behavior_t = uint32_t;
+ using task_set_exc_guard_behavior_t =
+ kern_return_t(task_t task, task_exc_guard_behavior_t behavior);
+ auto *set_behavior = (task_set_exc_guard_behavior_t *)dlsym(
+ RTLD_DEFAULT, "task_set_exc_guard_behavior");
+ if (set_behavior == nullptr) return;
+ const task_exc_guard_behavior_t task_exc_guard_none = 0;
+ set_behavior(mach_task_self(), task_exc_guard_none);
+}
+
void InitializePlatformEarly() {
// Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
use_xnu_fast_mmap =
#else
false;
#endif
+ if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
+ DisableMmapExcGuardExceptions();
}
#if !SANITIZER_GO
return false;
}
-extern "C" SANITIZER_WEAK_ATTRIBUTE double dyldVersionNumber;
-static const double kMinDyldVersionWithAutoInterposition = 360.0;
-
-bool DyldNeedsEnvVariable() {
- // Although sanitizer support was added to LLVM on OS X 10.7+, GCC users
- // still may want use them on older systems. On older Darwin platforms, dyld
- // doesn't export dyldVersionNumber symbol and we simply return true.
- if (!&dyldVersionNumber) return true;
+static bool DyldNeedsEnvVariable() {
// If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
- // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via
- // GetMacosAlignedVersion() doesn't work for the simulator. Let's instead
- // check `dyldVersionNumber`, which is exported by dyld, against a known
- // version number from the first OS release where this appeared.
- return dyldVersionNumber < kMinDyldVersionWithAutoInterposition;
+ // DYLD_INSERT_LIBRARIES is not set.
+ return GetMacosAlignedVersion() < MacosVersion(10, 11);
}
void MaybeReexec() {
if (DyldNeedsEnvVariable() && !lib_is_in_env) {
// DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
// library.
- InternalScopedString program_name(1024);
+ InternalMmapVector<char> program_name(1024);
uint32_t buf_size = program_name.size();
_NSGetExecutablePath(program_name.data(), &buf_size);
char *new_env = const_cast<char*>(info.dli_fname);
return *_NSGetArgv();
}
-#if SANITIZER_IOS
+#if SANITIZER_IOS && !SANITIZER_IOSSIM
// The task_vm_info struct is normally provided by the macOS SDK, but we need
// fields only available in 10.12+. Declare the struct manually to be able to
// build against older SDKs.
uptr GetMaxUserVirtualAddress() {
static uptr max_vm = GetTaskInfoMaxAddress();
- if (max_vm != 0)
- return max_vm - 1;
+ if (max_vm != 0) {
+ const uptr ret_value = max_vm - 1;
+ CHECK_LE(ret_value, SANITIZER_MMAP_RANGE_SIZE);
+ return ret_value;
+ }
// xnu cannot provide vm address limit
# if SANITIZER_WORDSIZE == 32
- return 0xffe00000 - 1;
+ constexpr uptr fallback_max_vm = 0xffe00000 - 1;
# else
- return 0x200000000 - 1;
+ constexpr uptr fallback_max_vm = 0x200000000 - 1;
# endif
+ static_assert(fallback_max_vm <= SANITIZER_MMAP_RANGE_SIZE,
+ "Max virtual address must be less than mmap range size.");
+ return fallback_max_vm;
}
#else // !SANITIZER_IOS
uptr GetMaxUserVirtualAddress() {
# if SANITIZER_WORDSIZE == 64
- return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
+ constexpr uptr max_vm = (1ULL << 47) - 1; // 0x00007fffffffffffUL;
# else // SANITIZER_WORDSIZE == 32
static_assert(SANITIZER_WORDSIZE == 32, "Wrong wordsize");
- return (1ULL << 32) - 1; // 0xffffffff;
+ constexpr uptr max_vm = (1ULL << 32) - 1; // 0xffffffff;
# endif
+ static_assert(max_vm <= SANITIZER_MMAP_RANGE_SIZE,
+ "Max virtual address must be less than mmap range size.");
+ return max_vm;
}
#endif
return GetMaxUserVirtualAddress();
}
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment, uptr &high_mem_end) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment =
+ Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
+ const uptr left_padding =
+ Max<uptr>(granularity, 1ULL << min_shadow_base_alignment);
+
+ uptr space_size = shadow_size_bytes + left_padding;
+
+ uptr largest_gap_found = 0;
+ uptr max_occupied_addr = 0;
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ uptr shadow_start =
+ FindAvailableMemoryRange(space_size, alignment, granularity,
+ &largest_gap_found, &max_occupied_addr);
+ // If the shadow doesn't fit, restrict the address space to make it fit.
+ if (shadow_start == 0) {
+ VReport(
+ 2,
+ "Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
+ largest_gap_found, max_occupied_addr);
+ uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
+ if (new_max_vm < max_occupied_addr) {
+ Report("Unable to find a memory range for dynamic shadow.\n");
+ Report(
+ "space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
+ "new_max_vm = %p\n",
+ space_size, largest_gap_found, max_occupied_addr, new_max_vm);
+ CHECK(0 && "cannot place shadow");
+ }
+ RestrictMemoryToMaxAddress(new_max_vm);
+ high_mem_end = new_max_vm - 1;
+ space_size = (high_mem_end >> shadow_scale) + left_padding;
+ VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
+ shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
+ nullptr, nullptr);
+ if (shadow_start == 0) {
+ Report("Unable to find a memory range after restricting VM.\n");
+ CHECK(0 && "cannot place shadow after restricting vm");
+ }
+ }
+ CHECK_NE((uptr)0, shadow_start);
+ CHECK(IsAligned(shadow_start, alignment));
+ return shadow_start;
+}
+
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+ uptr num_aliases, uptr ring_buffer_size) {
+ CHECK(false && "HWASan aliasing is unimplemented on Mac");
+ return 0;
+}
+
uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
uptr *largest_gap_found,
uptr *max_occupied_addr) {
uuid[12], uuid[13], uuid[14], uuid[15]);
}
-void PrintModuleMap() {
+void DumpProcessMap() {
Printf("Process module map:\n");
MemoryMappingLayout memory_mapping(false);
InternalMmapVector<LoadedModule> modules;
return (u32)sysconf(_SC_NPROCESSORS_ONLN);
}
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
#endif // SANITIZER_MAC
return major > other.major ||
(major == other.major && minor >= other.minor);
}
+ bool operator<(const VersionType &other) const { return !(*this >= other); }
};
struct MacosVersion : VersionBase<MacosVersion> {
} // namespace __sanitizer
-extern "C" {
-static char __crashreporter_info_buff__[__sanitizer::kErrorMessageBufferSize] =
- {};
-static const char *__crashreporter_info__ __attribute__((__used__)) =
- &__crashreporter_info_buff__[0];
-asm(".desc ___crashreporter_info__, 0x10");
-} // extern "C"
-
-namespace __sanitizer {
-static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
-
-INLINE void CRAppendCrashLogMessage(const char *msg) {
- BlockingMutexLock l(&crashreporter_info_mutex);
- internal_strlcat(__crashreporter_info_buff__, msg,
- sizeof(__crashreporter_info_buff__)); }
-} // namespace __sanitizer
-
#endif // SANITIZER_MAC
#endif // SANITIZER_MAC_H
INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) {
COMMON_MALLOC_ENTER();
- // Allocate |sizeof(COMMON_MALLOC_ZONE_NAME "-") + internal_strlen(name)|
- // bytes.
- size_t buflen =
- sizeof(COMMON_MALLOC_ZONE_NAME "-") + (name ? internal_strlen(name) : 0);
- InternalScopedString new_name(buflen);
+ InternalScopedString new_name;
if (name && zone->introspect == sanitizer_zone.introspect) {
new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name);
name = new_name.data();
--- /dev/null
+//===-- sanitizer_mutex.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between AddressSanitizer and ThreadSanitizer
+// run-time libraries.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_mutex.h"
+
+#include "sanitizer_common.h"
+
+namespace __sanitizer {
+
+void StaticSpinMutex::LockSlow() {
+ for (int i = 0;; i++) {
+ if (i < 100)
+ proc_yield(1);
+ else
+ internal_sched_yield();
+ if (atomic_load(&state_, memory_order_relaxed) == 0 &&
+ atomic_exchange(&state_, 1, memory_order_acquire) == 0)
+ return;
+ }
+}
+
+void Semaphore::Wait() {
+ u32 count = atomic_load(&state_, memory_order_relaxed);
+ for (;;) {
+ if (count == 0) {
+ FutexWait(&state_, 0);
+ count = atomic_load(&state_, memory_order_relaxed);
+ continue;
+ }
+ if (atomic_compare_exchange_weak(&state_, &count, count - 1,
+ memory_order_acquire))
+ break;
+ }
+}
+
+void Semaphore::Post(u32 count) {
+ CHECK_NE(count, 0);
+ atomic_fetch_add(&state_, count, memory_order_release);
+ FutexWake(&state_, count);
+}
+
+#if SANITIZER_CHECK_DEADLOCKS
+// An empty mutex meta table, it effectively disables deadlock detection.
+// Each tool can override the table to define own mutex hierarchy and
+// enable deadlock detection.
+// The table defines a static mutex type hierarchy (what mutex types can be locked
+// under what mutex types). This table is checked to be acyclic and then
+// actual mutex lock/unlock operations are checked to adhere to this hierarchy.
+// The checking happens on mutex types rather than on individual mutex instances
+// because doing it on mutex instances will both significantly complicate
+// the implementation, worsen performance and memory overhead and is mostly
+// unnecessary (we almost never lock multiple mutexes of the same type recursively).
+static constexpr int kMutexTypeMax = 20;
+SANITIZER_WEAK_ATTRIBUTE MutexMeta mutex_meta[kMutexTypeMax] = {};
+SANITIZER_WEAK_ATTRIBUTE void PrintMutexPC(uptr pc) {}
+static StaticSpinMutex mutex_meta_mtx;
+static int mutex_type_count = -1;
+// Adjacency matrix of what mutexes can be locked under what mutexes.
+static bool mutex_can_lock[kMutexTypeMax][kMutexTypeMax];
+// Mutex types with MutexMulti mark.
+static bool mutex_multi[kMutexTypeMax];
+
+void DebugMutexInit() {
+ // Build adjacency matrix.
+ bool leaf[kMutexTypeMax];
+ internal_memset(&leaf, 0, sizeof(leaf));
+ int cnt[kMutexTypeMax] = {};
+ internal_memset(&cnt, 0, sizeof(cnt));
+ for (int t = 0; t < kMutexTypeMax; t++) {
+ mutex_type_count = t;
+ if (!mutex_meta[t].name)
+ break;
+ CHECK_EQ(t, mutex_meta[t].type);
+ for (uptr j = 0; j < ARRAY_SIZE(mutex_meta[t].can_lock); j++) {
+ MutexType z = mutex_meta[t].can_lock[j];
+ if (z == MutexInvalid)
+ break;
+ if (z == MutexLeaf) {
+ CHECK(!leaf[t]);
+ leaf[t] = true;
+ continue;
+ }
+ if (z == MutexMulti) {
+ mutex_multi[t] = true;
+ continue;
+ }
+ CHECK_LT(z, kMutexTypeMax);
+ CHECK(!mutex_can_lock[t][z]);
+ mutex_can_lock[t][z] = true;
+ cnt[t]++;
+ }
+ }
+ // Indicates the array is not properly terminated.
+ CHECK_LT(mutex_type_count, kMutexTypeMax);
+ // Add leaf mutexes.
+ for (int t = 0; t < mutex_type_count; t++) {
+ if (!leaf[t])
+ continue;
+ CHECK_EQ(cnt[t], 0);
+ for (int z = 0; z < mutex_type_count; z++) {
+ if (z == MutexInvalid || t == z || leaf[z])
+ continue;
+ CHECK(!mutex_can_lock[z][t]);
+ mutex_can_lock[z][t] = true;
+ }
+ }
+ // Build the transitive closure and check that the graphs is acyclic.
+ u32 trans[kMutexTypeMax];
+ static_assert(sizeof(trans[0]) * 8 >= kMutexTypeMax,
+ "kMutexTypeMax does not fit into u32, switch to u64");
+ internal_memset(&trans, 0, sizeof(trans));
+ for (int i = 0; i < mutex_type_count; i++) {
+ for (int j = 0; j < mutex_type_count; j++)
+ if (mutex_can_lock[i][j])
+ trans[i] |= 1 << j;
+ }
+ for (int k = 0; k < mutex_type_count; k++) {
+ for (int i = 0; i < mutex_type_count; i++) {
+ if (trans[i] & (1 << k))
+ trans[i] |= trans[k];
+ }
+ }
+ for (int i = 0; i < mutex_type_count; i++) {
+ if (trans[i] & (1 << i)) {
+ Printf("Mutex %s participates in a cycle\n", mutex_meta[i].name);
+ Die();
+ }
+ }
+}
+
+struct InternalDeadlockDetector {
+ struct LockDesc {
+ u64 seq;
+ uptr pc;
+ int recursion;
+ };
+ int initialized;
+ u64 sequence;
+ LockDesc locked[kMutexTypeMax];
+
+ void Lock(MutexType type, uptr pc) {
+ if (!Initialize(type))
+ return;
+ CHECK_LT(type, mutex_type_count);
+ // Find the last locked mutex type.
+ // This is the type we will use for hierarchy checks.
+ u64 max_seq = 0;
+ MutexType max_idx = MutexInvalid;
+ for (int i = 0; i != mutex_type_count; i++) {
+ if (locked[i].seq == 0)
+ continue;
+ CHECK_NE(locked[i].seq, max_seq);
+ if (max_seq < locked[i].seq) {
+ max_seq = locked[i].seq;
+ max_idx = (MutexType)i;
+ }
+ }
+ if (max_idx == type && mutex_multi[type]) {
+ // Recursive lock of the same type.
+ CHECK_EQ(locked[type].seq, max_seq);
+ CHECK(locked[type].pc);
+ locked[type].recursion++;
+ return;
+ }
+ if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {
+ Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,
+ mutex_meta[type].name, mutex_meta[max_idx].name);
+ PrintMutexPC(pc);
+ CHECK(0);
+ }
+ locked[type].seq = ++sequence;
+ locked[type].pc = pc;
+ locked[type].recursion = 1;
+ }
+
+ void Unlock(MutexType type) {
+ if (!Initialize(type))
+ return;
+ CHECK_LT(type, mutex_type_count);
+ CHECK(locked[type].seq);
+ CHECK_GT(locked[type].recursion, 0);
+ if (--locked[type].recursion)
+ return;
+ locked[type].seq = 0;
+ locked[type].pc = 0;
+ }
+
+ void CheckNoLocks() {
+ for (int i = 0; i < mutex_type_count; i++) CHECK_EQ(locked[i].recursion, 0);
+ }
+
+ bool Initialize(MutexType type) {
+ if (type == MutexUnchecked || type == MutexInvalid)
+ return false;
+ CHECK_GT(type, MutexInvalid);
+ if (initialized != 0)
+ return initialized > 0;
+ initialized = -1;
+ SpinMutexLock lock(&mutex_meta_mtx);
+ if (mutex_type_count < 0)
+ DebugMutexInit();
+ initialized = mutex_type_count ? 1 : -1;
+ return initialized > 0;
+ }
+};
+
+static THREADLOCAL InternalDeadlockDetector deadlock_detector;
+
+void CheckedMutex::LockImpl(uptr pc) { deadlock_detector.Lock(type_, pc); }
+
+void CheckedMutex::UnlockImpl() { deadlock_detector.Unlock(type_); }
+
+void CheckedMutex::CheckNoLocksImpl() { deadlock_detector.CheckNoLocks(); }
+#endif
+
+} // namespace __sanitizer
#include "sanitizer_atomic.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
+#include "sanitizer_thread_safety.h"
namespace __sanitizer {
-class StaticSpinMutex {
+class MUTEX StaticSpinMutex {
public:
void Init() {
atomic_store(&state_, 0, memory_order_relaxed);
}
- void Lock() {
- if (TryLock())
+ void Lock() ACQUIRE() {
+ if (LIKELY(TryLock()))
return;
LockSlow();
}
- bool TryLock() {
+ bool TryLock() TRY_ACQUIRE(true) {
return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
}
- void Unlock() {
- atomic_store(&state_, 0, memory_order_release);
- }
+ void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); }
- void CheckLocked() {
+ void CheckLocked() const CHECK_LOCKED() {
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
}
private:
atomic_uint8_t state_;
- void NOINLINE LockSlow() {
- for (int i = 0;; i++) {
- if (i < 10)
- proc_yield(10);
- else
- internal_sched_yield();
- if (atomic_load(&state_, memory_order_relaxed) == 0
- && atomic_exchange(&state_, 1, memory_order_acquire) == 0)
- return;
- }
- }
+ void LockSlow();
};
-class SpinMutex : public StaticSpinMutex {
+class MUTEX SpinMutex : public StaticSpinMutex {
public:
SpinMutex() {
Init();
}
+ SpinMutex(const SpinMutex &) = delete;
+ void operator=(const SpinMutex &) = delete;
+};
+
+// Semaphore provides an OS-dependent way to park/unpark threads.
+// The last thread returned from Wait can destroy the object
+// (destruction-safety).
+class Semaphore {
+ public:
+ constexpr Semaphore() {}
+ Semaphore(const Semaphore &) = delete;
+ void operator=(const Semaphore &) = delete;
+
+ void Wait();
+ void Post(u32 count = 1);
+
private:
- SpinMutex(const SpinMutex&);
- void operator=(const SpinMutex&);
+ atomic_uint32_t state_ = {0};
};
-class BlockingMutex {
+typedef int MutexType;
+
+enum {
+ // Used as sentinel and to catch unassigned types
+ // (should not be used as real Mutex type).
+ MutexInvalid = 0,
+ MutexThreadRegistry,
+ // Each tool own mutexes must start at this number.
+ MutexLastCommon,
+ // Type for legacy mutexes that are not checked for deadlocks.
+ MutexUnchecked = -1,
+ // Special marks that can be used in MutexMeta::can_lock table.
+ // The leaf mutexes can be locked under any other non-leaf mutex,
+ // but no other mutex can be locked while under a leaf mutex.
+ MutexLeaf = -1,
+ // Multiple mutexes of this type can be locked at the same time.
+ MutexMulti = -3,
+};
+
+// Go linker does not support THREADLOCAL variables,
+// so we can't use per-thread state.
+#define SANITIZER_CHECK_DEADLOCKS (SANITIZER_DEBUG && !SANITIZER_GO)
+
+#if SANITIZER_CHECK_DEADLOCKS
+struct MutexMeta {
+ MutexType type;
+ const char *name;
+ // The table fixes what mutexes can be locked under what mutexes.
+ // If the entry for MutexTypeFoo contains MutexTypeBar,
+ // then Bar mutex can be locked while under Foo mutex.
+ // Can also contain the special MutexLeaf/MutexMulti marks.
+ MutexType can_lock[10];
+};
+#endif
+
+class CheckedMutex {
+ public:
+ constexpr CheckedMutex(MutexType type)
+#if SANITIZER_CHECK_DEADLOCKS
+ : type_(type)
+#endif
+ {
+ }
+
+ ALWAYS_INLINE void Lock() {
+#if SANITIZER_CHECK_DEADLOCKS
+ LockImpl(GET_CALLER_PC());
+#endif
+ }
+
+ ALWAYS_INLINE void Unlock() {
+#if SANITIZER_CHECK_DEADLOCKS
+ UnlockImpl();
+#endif
+ }
+
+ // Checks that the current thread does not hold any mutexes
+ // (e.g. when returning from a runtime function to user code).
+ static void CheckNoLocks() {
+#if SANITIZER_CHECK_DEADLOCKS
+ CheckNoLocksImpl();
+#endif
+ }
+
+ private:
+#if SANITIZER_CHECK_DEADLOCKS
+ const MutexType type_;
+
+ void LockImpl(uptr pc);
+ void UnlockImpl();
+ static void CheckNoLocksImpl();
+#endif
+};
+
+// Reader-writer mutex.
+// Derive from CheckedMutex for the purposes of EBO.
+// We could make it a field marked with [[no_unique_address]],
+// but this attribute is not supported by some older compilers.
+class MUTEX Mutex : CheckedMutex {
+ public:
+ constexpr Mutex(MutexType type = MutexUnchecked) : CheckedMutex(type) {}
+
+ void Lock() ACQUIRE() {
+ CheckedMutex::Lock();
+ u64 reset_mask = ~0ull;
+ u64 state = atomic_load_relaxed(&state_);
+ const uptr kMaxSpinIters = 1500;
+ for (uptr spin_iters = 0;; spin_iters++) {
+ u64 new_state;
+ bool locked = (state & (kWriterLock | kReaderLockMask)) != 0;
+ if (LIKELY(!locked)) {
+ // The mutex is not read-/write-locked, try to lock.
+ new_state = (state | kWriterLock) & reset_mask;
+ } else if (spin_iters > kMaxSpinIters) {
+ // We've spun enough, increment waiting writers count and block.
+ // The counter will be decremented by whoever wakes us.
+ new_state = (state + kWaitingWriterInc) & reset_mask;
+ } else if ((state & kWriterSpinWait) == 0) {
+ // Active spinning, but denote our presence so that unlocking
+ // thread does not wake up other threads.
+ new_state = state | kWriterSpinWait;
+ } else {
+ // Active spinning.
+ state = atomic_load(&state_, memory_order_relaxed);
+ continue;
+ }
+ if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_acquire)))
+ continue;
+ if (LIKELY(!locked))
+ return; // We've locked the mutex.
+ if (spin_iters > kMaxSpinIters) {
+ // We've incremented waiting writers, so now block.
+ writers_.Wait();
+ spin_iters = 0;
+ state = atomic_load(&state_, memory_order_relaxed);
+ DCHECK_NE(state & kWriterSpinWait, 0);
+ } else {
+ // We've set kWriterSpinWait, but we are still in active spinning.
+ }
+ // We either blocked and were unblocked,
+ // or we just spun but set kWriterSpinWait.
+ // Either way we need to reset kWriterSpinWait
+ // next time we take the lock or block again.
+ reset_mask = ~kWriterSpinWait;
+ }
+ }
+
+ void Unlock() RELEASE() {
+ CheckedMutex::Unlock();
+ bool wake_writer;
+ u64 wake_readers;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ DCHECK_NE(state & kWriterLock, 0);
+ DCHECK_EQ(state & kReaderLockMask, 0);
+ new_state = state & ~kWriterLock;
+ wake_writer =
+ (state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0;
+ if (wake_writer)
+ new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
+ wake_readers =
+ (state & (kWriterSpinWait | kWaitingWriterMask)) != 0
+ ? 0
+ : ((state & kWaitingReaderMask) >> kWaitingReaderShift);
+ if (wake_readers)
+ new_state = (new_state & ~kWaitingReaderMask) +
+ (wake_readers << kReaderLockShift);
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_release)));
+ if (UNLIKELY(wake_writer))
+ writers_.Post();
+ else if (UNLIKELY(wake_readers))
+ readers_.Post(wake_readers);
+ }
+
+ void ReadLock() ACQUIRE_SHARED() {
+ CheckedMutex::Lock();
+ bool locked;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ locked =
+ (state & kReaderLockMask) == 0 &&
+ (state & (kWriterLock | kWriterSpinWait | kWaitingWriterMask)) != 0;
+ if (LIKELY(!locked))
+ new_state = state + kReaderLockInc;
+ else
+ new_state = state + kWaitingReaderInc;
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_acquire)));
+ if (UNLIKELY(locked))
+ readers_.Wait();
+ DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0);
+ DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0);
+ }
+
+ void ReadUnlock() RELEASE_SHARED() {
+ CheckedMutex::Unlock();
+ bool wake;
+ u64 new_state;
+ u64 state = atomic_load_relaxed(&state_);
+ do {
+ DCHECK_NE(state & kReaderLockMask, 0);
+ DCHECK_EQ(state & (kWaitingReaderMask | kWriterLock), 0);
+ new_state = state - kReaderLockInc;
+ wake = (new_state & (kReaderLockMask | kWriterSpinWait)) == 0 &&
+ (new_state & kWaitingWriterMask) != 0;
+ if (wake)
+ new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
+ } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+ memory_order_release)));
+ if (UNLIKELY(wake))
+ writers_.Post();
+ }
+
+ // This function does not guarantee an explicit check that the calling thread
+ // is the thread which owns the mutex. This behavior, while more strictly
+ // correct, causes problems in cases like StopTheWorld, where a parent thread
+ // owns the mutex but a child checks that it is locked. Rather than
+ // maintaining complex state to work around those situations, the check only
+ // checks that the mutex is owned.
+ void CheckWriteLocked() const CHECK_LOCKED() {
+ CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
+ }
+
+ void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); }
+
+ void CheckReadLocked() const CHECK_LOCKED() {
+ CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
+ }
+
+ private:
+ atomic_uint64_t state_ = {0};
+ Semaphore writers_;
+ Semaphore readers_;
+
+ // The state has 3 counters:
+ // - number of readers holding the lock,
+ // if non zero, the mutex is read-locked
+ // - number of waiting readers,
+ // if not zero, the mutex is write-locked
+ // - number of waiting writers,
+ // if non zero, the mutex is read- or write-locked
+ // And 2 flags:
+ // - writer lock
+ // if set, the mutex is write-locked
+ // - a writer is awake and spin-waiting
+ // the flag is used to prevent thundering herd problem
+ // (new writers are not woken if this flag is set)
+ //
+ // Writer support active spinning, readers does not.
+ // But readers are more aggressive and always take the mutex
+ // if there are any other readers.
+ // Writers hand off the mutex to readers: after wake up readers
+ // already assume ownership of the mutex (don't need to do any
+ // state updates). But the mutex is not handed off to writers,
+ // after wake up writers compete to lock the mutex again.
+ // This is needed to allow repeated write locks even in presence
+ // of other blocked writers.
+ static constexpr u64 kCounterWidth = 20;
+ static constexpr u64 kReaderLockShift = 0;
+ static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
+ static constexpr u64 kReaderLockMask = ((1ull << kCounterWidth) - 1)
+ << kReaderLockShift;
+ static constexpr u64 kWaitingReaderShift = kCounterWidth;
+ static constexpr u64 kWaitingReaderInc = 1ull << kWaitingReaderShift;
+ static constexpr u64 kWaitingReaderMask = ((1ull << kCounterWidth) - 1)
+ << kWaitingReaderShift;
+ static constexpr u64 kWaitingWriterShift = 2 * kCounterWidth;
+ static constexpr u64 kWaitingWriterInc = 1ull << kWaitingWriterShift;
+ static constexpr u64 kWaitingWriterMask = ((1ull << kCounterWidth) - 1)
+ << kWaitingWriterShift;
+ static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
+ static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
+
+ Mutex(const Mutex &) = delete;
+ void operator=(const Mutex &) = delete;
+};
+
+void FutexWait(atomic_uint32_t *p, u32 cmp);
+void FutexWake(atomic_uint32_t *p, u32 count);
+
+class MUTEX BlockingMutex {
public:
explicit constexpr BlockingMutex(LinkerInitialized)
: opaque_storage_ {0, }, owner_ {0} {}
BlockingMutex();
- void Lock();
- void Unlock();
+ void Lock() ACQUIRE();
+ void Unlock() RELEASE();
// This function does not guarantee an explicit check that the calling thread
// is the thread which owns the mutex. This behavior, while more strictly
// maintaining complex state to work around those situations, the check only
// checks that the mutex is owned, and assumes callers to be generally
// well-behaved.
- void CheckLocked();
+ void CheckLocked() const CHECK_LOCKED();
private:
// Solaris mutex_t has a member that requires 64-bit alignment.
};
// Reader-writer spin mutex.
-class RWMutex {
+class MUTEX RWMutex {
public:
RWMutex() {
atomic_store(&state_, kUnlocked, memory_order_relaxed);
CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
}
- void Lock() {
+ void Lock() ACQUIRE() {
u32 cmp = kUnlocked;
if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
memory_order_acquire))
LockSlow();
}
- void Unlock() {
+ void Unlock() RELEASE() {
u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
DCHECK_NE(prev & kWriteLock, 0);
(void)prev;
}
- void ReadLock() {
+ void ReadLock() ACQUIRE_SHARED() {
u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
if ((prev & kWriteLock) == 0)
return;
ReadLockSlow();
}
- void ReadUnlock() {
+ void ReadUnlock() RELEASE_SHARED() {
u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
DCHECK_EQ(prev & kWriteLock, 0);
DCHECK_GT(prev & ~kWriteLock, 0);
(void)prev;
}
- void CheckLocked() {
+ void CheckLocked() const CHECK_LOCKED() {
CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
}
}
}
- RWMutex(const RWMutex&);
- void operator = (const RWMutex&);
+ RWMutex(const RWMutex &) = delete;
+ void operator=(const RWMutex &) = delete;
};
-template<typename MutexType>
-class GenericScopedLock {
+template <typename MutexType>
+class SCOPED_LOCK GenericScopedLock {
public:
- explicit GenericScopedLock(MutexType *mu)
- : mu_(mu) {
+ explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
mu_->Lock();
}
- ~GenericScopedLock() {
- mu_->Unlock();
- }
+ ~GenericScopedLock() RELEASE() { mu_->Unlock(); }
private:
MutexType *mu_;
- GenericScopedLock(const GenericScopedLock&);
- void operator=(const GenericScopedLock&);
+ GenericScopedLock(const GenericScopedLock &) = delete;
+ void operator=(const GenericScopedLock &) = delete;
};
-template<typename MutexType>
-class GenericScopedReadLock {
+template <typename MutexType>
+class SCOPED_LOCK GenericScopedReadLock {
public:
- explicit GenericScopedReadLock(MutexType *mu)
- : mu_(mu) {
+ explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
mu_->ReadLock();
}
- ~GenericScopedReadLock() {
- mu_->ReadUnlock();
- }
+ ~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); }
private:
MutexType *mu_;
- GenericScopedReadLock(const GenericScopedReadLock&);
- void operator=(const GenericScopedReadLock&);
+ GenericScopedReadLock(const GenericScopedReadLock &) = delete;
+ void operator=(const GenericScopedReadLock &) = delete;
};
typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
typedef GenericScopedLock<BlockingMutex> BlockingMutexLock;
typedef GenericScopedLock<RWMutex> RWMutexLock;
typedef GenericScopedReadLock<RWMutex> RWMutexReadLock;
+typedef GenericScopedLock<Mutex> Lock;
+typedef GenericScopedReadLock<Mutex> ReadLock;
} // namespace __sanitizer
return _REAL(munmap, addr, length);
}
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+ void *new_address) {
+ CHECK(false && "internal_mremap is unimplemented on NetBSD");
+ return 0;
+}
+
int internal_mprotect(void *addr, uptr length, int prot) {
DEFINE__REAL(int, mprotect, void *a, uptr b, int c);
return _REAL(mprotect, addr, length, prot);
}
+int internal_madvise(uptr addr, uptr length, int advice) {
+ DEFINE__REAL(int, madvise, void *a, uptr b, int c);
+ return _REAL(madvise, (void *)addr, length, advice);
+}
+
uptr internal_close(fd_t fd) {
CHECK(&_sys_close);
return _sys_close(fd);
Die(); // Unreachable.
}
-unsigned int internal_sleep(unsigned int seconds) {
+void internal_usleep(u64 useconds) {
struct timespec ts;
- ts.tv_sec = seconds;
- ts.tv_nsec = 0;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
CHECK(&_sys___nanosleep50);
- int res = _sys___nanosleep50(&ts, &ts);
- if (res)
- return ts.tv_sec;
- return 0;
+ _sys___nanosleep50(&ts, &ts);
}
uptr internal_execve(const char *filename, char *const argv[],
-//===-- sanitizer_openbsd.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between various sanitizers' runtime libraries and
-// implements Solaris-specific functions.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_platform.h"
-#if SANITIZER_OPENBSD
-
-#include <stdio.h>
-
-#include "sanitizer_common.h"
-#include "sanitizer_flags.h"
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_placement_new.h"
-#include "sanitizer_platform_limits_posix.h"
-#include "sanitizer_procmaps.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <pthread.h>
-#include <sched.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <sys/shm.h>
-#include <sys/sysctl.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-extern char **environ;
-
-namespace __sanitizer {
-
-uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd,
- u64 offset) {
- return (uptr)mmap(addr, length, prot, flags, fd, offset);
-}
-
-uptr internal_munmap(void *addr, uptr length) { return munmap(addr, length); }
-
-int internal_mprotect(void *addr, uptr length, int prot) {
- return mprotect(addr, length, prot);
-}
-
-int internal_sysctlbyname(const char *sname, void *oldp, uptr *oldlenp,
- const void *newp, uptr newlen) {
- Printf("internal_sysctlbyname not implemented for OpenBSD");
- Die();
- return 0;
-}
-
-uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
- // On OpenBSD we cannot get the full path
- struct kinfo_proc kp;
- uptr kl;
- const int Mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()};
- if (internal_sysctl(Mib, ARRAY_SIZE(Mib), &kp, &kl, NULL, 0) != -1)
- return internal_snprintf(buf,
- (KI_MAXCOMLEN < buf_len ? KI_MAXCOMLEN : buf_len),
- "%s", kp.p_comm);
- return (uptr)0;
-}
-
-static void GetArgsAndEnv(char ***argv, char ***envp) {
- uptr nargv;
- uptr nenv;
- int argvmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ARGV};
- int envmib[4] = {CTL_KERN, KERN_PROC_ARGS, getpid(), KERN_PROC_ENV};
- if (internal_sysctl(argvmib, 4, NULL, &nargv, NULL, 0) == -1) {
- Printf("sysctl KERN_PROC_NARGV failed\n");
- Die();
- }
- if (internal_sysctl(envmib, 4, NULL, &nenv, NULL, 0) == -1) {
- Printf("sysctl KERN_PROC_NENV failed\n");
- Die();
- }
- if (internal_sysctl(argvmib, 4, &argv, &nargv, NULL, 0) == -1) {
- Printf("sysctl KERN_PROC_ARGV failed\n");
- Die();
- }
- if (internal_sysctl(envmib, 4, &envp, &nenv, NULL, 0) == -1) {
- Printf("sysctl KERN_PROC_ENV failed\n");
- Die();
- }
-}
-
-char **GetArgv() {
- char **argv, **envp;
- GetArgsAndEnv(&argv, &envp);
- return argv;
-}
-
-char **GetEnviron() {
- char **argv, **envp;
- GetArgsAndEnv(&argv, &envp);
- return envp;
-}
-
-void ReExec() {
- UNIMPLEMENTED();
-}
-
-} // namespace __sanitizer
-
-#endif // SANITIZER_OPENBSD
#define SANITIZER_PLATFORM_H
#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
- !defined(__OpenBSD__) && !defined(__APPLE__) && !defined(_WIN32) && \
- !defined(__Fuchsia__) && !defined(__rtems__) && \
- !(defined(__sun__) && defined(__svr4__))
-# error "This operating system is not supported"
+ !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \
+ !(defined(__sun__) && defined(__svr4__))
+# error "This operating system is not supported"
+#endif
+
+// Get __GLIBC__ on a glibc platform. Exclude Android: features.h includes C
+// function declarations into a .S file which doesn't compile.
+// https://crbug.com/1162741
+#if __has_include(<features.h>) && !defined(__ANDROID__)
+#include <features.h>
#endif
#if defined(__linux__)
# define SANITIZER_LINUX 0
#endif
+#if defined(__GLIBC__)
+# define SANITIZER_GLIBC 1
+#else
+# define SANITIZER_GLIBC 0
+#endif
+
#if defined(__FreeBSD__)
# define SANITIZER_FREEBSD 1
#else
# define SANITIZER_NETBSD 0
#endif
-#if defined(__OpenBSD__)
-# define SANITIZER_OPENBSD 1
-#else
-# define SANITIZER_OPENBSD 0
-#endif
-
#if defined(__sun__) && defined(__svr4__)
# define SANITIZER_SOLARIS 1
#else
#if defined(__APPLE__)
# define SANITIZER_MAC 1
# include <TargetConditionals.h>
+# if TARGET_OS_OSX
+# define SANITIZER_OSX 1
+# else
+# define SANITIZER_OSX 0
+# endif
# if TARGET_OS_IPHONE
# define SANITIZER_IOS 1
# else
# define SANITIZER_MAC 0
# define SANITIZER_IOS 0
# define SANITIZER_IOSSIM 0
+# define SANITIZER_OSX 0
#endif
#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH
# define SANITIZER_FUCHSIA 0
#endif
-#if defined(__rtems__)
-# define SANITIZER_RTEMS 1
-#else
-# define SANITIZER_RTEMS 0
-#endif
-
#define SANITIZER_POSIX \
(SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \
- SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_SOLARIS)
+ SANITIZER_NETBSD || SANITIZER_SOLARIS)
#if __LP64__ || defined(_WIN64)
# define SANITIZER_WORDSIZE 64
# define SANITIZER_SOLARIS32 0
#endif
-#if defined(__myriad2__)
-# define SANITIZER_MYRIAD2 1
+#if defined(__riscv) && (__riscv_xlen == 64)
+#define SANITIZER_RISCV64 1
#else
-# define SANITIZER_MYRIAD2 0
+#define SANITIZER_RISCV64 0
#endif
// By default we allow to use SizeClassAllocator64 on 64-bit platform.
// FIXME: this value should be different on different platforms. Larger values
// will still work but will consume more memory for TwoLevelByteMap.
#if defined(__mips__)
+#if SANITIZER_GO && defined(__mips64)
+#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+#else
# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
+#endif
+#elif SANITIZER_RISCV64
+#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
#elif defined(__aarch64__)
# if SANITIZER_MAC
-// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
-# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
+# if SANITIZER_OSX || SANITIZER_IOSSIM
+# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
+# else
+ // Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
+# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
+# endif
# else
# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
# endif
#endif
#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
# define SANITIZER_MADVISE_DONTNEED MADV_FREE
#else
# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
# define SANITIZER_CACHE_LINE_SIZE 64
#endif
-// Enable offline markup symbolizer for Fuchsia and RTEMS.
-#if SANITIZER_FUCHSIA || SANITIZER_RTEMS
-#define SANITIZER_SYMBOLIZER_MARKUP 1
+// Enable offline markup symbolizer for Fuchsia.
+#if SANITIZER_FUCHSIA
+# define SANITIZER_SYMBOLIZER_MARKUP 1
#else
#define SANITIZER_SYMBOLIZER_MARKUP 0
#endif
#include "sanitizer_glibc_version.h"
#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
#if SANITIZER_POSIX
-# define SI_POSIX 1
+#define SI_POSIX 1
#else
-# define SI_POSIX 0
+#define SI_POSIX 0
#endif
#if !SANITIZER_WINDOWS
-# define SI_WINDOWS 0
+#define SI_WINDOWS 0
#else
-# define SI_WINDOWS 1
+#define SI_WINDOWS 1
#endif
#if SI_WINDOWS && SI_POSIX
-# error "Windows is not POSIX!"
+#error "Windows is not POSIX!"
#endif
#if SI_POSIX
-# include "sanitizer_platform_limits_freebsd.h"
-# include "sanitizer_platform_limits_netbsd.h"
-# include "sanitizer_platform_limits_openbsd.h"
-# include "sanitizer_platform_limits_posix.h"
-# include "sanitizer_platform_limits_solaris.h"
+#include "sanitizer_platform_limits_freebsd.h"
+#include "sanitizer_platform_limits_netbsd.h"
+#include "sanitizer_platform_limits_posix.h"
+#include "sanitizer_platform_limits_solaris.h"
#endif
#if SANITIZER_LINUX && !SANITIZER_ANDROID
-# define SI_LINUX_NOT_ANDROID 1
+#define SI_LINUX_NOT_ANDROID 1
#else
-# define SI_LINUX_NOT_ANDROID 0
+#define SI_LINUX_NOT_ANDROID 0
#endif
-#if SANITIZER_ANDROID
-# define SI_ANDROID 1
+#if SANITIZER_GLIBC
+#define SI_GLIBC 1
#else
-# define SI_ANDROID 0
+#define SI_GLIBC 0
#endif
-#if SANITIZER_FREEBSD
-# define SI_FREEBSD 1
+#if SANITIZER_ANDROID
+#define SI_ANDROID 1
#else
-# define SI_FREEBSD 0
+#define SI_ANDROID 0
#endif
-#if SANITIZER_NETBSD
-# define SI_NETBSD 1
+#if SANITIZER_FREEBSD
+#define SI_FREEBSD 1
#else
-# define SI_NETBSD 0
+#define SI_FREEBSD 0
#endif
-#if SANITIZER_OPENBSD
-#define SI_OPENBSD 1
+#if SANITIZER_NETBSD
+#define SI_NETBSD 1
#else
-#define SI_OPENBSD 0
+#define SI_NETBSD 0
#endif
#if SANITIZER_LINUX
-# define SI_LINUX 1
+#define SI_LINUX 1
#else
-# define SI_LINUX 0
+#define SI_LINUX 0
#endif
#if SANITIZER_MAC
-# define SI_MAC 1
-# define SI_NOT_MAC 0
+#define SI_MAC 1
+#define SI_NOT_MAC 0
#else
-# define SI_MAC 0
-# define SI_NOT_MAC 1
+#define SI_MAC 0
+#define SI_NOT_MAC 1
#endif
#if SANITIZER_IOS
-# define SI_IOS 1
+#define SI_IOS 1
#else
-# define SI_IOS 0
+#define SI_IOS 0
#endif
#if SANITIZER_IOSSIM
-# define SI_IOSSIM 1
+#define SI_IOSSIM 1
#else
-# define SI_IOSSIM 0
+#define SI_IOSSIM 0
#endif
#if SANITIZER_WATCHOS
-# define SI_WATCHOS 1
+#define SI_WATCHOS 1
#else
-# define SI_WATCHOS 0
+#define SI_WATCHOS 0
#endif
#if SANITIZER_TVOS
-# define SI_TVOS 1
+#define SI_TVOS 1
#else
-# define SI_TVOS 0
+#define SI_TVOS 0
#endif
#if SANITIZER_FUCHSIA
-# define SI_NOT_FUCHSIA 0
-#else
-# define SI_NOT_FUCHSIA 1
-#endif
-
-#if SANITIZER_RTEMS
-# define SI_NOT_RTEMS 0
+#define SI_NOT_FUCHSIA 0
#else
-# define SI_NOT_RTEMS 1
+#define SI_NOT_FUCHSIA 1
#endif
#if SANITIZER_SOLARIS
-# define SI_SOLARIS 1
+#define SI_SOLARIS 1
#else
-# define SI_SOLARIS 0
+#define SI_SOLARIS 0
#endif
#if SANITIZER_SOLARIS32
-# define SI_SOLARIS32 1
+#define SI_SOLARIS32 1
#else
-# define SI_SOLARIS32 0
+#define SI_SOLARIS32 0
#endif
#if SANITIZER_POSIX && !SANITIZER_MAC
-# define SI_POSIX_NOT_MAC 1
+#define SI_POSIX_NOT_MAC 1
#else
-# define SI_POSIX_NOT_MAC 0
+#define SI_POSIX_NOT_MAC 0
#endif
#if SANITIZER_LINUX && !SANITIZER_FREEBSD
-# define SI_LINUX_NOT_FREEBSD 1
-# else
-# define SI_LINUX_NOT_FREEBSD 0
+#define SI_LINUX_NOT_FREEBSD 1
+#else
+#define SI_LINUX_NOT_FREEBSD 0
#endif
#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
#define SANITIZER_INTERCEPT_BCMP \
SANITIZER_INTERCEPT_MEMCMP && \
- ((SI_POSIX && _GNU_SOURCE) || SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+ ((SI_POSIX && _GNU_SOURCE) || SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_STRNDUP SI_POSIX
-#define SANITIZER_INTERCEPT___STRNDUP SI_LINUX_NOT_FREEBSD
+#define SANITIZER_INTERCEPT___STRNDUP SI_GLIBC
#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070
-# define SI_MAC_DEPLOYMENT_BELOW_10_7 1
+#define SI_MAC_DEPLOYMENT_BELOW_10_7 1
#else
-# define SI_MAC_DEPLOYMENT_BELOW_10_7 0
+#define SI_MAC_DEPLOYMENT_BELOW_10_7 0
#endif
// memmem on Darwin doesn't exist on 10.6
// FIXME: enable memmem on Windows.
#define SANITIZER_INTERCEPT_MEMMEM (SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_7)
#define SANITIZER_INTERCEPT_MEMCHR SI_NOT_FUCHSIA
-#define SANITIZER_INTERCEPT_MEMRCHR \
- (SI_FREEBSD || SI_LINUX || SI_NETBSD || SI_OPENBSD)
+#define SANITIZER_INTERCEPT_MEMRCHR (SI_FREEBSD || SI_LINUX || SI_NETBSD)
#define SANITIZER_INTERCEPT_READ SI_POSIX
#define SANITIZER_INTERCEPT_PREAD SI_POSIX
#define SANITIZER_INTERCEPT_FPUTS SI_POSIX
#define SANITIZER_INTERCEPT_PUTS SI_POSIX
-#define SANITIZER_INTERCEPT_PREAD64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
-#define SANITIZER_INTERCEPT_PWRITE64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_PREAD64 (SI_GLIBC || SI_SOLARIS32)
+#define SANITIZER_INTERCEPT_PWRITE64 (SI_GLIBC || SI_SOLARIS32)
#define SANITIZER_INTERCEPT_READV SI_POSIX
#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
#define SANITIZER_INTERCEPT_PREADV \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_PWRITEV SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PREADV64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PWRITEV64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PREADV64 SI_GLIBC
+#define SANITIZER_INTERCEPT_PWRITEV64 SI_GLIBC
-#define SANITIZER_INTERCEPT_PRCTL SI_LINUX
+#define SANITIZER_INTERCEPT_PRCTL SI_LINUX
#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS SI_POSIX
#define SANITIZER_INTERCEPT_STRPTIME SI_POSIX
#define SANITIZER_INTERCEPT_SCANF SI_POSIX
-#define SANITIZER_INTERCEPT_ISOC99_SCANF SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_ISOC99_SCANF SI_GLIBC
#ifndef SANITIZER_INTERCEPT_PRINTF
-# define SANITIZER_INTERCEPT_PRINTF SI_POSIX
-# define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
-# define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PRINTF SI_POSIX
+#define SANITIZER_INTERCEPT_PRINTF_L (SI_FREEBSD || SI_NETBSD)
+#define SANITIZER_INTERCEPT_ISOC99_PRINTF SI_GLIBC
#endif
#define SANITIZER_INTERCEPT___PRINTF_CHK \
- (SANITIZER_INTERCEPT_PRINTF && SI_LINUX_NOT_ANDROID)
+ (SANITIZER_INTERCEPT_PRINTF && SI_GLIBC)
#define SANITIZER_INTERCEPT_FREXP SI_NOT_FUCHSIA
#define SANITIZER_INTERCEPT_FREXPF_FREXPL SI_POSIX
#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS SI_POSIX
-#define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
-#define SANITIZER_INTERCEPT_GETPWENT \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
-#define SANITIZER_INTERCEPT_FGETGRENT_R \
- (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETPWENT \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETGRENT_R (SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_FGETPWENT SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_GETPWENT_R \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_FGETPWENT_R \
- (SI_FREEBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_FGETPWENT_R (SI_FREEBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_SETPWENT \
(SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CLOCK_GETTIME \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID SI_LINUX
#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
#define SANITIZER_INTERCEPT_TIME SI_POSIX
-#define SANITIZER_INTERCEPT_GLOB SI_LINUX_NOT_ANDROID || SI_SOLARIS
-#define SANITIZER_INTERCEPT_GLOB64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
#define SANITIZER_INTERCEPT_WAIT SI_POSIX
#define SANITIZER_INTERCEPT_INET SI_POSIX
-#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
#define SANITIZER_INTERCEPT_GETADDRINFO SI_POSIX
#define SANITIZER_INTERCEPT_GETNAMEINFO SI_POSIX
#define SANITIZER_INTERCEPT_GETSOCKNAME SI_POSIX
(SI_FREEBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R \
(SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_GETHOSTENT_R \
- (SI_FREEBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
-#define SANITIZER_INTERCEPT_ACCEPT4 \
- (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_OPENBSD)
+#define SANITIZER_INTERCEPT_ACCEPT4 (SI_LINUX_NOT_ANDROID || SI_NETBSD)
#define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
#define SANITIZER_INTERCEPT_MODF SI_POSIX
#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
#define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
#define SANITIZER_INTERCEPT_READDIR SI_POSIX
#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
-#if SI_LINUX_NOT_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__))
+#if SI_LINUX_NOT_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+ defined(__s390__) || SANITIZER_RISCV64)
#define SANITIZER_INTERCEPT_PTRACE 1
#else
#define SANITIZER_INTERCEPT_PTRACE 0
#define SANITIZER_INTERCEPT___STRXFRM_L SI_LINUX
#define SANITIZER_INTERCEPT_WCSXFRM SI_POSIX
#define SANITIZER_INTERCEPT___WCSXFRM_L SI_LINUX
-#define SANITIZER_INTERCEPT_WCSNRTOMBS \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
-#define SANITIZER_INTERCEPT_WCRTOMB \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
-#define SANITIZER_INTERCEPT_WCTOMB \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCSNRTOMBS \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCRTOMB \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_WCTOMB \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_REALPATH SI_POSIX
-#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME \
- (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_CONFSTR \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME (SI_GLIBC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_CONFSTR \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_SCHED_GETPARAM SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_STRERROR SI_POSIX
#define SANITIZER_INTERCEPT_STRERROR_R SI_POSIX
#define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_SCANDIR \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
#define SANITIZER_INTERCEPT_POLL SI_POSIX
#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS
-#define SANITIZER_INTERCEPT_WORDEXP \
+#define SANITIZER_INTERCEPT_WORDEXP \
(SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
+ SI_SOLARIS) // NOLINT
#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS
#define SANITIZER_INTERCEPT_SIGSETOPS \
(SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_SIGSET_LOGICOPS SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_SIGPENDING SI_POSIX
#define SANITIZER_INTERCEPT_SIGPROCMASK SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_SIGMASK SI_POSIX
#define SANITIZER_INTERCEPT_BACKTRACE \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_GETMNTENT SI_LINUX
#define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_STATFS \
#define SANITIZER_INTERCEPT_STATFS64 \
(((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_STATVFS \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
-#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
#define SANITIZER_INTERCEPT_ETHER_HOST \
(SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
#define SANITIZER_INTERCEPT_SHMCTL \
(((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) || \
- SI_NETBSD || SI_OPENBSD || SI_SOLARIS) // NOLINT
-#define SANITIZER_INTERCEPT_RANDOM_R SI_LINUX_NOT_ANDROID
+ SI_NETBSD || SI_SOLARIS) // NOLINT
+#define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
(SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED (SI_POSIX && !SI_OPENBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC
+#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
- (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE (SI_POSIX && !SI_OPENBSD)
+ (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL \
(SI_MAC || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING \
(SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED \
- (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
-#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED \
- (SI_POSIX && !SI_NETBSD && !SI_OPENBSD)
+ (SI_POSIX && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP SI_GLIBC
+#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED (SI_POSIX && !SI_NETBSD)
#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK \
(SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
- (SI_LINUX_NOT_ANDROID && !SI_NETBSD && !SI_OPENBSD)
+ (SI_LINUX_NOT_ANDROID && !SI_NETBSD)
#define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD
#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
-#define SANITIZER_INTERCEPT_TMPNAM_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_PTSNAME SI_LINUX
+#define SANITIZER_INTERCEPT_PTSNAME_R SI_LINUX
#define SANITIZER_INTERCEPT_TTYNAME SI_POSIX
#define SANITIZER_INTERCEPT_TTYNAME_R SI_POSIX
#define SANITIZER_INTERCEPT_TEMPNAM SI_POSIX
#define SANITIZER_INTERCEPT_LGAMMAL (SI_POSIX && !SI_NETBSD)
#define SANITIZER_INTERCEPT_LGAMMA_R (SI_FREEBSD || SI_LINUX || SI_SOLARIS)
#define SANITIZER_INTERCEPT_LGAMMAL_R SI_LINUX_NOT_ANDROID || SI_SOLARIS
-#define SANITIZER_INTERCEPT_DRAND48_R SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_RAND_R \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_LINUX_NOT_ANDROID || \
- SI_SOLARIS)
+#define SANITIZER_INTERCEPT_DRAND48_R SI_GLIBC
+#define SANITIZER_INTERCEPT_RAND_R \
+ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_ICONV \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_TIMES SI_POSIX
// FIXME: getline seems to be available on OSX 10.7
#define SANITIZER_INTERCEPT_GETLINE \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT__EXIT \
- (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_MAC || SI_SOLARIS)
+ (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_MUTEX SI_POSIX
-#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_GLIBC
#define SANITIZER_INTERCEPT___LIBC_MUTEX SI_NETBSD
#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP \
- (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_TLS_GET_ADDR \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
#define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
#define SANITIZER_INTERCEPT_GETRESID SI_LINUX
-#define SANITIZER_INTERCEPT_GETIFADDRS \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
- SI_SOLARIS)
-#define SANITIZER_INTERCEPT_IF_INDEXTONAME \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
- SI_SOLARIS)
+#define SANITIZER_INTERCEPT_GETIFADDRS \
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_IF_INDEXTONAME \
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CAPGET SI_LINUX_NOT_ANDROID
#if SI_LINUX && defined(__arm__)
#define SANITIZER_INTERCEPT_AEABI_MEM 1
#else
#define SANITIZER_INTERCEPT_AEABI_MEM 0
#endif
-#define SANITIZER_INTERCEPT___BZERO SI_MAC || SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___BZERO SI_MAC || SI_GLIBC
#define SANITIZER_INTERCEPT_BZERO SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_FTIME \
- (!SI_FREEBSD && !SI_NETBSD && !SI_OPENBSD && SI_POSIX)
-#define SANITIZER_INTERCEPT_XDR SI_LINUX_NOT_ANDROID || SI_SOLARIS
+#define SANITIZER_INTERCEPT_FTIME (!SI_FREEBSD && !SI_NETBSD && SI_POSIX)
+#define SANITIZER_INTERCEPT_XDR (SI_GLIBC || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_XDRREC SI_GLIBC
#define SANITIZER_INTERCEPT_TSEARCH \
- (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_LINUX_NOT_ANDROID
+ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_LIBIO_INTERNALS SI_GLIBC
#define SANITIZER_INTERCEPT_FOPEN SI_POSIX
-#define SANITIZER_INTERCEPT_FOPEN64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_FOPEN64 (SI_GLIBC || SI_SOLARIS32)
#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM \
- (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_OBSTACK SI_LINUX_NOT_ANDROID
+ (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_OBSTACK SI_GLIBC
#define SANITIZER_INTERCEPT_FFLUSH SI_POSIX
#define SANITIZER_INTERCEPT_FCLOSE SI_POSIX
#ifndef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE
-#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \
- (SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_LINUX_NOT_ANDROID || SI_MAC || \
- SI_SOLARIS)
+#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE \
+ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_MAC || SI_SOLARIS)
#endif
#define SANITIZER_INTERCEPT_GETPASS \
- (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD || SI_OPENBSD)
+ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_NETBSD)
#define SANITIZER_INTERCEPT_TIMERFD SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_MLOCKX SI_POSIX
#define SANITIZER_INTERCEPT_SEM \
(SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
-#define SANITIZER_INTERCEPT_MINCORE \
- (SI_LINUX || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
#define SANITIZER_INTERCEPT_CTERMID \
- (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+ (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_CTERMID_R (SI_MAC || SI_FREEBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPTOR_HOOKS \
- (SI_LINUX || SI_MAC || SI_WINDOWS || SI_NETBSD)
+ (SI_LINUX || SI_MAC || SI_WINDOWS || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_RECV_RECVFROM SI_POSIX
#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
#define SANITIZER_INTERCEPT_STAT \
- (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_OPENBSD || SI_SOLARIS)
+ (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS)
#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
(SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_GETLOADAVG \
- (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_OPENBSD)
+ (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
#define SANITIZER_INTERCEPT_MMAP SI_POSIX
#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
-#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
- SI_NOT_RTEMS)
-#define SANITIZER_INTERCEPT_MEMALIGN \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_RTEMS)
-#define SANITIZER_INTERCEPT_PVALLOC \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
- SI_NOT_RTEMS)
-#define SANITIZER_INTERCEPT_CFREE \
- (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && !SI_OPENBSD && SI_NOT_FUCHSIA && \
- SI_NOT_RTEMS)
+#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID)
+#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
+#define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
+#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID)
+#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
-#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS)
-#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE \
- (!SI_MAC && !SI_OPENBSD && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
+#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID
-#define SANITIZER_INTERCEPT_ACCT (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_ACCT (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_USER_FROM_UID SI_NETBSD
#define SANITIZER_INTERCEPT_UID_FROM_USER SI_NETBSD
#define SANITIZER_INTERCEPT_GROUP_FROM_GID SI_NETBSD
#define SANITIZER_INTERCEPT_GID_FROM_GROUP SI_NETBSD
-#define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_GETGROUPLIST (SI_NETBSD || SI_OPENBSD)
-#define SANITIZER_INTERCEPT_STRLCPY \
- (SI_NETBSD || SI_FREEBSD || SI_OPENBSD || SI_MAC || SI_ANDROID)
+#define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_GETGROUPLIST SI_NETBSD
+#define SANITIZER_INTERCEPT_STRLCPY \
+ (SI_NETBSD || SI_FREEBSD || SI_MAC || SI_ANDROID)
#define SANITIZER_INTERCEPT_NAME_TO_HANDLE_AT SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_OPEN_BY_HANDLE_AT SI_LINUX_NOT_ANDROID
#define SANITIZER_INTERCEPT_READLINK SI_POSIX
#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000
-# define SI_MAC_DEPLOYMENT_BELOW_10_10 1
+#define SI_MAC_DEPLOYMENT_BELOW_10_10 1
#else
-# define SI_MAC_DEPLOYMENT_BELOW_10_10 0
+#define SI_MAC_DEPLOYMENT_BELOW_10_10 0
#endif
#define SANITIZER_INTERCEPT_READLINKAT \
(SI_POSIX && !SI_MAC_DEPLOYMENT_BELOW_10_10)
-#define SANITIZER_INTERCEPT_DEVNAME (SI_NETBSD || SI_OPENBSD || SI_FREEBSD)
+#define SANITIZER_INTERCEPT_DEVNAME (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_DEVNAME_R (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD || SI_FREEBSD)
#define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD || SI_LINUX)
-#define SANITIZER_INTERCEPT_PROTOENT_R (SI_LINUX_NOT_ANDROID)
+#define SANITIZER_INTERCEPT_PROTOENT_R SI_GLIBC
#define SANITIZER_INTERCEPT_NETENT SI_NETBSD
-#define SANITIZER_INTERCEPT_SETVBUF (SI_NETBSD || SI_FREEBSD || \
- SI_LINUX || SI_MAC)
+#define SANITIZER_INTERCEPT_SETVBUF \
+ (SI_NETBSD || SI_FREEBSD || SI_LINUX || SI_MAC)
#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
#define SANITIZER_INTERCEPT_MI_VECTOR_HASH SI_NETBSD
#define SANITIZER_INTERCEPT_GETVFSSTAT SI_NETBSD
#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD
#define SANITIZER_INTERCEPT_QSORT \
(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
-#define SANITIZER_INTERCEPT_QSORT_R (SI_LINUX && !SI_ANDROID)
+#define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
// sigaltstack on i386 macOS cannot be intercepted due to setjmp()
// calling it and assuming that it does not clobber registers.
#define SANITIZER_INTERCEPT_SIGALTSTACK \
(SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
#define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
+#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
+
+// This macro gives a way for downstream users to override the above
+// interceptor macros irrespective of the platform they are on. They have
+// to do two things:
+// 1. Build compiler-rt with -DSANITIZER_OVERRIDE_INTERCEPTORS.
+// 2. Provide a header file named sanitizer_intercept_overriders.h in the
+// include path for their compiler-rt build.
+// An example of an overrider for strlen interceptor that one can list in
+// sanitizer_intercept_overriders.h is as follows:
+//
+// #ifdef SANITIZER_INTERCEPT_STRLEN
+// #undef SANITIZER_INTERCEPT_STRLEN
+// #define SANITIZER_INTERCEPT_STRLEN <value of choice>
+// #endif
+//
+// This "feature" is useful for downstream users who do not want some of
+// their libc funtions to be intercepted. They can selectively disable
+// interception of those functions.
+#ifdef SANITIZER_OVERRIDE_INTERCEPTORS
+#include <sanitizer_intercept_overriders.h>
+#endif
#endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/time.h>
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-W#warnings"
#include <sys/timeb.h>
+#pragma clang diagnostic pop
#include <sys/times.h>
#include <sys/timespec.h>
#include <sys/types.h>
#include <sys/shm.h>
#undef _KERNEL
-#undef INLINE // to avoid clashes with sanitizers' definitions
-
#undef IOC_DIRMASK
// Include these after system headers to avoid name clashes and ambiguities.
#include <sys/chio.h>
#include <sys/clockctl.h>
#include <sys/cpuio.h>
+#include <sys/dkbad.h>
#include <sys/dkio.h>
#include <sys/drvctlio.h>
#include <sys/dvdio.h>
#include <sys/resource.h>
#include <sys/sem.h>
+#include <sys/scsiio.h>
#include <sys/sha1.h>
#include <sys/sha2.h>
#include <sys/shm.h>
#include <dev/ir/irdaio.h>
#include <dev/isa/isvio.h>
#include <dev/isa/wtreg.h>
+#if __has_include(<dev/iscsi/iscsi_ioctl.h>)
#include <dev/iscsi/iscsi_ioctl.h>
+#else
+/* Fallback for MKISCSI=no */
+
+typedef struct {
+ uint32_t status;
+ uint32_t session_id;
+ uint32_t connection_id;
+} iscsi_conn_status_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint16_t interface_version;
+ uint16_t major;
+ uint16_t minor;
+ uint8_t version_string[224];
+} iscsi_get_version_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t session_id;
+ uint32_t connection_id;
+ struct {
+ unsigned int immediate : 1;
+ } options;
+ uint64_t lun;
+ scsireq_t req; /* from <sys/scsiio.h> */
+} iscsi_iocommand_parameters_t;
+
+typedef enum {
+ ISCSI_AUTH_None = 0,
+ ISCSI_AUTH_CHAP = 1,
+ ISCSI_AUTH_KRB5 = 2,
+ ISCSI_AUTH_SRP = 3
+} iscsi_auth_types_t;
+
+typedef enum {
+ ISCSI_LOGINTYPE_DISCOVERY = 0,
+ ISCSI_LOGINTYPE_NOMAP = 1,
+ ISCSI_LOGINTYPE_MAP = 2
+} iscsi_login_session_type_t;
+
+typedef enum { ISCSI_DIGEST_None = 0, ISCSI_DIGEST_CRC32C = 1 } iscsi_digest_t;
+
+typedef enum {
+ ISCSI_SESSION_TERMINATED = 1,
+ ISCSI_CONNECTION_TERMINATED,
+ ISCSI_RECOVER_CONNECTION,
+ ISCSI_DRIVER_TERMINATING
+} iscsi_event_t;
+
+typedef struct {
+ unsigned int mutual_auth : 1;
+ unsigned int is_secure : 1;
+ unsigned int auth_number : 4;
+ iscsi_auth_types_t auth_type[4];
+} iscsi_auth_info_t;
+
+typedef struct {
+ uint32_t status;
+ int socket;
+ struct {
+ unsigned int HeaderDigest : 1;
+ unsigned int DataDigest : 1;
+ unsigned int MaxConnections : 1;
+ unsigned int DefaultTime2Wait : 1;
+ unsigned int DefaultTime2Retain : 1;
+ unsigned int MaxRecvDataSegmentLength : 1;
+ unsigned int auth_info : 1;
+ unsigned int user_name : 1;
+ unsigned int password : 1;
+ unsigned int target_password : 1;
+ unsigned int TargetName : 1;
+ unsigned int TargetAlias : 1;
+ unsigned int ErrorRecoveryLevel : 1;
+ } is_present;
+ iscsi_auth_info_t auth_info;
+ iscsi_login_session_type_t login_type;
+ iscsi_digest_t HeaderDigest;
+ iscsi_digest_t DataDigest;
+ uint32_t session_id;
+ uint32_t connection_id;
+ uint32_t MaxRecvDataSegmentLength;
+ uint16_t MaxConnections;
+ uint16_t DefaultTime2Wait;
+ uint16_t DefaultTime2Retain;
+ uint16_t ErrorRecoveryLevel;
+ void *user_name;
+ void *password;
+ void *target_password;
+ void *TargetName;
+ void *TargetAlias;
+} iscsi_login_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t session_id;
+} iscsi_logout_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t event_id;
+} iscsi_register_event_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t session_id;
+ uint32_t connection_id;
+} iscsi_remove_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t session_id;
+ void *response_buffer;
+ uint32_t response_size;
+ uint32_t response_used;
+ uint32_t response_total;
+ uint8_t key[224];
+} iscsi_send_targets_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint8_t InitiatorName[224];
+ uint8_t InitiatorAlias[224];
+ uint8_t ISID[6];
+} iscsi_set_node_name_parameters_t;
+
+typedef struct {
+ uint32_t status;
+ uint32_t event_id;
+ iscsi_event_t event_kind;
+ uint32_t session_id;
+ uint32_t connection_id;
+ uint32_t reason;
+} iscsi_wait_event_parameters_t;
+
+#define ISCSI_GET_VERSION _IOWR(0, 1, iscsi_get_version_parameters_t)
+#define ISCSI_LOGIN _IOWR(0, 2, iscsi_login_parameters_t)
+#define ISCSI_LOGOUT _IOWR(0, 3, iscsi_logout_parameters_t)
+#define ISCSI_ADD_CONNECTION _IOWR(0, 4, iscsi_login_parameters_t)
+#define ISCSI_RESTORE_CONNECTION _IOWR(0, 5, iscsi_login_parameters_t)
+#define ISCSI_REMOVE_CONNECTION _IOWR(0, 6, iscsi_remove_parameters_t)
+#define ISCSI_CONNECTION_STATUS _IOWR(0, 7, iscsi_conn_status_parameters_t)
+#define ISCSI_SEND_TARGETS _IOWR(0, 8, iscsi_send_targets_parameters_t)
+#define ISCSI_SET_NODE_NAME _IOWR(0, 9, iscsi_set_node_name_parameters_t)
+#define ISCSI_IO_COMMAND _IOWR(0, 10, iscsi_iocommand_parameters_t)
+#define ISCSI_REGISTER_EVENT _IOWR(0, 11, iscsi_register_event_parameters_t)
+#define ISCSI_DEREGISTER_EVENT _IOWR(0, 12, iscsi_register_event_parameters_t)
+#define ISCSI_WAIT_EVENT _IOWR(0, 13, iscsi_wait_event_parameters_t)
+#define ISCSI_POLL_EVENT _IOWR(0, 14, iscsi_wait_event_parameters_t)
+#endif
#include <dev/ofw/openfirmio.h>
#include <dev/pci/amrio.h>
#include <dev/pci/mlyreg.h>
#include "sanitizer_platform_limits_netbsd.h"
namespace __sanitizer {
-void *__sanitizer_get_link_map_by_dlopen_handle(void* handle) {
+void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) {
void *p = nullptr;
return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
}
namespace __sanitizer {
void *__sanitizer_get_link_map_by_dlopen_handle(void *handle);
-# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
- (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
+#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
+ (link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
extern unsigned struct_utsname_sz;
extern unsigned struct_stat_sz;
extern unsigned struct_nvlist_ref_sz;
extern unsigned struct_StringList_sz;
-
// A special value to mark ioctls that are not present on the target platform,
// when it can not be determined without including any system headers.
extern const unsigned IOCTL_NOT_PRESENT;
-
extern unsigned IOCTL_AFM_ADDFMAP;
extern unsigned IOCTL_AFM_DELFMAP;
extern unsigned IOCTL_AFM_CLEANFMAP;
-//===-- sanitizer_platform_limits_openbsd.cpp -----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of Sanitizer common code.
-//
-// Sizes and layouts of platform-specific NetBSD data structures.
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_platform.h"
-
-#if SANITIZER_OPENBSD
-#include <arpa/inet.h>
-#include <dirent.h>
-#include <glob.h>
-#include <grp.h>
-#include <ifaddrs.h>
-#include <limits.h>
-#include <link_elf.h>
-#include <sys/socket.h>
-#include <net/if.h>
-#include <net/ppp_defs.h>
-#include <net/route.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/ip_mroute.h>
-#include <poll.h>
-#include <pthread.h>
-#include <pwd.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <soundcard.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <sys/filio.h>
-#include <sys/ipc.h>
-#include <sys/mman.h>
-#include <sys/mount.h>
-#include <sys/msg.h>
-#include <sys/mtio.h>
-#include <sys/ptrace.h>
-#include <sys/resource.h>
-#include <sys/shm.h>
-#include <sys/signal.h>
-#include <sys/sockio.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
-#include <sys/time.h>
-#include <sys/times.h>
-#include <sys/types.h>
-#include <sys/utsname.h>
-#include <term.h>
-#include <time.h>
-#include <utime.h>
-#include <utmp.h>
-#include <wchar.h>
-
-// Include these after system headers to avoid name clashes and ambiguities.
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_platform_limits_openbsd.h"
-
-namespace __sanitizer {
-unsigned struct_utsname_sz = sizeof(struct utsname);
-unsigned struct_stat_sz = sizeof(struct stat);
-unsigned struct_rusage_sz = sizeof(struct rusage);
-unsigned struct_tm_sz = sizeof(struct tm);
-unsigned struct_passwd_sz = sizeof(struct passwd);
-unsigned struct_group_sz = sizeof(struct group);
-unsigned siginfo_t_sz = sizeof(siginfo_t);
-unsigned struct_sigaction_sz = sizeof(struct sigaction);
-unsigned struct_stack_t_sz = sizeof(stack_t);
-unsigned struct_itimerval_sz = sizeof(struct itimerval);
-unsigned pthread_t_sz = sizeof(pthread_t);
-unsigned pthread_mutex_t_sz = sizeof(pthread_mutex_t);
-unsigned pthread_cond_t_sz = sizeof(pthread_cond_t);
-unsigned pid_t_sz = sizeof(pid_t);
-unsigned timeval_sz = sizeof(timeval);
-unsigned uid_t_sz = sizeof(uid_t);
-unsigned gid_t_sz = sizeof(gid_t);
-unsigned mbstate_t_sz = sizeof(mbstate_t);
-unsigned sigset_t_sz = sizeof(sigset_t);
-unsigned struct_timezone_sz = sizeof(struct timezone);
-unsigned struct_tms_sz = sizeof(struct tms);
-unsigned struct_sched_param_sz = sizeof(struct sched_param);
-unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
-unsigned struct_rlimit_sz = sizeof(struct rlimit);
-unsigned struct_timespec_sz = sizeof(struct timespec);
-unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
-unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
-unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
-unsigned struct_statvfs_sz = sizeof(struct statvfs);
-
-const uptr sig_ign = (uptr)SIG_IGN;
-const uptr sig_dfl = (uptr)SIG_DFL;
-const uptr sig_err = (uptr)SIG_ERR;
-const uptr sa_siginfo = (uptr)SA_SIGINFO;
-
-int shmctl_ipc_stat = (int)IPC_STAT;
-
-unsigned struct_utmp_sz = sizeof(struct utmp);
-
-int map_fixed = MAP_FIXED;
-
-int af_inet = (int)AF_INET;
-int af_inet6 = (int)AF_INET6;
-
-uptr __sanitizer_in_addr_sz(int af) {
- if (af == AF_INET)
- return sizeof(struct in_addr);
- else if (af == AF_INET6)
- return sizeof(struct in6_addr);
- else
- return 0;
-}
-
-unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
-
-int glob_nomatch = GLOB_NOMATCH;
-int glob_altdirfunc = GLOB_ALTDIRFUNC;
-
-unsigned path_max = PATH_MAX;
-
-const int si_SEGV_MAPERR = SEGV_MAPERR;
-const int si_SEGV_ACCERR = SEGV_ACCERR;
-} // namespace __sanitizer
-
-using namespace __sanitizer;
-
-COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t));
-
-COMPILER_CHECK(sizeof(socklen_t) == sizeof(unsigned));
-CHECK_TYPE_SIZE(pthread_key_t);
-
-CHECK_TYPE_SIZE(dl_phdr_info);
-CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr);
-CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name);
-CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
-CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
-
-CHECK_TYPE_SIZE(glob_t);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_flags);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_closedir);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_readdir);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
-CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
-
-CHECK_TYPE_SIZE(addrinfo);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_family);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_socktype);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
-CHECK_SIZE_AND_OFFSET(addrinfo, ai_next);
-
-CHECK_TYPE_SIZE(hostent);
-CHECK_SIZE_AND_OFFSET(hostent, h_name);
-CHECK_SIZE_AND_OFFSET(hostent, h_aliases);
-CHECK_SIZE_AND_OFFSET(hostent, h_addrtype);
-CHECK_SIZE_AND_OFFSET(hostent, h_length);
-CHECK_SIZE_AND_OFFSET(hostent, h_addr_list);
-
-CHECK_TYPE_SIZE(iovec);
-CHECK_SIZE_AND_OFFSET(iovec, iov_base);
-CHECK_SIZE_AND_OFFSET(iovec, iov_len);
-
-CHECK_TYPE_SIZE(msghdr);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
-CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
-
-CHECK_TYPE_SIZE(cmsghdr);
-CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
-CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
-CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
-
-COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
-CHECK_SIZE_AND_OFFSET(dirent, d_fileno);
-CHECK_SIZE_AND_OFFSET(dirent, d_off);
-CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
-
-CHECK_TYPE_SIZE(ifconf);
-CHECK_SIZE_AND_OFFSET(ifconf, ifc_len);
-CHECK_SIZE_AND_OFFSET(ifconf, ifc_ifcu);
-
-CHECK_TYPE_SIZE(pollfd);
-CHECK_SIZE_AND_OFFSET(pollfd, fd);
-CHECK_SIZE_AND_OFFSET(pollfd, events);
-CHECK_SIZE_AND_OFFSET(pollfd, revents);
-
-CHECK_TYPE_SIZE(nfds_t);
-
-CHECK_TYPE_SIZE(sigset_t);
-
-COMPILER_CHECK(sizeof(__sanitizer_sigaction) == sizeof(struct sigaction));
-// Can't write checks for sa_handler and sa_sigaction due to them being
-// preprocessor macros.
-CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
-
-CHECK_TYPE_SIZE(tm);
-CHECK_SIZE_AND_OFFSET(tm, tm_sec);
-CHECK_SIZE_AND_OFFSET(tm, tm_min);
-CHECK_SIZE_AND_OFFSET(tm, tm_hour);
-CHECK_SIZE_AND_OFFSET(tm, tm_mday);
-CHECK_SIZE_AND_OFFSET(tm, tm_mon);
-CHECK_SIZE_AND_OFFSET(tm, tm_year);
-CHECK_SIZE_AND_OFFSET(tm, tm_wday);
-CHECK_SIZE_AND_OFFSET(tm, tm_yday);
-CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
-CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
-CHECK_SIZE_AND_OFFSET(tm, tm_zone);
-
-CHECK_TYPE_SIZE(ipc_perm);
-CHECK_SIZE_AND_OFFSET(ipc_perm, cuid);
-CHECK_SIZE_AND_OFFSET(ipc_perm, cgid);
-CHECK_SIZE_AND_OFFSET(ipc_perm, uid);
-CHECK_SIZE_AND_OFFSET(ipc_perm, gid);
-CHECK_SIZE_AND_OFFSET(ipc_perm, mode);
-CHECK_SIZE_AND_OFFSET(ipc_perm, seq);
-CHECK_SIZE_AND_OFFSET(ipc_perm, key);
-
-CHECK_TYPE_SIZE(shmid_ds);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_perm);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_segsz);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_atime);
-CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_atimensec);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_dtime);
-CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_dtimensec);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_ctime);
-CHECK_SIZE_AND_OFFSET(shmid_ds, __shm_ctimensec);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_cpid);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_lpid);
-CHECK_SIZE_AND_OFFSET(shmid_ds, shm_nattch);
-
-CHECK_TYPE_SIZE(clock_t);
-
-CHECK_TYPE_SIZE(ifaddrs);
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_addr);
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask);
-// Compare against the union, because we can't reach into the union in a
-// compliant way.
-#ifdef ifa_dstaddr
-#undef ifa_dstaddr
-#endif
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr);
-CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
-
-CHECK_TYPE_SIZE(passwd);
-CHECK_SIZE_AND_OFFSET(passwd, pw_name);
-CHECK_SIZE_AND_OFFSET(passwd, pw_passwd);
-CHECK_SIZE_AND_OFFSET(passwd, pw_uid);
-CHECK_SIZE_AND_OFFSET(passwd, pw_gid);
-CHECK_SIZE_AND_OFFSET(passwd, pw_dir);
-CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
-
-CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
-
-CHECK_TYPE_SIZE(group);
-CHECK_SIZE_AND_OFFSET(group, gr_name);
-CHECK_SIZE_AND_OFFSET(group, gr_passwd);
-CHECK_SIZE_AND_OFFSET(group, gr_gid);
-CHECK_SIZE_AND_OFFSET(group, gr_mem);
-
-#endif // SANITIZER_OPENBSD
-//===-- sanitizer_platform_limits_openbsd.h -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of Sanitizer common code.
-//
-// Sizes and layouts of platform-specific OpenBSD data structures.
-//===----------------------------------------------------------------------===//
-
-#ifndef SANITIZER_PLATFORM_LIMITS_OPENBSD_H
-#define SANITIZER_PLATFORM_LIMITS_OPENBSD_H
-
-#if SANITIZER_OPENBSD
-
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_platform.h"
-
-#define _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, shift) \
- ((link_map *)((handle) == nullptr ? nullptr : ((char *)(handle) + (shift))))
-
-#if defined(__x86_64__)
-#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
- _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 312)
-#elif defined(__i386__)
-#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
- _GET_LINK_MAP_BY_DLOPEN_HANDLE(handle, 164)
-#endif
-
-#define RLIMIT_AS RLIMIT_DATA
-
-namespace __sanitizer {
-extern unsigned struct_utsname_sz;
-extern unsigned struct_stat_sz;
-extern unsigned struct_rusage_sz;
-extern unsigned siginfo_t_sz;
-extern unsigned struct_itimerval_sz;
-extern unsigned pthread_t_sz;
-extern unsigned pthread_mutex_t_sz;
-extern unsigned pthread_cond_t_sz;
-extern unsigned pid_t_sz;
-extern unsigned timeval_sz;
-extern unsigned uid_t_sz;
-extern unsigned gid_t_sz;
-extern unsigned mbstate_t_sz;
-extern unsigned struct_timezone_sz;
-extern unsigned struct_tms_sz;
-extern unsigned struct_itimerspec_sz;
-extern unsigned struct_sigevent_sz;
-extern unsigned struct_stack_t_sz;
-extern unsigned struct_statfs_sz;
-extern unsigned struct_sockaddr_sz;
-
-extern unsigned struct_rlimit_sz;
-extern unsigned struct_utimbuf_sz;
-extern unsigned struct_timespec_sz;
-
-struct __sanitizer_iocb {
- u64 aio_offset;
- uptr aio_buf;
- long aio_nbytes;
- u32 aio_fildes;
- u32 aio_lio_opcode;
- long aio_reqprio;
-#if SANITIZER_WORDSIZE == 64
- u8 aio_sigevent[32];
-#else
- u8 aio_sigevent[20];
-#endif
- u32 _state;
- u32 _errno;
- long _retval;
-};
-
-struct __sanitizer___sysctl_args {
- int *name;
- int nlen;
- void *oldval;
- uptr *oldlenp;
- void *newval;
- uptr newlen;
-};
-
-struct __sanitizer_sem_t {
- uptr data[5];
-};
-
-struct __sanitizer_ipc_perm {
- u32 cuid;
- u32 cgid;
- u32 uid;
- u32 gid;
- u32 mode;
- unsigned short seq;
- long key;
-};
-
-struct __sanitizer_shmid_ds {
- __sanitizer_ipc_perm shm_perm;
- int shm_segsz;
- u32 shm_lpid;
- u32 shm_cpid;
- short shm_nattch;
- u64 shm_atime;
- long __shm_atimensec;
- u64 shm_dtime;
- long __shm_dtimensec;
- u64 shm_ctime;
- long __shm_ctimensec;
- void *_shm_internal;
-};
-
-extern unsigned struct_msqid_ds_sz;
-extern unsigned struct_mq_attr_sz;
-extern unsigned struct_timex_sz;
-extern unsigned struct_statvfs_sz;
-
-struct __sanitizer_iovec {
- void *iov_base;
- uptr iov_len;
-};
-
-struct __sanitizer_ifaddrs {
- struct __sanitizer_ifaddrs *ifa_next;
- char *ifa_name;
- unsigned int ifa_flags;
- struct __sanitizer_sockaddr *ifa_addr; // (struct sockaddr *)
- struct __sanitizer_sockaddr *ifa_netmask; // (struct sockaddr *)
- struct __sanitizer_sockaddr *ifa_dstaddr; // (struct sockaddr *)
- void *ifa_data;
-};
-
-typedef unsigned __sanitizer_pthread_key_t;
-
-typedef long long __sanitizer_time_t;
-typedef int __sanitizer_suseconds_t;
-
-struct __sanitizer_timeval {
- __sanitizer_time_t tv_sec;
- __sanitizer_suseconds_t tv_usec;
-};
-
-struct __sanitizer_itimerval {
- struct __sanitizer_timeval it_interval;
- struct __sanitizer_timeval it_value;
-};
-
-struct __sanitizer_passwd {
- char *pw_name;
- char *pw_passwd;
- int pw_uid;
- int pw_gid;
- __sanitizer_time_t pw_change;
- char *pw_class;
- char *pw_gecos;
- char *pw_dir;
- char *pw_shell;
- __sanitizer_time_t pw_expire;
-};
-
-struct __sanitizer_group {
- char *gr_name;
- char *gr_passwd;
- int gr_gid;
- char **gr_mem;
-};
-
-struct __sanitizer_ether_addr {
- u8 octet[6];
-};
-
-struct __sanitizer_tm {
- int tm_sec;
- int tm_min;
- int tm_hour;
- int tm_mday;
- int tm_mon;
- int tm_year;
- int tm_wday;
- int tm_yday;
- int tm_isdst;
- long int tm_gmtoff;
- const char *tm_zone;
-};
-
-struct __sanitizer_msghdr {
- void *msg_name;
- unsigned msg_namelen;
- struct __sanitizer_iovec *msg_iov;
- unsigned msg_iovlen;
- void *msg_control;
- unsigned msg_controllen;
- int msg_flags;
-};
-struct __sanitizer_cmsghdr {
- unsigned cmsg_len;
- int cmsg_level;
- int cmsg_type;
-};
-
-struct __sanitizer_dirent {
- u64 d_fileno;
- u64 d_off;
- u16 d_reclen;
-};
-
-typedef u64 __sanitizer_clock_t;
-typedef u32 __sanitizer_clockid_t;
-
-typedef u32 __sanitizer___kernel_uid_t;
-typedef u32 __sanitizer___kernel_gid_t;
-typedef u64 __sanitizer___kernel_off_t;
-typedef struct {
- u32 fds_bits[8];
-} __sanitizer___kernel_fd_set;
-
-typedef struct {
- unsigned int pta_magic;
- int pta_flags;
- void *pta_private;
-} __sanitizer_pthread_attr_t;
-
-typedef unsigned int __sanitizer_sigset_t;
-
-struct __sanitizer_siginfo {
- // The size is determined by looking at sizeof of real siginfo_t on linux.
- u64 opaque[128 / sizeof(u64)];
-};
-
-using __sanitizer_sighandler_ptr = void (*)(int sig);
-using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
- __sanitizer_siginfo *siginfo,
- void *uctx);
-
-struct __sanitizer_sigaction {
- union {
- __sanitizer_sighandler_ptr handler;
- __sanitizer_sigactionhandler_ptr sigaction;
- };
- __sanitizer_sigset_t sa_mask;
- int sa_flags;
-};
-
-typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t;
-
-struct __sanitizer_kernel_sigaction_t {
- union {
- void (*handler)(int signo);
- void (*sigaction)(int signo, void *info, void *ctx);
- };
- unsigned long sa_flags;
- void (*sa_restorer)(void);
- __sanitizer_kernel_sigset_t sa_mask;
-};
-
-extern const uptr sig_ign;
-extern const uptr sig_dfl;
-extern const uptr sig_err;
-extern const uptr sa_siginfo;
-
-extern int af_inet;
-extern int af_inet6;
-uptr __sanitizer_in_addr_sz(int af);
-
-struct __sanitizer_dl_phdr_info {
-#if SANITIZER_WORDSIZE == 64
- u64 dlpi_addr;
-#else
- u32 dlpi_addr;
-#endif
- const char *dlpi_name;
- const void *dlpi_phdr;
-#if SANITIZER_WORDSIZE == 64
- u32 dlpi_phnum;
-#else
- u16 dlpi_phnum;
-#endif
-};
-
-extern unsigned struct_ElfW_Phdr_sz;
-
-struct __sanitizer_addrinfo {
- int ai_flags;
- int ai_family;
- int ai_socktype;
- int ai_protocol;
- unsigned ai_addrlen;
- struct __sanitizer_sockaddr *ai_addr;
- char *ai_canonname;
- struct __sanitizer_addrinfo *ai_next;
-};
-
-struct __sanitizer_hostent {
- char *h_name;
- char **h_aliases;
- int h_addrtype;
- int h_length;
- char **h_addr_list;
-};
-
-struct __sanitizer_pollfd {
- int fd;
- short events;
- short revents;
-};
-
-typedef unsigned __sanitizer_nfds_t;
-
-struct __sanitizer_glob_t {
- int gl_pathc;
- int gl_matchc;
- int gl_offs;
- int gl_flags;
- char **gl_pathv;
- void **gl_statv;
- int (*gl_errfunc)(const char *, int);
- void (*gl_closedir)(void *dirp);
- struct dirent *(*gl_readdir)(void *dirp);
- void *(*gl_opendir)(const char *);
- int (*gl_lstat)(const char *, void * /* struct stat* */);
- int (*gl_stat)(const char *, void * /* struct stat* */);
-};
-
-extern int glob_nomatch;
-extern int glob_altdirfunc;
-
-extern unsigned path_max;
-
-typedef char __sanitizer_FILE;
-#define SANITIZER_HAS_STRUCT_FILE 0
-
-extern int shmctl_ipc_stat;
-
-// This simplifies generic code
-#define struct_shminfo_sz -1
-#define struct_shm_info_sz -1
-#define shmctl_shm_stat -1
-#define shmctl_ipc_info -1
-#define shmctl_shm_info -1
-
-extern unsigned struct_utmp_sz;
-extern unsigned struct_utmpx_sz;
-
-extern int map_fixed;
-
-// ioctl arguments
-struct __sanitizer_ifconf {
- int ifc_len;
- union {
- void *ifcu_req;
- } ifc_ifcu;
-};
-
-extern const int si_SEGV_MAPERR;
-extern const int si_SEGV_ACCERR;
-} // namespace __sanitizer
-
-#define CHECK_TYPE_SIZE(TYPE) \
- COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
-
-#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
- COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
- sizeof(((CLASS *)NULL)->MEMBER)); \
- COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
- offsetof(CLASS, MEMBER))
-
-// For sigaction, which is a function and struct at the same time,
-// and thus requires explicit "struct" in sizeof() expression.
-#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
- COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
- sizeof(((struct CLASS *)NULL)->MEMBER)); \
- COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
- offsetof(struct CLASS, MEMBER))
-
-#define SIGACTION_SYMNAME __sigaction14
-
-#endif // SANITIZER_OPENBSD
-
-#endif
// Sizes and layouts of platform-specific POSIX data structures.
//===----------------------------------------------------------------------===//
-#include "sanitizer_platform.h"
-
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if defined(__linux__) || defined(__APPLE__)
// Tests in this file assume that off_t-dependent data structures match the
// libc ABI. For example, struct dirent here is what readdir() function (as
// exported from libc) returns, and not the user-facing "dirent", which
// depends on _FILE_OFFSET_BITS setting.
// To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below.
-#ifdef _FILE_OFFSET_BITS
#undef _FILE_OFFSET_BITS
#endif
+// Must go after undef _FILE_OFFSET_BITS.
+#include "sanitizer_platform.h"
+
+#if SANITIZER_LINUX || SANITIZER_MAC
// Must go after undef _FILE_OFFSET_BITS.
#include "sanitizer_glibc_version.h"
#include <pwd.h>
#include <signal.h>
#include <stddef.h>
+#include <stdio.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/socket.h>
#endif
#if !SANITIZER_ANDROID
-#include <fstab.h>
#include <sys/mount.h>
#include <sys/timeb.h>
#include <utmpx.h>
#if SANITIZER_LINUX
# include <utime.h>
# include <sys/ptrace.h>
-# if defined(__mips64) || defined(__aarch64__) || defined(__arm__)
+#if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
+ SANITIZER_RISCV64
# include <asm/ptrace.h>
# ifdef __arm__
typedef struct user_fpregs elf_fpregset_t;
#include <wordexp.h>
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-#include <glob.h>
-#include <obstack.h>
-#include <mqueue.h>
+#if SANITIZER_LINUX
+#if SANITIZER_GLIBC
+#include <fstab.h>
#include <net/if_ppp.h>
#include <netax25/ax25.h>
#include <netipx/ipx.h>
#include <netrom/netrom.h>
+#include <obstack.h>
#if HAVE_RPC_XDR_H
# include <rpc/xdr.h>
#endif
#include <scsi/scsi.h>
-#include <sys/mtio.h>
+#else
+#include <linux/if_ppp.h>
+#include <linux/kd.h>
+#include <linux/ppp_defs.h>
+#endif // SANITIZER_GLIBC
+
+#if SANITIZER_ANDROID
+#include <linux/mtio.h>
+#else
+#include <glob.h>
+#include <mqueue.h>
#include <sys/kd.h>
+#include <sys/mtio.h>
#include <sys/shm.h>
#include <sys/statvfs.h>
#include <sys/timex.h>
# include <sys/procfs.h>
#endif
#include <sys/user.h>
-#include <linux/cyclades.h>
#include <linux/if_eql.h>
#include <linux/if_plip.h>
#include <linux/lp.h>
#include <sys/msg.h>
#include <sys/ipc.h>
#include <crypt.h>
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
+#endif // SANITIZER_ANDROID
-#if SANITIZER_ANDROID
-#include <linux/kd.h>
-#include <linux/mtio.h>
-#include <linux/ppp_defs.h>
-#include <linux/if_ppp.h>
-#endif
-
-#if SANITIZER_LINUX
#include <link.h>
#include <sys/vfs.h>
#include <sys/epoll.h>
#include <linux/capability.h>
+#else
+#include <fstab.h>
#endif // SANITIZER_LINUX
#if SANITIZER_MAC
unsigned struct_statfs64_sz = sizeof(struct statfs64);
#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
-#if !SANITIZER_ANDROID
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
unsigned struct_fstab_sz = sizeof(struct fstab);
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
+ // SANITIZER_MAC
+#if !SANITIZER_ANDROID
unsigned struct_statfs_sz = sizeof(struct statfs);
unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
unsigned ucontext_t_sz = sizeof(ucontext_t);
#if SANITIZER_LINUX && !SANITIZER_ANDROID
// Use pre-computed size of struct ustat to avoid <sys/ustat.h> which
// has been removed from glibc 2.28.
-#if defined(__aarch64__) || defined(__s390x__) || defined (__mips64) \
- || defined(__powerpc64__) || defined(__arch64__) || defined(__sparcv9) \
- || defined(__x86_64__) || (defined(__riscv) && __riscv_xlen == 64)
+#if defined(__aarch64__) || defined(__s390x__) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__arch64__) || defined(__sparcv9) || \
+ defined(__x86_64__) || SANITIZER_RISCV64
#define SIZEOF_STRUCT_USTAT 32
#elif defined(__arm__) || defined(__i386__) || defined(__mips__) \
|| defined(__powerpc__) || defined(__s390__) || defined(__sparc__)
unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
int glob_nomatch = GLOB_NOMATCH;
int glob_altdirfunc = GLOB_ALTDIRFUNC;
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
- (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
- defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__))
+#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
+ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \
+ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+ defined(__s390__) || SANITIZER_RISCV64)
#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
+#elif SANITIZER_RISCV64
+ unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
+ unsigned struct_user_fpregs_struct_sz = sizeof(struct __riscv_q_ext_state);
#elif defined(__aarch64__)
unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state);
unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
#endif // __mips64 || __powerpc64__ || __aarch64__
#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
- defined(__aarch64__) || defined(__arm__) || defined(__s390__)
+ defined(__aarch64__) || defined(__arm__) || defined(__s390__) || \
+ SANITIZER_RISCV64
unsigned struct_user_fpxregs_struct_sz = 0;
#else
unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
unsigned struct_input_id_sz = sizeof(struct input_id);
unsigned struct_mtpos_sz = sizeof(struct mtpos);
unsigned struct_rtentry_sz = sizeof(struct rtentry);
+#if SANITIZER_GLIBC || SANITIZER_ANDROID
unsigned struct_termio_sz = sizeof(struct termio);
+#endif
unsigned struct_vt_consize_sz = sizeof(struct vt_consize);
unsigned struct_vt_sizes_sz = sizeof(struct vt_sizes);
unsigned struct_vt_stat_sz = sizeof(struct vt_stat);
unsigned struct_vt_mode_sz = sizeof(struct vt_mode);
#endif // SANITIZER_LINUX
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct);
- unsigned struct_cyclades_monitor_sz = sizeof(struct cyclades_monitor);
#if EV_VERSION > (0x010000)
unsigned struct_input_keymap_entry_sz = sizeof(struct input_keymap_entry);
#else
unsigned struct_sockaddr_ax25_sz = sizeof(struct sockaddr_ax25);
unsigned struct_unimapdesc_sz = sizeof(struct unimapdesc);
unsigned struct_unimapinit_sz = sizeof(struct unimapinit);
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
-#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+#endif // SANITIZER_GLIBC
#if !SANITIZER_ANDROID && !SANITIZER_MAC
unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
#endif // SANITIZER_LINUX
#if SANITIZER_LINUX && !SANITIZER_ANDROID
- unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH;
- unsigned IOCTL_CYGETDEFTIMEOUT = CYGETDEFTIMEOUT;
- unsigned IOCTL_CYGETMON = CYGETMON;
- unsigned IOCTL_CYGETTHRESH = CYGETTHRESH;
- unsigned IOCTL_CYGETTIMEOUT = CYGETTIMEOUT;
- unsigned IOCTL_CYSETDEFTHRESH = CYSETDEFTHRESH;
- unsigned IOCTL_CYSETDEFTIMEOUT = CYSETDEFTIMEOUT;
- unsigned IOCTL_CYSETTHRESH = CYSETTHRESH;
- unsigned IOCTL_CYSETTIMEOUT = CYSETTIMEOUT;
unsigned IOCTL_EQL_EMANCIPATE = EQL_EMANCIPATE;
unsigned IOCTL_EQL_ENSLAVE = EQL_ENSLAVE;
unsigned IOCTL_EQL_GETMASTRCFG = EQL_GETMASTRCFG;
unsigned IOCTL_PIO_UNIMAP = PIO_UNIMAP;
unsigned IOCTL_PIO_UNIMAPCLR = PIO_UNIMAPCLR;
unsigned IOCTL_PIO_UNISCRNMAP = PIO_UNISCRNMAP;
+#if SANITIZER_GLIBC
unsigned IOCTL_SCSI_IOCTL_GET_IDLUN = SCSI_IOCTL_GET_IDLUN;
unsigned IOCTL_SCSI_IOCTL_PROBE_HOST = SCSI_IOCTL_PROBE_HOST;
unsigned IOCTL_SCSI_IOCTL_TAGGED_DISABLE = SCSI_IOCTL_TAGGED_DISABLE;
unsigned IOCTL_SIOCNRGETPARMS = SIOCNRGETPARMS;
unsigned IOCTL_SIOCNRRTCTL = SIOCNRRTCTL;
unsigned IOCTL_SIOCNRSETPARMS = SIOCNRSETPARMS;
+#endif
unsigned IOCTL_TIOCGSERIAL = TIOCGSERIAL;
unsigned IOCTL_TIOCSERGETMULTI = TIOCSERGETMULTI;
unsigned IOCTL_TIOCSERSETMULTI = TIOCSERSETMULTI;
CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
#endif // SANITIZER_LINUX || SANITIZER_FREEBSD
-#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD
CHECK_TYPE_SIZE(glob_t);
CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
CHECK_SIZE_AND_OFFSET(glob_t, gl_opendir);
CHECK_SIZE_AND_OFFSET(glob_t, gl_lstat);
CHECK_SIZE_AND_OFFSET(glob_t, gl_stat);
-#endif
+#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD
CHECK_TYPE_SIZE(addrinfo);
CHECK_SIZE_AND_OFFSET(addrinfo, ai_flags);
CHECK_SIZE_AND_OFFSET(iovec, iov_base);
CHECK_SIZE_AND_OFFSET(iovec, iov_len);
+// In POSIX, int msg_iovlen; socklen_t msg_controllen; socklen_t cmsg_len; but
+// many implementations don't conform to the standard. Since we pick the
+// non-conforming glibc definition, exclude the checks for musl (incompatible
+// sizes but compatible offsets).
CHECK_TYPE_SIZE(msghdr);
CHECK_SIZE_AND_OFFSET(msghdr, msg_name);
CHECK_SIZE_AND_OFFSET(msghdr, msg_namelen);
CHECK_SIZE_AND_OFFSET(msghdr, msg_iov);
+#if SANITIZER_GLIBC || SANITIZER_ANDROID
CHECK_SIZE_AND_OFFSET(msghdr, msg_iovlen);
+#endif
CHECK_SIZE_AND_OFFSET(msghdr, msg_control);
+#if SANITIZER_GLIBC || SANITIZER_ANDROID
CHECK_SIZE_AND_OFFSET(msghdr, msg_controllen);
+#endif
CHECK_SIZE_AND_OFFSET(msghdr, msg_flags);
CHECK_TYPE_SIZE(cmsghdr);
+#if SANITIZER_GLIBC || SANITIZER_ANDROID
CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
+#endif
CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
CHECK_TYPE_SIZE(ether_addr);
-#if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD
CHECK_TYPE_SIZE(ipc_perm);
# if SANITIZER_FREEBSD
CHECK_SIZE_AND_OFFSET(ipc_perm, key);
CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data);
#endif
-#if SANITIZER_LINUX
+#if SANITIZER_GLIBC || SANITIZER_ANDROID
COMPILER_CHECK(sizeof(__sanitizer_struct_mallinfo) == sizeof(struct mallinfo));
#endif
COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
COMPILER_CHECK(sizeof(__sanitizer_FILE) <= sizeof(FILE));
CHECK_SIZE_AND_OFFSET(FILE, _flags);
CHECK_SIZE_AND_OFFSET(FILE, _IO_read_ptr);
CHECK_SIZE_AND_OFFSET(FILE, _markers);
CHECK_SIZE_AND_OFFSET(FILE, _chain);
CHECK_SIZE_AND_OFFSET(FILE, _fileno);
-#endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
COMPILER_CHECK(sizeof(__sanitizer__obstack_chunk) <= sizeof(_obstack_chunk));
CHECK_SIZE_AND_OFFSET(_obstack_chunk, limit);
CHECK_SIZE_AND_OFFSET(_obstack_chunk, prev);
CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, write);
CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, seek);
CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, close);
-#endif
+#endif // SANITIZER_GLIBC
#if SANITIZER_LINUX || SANITIZER_FREEBSD
CHECK_TYPE_SIZE(sem_t);
const unsigned struct___old_kernel_stat_sz = 0;
const unsigned struct_kernel_stat_sz = 64;
const unsigned struct_kernel_stat64_sz = 104;
-#elif defined(__riscv) && __riscv_xlen == 64
+#elif SANITIZER_RISCV64
const unsigned struct_kernel_stat_sz = 128;
-const unsigned struct_kernel_stat64_sz = 104;
+const unsigned struct_kernel_stat64_sz = 0; // RISCV64 does not use stat64
#endif
struct __sanitizer_perf_event_attr {
unsigned type;
int cmsg_type;
};
#else
+// In POSIX, int msg_iovlen; socklen_t msg_controllen; socklen_t cmsg_len; but
+// many implementations don't conform to the standard.
struct __sanitizer_msghdr {
void *msg_name;
unsigned msg_namelen;
#endif // !SANITIZER_ANDROID
#if defined(__mips__)
-struct __sanitizer_kernel_sigset_t {
- uptr sig[2];
-};
+#define __SANITIZER_KERNEL_NSIG 128
#else
+#define __SANITIZER_KERNEL_NSIG 64
+#endif
+
struct __sanitizer_kernel_sigset_t {
- u8 sig[8];
+ uptr sig[__SANITIZER_KERNEL_NSIG / (sizeof(uptr) * 8)];
};
-#endif
// Linux system headers define the 'sa_handler' and 'sa_sigaction' macros.
#if SANITIZER_MIPS
#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
(defined(__i386) || defined(__x86_64) || defined(__mips64) || \
defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
- defined(__s390__))
+ defined(__s390__) || SANITIZER_RISCV64)
extern unsigned struct_user_regs_struct_sz;
extern unsigned struct_user_fpregs_struct_sz;
extern unsigned struct_user_fpxregs_struct_sz;
#if SANITIZER_LINUX && !SANITIZER_ANDROID
extern unsigned struct_ax25_parms_struct_sz;
-extern unsigned struct_cyclades_monitor_sz;
extern unsigned struct_input_keymap_entry_sz;
extern unsigned struct_ipx_config_data_sz;
extern unsigned struct_kbdiacrs_sz;
#endif // SANITIZER_LINUX
#if SANITIZER_LINUX && !SANITIZER_ANDROID
-extern unsigned IOCTL_CYGETDEFTHRESH;
-extern unsigned IOCTL_CYGETDEFTIMEOUT;
-extern unsigned IOCTL_CYGETMON;
-extern unsigned IOCTL_CYGETTHRESH;
-extern unsigned IOCTL_CYGETTIMEOUT;
-extern unsigned IOCTL_CYSETDEFTHRESH;
-extern unsigned IOCTL_CYSETDEFTIMEOUT;
-extern unsigned IOCTL_CYSETTHRESH;
-extern unsigned IOCTL_CYSETTIMEOUT;
extern unsigned IOCTL_EQL_EMANCIPATE;
extern unsigned IOCTL_EQL_ENSLAVE;
extern unsigned IOCTL_EQL_GETMASTRCFG;
CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr);
CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum);
-CHECK_TYPE_SIZE(glob_t);
+// There are additional fields we are not interested in.
+COMPILER_CHECK(sizeof(__sanitizer_glob_t) <= sizeof(glob_t));
CHECK_SIZE_AND_OFFSET(glob_t, gl_pathc);
CHECK_SIZE_AND_OFFSET(glob_t, gl_pathv);
CHECK_SIZE_AND_OFFSET(glob_t, gl_offs);
return true;
}
+#if !SANITIZER_MAC
void DumpProcessMap() {
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
const sptr kBufSize = 4095;
Report("End of process memory map.\n");
UnmapOrDie(filename, kBufSize);
}
+#endif
const char *GetPwd() {
return GetEnv("PWD");
bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
MemoryMappingLayout proc_maps(/*cache_enabled*/false);
- InternalScopedString buff(kMaxPathLength);
- MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+ InternalMmapVector<char> buff(kMaxPathLength);
+ MemoryMappedSegment segment(buff.data(), buff.size());
while (proc_maps.Next(&segment)) {
if (segment.IsExecutable() &&
internal_strcmp(module, segment.filename) == 0) {
bool SignalContext::IsMemoryAccess() const {
auto si = static_cast<const siginfo_t *>(siginfo);
- return si->si_signo == SIGSEGV;
+ return si->si_signo == SIGSEGV || si->si_signo == SIGBUS;
}
int SignalContext::GetType() const {
int fd = ReserveStandardFds(
internal_open(shmname, O_RDWR | O_CREAT | O_TRUNC | o_cloexec, S_IRWXU));
CHECK_GE(fd, 0);
- if (!o_cloexec) {
- int res = fcntl(fd, F_SETFD, FD_CLOEXEC);
- CHECK_EQ(0, res);
- }
int res = internal_ftruncate(fd, size);
+#if !defined(O_CLOEXEC)
+ res = fcntl(fd, F_SETFD, FD_CLOEXEC);
+ CHECK_EQ(0, res);
+#endif
CHECK_EQ(0, res);
res = internal_unlink(shmname);
CHECK_EQ(0, res);
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform_limits_freebsd.h"
#include "sanitizer_platform_limits_netbsd.h"
-#include "sanitizer_platform_limits_openbsd.h"
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_platform_limits_solaris.h"
uptr internal_mmap(void *addr, uptr length, int prot, int flags,
int fd, u64 offset);
uptr internal_munmap(void *addr, uptr length);
+#if SANITIZER_LINUX
+uptr internal_mremap(void *old_address, uptr old_size, uptr new_size, int flags,
+ void *new_address);
+#endif
int internal_mprotect(void *addr, uptr length, int prot);
+int internal_madvise(uptr addr, uptr length, int advice);
// OS
uptr internal_filesize(fd_t fd); // -1 on error.
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
#include "sanitizer_platform_limits_netbsd.h"
-#include "sanitizer_platform_limits_openbsd.h"
#include "sanitizer_platform_limits_posix.h"
#include "sanitizer_platform_limits_solaris.h"
#include "sanitizer_posix.h"
uptr beg_aligned = RoundUpTo(beg, page_size);
uptr end_aligned = RoundDownTo(end, page_size);
if (beg_aligned < end_aligned)
- // In the default Solaris compilation environment, madvise() is declared
- // to take a caddr_t arg; casting it to void * results in an invalid
- // conversion error, so use char * instead.
- madvise((char *)beg_aligned, end_aligned - beg_aligned,
- SANITIZER_MADVISE_DONTNEED);
+ internal_madvise(beg_aligned, end_aligned - beg_aligned,
+ SANITIZER_MADVISE_DONTNEED);
}
void SetShadowRegionHugePageMode(uptr addr, uptr size) {
#ifdef MADV_NOHUGEPAGE // May not be defined on old systems.
if (common_flags()->no_huge_pages_for_shadow)
- madvise((char *)addr, size, MADV_NOHUGEPAGE);
+ internal_madvise(addr, size, MADV_NOHUGEPAGE);
else
- madvise((char *)addr, size, MADV_HUGEPAGE);
+ internal_madvise(addr, size, MADV_HUGEPAGE);
#endif // MADV_NOHUGEPAGE
}
bool DontDumpShadowMemory(uptr addr, uptr length) {
#if defined(MADV_DONTDUMP)
- return madvise((char *)addr, length, MADV_DONTDUMP) == 0;
+ return internal_madvise(addr, length, MADV_DONTDUMP) == 0;
#elif defined(MADV_NOCORE)
- return madvise((char *)addr, length, MADV_NOCORE) == 0;
+ return internal_madvise(addr, length, MADV_NOCORE) == 0;
#else
return true;
#endif // MADV_DONTDUMP
CHECK(AddressSpaceIsUnlimited());
}
-void SleepForSeconds(int seconds) {
- sleep(seconds);
-}
-
-void SleepForMillis(int millis) {
- usleep(millis * 1000);
-}
-
void Abort() {
#if !SANITIZER_GO
// If we are handling SIGABRT, unhandle it first.
if (GetHandleSignalMode(SIGABRT) != kHandleSignalNo) {
struct sigaction sigact;
internal_memset(&sigact, 0, sizeof(sigact));
- sigact.sa_sigaction = (sa_sigaction_t)SIG_DFL;
+ sigact.sa_handler = SIG_DFL;
internal_sigaction(SIGABRT, &sigact, nullptr);
}
#endif
#if !SANITIZER_GO
// TODO(glider): different tools may require different altstack size.
-static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough.
+static uptr GetAltStackSize() {
+ // Note: since GLIBC_2.31, SIGSTKSZ may be a function call, so this may be
+ // more costly that you think. However GetAltStackSize is only call 2-3 times
+ // per thread so don't cache the evaluation.
+ return SIGSTKSZ * 4;
+}
void SetAlternateSignalStack() {
stack_t altstack, oldstack;
// TODO(glider): the mapped stack should have the MAP_STACK flag in the
// future. It is not required by man 2 sigaltstack now (they're using
// malloc()).
- void* base = MmapOrDie(kAltStackSize, __func__);
- altstack.ss_sp = (char*) base;
+ altstack.ss_size = GetAltStackSize();
+ altstack.ss_sp = (char *)MmapOrDie(altstack.ss_size, __func__);
altstack.ss_flags = 0;
- altstack.ss_size = kAltStackSize;
CHECK_EQ(0, sigaltstack(&altstack, nullptr));
}
stack_t altstack, oldstack;
altstack.ss_sp = nullptr;
altstack.ss_flags = SS_DISABLE;
- altstack.ss_size = kAltStackSize; // Some sane value required on Darwin.
+ altstack.ss_size = GetAltStackSize(); // Some sane value required on Darwin.
CHECK_EQ(0, sigaltstack(&altstack, &oldstack));
UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
}
#include <stdio.h>
#include <stdarg.h>
+#if defined(__x86_64__)
+# include <emmintrin.h>
+#endif
+
#if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 && \
!defined(va_copy)
# define va_copy(dst, src) ((dst) = (src))
int VSNPrintf(char *buff, int buff_length,
const char *format, va_list args) {
static const char *kPrintfFormatsHelp =
- "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
+ "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X,V}; %p; "
"%[-]([0-9]*)?(\\.\\*)?s; %c\n";
RAW_CHECK(format);
RAW_CHECK(buff_length > 0);
cur += have_z;
bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
cur += have_ll * 2;
- s64 dval;
- u64 uval;
const bool have_length = have_z || have_ll;
const bool have_flags = have_width || have_length;
// At the moment only %s supports precision and left-justification.
CHECK(!((precision >= 0 || left_justified) && *cur != 's'));
switch (*cur) {
case 'd': {
- dval = have_ll ? va_arg(args, s64)
- : have_z ? va_arg(args, sptr)
- : va_arg(args, int);
+ s64 dval = have_ll ? va_arg(args, s64)
+ : have_z ? va_arg(args, sptr)
+ : va_arg(args, int);
result += AppendSignedDecimal(&buff, buff_end, dval, width,
pad_with_zero);
break;
case 'u':
case 'x':
case 'X': {
- uval = have_ll ? va_arg(args, u64)
- : have_z ? va_arg(args, uptr)
- : va_arg(args, unsigned);
+ u64 uval = have_ll ? va_arg(args, u64)
+ : have_z ? va_arg(args, uptr)
+ : va_arg(args, unsigned);
bool uppercase = (*cur == 'X');
result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
width, pad_with_zero, uppercase);
break;
}
+ case 'V': {
+ for (uptr i = 0; i < 16; i++) {
+ unsigned x = va_arg(args, unsigned);
+ result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false);
+ }
+ break;
+ }
case 'p': {
RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
va_list args) {
va_list args2;
va_copy(args2, args);
- const int kLen = 16 * 1024;
- int needed_length;
+ InternalMmapVector<char> v;
+ int needed_length = 0;
char *buffer = local_buffer;
// First try to print a message using a local buffer, and then fall back to
// mmaped buffer.
- for (int use_mmap = 0; use_mmap < 2; use_mmap++) {
+ for (int use_mmap = 0;; use_mmap++) {
if (use_mmap) {
va_end(args);
va_copy(args, args2);
- buffer = (char*)MmapOrDie(kLen, "Report");
- buffer_size = kLen;
+ v.resize(needed_length + 1);
+ buffer_size = v.capacity();
+ v.resize(buffer_size);
+ buffer = &v[0];
}
needed_length = 0;
- // Check that data fits into the current buffer.
-# define CHECK_NEEDED_LENGTH \
- if (needed_length >= buffer_size) { \
- if (!use_mmap) continue; \
- RAW_CHECK_MSG(needed_length < kLen, \
- "Buffer in Report is too short!\n"); \
- }
// Fuchsia's logging infrastructure always keeps track of the logging
// process, thread, and timestamp, so never prepend such information.
if (!SANITIZER_FUCHSIA && append_pid) {
if (common_flags()->log_exe_name && exe_name) {
needed_length += internal_snprintf(buffer, buffer_size,
"==%s", exe_name);
- CHECK_NEEDED_LENGTH
+ if (needed_length >= buffer_size)
+ continue;
}
needed_length += internal_snprintf(
buffer + needed_length, buffer_size - needed_length, "==%d==", pid);
- CHECK_NEEDED_LENGTH
+ if (needed_length >= buffer_size)
+ continue;
}
needed_length += VSNPrintf(buffer + needed_length,
buffer_size - needed_length, format, args);
- CHECK_NEEDED_LENGTH
+ if (needed_length >= buffer_size)
+ continue;
// If the message fit into the buffer, print it and exit.
break;
-# undef CHECK_NEEDED_LENGTH
}
RawWrite(buffer);
CallPrintfAndReportCallback(buffer);
LogMessageOnPrintf(buffer);
- // If we had mapped any memory, clean up.
- if (buffer != local_buffer)
- UnmapOrDie((void *)buffer, buffer_size);
va_end(args2);
}
FORMAT(2, 3)
void InternalScopedString::append(const char *format, ...) {
- CHECK_LT(length_, size());
- va_list args;
- va_start(args, format);
- VSNPrintf(data() + length_, size() - length_, format, args);
- va_end(args);
- length_ += internal_strlen(data() + length_);
- CHECK_LT(length_, size());
+ uptr prev_len = length();
+
+ while (true) {
+ buffer_.resize(buffer_.capacity());
+
+ va_list args;
+ va_start(args, format);
+ uptr sz = VSNPrintf(buffer_.data() + prev_len, buffer_.size() - prev_len,
+ format, args);
+ va_end(args);
+ if (sz < buffer_.size() - prev_len) {
+ buffer_.resize(prev_len + sz + 1);
+ break;
+ }
+
+ buffer_.reserve(buffer_.capacity() * 2);
+ }
+ CHECK_EQ(buffer_[length()], '\0');
}
} // namespace __sanitizer
#include "sanitizer_platform.h"
#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_MAC || SANITIZER_SOLARIS || \
+ SANITIZER_MAC || SANITIZER_SOLARIS || \
SANITIZER_FUCHSIA
#include "sanitizer_common.h"
//===----------------------------------------------------------------------===//
//
// Information about the process mappings
-// (FreeBSD, OpenBSD and NetBSD-specific parts).
+// (FreeBSD and NetBSD-specific parts).
//===----------------------------------------------------------------------===//
#include "sanitizer_platform.h"
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD
#include "sanitizer_common.h"
#if SANITIZER_FREEBSD
#include "sanitizer_freebsd.h"
#endif
#include <limits.h>
-#if SANITIZER_OPENBSD
-#define KVME_PROT_READ KVE_PROT_READ
-#define KVME_PROT_WRITE KVE_PROT_WRITE
-#define KVME_PROT_EXEC KVE_PROT_EXEC
-#endif
// Fix 'kinfo_vmentry' definition on FreeBSD prior v9.2 in 32-bit mode.
#if SANITIZER_FREEBSD && (SANITIZER_WORDSIZE == 32)
KERN_PROC,
KERN_PROC_VMMAP,
getpid()
-#elif SANITIZER_OPENBSD
- CTL_KERN,
- KERN_PROC_VMMAP,
- getpid()
#elif SANITIZER_NETBSD
CTL_VM,
VM_PROC,
CHECK_EQ(Err, 0);
CHECK_GT(Size, 0);
-#if !SANITIZER_OPENBSD
size_t MmapedSize = Size * 4 / 3;
void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
Size = MmapedSize;
Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), VmMap, &Size, NULL, 0);
CHECK_EQ(Err, 0);
proc_maps->data = (char *)VmMap;
-#else
- size_t PageSize = GetPageSize();
- size_t MmapedSize = Size;
- MmapedSize = ((MmapedSize - 1) / PageSize + 1) * PageSize;
- char *Mem = (char *)MmapOrDie(MmapedSize, "ReadProcMaps()");
- Size = 2 * Size + 10 * sizeof(struct kinfo_vmentry);
- if (Size > 0x10000)
- Size = 0x10000;
- Size = (Size / sizeof(struct kinfo_vmentry)) * sizeof(struct kinfo_vmentry);
- Err = internal_sysctl(Mib, ARRAY_SIZE(Mib), Mem, &Size, NULL, 0);
- CHECK_EQ(Err, 0);
- MmapedSize = Size;
- proc_maps->data = Mem;
-#endif
-
proc_maps->mmaped_size = MmapedSize;
proc_maps->len = Size;
}
if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0)
segment->protection |= kProtectionExecute;
-#if !SANITIZER_OPENBSD
if (segment->filename != NULL && segment->filename_size > 0) {
internal_snprintf(segment->filename,
Min(segment->filename_size, (uptr)PATH_MAX), "%s",
VmEntry->kve_path);
}
-#endif
#if SANITIZER_FREEBSD
data_.current += VmEntry->kve_structsize;
#include "sanitizer_platform.h"
#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD || SANITIZER_SOLARIS
+ SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_placement_new.h"
void MemoryMappingLayout::DumpListOfModules(
InternalMmapVectorNoCtor<LoadedModule> *modules) {
Reset();
- InternalScopedString module_name(kMaxPathLength);
+ InternalMmapVector<char> module_name(kMaxPathLength);
MemoryMappedSegment segment(module_name.data(), module_name.size());
for (uptr i = 0; Next(&segment); i++) {
const char *cur_name = segment.filename;
void MemoryMappingLayout::DumpListOfModules(
InternalMmapVectorNoCtor<LoadedModule> *modules) {
Reset();
- InternalScopedString module_name(kMaxPathLength);
- MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+ InternalMmapVector<char> module_name(kMaxPathLength);
+ MemoryMappedSegment segment(module_name.data(), module_name.size());
MemoryMappedSegmentData data;
segment.data_ = &data;
while (Next(&segment)) {
// Information about the process mappings (Solaris-specific parts).
//===----------------------------------------------------------------------===//
+// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
+#undef _FILE_OFFSET_BITS
#include "sanitizer_platform.h"
#if SANITIZER_SOLARIS
#include "sanitizer_common.h"
#include "sanitizer_procmaps.h"
-// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
-#undef _FILE_OFFSET_BITS
#include <procfs.h>
#include <limits.h>
char *last = data_.proc_self_maps.data + data_.proc_self_maps.len;
if (data_.current >= last) return false;
- prxmap_t *xmapentry = (prxmap_t*)data_.current;
+ prxmap_t *xmapentry =
+ const_cast<prxmap_t *>(reinterpret_cast<const prxmap_t *>(data_.current));
segment->start = (uptr)xmapentry->pr_vaddr;
segment->end = (uptr)(xmapentry->pr_vaddr + xmapentry->pr_size);
#if __has_feature(ptrauth_calls)
#include <ptrauth.h>
+#elif defined(__ARM_FEATURE_PAC_DEFAULT) && !defined(__APPLE__)
+inline unsigned long ptrauth_strip(void* __value, unsigned int __key) {
+ // On the stack the link register is protected with Pointer
+ // Authentication Code when compiled with -mbranch-protection.
+ // Let's stripping the PAC unconditionally because xpaclri is in
+ // the NOP space so will do nothing when it is not enabled or not available.
+ unsigned long ret;
+ asm volatile(
+ "mov x30, %1\n\t"
+ "hint #7\n\t" // xpaclri
+ "mov %0, x30\n\t"
+ : "=r"(ret)
+ : "r"(__value)
+ : "x30");
+ return ret;
+}
+#define ptrauth_auth_data(__value, __old_key, __old_data) __value
+#define ptrauth_string_discriminator(__string) ((int)0)
#else
// Copied from <ptrauth.h>
#define ptrauth_strip(__value, __key) __value
#define ptrauth_string_discriminator(__string) ((int)0)
#endif
+#define STRIP_PAC_PC(pc) ((uptr)ptrauth_strip(pc, 0))
+
#endif // SANITIZER_PTRAUTH_H
Cache cache_;
char pad2_[kCacheLineSize];
- void NOINLINE Recycle(uptr min_size, Callback cb) {
+ void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_)
+ RELEASE(recycle_mutex_) {
Cache tmp;
{
SpinMutexLock l(&cache_mutex_);
INTERCEPTOR(int, sigaction_symname, int signum,
const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) {
- if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
+ if (GetHandleSignalMode(signum) == kHandleSignalExclusive) {
+ if (!oldact) return 0;
+ act = nullptr;
+ }
SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact);
}
#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname)
return _REAL(mprotect)(addr, length, prot);
}
+// Illumos' declaration of madvise cannot be made visible if _XOPEN_SOURCE
+// is defined as g++ does on Solaris.
+//
+// This declaration is consistent with Solaris 11.4. Both Illumos and Solaris
+// versions older than 11.4 declared madvise with a caddr_t as the first
+// argument, but we don't currently support Solaris versions older than 11.4,
+// and as mentioned above the declaration is not visible on Illumos so we can
+// use any declaration we like on Illumos.
+extern "C" int madvise(void *, size_t, int);
+
+int internal_madvise(uptr addr, uptr length, int advice) {
+ return madvise((void *)addr, length, advice);
+}
+
DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) {
return _REAL(close)(fd);
}
return sched_yield();
}
-DECLARE__REAL_AND_INTERNAL(void, _exit, int exitcode) {
- _exit(exitcode);
+DECLARE__REAL_AND_INTERNAL(void, usleep, u64 useconds) {
+ struct timespec ts;
+ ts.tv_sec = useconds / 1000000;
+ ts.tv_nsec = (useconds % 1000000) * 1000;
+ nanosleep(&ts, nullptr);
}
DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename,
}
// ----------------- sanitizer_common.h
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ // FIXME: implement actual blocking.
+ sched_yield();
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {}
+
BlockingMutex::BlockingMutex() {
CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
internal_memset(this, 0, sizeof(*this));
CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
}
-void BlockingMutex::CheckLocked() {
- CHECK_EQ((uptr)thr_self(), owner_);
-}
+void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); }
} // namespace __sanitizer
theDepot.UnlockAll();
}
+void StackDepotPrintAll() {
+#if !SANITIZER_GO
+ theDepot.PrintAll();
+#endif
+}
+
bool StackDepotReverseMap::IdDescPair::IdComparator(
const StackDepotReverseMap::IdDescPair &a,
const StackDepotReverseMap::IdDescPair &b) {
if (!map_.size())
return StackTrace();
IdDescPair pair = {id, nullptr};
- uptr idx =
- InternalLowerBound(map_, 0, map_.size(), pair, IdDescPair::IdComparator);
+ uptr idx = InternalLowerBound(map_, pair, IdDescPair::IdComparator);
if (idx > map_.size() || map_[idx].id != id)
return StackTrace();
return map_[idx].desc->load();
void StackDepotLockAll();
void StackDepotUnlockAll();
+void StackDepotPrintAll();
// Instantiating this class creates a snapshot of StackDepot which can be
// efficiently queried with StackDepotGet(). You can use it concurrently with
#ifndef SANITIZER_STACKDEPOTBASE_H
#define SANITIZER_STACKDEPOTBASE_H
+#include <stdio.h>
+
+#include "sanitizer_atomic.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_mutex.h"
-#include "sanitizer_atomic.h"
#include "sanitizer_persistent_allocator.h"
namespace __sanitizer {
void LockAll();
void UnlockAll();
+ void PrintAll();
private:
static Node *find(Node *s, args_type args, u32 hash);
}
}
+template <class Node, int kReservedBits, int kTabSizeLog>
+void StackDepotBase<Node, kReservedBits, kTabSizeLog>::PrintAll() {
+ for (int i = 0; i < kTabSize; ++i) {
+ atomic_uintptr_t *p = &tab[i];
+ lock(p);
+ uptr v = atomic_load(p, memory_order_relaxed);
+ Node *s = (Node *)(v & ~1UL);
+ for (; s; s = s->link) {
+ Printf("Stack for id %u:\n", s->id);
+ s->load().Print();
+ }
+ unlock(p, s);
+ }
+}
+
} // namespace __sanitizer
#endif // SANITIZER_STACKDEPOTBASE_H
// run-time libraries.
//===----------------------------------------------------------------------===//
+#include "sanitizer_stacktrace.h"
+
#include "sanitizer_common.h"
#include "sanitizer_flags.h"
-#include "sanitizer_stacktrace.h"
+#include "sanitizer_platform.h"
+#include "sanitizer_ptrauth.h"
namespace __sanitizer {
return pc + 8;
#elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__)
return pc + 4;
+#elif SANITIZER_RISCV64
+ // Current check order is 4 -> 2 -> 6 -> 8
+ u8 InsnByte = *(u8 *)(pc);
+ if (((InsnByte & 0x3) == 0x3) && ((InsnByte & 0x1c) != 0x1c)) {
+ // xxxxxxxxxxxbbb11 | 32 bit | bbb != 111
+ return pc + 4;
+ }
+ if ((InsnByte & 0x3) != 0x3) {
+ // xxxxxxxxxxxxxxaa | 16 bit | aa != 11
+ return pc + 2;
+ }
+ // RISC-V encoding allows instructions to be up to 8 bytes long
+ if ((InsnByte & 0x3f) == 0x1f) {
+ // xxxxxxxxxx011111 | 48 bit |
+ return pc + 6;
+ }
+ if ((InsnByte & 0x7f) == 0x3f) {
+ // xxxxxxxxx0111111 | 64 bit |
+ return pc + 8;
+ }
+ // bail-out if could not figure out the instruction size
+ return 0;
#else
return pc + 1;
#endif
uhwptr pc1 = caller_frame[2];
#elif defined(__s390__)
uhwptr pc1 = frame[14];
+#elif defined(__riscv)
+ // frame[-1] contains the return address
+ uhwptr pc1 = frame[-1];
#else
- uhwptr pc1 = frame[1];
+ uhwptr pc1 = STRIP_PAC_PC((void *)frame[1]);
#endif
// Let's assume that any pointer in the 0th page (i.e. <0x1000 on i386 and
// x86_64) is invalid and stop unwinding here. If we're adding support for
trace_buffer[size++] = (uptr) pc1;
}
bottom = (uptr)frame;
- frame = GetCanonicFrame((uptr)frame[0], stack_top, bottom);
+#if defined(__riscv)
+ // frame[-2] contain fp of the previous frame
+ uptr new_bp = (uptr)frame[-2];
+#else
+ uptr new_bp = (uptr)frame[0];
+#endif
+ frame = GetCanonicFrame(new_bp, stack_top, bottom);
}
}
#ifndef SANITIZER_STACKTRACE_H
#define SANITIZER_STACKTRACE_H
+#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
namespace __sanitizer {
# define SANITIZER_CAN_FAST_UNWIND 0
#elif SANITIZER_WINDOWS
# define SANITIZER_CAN_FAST_UNWIND 0
-#elif SANITIZER_OPENBSD
-# define SANITIZER_CAN_FAST_UNWIND 0
#else
# define SANITIZER_CAN_FAST_UNWIND 1
#endif
// Fast unwind is the only option on Mac for now; we will need to
// revisit this macro when slow unwind works on Mac, see
// https://github.com/google/sanitizers/issues/137
-#if SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_RTEMS
-# define SANITIZER_CAN_SLOW_UNWIND 0
+#if SANITIZER_MAC
+# define SANITIZER_CAN_SLOW_UNWIND 0
#else
# define SANITIZER_CAN_SLOW_UNWIND 1
#endif
// Prints a symbolized stacktrace, followed by an empty line.
void Print() const;
+ // Prints a symbolized stacktrace to the output string, followed by an empty
+ // line.
+ void PrintTo(InternalScopedString *output) const;
+
+ // Prints a symbolized stacktrace to the output buffer, followed by an empty
+ // line. Returns the number of symbols that should have been written to buffer
+ // (not including trailing '\0'). Thus, the string is truncated iff return
+ // value is not less than "out_buf_size".
+ uptr PrintTo(char *out_buf, uptr out_buf_size) const;
+
static bool WillUseFastUnwind(bool request_fast_unwind) {
if (!SANITIZER_CAN_FAST_UNWIND)
return false;
static uptr GetCurrentPc();
static inline uptr GetPreviousInstructionPc(uptr pc);
static uptr GetNextInstructionPc(uptr pc);
- typedef bool (*SymbolizeCallback)(const void *pc, char *out_buffer,
- int out_size);
};
// Performance-critical, must be in the header.
return pc - 4;
#elif defined(__sparc__) || defined(__mips__)
return pc - 8;
+#elif SANITIZER_RISCV64
+ // RV-64 has variable instruciton length...
+ // C extentions gives us 2-byte instructoins
+ // RV-64 has 4-byte instructions
+ // + RISCV architecture allows instructions up to 8 bytes
+ // It seems difficult to figure out the exact instruction length -
+ // pc - 2 seems like a safe option for the purposes of stack tracing
+ return pc - 2;
#else
return pc - 1;
#endif
friend class FastUnwindTest;
};
+#if defined(__s390x__)
+static const uptr kFrameSize = 160;
+#elif defined(__s390__)
+static const uptr kFrameSize = 96;
+#else
+static const uptr kFrameSize = 2 * sizeof(uhwptr);
+#endif
+
// Check if given pointer points into allocated stack area.
static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
- return frame > stack_bottom && frame < stack_top - 2 * sizeof (uhwptr);
+ return frame > stack_bottom && frame < stack_top - kFrameSize;
}
} // namespace __sanitizer
uptr local_stack; \
uptr sp = (uptr)&local_stack
+// GET_CURRENT_PC() is equivalent to StackTrace::GetCurrentPc().
+// Optimized x86 version is faster than GetCurrentPc because
+// it does not involve a function call, instead it reads RIP register.
+// Reads of RIP by an instruction return RIP pointing to the next
+// instruction, which is exactly what we want here, thus 0 offset.
+// It needs to be a macro because otherwise we will get the name
+// of this function on the top of most stacks. Attribute artificial
+// does not do what it claims to do, unfortunatley. And attribute
+// __nodebug__ is clang-only. If we would have an attribute that
+// would remove this function from debug info, we could simply make
+// StackTrace::GetCurrentPc() faster.
+#if defined(__x86_64__)
+# define GET_CURRENT_PC() \
+ ({ \
+ uptr pc; \
+ asm("lea 0(%%rip), %0" : "=r"(pc)); \
+ pc; \
+ })
+#else
+# define GET_CURRENT_PC() StackTrace::GetCurrentPc()
+#endif
#endif // SANITIZER_STACKTRACE_H
namespace __sanitizer {
-void StackTrace::Print() const {
+namespace {
+
+class StackTraceTextPrinter {
+ public:
+ StackTraceTextPrinter(const char *stack_trace_fmt, char frame_delimiter,
+ InternalScopedString *output,
+ InternalScopedString *dedup_token)
+ : stack_trace_fmt_(stack_trace_fmt),
+ frame_delimiter_(frame_delimiter),
+ output_(output),
+ dedup_token_(dedup_token),
+ symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {}
+
+ bool ProcessAddressFrames(uptr pc) {
+ SymbolizedStack *frames = symbolize_
+ ? Symbolizer::GetOrInit()->SymbolizePC(pc)
+ : SymbolizedStack::New(pc);
+ if (!frames)
+ return false;
+
+ for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
+ uptr prev_len = output_->length();
+ RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address,
+ symbolize_ ? &cur->info : nullptr,
+ common_flags()->symbolize_vs_style,
+ common_flags()->strip_path_prefix);
+
+ if (prev_len != output_->length())
+ output_->append("%c", frame_delimiter_);
+
+ ExtendDedupToken(cur);
+ }
+ frames->ClearAll();
+ return true;
+ }
+
+ private:
+ // Extend the dedup token by appending a new frame.
+ void ExtendDedupToken(SymbolizedStack *stack) {
+ if (!dedup_token_)
+ return;
+
+ if (dedup_frames_-- > 0) {
+ if (dedup_token_->length())
+ dedup_token_->append("--");
+ if (stack->info.function != nullptr)
+ dedup_token_->append(stack->info.function);
+ }
+ }
+
+ const char *stack_trace_fmt_;
+ const char frame_delimiter_;
+ int dedup_frames_ = common_flags()->dedup_token_length;
+ uptr frame_num_ = 0;
+ InternalScopedString *output_;
+ InternalScopedString *dedup_token_;
+ const bool symbolize_ = false;
+};
+
+static void CopyStringToBuffer(const InternalScopedString &str, char *out_buf,
+ uptr out_buf_size) {
+ if (!out_buf_size)
+ return;
+
+ CHECK_GT(out_buf_size, 0);
+ uptr copy_size = Min(str.length(), out_buf_size - 1);
+ internal_memcpy(out_buf, str.data(), copy_size);
+ out_buf[copy_size] = '\0';
+}
+
+} // namespace
+
+void StackTrace::PrintTo(InternalScopedString *output) const {
+ CHECK(output);
+
+ InternalScopedString dedup_token;
+ StackTraceTextPrinter printer(common_flags()->stack_trace_format, '\n',
+ output, &dedup_token);
+
if (trace == nullptr || size == 0) {
- Printf(" <empty stack>\n\n");
+ output->append(" <empty stack>\n\n");
return;
}
- InternalScopedString frame_desc(GetPageSizeCached() * 2);
- InternalScopedString dedup_token(GetPageSizeCached());
- int dedup_frames = common_flags()->dedup_token_length;
- uptr frame_num = 0;
+
for (uptr i = 0; i < size && trace[i]; i++) {
// PCs in stack traces are actually the return addresses, that is,
// addresses of the next instructions after the call.
uptr pc = GetPreviousInstructionPc(trace[i]);
- SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(pc);
- CHECK(frames);
- for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
- frame_desc.clear();
- RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++,
- cur->info, common_flags()->symbolize_vs_style,
- common_flags()->strip_path_prefix);
- Printf("%s\n", frame_desc.data());
- if (dedup_frames-- > 0) {
- if (dedup_token.length())
- dedup_token.append("--");
- if (cur->info.function != nullptr)
- dedup_token.append(cur->info.function);
- }
- }
- frames->ClearAll();
+ CHECK(printer.ProcessAddressFrames(pc));
}
- // Always print a trailing empty line after stack trace.
- Printf("\n");
+
+ // Always add a trailing empty line after stack trace.
+ output->append("\n");
+
+ // Append deduplication token, if non-empty.
if (dedup_token.length())
- Printf("DEDUP_TOKEN: %s\n", dedup_token.data());
+ output->append("DEDUP_TOKEN: %s\n", dedup_token.data());
+}
+
+uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const {
+ CHECK(out_buf);
+
+ InternalScopedString output;
+ PrintTo(&output);
+ CopyStringToBuffer(output, out_buf, out_buf_size);
+
+ return output.length();
+}
+
+void StackTrace::Print() const {
+ InternalScopedString output;
+ PrintTo(&output);
+ Printf("%s", output.data());
}
void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
UnwindSlow(pc, context, max_depth);
else
UnwindSlow(pc, max_depth);
+ // If there are too few frames, the program may be built with
+ // -fno-asynchronous-unwind-tables. Fall back to fast unwinder below.
+ if (size > 2 || size >= max_depth)
+ return;
#else
UNREACHABLE("slow unwind requested but not available");
#endif
- } else {
- UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
}
+ UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
}
static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf,
uptr out_buf_size) {
- if (!out_buf_size) return;
- pc = StackTrace::GetPreviousInstructionPc(pc);
- SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc);
- if (!frame) {
- internal_strncpy(out_buf, "<can't symbolize>", out_buf_size);
- out_buf[out_buf_size - 1] = 0;
+ if (!out_buf_size)
return;
+
+ pc = StackTrace::GetPreviousInstructionPc(pc);
+
+ InternalScopedString output;
+ StackTraceTextPrinter printer(fmt, '\0', &output, nullptr);
+ if (!printer.ProcessAddressFrames(pc)) {
+ output.clear();
+ output.append("<can't symbolize>");
}
- InternalScopedString frame_desc(GetPageSizeCached());
- uptr frame_num = 0;
- // Reserve one byte for the final 0.
- char *out_end = out_buf + out_buf_size - 1;
- for (SymbolizedStack *cur = frame; cur && out_buf < out_end;
- cur = cur->next) {
- frame_desc.clear();
- RenderFrame(&frame_desc, fmt, frame_num++, cur->info,
- common_flags()->symbolize_vs_style,
- common_flags()->strip_path_prefix);
- if (!frame_desc.length())
- continue;
- // Reserve one byte for the terminating 0.
- uptr n = out_end - out_buf - 1;
- internal_strncpy(out_buf, frame_desc.data(), n);
- out_buf += __sanitizer::Min<uptr>(n, frame_desc.length());
- *out_buf++ = 0;
- }
- CHECK(out_buf <= out_end);
- *out_buf = 0;
+ CopyStringToBuffer(output, out_buf, out_buf_size);
}
SANITIZER_INTERFACE_ATTRIBUTE
out_buf[0] = 0;
DataInfo DI;
if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return;
- InternalScopedString data_desc(GetPageSizeCached());
+ InternalScopedString data_desc;
RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix);
internal_strncpy(out_buf, data_desc.data(), out_buf_size);
out_buf[out_buf_size - 1] = 0;
static const char kDefaultFormat[] = " #%n %p %F %L";
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
- const AddressInfo &info, bool vs_style,
+ uptr address, const AddressInfo *info, bool vs_style,
const char *strip_path_prefix, const char *strip_func_prefix) {
+ // info will be null in the case where symbolization is not needed for the
+ // given format. This ensures that the code below will get a hard failure
+ // rather than print incorrect information in case RenderNeedsSymbolization
+ // ever ends up out of sync with this function. If non-null, the addresses
+ // should match.
+ CHECK(!info || address == info->address);
if (0 == internal_strcmp(format, "DEFAULT"))
format = kDefaultFormat;
for (const char *p = format; *p != '\0'; p++) {
buffer->append("%zu", frame_no);
break;
case 'p':
- buffer->append("0x%zx", info.address);
+ buffer->append("0x%zx", address);
break;
case 'm':
- buffer->append("%s", StripPathPrefix(info.module, strip_path_prefix));
+ buffer->append("%s", StripPathPrefix(info->module, strip_path_prefix));
break;
case 'o':
- buffer->append("0x%zx", info.module_offset);
+ buffer->append("0x%zx", info->module_offset);
break;
case 'f':
- buffer->append("%s",
- DemangleFunctionName(
- StripFunctionName(info.function, strip_func_prefix)));
+ buffer->append("%s", DemangleFunctionName(StripFunctionName(
+ info->function, strip_func_prefix)));
break;
case 'q':
- buffer->append("0x%zx", info.function_offset != AddressInfo::kUnknown
- ? info.function_offset
+ buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
+ ? info->function_offset
: 0x0);
break;
case 's':
- buffer->append("%s", StripPathPrefix(info.file, strip_path_prefix));
+ buffer->append("%s", StripPathPrefix(info->file, strip_path_prefix));
break;
case 'l':
- buffer->append("%d", info.line);
+ buffer->append("%d", info->line);
break;
case 'c':
- buffer->append("%d", info.column);
+ buffer->append("%d", info->column);
break;
// Smarter special cases.
case 'F':
// Function name and offset, if file is unknown.
- if (info.function) {
- buffer->append("in %s",
- DemangleFunctionName(
- StripFunctionName(info.function, strip_func_prefix)));
- if (!info.file && info.function_offset != AddressInfo::kUnknown)
- buffer->append("+0x%zx", info.function_offset);
+ if (info->function) {
+ buffer->append("in %s", DemangleFunctionName(StripFunctionName(
+ info->function, strip_func_prefix)));
+ if (!info->file && info->function_offset != AddressInfo::kUnknown)
+ buffer->append("+0x%zx", info->function_offset);
}
break;
case 'S':
// File/line information.
- RenderSourceLocation(buffer, info.file, info.line, info.column, vs_style,
- strip_path_prefix);
+ RenderSourceLocation(buffer, info->file, info->line, info->column,
+ vs_style, strip_path_prefix);
break;
case 'L':
// Source location, or module location.
- if (info.file) {
- RenderSourceLocation(buffer, info.file, info.line, info.column,
+ if (info->file) {
+ RenderSourceLocation(buffer, info->file, info->line, info->column,
vs_style, strip_path_prefix);
- } else if (info.module) {
- RenderModuleLocation(buffer, info.module, info.module_offset,
- info.module_arch, strip_path_prefix);
+ } else if (info->module) {
+ RenderModuleLocation(buffer, info->module, info->module_offset,
+ info->module_arch, strip_path_prefix);
} else {
buffer->append("(<unknown module>)");
}
break;
case 'M':
// Module basename and offset, or PC.
- if (info.address & kExternalPCBit)
- {} // There PCs are not meaningful.
- else if (info.module)
+ if (address & kExternalPCBit) {
+ // There PCs are not meaningful.
+ } else if (info->module) {
// Always strip the module name for %M.
- RenderModuleLocation(buffer, StripModuleName(info.module),
- info.module_offset, info.module_arch, "");
- else
- buffer->append("(%p)", (void *)info.address);
+ RenderModuleLocation(buffer, StripModuleName(info->module),
+ info->module_offset, info->module_arch, "");
+ } else {
+ buffer->append("(%p)", (void *)address);
+ }
break;
default:
Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
}
}
+bool RenderNeedsSymbolization(const char *format) {
+ if (0 == internal_strcmp(format, "DEFAULT"))
+ format = kDefaultFormat;
+ for (const char *p = format; *p != '\0'; p++) {
+ if (*p != '%')
+ continue;
+ p++;
+ switch (*p) {
+ case '%':
+ break;
+ case 'n':
+ // frame_no
+ break;
+ case 'p':
+ // address
+ break;
+ default:
+ return true;
+ }
+ }
+ return false;
+}
+
void RenderData(InternalScopedString *buffer, const char *format,
const DataInfo *DI, const char *strip_path_prefix) {
for (const char *p = format; *p != '\0'; p++) {
// module+offset if it is known, or (<unknown module>) string.
// %M - prints module basename and offset, if it is known, or PC.
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
- const AddressInfo &info, bool vs_style,
+ uptr address, const AddressInfo *info, bool vs_style,
const char *strip_path_prefix = "",
const char *strip_func_prefix = "");
+bool RenderNeedsSymbolization(const char *format);
+
void RenderSourceLocation(InternalScopedString *buffer, const char *file,
int line, int column, bool vs_style,
const char *strip_path_prefix);
// Can't declare pure virtual functions in sanitizer runtimes:
// __cxa_pure_virtual might be unavailable. Use UNIMPLEMENTED() instead.
- virtual PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
- uptr *sp) const {
+ virtual PtraceRegistersStatus GetRegistersAndSP(
+ uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
UNIMPLEMENTED();
}
- // The buffer in GetRegistersAndSP should be at least this big.
- virtual uptr RegisterCount() const { UNIMPLEMENTED(); }
virtual uptr ThreadCount() const { UNIMPLEMENTED(); }
virtual tid_t GetThreadID(uptr index) const { UNIMPLEMENTED(); }
+ protected:
+ ~SuspendedThreadsList() {}
+
private:
// Prohibit copy and assign.
- SuspendedThreadsList(const SuspendedThreadsList&);
- void operator=(const SuspendedThreadsList&);
+ SuspendedThreadsList(const SuspendedThreadsList &) = delete;
+ void operator=(const SuspendedThreadsList &) = delete;
};
typedef void (*StopTheWorldCallback)(
#include <zircon/sanitizer.h>
#include "sanitizer_stoptheworld.h"
+#include "sanitizer_stoptheworld_fuchsia.h"
namespace __sanitizer {
nullptr, nullptr, nullptr, nullptr,
[](zx_status_t, void *data) {
auto params = reinterpret_cast<Params *>(data);
- params->callback({}, params->argument);
+ params->callback(SuspendedThreadsListFuchsia(), params->argument);
},
¶ms);
}
--- /dev/null
+//===-- sanitizer_stoptheworld_fuchsia.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_STOPTHEWORLD_FUCHSIA_H
+#define SANITIZER_STOPTHEWORLD_FUCHSIA_H
+
+#include "sanitizer_stoptheworld.h"
+
+namespace __sanitizer {
+
+class SuspendedThreadsListFuchsia final : public SuspendedThreadsList {};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_STOPTHEWORLD_FUCHSIA_H
#include "sanitizer_platform.h"
-#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) || \
- defined(__aarch64__) || defined(__powerpc64__) || \
- defined(__s390__) || defined(__i386__) || \
- defined(__arm__))
+#if SANITIZER_LINUX && \
+ (defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \
+ defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \
+ defined(__arm__) || SANITIZER_RISCV64)
#include "sanitizer_stoptheworld.h"
#include <sys/types.h> // for pid_t
#include <sys/uio.h> // for iovec
#include <elf.h> // for NT_PRSTATUS
-#if defined(__aarch64__) && !SANITIZER_ANDROID
+#if (defined(__aarch64__) || SANITIZER_RISCV64) && !SANITIZER_ANDROID
// GLIBC 2.20+ sys/user does not include asm/ptrace.h
# include <asm/ptrace.h>
#endif
namespace __sanitizer {
-class SuspendedThreadsListLinux : public SuspendedThreadsList {
+class SuspendedThreadsListLinux final : public SuspendedThreadsList {
public:
SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
- tid_t GetThreadID(uptr index) const;
- uptr ThreadCount() const;
+ tid_t GetThreadID(uptr index) const override;
+ uptr ThreadCount() const override;
bool ContainsTid(tid_t thread_id) const;
void Append(tid_t tid);
- PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
- uptr *sp) const;
- uptr RegisterCount() const;
+ PtraceRegistersStatus GetRegistersAndSP(uptr index,
+ InternalMmapVector<uptr> *buffer,
+ uptr *sp) const override;
private:
InternalMmapVector<tid_t> thread_ids_;
#else
#define REG_SP rsp
#endif
+#define ARCH_IOVEC_FOR_GETREGSET
+// Support ptrace extensions even when compiled without required kernel support
+#ifndef NT_X86_XSTATE
+#define NT_X86_XSTATE 0x202
+#endif
+#ifndef PTRACE_GETREGSET
+#define PTRACE_GETREGSET 0x4204
+#endif
+// Compiler may use FP registers to store pointers.
+static constexpr uptr kExtraRegs[] = {NT_X86_XSTATE, NT_FPREGSET};
#elif defined(__powerpc__) || defined(__powerpc64__)
typedef pt_regs regs_struct;
#elif defined(__aarch64__)
typedef struct user_pt_regs regs_struct;
#define REG_SP sp
+static constexpr uptr kExtraRegs[] = {0};
+#define ARCH_IOVEC_FOR_GETREGSET
+
+#elif SANITIZER_RISCV64
+typedef struct user_regs_struct regs_struct;
+// sys/ucontext.h already defines REG_SP as 2. Undefine it first.
+#undef REG_SP
+#define REG_SP sp
+static constexpr uptr kExtraRegs[] = {0};
#define ARCH_IOVEC_FOR_GETREGSET
#elif defined(__s390__)
typedef _user_regs_struct regs_struct;
#define REG_SP gprs[15]
+static constexpr uptr kExtraRegs[] = {0};
#define ARCH_IOVEC_FOR_GETREGSET
#else
}
PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
- uptr index, uptr *buffer, uptr *sp) const {
+ uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
pid_t tid = GetThreadID(index);
- regs_struct regs;
+ constexpr uptr uptr_sz = sizeof(uptr);
int pterrno;
#ifdef ARCH_IOVEC_FOR_GETREGSET
- struct iovec regset_io;
- regset_io.iov_base = ®s;
- regset_io.iov_len = sizeof(regs_struct);
- bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid,
- (void*)NT_PRSTATUS, (void*)®set_io),
- &pterrno);
+ auto append = [&](uptr regset) {
+ uptr size = buffer->size();
+ // NT_X86_XSTATE requires 64bit alignment.
+ uptr size_up = RoundUpTo(size, 8 / uptr_sz);
+ buffer->reserve(Max<uptr>(1024, size_up));
+ struct iovec regset_io;
+ for (;; buffer->resize(buffer->capacity() * 2)) {
+ buffer->resize(buffer->capacity());
+ uptr available_bytes = (buffer->size() - size_up) * uptr_sz;
+ regset_io.iov_base = buffer->data() + size_up;
+ regset_io.iov_len = available_bytes;
+ bool fail =
+ internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid,
+ (void *)regset, (void *)®set_io),
+ &pterrno);
+ if (fail) {
+ VReport(1, "Could not get regset %p from thread %d (errno %d).\n",
+ (void *)regset, tid, pterrno);
+ buffer->resize(size);
+ return false;
+ }
+
+ // Far enough from the buffer size, no need to resize and repeat.
+ if (regset_io.iov_len + 64 < available_bytes)
+ break;
+ }
+ buffer->resize(size_up + RoundUpTo(regset_io.iov_len, uptr_sz) / uptr_sz);
+ return true;
+ };
+
+ buffer->clear();
+ bool fail = !append(NT_PRSTATUS);
+ if (!fail) {
+ // Accept the first available and do not report errors.
+ for (uptr regs : kExtraRegs)
+ if (regs && append(regs))
+ break;
+ }
#else
- bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, nullptr,
- ®s), &pterrno);
-#endif
- if (isErr) {
+ buffer->resize(RoundUpTo(sizeof(regs_struct), uptr_sz) / uptr_sz);
+ bool fail = internal_iserror(
+ internal_ptrace(PTRACE_GETREGS, tid, nullptr, buffer->data()), &pterrno);
+ if (fail)
VReport(1, "Could not get registers from thread %d (errno %d).\n", tid,
pterrno);
+#endif
+ if (fail) {
// ESRCH means that the given thread is not suspended or already dead.
// Therefore it's unsafe to inspect its data (e.g. walk through stack) and
// we should notify caller about this.
: REGISTERS_UNAVAILABLE;
}
- *sp = regs.REG_SP;
- internal_memcpy(buffer, ®s, sizeof(regs));
+ *sp = reinterpret_cast<regs_struct *>(buffer->data())[0].REG_SP;
return REGISTERS_AVAILABLE;
}
-uptr SuspendedThreadsListLinux::RegisterCount() const {
- return sizeof(regs_struct) / sizeof(uptr);
-}
} // namespace __sanitizer
#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)
thread_t thread;
} SuspendedThreadInfo;
-class SuspendedThreadsListMac : public SuspendedThreadsList {
+class SuspendedThreadsListMac final : public SuspendedThreadsList {
public:
SuspendedThreadsListMac() : threads_(1024) {}
- tid_t GetThreadID(uptr index) const;
+ tid_t GetThreadID(uptr index) const override;
thread_t GetThread(uptr index) const;
- uptr ThreadCount() const;
+ uptr ThreadCount() const override;
bool ContainsThread(thread_t thread) const;
void Append(thread_t thread);
- PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
- uptr *sp) const;
- uptr RegisterCount() const;
+ PtraceRegistersStatus GetRegistersAndSP(uptr index,
+ InternalMmapVector<uptr> *buffer,
+ uptr *sp) const override;
private:
InternalMmapVector<SuspendedThreadInfo> threads_;
}
PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
- uptr index, uptr *buffer, uptr *sp) const {
+ uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
thread_t thread = GetThread(index);
regs_struct regs;
int err;
: REGISTERS_UNAVAILABLE;
}
- internal_memcpy(buffer, ®s, sizeof(regs));
+ buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr));
+ internal_memcpy(buffer->data(), ®s, sizeof(regs));
#if defined(__aarch64__) && defined(arm_thread_state64_get_sp)
*sp = arm_thread_state64_get_sp(regs);
#else
return REGISTERS_AVAILABLE;
}
-uptr SuspendedThreadsListMac::RegisterCount() const {
- return MACHINE_THREAD_STATE_COUNT;
-}
} // namespace __sanitizer
#endif // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) ||
namespace __sanitizer {
-class SuspendedThreadsListNetBSD : public SuspendedThreadsList {
+class SuspendedThreadsListNetBSD final : public SuspendedThreadsList {
public:
SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); }
bool ContainsTid(tid_t thread_id) const;
void Append(tid_t tid);
- PtraceRegistersStatus GetRegistersAndSP(uptr index, uptr *buffer,
+ PtraceRegistersStatus GetRegistersAndSP(uptr index,
+ InternalMmapVector<uptr> *buffer,
uptr *sp) const;
- uptr RegisterCount() const;
private:
InternalMmapVector<tid_t> thread_ids_;
pl.pl_lwpid = 0;
int val;
- while ((val = ptrace(op, pid_, (void *)&pl, sizeof(pl))) != -1 &&
+ while ((val = internal_ptrace(op, pid_, (void *)&pl, sizeof(pl))) != -1 &&
pl.pl_lwpid != 0) {
suspended_threads_list_.Append(pl.pl_lwpid);
VReport(2, "Appended thread %d in process %d.\n", pl.pl_lwpid, pid_);
}
PtraceRegistersStatus SuspendedThreadsListNetBSD::GetRegistersAndSP(
- uptr index, uptr *buffer, uptr *sp) const {
+ uptr index, InternalMmapVector<uptr> *buffer, uptr *sp) const {
lwpid_t tid = GetThreadID(index);
pid_t ppid = internal_getppid();
struct reg regs;
}
*sp = PTRACE_REG_SP(®s);
- internal_memcpy(buffer, ®s, sizeof(regs));
+ buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr));
+ internal_memcpy(buffer->data(), ®s, sizeof(regs));
return REGISTERS_AVAILABLE;
}
-uptr SuspendedThreadsListNetBSD::RegisterCount() const {
- return sizeof(struct reg) / sizeof(uptr);
-}
} // namespace __sanitizer
#endif
static bool GetPathAssumingFileIsRelativeToExec(const char *file_path,
/*out*/char *new_file_path,
uptr new_file_path_size) {
- InternalScopedString exec(kMaxPathLength);
+ InternalMmapVector<char> exec(kMaxPathLength);
if (ReadBinaryNameCached(exec.data(), exec.size())) {
const char *file_name_pos = StripModuleName(exec.data());
uptr path_to_exec_len = file_name_pos - exec.data();
if (filename[0] == '\0')
return;
- InternalScopedString new_file_path(kMaxPathLength);
+ InternalMmapVector<char> new_file_path(kMaxPathLength);
filename = FindFile(filename, new_file_path.data(), new_file_path.size());
// Read the file.
// Usually this is a safe place to call code that might need to use user
// memory allocators.
virtual void LateInitialize() {}
+
+ protected:
+ ~SymbolizerTool() {}
};
// SymbolizerProcess encapsulates communication between the tool and
const char *SendCommand(const char *command);
protected:
+ ~SymbolizerProcess() {}
+
/// The maximum number of arguments required to invoke a tool process.
static const unsigned kArgVMax = 6;
// This tool invokes llvm-symbolizer in a subprocess. It should be as portable
// as the llvm-symbolizer tool is.
-class LLVMSymbolizer : public SymbolizerTool {
+class LLVMSymbolizer final : public SymbolizerTool {
public:
explicit LLVMSymbolizer(const char *path, LowLevelAllocator *allocator);
namespace __sanitizer {
-class LibbacktraceSymbolizer : public SymbolizerTool {
+class LibbacktraceSymbolizer final : public SymbolizerTool {
public:
static LibbacktraceSymbolizer *get(LowLevelAllocator *alloc);
#include "sanitizer_allocator_internal.h"
#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
#include "sanitizer_symbolizer_internal.h"
namespace __sanitizer {
// <file_name>:<line_number>:<column_number>
// ...
// <empty line>
-class LLVMSymbolizerProcess : public SymbolizerProcess {
+class LLVMSymbolizerProcess final : public SymbolizerProcess {
public:
explicit LLVMSymbolizerProcess(const char *path)
: SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_MAC) {}
const char* const kSymbolizerArch = "--default-arch=x86_64";
#elif defined(__i386__)
const char* const kSymbolizerArch = "--default-arch=i386";
+#elif SANITIZER_RISCV64
+ const char *const kSymbolizerArch = "--default-arch=riscv64";
#elif defined(__aarch64__)
const char* const kSymbolizerArch = "--default-arch=arm64";
#elif defined(__arm__)
#endif
const char *const inline_flag = common_flags()->symbolize_inline_frames
- ? "--inlining=true"
- : "--inlining=false";
+ ? "--inlines"
+ : "--no-inlines";
int i = 0;
argv[i++] = path_to_binary;
argv[i++] = inline_flag;
InternalFree(info->function);
info->function = 0;
}
- if (0 == internal_strcmp(info->file, "??")) {
+ if (info->file && 0 == internal_strcmp(info->file, "??")) {
InternalFree(info->file);
info->file = 0;
}
int result = dladdr((const void *)addr, &info);
if (!result) return false;
- CHECK(addr >= reinterpret_cast<uptr>(info.dli_saddr));
- stack->info.function_offset = addr - reinterpret_cast<uptr>(info.dli_saddr);
+ // Compute offset if possible. `dladdr()` doesn't always ensure that `addr >=
+ // sym_addr` so only compute the offset when this holds. Failure to find the
+ // function offset is not treated as a failure because it might still be
+ // possible to get the symbol name.
+ uptr sym_addr = reinterpret_cast<uptr>(info.dli_saddr);
+ if (addr >= sym_addr) {
+ stack->info.function_offset = addr - sym_addr;
+ }
+
const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
if (!demangled) return false;
stack->info.function = internal_strdup(demangled);
// kAsanInternalHeapMagic.
static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
-class AtosSymbolizerProcess : public SymbolizerProcess {
+class AtosSymbolizerProcess final : public SymbolizerProcess {
public:
explicit AtosSymbolizerProcess(const char *path)
: SymbolizerProcess(path, /*use_posix_spawn*/ true) {
start_address = reinterpret_cast<uptr>(info.dli_saddr);
}
- // Only assig to `function_offset` if we were able to get the function's
- // start address.
- if (start_address != AddressInfo::kUnknown) {
- CHECK(addr >= start_address);
+ // Only assign to `function_offset` if we were able to get the function's
+ // start address and we got a sensible `start_address` (dladdr doesn't always
+ // ensure that `addr >= sym_addr`).
+ if (start_address != AddressInfo::kUnknown && addr >= start_address) {
stack->info.function_offset = addr - start_address;
}
return true;
namespace __sanitizer {
-class DlAddrSymbolizer : public SymbolizerTool {
+class DlAddrSymbolizer final : public SymbolizerTool {
public:
bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
bool SymbolizeData(uptr addr, DataInfo *info) override;
class AtosSymbolizerProcess;
-class AtosSymbolizer : public SymbolizerTool {
+class AtosSymbolizer final : public SymbolizerTool {
public:
explicit AtosSymbolizer(const char *path, LowLevelAllocator *allocator);
#if SANITIZER_FUCHSIA
#include "sanitizer_symbolizer_fuchsia.h"
-#elif SANITIZER_RTEMS
-#include "sanitizer_symbolizer_rtems.h"
-#endif
-#include "sanitizer_stacktrace.h"
-#include "sanitizer_symbolizer.h"
+# endif
-#include <limits.h>
-#include <unwind.h>
+# include <limits.h>
+# include <unwind.h>
+
+# include "sanitizer_stacktrace.h"
+# include "sanitizer_symbolizer.h"
namespace __sanitizer {
return false;
}
+// This is mainly used by hwasan for online symbolization. This isn't needed
+// since hwasan can always just dump stack frames for offline symbolization.
+bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return false; }
+
// This is used in some places for suppression checking, which we
// don't really support for Fuchsia. It's also used in UBSan to
// identify a PC location to a function name, so we always fill in
buffer->append(kFormatData, DI->start);
}
+bool RenderNeedsSymbolization(const char *format) { return false; }
+
// We don't support the stack_trace_format flag at all.
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
- const AddressInfo &info, bool vs_style,
+ uptr address, const AddressInfo *info, bool vs_style,
const char *strip_path_prefix, const char *strip_func_prefix) {
- buffer->append(kFormatFrame, frame_no, info.address);
+ CHECK(!RenderNeedsSymbolization(format));
+ buffer->append(kFormatFrame, frame_no, address);
}
Symbolizer *Symbolizer::PlatformInit() {
return true;
}
-class Addr2LineProcess : public SymbolizerProcess {
+class Addr2LineProcess final : public SymbolizerProcess {
public:
Addr2LineProcess(const char *path, const char *module_name)
: SymbolizerProcess(path), module_name_(internal_strdup(module_name)) {}
output_terminator_, kTerminatorLen);
}
-class Addr2LinePool : public SymbolizerTool {
+class Addr2LinePool final : public SymbolizerTool {
public:
explicit Addr2LinePool(const char *addr2line_path,
LowLevelAllocator *allocator)
int MaxLength);
} // extern "C"
-class InternalSymbolizer : public SymbolizerTool {
+class InternalSymbolizer final : public SymbolizerTool {
public:
static InternalSymbolizer *get(LowLevelAllocator *alloc) {
if (__sanitizer_symbolize_code != 0 &&
};
#else // SANITIZER_SUPPORTS_WEAK_HOOKS
-class InternalSymbolizer : public SymbolizerTool {
+class InternalSymbolizer final : public SymbolizerTool {
public:
static InternalSymbolizer *get(LowLevelAllocator *alloc) { return 0; }
};
static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) {
const char *path = common_flags()->external_symbolizer_path;
+
+ if (path && internal_strchr(path, '%')) {
+ char *new_path = (char *)InternalAlloc(kMaxPathLength);
+ SubstituteForFlagValue(path, new_path, kMaxPathLength);
+ path = new_path;
+ }
+
const char *binary_name = path ? StripModuleName(path) : "";
+ static const char kLLVMSymbolizerPrefix[] = "llvm-symbolizer";
if (path && path[0] == '\0') {
VReport(2, "External symbolizer is explicitly disabled.\n");
return nullptr;
- } else if (!internal_strcmp(binary_name, "llvm-symbolizer")) {
+ } else if (!internal_strncmp(binary_name, kLLVMSymbolizerPrefix,
+ internal_strlen(kLLVMSymbolizerPrefix))) {
VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
return new(*allocator) LLVMSymbolizer(path, allocator);
} else if (!internal_strcmp(binary_name, "atos")) {
void ReportErrorSummary(const char *error_type, const AddressInfo &info,
const char *alt_tool_name) {
if (!common_flags()->print_summary) return;
- InternalScopedString buff(kMaxSummaryLength);
+ InternalScopedString buff;
buff.append("%s ", error_type);
- RenderFrame(&buff, "%L %F", 0, info, common_flags()->symbolize_vs_style,
+ RenderFrame(&buff, "%L %F", 0, info.address, &info,
+ common_flags()->symbolize_vs_style,
common_flags()->strip_path_prefix);
ReportErrorSummary(buff.data(), alt_tool_name);
}
return SupportsColoredOutput(fd);
}
-static INLINE bool ReportSupportsColors() {
+static inline bool ReportSupportsColors() {
return report_file.SupportsColors();
}
#else // SANITIZER_FUCHSIA
// Fuchsia's logs always go through post-processing that handles colorization.
-static INLINE bool ReportSupportsColors() { return true; }
+static inline bool ReportSupportsColors() { return true; }
#endif // !SANITIZER_FUCHSIA
#endif
}
-#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO
+#if !SANITIZER_FUCHSIA && !SANITIZER_GO
void StartReportDeadlySignal() {
// Write the first message using fd=2, just in case.
// It may actually fail to write in case stderr is closed.
static void MaybeDumpInstructionBytes(uptr pc) {
if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached()))
return;
- InternalScopedString str(1024);
+ InternalScopedString str;
str.append("First 16 instruction bytes at pc: ");
if (IsAccessibleMemoryRange(pc, 16)) {
for (int i = 0; i < 16; ++i) {
Report("The signal is caused by a %s memory access.\n", access_type);
if (!sig.is_true_faulting_addr)
Report("Hint: this fault was caused by a dereference of a high value "
- "address (see register values below). Dissassemble the provided "
+ "address (see register values below). Disassemble the provided "
"pc to learn which register was used.\n");
else if (sig.addr < GetPageSizeCached())
Report("Hint: address points to the zero page.\n");
#endif // !SANITIZER_FUCHSIA && !SANITIZER_GO
-static atomic_uintptr_t reporting_thread = {0};
-static StaticSpinMutex CommonSanitizerReportMutex;
+atomic_uintptr_t ScopedErrorReportLock::reporting_thread_ = {0};
+StaticSpinMutex ScopedErrorReportLock::mutex_;
-ScopedErrorReportLock::ScopedErrorReportLock() {
+void ScopedErrorReportLock::Lock() {
uptr current = GetThreadSelf();
for (;;) {
uptr expected = 0;
- if (atomic_compare_exchange_strong(&reporting_thread, &expected, current,
+ if (atomic_compare_exchange_strong(&reporting_thread_, &expected, current,
memory_order_relaxed)) {
// We've claimed reporting_thread so proceed.
- CommonSanitizerReportMutex.Lock();
+ mutex_.Lock();
return;
}
}
}
-ScopedErrorReportLock::~ScopedErrorReportLock() {
- CommonSanitizerReportMutex.Unlock();
- atomic_store_relaxed(&reporting_thread, 0);
+void ScopedErrorReportLock::Unlock() {
+ mutex_.Unlock();
+ atomic_store_relaxed(&reporting_thread_, 0);
}
-void ScopedErrorReportLock::CheckLocked() {
- CommonSanitizerReportMutex.CheckLocked();
-}
+void ScopedErrorReportLock::CheckLocked() { mutex_.CheckLocked(); }
} // namespace __sanitizer
namespace {
-class WinSymbolizerTool : public SymbolizerTool {
+class WinSymbolizerTool final : public SymbolizerTool {
public:
// The constructor is provided to avoid synthesized memsets.
WinSymbolizerTool() {}
bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) {
InitializeDbgHelpIfNeeded();
- // See http://msdn.microsoft.com/en-us/library/ms680578(VS.85).aspx
- char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)];
- PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer;
+ // See https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-symbol-information-by-address
+ InternalMmapVector<char> buffer(sizeof(SYMBOL_INFO) +
+ MAX_SYM_NAME * sizeof(CHAR));
+ PSYMBOL_INFO symbol = (PSYMBOL_INFO)&buffer[0];
symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
symbol->MaxNameLen = MAX_SYM_NAME;
DWORD64 offset = 0;
// Compute the command line. Wrap double quotes around everything.
const char *argv[kArgVMax];
GetArgV(path_, argv);
- InternalScopedString command_line(kMaxPathLength * 3);
+ InternalScopedString command_line;
for (int i = 0; argv[i]; i++) {
const char *arg = argv[i];
int arglen = internal_strlen(arg);
return;
}
- // Add llvm-symbolizer in case the binary has dwarf.
+ // Add llvm-symbolizer.
const char *user_path = common_flags()->external_symbolizer_path;
+
+ if (user_path && internal_strchr(user_path, '%')) {
+ char *new_path = (char *)InternalAlloc(kMaxPathLength);
+ SubstituteForFlagValue(user_path, new_path, kMaxPathLength);
+ user_path = new_path;
+ }
+
const char *path =
user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
if (path) {
// NetBSD uses libc calls directly
#if !SANITIZER_NETBSD
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_OPENBSD || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_SOLARIS
# define SYSCALL(name) SYS_ ## name
#else
# define SYSCALL(name) __NR_ ## name
--- /dev/null
+//===-- sanitizer_syscall_linux_riscv64.inc ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of internal_syscall and internal_iserror for Linux/riscv64.
+//
+//===----------------------------------------------------------------------===//
+
+// About local register variables:
+// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables
+//
+// Kernel ABI...
+// To my surprise I haven't found much information regarding it.
+// Kernel source and internet browsing shows that:
+// syscall number is passed in a7
+// (http://man7.org/linux/man-pages/man2/syscall.2.html) results are return in
+// a0 and a1 (http://man7.org/linux/man-pages/man2/syscall.2.html) arguments
+// are passed in: a0-a7 (see below)
+//
+// Regarding the arguments. The only "documentation" I could find is
+// this comment (!!!) by Bruce Hold on google forums (!!!):
+// https://groups.google.com/a/groups.riscv.org/forum/#!topic/sw-dev/exbrzM3GZDQ
+// Confirmed by inspecting glibc sources.
+// Great way to document things.
+#define SYSCALL(name) __NR_##name
+
+#define INTERNAL_SYSCALL_CLOBBERS "memory"
+
+static uptr __internal_syscall(u64 nr) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0");
+ __asm__ volatile("ecall\n\t"
+ : "=r"(a0)
+ : "r"(a7)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall0(n) (__internal_syscall)(n)
+
+static uptr __internal_syscall(u64 nr, u64 arg1) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall2(n, a1, a2) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ register u64 a2 asm("a2") = arg3;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall3(n, a1, a2, a3) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3,
+ u64 arg4) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ register u64 a2 asm("a2") = arg3;
+ register u64 a3 asm("a3") = arg4;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall4(n, a1, a2, a3, a4) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ register u64 a2 asm("a2") = arg3;
+ register u64 a3 asm("a3") = arg4;
+ register u64 a4 asm("a4") = arg5;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall5(n, a1, a2, a3, a4, a5) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5, long arg6) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ register u64 a2 asm("a2") = arg3;
+ register u64 a3 asm("a3") = arg4;
+ register u64 a4 asm("a4") = arg5;
+ register u64 a5 asm("a5") = arg6;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5), (long)(a6))
+
+static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4,
+ long arg5, long arg6, long arg7) {
+ register u64 a7 asm("a7") = nr;
+ register u64 a0 asm("a0") = arg1;
+ register u64 a1 asm("a1") = arg2;
+ register u64 a2 asm("a2") = arg3;
+ register u64 a3 asm("a3") = arg4;
+ register u64 a4 asm("a4") = arg5;
+ register u64 a5 asm("a5") = arg6;
+ register u64 a6 asm("a6") = arg7;
+ __asm__ volatile("ecall\n\t"
+ : "+r"(a0)
+ : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
+ "r"(a6)
+ : INTERNAL_SYSCALL_CLOBBERS);
+ return a0;
+}
+#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \
+ (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \
+ (u64)(a5), (long)(a6), (long)(a7))
+
+#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n
+#define __SYSCALL_NARGS(...) \
+ __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, )
+#define __SYSCALL_CONCAT_X(a, b) a##b
+#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b)
+#define __SYSCALL_DISP(b, ...) \
+ __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__)
+
+#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__)
+
+// Helper function used to avoid clobbering of errno.
+bool internal_iserror(uptr retval, int *rverrno) {
+ if (retval >= (uptr)-4095) {
+ if (rverrno)
+ *rverrno = -retval;
+ return true;
+ }
+ return false;
+}
// DO NOT EDIT! THIS FILE HAS BEEN GENERATED!
//
// Generated with: generate_netbsd_syscalls.awk
-// Generated date: 2019-12-24
-// Generated from: syscalls.master,v 1.296 2019/09/22 22:59:39 christos Exp
+// Generated date: 2020-09-10
+// Generated from: syscalls.master,v 1.306 2020/08/14 00:53:16 riastradh Exp
//
//===----------------------------------------------------------------------===//
POST_SYSCALL(dup2)(long long res, long long from_, long long to_) {
/* Nothing to do */
}
-/* syscall 91 has been skipped */
+PRE_SYSCALL(getrandom)(void *buf_, long long buflen_, long long flags_) {
+ /* TODO */
+}
+POST_SYSCALL(getrandom)
+(long long res, void *buf_, long long buflen_, long long flags_) {
+ /* TODO */
+}
PRE_SYSCALL(fcntl)(long long fd_, long long cmd_, void *arg_) {
/* Nothing to do */
}
POST_SYSCALL(compat_09_ouname)(long long res, void *name_) { /* TODO */ }
PRE_SYSCALL(sysarch)(long long op_, void *parms_) { /* TODO */ }
POST_SYSCALL(sysarch)(long long res, long long op_, void *parms_) { /* TODO */ }
-/* syscall 166 has been skipped */
-/* syscall 167 has been skipped */
-/* syscall 168 has been skipped */
+PRE_SYSCALL(__futex)
+(void *uaddr_, long long op_, long long val_, void *timeout_, void *uaddr2_,
+ long long val2_, long long val3_) {
+ /* TODO */
+}
+POST_SYSCALL(__futex)
+(long long res, void *uaddr_, long long op_, long long val_, void *timeout_,
+ void *uaddr2_, long long val2_, long long val3_) {
+ /* TODO */
+}
+PRE_SYSCALL(__futex_set_robust_list)(void *head_, long long len_) { /* TODO */ }
+POST_SYSCALL(__futex_set_robust_list)
+(long long res, void *head_, long long len_) {
+ /* TODO */
+}
+PRE_SYSCALL(__futex_get_robust_list)
+(long long lwpid_, void **headp_, void *lenp_) {
+ /* TODO */
+}
+POST_SYSCALL(__futex_get_robust_list)
+(long long res, long long lwpid_, void **headp_, void *lenp_) {
+ /* TODO */
+}
#if !defined(_LP64)
PRE_SYSCALL(compat_10_osemsys)
(long long which_, long long a2_, long long a3_, long long a4_, long long a5_) {
}
POST_SYSCALL(__fhstatvfs190)
(long long res, void *fhp_, long long fh_size_, void *buf_, long long flags_) {}
+PRE_SYSCALL(__acl_get_link)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_get_link)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_set_link)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_set_link)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_delete_link)(void *path_, long long type_) { /* TODO */ }
+POST_SYSCALL(__acl_delete_link)(long long res, void *path_, long long type_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_aclcheck_link)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_aclcheck_link)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_get_file)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_get_file)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_set_file)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_set_file)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_get_fd)(long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_get_fd)
+(long long res, long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_set_fd)(long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_set_fd)
+(long long res, long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_delete_file)(void *path_, long long type_) { /* TODO */ }
+POST_SYSCALL(__acl_delete_file)(long long res, void *path_, long long type_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_delete_fd)(long long filedes_, long long type_) { /* TODO */ }
+POST_SYSCALL(__acl_delete_fd)
+(long long res, long long filedes_, long long type_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_aclcheck_file)(void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_aclcheck_file)
+(long long res, void *path_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(__acl_aclcheck_fd)
+(long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+POST_SYSCALL(__acl_aclcheck_fd)
+(long long res, long long filedes_, long long type_, void *aclp_) {
+ /* TODO */
+}
+PRE_SYSCALL(lpathconf)(void *path_, long long name_) { /* TODO */ }
+POST_SYSCALL(lpathconf)(long long res, void *path_, long long name_) {
+ /* TODO */
+}
#undef SYS_MAXSYSARGS
} // extern "C"
internal__exit(common_flags()->exitcode);
}
-static CheckFailedCallbackType CheckFailedCallback;
-void SetCheckFailedCallback(CheckFailedCallbackType callback) {
- CheckFailedCallback = callback;
+static void (*CheckUnwindCallback)();
+void SetCheckUnwindCallback(void (*callback)()) {
+ CheckUnwindCallback = callback;
}
-const int kSecondsToSleepWhenRecursiveCheckFailed = 2;
-
void NORETURN CheckFailed(const char *file, int line, const char *cond,
u64 v1, u64 v2) {
- static atomic_uint32_t num_calls;
- if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) > 10) {
- SleepForSeconds(kSecondsToSleepWhenRecursiveCheckFailed);
+ u32 tid = GetTid();
+ Printf("%s: CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx) (tid=%u)\n",
+ SanitizerToolName, StripModuleName(file), line, cond, (uptr)v1,
+ (uptr)v2, tid);
+ static atomic_uint32_t first_tid;
+ u32 cmp = 0;
+ if (!atomic_compare_exchange_strong(&first_tid, &cmp, tid,
+ memory_order_relaxed)) {
+ if (cmp == tid) {
+ // Recursing into CheckFailed.
+ } else {
+ // Another thread fails already, let it print the stack and terminate.
+ SleepForSeconds(2);
+ }
Trap();
}
-
- if (CheckFailedCallback) {
- CheckFailedCallback(file, line, cond, v1, v2);
- }
- Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond,
- v1, v2);
+ if (CheckUnwindCallback)
+ CheckUnwindCallback();
Die();
}
unique_id = _unique_id;
detached = _detached;
// Parent tid makes no sense for the main thread.
- if (tid != 0)
+ if (tid != kMainTid)
parent_tid = _parent_tid;
OnCreated(arg);
}
// ThreadRegistry implementation.
-const u32 ThreadRegistry::kUnknownTid = ~0U;
+ThreadRegistry::ThreadRegistry(ThreadContextFactory factory)
+ : ThreadRegistry(factory, UINT32_MAX, UINT32_MAX, 0) {}
ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
u32 thread_quarantine_size, u32 max_reuse)
thread_quarantine_size_(thread_quarantine_size),
max_reuse_(max_reuse),
mtx_(),
- n_contexts_(0),
total_threads_(0),
alive_threads_(0),
max_alive_threads_(0),
running_threads_(0) {
- threads_ = (ThreadContextBase **)MmapOrDie(max_threads_ * sizeof(threads_[0]),
- "ThreadRegistry");
dead_threads_.clear();
invalid_threads_.clear();
}
void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
uptr *alive) {
BlockingMutexLock l(&mtx_);
- if (total) *total = n_contexts_;
+ if (total)
+ *total = threads_.size();
if (running) *running = running_threads_;
if (alive) *alive = alive_threads_;
}
u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
void *arg) {
BlockingMutexLock l(&mtx_);
- u32 tid = kUnknownTid;
+ u32 tid = kInvalidTid;
ThreadContextBase *tctx = QuarantinePop();
if (tctx) {
tid = tctx->tid;
- } else if (n_contexts_ < max_threads_) {
+ } else if (threads_.size() < max_threads_) {
// Allocate new thread context and tid.
- tid = n_contexts_++;
+ tid = threads_.size();
tctx = context_factory_(tid);
- threads_[tid] = tctx;
+ threads_.push_back(tctx);
} else {
#if !SANITIZER_GO
Report("%s: Thread limit (%u threads) exceeded. Dying.\n",
Die();
}
CHECK_NE(tctx, 0);
- CHECK_NE(tid, kUnknownTid);
+ CHECK_NE(tid, kInvalidTid);
CHECK_LT(tid, max_threads_);
CHECK_EQ(tctx->status, ThreadStatusInvalid);
alive_threads_++;
void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
void *arg) {
CheckLocked();
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx == 0)
continue;
u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
BlockingMutexLock l(&mtx_);
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && cb(tctx, arg))
return tctx->tid;
}
- return kUnknownTid;
+ return kInvalidTid;
}
ThreadContextBase *
ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
CheckLocked();
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && cb(tctx, arg))
return tctx;
void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
BlockingMutexLock l(&mtx_);
- for (u32 tid = 0; tid < n_contexts_; tid++) {
+ for (u32 tid = 0; tid < threads_.size(); tid++) {
ThreadContextBase *tctx = threads_[tid];
if (tctx != 0 && tctx->user_id == user_id &&
tctx->status != ThreadStatusInvalid) {
void ThreadRegistry::DetachThread(u32 tid, void *arg) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
do {
{
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
if (tctx->status == ThreadStatusInvalid) {
// really started. We just did CreateThread for a prospective new
// thread before trying to create it, and then failed to actually
// create it, and so never called StartThread.
-void ThreadRegistry::FinishThread(u32 tid) {
+ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
BlockingMutexLock l(&mtx_);
CHECK_GT(alive_threads_, 0);
alive_threads_--;
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
bool dead = tctx->detached;
+ ThreadStatus prev_status = tctx->status;
if (tctx->status == ThreadStatusRunning) {
CHECK_GT(running_threads_, 0);
running_threads_--;
QuarantinePush(tctx);
}
tctx->SetDestroyed();
+ return prev_status;
}
void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
void *arg) {
BlockingMutexLock l(&mtx_);
running_threads_++;
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_EQ(ThreadStatusCreated, tctx->status);
void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
BlockingMutexLock l(&mtx_);
- CHECK_LT(tid, n_contexts_);
ThreadContextBase *tctx = threads_[tid];
CHECK_NE(tctx, 0);
CHECK_NE(tctx->status, ThreadStatusInvalid);
class ThreadContextBase {
public:
explicit ThreadContextBase(u32 tid);
- ~ThreadContextBase(); // Should never be called.
-
const u32 tid; // Thread ID. Main thread should have tid = 0.
u64 unique_id; // Unique thread ID.
u32 reuse_count; // Number of times this tid was reused.
virtual void OnCreated(void *arg) {}
virtual void OnReset() {}
virtual void OnDetached(void *arg) {}
+
+ protected:
+ ~ThreadContextBase();
};
typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
-class ThreadRegistry {
+class MUTEX ThreadRegistry {
public:
- static const u32 kUnknownTid;
-
+ ThreadRegistry(ThreadContextFactory factory);
ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
- u32 thread_quarantine_size, u32 max_reuse = 0);
+ u32 thread_quarantine_size, u32 max_reuse);
void GetNumberOfThreads(uptr *total = nullptr, uptr *running = nullptr,
uptr *alive = nullptr);
uptr GetMaxAliveThreads();
- void Lock() { mtx_.Lock(); }
- void CheckLocked() { mtx_.CheckLocked(); }
- void Unlock() { mtx_.Unlock(); }
+ void Lock() ACQUIRE() { mtx_.Lock(); }
+ void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); }
+ void Unlock() RELEASE() { mtx_.Unlock(); }
// Should be guarded by ThreadRegistryLock.
ThreadContextBase *GetThreadLocked(u32 tid) {
- DCHECK_LT(tid, n_contexts_);
- return threads_[tid];
+ return threads_.empty() ? nullptr : threads_[tid];
}
u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
void RunCallbackForEachThreadLocked(ThreadCallback cb, void *arg);
typedef bool (*FindThreadCallback)(ThreadContextBase *tctx, void *arg);
- // Finds a thread using the provided callback. Returns kUnknownTid if no
+ // Finds a thread using the provided callback. Returns kInvalidTid if no
// thread is found.
u32 FindThread(FindThreadCallback cb, void *arg);
// Should be guarded by ThreadRegistryLock. Return 0 if no thread
void SetThreadNameByUserId(uptr user_id, const char *name);
void DetachThread(u32 tid, void *arg);
void JoinThread(u32 tid, void *arg);
- void FinishThread(u32 tid);
+ // Finishes thread and returns previous status.
+ ThreadStatus FinishThread(u32 tid);
void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
void SetThreadUserId(u32 tid, uptr user_id);
BlockingMutex mtx_;
- u32 n_contexts_; // Number of created thread contexts,
- // at most max_threads_.
u64 total_threads_; // Total number of created threads. May be greater than
// max_threads_ if contexts were reused.
uptr alive_threads_; // Created or running.
uptr max_alive_threads_;
uptr running_threads_;
- ThreadContextBase **threads_; // Array of thread contexts is leaked.
+ InternalMmapVector<ThreadContextBase *> threads_;
IntrusiveList<ThreadContextBase> dead_threads_;
IntrusiveList<ThreadContextBase> invalid_threads_;
--- /dev/null
+//===-- sanitizer_thread_safety.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// Wrappers around thread safety annotations.
+// https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_THREAD_SAFETY_H
+#define SANITIZER_THREAD_SAFETY_H
+
+#if defined(__clang__)
+# define THREAD_ANNOTATION(x) __attribute__((x))
+#else
+# define THREAD_ANNOTATION(x)
+#endif
+
+#define MUTEX THREAD_ANNOTATION(capability("mutex"))
+#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable)
+#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x))
+#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x))
+#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
+#define REQUIRES_SHARED(...) \
+ THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
+#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
+#define ACQUIRE_SHARED(...) \
+ THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
+#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
+#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__))
+#define RELEASE_SHARED(...) \
+ THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
+#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
+#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
+#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis)
+
+#endif
#include "sanitizer_tls_get_addr.h"
+#include "sanitizer_atomic.h"
#include "sanitizer_flags.h"
#include "sanitizer_platform_interceptors.h"
static const uptr kDestroyedThread = -1;
-static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
- if (!size) return;
- VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
- UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
+static void DTLS_Deallocate(DTLS::DTVBlock *block) {
+ VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+ UnmapOrDie(block, sizeof(DTLS::DTVBlock));
atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
}
-static inline void DTLS_Resize(uptr new_size) {
- if (dtls.dtv_size >= new_size) return;
- new_size = RoundUpToPowerOfTwo(new_size);
- new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV));
- DTLS::DTV *new_dtv =
- (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
+static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
+ uptr v = atomic_load(cur, memory_order_acquire);
+ if (v == kDestroyedThread)
+ return nullptr;
+ DTLS::DTVBlock *next = (DTLS::DTVBlock *)v;
+ if (next)
+ return next;
+ DTLS::DTVBlock *new_dtv =
+ (DTLS::DTVBlock *)MmapOrDie(sizeof(DTLS::DTVBlock), "DTLS_NextBlock");
+ uptr prev = 0;
+ if (!atomic_compare_exchange_strong(cur, &prev, (uptr)new_dtv,
+ memory_order_seq_cst)) {
+ UnmapOrDie(new_dtv, sizeof(DTLS::DTVBlock));
+ return (DTLS::DTVBlock *)prev;
+ }
uptr num_live_dtls =
atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
- VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
- CHECK_LT(num_live_dtls, 1 << 20);
- uptr old_dtv_size = dtls.dtv_size;
- DTLS::DTV *old_dtv = dtls.dtv;
- if (old_dtv_size)
- internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV));
- dtls.dtv = new_dtv;
- dtls.dtv_size = new_size;
- if (old_dtv_size)
- DTLS_Deallocate(old_dtv, old_dtv_size);
+ VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+ return new_dtv;
+}
+
+static DTLS::DTV *DTLS_Find(uptr id) {
+ VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+ static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
+ DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
+ if (!cur)
+ return nullptr;
+ for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(&cur->next);
+ return cur->dtvs + id;
}
void DTLS_Destroy() {
if (!common_flags()->intercept_tls_get_addr) return;
- VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
- uptr s = dtls.dtv_size;
- dtls.dtv_size = kDestroyedThread; // Do this before unmap for AS-safety.
- DTLS_Deallocate(dtls.dtv, s);
+ VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+ DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
+ &dtls.dtv_block, kDestroyedThread, memory_order_release);
+ while (block) {
+ DTLS::DTVBlock *next =
+ (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+ DTLS_Deallocate(block);
+ block = next;
+ }
}
#if defined(__powerpc64__) || defined(__mips__)
// This is glibc's TLS_DTV_OFFSET:
// "Dynamic thread vector pointers point 0x8000 past the start of each
-// TLS block."
+// TLS block." (sysdeps/<arch>/dl-tls.h)
static const uptr kDtvOffset = 0x8000;
+#elif defined(__riscv)
+// This is glibc's TLS_DTV_OFFSET:
+// "Dynamic thread vector pointers point 0x800 past the start of each
+// TLS block." (sysdeps/riscv/dl-tls.h)
+static const uptr kDtvOffset = 0x800;
#else
static const uptr kDtvOffset = 0;
#endif
if (!common_flags()->intercept_tls_get_addr) return 0;
TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
uptr dso_id = arg->dso_id;
- if (dtls.dtv_size == kDestroyedThread) return 0;
- DTLS_Resize(dso_id + 1);
- if (dtls.dtv[dso_id].beg) return 0;
+ DTLS::DTV *dtv = DTLS_Find(dso_id);
+ if (!dtv || dtv->beg)
+ return 0;
uptr tls_size = 0;
uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
// This may happen inside the DTOR of main thread, so just ignore it.
tls_size = 0;
}
- dtls.dtv[dso_id].beg = tls_beg;
- dtls.dtv[dso_id].size = tls_size;
- return dtls.dtv + dso_id;
+ dtv->beg = tls_beg;
+ dtv->size = tls_size;
+ return dtv;
}
void DTLS_on_libc_memalign(void *ptr, uptr size) {
DTLS *DTLS_Get() { return &dtls; }
bool DTLSInDestruction(DTLS *dtls) {
- return dtls->dtv_size == kDestroyedThread;
+ return atomic_load(&dtls->dtv_block, memory_order_relaxed) ==
+ kDestroyedThread;
}
#else
#ifndef SANITIZER_TLS_GET_ADDR_H
#define SANITIZER_TLS_GET_ADDR_H
+#include "sanitizer_atomic.h"
#include "sanitizer_common.h"
namespace __sanitizer {
struct DTV {
uptr beg, size;
};
+ struct DTVBlock {
+ atomic_uintptr_t next;
+ DTV dtvs[(4096UL - sizeof(next)) / sizeof(DTLS::DTV)];
+ };
+
+ static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
- uptr dtv_size;
- DTV *dtv; // dtv_size elements, allocated by MmapOrDie.
+ atomic_uintptr_t dtv_block;
// Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
uptr last_memalign_size;
uptr last_memalign_ptr;
};
+template <typename Fn>
+void ForEachDVT(DTLS *dtls, const Fn &fn) {
+ DTLS::DTVBlock *block =
+ (DTLS::DTVBlock *)atomic_load(&dtls->dtv_block, memory_order_acquire);
+ while (block) {
+ int id = 0;
+ for (auto &d : block->dtvs) fn(d, id++);
+ block = (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+ }
+}
+
// Returns pointer and size of a linker-allocated TLS block.
// Each block is returned exactly once.
DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res, uptr static_tls_begin,
// Skip the RTL frames by searching for the PC in the stacktrace.
uptr pc_location = LocatePcInTrace(pc);
PopStackFrames(pc_location);
+
+ // Replace the first frame with the PC because the frame in the
+ // stacktrace might be incorrect.
+ trace_buffer[0] = pc;
}
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wframe-larger-than="
+#endif
void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
CHECK(context);
CHECK_GE(max_depth, 2);
trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
}
}
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
#endif // #if !SANITIZER_GO
#endif // SANITIZER_WINDOWS
#define TraceLoggingUnregister(x)
#endif
+// For WaitOnAddress
+# pragma comment(lib, "synchronization.lib")
+
// A macro to tell the compiler that this part of the code cannot be reached,
// if the compiler supports this feature. Since we're using this in
// code that is called when terminating the process, the expansion of the
}
void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
- // This is almost useless on 32-bits.
- // FIXME: add madvise-analog when we move to 64-bits.
+ uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()),
+ end_aligned = RoundDownTo(end, GetPageSizeCached());
+ CHECK(beg < end); // make sure the region is sane
+ if (beg_aligned == end_aligned) // make sure we're freeing at least 1 page;
+ return;
+ UnmapOrDie((void *)beg, end_aligned - beg_aligned);
}
void SetShadowRegionHugePageMode(uptr addr, uptr size) {
return true;
}
+uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
+ uptr min_shadow_base_alignment,
+ UNUSED uptr &high_mem_end) {
+ const uptr granularity = GetMmapGranularity();
+ const uptr alignment =
+ Max<uptr>(granularity << shadow_scale, 1ULL << min_shadow_base_alignment);
+ const uptr left_padding =
+ Max<uptr>(granularity, 1ULL << min_shadow_base_alignment);
+ uptr space_size = shadow_size_bytes + left_padding;
+ uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
+ granularity, nullptr, nullptr);
+ CHECK_NE((uptr)0, shadow_start);
+ CHECK(IsAligned(shadow_start, alignment));
+ return shadow_start;
+}
+
uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
uptr *largest_gap_found,
uptr *max_occupied_addr) {
return 0;
}
+uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
+ uptr num_aliases, uptr ring_buffer_size) {
+ CHECK(false && "HWASan aliasing is unimplemented on Windows");
+ return 0;
+}
+
bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
MEMORY_BASIC_INFORMATION mbi;
CHECK(VirtualQuery((void *)range_start, &mbi, sizeof(mbi)));
}
#endif
-void PrintModuleMap() { }
-
void DisableCoreDumperIfNecessary() {
// Do nothing.
}
IsPathSeparator(path[2]);
}
-void SleepForSeconds(int seconds) {
- Sleep(seconds * 1000);
-}
-
-void SleepForMillis(int millis) {
- Sleep(millis);
-}
+void internal_usleep(u64 useconds) { Sleep(useconds / 1000); }
u64 NanoTime() {
static LARGE_INTEGER frequency = {};
// load the image at this address. Therefore, we call it the preferred base. Any
// addresses in the DWARF typically assume that the object has been loaded at
// this address.
-static uptr GetPreferredBase(const char *modname) {
+static uptr GetPreferredBase(const char *modname, char *buf, size_t buf_size) {
fd_t fd = OpenFile(modname, RdOnly, nullptr);
if (fd == kInvalidFd)
return 0;
// IMAGE_FILE_HEADER
// IMAGE_OPTIONAL_HEADER
// Seek to e_lfanew and read all that data.
- char buf[4 + sizeof(IMAGE_FILE_HEADER) + sizeof(IMAGE_OPTIONAL_HEADER)];
if (::SetFilePointer(fd, dos_header.e_lfanew, nullptr, FILE_BEGIN) ==
INVALID_SET_FILE_POINTER)
return 0;
- if (!ReadFromFile(fd, &buf[0], sizeof(buf), &bytes_read) ||
- bytes_read != sizeof(buf))
+ if (!ReadFromFile(fd, buf, buf_size, &bytes_read) || bytes_read != buf_size)
return 0;
// Check for "PE\0\0" before the PE header.
}
}
+ InternalMmapVector<char> buf(4 + sizeof(IMAGE_FILE_HEADER) +
+ sizeof(IMAGE_OPTIONAL_HEADER));
+ InternalMmapVector<wchar_t> modname_utf16(kMaxPathLength);
+ InternalMmapVector<char> module_name(kMaxPathLength);
// |num_modules| is the number of modules actually present,
size_t num_modules = bytes_required / sizeof(HMODULE);
for (size_t i = 0; i < num_modules; ++i) {
continue;
// Get the UTF-16 path and convert to UTF-8.
- wchar_t modname_utf16[kMaxPathLength];
int modname_utf16_len =
- GetModuleFileNameW(handle, modname_utf16, kMaxPathLength);
+ GetModuleFileNameW(handle, &modname_utf16[0], kMaxPathLength);
if (modname_utf16_len == 0)
modname_utf16[0] = '\0';
- char module_name[kMaxPathLength];
- int module_name_len =
- ::WideCharToMultiByte(CP_UTF8, 0, modname_utf16, modname_utf16_len + 1,
- &module_name[0], kMaxPathLength, NULL, NULL);
+ int module_name_len = ::WideCharToMultiByte(
+ CP_UTF8, 0, &modname_utf16[0], modname_utf16_len + 1, &module_name[0],
+ kMaxPathLength, NULL, NULL);
module_name[module_name_len] = '\0';
uptr base_address = (uptr)mi.lpBaseOfDll;
// RVA when computing the module offset. This helps llvm-symbolizer find the
// right DWARF CU. In the common case that the image is loaded at it's
// preferred address, we will now print normal virtual addresses.
- uptr preferred_base = GetPreferredBase(&module_name[0]);
+ uptr preferred_base =
+ GetPreferredBase(&module_name[0], &buf[0], buf.size());
uptr adjusted_base = base_address - preferred_base;
- LoadedModule cur_module;
- cur_module.set(module_name, adjusted_base);
+ modules_.push_back(LoadedModule());
+ LoadedModule &cur_module = modules_.back();
+ cur_module.set(&module_name[0], adjusted_base);
// We add the whole module as one single address range.
cur_module.addAddressRange(base_address, end_address, /*executable*/ true,
/*writable*/ true);
- modules_.push_back(cur_module);
}
UnmapOrDie(hmodules, modules_buffer_size);
}
void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
void internal_join_thread(void *th) { }
+void FutexWait(atomic_uint32_t *p, u32 cmp) {
+ WaitOnAddress(p, &cmp, sizeof(cmp), INFINITE);
+}
+
+void FutexWake(atomic_uint32_t *p, u32 count) {
+ if (count == 1)
+ WakeByAddressSingle(p);
+ else
+ WakeByAddressAll(p);
+}
+
// ---------------------- BlockingMutex ---------------- {{{1
BlockingMutex::BlockingMutex() {
ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
}
-void BlockingMutex::CheckLocked() {
- CHECK_EQ(owner_, GetThreadSelf());
-}
+void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); }
uptr GetTlsSize() {
return 0;
uptr SignalContext::GetAddress() const {
EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
- return exception_record->ExceptionInformation[1];
+ if (exception_record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION)
+ return exception_record->ExceptionInformation[1];
+ return (uptr)exception_record->ExceptionAddress;
}
bool SignalContext::IsMemoryAccess() const {
- return GetWriteFlag() != SignalContext::UNKNOWN;
+ return ((EXCEPTION_RECORD *)siginfo)->ExceptionCode ==
+ EXCEPTION_ACCESS_VIOLATION;
}
-bool SignalContext::IsTrueFaultingAddress() const {
- // FIXME: Provide real implementation for this. See Linux and Mac variants.
- return IsMemoryAccess();
-}
+bool SignalContext::IsTrueFaultingAddress() const { return true; }
SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+
+ // The write flag is only available for access violation exceptions.
+ if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
+ return SignalContext::UNKNOWN;
+
// The contents of this array are documented at
- // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx
+ // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
// The first element indicates read as 0, write as 1, or execute as 8. The
// second element is the faulting address.
switch (exception_record->ExceptionInformation[0]) {
}
uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) {
- // FIXME: Actually implement this function.
- CHECK_GT(buf_len, 0);
- buf[0] = 0;
- return 0;
+ if (buf_len == 0)
+ return 0;
+
+ // Get the UTF-16 path and convert to UTF-8.
+ InternalMmapVector<wchar_t> binname_utf16(kMaxPathLength);
+ int binname_utf16_len =
+ GetModuleFileNameW(NULL, &binname_utf16[0], kMaxPathLength);
+ if (binname_utf16_len == 0) {
+ buf[0] = '\0';
+ return 0;
+ }
+ int binary_name_len =
+ ::WideCharToMultiByte(CP_UTF8, 0, &binname_utf16[0], binname_utf16_len,
+ buf, buf_len, NULL, NULL);
+ if ((unsigned)binary_name_len == buf_len)
+ --binary_name_len;
+ buf[binary_name_len] = '\0';
+ return binary_name_len;
}
uptr ReadLongProcessName(/*out*/char *buf, uptr buf_len) {
}
#endif // SANITIZER_WIN_TRACE
+void InitializePlatformCommonFlags(CommonFlags *cf) {}
+
} // namespace __sanitizer
#endif // _WIN32
# Filters
# TODO: remove some of these filters
COMMON_LINT_FILTER=-build/include,-build/header_guard,-legal/copyright,-whitespace/comments,-readability/casting,\
--build/namespaces,-build/c++11,-runtime/int
+-build/namespaces,-build/c++11,-runtime/int,-runtime/references,-readability/todo,-whitespace/parens
COMMON_LIT_TEST_LINT_FILTER=-whitespace/indent,-whitespace/line_length,-runtime/arrays,-readability/braces
import optparse
import re
import sys
+from io import open
# Compile regex once for all files
runRegex = re.compile(r'(?<!-o)(?<!%run) %t\s')
The number of errors detected.
"""
errs = 0
- with open(p, 'r') as f:
+ with open(p, 'r', encoding='utf-8') as f:
for i, s in enumerate(f.readlines(), start=1):
msg, col = LintLine(s)
if msg != None:
-#!/usr/bin/python
+#!/usr/bin/env python
# Tests for litlint.py
#
# - __sanitizer_cov() or __sanitizer_cov_with_check(),
# - with call or callq,
# - directly or via PLT.
- cmd = "objdump -d %s | " \
- "grep '^\s\+[0-9a-f]\+:.*\scall\(q\|\)\s\+[0-9a-f]\+ <__sanitizer_cov\(_with_check\|\|_trace_pc_guard\)\(@plt\|\)>' | " \
- "grep '^\s\+[0-9a-f]\+' -o" % binary
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
- shell=True)
- proc.stdin.close()
+ cmd = r"objdump --no-show-raw-insn -d %s | " \
+ r"grep '^\s\+[0-9a-f]\+:\s\+call\(q\|\)\s\+\(0x\|\)[0-9a-f]\+ <__sanitizer_cov\(_with_check\|\|_trace_pc_guard\)\(@plt\|\)>' | " \
+ r"grep -o '^\s\+[0-9a-f]\+'" % binary
+ lines = subprocess.check_output(cmd, stdin=subprocess.PIPE, shell=True).splitlines()
# The PCs we get from objdump are off by 4 bytes, as they point to the
# beginning of the callq instruction. Empirically this is true on x86 and
# x86_64.
- return set(int(line.strip(), 16) + 4 for line in proc.stdout)
+ return set(int(line.strip(), 16) + 4 for line in lines)
def PrintMissing(binary):
if not os.path.isfile(binary):
return DefaultSymbolizer;
}
+static llvm::symbolize::PrinterConfig getDefaultPrinterConfig() {
+ llvm::symbolize::PrinterConfig Config;
+ Config.Pretty = false;
+ Config.Verbose = false;
+ Config.PrintFunctions = true;
+ Config.PrintAddress = false;
+ Config.SourceContextLines = 0;
+ return Config;
+}
+
namespace __sanitizer {
int internal_snprintf(char *buffer, unsigned long length, const char *format,
...);
std::string Result;
{
llvm::raw_string_ostream OS(Result);
- llvm::symbolize::DIPrinter Printer(OS);
+ llvm::symbolize::PrinterConfig Config = getDefaultPrinterConfig();
+ llvm::symbolize::Request Request{ModuleName, ModuleOffset};
+ auto Printer =
+ std::make_unique<llvm::symbolize::LLVMPrinter>(OS, OS, Config);
+
// TODO: it is neccessary to set proper SectionIndex here.
// object::SectionedAddress::UndefSection works for only absolute addresses.
if (SymbolizeInlineFrames) {
auto ResOrErr = getDefaultSymbolizer()->symbolizeInlinedCode(
ModuleName,
{ModuleOffset, llvm::object::SectionedAddress::UndefSection});
- Printer << (ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo());
+ Printer->print(Request,
+ ResOrErr ? ResOrErr.get() : llvm::DIInliningInfo());
} else {
auto ResOrErr = getDefaultSymbolizer()->symbolizeCode(
ModuleName,
{ModuleOffset, llvm::object::SectionedAddress::UndefSection});
- Printer << (ResOrErr ? ResOrErr.get() : llvm::DILineInfo());
+ Printer->print(Request, ResOrErr ? ResOrErr.get() : llvm::DILineInfo());
}
}
return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s",
char *Buffer, int MaxLength) {
std::string Result;
{
+ llvm::symbolize::PrinterConfig Config = getDefaultPrinterConfig();
llvm::raw_string_ostream OS(Result);
- llvm::symbolize::DIPrinter Printer(OS);
+ llvm::symbolize::Request Request{ModuleName, ModuleOffset};
+ auto Printer =
+ std::make_unique<llvm::symbolize::LLVMPrinter>(OS, OS, Config);
+
// TODO: it is neccessary to set proper SectionIndex here.
// object::SectionedAddress::UndefSection works for only absolute addresses.
auto ResOrErr = getDefaultSymbolizer()->symbolizeData(
ModuleName,
{ModuleOffset, llvm::object::SectionedAddress::UndefSection});
- Printer << (ResOrErr ? ResOrErr.get() : llvm::DIGlobal());
+ Printer->print(Request, ResOrErr ? ResOrErr.get() : llvm::DIGlobal());
}
return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s",
Result.c_str()) < MaxLength;
: 0;
}
+// Override __cxa_atexit and ignore callbacks.
+// This prevents crashes in a configuration when the symbolizer
+// is built into sanitizer runtime and consequently into the test process.
+// LLVM libraries have some global objects destroyed during exit,
+// so if the test process triggers any bugs after that, the symbolizer crashes.
+// An example stack trace of such crash:
+//
+// #1 __cxa_throw
+// #2 std::__u::__throw_system_error
+// #3 std::__u::recursive_mutex::lock
+// #4 __sanitizer_llvm::ManagedStaticBase::RegisterManagedStatic
+// #5 __sanitizer_llvm::errorToErrorCode
+// #6 __sanitizer_llvm::getFileAux
+// #7 __sanitizer_llvm::MemoryBuffer::getFileOrSTDIN
+// #10 __sanitizer_llvm::symbolize::LLVMSymbolizer::getOrCreateModuleInfo
+// #13 __sanitizer::Symbolizer::SymbolizeData
+// #14 __tsan::SymbolizeData
+// #16 __tsan::ReportRace
+// #18 __tsan_write4
+// #19 race() () at test/tsan/atexit4.cpp
+// #20 cxa_at_exit_wrapper
+// #21 __cxa_finalize
+// #22 __do_fini
+//
+// For the standalone llvm-symbolizer this does not hurt,
+// we just don't destroy few global objects on exit.
+int __cxa_atexit(void (*f)(void *a), void *arg, void *dso) { return 0; }
+
} // extern "C"
if [[ ! -d ${LIBCXX_BUILD} ]]; then
mkdir -p ${LIBCXX_BUILD}
cd ${LIBCXX_BUILD}
- LIBCXX_FLAGS="${FLAGS} -Wno-macro-redefined -I${LIBCXX_SRC}/include"
+ LIBCXX_FLAGS="${FLAGS} -Wno-macro-redefined"
PROJECTS=
if [[ ! -d $LLVM_SRC/projects/libcxxabi ]] ; then
PROJECTS="-DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi'"
ninja cxx cxxabi
FLAGS="${FLAGS} -fno-rtti -fno-exceptions"
-LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1"
+LLVM_FLAGS="${FLAGS} -nostdinc++ -I${ZLIB_BUILD} -I${LIBCXX_BUILD}/include/c++/v1 -Wno-error=global-constructors"
# Build LLVM.
if [[ ! -d ${LLVM_BUILD} ]]; then
__interceptor_read w
__interceptor_realpath w
__isinf U
+__isoc99_sscanf U
+__isoc99_vsscanf U
__moddi3 U
__sanitizer_symbolize_code T
__sanitizer_symbolize_data T
clang_compiler_add_cxx_check()
# FIXME: use SANITIZER_COMMON_SUPPORTED_ARCH here
-filter_available_targets(SANITIZER_UNITTEST_SUPPORTED_ARCH x86_64 i386 mips64 mips64el)
+filter_available_targets(SANITIZER_UNITTEST_SUPPORTED_ARCH x86_64 i386 mips64 mips64el riscv64)
if(APPLE)
darwin_filter_host_archs(SANITIZER_UNITTEST_SUPPORTED_ARCH SANITIZER_UNITTEST_SUPPORTED_ARCH)
endif()
sanitizer_atomic_test.cpp
sanitizer_bitvector_test.cpp
sanitizer_bvgraph_test.cpp
+ sanitizer_chained_origin_depot_test.cpp
sanitizer_common_test.cpp
sanitizer_deadlock_detector_test.cpp
sanitizer_flags_test.cpp
set(SANITIZER_TEST_CFLAGS_COMMON
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
+ ${COMPILER_RT_GMOCK_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/sanitizer_common
-fno-rtti
-O2
-Werror=sign-compare
- -Wno-non-virtual-dtor
-Wno-gnu-zero-variadic-macro-arguments
)
generate_compiler_rt_tests(SANITIZER_TEST_OBJECTS SanitizerUnitTests
"Sanitizer-${arch}-Test" ${arch}
RUNTIME "${SANITIZER_COMMON_LIB}"
- SOURCES ${SANITIZER_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE}
+ SOURCES ${SANITIZER_UNITTESTS} ${COMPILER_RT_GTEST_SOURCE} ${COMPILER_RT_GMOCK_SOURCE}
COMPILE_DEPS ${SANITIZER_TEST_HEADERS}
DEPS gtest
CFLAGS ${SANITIZER_TEST_CFLAGS_COMMON} ${extra_flags}
# Test that the libc-independent part of sanitizer_common is indeed
# independent of libc, by linking this binary without libc (here) and
# executing it (unit test in sanitizer_nolibc_test.cpp).
+ get_target_flags_for_arch(${arch} TARGET_FLAGS)
clang_compile(sanitizer_nolibc_test_main.${arch}.o
sanitizer_nolibc_test_main.cpp
CFLAGS ${SANITIZER_TEST_CFLAGS_COMMON} ${TARGET_FLAGS}
add_executable(SanitizerTest
${SANITIZER_UNITTESTS}
${COMPILER_RT_GTEST_SOURCE}
+ ${COMPILER_RT_GMOCK_SOURCE}
$<TARGET_OBJECTS:RTSanitizerCommon.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>
$<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>)
- set_target_compile_flags(SanitizerTest
- ${SANITIZER_COMMON_CFLAGS}
- ${SANITIZER_TEST_CFLAGS_COMMON})
+ set_target_compile_flags(SanitizerTest ${SANITIZER_TEST_CFLAGS_COMMON})
# Setup correct output directory and link flags.
set_target_properties(SanitizerTest PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
using namespace __sanitizer;
+#if SANITIZER_SOLARIS && defined(__sparcv9)
+// FIXME: These tests probably fail because Solaris/sparcv9 uses the full
+// 64-bit address space. Needs more investigation
+#define SKIP_ON_SOLARIS_SPARCV9(x) DISABLED_##x
+#else
+#define SKIP_ON_SOLARIS_SPARCV9(x) x
+#endif
+
+// On 64-bit systems with small virtual address spaces (e.g. 39-bit) we can't
+// use size class maps with a large number of classes, as that will make the
+// SizeClassAllocator64 region size too small (< 2^32).
+#if SANITIZER_ANDROID && defined(__aarch64__)
+#define ALLOCATOR64_SMALL_SIZE 1
+#elif SANITIZER_RISCV64
+#define ALLOCATOR64_SMALL_SIZE 1
+#else
+#define ALLOCATOR64_SMALL_SIZE 0
+#endif
+
// Too slow for debug build
#if !SANITIZER_DEBUG
static const uptr kAllocatorSize = 0x2000000000ULL;
static const u64 kAddressSpaceSize = 1ULL << 39;
typedef VeryCompactSizeClassMap SizeClassMap;
+#elif SANITIZER_RISCV64
+const uptr kAllocatorSpace = ~(uptr)0;
+const uptr kAllocatorSize = 0x2000000000ULL; // 128G.
+static const u64 kAddressSpaceSize = 1ULL << 38;
+typedef VeryDenseSizeClassMap SizeClassMap;
#else
static const uptr kAllocatorSpace = 0x700000000000ULL;
static const uptr kAllocatorSize = 0x010000000000ULL; // 1T.
}
template <class Allocator>
-void TestSizeClassAllocator() {
+void TestSizeClassAllocator(uptr premapped_heap = 0) {
Allocator *a = new Allocator;
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
typename Allocator::AllocatorCache cache;
memset(&cache, 0, sizeof(cache));
cache.Init(0);
}
#if SANITIZER_CAN_USE_ALLOCATOR64
+
+// Allocates kAllocatorSize aligned bytes on construction and frees it on
+// destruction.
+class ScopedPremappedHeap {
+ public:
+ ScopedPremappedHeap() {
+ BasePtr = MmapNoReserveOrDie(2 * kAllocatorSize, "preallocated heap");
+ AlignedAddr = RoundUpTo(reinterpret_cast<uptr>(BasePtr), kAllocatorSize);
+ }
+
+ ~ScopedPremappedHeap() { UnmapOrDie(BasePtr, kAllocatorSize); }
+
+ uptr Addr() { return AlignedAddr; }
+
+ private:
+ void *BasePtr;
+ uptr AlignedAddr;
+};
+
// These tests can fail on Windows if memory is somewhat full and lit happens
// to run them all at the same time. FIXME: Make them not flaky and reenable.
#if !SANITIZER_WINDOWS
TestSizeClassAllocator<Allocator64Dynamic>();
}
-#if !SANITIZER_ANDROID
-//FIXME(kostyak): find values so that those work on Android as well.
+#if !ALLOCATOR64_SMALL_SIZE
+// Android only has 39-bit address space, so mapping 2 * kAllocatorSize
+// sometimes fails.
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremapped) {
+ ScopedPremappedHeap h;
+ TestSizeClassAllocator<Allocator64Dynamic>(h.Addr());
+}
+
TEST(SanitizerCommon, SizeClassAllocator64Compact) {
TestSizeClassAllocator<Allocator64Compact>();
}
}
template <class Allocator>
-void SizeClassAllocatorMetadataStress() {
+void SizeClassAllocatorMetadataStress(uptr premapped_heap = 0) {
Allocator *a = new Allocator;
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
typename Allocator::AllocatorCache cache;
memset(&cache, 0, sizeof(cache));
cache.Init(0);
SizeClassAllocatorMetadataStress<Allocator64Dynamic>();
}
-#if !SANITIZER_ANDROID
+#if !ALLOCATOR64_SMALL_SIZE
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedMetadataStress) {
+ ScopedPremappedHeap h;
+ SizeClassAllocatorMetadataStress<Allocator64Dynamic>(h.Addr());
+}
+
TEST(SanitizerCommon, SizeClassAllocator64CompactMetadataStress) {
SizeClassAllocatorMetadataStress<Allocator64Compact>();
}
}
template <class Allocator>
-void SizeClassAllocatorGetBlockBeginStress(u64 TotalSize) {
+void SizeClassAllocatorGetBlockBeginStress(u64 TotalSize,
+ uptr premapped_heap = 0) {
Allocator *a = new Allocator;
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
typename Allocator::AllocatorCache cache;
memset(&cache, 0, sizeof(cache));
cache.Init(0);
SizeClassAllocatorGetBlockBeginStress<Allocator64Dynamic>(
1ULL << (SANITIZER_ANDROID ? 31 : 33));
}
-#if !SANITIZER_ANDROID
+#if !ALLOCATOR64_SMALL_SIZE
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedGetBlockBegin) {
+ ScopedPremappedHeap h;
+ SizeClassAllocatorGetBlockBeginStress<Allocator64Dynamic>(
+ 1ULL << (SANITIZER_ANDROID ? 31 : 33), h.Addr());
+}
TEST(SanitizerCommon, SizeClassAllocator64CompactGetBlockBegin) {
SizeClassAllocatorGetBlockBeginStress<Allocator64Compact>(1ULL << 33);
}
// Don't test OOM conditions on Win64 because it causes other tests on the same
// machine to OOM.
-#if SANITIZER_CAN_USE_ALLOCATOR64 && !SANITIZER_WINDOWS64 && !SANITIZER_ANDROID
+#if SANITIZER_CAN_USE_ALLOCATOR64 && !SANITIZER_WINDOWS64
TEST(SanitizerCommon, SizeClassAllocator64Overflow) {
Allocator64 a;
a.Init(kReleaseToOSIntervalNever);
uint32_t chunks[kNumChunks];
bool allocation_failed = false;
for (int i = 0; i < 1000000; i++) {
- if (!a.GetFromAllocator(&stats, 52, chunks, kNumChunks)) {
+ uptr class_id = a.kNumClasses - 1;
+ if (!a.GetFromAllocator(&stats, class_id, chunks, kNumChunks)) {
allocation_failed = true;
break;
}
}
template <class PrimaryAllocator>
-void TestCombinedAllocator() {
+void TestCombinedAllocator(uptr premapped_heap = 0) {
typedef CombinedAllocator<PrimaryAllocator> Allocator;
Allocator *a = new Allocator;
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
std::mt19937 r;
typename Allocator::AllocatorCache cache;
TestCombinedAllocator<Allocator64Dynamic>();
}
-#if !SANITIZER_ANDROID
+#if !ALLOCATOR64_SMALL_SIZE
+#if !SANITIZER_WINDOWS
+// Windows fails to map 1TB, so disable this test.
+TEST(SanitizerCommon, CombinedAllocator64DynamicPremapped) {
+ ScopedPremappedHeap h;
+ TestCombinedAllocator<Allocator64Dynamic>(h.Addr());
+}
+#endif
+
TEST(SanitizerCommon, CombinedAllocator64Compact) {
TestCombinedAllocator<Allocator64Compact>();
}
}
#endif
-TEST(SanitizerCommon, CombinedAllocator32Compact) {
+TEST(SanitizerCommon, SKIP_ON_SOLARIS_SPARCV9(CombinedAllocator32Compact)) {
TestCombinedAllocator<Allocator32Compact>();
}
template <class Allocator>
-void TestSizeClassAllocatorLocalCache() {
+void TestSizeClassAllocatorLocalCache(uptr premapped_heap = 0) {
using AllocatorCache = typename Allocator::AllocatorCache;
AllocatorCache cache;
Allocator *a = new Allocator();
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
memset(&cache, 0, sizeof(cache));
cache.Init(0);
TestSizeClassAllocatorLocalCache<Allocator64Dynamic>();
}
-#if !SANITIZER_ANDROID
+#if !ALLOCATOR64_SMALL_SIZE
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedLocalCache) {
+ ScopedPremappedHeap h;
+ TestSizeClassAllocatorLocalCache<Allocator64Dynamic>(h.Addr());
+}
+
TEST(SanitizerCommon, SizeClassAllocator64CompactLocalCache) {
TestSizeClassAllocatorLocalCache<Allocator64Compact>();
}
}
template <class Allocator>
-void TestSizeClassAllocatorIteration() {
+void TestSizeClassAllocatorIteration(uptr premapped_heap = 0) {
Allocator *a = new Allocator;
- a->Init(kReleaseToOSIntervalNever);
+ a->Init(kReleaseToOSIntervalNever, premapped_heap);
typename Allocator::AllocatorCache cache;
memset(&cache, 0, sizeof(cache));
cache.Init(0);
TEST(SanitizerCommon, SizeClassAllocator64DynamicIteration) {
TestSizeClassAllocatorIteration<Allocator64Dynamic>();
}
+#if !ALLOCATOR64_SMALL_SIZE
+TEST(SanitizerCommon, SizeClassAllocator64DynamicPremappedIteration) {
+ ScopedPremappedHeap h;
+ TestSizeClassAllocatorIteration<Allocator64Dynamic>(h.Addr());
+}
+#endif
#endif
#endif
-TEST(SanitizerCommon, SizeClassAllocator32Iteration) {
+TEST(SanitizerCommon, SKIP_ON_SOLARIS_SPARCV9(SizeClassAllocator32Iteration)) {
TestSizeClassAllocatorIteration<Allocator32Compact>();
}
// Don't test OOM conditions on Win64 because it causes other tests on the same
// machine to OOM.
-#if SANITIZER_CAN_USE_ALLOCATOR64 && !SANITIZER_WINDOWS64 && !SANITIZER_ANDROID
-typedef __sanitizer::SizeClassMap<3, 4, 8, 38, 128, 16> SpecialSizeClassMap;
+#if SANITIZER_CAN_USE_ALLOCATOR64 && !SANITIZER_WINDOWS64
+typedef __sanitizer::SizeClassMap<2, 22, 22, 34, 128, 16> SpecialSizeClassMap;
template <typename AddressSpaceViewTy = LocalAddressSpaceView>
struct AP64_SpecialSizeClassMap {
static const uptr kSpaceBeg = kAllocatorSpace;
// ...one man is on a mission to overflow a region with a series of
// successive allocations.
- const uptr kClassID = 107;
+ const uptr kClassID = ALLOCATOR64_SMALL_SIZE ? 18 : 24;
const uptr kAllocationSize = SpecialSizeClassMap::Size(kClassID);
ASSERT_LT(2 * kAllocationSize, kRegionSize);
ASSERT_GT(3 * kAllocationSize, kRegionSize);
EXPECT_NE(cache.Allocate(a, kClassID), nullptr);
EXPECT_EQ(cache.Allocate(a, kClassID), nullptr);
- const uptr Class2 = 100;
+ const uptr Class2 = ALLOCATOR64_SMALL_SIZE ? 15 : 21;
const uptr Size2 = SpecialSizeClassMap::Size(Class2);
ASSERT_EQ(Size2 * 8, kRegionSize);
char *p[7];
class NoMemoryMapper {
public:
- uptr last_request_buffer_size;
-
- NoMemoryMapper() : last_request_buffer_size(0) {}
+ uptr last_request_buffer_size = 0;
- uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
- last_request_buffer_size = buffer_size;
- return 0;
+ u64 *MapPackedCounterArrayBuffer(uptr buffer_size) {
+ last_request_buffer_size = buffer_size * sizeof(u64);
+ return nullptr;
}
- void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {}
};
class RedZoneMemoryMapper {
MprotectNoAccess(reinterpret_cast<uptr>(buffer), page_size);
MprotectNoAccess(reinterpret_cast<uptr>(buffer) + page_size * 2, page_size);
}
- ~RedZoneMemoryMapper() {
- UnmapOrDie(buffer, 3 * GetPageSize());
- }
+ ~RedZoneMemoryMapper() { UnmapOrDie(buffer, 3 * GetPageSize()); }
- uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
+ u64 *MapPackedCounterArrayBuffer(uptr buffer_size) {
+ buffer_size *= sizeof(u64);
const auto page_size = GetPageSize();
CHECK_EQ(buffer_size, page_size);
- memset(reinterpret_cast<void*>(reinterpret_cast<uptr>(buffer) + page_size),
- 0, page_size);
- return reinterpret_cast<uptr>(buffer) + page_size;
+ u64 *p =
+ reinterpret_cast<u64 *>(reinterpret_cast<uptr>(buffer) + page_size);
+ memset(p, 0, page_size);
+ return p;
}
- void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {}
private:
void *buffer;
TEST(SanitizerCommon, SizeClassAllocator64PackedCounterArray) {
NoMemoryMapper no_memory_mapper;
- typedef Allocator64::PackedCounterArray<NoMemoryMapper>
- NoMemoryPackedCounterArray;
-
for (int i = 0; i < 64; i++) {
// Various valid counter's max values packed into one word.
- NoMemoryPackedCounterArray counters_2n(1, 1ULL << i, &no_memory_mapper);
+ Allocator64::PackedCounterArray counters_2n(1, 1ULL << i,
+ &no_memory_mapper);
EXPECT_EQ(8ULL, no_memory_mapper.last_request_buffer_size);
// Check the "all bit set" values too.
- NoMemoryPackedCounterArray counters_2n1_1(1, ~0ULL >> i, &no_memory_mapper);
+ Allocator64::PackedCounterArray counters_2n1_1(1, ~0ULL >> i,
+ &no_memory_mapper);
EXPECT_EQ(8ULL, no_memory_mapper.last_request_buffer_size);
// Verify the packing ratio, the counter is expected to be packed into the
// closest power of 2 bits.
- NoMemoryPackedCounterArray counters(64, 1ULL << i, &no_memory_mapper);
+ Allocator64::PackedCounterArray counters(64, 1ULL << i, &no_memory_mapper);
EXPECT_EQ(8ULL * RoundUpToPowerOfTwo(i + 1),
no_memory_mapper.last_request_buffer_size);
}
RedZoneMemoryMapper memory_mapper;
- typedef Allocator64::PackedCounterArray<RedZoneMemoryMapper>
- RedZonePackedCounterArray;
// Go through 1, 2, 4, 8, .. 64 bits per counter.
for (int i = 0; i < 7; i++) {
// Make sure counters request one memory page for the buffer.
const u64 kNumCounters = (GetPageSize() / 8) * (64 >> i);
- RedZonePackedCounterArray counters(kNumCounters,
- 1ULL << ((1 << i) - 1),
- &memory_mapper);
+ Allocator64::PackedCounterArray counters(
+ kNumCounters, 1ULL << ((1 << i) - 1), &memory_mapper);
counters.Inc(0);
for (u64 c = 1; c < kNumCounters - 1; c++) {
ASSERT_EQ(0ULL, counters.Get(c));
Log2(GetPageSizeCached() >> Allocator64::kCompactPtrScale)),
last_page_reported(0) {}
- void ReleasePageRangeToOS(u32 from, u32 to) {
+ void ReleasePageRangeToOS(u32 class_id, u32 from, u32 to) {
from >>= page_size_scaled_log;
to >>= page_size_scaled_log;
ASSERT_LT(from, to);
reported_pages.append(to - from, 'x');
last_page_reported = to;
}
+
private:
const uptr page_size_scaled_log;
u32 last_page_reported;
for (auto test_case : test_cases) {
RangeRecorder range_recorder;
- RangeTracker tracker(&range_recorder);
+ RangeTracker tracker(&range_recorder, 1);
for (int i = 0; test_case[i] != 0; i++)
tracker.NextPage(test_case[i] == 'x');
tracker.Done();
class ReleasedPagesTrackingMemoryMapper {
public:
std::set<u32> reported_pages;
+ std::vector<u64> buffer;
- uptr MapPackedCounterArrayBuffer(uptr buffer_size) {
+ u64 *MapPackedCounterArrayBuffer(uptr buffer_size) {
reported_pages.clear();
- return reinterpret_cast<uptr>(calloc(1, buffer_size));
+ buffer.assign(buffer_size, 0);
+ return buffer.data();
}
- void UnmapPackedCounterArrayBuffer(uptr buffer, uptr buffer_size) {
- free(reinterpret_cast<void*>(buffer));
- }
-
- void ReleasePageRangeToOS(u32 from, u32 to) {
+ void ReleasePageRangeToOS(u32 class_id, u32 from, u32 to) {
uptr page_size_scaled =
GetPageSizeCached() >> Allocator64::kCompactPtrScale;
for (u32 i = from; i < to; i += page_size_scaled)
Allocator::ReleaseFreeMemoryToOS(&free_array[0], free_array.size(),
chunk_size, kAllocatedPagesCount,
- &memory_mapper);
+ &memory_mapper, class_id);
// Verify that there are no released pages touched by used chunks and all
// ranges of free chunks big enough to contain the entire memory pages had
TestReleaseFreeMemoryToOS<Allocator64>();
}
-#if !SANITIZER_ANDROID
+#if !ALLOCATOR64_SMALL_SIZE
TEST(SanitizerCommon, SizeClassAllocator64CompactReleaseFreeMemoryToOS) {
TestReleaseFreeMemoryToOS<Allocator64Compact>();
}
TEST(SanitizerCommon, SizeClassAllocator64VeryCompactReleaseFreeMemoryToOS) {
TestReleaseFreeMemoryToOS<Allocator64VeryCompact>();
}
-#endif // !SANITIZER_ANDROID
+#endif // !ALLOCATOR64_SMALL_SIZE
#endif // SANITIZER_CAN_USE_ALLOCATOR64
#include "sanitizer_common/sanitizer_atomic.h"
#include "gtest/gtest.h"
+#ifndef __has_extension
+#define __has_extension(x) 0
+#endif
+
+#ifndef ATOMIC_LLONG_LOCK_FREE
+# if __has_extension(c_atomic) || __has_extension(cxx_atomic)
+# define ATOMIC_LLONG_LOCK_FREE __CLANG_ATOMIC_LLONG_LOCK_FREE
+# elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7)
+# define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE
+# else
+# error Unsupported compiler.
+# endif
+#endif
+
namespace __sanitizer {
template<typename T>
CheckStoreLoad<atomic_uint32_t, memory_order_relaxed, memory_order_release>();
CheckStoreLoad<atomic_uint32_t, memory_order_seq_cst, memory_order_seq_cst>();
+ // Avoid fallbacking to software emulated compiler atomics, that are usually
+ // provided by libatomic, which is not always present.
+#if ATOMIC_LLONG_LOCK_FREE == 2
CheckStoreLoad<atomic_uint64_t, memory_order_relaxed, memory_order_relaxed>();
CheckStoreLoad<atomic_uint64_t, memory_order_consume, memory_order_relaxed>();
CheckStoreLoad<atomic_uint64_t, memory_order_acquire, memory_order_relaxed>();
CheckStoreLoad<atomic_uint64_t, memory_order_relaxed, memory_order_release>();
CheckStoreLoad<atomic_uint64_t, memory_order_seq_cst, memory_order_seq_cst>();
+#endif
CheckStoreLoad<atomic_uintptr_t, memory_order_relaxed, memory_order_relaxed>
();
CheckAtomicCompareExchange<atomic_uint8_t>();
CheckAtomicCompareExchange<atomic_uint16_t>();
CheckAtomicCompareExchange<atomic_uint32_t>();
+#if ATOMIC_LLONG_LOCK_FREE == 2
CheckAtomicCompareExchange<atomic_uint64_t>();
+#endif
CheckAtomicCompareExchange<atomic_uintptr_t>();
}
#endif //!SANITIZER_ANDROID
--- /dev/null
+//===-- sanitizer_chained_origin_depot_test.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer runtime.
+// Tests for sanitizer_chained_origin_depot.h.
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_chained_origin_depot.h"
+
+#include "gtest/gtest.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_libc.h"
+
+namespace __sanitizer {
+
+static ChainedOriginDepot chainedOriginDepot;
+
+TEST(SanitizerCommon, ChainedOriginDepotBasic) {
+ u32 new_id;
+ EXPECT_TRUE(chainedOriginDepot.Put(1, 2, &new_id));
+ u32 prev_id;
+ EXPECT_EQ(chainedOriginDepot.Get(new_id, &prev_id), 1U);
+ EXPECT_EQ(prev_id, 2U);
+}
+
+TEST(SanitizerCommon, ChainedOriginDepotAbsent) {
+ u32 prev_id;
+ EXPECT_EQ(0U, chainedOriginDepot.Get(99, &prev_id));
+ EXPECT_EQ(0U, prev_id);
+}
+
+TEST(SanitizerCommon, ChainedOriginDepotZeroId) {
+ u32 prev_id;
+ EXPECT_EQ(0U, chainedOriginDepot.Get(0, &prev_id));
+ EXPECT_EQ(0U, prev_id);
+}
+
+TEST(SanitizerCommon, ChainedOriginDepotSame) {
+ u32 new_id1;
+ EXPECT_TRUE(chainedOriginDepot.Put(11, 12, &new_id1));
+ u32 new_id2;
+ EXPECT_FALSE(chainedOriginDepot.Put(11, 12, &new_id2));
+ EXPECT_EQ(new_id1, new_id2);
+
+ u32 prev_id;
+ EXPECT_EQ(chainedOriginDepot.Get(new_id1, &prev_id), 11U);
+ EXPECT_EQ(prev_id, 12U);
+}
+
+TEST(SanitizerCommon, ChainedOriginDepotDifferent) {
+ u32 new_id1;
+ EXPECT_TRUE(chainedOriginDepot.Put(21, 22, &new_id1));
+ u32 new_id2;
+ EXPECT_TRUE(chainedOriginDepot.Put(21, 23, &new_id2));
+ EXPECT_NE(new_id1, new_id2);
+
+ u32 prev_id;
+ EXPECT_EQ(chainedOriginDepot.Get(new_id1, &prev_id), 21U);
+ EXPECT_EQ(prev_id, 22U);
+ EXPECT_EQ(chainedOriginDepot.Get(new_id2, &prev_id), 21U);
+ EXPECT_EQ(prev_id, 23U);
+}
+
+TEST(SanitizerCommon, ChainedOriginDepotStats) {
+ StackDepotStats stats0 = *chainedOriginDepot.GetStats();
+
+ u32 new_id;
+ EXPECT_TRUE(chainedOriginDepot.Put(33, 34, &new_id));
+ StackDepotStats stats1 = *chainedOriginDepot.GetStats();
+ EXPECT_EQ(stats1.n_uniq_ids, stats0.n_uniq_ids + 1);
+ EXPECT_GT(stats1.allocated, stats0.allocated);
+
+ EXPECT_FALSE(chainedOriginDepot.Put(33, 34, &new_id));
+ StackDepotStats stats2 = *chainedOriginDepot.GetStats();
+ EXPECT_EQ(stats2.n_uniq_ids, stats1.n_uniq_ids);
+ EXPECT_EQ(stats2.allocated, stats1.allocated);
+
+ EXPECT_TRUE(chainedOriginDepot.Put(35, 36, &new_id));
+ StackDepotStats stats3 = *chainedOriginDepot.GetStats();
+ EXPECT_EQ(stats3.n_uniq_ids, stats2.n_uniq_ids + 1);
+ EXPECT_GT(stats3.allocated, stats2.allocated);
+}
+
+} // namespace __sanitizer
}
TEST(SanitizerCommon, InternalLowerBound) {
- static const uptr kSize = 5;
- int arr[kSize];
- arr[0] = 1;
- arr[1] = 3;
- arr[2] = 5;
- arr[3] = 7;
- arr[4] = 11;
-
- EXPECT_EQ(0u, InternalLowerBound(arr, 0, kSize, 0, UptrLess));
- EXPECT_EQ(0u, InternalLowerBound(arr, 0, kSize, 1, UptrLess));
- EXPECT_EQ(1u, InternalLowerBound(arr, 0, kSize, 2, UptrLess));
- EXPECT_EQ(1u, InternalLowerBound(arr, 0, kSize, 3, UptrLess));
- EXPECT_EQ(2u, InternalLowerBound(arr, 0, kSize, 4, UptrLess));
- EXPECT_EQ(2u, InternalLowerBound(arr, 0, kSize, 5, UptrLess));
- EXPECT_EQ(3u, InternalLowerBound(arr, 0, kSize, 6, UptrLess));
- EXPECT_EQ(3u, InternalLowerBound(arr, 0, kSize, 7, UptrLess));
- EXPECT_EQ(4u, InternalLowerBound(arr, 0, kSize, 8, UptrLess));
- EXPECT_EQ(4u, InternalLowerBound(arr, 0, kSize, 9, UptrLess));
- EXPECT_EQ(4u, InternalLowerBound(arr, 0, kSize, 10, UptrLess));
- EXPECT_EQ(4u, InternalLowerBound(arr, 0, kSize, 11, UptrLess));
- EXPECT_EQ(5u, InternalLowerBound(arr, 0, kSize, 12, UptrLess));
+ std::vector<int> arr = {1, 3, 5, 7, 11};
+
+ EXPECT_EQ(0u, InternalLowerBound(arr, 0));
+ EXPECT_EQ(0u, InternalLowerBound(arr, 1));
+ EXPECT_EQ(1u, InternalLowerBound(arr, 2));
+ EXPECT_EQ(1u, InternalLowerBound(arr, 3));
+ EXPECT_EQ(2u, InternalLowerBound(arr, 4));
+ EXPECT_EQ(2u, InternalLowerBound(arr, 5));
+ EXPECT_EQ(3u, InternalLowerBound(arr, 6));
+ EXPECT_EQ(3u, InternalLowerBound(arr, 7));
+ EXPECT_EQ(4u, InternalLowerBound(arr, 8));
+ EXPECT_EQ(4u, InternalLowerBound(arr, 9));
+ EXPECT_EQ(4u, InternalLowerBound(arr, 10));
+ EXPECT_EQ(4u, InternalLowerBound(arr, 11));
+ EXPECT_EQ(5u, InternalLowerBound(arr, 12));
}
TEST(SanitizerCommon, InternalLowerBoundVsStdLowerBound) {
for (auto to_find : {val - 1, val, val + 1}) {
uptr expected =
std::lower_bound(data.begin(), data.end(), to_find) - data.begin();
- EXPECT_EQ(expected, InternalLowerBound(data.data(), 0, data.size(),
- to_find, std::less<int>()));
+ EXPECT_EQ(expected,
+ InternalLowerBound(data, to_find, std::less<int>()));
}
}
}
}
+class SortAndDedupTest : public ::testing::TestWithParam<std::vector<int>> {};
+
+TEST_P(SortAndDedupTest, SortAndDedup) {
+ std::vector<int> v_std = GetParam();
+ std::sort(v_std.begin(), v_std.end());
+ v_std.erase(std::unique(v_std.begin(), v_std.end()), v_std.end());
+
+ std::vector<int> v = GetParam();
+ SortAndDedup(v);
+
+ EXPECT_EQ(v_std, v);
+}
+
+const std::vector<int> kSortAndDedupTests[] = {
+ {},
+ {1},
+ {1, 1},
+ {1, 1, 1},
+ {1, 2, 3},
+ {3, 2, 1},
+ {1, 2, 2, 3},
+ {3, 3, 2, 1, 2},
+ {3, 3, 2, 1, 2},
+ {1, 2, 1, 1, 2, 1, 1, 1, 2, 2},
+ {1, 3, 3, 2, 3, 1, 3, 1, 4, 4, 2, 1, 4, 1, 1, 2, 2},
+};
+INSTANTIATE_TEST_SUITE_P(SortAndDedupTest, SortAndDedupTest,
+ ::testing::ValuesIn(kSortAndDedupTests));
+
#if SANITIZER_LINUX && !SANITIZER_ANDROID
TEST(SanitizerCommon, FindPathToBinary) {
char *true_path = FindPathToBinary("true");
}
TEST(SanitizerCommon, InternalScopedString) {
- InternalScopedString str(10);
+ InternalScopedString str;
EXPECT_EQ(0U, str.length());
EXPECT_STREQ("", str.data());
EXPECT_STREQ("foo1234", str.data());
str.append("%d", x);
- EXPECT_EQ(9U, str.length());
- EXPECT_STREQ("foo123412", str.data());
+ EXPECT_EQ(11U, str.length());
+ EXPECT_STREQ("foo12341234", str.data());
str.clear();
EXPECT_EQ(0U, str.length());
EXPECT_STREQ("", str.data());
+}
+
+TEST(SanitizerCommon, InternalScopedStringLarge) {
+ InternalScopedString str;
+ std::string expected;
+ for (int i = 0; i < 1000; ++i) {
+ std::string append(i, 'a' + i % 26);
+ expected += append;
+ str.append(append.c_str());
+ EXPECT_EQ(expected, str.data());
+ }
+}
- str.append("0123456789");
- EXPECT_EQ(9U, str.length());
- EXPECT_STREQ("012345678", str.data());
+TEST(SanitizerCommon, InternalScopedStringLargeFormat) {
+ InternalScopedString str;
+ std::string expected;
+ for (int i = 0; i < 1000; ++i) {
+ std::string append(i, 'a' + i % 26);
+ expected += append;
+ str.append("%s", append.c_str());
+ EXPECT_EQ(expected, str.data());
+ }
}
-#if SANITIZER_LINUX || SANITIZER_FREEBSD || \
- SANITIZER_OPENBSD || SANITIZER_MAC || SANITIZER_IOS
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_IOS
TEST(SanitizerCommon, GetRandom) {
u8 buffer_1[32], buffer_2[32];
for (bool blocking : { false, true }) {
EXPECT_DEATH(address_range.Unmap(base_addr + (PageSize * 2), PageSize), ".*");
}
-// Windows has no working ReadBinaryName.
-#if !SANITIZER_WINDOWS
TEST(SanitizerCommon, ReadBinaryNameCached) {
char buf[256];
EXPECT_NE((uptr)0, ReadBinaryNameCached(buf, sizeof(buf)));
}
-#endif
} // namespace __sanitizer
static void verifyFormatResults(const char *format, unsigned n,
const std::vector<unsigned> &computed_sizes,
- va_list expected_sizes) {
- // "+ 1" because of format string
+ const std::vector<unsigned> &expected_sizes) {
+ // "+ 1" because of the format string
ASSERT_EQ(n + 1,
computed_sizes.size()) << "Unexpected number of format arguments: '"
<< format << "'";
for (unsigned i = 0; i < n; ++i)
- EXPECT_EQ(va_arg(expected_sizes, unsigned), computed_sizes[i + 1])
+ EXPECT_EQ(expected_sizes[i], computed_sizes[i + 1])
<< "Unexpect write size for argument " << i << ", format string '"
<< format << "'";
}
static void testScanf2(const char *format, int scanf_result,
bool allowGnuMalloc, unsigned n,
- va_list expected_sizes) {
- std::vector<unsigned> scanf_sizes;
+ va_list expected_sizes_va) {
+ std::vector<unsigned> scanf_sizes, expected_sizes;
+ for (unsigned i = 0; i < n; ++i)
+ expected_sizes.push_back(va_arg(expected_sizes_va, unsigned));
+
// 16 args should be enough.
testScanf3((void *)&scanf_sizes, scanf_result, allowGnuMalloc, format,
test_buf, test_buf, test_buf, test_buf, test_buf, test_buf,
testScanf("%c%d", 2, C, I);
testScanf("%A%lf", 2, F, D);
- testScanf("%ms %Lf", 2, P, LD);
testScanf("s%Las", 1, LD);
testScanf("%ar", 1, F);
test_buf_size);
}
+TEST(SanitizerCommonInterceptors, ScanfAllocate) {
+ const char *buf = "123456";
+
+ // Can not use testScanf() because this case needs a valid pointer to a string
+ // in the scanf argument.
+ {
+ std::vector<unsigned> scanf_sizes;
+ testScanf3((void *)&scanf_sizes, 2, /*allowGnuMalloc=*/false, "%ms", &buf);
+ verifyFormatResults("%ms", 2, scanf_sizes,
+ {P, (unsigned)(strlen(buf) + 1)});
+ }
+
+ {
+ std::vector<unsigned> scanf_sizes;
+ testScanf3((void *)&scanf_sizes, 2, /*allowGnuMalloc=*/false, "%mc", &buf);
+ verifyFormatResults("%mc", 2, scanf_sizes,
+ {P, (unsigned)(strlen(buf) + 1)});
+ }
+}
+
static void testPrintf3(void *ctx, const char *format, ...) {
va_list ap;
va_start(ap, format);
}
static void testPrintf2(const char *format, unsigned n,
- va_list expected_sizes) {
- std::vector<unsigned> printf_sizes;
+ va_list expected_sizes_va) {
+ std::vector<unsigned> printf_sizes, expected_sizes;
+ for (unsigned i = 0; i < n; ++i)
+ expected_sizes.push_back(va_arg(expected_sizes_va, unsigned));
+
// 16 args should be enough.
testPrintf3((void *)&printf_sizes, format,
test_buf, test_buf, test_buf, test_buf, test_buf, test_buf,
EXPECT_EQ(data_, std::vector<char>(buff.begin(), buff.end()));
}
-INSTANTIATE_TEST_CASE_P(FileSizes, SanitizerCommonFileTest,
- ::testing::Values(0, 1, 7, 13, 32, 4096, 4097, 1048575,
- 1048576, 1048577));
+INSTANTIATE_TEST_SUITE_P(FileSizes, SanitizerCommonFileTest,
+ ::testing::Values(0, 1, 7, 13, 32, 4096, 4097, 1048575,
+ 1048576, 1048577));
static const size_t kStrlcpyBufSize = 8;
void test_internal_strlcpy(char *dbuf, const char *sbuf) {
}
#if (defined(__x86_64__) || defined(__i386__)) && !SANITIZER_ANDROID
-void *thread_self_offset_test_func(void *arg) {
- bool result =
- *(uptr *)((char *)ThreadSelf() + ThreadSelfOffset()) == ThreadSelf();
- return (void *)result;
-}
-
-TEST(SanitizerLinux, ThreadSelfOffset) {
- EXPECT_TRUE((bool)thread_self_offset_test_func(0));
- pthread_t tid;
- void *result;
- ASSERT_EQ(0, pthread_create(&tid, 0, thread_self_offset_test_func, 0));
- ASSERT_EQ(0, pthread_join(tid, &result));
- EXPECT_TRUE((bool)result);
-}
-
// libpthread puts the thread descriptor at the end of stack space.
void *thread_descriptor_size_test_func(void *arg) {
- uptr descr_addr = ThreadSelf();
+ uptr descr_addr = (uptr)pthread_self();
pthread_attr_t attr;
pthread_getattr_np(pthread_self(), &attr);
void *stackaddr;
namespace __sanitizer {
-TEST(SanitizerMac, GetMacosAlignedVersion) {
- MacosVersion vers = GetMacosAlignedVersion();
- u16 kernel_major = GetDarwinKernelVersion().major;
- bool macos_11 = (kernel_major >= 20);
- u16 expected_major = macos_11 ? (kernel_major - 9) : 10;
- u16 expected_minor = macos_11 ? 0 : (kernel_major - 4);
- EXPECT_EQ(vers.major, expected_major);
- EXPECT_EQ(vers.minor, expected_minor);
-}
-
void ParseVersion(const char *vers, u16 *major, u16 *minor);
TEST(SanitizerMac, ParseVersion) {
u16 major, minor;
+
ParseVersion("11.22.33", &major, &minor);
EXPECT_EQ(major, 11);
EXPECT_EQ(minor, 22);
+
+ ParseVersion("1.2", &major, &minor);
+ EXPECT_EQ(major, 1);
+ EXPECT_EQ(minor, 2);
+}
+
+// TODO(yln): Run sanitizer unit tests for the simulators (rdar://65680742)
+#if SANITIZER_IOSSIM
+TEST(SanitizerMac, GetMacosAlignedVersion) {
+ const char *vers_str;
+ if (SANITIZER_IOS || SANITIZER_TVOS) {
+ vers_str = "13.0";
+ } else if (SANITIZER_WATCHOS) {
+ vers_str = "6.5";
+ } else {
+ FAIL() << "unsupported simulator runtime";
+ }
+ setenv("SIMULATOR_RUNTIME_VERSION", vers_str, /*overwrite=*/1);
+
+ MacosVersion vers = GetMacosAlignedVersion();
+ EXPECT_EQ(vers.major, 10);
+ EXPECT_EQ(vers.minor, 15);
+}
+#else
+TEST(SanitizerMac, GetMacosAlignedVersion) {
+ MacosVersion vers = GetMacosAlignedVersion();
+ std::ostringstream oss;
+ oss << vers.major << '.' << vers.minor;
+ std::string actual = oss.str();
+
+ char buf[100];
+ size_t len = sizeof(buf);
+ int res = sysctlbyname("kern.osproductversion", buf, &len, nullptr, 0);
+ ASSERT_EQ(res, KERN_SUCCESS);
+ std::string expected(buf);
+
+ // Prefix match
+ ASSERT_EQ(expected.compare(0, actual.size(), actual), 0);
}
+#endif
TEST(SanitizerMac, GetDarwinKernelVersion) {
DarwinKernelVersion vers = GetDarwinKernelVersion();
Lock l(mtx_);
T v0 = data_[0];
for (int i = 0; i < kSize; i++) {
+ mtx_->CheckLocked();
CHECK_EQ(data_[i], v0);
data_[i]++;
}
return;
T v0 = data_[0];
for (int i = 0; i < kSize; i++) {
+ mtx_->CheckLocked();
CHECK_EQ(data_[i], v0);
data_[i]++;
}
mtx_->Unlock();
}
+ void Read() {
+ ReadLock l(mtx_);
+ T v0 = data_[0];
+ for (int i = 0; i < kSize; i++) {
+ mtx_->CheckReadLocked();
+ CHECK_EQ(data_[i], v0);
+ }
+ }
+
void Backoff() {
volatile T data[kSize] = {};
for (int i = 0; i < kSize; i++) {
private:
typedef GenericScopedLock<MutexType> Lock;
+ typedef GenericScopedReadLock<MutexType> ReadLock;
static const int kSize = 64;
typedef u64 T;
MutexType *mtx_;
return 0;
}
+template <typename MutexType>
+static void *read_write_thread(void *param) {
+ TestData<MutexType> *data = (TestData<MutexType> *)param;
+ for (int i = 0; i < kIters; i++) {
+ if ((i % 10) == 0)
+ data->Write();
+ else
+ data->Read();
+ data->Backoff();
+ }
+ return 0;
+}
+
template<typename MutexType>
static void check_locked(MutexType *mtx) {
GenericScopedLock<MutexType> l(mtx);
check_locked(mtx);
}
+TEST(SanitizerCommon, Mutex) {
+ Mutex mtx;
+ TestData<Mutex> data(&mtx);
+ pthread_t threads[kThreads];
+ for (int i = 0; i < kThreads; i++)
+ PTHREAD_CREATE(&threads[i], 0, read_write_thread<Mutex>, &data);
+ for (int i = 0; i < kThreads; i++) PTHREAD_JOIN(threads[i], 0);
+}
+
+struct SemaphoreData {
+ Semaphore *sem;
+ bool done;
+};
+
+void *SemaphoreThread(void *arg) {
+ auto data = static_cast<SemaphoreData *>(arg);
+ data->sem->Wait();
+ data->done = true;
+ return nullptr;
+}
+
+TEST(SanitizerCommon, Semaphore) {
+ Semaphore sem;
+ sem.Post(1);
+ sem.Wait();
+ sem.Post(3);
+ sem.Wait();
+ sem.Wait();
+ sem.Wait();
+
+ SemaphoreData data = {&sem, false};
+ pthread_t thread;
+ PTHREAD_CREATE(&thread, nullptr, SemaphoreThread, &data);
+ internal_sleep(1);
+ CHECK(!data.done);
+ sem.Post(1);
+ PTHREAD_JOIN(thread, nullptr);
+}
+
} // namespace __sanitizer
#include <string.h>
#include <limits.h>
+#ifdef __x86_64__
+# include <emmintrin.h>
+#endif
+
namespace __sanitizer {
TEST(Printf, Basic) {
EXPECT_STREQ("12345 ", buf);
}
+#ifdef __x86_64__
+TEST(Printf, M128) {
+ __m128i v = _mm_set_epi32(0x12345678, 0x0a0a0a0a, 0xb0b0b0b0, 0xaabbccdd);
+ char buf[128];
+ internal_snprintf(buf, sizeof(buf), "%V", PRINTF_128(v));
+ EXPECT_STREQ("ddccbbaab0b0b0b00a0a0a0a78563412", buf);
+ v = _mm_cvtsi32_si128(0x12345678);
+ internal_snprintf(buf, sizeof(buf), "%V", PRINTF_128(v));
+ EXPECT_STREQ("78563412000000000000000000000000", buf);
+ internal_snprintf(buf, sizeof(buf), "%d %V", 0, PRINTF_128(v));
+ EXPECT_STREQ("0 78563412000000000000000000000000", buf);
+}
+#endif
+
} // namespace __sanitizer
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_stackdepot.h"
+
+#include "gtest/gtest.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include "gtest/gtest.h"
namespace __sanitizer {
EXPECT_NE(i1, i2);
}
+#if SANITIZER_WINDOWS
+// CaptureStderr does not work on Windows.
+#define Maybe_StackDepotPrint DISABLED_StackDepotPrint
+#else
+#define Maybe_StackDepotPrint StackDepotPrint
+#endif
+TEST(SanitizerCommon, Maybe_StackDepotPrint) {
+ uptr array1[] = {0x111, 0x222, 0x333, 0x444, 0x777};
+ StackTrace s1(array1, ARRAY_SIZE(array1));
+ u32 i1 = StackDepotPut(s1);
+ uptr array2[] = {0x1111, 0x2222, 0x3333, 0x4444, 0x8888, 0x9999};
+ StackTrace s2(array2, ARRAY_SIZE(array2));
+ u32 i2 = StackDepotPut(s2);
+ EXPECT_NE(i1, i2);
+ EXPECT_EXIT((StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
+ "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x7.*");
+ EXPECT_EXIT(
+ (StackDepotPrintAll(), exit(0)), ::testing::ExitedWithCode(0),
+ "Stack for id .*#0 0x1.*#1 0x2.*#2 0x3.*#3 0x4.*#4 0x8.*#5 0x9.*");
+}
+
TEST(SanitizerCommon, StackDepotReverseMap) {
uptr array1[] = {1, 2, 3, 4, 5};
uptr array2[] = {7, 1, 3, 0};
namespace __sanitizer {
TEST(SanitizerStacktracePrinter, RenderSourceLocation) {
- InternalScopedString str(128);
+ InternalScopedString str;
RenderSourceLocation(&str, "/dir/file.cc", 10, 5, false, "");
EXPECT_STREQ("/dir/file.cc:10:5", str.data());
}
TEST(SanitizerStacktracePrinter, RenderModuleLocation) {
- InternalScopedString str(128);
+ InternalScopedString str;
RenderModuleLocation(&str, "/dir/exe", 0x123, kModuleArchUnknown, "");
EXPECT_STREQ("(/dir/exe+0x123)", str.data());
info.file = internal_strdup("/path/to/my/source");
info.line = 10;
info.column = 5;
- InternalScopedString str(256);
+ InternalScopedString str;
// Dump all the AddressInfo fields.
- RenderFrame(&str, "%% Frame:%n PC:%p Module:%m ModuleOffset:%o "
- "Function:%f FunctionOffset:%q Source:%s Line:%l "
- "Column:%c",
- frame_no, info, false, "/path/to/", "function_");
+ RenderFrame(&str,
+ "%% Frame:%n PC:%p Module:%m ModuleOffset:%o "
+ "Function:%f FunctionOffset:%q Source:%s Line:%l "
+ "Column:%c",
+ frame_no, info.address, &info, false, "/path/to/", "function_");
EXPECT_STREQ("% Frame:42 PC:0x400000 Module:my/module ModuleOffset:0x200 "
"Function:foo FunctionOffset:0x100 Source:my/source Line:10 "
"Column:5",
// Test special format specifiers.
info.address = 0x400000;
- RenderFrame(&str, "%M", frame_no, info, false);
+ RenderFrame(&str, "%M", frame_no, info.address, &info, false);
EXPECT_NE(nullptr, internal_strstr(str.data(), "400000"));
str.clear();
- RenderFrame(&str, "%L", frame_no, info, false);
+ RenderFrame(&str, "%L", frame_no, info.address, &info, false);
EXPECT_STREQ("(<unknown module>)", str.data());
str.clear();
info.module = internal_strdup("/path/to/module");
info.module_offset = 0x200;
- RenderFrame(&str, "%M", frame_no, info, false);
+ RenderFrame(&str, "%M", frame_no, info.address, &info, false);
EXPECT_NE(nullptr, internal_strstr(str.data(), "(module+0x"));
EXPECT_NE(nullptr, internal_strstr(str.data(), "200"));
str.clear();
- RenderFrame(&str, "%L", frame_no, info, false);
+ RenderFrame(&str, "%L", frame_no, info.address, &info, false);
EXPECT_STREQ("(/path/to/module+0x200)", str.data());
str.clear();
info.function = internal_strdup("my_function");
- RenderFrame(&str, "%F", frame_no, info, false);
+ RenderFrame(&str, "%F", frame_no, info.address, &info, false);
EXPECT_STREQ("in my_function", str.data());
str.clear();
info.function_offset = 0x100;
- RenderFrame(&str, "%F %S", frame_no, info, false);
+ RenderFrame(&str, "%F %S", frame_no, info.address, &info, false);
EXPECT_STREQ("in my_function+0x100 <null>", str.data());
str.clear();
info.file = internal_strdup("my_file");
- RenderFrame(&str, "%F %S", frame_no, info, false);
+ RenderFrame(&str, "%F %S", frame_no, info.address, &info, false);
EXPECT_STREQ("in my_function my_file", str.data());
str.clear();
info.line = 10;
- RenderFrame(&str, "%F %S", frame_no, info, false);
+ RenderFrame(&str, "%F %S", frame_no, info.address, &info, false);
EXPECT_STREQ("in my_function my_file:10", str.data());
str.clear();
info.column = 5;
- RenderFrame(&str, "%S %L", frame_no, info, false);
+ RenderFrame(&str, "%S %L", frame_no, info.address, &info, false);
EXPECT_STREQ("my_file:10:5 my_file:10:5", str.data());
str.clear();
- RenderFrame(&str, "%S %L", frame_no, info, true);
+ RenderFrame(&str, "%S %L", frame_no, info.address, &info, true);
EXPECT_STREQ("my_file(10,5) my_file(10,5)", str.data());
str.clear();
info.column = 0;
- RenderFrame(&str, "%F %S", frame_no, info, true);
+ RenderFrame(&str, "%F %S", frame_no, info.address, &info, true);
EXPECT_STREQ("in my_function my_file(10)", str.data());
str.clear();
info.line = 0;
- RenderFrame(&str, "%F %S", frame_no, info, true);
+ RenderFrame(&str, "%F %S", frame_no, info.address, &info, true);
EXPECT_STREQ("in my_function my_file", str.data());
str.clear();
//
//===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <string>
+
+#include "gmock/gmock.h"
#include "gtest/gtest.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+
+using testing::ContainsRegex;
+using testing::MatchesRegex;
namespace __sanitizer {
uhwptr fake_top;
uhwptr fake_bottom;
BufferedStackTrace trace;
+
+#if defined(__riscv)
+ const uptr kFpOffset = 4;
+ const uptr kBpOffset = 2;
+#else
+ const uptr kFpOffset = 2;
+ const uptr kBpOffset = 0;
+#endif
+
+ private:
+ CommonFlags tmp_flags_;
};
static uptr PC(uptr idx) {
// Fill an array of pointers with fake fp+retaddr pairs. Frame pointers have
// even indices.
for (uptr i = 0; i + 1 < fake_stack_size; i += 2) {
- fake_stack[i] = (uptr)&fake_stack[i+2]; // fp
+ fake_stack[i] = (uptr)&fake_stack[i + kFpOffset]; // fp
fake_stack[i+1] = PC(i + 1); // retaddr
}
// Mark the last fp point back up to terminate the stack trace.
fake_stack[RoundDownTo(fake_stack_size - 1, 2)] = (uhwptr)&fake_stack[0];
// Top is two slots past the end because UnwindFast subtracts two.
- fake_top = (uhwptr)&fake_stack[fake_stack_size + 2];
+ fake_top = (uhwptr)&fake_stack[fake_stack_size + kFpOffset];
// Bottom is one slot before the start because UnwindFast uses >.
fake_bottom = (uhwptr)mapping;
- fake_bp = (uptr)&fake_stack[0];
+ fake_bp = (uptr)&fake_stack[kBpOffset];
start_pc = PC(0);
+
+ tmp_flags_.CopyFrom(*common_flags());
}
void FastUnwindTest::TearDown() {
size_t ps = GetPageSize();
UnmapOrDie(mapping, 2 * ps);
+
+ // Restore default flags.
+ OverrideCommonFlags(tmp_flags_);
}
#if SANITIZER_CAN_FAST_UNWIND
+#ifdef __sparc__
+// Fake stacks don't meet SPARC UnwindFast requirements.
+#define SKIP_ON_SPARC(x) DISABLED_##x
+#else
+#define SKIP_ON_SPARC(x) x
+#endif
+
void FastUnwindTest::UnwindFast() {
trace.UnwindFast(start_pc, fake_bp, fake_top, fake_bottom, kStackTraceMax);
}
-TEST_F(FastUnwindTest, Basic) {
+TEST_F(FastUnwindTest, SKIP_ON_SPARC(Basic)) {
UnwindFast();
// Should get all on-stack retaddrs and start_pc.
EXPECT_EQ(6U, trace.size);
}
// From: https://github.com/google/sanitizers/issues/162
-TEST_F(FastUnwindTest, FramePointerLoop) {
+TEST_F(FastUnwindTest, SKIP_ON_SPARC(FramePointerLoop)) {
// Make one fp point to itself.
fake_stack[4] = (uhwptr)&fake_stack[4];
UnwindFast();
}
}
-TEST_F(FastUnwindTest, MisalignedFramePointer) {
+TEST_F(FastUnwindTest, SKIP_ON_SPARC(MisalignedFramePointer)) {
// Make one fp misaligned.
fake_stack[4] += 3;
UnwindFast();
trace.Unwind(start_pc, fake_bp, nullptr, true, 1);
EXPECT_EQ(1U, trace.size);
EXPECT_EQ(start_pc, trace.trace[0]);
- EXPECT_EQ((uhwptr)&fake_stack[0], trace.top_frame_bp);
+ EXPECT_EQ((uhwptr)&fake_stack[kBpOffset], trace.top_frame_bp);
}
TEST_F(FastUnwindTest, ZeroFramesStackTrace) {
EXPECT_EQ(0U, trace.top_frame_bp);
}
-TEST_F(FastUnwindTest, FPBelowPrevFP) {
+TEST_F(FastUnwindTest, SKIP_ON_SPARC(FPBelowPrevFP)) {
// The next FP points to unreadable memory inside the stack limits, but below
// current FP.
fake_stack[0] = (uhwptr)&fake_stack[-50];
EXPECT_EQ(PC(1), trace.trace[1]);
}
-TEST_F(FastUnwindTest, CloseToZeroFrame) {
+TEST_F(FastUnwindTest, SKIP_ON_SPARC(CloseToZeroFrame)) {
// Make one pc a NULL pointer.
fake_stack[5] = 0x0;
UnwindFast();
}
}
+using StackPrintTest = FastUnwindTest;
+
+TEST_F(StackPrintTest, SKIP_ON_SPARC(ContainsFullTrace)) {
+ // Override stack trace format to make testing code independent of default
+ // flag values.
+ CommonFlags flags;
+ flags.CopyFrom(*common_flags());
+ flags.stack_trace_format = "#%n %p";
+ OverrideCommonFlags(flags);
+
+ UnwindFast();
+
+ char buf[3000];
+ trace.PrintTo(buf, sizeof(buf));
+ EXPECT_THAT(std::string(buf),
+ MatchesRegex("(#[0-9]+ 0x[0-9a-f]+\n){" +
+ std::to_string(trace.size) + "}\n"));
+}
+
+TEST_F(StackPrintTest, SKIP_ON_SPARC(TruncatesContents)) {
+ UnwindFast();
+
+ char buf[3000];
+ uptr actual_len = trace.PrintTo(buf, sizeof(buf));
+ ASSERT_LT(actual_len, sizeof(buf));
+
+ char tinybuf[10];
+ trace.PrintTo(tinybuf, sizeof(tinybuf));
+
+ // This the the truncation case.
+ ASSERT_GT(actual_len, sizeof(tinybuf));
+
+ // The truncated contents should be a prefix of the full contents.
+ size_t lastpos = sizeof(tinybuf) - 1;
+ EXPECT_EQ(strncmp(buf, tinybuf, lastpos), 0);
+ EXPECT_EQ(tinybuf[lastpos], '\0');
+
+ // Full bufffer has more contents...
+ EXPECT_NE(buf[lastpos], '\0');
+}
+
+TEST_F(StackPrintTest, SKIP_ON_SPARC(WorksWithEmptyStack)) {
+ char buf[3000];
+ trace.PrintTo(buf, sizeof(buf));
+ EXPECT_NE(strstr(buf, "<empty stack>"), nullptr);
+}
+
+TEST_F(StackPrintTest, SKIP_ON_SPARC(ReturnsCorrectLength)) {
+ UnwindFast();
+
+ char buf[3000];
+ uptr len = trace.PrintTo(buf, sizeof(buf));
+ size_t actual_len = strlen(buf);
+ ASSERT_LT(len, sizeof(buf));
+ EXPECT_EQ(len, actual_len);
+
+ char tinybuf[5];
+ len = trace.PrintTo(tinybuf, sizeof(tinybuf));
+ size_t truncated_len = strlen(tinybuf);
+ ASSERT_GE(len, sizeof(tinybuf));
+ EXPECT_EQ(len, actual_len);
+ EXPECT_EQ(truncated_len, sizeof(tinybuf) - 1);
+}
+
+TEST_F(StackPrintTest, SKIP_ON_SPARC(AcceptsZeroSize)) {
+ UnwindFast();
+ char buf[1];
+ EXPECT_GT(trace.PrintTo(buf, 0), 0u);
+}
+
+using StackPrintDeathTest = StackPrintTest;
+
+TEST_F(StackPrintDeathTest, SKIP_ON_SPARC(RequiresNonNullBuffer)) {
+ UnwindFast();
+ EXPECT_DEATH(trace.PrintTo(NULL, 100), "");
+}
+
#endif // SANITIZER_CAN_FAST_UNWIND
TEST(SlowUnwindTest, ShortStackTrace) {
EXPECT_EQ(bp, stack.top_frame_bp);
}
+TEST(GetCurrentPc, Basic) {
+ // Test that PCs obtained via GET_CURRENT_PC()
+ // and StackTrace::GetCurrentPc() are all different
+ // and are close to the function start.
+ struct Local {
+ static NOINLINE void Test() {
+ const uptr pcs[] = {
+ (uptr)&Local::Test,
+ GET_CURRENT_PC(),
+ StackTrace::GetCurrentPc(),
+ StackTrace::GetCurrentPc(),
+ };
+ for (uptr i = 0; i < ARRAY_SIZE(pcs); i++)
+ Printf("pc%zu: %p\n", i, pcs[i]);
+ for (uptr i = 1; i < ARRAY_SIZE(pcs); i++) {
+ EXPECT_GT(pcs[i], pcs[0]);
+ EXPECT_LT(pcs[i], pcs[0] + 1000);
+ for (uptr j = 0; j < i; j++) EXPECT_NE(pcs[i], pcs[j]);
+ }
+ }
+ };
+ Local::Test();
+}
+
+// Dummy implementation. This should never be called, but is required to link
+// non-optimized builds of this test.
+void BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, void *context,
+ bool request_fast, u32 max_depth) {
+ UNIMPLEMENTED();
+}
+
} // namespace __sanitizer
# define SANITIZER_TEST_HAS_MEMALIGN 0
#endif
-#if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__ANDROID__) && \
- !defined(__NetBSD__) && !defined(_WIN32) && \
- !(defined(__sun__) && defined(__svr4__))
+#if defined(__GLIBC__)
# define SANITIZER_TEST_HAS_PVALLOC 1
# define SANITIZER_TEST_HAS_MALLOC_USABLE_SIZE 1
#else
registry->SetThreadName(6, "six");
registry->SetThreadName(7, "seven");
EXPECT_EQ(7U, registry->FindThread(HasName, (void*)"seven"));
- EXPECT_EQ(ThreadRegistry::kUnknownTid,
- registry->FindThread(HasName, (void*)"none"));
+ EXPECT_EQ(kInvalidTid, registry->FindThread(HasName, (void *)"none"));
EXPECT_EQ(0U, registry->FindThread(HasUid, (void*)get_uid(0)));
EXPECT_EQ(10U, registry->FindThread(HasUid, (void*)get_uid(10)));
- EXPECT_EQ(ThreadRegistry::kUnknownTid,
- registry->FindThread(HasUid, (void*)0x1234));
+ EXPECT_EQ(kInvalidTid, registry->FindThread(HasUid, (void *)0x1234));
// Detach and finish and join remaining threads.
for (u32 i = 6; i <= 10; i++) {
registry->DetachThread(i, 0);
TEST(SanitizerCommon, ThreadRegistryTest) {
ThreadRegistry quarantine_registry(GetThreadContext<ThreadContextBase>,
- kMaxRegistryThreads,
- kRegistryQuarantine);
+ kMaxRegistryThreads, kRegistryQuarantine,
+ 0);
TestRegistry(&quarantine_registry, true);
ThreadRegistry no_quarantine_registry(GetThreadContext<ThreadContextBase>,
kMaxRegistryThreads,
- kMaxRegistryThreads);
+ kMaxRegistryThreads, 0);
TestRegistry(&no_quarantine_registry, false);
}
uptr shard; // started from 1.
};
-class TestThreadContext : public ThreadContextBase {
+class TestThreadContext final : public ThreadContextBase {
public:
explicit TestThreadContext(int tid) : ThreadContextBase(tid) {}
void OnJoined(void *arg) {
memset(&num_joined, 0, sizeof(num_created));
ThreadRegistry registry(GetThreadContext<TestThreadContext>,
- kThreadsPerShard * kNumShards + 1, 10);
+ kThreadsPerShard * kNumShards + 1, 10, 0);
ThreadedTestRegistry(®istry);
}
RTInterception)
if (COMPILER_RT_HAS_GWP_ASAN)
- # Currently, Scudo uses the GwpAsan flag parser. This backs onto the flag
- # parsing mechanism of sanitizer_common. Once Scudo has its own flag parsing,
- # and parses GwpAsan options, you can remove this dependency.
- list(APPEND SCUDO_MINIMAL_OBJECT_LIBS RTGwpAsan RTGwpAsanOptionsParser
- RTGwpAsanBacktraceLibc
- RTGwpAsanSegvHandler)
+ list(APPEND SCUDO_MINIMAL_OBJECT_LIBS
+ RTGwpAsan RTGwpAsanOptionsParser RTGwpAsanBacktraceLibc
+ RTGwpAsanSegvHandler)
list(APPEND SCUDO_CFLAGS -DGWP_ASAN_HOOKS)
endif()
# include "gwp_asan/guarded_pool_allocator.h"
# include "gwp_asan/optional/backtrace.h"
# include "gwp_asan/optional/options_parser.h"
+#include "gwp_asan/optional/segv_handler.h"
#endif // GWP_ASAN_HOOKS
#include <errno.h>
// at compilation or at runtime.
static atomic_uint8_t HashAlgorithm = { CRC32Software };
-INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
+inline u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
// If the hardware CRC32 feature is defined here, it was enabled everywhere,
// as opposed to only for scudo_crc32.cpp. This means that other hardware
// specific instructions were likely emitted at other places, and as a
static BackendT &getBackend();
namespace Chunk {
- static INLINE AtomicPackedHeader *getAtomicHeader(void *Ptr) {
+ static inline AtomicPackedHeader *getAtomicHeader(void *Ptr) {
return reinterpret_cast<AtomicPackedHeader *>(reinterpret_cast<uptr>(Ptr) -
getHeaderSize());
}
- static INLINE
+ static inline
const AtomicPackedHeader *getConstAtomicHeader(const void *Ptr) {
return reinterpret_cast<const AtomicPackedHeader *>(
reinterpret_cast<uptr>(Ptr) - getHeaderSize());
}
- static INLINE bool isAligned(const void *Ptr) {
+ static inline bool isAligned(const void *Ptr) {
return IsAligned(reinterpret_cast<uptr>(Ptr), MinAlignment);
}
// We can't use the offset member of the chunk itself, as we would double
// fetch it without any warranty that it wouldn't have been tampered. To
// prevent this, we work with a local copy of the header.
- static INLINE void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
+ static inline void *getBackendPtr(const void *Ptr, UnpackedHeader *Header) {
return reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -
getHeaderSize() - (Header->Offset << MinAlignmentLog));
}
// Returns the usable size for a chunk, meaning the amount of bytes from the
// beginning of the user data to the end of the backend allocated chunk.
- static INLINE uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
+ static inline uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
const uptr ClassId = Header->ClassId;
if (ClassId)
return PrimaryT::ClassIdToSize(ClassId) - getHeaderSize() -
}
// Returns the size the user requested when allocating the chunk.
- static INLINE uptr getSize(const void *Ptr, UnpackedHeader *Header) {
+ static inline uptr getSize(const void *Ptr, UnpackedHeader *Header) {
const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
if (Header->ClassId)
return SizeOrUnusedBytes;
}
// Compute the checksum of the chunk pointer and its header.
- static INLINE u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) {
+ static inline u16 computeChecksum(const void *Ptr, UnpackedHeader *Header) {
UnpackedHeader ZeroChecksumHeader = *Header;
ZeroChecksumHeader.Checksum = 0;
uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
// Checks the validity of a chunk by verifying its checksum. It doesn't
// incur termination in the event of an invalid chunk.
- static INLINE bool isValid(const void *Ptr) {
+ static inline bool isValid(const void *Ptr) {
PackedHeader NewPackedHeader =
atomic_load_relaxed(getConstAtomicHeader(Ptr));
UnpackedHeader NewUnpackedHeader =
COMPILER_CHECK(ChunkAvailable == 0);
// Loads and unpacks the header, verifying the checksum in the process.
- static INLINE
+ static inline
void loadHeader(const void *Ptr, UnpackedHeader *NewUnpackedHeader) {
PackedHeader NewPackedHeader =
atomic_load_relaxed(getConstAtomicHeader(Ptr));
}
// Packs and stores the header, computing the checksum in the process.
- static INLINE void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) {
+ static inline void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) {
NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader);
// Packs and stores the header, computing the checksum in the process. We
// compare the current header with the expected provided one to ensure that
// we are not being raced by a corruption occurring in another thread.
- static INLINE void compareExchangeHeader(void *Ptr,
+ static inline void compareExchangeHeader(void *Ptr,
UnpackedHeader *NewUnpackedHeader,
UnpackedHeader *OldUnpackedHeader) {
NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader);
// Allocates a chunk.
void *allocate(uptr Size, uptr Alignment, AllocType Type,
- bool ForceZeroContents = false) {
+ bool ForceZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {
initThreadMaybe();
-#ifdef GWP_ASAN_HOOKS
- if (UNLIKELY(GuardedAlloc.shouldSample())) {
- if (void *Ptr = GuardedAlloc.allocate(Size))
- return Ptr;
- }
-#endif // GWP_ASAN_HOOKS
-
if (UNLIKELY(Alignment > MaxAlignment)) {
if (AllocatorMayReturnNull())
return nullptr;
if (UNLIKELY(Alignment < MinAlignment))
Alignment = MinAlignment;
+#ifdef GWP_ASAN_HOOKS
+ if (UNLIKELY(GuardedAlloc.shouldSample())) {
+ if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
+ if (SCUDO_CAN_USE_HOOKS && &__sanitizer_malloc_hook)
+ __sanitizer_malloc_hook(Ptr, Size);
+ return Ptr;
+ }
+ }
+#endif // GWP_ASAN_HOOKS
+
const uptr NeededSize = RoundUpTo(Size ? Size : 1, MinAlignment) +
Chunk::getHeaderSize();
const uptr AlignedSize = (Alignment > MinAlignment) ?
// a zero-sized quarantine, or if the size of the chunk is greater than the
// quarantine chunk size threshold.
void quarantineOrDeallocateChunk(void *Ptr, UnpackedHeader *Header,
- uptr Size) {
+ uptr Size) NO_THREAD_SAFETY_ANALYSIS {
const bool BypassQuarantine = !Size || (Size > QuarantineChunksUpToSize);
if (BypassQuarantine) {
UnpackedHeader NewHeader = *Header;
void initScudo() {
Instance.init();
#ifdef GWP_ASAN_HOOKS
- gwp_asan::options::initOptions();
+ gwp_asan::options::initOptions(__sanitizer::GetEnv("GWP_ASAN_OPTIONS"),
+ Printf);
gwp_asan::options::Options &Opts = gwp_asan::options::getOptions();
- Opts.Backtrace = gwp_asan::options::getBacktraceFunction();
+ Opts.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
GuardedAlloc.init(Opts);
if (Opts.InstallSignalHandlers)
- gwp_asan::crash_handler::installSignalHandlers(
+ gwp_asan::segv_handler::installSignalHandlers(
&GuardedAlloc, __sanitizer::Printf,
- gwp_asan::options::getPrintBacktraceFunction(), Opts.Backtrace);
+ gwp_asan::backtrace::getPrintBacktraceFunction(),
+ gwp_asan::backtrace::getSegvBacktraceFunction());
#endif // GWP_ASAN_HOOKS
}
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
};
-INLINE u32 computeSoftwareCRC32(u32 Crc, uptr Data) {
+inline u32 computeSoftwareCRC32(u32 Crc, uptr Data) {
for (uptr i = 0; i < sizeof(Data); i++) {
Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8);
Data >>= 8;
internal__exit(common_flags()->exitcode);
}
-void SetCheckFailedCallback(CheckFailedCallbackType callback) {}
+void SetCheckUnwindCallback(void (*callback)()) {}
void NORETURN CheckFailed(const char *File, int Line, const char *Condition,
u64 Value1, u64 Value2) {
void init();
void commitBack();
- INLINE bool tryLock() {
+ inline bool tryLock() TRY_ACQUIRE(true, Mutex) {
if (Mutex.TryLock()) {
atomic_store_relaxed(&Precedence, 0);
return true;
return false;
}
- INLINE void lock() {
+ inline void lock() ACQUIRE(Mutex) {
atomic_store_relaxed(&Precedence, 0);
Mutex.Lock();
}
- INLINE void unlock() { Mutex.Unlock(); }
+ inline void unlock() RELEASE(Mutex) { Mutex.Unlock(); }
- INLINE uptr getPrecedence() { return atomic_load_relaxed(&Precedence); }
+ inline uptr getPrecedence() { return atomic_load_relaxed(&Precedence); }
private:
StaticSpinMutex Mutex;
//===----------------------------------------------------------------------===//
#ifndef SCUDO_TSD_H_
-# error "This file must be included inside scudo_tsd.h."
-#endif // SCUDO_TSD_H_
+#error "This file must be included inside scudo_tsd.h."
+#endif // SCUDO_TSD_H_
#if SCUDO_TSD_EXCLUSIVE
ThreadInitialized,
ThreadTornDown,
};
-__attribute__((tls_model("initial-exec")))
-extern THREADLOCAL ThreadState ScudoThreadState;
-__attribute__((tls_model("initial-exec")))
-extern THREADLOCAL ScudoTSD TSD;
+__attribute__((
+ tls_model("initial-exec"))) extern THREADLOCAL ThreadState ScudoThreadState;
+__attribute__((tls_model("initial-exec"))) extern THREADLOCAL ScudoTSD TSD;
extern ScudoTSD FallbackTSD;
initThread(MinimalInit);
}
-ALWAYS_INLINE ScudoTSD *getTSDAndLock(bool *UnlockRequired) {
+ALWAYS_INLINE ScudoTSD *
+getTSDAndLock(bool *UnlockRequired) NO_THREAD_SAFETY_ANALYSIS {
if (UNLIKELY(ScudoThreadState != ThreadInitialized)) {
FallbackTSD.lock();
*UnlockRequired = true;
return &TSD;
}
-#endif // SCUDO_TSD_EXCLUSIVE
+#endif // SCUDO_TSD_EXCLUSIVE
setCurrentTSD(&TSDs[Index % NumberOfTSDs]);
}
-ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) {
+ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD) NO_THREAD_SAFETY_ANALYSIS {
if (NumberOfTSDs > 1) {
// Use the Precedence of the current TSD as our random seed. Since we are in
// the slow path, it means that tryLock failed, and as a result it's very
ScudoTSD *getTSDAndLockSlow(ScudoTSD *TSD);
-ALWAYS_INLINE ScudoTSD *getTSDAndLock(bool *UnlockRequired) {
+ALWAYS_INLINE ScudoTSD *
+getTSDAndLock(bool *UnlockRequired) NO_THREAD_SAFETY_ANALYSIS {
ScudoTSD *TSD = getCurrentTSD();
DCHECK(TSD && "No TSD associated with the current thread!");
*UnlockRequired = true;
// initialized after the other globals, so we can check its value to know if
// calling getauxval is safe.
extern "C" SANITIZER_WEAK_ATTRIBUTE char *__progname;
-INLINE bool areBionicGlobalsInitialized() {
+inline bool areBionicGlobalsInitialized() {
return !SANITIZER_ANDROID || (&__progname && __progname);
}
namespace __scudo {
template <class Dest, class Source>
-INLINE Dest bit_cast(const Source& source) {
+inline Dest bit_cast(const Source& source) {
static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!");
Dest dest;
memcpy(&dest, &source, sizeof(dest));
add_compiler_rt_component(scudo_standalone)
-if (COMPILER_RT_HAS_GWP_ASAN)
- add_dependencies(scudo_standalone gwp_asan)
-endif()
include_directories(../.. include)
list(APPEND SCUDO_CFLAGS
-Werror=conversion
-Wall
+ -g
-nostdinc++)
# Remove -stdlib= which is unused when passing -nostdinc++.
string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-append_list_if(COMPILER_RT_HAS_FFREESTANDING_FLAG -ffreestanding SCUDO_CFLAGS)
-
append_list_if(COMPILER_RT_HAS_FVISIBILITY_HIDDEN_FLAG -fvisibility=hidden SCUDO_CFLAGS)
-if (COMPILER_RT_HAS_GWP_ASAN)
- append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
- SCUDO_CFLAGS)
- append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG
- -mno-omit-leaf-frame-pointer SCUDO_CFLAGS)
-endif() # COMPILER_RT_HAS_GWP_ASAN
+append_list_if(COMPILER_RT_HAS_FNO_EXCEPTIONS_FLAG -fno-exceptions SCUDO_CFLAGS)
+
+append_list_if(COMPILER_RT_HAS_WNO_PEDANTIC -Wno-pedantic SCUDO_CFLAGS)
+
+# FIXME: find cleaner way to agree with GWPAsan flags
+append_list_if(COMPILER_RT_HAS_FNO_LTO_FLAG -fno-lto SCUDO_CFLAGS)
if(COMPILER_RT_DEBUG)
- list(APPEND SCUDO_CFLAGS -O0)
+ list(APPEND SCUDO_CFLAGS -O0 -DSCUDO_DEBUG=1)
else()
list(APPEND SCUDO_CFLAGS -O3)
endif()
list(APPEND SCUDO_LINK_FLAGS -Wl,-z,defs,-z,now,-z,relro)
-append_list_if(COMPILER_RT_HAS_NODEFAULTLIBS_FLAG -nodefaultlibs SCUDO_LINK_FLAGS)
+list(APPEND SCUDO_LINK_FLAGS -ffunction-sections -fdata-sections -Wl,--gc-sections)
+
+# We don't use the C++ standard library, so avoid including it by mistake.
+append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ SCUDO_LINK_FLAGS)
if(ANDROID)
list(APPEND SCUDO_CFLAGS -fno-emulated-tls)
checksum.h
chunk.h
combined.h
- flags.h
+ common.h
flags_parser.h
+ flags.h
fuchsia.h
internal_defs.h
linux.h
list.h
local_cache.h
+ memtag.h
mutex.h
+ options.h
platform.h
primary32.h
primary64.h
report.h
secondary.h
size_class_map.h
+ stack_depot.h
stats.h
string_utils.h
- tsd.h
tsd_exclusive.h
tsd_shared.h
+ tsd.h
vector.h
wrappers_c_checks.h
wrappers_c.h
set(SCUDO_SOURCES
checksum.cpp
- crc32_hw.cpp
common.cpp
- flags.cpp
+ crc32_hw.cpp
flags_parser.cpp
+ flags.cpp
fuchsia.cpp
linux.cpp
release.cpp
set(SCUDO_OBJECT_LIBS)
if (COMPILER_RT_HAS_GWP_ASAN)
+ add_dependencies(scudo_standalone gwp_asan)
list(APPEND SCUDO_OBJECT_LIBS
- RTGwpAsan RTGwpAsanBacktraceLibc RTGwpAsanSegvHandler)
+ RTGwpAsan RTGwpAsanBacktraceLibc RTGwpAsanSegvHandler
+ RTGwpAsanOptionsParser)
+
+ append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer
+ -mno-omit-leaf-frame-pointer
+ SCUDO_CFLAGS)
list(APPEND SCUDO_CFLAGS -DGWP_ASAN_HOOKS)
+
endif()
+set(SCUDO_LINK_LIBS)
+
+append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread SCUDO_LINK_FLAGS)
+
+append_list_if(FUCHSIA zircon SCUDO_LINK_LIBS)
+
if(COMPILER_RT_HAS_SCUDO_STANDALONE)
add_compiler_rt_object_libraries(RTScudoStandalone
ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
SOURCES ${SCUDO_SOURCES_CXX_WRAPPERS}
ADDITIONAL_HEADERS ${SCUDO_HEADERS}
CFLAGS ${SCUDO_CFLAGS}
+ PARENT_TARGET scudo_standalone)
+
+ add_compiler_rt_runtime(clang_rt.scudo_standalone
+ SHARED
+ ARCHS ${SCUDO_STANDALONE_SUPPORTED_ARCH}
+ SOURCES ${SCUDO_SOURCES} ${SCUDO_SOURCES_C_WRAPPERS} ${SCUDO_SOURCES_CXX_WRAPPERS}
+ ADDITIONAL_HEADERS ${SCUDO_HEADERS}
+ CFLAGS ${SCUDO_CFLAGS}
OBJECT_LIBS ${SCUDO_OBJECT_LIBS}
+ LINK_FLAGS ${SCUDO_LINK_FLAGS}
+ LINK_LIBS ${SCUDO_LINK_LIBS}
PARENT_TARGET scudo_standalone)
add_subdirectory(benchmarks)
namespace scudo {
+// The combined allocator uses a structure as a template argument that
+// specifies the configuration options for the various subcomponents of the
+// allocator.
+//
+// struct ExampleConfig {
+// // SizeClasMmap to use with the Primary.
+// using SizeClassMap = DefaultSizeClassMap;
+// // Indicates possible support for Memory Tagging.
+// static const bool MaySupportMemoryTagging = false;
+// // Defines the Primary allocator to use.
+// typedef SizeClassAllocator64<ExampleConfig> Primary;
+// // Log2 of the size of a size class region, as used by the Primary.
+// static const uptr PrimaryRegionSizeLog = 30U;
+// // Defines the type and scale of a compact pointer. A compact pointer can
+// // be understood as the offset of a pointer within the region it belongs
+// // to, in increments of a power-of-2 scale.
+// // eg: Ptr = Base + (CompactPtr << Scale).
+// typedef u32 PrimaryCompactPtrT;
+// static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+// // Indicates support for offsetting the start of a region by
+// // a random number of pages. Only used with primary64.
+// static const bool PrimaryEnableRandomOffset = true;
+// // Call map for user memory with at least this size. Only used with
+// // primary64.
+// static const uptr PrimaryMapSizeIncrement = 1UL << 18;
+// // Defines the minimal & maximal release interval that can be set.
+// static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+// static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+// // Defines the type of cache used by the Secondary. Some additional
+// // configuration entries can be necessary depending on the Cache.
+// typedef MapAllocatorNoCache SecondaryCache;
+// // Thread-Specific Data Registry used, shared or exclusive.
+// template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>;
+// };
+
// Default configurations for various platforms.
struct DefaultConfig {
using SizeClassMap = DefaultSizeClassMap;
+ static const bool MaySupportMemoryTagging = true;
+
#if SCUDO_CAN_USE_PRIMARY64
- // 1GB Regions
- typedef SizeClassAllocator64<SizeClassMap, 30U> Primary;
+ typedef SizeClassAllocator64<DefaultConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 32U;
+ typedef uptr PrimaryCompactPtrT;
+ static const uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
- // 512KB regions
- typedef SizeClassAllocator32<SizeClassMap, 19U> Primary;
+ typedef SizeClassAllocator32<DefaultConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 19U;
+ typedef uptr PrimaryCompactPtrT;
#endif
- typedef MapAllocator<MapAllocatorCache<>> Secondary;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+
+ typedef MapAllocatorCache<DefaultConfig> SecondaryCache;
+ static const u32 SecondaryCacheEntriesArraySize = 32U;
+ static const u32 SecondaryCacheQuarantineSize = 0U;
+ static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U;
+ static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 19;
+ static const s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN;
+ static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX;
+
template <class A> using TSDRegistryT = TSDRegistryExT<A>; // Exclusive
};
-
struct AndroidConfig {
using SizeClassMap = AndroidSizeClassMap;
+ static const bool MaySupportMemoryTagging = true;
+
#if SCUDO_CAN_USE_PRIMARY64
- // 256MB regions
- typedef SizeClassAllocator64<SizeClassMap, 28U, 1000, 1000,
- /*MaySupportMemoryTagging=*/true>
- Primary;
+ typedef SizeClassAllocator64<AndroidConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 28U;
+ typedef u32 PrimaryCompactPtrT;
+ static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
- // 256KB regions
- typedef SizeClassAllocator32<SizeClassMap, 18U, 1000, 1000> Primary;
+ typedef SizeClassAllocator32<AndroidConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 18U;
+ typedef uptr PrimaryCompactPtrT;
#endif
- // Cache blocks up to 2MB
- typedef MapAllocator<MapAllocatorCache<32U, 2UL << 20, 0, 1000>> Secondary;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = 1000;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000;
+
+ typedef MapAllocatorCache<AndroidConfig> SecondaryCache;
+ static const u32 SecondaryCacheEntriesArraySize = 256U;
+ static const u32 SecondaryCacheQuarantineSize = 32U;
+ static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U;
+ static const uptr SecondaryCacheDefaultMaxEntrySize = 2UL << 20;
+ static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0;
+ static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = 1000;
+
template <class A>
- using TSDRegistryT = TSDRegistrySharedT<A, 2U>; // Shared, max 2 TSDs.
+ using TSDRegistryT = TSDRegistrySharedT<A, 8U, 2U>; // Shared, max 8 TSDs.
};
struct AndroidSvelteConfig {
using SizeClassMap = SvelteSizeClassMap;
+ static const bool MaySupportMemoryTagging = false;
+
#if SCUDO_CAN_USE_PRIMARY64
- // 128MB regions
- typedef SizeClassAllocator64<SizeClassMap, 27U, 1000, 1000> Primary;
+ typedef SizeClassAllocator64<AndroidSvelteConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 27U;
+ typedef u32 PrimaryCompactPtrT;
+ static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
- // 64KB regions
- typedef SizeClassAllocator32<SizeClassMap, 16U, 1000, 1000> Primary;
+ typedef SizeClassAllocator32<AndroidSvelteConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 16U;
+ typedef uptr PrimaryCompactPtrT;
#endif
- typedef MapAllocator<MapAllocatorCache<4U, 1UL << 18, 0, 0>> Secondary;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = 1000;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000;
+
+ typedef MapAllocatorCache<AndroidSvelteConfig> SecondaryCache;
+ static const u32 SecondaryCacheEntriesArraySize = 16U;
+ static const u32 SecondaryCacheQuarantineSize = 32U;
+ static const u32 SecondaryCacheDefaultMaxEntriesCount = 4U;
+ static const uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 18;
+ static const s32 SecondaryCacheMinReleaseToOsIntervalMs = 0;
+ static const s32 SecondaryCacheMaxReleaseToOsIntervalMs = 0;
+
template <class A>
- using TSDRegistryT = TSDRegistrySharedT<A, 1U>; // Shared, only 1 TSD.
+ using TSDRegistryT = TSDRegistrySharedT<A, 2U, 1U>; // Shared, max 2 TSDs.
};
#if SCUDO_CAN_USE_PRIMARY64
struct FuchsiaConfig {
- // 1GB Regions
- typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary;
- typedef MapAllocator<MapAllocatorNoCache> Secondary;
+ using SizeClassMap = FuchsiaSizeClassMap;
+ static const bool MaySupportMemoryTagging = false;
+
+ typedef SizeClassAllocator64<FuchsiaConfig> Primary;
+ static const uptr PrimaryRegionSizeLog = 30U;
+ typedef u32 PrimaryCompactPtrT;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
+ static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+
+ typedef MapAllocatorNoCache SecondaryCache;
+ template <class A>
+ using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.
+};
+
+struct TrustyConfig {
+ using SizeClassMap = TrustySizeClassMap;
+ static const bool MaySupportMemoryTagging = false;
+
+ typedef SizeClassAllocator64<TrustyConfig> Primary;
+ // Some apps have 1 page of heap total so small regions are necessary.
+ static const uptr PrimaryRegionSizeLog = 10U;
+ typedef u32 PrimaryCompactPtrT;
+ static const bool PrimaryEnableRandomOffset = false;
+ // Trusty is extremely memory-constrained so minimally round up map calls.
+ static const uptr PrimaryMapSizeIncrement = 1UL << 4;
+ static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+
+ typedef MapAllocatorNoCache SecondaryCache;
template <class A>
- using TSDRegistryT = TSDRegistrySharedT<A, 8U>; // Shared, max 8 TSDs.
+ using TSDRegistryT = TSDRegistrySharedT<A, 1U, 1U>; // Shared, max 1 TSD.
};
#endif
typedef AndroidConfig Config;
#elif SCUDO_FUCHSIA
typedef FuchsiaConfig Config;
+#elif SCUDO_TRUSTY
+typedef TrustyConfig Config;
#else
typedef DefaultConfig Config;
#endif
return __atomic_fetch_sub(&A->ValDoNotUse, V, MO);
}
+template <typename T>
+inline typename T::Type atomic_fetch_and(volatile T *A, typename T::Type V,
+ memory_order MO) {
+ DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
+ return __atomic_fetch_and(&A->ValDoNotUse, V, MO);
+}
+
+template <typename T>
+inline typename T::Type atomic_fetch_or(volatile T *A, typename T::Type V,
+ memory_order MO) {
+ DCHECK(!(reinterpret_cast<uptr>(A) % sizeof(*A)));
+ return __atomic_fetch_or(&A->ValDoNotUse, V, MO);
+}
+
template <typename T>
inline typename T::Type atomic_exchange(volatile T *A, typename T::Type V,
memory_order MO) {
__ATOMIC_RELAXED);
}
-template <typename T>
-inline bool atomic_compare_exchange_weak(volatile T *A, typename T::Type *Cmp,
- typename T::Type Xchg,
- memory_order MO) {
- return __atomic_compare_exchange(&A->ValDoNotUse, Cmp, &Xchg, true, MO,
- __ATOMIC_RELAXED);
-}
-
// Clutter-reducing helpers.
template <typename T>
$<TARGET_OBJECTS:RTScudoStandalone.${arch}>)
set_property(TARGET ScudoBenchmarks.${arch} APPEND_STRING PROPERTY
COMPILE_FLAGS "${SCUDO_BENCHMARK_CFLAGS}")
+
+ if (COMPILER_RT_HAS_GWP_ASAN)
+ add_benchmark(
+ ScudoBenchmarksWithGwpAsan.${arch} malloc_benchmark.cpp
+ $<TARGET_OBJECTS:RTScudoStandalone.${arch}>
+ $<TARGET_OBJECTS:RTGwpAsan.${arch}>
+ $<TARGET_OBJECTS:RTGwpAsanBacktraceLibc.${arch}>
+ $<TARGET_OBJECTS:RTGwpAsanSegvHandler.${arch}>)
+ set_property(
+ TARGET ScudoBenchmarksWithGwpAsan.${arch} APPEND_STRING PROPERTY
+ COMPILE_FLAGS "${SCUDO_BENCHMARK_CFLAGS} -DGWP_ASAN_HOOKS")
+ endif()
endforeach()
#include "benchmark/benchmark.h"
#include <memory>
+#include <vector>
+
+void *CurrentAllocator;
+template <typename Config> void PostInitCallback() {
+ reinterpret_cast<scudo::Allocator<Config> *>(CurrentAllocator)->initGwpAsan();
+}
template <typename Config> static void BM_malloc_free(benchmark::State &State) {
- using AllocatorT = scudo::Allocator<Config>;
+ using AllocatorT = scudo::Allocator<Config, PostInitCallback<Config>>;
auto Deleter = [](AllocatorT *A) {
A->unmapTestOnly();
delete A;
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
+ CurrentAllocator = Allocator.get();
const size_t NBytes = State.range(0);
size_t PageSize = scudo::getPageSizeCached();
template <typename Config>
static void BM_malloc_free_loop(benchmark::State &State) {
- using AllocatorT = scudo::Allocator<Config>;
+ using AllocatorT = scudo::Allocator<Config, PostInitCallback<Config>>;
auto Deleter = [](AllocatorT *A) {
A->unmapTestOnly();
delete A;
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
+ CurrentAllocator = Allocator.get();
const size_t NumIters = State.range(0);
size_t PageSize = scudo::getPageSizeCached();
- void *Ptrs[NumIters];
+ std::vector<void *> Ptrs(NumIters);
for (auto _ : State) {
size_t SizeLog2 = 0;
template <uptr Size> class FlatByteMap {
public:
- void initLinkerInitialized() {}
- void init() { memset(Map, 0, sizeof(Map)); }
+ void init() { DCHECK(Size == 0 || Map[0] == 0); }
- void unmapTestOnly() {}
+ void unmapTestOnly() { memset(Map, 0, Size); }
void set(uptr Index, u8 Value) {
DCHECK_LT(Index, Size);
void enable() {}
private:
- u8 Map[Size];
+ u8 Map[Size] = {};
};
} // namespace scudo
struct UnpackedHeader {
uptr ClassId : 8;
u8 State : 2;
- u8 Origin : 2;
+ // Origin if State == Allocated, or WasZeroed otherwise.
+ u8 OriginOrWasZeroed : 2;
uptr SizeOrUnusedBytes : 20;
uptr Offset : 16;
uptr Checksum : 16;
#include "flags_parser.h"
#include "local_cache.h"
#include "memtag.h"
+#include "options.h"
#include "quarantine.h"
#include "report.h"
#include "secondary.h"
namespace scudo {
-enum class Option { ReleaseInterval };
-
template <class Params, void (*PostInitCallback)(void) = EmptyCallback>
class Allocator {
public:
typedef typename Params::template TSDRegistryT<ThisT> TSDRegistryT;
void callPostInitCallback() {
- static pthread_once_t OnceControl = PTHREAD_ONCE_INIT;
- pthread_once(&OnceControl, PostInitCallback);
+ pthread_once(&PostInitNonce, PostInitCallback);
}
struct QuarantineCallback {
NewHeader.State = Chunk::State::Available;
Chunk::compareExchangeHeader(Allocator.Cookie, Ptr, &NewHeader, &Header);
+ if (allocatorSupportsMemoryTagging<Params>())
+ Ptr = untagPointer(Ptr);
void *BlockBegin = Allocator::getBlockBegin(Ptr, &NewHeader);
- const uptr ClassId = NewHeader.ClassId;
- if (LIKELY(ClassId))
- Cache.deallocate(ClassId, BlockBegin);
- else
- Allocator.Secondary.deallocate(BlockBegin);
+ Cache.deallocate(NewHeader.ClassId, BlockBegin);
}
// We take a shortcut when allocating a quarantine batch by working with the
Header.State = Chunk::State::Allocated;
Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);
+ // Reset tag to 0 as this chunk may have been previously used for a tagged
+ // user allocation.
+ if (UNLIKELY(useMemoryTagging<Params>(Allocator.Primary.Options.load())))
+ storeTags(reinterpret_cast<uptr>(Ptr),
+ reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
+
return Ptr;
}
typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
typedef typename QuarantineT::CacheT QuarantineCacheT;
- void initLinkerInitialized() {
+ void init() {
performSanityChecks();
// Check if hardware CRC32 is supported in the binary and by the platform,
reportUnrecognizedFlags();
// Store some flags locally.
- Options.MayReturnNull = getFlags()->may_return_null;
- Options.FillContents =
- getFlags()->zero_contents
- ? ZeroFill
- : (getFlags()->pattern_fill_contents ? PatternOrZeroFill : NoFill);
- Options.DeallocTypeMismatch = getFlags()->dealloc_type_mismatch;
- Options.DeleteSizeMismatch = getFlags()->delete_size_mismatch;
- Options.TrackAllocationStacks = false;
- Options.QuarantineMaxChunkSize =
+ if (getFlags()->may_return_null)
+ Primary.Options.set(OptionBit::MayReturnNull);
+ if (getFlags()->zero_contents)
+ Primary.Options.setFillContentsMode(ZeroFill);
+ else if (getFlags()->pattern_fill_contents)
+ Primary.Options.setFillContentsMode(PatternOrZeroFill);
+ if (getFlags()->dealloc_type_mismatch)
+ Primary.Options.set(OptionBit::DeallocTypeMismatch);
+ if (getFlags()->delete_size_mismatch)
+ Primary.Options.set(OptionBit::DeleteSizeMismatch);
+ if (allocatorSupportsMemoryTagging<Params>() &&
+ systemSupportsMemoryTagging())
+ Primary.Options.set(OptionBit::UseMemoryTagging);
+ Primary.Options.set(OptionBit::UseOddEvenTags);
+
+ QuarantineMaxChunkSize =
static_cast<u32>(getFlags()->quarantine_max_chunk_size);
- Stats.initLinkerInitialized();
+ Stats.init();
const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
- Primary.initLinkerInitialized(ReleaseToOsIntervalMs);
- Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs);
-
+ Primary.init(ReleaseToOsIntervalMs);
+ Secondary.init(&Stats, ReleaseToOsIntervalMs);
Quarantine.init(
static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
#ifdef GWP_ASAN_HOOKS
gwp_asan::options::Options Opt;
Opt.Enabled = getFlags()->GWP_ASAN_Enabled;
- // Bear in mind - Scudo has its own alignment guarantees that are strictly
- // enforced. Scudo exposes the same allocation function for everything from
- // malloc() to posix_memalign, so in general this flag goes unused, as Scudo
- // will always ask GWP-ASan for an aligned amount of bytes.
- Opt.PerfectlyRightAlign = getFlags()->GWP_ASAN_PerfectlyRightAlign;
Opt.MaxSimultaneousAllocations =
getFlags()->GWP_ASAN_MaxSimultaneousAllocations;
Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;
// Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork
// handler.
Opt.InstallForkHandlers = false;
- Opt.Backtrace = gwp_asan::options::getBacktraceFunction();
+ Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();
GuardedAlloc.init(Opt);
if (Opt.InstallSignalHandlers)
- gwp_asan::crash_handler::installSignalHandlers(
- &GuardedAlloc, Printf, gwp_asan::options::getPrintBacktraceFunction(),
- Opt.Backtrace);
+ gwp_asan::segv_handler::installSignalHandlers(
+ &GuardedAlloc, Printf,
+ gwp_asan::backtrace::getPrintBacktraceFunction(),
+ gwp_asan::backtrace::getSegvBacktraceFunction());
+
+ GuardedAllocSlotSize =
+ GuardedAlloc.getAllocatorState()->maximumAllocationSize();
+ Stats.add(StatFree, static_cast<uptr>(Opt.MaxSimultaneousAllocations) *
+ GuardedAllocSlotSize);
#endif // GWP_ASAN_HOOKS
}
- void reset() { memset(this, 0, sizeof(*this)); }
+ ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
+ TSDRegistry.initThreadMaybe(this, MinimalInit);
+ }
void unmapTestOnly() {
- TSDRegistry.unmapTestOnly();
+ TSDRegistry.unmapTestOnly(this);
Primary.unmapTestOnly();
+ Secondary.unmapTestOnly();
#ifdef GWP_ASAN_HOOKS
if (getFlags()->GWP_ASAN_InstallSignalHandlers)
- gwp_asan::crash_handler::uninstallSignalHandlers();
+ gwp_asan::segv_handler::uninstallSignalHandlers();
GuardedAlloc.uninitTestOnly();
#endif // GWP_ASAN_HOOKS
}
TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
// The Cache must be provided zero-initialized.
- void initCache(CacheT *Cache) {
- Cache->initLinkerInitialized(&Stats, &Primary);
- }
+ void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
// Release the resources used by a TSD, which involves:
// - draining the local quarantine cache to the global quarantine;
TSD->Cache.destroy(&Stats);
}
- ALWAYS_INLINE void *untagPointerMaybe(void *Ptr) {
- if (Primary.SupportsMemoryTagging)
- return reinterpret_cast<void *>(
- untagPointer(reinterpret_cast<uptr>(Ptr)));
- return Ptr;
+ ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
+ if (!allocatorSupportsMemoryTagging<Params>())
+ return Ptr;
+ auto UntaggedPtr = untagPointer(Ptr);
+ if (UntaggedPtr != Ptr)
+ return UntaggedPtr;
+ // Secondary, or pointer allocated while memory tagging is unsupported or
+ // disabled. The tag mismatch is okay in the latter case because tags will
+ // not be checked.
+ return addHeaderTag(Ptr);
+ }
+
+ ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {
+ if (!allocatorSupportsMemoryTagging<Params>())
+ return Ptr;
+ return addFixedTag(Ptr, 2);
+ }
+
+ ALWAYS_INLINE void *addHeaderTag(void *Ptr) {
+ return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));
}
NOINLINE u32 collectStackTrace() {
#endif
}
+ uptr computeOddEvenMaskForPointerMaybe(Options Options, uptr Ptr,
+ uptr ClassId) {
+ if (!Options.get(OptionBit::UseOddEvenTags))
+ return 0;
+
+ // If a chunk's tag is odd, we want the tags of the surrounding blocks to be
+ // even, and vice versa. Blocks are laid out Size bytes apart, and adding
+ // Size to Ptr will flip the least significant set bit of Size in Ptr, so
+ // that bit will have the pattern 010101... for consecutive blocks, which we
+ // can use to determine which tag mask to use.
+ return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);
+ }
+
NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
uptr Alignment = MinAlignment,
bool ZeroContents = false) {
initThreadMaybe();
-#ifdef GWP_ASAN_HOOKS
- if (UNLIKELY(GuardedAlloc.shouldSample())) {
- if (void *Ptr = GuardedAlloc.allocate(roundUpTo(Size, Alignment)))
- return Ptr;
- }
-#endif // GWP_ASAN_HOOKS
-
- FillContentsMode FillContents =
- ZeroContents ? ZeroFill : Options.FillContents;
-
+ const Options Options = Primary.Options.load();
if (UNLIKELY(Alignment > MaxAlignment)) {
- if (Options.MayReturnNull)
+ if (Options.get(OptionBit::MayReturnNull))
return nullptr;
reportAlignmentTooBig(Alignment, MaxAlignment);
}
if (Alignment < MinAlignment)
Alignment = MinAlignment;
+#ifdef GWP_ASAN_HOOKS
+ if (UNLIKELY(GuardedAlloc.shouldSample())) {
+ if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {
+ if (UNLIKELY(&__scudo_allocate_hook))
+ __scudo_allocate_hook(Ptr, Size);
+ Stats.lock();
+ Stats.add(StatAllocated, GuardedAllocSlotSize);
+ Stats.sub(StatFree, GuardedAllocSlotSize);
+ Stats.unlock();
+ return Ptr;
+ }
+ }
+#endif // GWP_ASAN_HOOKS
+
+ const FillContentsMode FillContents = ZeroContents ? ZeroFill
+ : TSDRegistry.getDisableMemInit()
+ ? NoFill
+ : Options.getFillContentsMode();
+
// If the requested size happens to be 0 (more common than you might think),
// allocate MinAlignment bytes on top of the header. Then add the extra
// bytes required to fulfill the alignment requirements: we allocate enough
// Takes care of extravagantly large sizes as well as integer overflows.
static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");
if (UNLIKELY(Size >= MaxAllowedMallocSize)) {
- if (Options.MayReturnNull)
+ if (Options.get(OptionBit::MayReturnNull))
return nullptr;
reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize);
}
void *Block = nullptr;
uptr ClassId = 0;
- uptr SecondaryBlockEnd;
+ uptr SecondaryBlockEnd = 0;
if (LIKELY(PrimaryT::canAllocate(NeededSize))) {
ClassId = SizeClassMap::getClassIdBySize(NeededSize);
DCHECK_NE(ClassId, 0U);
// larger class until it fits. If it fails to fit in the largest class,
// fallback to the Secondary.
if (UNLIKELY(!Block)) {
- while (ClassId < SizeClassMap::LargestClassId) {
+ while (ClassId < SizeClassMap::LargestClassId && !Block)
Block = TSD->Cache.allocate(++ClassId);
- if (LIKELY(Block)) {
- break;
- }
- }
- if (UNLIKELY(!Block)) {
+ if (!Block)
ClassId = 0;
- }
}
if (UnlockRequired)
TSD->unlock();
}
if (UNLIKELY(ClassId == 0))
- Block = Secondary.allocate(NeededSize, Alignment, &SecondaryBlockEnd,
+ Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,
FillContents);
if (UNLIKELY(!Block)) {
- if (Options.MayReturnNull)
+ if (Options.get(OptionBit::MayReturnNull))
return nullptr;
reportOutOfMemory(NeededSize);
}
void *Ptr = reinterpret_cast<void *>(UserPtr);
void *TaggedPtr = Ptr;
- if (ClassId) {
+ if (LIKELY(ClassId)) {
// We only need to zero or tag the contents for Primary backed
// allocations. We only set tags for primary allocations in order to avoid
// faulting potentially large numbers of pages for large secondary
//
// When memory tagging is enabled, zeroing the contents is done as part of
// setting the tag.
- if (UNLIKELY(useMemoryTagging())) {
+ if (UNLIKELY(useMemoryTagging<Params>(Options))) {
uptr PrevUserPtr;
Chunk::UnpackedHeader Header;
- const uptr BlockEnd = BlockUptr + PrimaryT::getSizeByClassId(ClassId);
+ const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);
+ const uptr BlockEnd = BlockUptr + BlockSize;
// If possible, try to reuse the UAF tag that was set by deallocate().
// For simplicity, only reuse tags if we have the same start address as
// the previous allocation. This handles the majority of cases since
if (NextPage < PrevEnd && loadTag(NextPage) != NextPage)
PrevEnd = NextPage;
TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);
- resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, BlockEnd);
- if (Size) {
+ resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, Size, BlockEnd);
+ if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {
+ // If an allocation needs to be zeroed (i.e. calloc) we can normally
+ // avoid zeroing the memory now since we can rely on memory having
+ // been zeroed on free, as this is normally done while setting the
+ // UAF tag. But if tagging was disabled per-thread when the memory
+ // was freed, it would not have been retagged and thus zeroed, and
+ // therefore it needs to be zeroed now.
+ memset(TaggedPtr, 0,
+ Min(Size, roundUpTo(PrevEnd - TaggedUserPtr,
+ archMemoryTagGranuleSize())));
+ } else if (Size) {
// Clear any stack metadata that may have previously been stored in
// the chunk data.
memset(TaggedPtr, 0, archMemoryTagGranuleSize());
}
} else {
- TaggedPtr = prepareTaggedChunk(Ptr, Size, BlockEnd);
+ const uptr OddEvenMask =
+ computeOddEvenMaskForPointerMaybe(Options, BlockUptr, ClassId);
+ TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);
}
- storeAllocationStackMaybe(Ptr);
- } else if (UNLIKELY(FillContents != NoFill)) {
- // This condition is not necessarily unlikely, but since memset is
- // costly, we might as well mark it as such.
- memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
- PrimaryT::getSizeByClassId(ClassId));
+ storePrimaryAllocationStackMaybe(Options, Ptr);
+ } else {
+ Block = addHeaderTag(Block);
+ Ptr = addHeaderTag(Ptr);
+ if (UNLIKELY(FillContents != NoFill)) {
+ // This condition is not necessarily unlikely, but since memset is
+ // costly, we might as well mark it as such.
+ memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,
+ PrimaryT::getSizeByClassId(ClassId));
+ }
+ }
+ } else {
+ Block = addHeaderTag(Block);
+ Ptr = addHeaderTag(Ptr);
+ if (UNLIKELY(useMemoryTagging<Params>(Options))) {
+ storeTags(reinterpret_cast<uptr>(Block), reinterpret_cast<uptr>(Ptr));
+ storeSecondaryAllocationStackMaybe(Options, Ptr, Size);
}
}
}
Header.ClassId = ClassId & Chunk::ClassIdMask;
Header.State = Chunk::State::Allocated;
- Header.Origin = Origin & Chunk::OriginMask;
+ Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;
Header.SizeOrUnusedBytes =
(ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size)) &
Chunk::SizeOrUnusedBytesMask;
Chunk::storeHeader(Cookie, Ptr, &Header);
- if (&__scudo_allocate_hook)
+ if (UNLIKELY(&__scudo_allocate_hook))
__scudo_allocate_hook(TaggedPtr, Size);
return TaggedPtr;
// being destroyed properly. Any other heap operation will do a full init.
initThreadMaybe(/*MinimalInit=*/true);
+ if (UNLIKELY(&__scudo_deallocate_hook))
+ __scudo_deallocate_hook(Ptr);
+
+ if (UNLIKELY(!Ptr))
+ return;
+
#ifdef GWP_ASAN_HOOKS
if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
GuardedAlloc.deallocate(Ptr);
+ Stats.lock();
+ Stats.add(StatFree, GuardedAllocSlotSize);
+ Stats.sub(StatAllocated, GuardedAllocSlotSize);
+ Stats.unlock();
return;
}
#endif // GWP_ASAN_HOOKS
- if (&__scudo_deallocate_hook)
- __scudo_deallocate_hook(Ptr);
-
- if (UNLIKELY(!Ptr))
- return;
if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))
reportMisalignedPointer(AllocatorAction::Deallocating, Ptr);
- Ptr = untagPointerMaybe(Ptr);
+ void *TaggedPtr = Ptr;
+ Ptr = getHeaderTaggedPointer(Ptr);
Chunk::UnpackedHeader Header;
Chunk::loadHeader(Cookie, Ptr, &Header);
if (UNLIKELY(Header.State != Chunk::State::Allocated))
reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);
- if (Options.DeallocTypeMismatch) {
- if (Header.Origin != Origin) {
+
+ const Options Options = Primary.Options.load();
+ if (Options.get(OptionBit::DeallocTypeMismatch)) {
+ if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {
// With the exception of memalign'd chunks, that can be still be free'd.
- if (UNLIKELY(Header.Origin != Chunk::Origin::Memalign ||
- Origin != Chunk::Origin::Malloc))
+ if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
+ Origin != Chunk::Origin::Malloc)
reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr,
- Header.Origin, Origin);
+ Header.OriginOrWasZeroed, Origin);
}
}
const uptr Size = getSize(Ptr, &Header);
- if (DeleteSize && Options.DeleteSizeMismatch) {
+ if (DeleteSize && Options.get(OptionBit::DeleteSizeMismatch)) {
if (UNLIKELY(DeleteSize != Size))
reportDeleteSizeMismatch(Ptr, DeleteSize, Size);
}
- quarantineOrDeallocateChunk(Ptr, &Header, Size);
+ quarantineOrDeallocateChunk(Options, TaggedPtr, &Header, Size);
}
void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
initThreadMaybe();
+ const Options Options = Primary.Options.load();
if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
- if (Options.MayReturnNull)
+ if (Options.get(OptionBit::MayReturnNull))
return nullptr;
reportAllocationSizeTooBig(NewSize, 0, MaxAllowedMallocSize);
}
- void *OldTaggedPtr = OldPtr;
- OldPtr = untagPointerMaybe(OldPtr);
-
// The following cases are handled by the C wrappers.
DCHECK_NE(OldPtr, nullptr);
DCHECK_NE(NewSize, 0);
if (NewPtr)
memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize);
GuardedAlloc.deallocate(OldPtr);
+ Stats.lock();
+ Stats.add(StatFree, GuardedAllocSlotSize);
+ Stats.sub(StatAllocated, GuardedAllocSlotSize);
+ Stats.unlock();
return NewPtr;
}
#endif // GWP_ASAN_HOOKS
+ void *OldTaggedPtr = OldPtr;
+ OldPtr = getHeaderTaggedPointer(OldPtr);
+
if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))
reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr);
// Pointer has to be allocated with a malloc-type function. Some
// applications think that it is OK to realloc a memalign'ed pointer, which
// will trigger this check. It really isn't.
- if (Options.DeallocTypeMismatch) {
- if (UNLIKELY(OldHeader.Origin != Chunk::Origin::Malloc))
+ if (Options.get(OptionBit::DeallocTypeMismatch)) {
+ if (UNLIKELY(OldHeader.OriginOrWasZeroed != Chunk::Origin::Malloc))
reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr,
- OldHeader.Origin, Chunk::Origin::Malloc);
+ OldHeader.OriginOrWasZeroed,
+ Chunk::Origin::Malloc);
}
- void *BlockBegin = getBlockBegin(OldPtr, &OldHeader);
+ void *BlockBegin = getBlockBegin(OldTaggedPtr, &OldHeader);
uptr BlockEnd;
uptr OldSize;
const uptr ClassId = OldHeader.ClassId;
OldSize = OldHeader.SizeOrUnusedBytes;
} else {
BlockEnd = SecondaryT::getBlockEnd(BlockBegin);
- OldSize = BlockEnd -
- (reinterpret_cast<uptr>(OldPtr) + OldHeader.SizeOrUnusedBytes);
+ OldSize = BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) +
+ OldHeader.SizeOrUnusedBytes);
}
// If the new chunk still fits in the previously allocated block (with a
// reasonable delta), we just keep the old block, and update the chunk
// header to reflect the size change.
- if (reinterpret_cast<uptr>(OldPtr) + NewSize <= BlockEnd) {
+ if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {
if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {
Chunk::UnpackedHeader NewHeader = OldHeader;
NewHeader.SizeOrUnusedBytes =
(ClassId ? NewSize
- : BlockEnd - (reinterpret_cast<uptr>(OldPtr) + NewSize)) &
+ : BlockEnd -
+ (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &
Chunk::SizeOrUnusedBytesMask;
Chunk::compareExchangeHeader(Cookie, OldPtr, &NewHeader, &OldHeader);
- if (UNLIKELY(ClassId && useMemoryTagging())) {
- resizeTaggedChunk(reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
- reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
- BlockEnd);
- storeAllocationStackMaybe(OldPtr);
+ if (UNLIKELY(useMemoryTagging<Params>(Options))) {
+ if (ClassId) {
+ resizeTaggedChunk(reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,
+ reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,
+ NewSize, untagPointer(BlockEnd));
+ storePrimaryAllocationStackMaybe(Options, OldPtr);
+ } else {
+ storeSecondaryAllocationStackMaybe(Options, OldPtr, NewSize);
+ }
}
return OldTaggedPtr;
}
// allow for potential further in-place realloc. The gains of such a trick
// are currently unclear.
void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
- if (NewPtr) {
- const uptr OldSize = getSize(OldPtr, &OldHeader);
+ if (LIKELY(NewPtr)) {
memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize));
- quarantineOrDeallocateChunk(OldPtr, &OldHeader, OldSize);
+ quarantineOrDeallocateChunk(Options, OldTaggedPtr, &OldHeader, OldSize);
}
return NewPtr;
}
// function. This can be called with a null buffer or zero size for buffer
// sizing purposes.
uptr getStats(char *Buffer, uptr Size) {
- ScopedString Str(1024);
+ ScopedString Str;
disable();
const uptr Length = getStats(&Str) + 1;
enable();
}
void printStats() {
- ScopedString Str(1024);
+ ScopedString Str;
disable();
getStats(&Str);
enable();
void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
void *Arg) {
initThreadMaybe();
+ if (archSupportsMemoryTagging())
+ Base = untagPointer(Base);
const uptr From = Base;
const uptr To = Base + Size;
- auto Lambda = [this, From, To, Callback, Arg](uptr Block) {
+ bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging<Params>() &&
+ systemSupportsMemoryTagging();
+ auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,
+ Arg](uptr Block) {
if (Block < From || Block >= To)
return;
uptr Chunk;
Chunk::UnpackedHeader Header;
- if (getChunkFromBlock(Block, &Chunk, &Header) &&
- Header.State == Chunk::State::Allocated) {
+ if (MayHaveTaggedPrimary) {
+ // A chunk header can either have a zero tag (tagged primary) or the
+ // header tag (secondary, or untagged primary). We don't know which so
+ // try both.
+ ScopedDisableMemoryTagChecks x;
+ if (!getChunkFromBlock(Block, &Chunk, &Header) &&
+ !getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header))
+ return;
+ } else {
+ if (!getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header))
+ return;
+ }
+ if (Header.State == Chunk::State::Allocated) {
uptr TaggedChunk = Chunk;
- if (useMemoryTagging())
+ if (allocatorSupportsMemoryTagging<Params>())
+ TaggedChunk = untagPointer(TaggedChunk);
+ if (useMemoryTagging<Params>(Primary.Options.load()))
TaggedChunk = loadTag(Chunk);
Callback(TaggedChunk, getSize(reinterpret_cast<void *>(Chunk), &Header),
Arg);
bool canReturnNull() {
initThreadMaybe();
- return Options.MayReturnNull;
+ return Primary.Options.load().get(OptionBit::MayReturnNull);
}
bool setOption(Option O, sptr Value) {
- if (O == Option::ReleaseInterval) {
- Primary.setReleaseToOsIntervalMs(static_cast<s32>(Value));
- Secondary.setReleaseToOsIntervalMs(static_cast<s32>(Value));
+ initThreadMaybe();
+ if (O == Option::MemtagTuning) {
+ // Enabling odd/even tags involves a tradeoff between use-after-free
+ // detection and buffer overflow detection. Odd/even tags make it more
+ // likely for buffer overflows to be detected by increasing the size of
+ // the guaranteed "red zone" around the allocation, but on the other hand
+ // use-after-free is less likely to be detected because the tag space for
+ // any particular chunk is cut in half. Therefore we use this tuning
+ // setting to control whether odd/even tags are enabled.
+ if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)
+ Primary.Options.set(OptionBit::UseOddEvenTags);
+ else if (Value == M_MEMTAG_TUNING_UAF)
+ Primary.Options.clear(OptionBit::UseOddEvenTags);
return true;
+ } else {
+ // We leave it to the various sub-components to decide whether or not they
+ // want to handle the option, but we do not want to short-circuit
+ // execution if one of the setOption was to return false.
+ const bool PrimaryResult = Primary.setOption(O, Value);
+ const bool SecondaryResult = Secondary.setOption(O, Value);
+ const bool RegistryResult = TSDRegistry.setOption(O, Value);
+ return PrimaryResult && SecondaryResult && RegistryResult;
}
return false;
}
return GuardedAlloc.getSize(Ptr);
#endif // GWP_ASAN_HOOKS
- Ptr = untagPointerMaybe(const_cast<void *>(Ptr));
+ Ptr = getHeaderTaggedPointer(const_cast<void *>(Ptr));
Chunk::UnpackedHeader Header;
Chunk::loadHeader(Cookie, Ptr, &Header);
// Getting the usable size of a chunk only makes sense if it's allocated.
#endif // GWP_ASAN_HOOKS
if (!Ptr || !isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment))
return false;
- Ptr = untagPointerMaybe(const_cast<void *>(Ptr));
+ Ptr = getHeaderTaggedPointer(const_cast<void *>(Ptr));
Chunk::UnpackedHeader Header;
return Chunk::isValid(Cookie, Ptr, &Header) &&
Header.State == Chunk::State::Allocated;
}
- bool useMemoryTagging() { return Primary.useMemoryTagging(); }
-
- void disableMemoryTagging() { Primary.disableMemoryTagging(); }
+ bool useMemoryTaggingTestOnly() const {
+ return useMemoryTagging<Params>(Primary.Options.load());
+ }
+ void disableMemoryTagging() {
+ // If we haven't been initialized yet, we need to initialize now in order to
+ // prevent a future call to initThreadMaybe() from enabling memory tagging
+ // based on feature detection. But don't call initThreadMaybe() because it
+ // may end up calling the allocator (via pthread_atfork, via the post-init
+ // callback), which may cause mappings to be created with memory tagging
+ // enabled.
+ TSDRegistry.initOnceMaybe(this);
+ if (allocatorSupportsMemoryTagging<Params>()) {
+ Secondary.disableMemoryTagging();
+ Primary.Options.clear(OptionBit::UseMemoryTagging);
+ }
+ }
void setTrackAllocationStacks(bool Track) {
initThreadMaybe();
- Options.TrackAllocationStacks = Track;
+ if (Track)
+ Primary.Options.set(OptionBit::TrackAllocationStacks);
+ else
+ Primary.Options.clear(OptionBit::TrackAllocationStacks);
}
void setFillContents(FillContentsMode FillContents) {
initThreadMaybe();
- Options.FillContents = FillContents;
+ Primary.Options.setFillContentsMode(FillContents);
+ }
+
+ void setAddLargeAllocationSlack(bool AddSlack) {
+ initThreadMaybe();
+ if (AddSlack)
+ Primary.Options.set(OptionBit::AddLargeAllocationSlack);
+ else
+ Primary.Options.clear(OptionBit::AddLargeAllocationSlack);
}
const char *getStackDepotAddress() const {
return PrimaryT::getRegionInfoArraySize();
}
+ const char *getRingBufferAddress() const {
+ return reinterpret_cast<const char *>(&RingBuffer);
+ }
+
+ static uptr getRingBufferSize() { return sizeof(RingBuffer); }
+
+ static const uptr MaxTraceSize = 64;
+
+ static void collectTraceMaybe(const StackDepot *Depot,
+ uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {
+ uptr RingPos, Size;
+ if (!Depot->find(Hash, &RingPos, &Size))
+ return;
+ for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
+ Trace[I] = (*Depot)[RingPos + I];
+ }
+
static void getErrorInfo(struct scudo_error_info *ErrorInfo,
uintptr_t FaultAddr, const char *DepotPtr,
- const char *RegionInfoPtr, const char *Memory,
- const char *MemoryTags, uintptr_t MemoryAddr,
- size_t MemorySize) {
+ const char *RegionInfoPtr, const char *RingBufferPtr,
+ const char *Memory, const char *MemoryTags,
+ uintptr_t MemoryAddr, size_t MemorySize) {
*ErrorInfo = {};
- if (!PrimaryT::SupportsMemoryTagging ||
+ if (!allocatorSupportsMemoryTagging<Params>() ||
MemoryAddr + MemorySize < MemoryAddr)
return;
- uptr UntaggedFaultAddr = untagPointer(FaultAddr);
- u8 FaultAddrTag = extractTag(FaultAddr);
- BlockInfo Info =
- PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
-
- auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
- if (Addr < MemoryAddr ||
- Addr + archMemoryTagGranuleSize() < Addr ||
- Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
- return false;
- *Data = &Memory[Addr - MemoryAddr];
- *Tag = static_cast<u8>(
- MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
- return true;
- };
-
- auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
- Chunk::UnpackedHeader *Header, const u32 **Data,
- u8 *Tag) {
- const char *BlockBegin;
- u8 BlockBeginTag;
- if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
- return false;
- uptr ChunkOffset = getChunkOffsetFromBlock(BlockBegin);
- *ChunkAddr = Addr + ChunkOffset;
-
- const char *ChunkBegin;
- if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
- return false;
- *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
- ChunkBegin - Chunk::getHeaderSize());
- *Data = reinterpret_cast<const u32 *>(ChunkBegin);
- return true;
- };
-
auto *Depot = reinterpret_cast<const StackDepot *>(DepotPtr);
-
- auto MaybeCollectTrace = [&](uintptr_t(&Trace)[MaxTraceSize], u32 Hash) {
- uptr RingPos, Size;
- if (!Depot->find(Hash, &RingPos, &Size))
- return;
- for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)
- Trace[I] = (*Depot)[RingPos + I];
- };
-
size_t NextErrorReport = 0;
- // First, check for UAF.
- {
- uptr ChunkAddr;
- Chunk::UnpackedHeader Header;
- const u32 *Data;
- uint8_t Tag;
- if (ReadBlock(Info.BlockBegin, &ChunkAddr, &Header, &Data, &Tag) &&
- Header.State != Chunk::State::Allocated &&
- Data[MemTagPrevTagIndex] == FaultAddrTag) {
- auto *R = &ErrorInfo->reports[NextErrorReport++];
- R->error_type = USE_AFTER_FREE;
- R->allocation_address = ChunkAddr;
- R->allocation_size = Header.SizeOrUnusedBytes;
- MaybeCollectTrace(R->allocation_trace,
- Data[MemTagAllocationTraceIndex]);
- R->allocation_tid = Data[MemTagAllocationTidIndex];
- MaybeCollectTrace(R->deallocation_trace,
- Data[MemTagDeallocationTraceIndex]);
- R->deallocation_tid = Data[MemTagDeallocationTidIndex];
- }
- }
-
- auto CheckOOB = [&](uptr BlockAddr) {
- if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
- return false;
-
- uptr ChunkAddr;
- Chunk::UnpackedHeader Header;
- const u32 *Data;
- uint8_t Tag;
- if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
- Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
- return false;
-
- auto *R = &ErrorInfo->reports[NextErrorReport++];
- R->error_type =
- UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
- R->allocation_address = ChunkAddr;
- R->allocation_size = Header.SizeOrUnusedBytes;
- MaybeCollectTrace(R->allocation_trace, Data[MemTagAllocationTraceIndex]);
- R->allocation_tid = Data[MemTagAllocationTidIndex];
- return NextErrorReport ==
- sizeof(ErrorInfo->reports) / sizeof(ErrorInfo->reports[0]);
- };
-
- if (CheckOOB(Info.BlockBegin))
- return;
-
- // Check for OOB in the 30 surrounding blocks. Beyond that we are likely to
- // hit false positives.
- for (int I = 1; I != 16; ++I)
- if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
- CheckOOB(Info.BlockBegin - I * Info.BlockSize))
- return;
+ // Check for OOB in the current block and the two surrounding blocks. Beyond
+ // that, UAF is more likely.
+ if (extractTag(FaultAddr) != 0)
+ getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
+ MemorySize, 0, 2);
+
+ // Check the ring buffer. For primary allocations this will only find UAF;
+ // for secondary allocations we can find either UAF or OOB.
+ getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ RingBufferPtr);
+
+ // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.
+ // Beyond that we are likely to hit false positives.
+ if (extractTag(FaultAddr) != 0)
+ getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,
+ RegionInfoPtr, Memory, MemoryTags, MemoryAddr,
+ MemorySize, 2, 16);
}
private:
- using SecondaryT = typename Params::Secondary;
+ using SecondaryT = MapAllocator<Params>;
typedef typename PrimaryT::SizeClassMap SizeClassMap;
static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),
"Minimal alignment must at least cover a chunk header.");
- static_assert(!PrimaryT::SupportsMemoryTagging ||
+ static_assert(!allocatorSupportsMemoryTagging<Params>() ||
MinAlignment >= archMemoryTagGranuleSize(),
"");
// These are indexes into an "array" of 32-bit values that store information
// inline with a chunk that is relevant to diagnosing memory tag faults, where
- // 0 corresponds to the address of the user memory. This means that negative
- // indexes may be used to store information about allocations, while positive
- // indexes may only be used to store information about deallocations, because
- // the user memory is in use until it has been deallocated. The smallest index
- // that may be used is -2, which corresponds to 8 bytes before the user
- // memory, because the chunk header size is 8 bytes and in allocators that
- // support memory tagging the minimum alignment is at least the tag granule
- // size (16 on aarch64), and the largest index that may be used is 3 because
- // we are only guaranteed to have at least a granule's worth of space in the
- // user memory.
+ // 0 corresponds to the address of the user memory. This means that only
+ // negative indexes may be used. The smallest index that may be used is -2,
+ // which corresponds to 8 bytes before the user memory, because the chunk
+ // header size is 8 bytes and in allocators that support memory tagging the
+ // minimum alignment is at least the tag granule size (16 on aarch64).
static const sptr MemTagAllocationTraceIndex = -2;
static const sptr MemTagAllocationTidIndex = -1;
- static const sptr MemTagDeallocationTraceIndex = 0;
- static const sptr MemTagDeallocationTidIndex = 1;
- static const sptr MemTagPrevTagIndex = 2;
- static const uptr MaxTraceSize = 64;
+ u32 Cookie = 0;
+ u32 QuarantineMaxChunkSize = 0;
GlobalStats Stats;
- TSDRegistryT TSDRegistry;
PrimaryT Primary;
SecondaryT Secondary;
QuarantineT Quarantine;
-
- u32 Cookie;
-
- struct {
- u8 MayReturnNull : 1; // may_return_null
- FillContentsMode FillContents : 2; // zero_contents, pattern_fill_contents
- u8 DeallocTypeMismatch : 1; // dealloc_type_mismatch
- u8 DeleteSizeMismatch : 1; // delete_size_mismatch
- u8 TrackAllocationStacks : 1;
- u32 QuarantineMaxChunkSize; // quarantine_max_chunk_size
- } Options;
+ TSDRegistryT TSDRegistry;
+ pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;
#ifdef GWP_ASAN_HOOKS
gwp_asan::GuardedPoolAllocator GuardedAlloc;
+ uptr GuardedAllocSlotSize = 0;
#endif // GWP_ASAN_HOOKS
StackDepot Depot;
+ struct AllocationRingBuffer {
+ struct Entry {
+ atomic_uptr Ptr;
+ atomic_uptr AllocationSize;
+ atomic_u32 AllocationTrace;
+ atomic_u32 AllocationTid;
+ atomic_u32 DeallocationTrace;
+ atomic_u32 DeallocationTid;
+ };
+
+ atomic_uptr Pos;
+#ifdef SCUDO_FUZZ
+ static const uptr NumEntries = 2;
+#else
+ static const uptr NumEntries = 32768;
+#endif
+ Entry Entries[NumEntries];
+ };
+ AllocationRingBuffer RingBuffer = {};
+
// The following might get optimized out by the compiler.
NOINLINE void performSanityChecks() {
// Verify that the header offset field can hold the maximum offset. In the
const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
if (LIKELY(Header->ClassId))
return SizeOrUnusedBytes;
+ if (allocatorSupportsMemoryTagging<Params>())
+ Ptr = untagPointer(const_cast<void *>(Ptr));
return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -
reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;
}
- ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {
- TSDRegistry.initThreadMaybe(this, MinimalInit);
- }
-
- void quarantineOrDeallocateChunk(void *Ptr, Chunk::UnpackedHeader *Header,
- uptr Size) {
+ void quarantineOrDeallocateChunk(Options Options, void *TaggedPtr,
+ Chunk::UnpackedHeader *Header, uptr Size) {
+ void *Ptr = getHeaderTaggedPointer(TaggedPtr);
Chunk::UnpackedHeader NewHeader = *Header;
- if (UNLIKELY(NewHeader.ClassId && useMemoryTagging())) {
- u8 PrevTag = extractTag(loadTag(reinterpret_cast<uptr>(Ptr)));
- uptr TaggedBegin, TaggedEnd;
- // Exclude the previous tag so that immediate use after free is detected
- // 100% of the time.
- setRandomTag(Ptr, Size, 1UL << PrevTag, &TaggedBegin, &TaggedEnd);
- storeDeallocationStackMaybe(Ptr, PrevTag);
- }
// If the quarantine is disabled, the actual size of a chunk is 0 or larger
// than the maximum allowed, we return a chunk directly to the backend.
- // Logical Or can be short-circuited, which introduces unnecessary
- // conditional jumps, so use bitwise Or and let the compiler be clever.
- const bool BypassQuarantine = !Quarantine.getCacheSize() | !Size |
- (Size > Options.QuarantineMaxChunkSize);
- if (BypassQuarantine) {
+ // This purposefully underflows for Size == 0.
+ const bool BypassQuarantine = !Quarantine.getCacheSize() ||
+ ((Size - 1) >= QuarantineMaxChunkSize) ||
+ !NewHeader.ClassId;
+ if (BypassQuarantine)
NewHeader.State = Chunk::State::Available;
- Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
+ else
+ NewHeader.State = Chunk::State::Quarantined;
+ NewHeader.OriginOrWasZeroed = useMemoryTagging<Params>(Options) &&
+ NewHeader.ClassId &&
+ !TSDRegistry.getDisableMemInit();
+ Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
+
+ if (UNLIKELY(useMemoryTagging<Params>(Options))) {
+ u8 PrevTag = extractTag(reinterpret_cast<uptr>(TaggedPtr));
+ storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);
+ if (NewHeader.ClassId) {
+ if (!TSDRegistry.getDisableMemInit()) {
+ uptr TaggedBegin, TaggedEnd;
+ const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(
+ Options, reinterpret_cast<uptr>(getBlockBegin(Ptr, &NewHeader)),
+ NewHeader.ClassId);
+ // Exclude the previous tag so that immediate use after free is
+ // detected 100% of the time.
+ setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,
+ &TaggedEnd);
+ }
+ }
+ }
+ if (BypassQuarantine) {
+ if (allocatorSupportsMemoryTagging<Params>())
+ Ptr = untagPointer(Ptr);
void *BlockBegin = getBlockBegin(Ptr, &NewHeader);
const uptr ClassId = NewHeader.ClassId;
if (LIKELY(ClassId)) {
if (UnlockRequired)
TSD->unlock();
} else {
- Secondary.deallocate(BlockBegin);
+ if (UNLIKELY(useMemoryTagging<Params>(Options)))
+ storeTags(reinterpret_cast<uptr>(BlockBegin),
+ reinterpret_cast<uptr>(Ptr));
+ Secondary.deallocate(Options, BlockBegin);
}
} else {
- NewHeader.State = Chunk::State::Quarantined;
- Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
bool UnlockRequired;
auto *TSD = TSDRegistry.getTSDAndLock(&UnlockRequired);
Quarantine.put(&TSD->QuarantineCache,
return Offset + Chunk::getHeaderSize();
}
- void storeAllocationStackMaybe(void *Ptr) {
- if (!UNLIKELY(Options.TrackAllocationStacks))
+ // Set the tag of the granule past the end of the allocation to 0, to catch
+ // linear overflows even if a previous larger allocation used the same block
+ // and tag. Only do this if the granule past the end is in our block, because
+ // this would otherwise lead to a SEGV if the allocation covers the entire
+ // block and our block is at the end of a mapping. The tag of the next block's
+ // header granule will be set to 0, so it will serve the purpose of catching
+ // linear overflows in this case.
+ //
+ // For allocations of size 0 we do not end up storing the address tag to the
+ // memory tag space, which getInlineErrorInfo() normally relies on to match
+ // address tags against chunks. To allow matching in this case we store the
+ // address tag in the first byte of the chunk.
+ void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {
+ DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));
+ uptr UntaggedEnd = untagPointer(End);
+ if (UntaggedEnd != BlockEnd) {
+ storeTag(UntaggedEnd);
+ if (Size == 0)
+ *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(End);
+ }
+ }
+
+ void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
+ uptr BlockEnd) {
+ // Prepare the granule before the chunk to store the chunk header by setting
+ // its tag to 0. Normally its tag will already be 0, but in the case where a
+ // chunk holding a low alignment allocation is reused for a higher alignment
+ // allocation, the chunk may already have a non-zero tag from the previous
+ // allocation.
+ storeTag(reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());
+
+ uptr TaggedBegin, TaggedEnd;
+ setRandomTag(Ptr, Size, ExcludeMask, &TaggedBegin, &TaggedEnd);
+
+ storeEndMarker(TaggedEnd, Size, BlockEnd);
+ return reinterpret_cast<void *>(TaggedBegin);
+ }
+
+ void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,
+ uptr BlockEnd) {
+ uptr RoundOldPtr = roundUpTo(OldPtr, archMemoryTagGranuleSize());
+ uptr RoundNewPtr;
+ if (RoundOldPtr >= NewPtr) {
+ // If the allocation is shrinking we just need to set the tag past the end
+ // of the allocation to 0. See explanation in storeEndMarker() above.
+ RoundNewPtr = roundUpTo(NewPtr, archMemoryTagGranuleSize());
+ } else {
+ // Set the memory tag of the region
+ // [RoundOldPtr, roundUpTo(NewPtr, archMemoryTagGranuleSize()))
+ // to the pointer tag stored in OldPtr.
+ RoundNewPtr = storeTags(RoundOldPtr, NewPtr);
+ }
+ storeEndMarker(RoundNewPtr, NewSize, BlockEnd);
+ }
+
+ void storePrimaryAllocationStackMaybe(Options Options, void *Ptr) {
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
return;
auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
Ptr32[MemTagAllocationTraceIndex] = collectStackTrace();
Ptr32[MemTagAllocationTidIndex] = getThreadID();
}
- void storeDeallocationStackMaybe(void *Ptr, uint8_t PrevTag) {
- if (!UNLIKELY(Options.TrackAllocationStacks))
+ void storeRingBufferEntry(void *Ptr, u32 AllocationTrace, u32 AllocationTid,
+ uptr AllocationSize, u32 DeallocationTrace,
+ u32 DeallocationTid) {
+ uptr Pos = atomic_fetch_add(&RingBuffer.Pos, 1, memory_order_relaxed);
+ typename AllocationRingBuffer::Entry *Entry =
+ &RingBuffer.Entries[Pos % AllocationRingBuffer::NumEntries];
+
+ // First invalidate our entry so that we don't attempt to interpret a
+ // partially written state in getSecondaryErrorInfo(). The fences below
+ // ensure that the compiler does not move the stores to Ptr in between the
+ // stores to the other fields.
+ atomic_store_relaxed(&Entry->Ptr, 0);
+
+ __atomic_signal_fence(__ATOMIC_SEQ_CST);
+ atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);
+ atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);
+ atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);
+ atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);
+ atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);
+ __atomic_signal_fence(__ATOMIC_SEQ_CST);
+
+ atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));
+ }
+
+ void storeSecondaryAllocationStackMaybe(Options Options, void *Ptr,
+ uptr Size) {
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
+ return;
+
+ u32 Trace = collectStackTrace();
+ u32 Tid = getThreadID();
+
+ auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
+ Ptr32[MemTagAllocationTraceIndex] = Trace;
+ Ptr32[MemTagAllocationTidIndex] = Tid;
+
+ storeRingBufferEntry(untagPointer(Ptr), Trace, Tid, Size, 0, 0);
+ }
+
+ void storeDeallocationStackMaybe(Options Options, void *Ptr, u8 PrevTag,
+ uptr Size) {
+ if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))
return;
- // Disable tag checks here so that we don't need to worry about zero sized
- // allocations.
- ScopedDisableMemoryTagChecks x;
auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);
- Ptr32[MemTagDeallocationTraceIndex] = collectStackTrace();
- Ptr32[MemTagDeallocationTidIndex] = getThreadID();
- Ptr32[MemTagPrevTagIndex] = PrevTag;
+ u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];
+ u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];
+
+ u32 DeallocationTrace = collectStackTrace();
+ u32 DeallocationTid = getThreadID();
+
+ storeRingBufferEntry(addFixedTag(untagPointer(Ptr), PrevTag),
+ AllocationTrace, AllocationTid, Size,
+ DeallocationTrace, DeallocationTid);
+ }
+
+ static const size_t NumErrorReports =
+ sizeof(((scudo_error_info *)0)->reports) /
+ sizeof(((scudo_error_info *)0)->reports[0]);
+
+ static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,
+ size_t &NextErrorReport, uintptr_t FaultAddr,
+ const StackDepot *Depot,
+ const char *RegionInfoPtr, const char *Memory,
+ const char *MemoryTags, uintptr_t MemoryAddr,
+ size_t MemorySize, size_t MinDistance,
+ size_t MaxDistance) {
+ uptr UntaggedFaultAddr = untagPointer(FaultAddr);
+ u8 FaultAddrTag = extractTag(FaultAddr);
+ BlockInfo Info =
+ PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);
+
+ auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {
+ if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||
+ Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)
+ return false;
+ *Data = &Memory[Addr - MemoryAddr];
+ *Tag = static_cast<u8>(
+ MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);
+ return true;
+ };
+
+ auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,
+ Chunk::UnpackedHeader *Header, const u32 **Data,
+ u8 *Tag) {
+ const char *BlockBegin;
+ u8 BlockBeginTag;
+ if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))
+ return false;
+ uptr ChunkOffset = getChunkOffsetFromBlock(BlockBegin);
+ *ChunkAddr = Addr + ChunkOffset;
+
+ const char *ChunkBegin;
+ if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))
+ return false;
+ *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(
+ ChunkBegin - Chunk::getHeaderSize());
+ *Data = reinterpret_cast<const u32 *>(ChunkBegin);
+
+ // Allocations of size 0 will have stashed the tag in the first byte of
+ // the chunk, see storeEndMarker().
+ if (Header->SizeOrUnusedBytes == 0)
+ *Tag = static_cast<u8>(*ChunkBegin);
+
+ return true;
+ };
+
+ if (NextErrorReport == NumErrorReports)
+ return;
+
+ auto CheckOOB = [&](uptr BlockAddr) {
+ if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)
+ return false;
+
+ uptr ChunkAddr;
+ Chunk::UnpackedHeader Header;
+ const u32 *Data;
+ uint8_t Tag;
+ if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||
+ Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)
+ return false;
+
+ auto *R = &ErrorInfo->reports[NextErrorReport++];
+ R->error_type =
+ UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;
+ R->allocation_address = ChunkAddr;
+ R->allocation_size = Header.SizeOrUnusedBytes;
+ collectTraceMaybe(Depot, R->allocation_trace,
+ Data[MemTagAllocationTraceIndex]);
+ R->allocation_tid = Data[MemTagAllocationTidIndex];
+ return NextErrorReport == NumErrorReports;
+ };
+
+ if (MinDistance == 0 && CheckOOB(Info.BlockBegin))
+ return;
+
+ for (size_t I = Max<size_t>(MinDistance, 1); I != MaxDistance; ++I)
+ if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||
+ CheckOOB(Info.BlockBegin - I * Info.BlockSize))
+ return;
+ }
+
+ static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,
+ size_t &NextErrorReport,
+ uintptr_t FaultAddr,
+ const StackDepot *Depot,
+ const char *RingBufferPtr) {
+ auto *RingBuffer =
+ reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);
+ uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);
+
+ for (uptr I = Pos - 1; I != Pos - 1 - AllocationRingBuffer::NumEntries &&
+ NextErrorReport != NumErrorReports;
+ --I) {
+ auto *Entry = &RingBuffer->Entries[I % AllocationRingBuffer::NumEntries];
+ uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);
+ if (!EntryPtr)
+ continue;
+
+ uptr UntaggedEntryPtr = untagPointer(EntryPtr);
+ uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);
+ u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);
+ u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);
+ u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);
+ u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);
+
+ if (DeallocationTid) {
+ // For UAF we only consider in-bounds fault addresses because
+ // out-of-bounds UAF is rare and attempting to detect it is very likely
+ // to result in false positives.
+ if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)
+ continue;
+ } else {
+ // Ring buffer OOB is only possible with secondary allocations. In this
+ // case we are guaranteed a guard region of at least a page on either
+ // side of the allocation (guard page on the right, guard page + tagged
+ // region on the left), so ignore any faults outside of that range.
+ if (FaultAddr < EntryPtr - getPageSizeCached() ||
+ FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())
+ continue;
+
+ // For UAF the ring buffer will contain two entries, one for the
+ // allocation and another for the deallocation. Don't report buffer
+ // overflow/underflow using the allocation entry if we have already
+ // collected a report from the deallocation entry.
+ bool Found = false;
+ for (uptr J = 0; J != NextErrorReport; ++J) {
+ if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {
+ Found = true;
+ break;
+ }
+ }
+ if (Found)
+ continue;
+ }
+
+ auto *R = &ErrorInfo->reports[NextErrorReport++];
+ if (DeallocationTid)
+ R->error_type = USE_AFTER_FREE;
+ else if (FaultAddr < EntryPtr)
+ R->error_type = BUFFER_UNDERFLOW;
+ else
+ R->error_type = BUFFER_OVERFLOW;
+
+ R->allocation_address = UntaggedEntryPtr;
+ R->allocation_size = EntrySize;
+ collectTraceMaybe(Depot, R->allocation_trace, AllocationTrace);
+ R->allocation_tid = AllocationTid;
+ collectTraceMaybe(Depot, R->deallocation_trace, DeallocationTrace);
+ R->deallocation_tid = DeallocationTid;
+ }
}
uptr getStats(ScopedString *Str) {
#include "common.h"
#include "atomic_helpers.h"
+#include "string_utils.h"
namespace scudo {
}
// Fatal internal map() or unmap() error (potentially OOM related).
-void NORETURN dieOnMapUnmapError(bool OutOfMemory) {
- outputRaw("Scudo ERROR: internal map or unmap failure");
- if (OutOfMemory)
- outputRaw(" (OOM)");
- outputRaw("\n");
+void NORETURN dieOnMapUnmapError(uptr SizeIfOOM) {
+ char Error[128] = "Scudo ERROR: internal map or unmap failure\n";
+ if (SizeIfOOM) {
+ formatString(
+ Error, sizeof(Error),
+ "Scudo ERROR: internal map failure (NO MEMORY) requesting %zuKB\n",
+ SizeIfOOM >> 10);
+ }
+ outputRaw(Error);
+ setAbortMessage(Error);
die();
}
#include "fuchsia.h"
#include "linux.h"
+#include "trusty.h"
#include <stddef.h>
#include <string.h>
void unmap(void *Addr, uptr Size, uptr Flags = 0,
MapPlatformData *Data = nullptr);
+void setMemoryPermission(uptr Addr, uptr Size, uptr Flags,
+ MapPlatformData *Data = nullptr);
+
void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
MapPlatformData *Data = nullptr);
-// Internal map & unmap fatal error. This must not call map().
-void NORETURN dieOnMapUnmapError(bool OutOfMemory = false);
+// Internal map & unmap fatal error. This must not call map(). SizeIfOOM shall
+// hold the requested size on an out-of-memory error, 0 otherwise.
+void NORETURN dieOnMapUnmapError(uptr SizeIfOOM = 0);
// Logging related functions.
uptr RegionEnd;
};
+enum class Option : u8 {
+ ReleaseInterval, // Release to OS interval in milliseconds.
+ MemtagTuning, // Whether to tune tagging for UAF or overflow.
+ ThreadDisableMemInit, // Whether to disable automatic heap initialization and,
+ // where possible, memory tagging, on this thread.
+ MaxCacheEntriesCount, // Maximum number of blocks that can be cached.
+ MaxCacheEntrySize, // Maximum size of a block that can be cached.
+ MaxTSDsCount, // Number of usable TSDs for the shared registry.
+};
+
constexpr unsigned char PatternFillByte = 0xAB;
enum FillContentsMode {
SCUDO_FLAG(bool, pattern_fill_contents, false,
"Pattern fill chunk contents on allocation.")
-SCUDO_FLAG(int, rss_limit_mb, -1,
- "Enforce an upper limit (in megabytes) to the process RSS. The "
- "allocator will terminate or return NULL when allocations are "
- "attempted past that limit (depending on may_return_null). Negative "
- "values disable the feature.")
-
SCUDO_FLAG(bool, may_return_null, true,
"Indicate whether the allocator should terminate instead of "
"returning NULL in otherwise non-fatal error scenarios, eg: OOM, "
void printFlagDescriptions();
private:
- static const u32 MaxFlags = 16;
+ static const u32 MaxFlags = 20;
struct Flag {
const char *Name;
const char *Desc;
#include "string_utils.h"
#include <lib/sync/mutex.h> // for sync_mutex_t
-#include <limits.h> // for PAGE_SIZE
#include <stdlib.h> // for getenv()
#include <zircon/compiler.h>
#include <zircon/sanitizer.h>
namespace scudo {
-uptr getPageSize() { return PAGE_SIZE; }
+uptr getPageSize() { return _zx_system_get_page_size(); }
void NORETURN die() { __builtin_trap(); }
Size, &Data->Vmar, &Data->VmarBase);
if (UNLIKELY(Status != ZX_OK)) {
if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
- dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
+ dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
return nullptr;
}
return reinterpret_cast<void *>(Data->VmarBase);
void *map(void *Addr, uptr Size, const char *Name, uptr Flags,
MapPlatformData *Data) {
- DCHECK_EQ(Size % PAGE_SIZE, 0);
+ DCHECK_EQ(Size % getPageSizeCached(), 0);
const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM);
// For MAP_NOACCESS, just allocate a Vmar and return.
Status = _zx_vmo_set_size(Vmo, VmoSize + Size);
if (Status != ZX_OK) {
if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
- dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
+ dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
return nullptr;
}
} else {
Status = _zx_vmo_create(Size, ZX_VMO_RESIZABLE, &Vmo);
if (UNLIKELY(Status != ZX_OK)) {
if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
- dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
+ dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
return nullptr;
}
_zx_object_set_property(Vmo, ZX_PROP_NAME, Name, strlen(Name));
// No need to track the Vmo if we don't intend on resizing it. Close it.
if (Flags & MAP_RESIZABLE) {
DCHECK(Data);
- DCHECK_EQ(Data->Vmo, ZX_HANDLE_INVALID);
- Data->Vmo = Vmo;
+ if (Data->Vmo == ZX_HANDLE_INVALID)
+ Data->Vmo = Vmo;
+ else
+ DCHECK_EQ(Data->Vmo, Vmo);
} else {
CHECK_EQ(_zx_handle_close(Vmo), ZX_OK);
}
if (UNLIKELY(Status != ZX_OK)) {
if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem)
- dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY);
+ dieOnMapUnmapError(Status == ZX_ERR_NO_MEMORY ? Size : 0);
return nullptr;
}
if (Data)
}
}
+void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags,
+ UNUSED MapPlatformData *Data) {
+ const zx_vm_option_t Prot =
+ (Flags & MAP_NOACCESS) ? 0 : (ZX_VM_PERM_READ | ZX_VM_PERM_WRITE);
+ DCHECK(Data);
+ DCHECK_NE(Data->Vmar, ZX_HANDLE_INVALID);
+ if (_zx_vmar_protect(Data->Vmar, Prot, Addr, Size) != ZX_OK)
+ dieOnMapUnmapError();
+}
+
void releasePagesToOS(UNUSED uptr BaseAddress, uptr Offset, uptr Size,
MapPlatformData *Data) {
DCHECK(Data);
uintptr_t FaultAddr = FDP.ConsumeIntegral<uintptr_t>();
uintptr_t MemoryAddr = FDP.ConsumeIntegral<uintptr_t>();
- std::string MemoryAndTags = FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
+ std::string MemoryAndTags =
+ FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
const char *Memory = MemoryAndTags.c_str();
// Assume 16-byte alignment.
size_t MemorySize = (MemoryAndTags.length() / 17) * 16;
const char *MemoryTags = Memory + MemorySize;
- std::string StackDepotBytes = FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
+ std::string StackDepotBytes =
+ FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
std::vector<char> StackDepot(sizeof(scudo::StackDepot), 0);
- for (size_t i = 0; i < StackDepotBytes.length() && i < StackDepot.size(); ++i) {
+ for (size_t i = 0; i < StackDepotBytes.length() && i < StackDepot.size();
+ ++i) {
StackDepot[i] = StackDepotBytes[i];
}
- std::string RegionInfoBytes = FDP.ConsumeRemainingBytesAsString();
+ std::string RegionInfoBytes =
+ FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
std::vector<char> RegionInfo(AllocatorT::getRegionInfoArraySize(), 0);
- for (size_t i = 0; i < RegionInfoBytes.length() && i < RegionInfo.size(); ++i) {
+ for (size_t i = 0; i < RegionInfoBytes.length() && i < RegionInfo.size();
+ ++i) {
RegionInfo[i] = RegionInfoBytes[i];
}
+ std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString();
+ std::vector<char> RingBuffer(AllocatorT::getRingBufferSize(), 0);
+ for (size_t i = 0; i < RingBufferBytes.length() && i < RingBuffer.size();
+ ++i) {
+ RingBuffer[i] = RingBufferBytes[i];
+ }
+
scudo_error_info ErrorInfo;
AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(),
- RegionInfo.data(), Memory, MemoryTags, MemoryAddr,
- MemorySize);
+ RegionInfo.data(), RingBuffer.data(), Memory,
+ MemoryTags, MemoryAddr, MemorySize);
return 0;
}
#define SCUDO_INTERFACE_H_
#include <stddef.h>
+#include <stdint.h>
extern "C" {
// the version in the process that analyzes the crash.
//
// fault_addr is the fault address. On aarch64 this is available in the system
-// register FAR_ELx, or far_context.far in an upcoming release of the Linux
-// kernel. This address must include the pointer tag; note that the kernel
-// strips the tag from the fields siginfo.si_addr and sigcontext.fault_address,
-// so these addresses are not suitable to be passed as fault_addr.
+// register FAR_ELx, or siginfo.si_addr in Linux 5.11 or above. This address
+// must include the pointer tag; this is available if SA_EXPOSE_TAGBITS was set
+// in sigaction.sa_flags when the signal handler was registered. Note that the
+// kernel strips the tag from the field sigcontext.fault_address, so this
+// address is not suitable to be passed as fault_addr.
//
// stack_depot is a pointer to the stack depot data structure, which may be
// obtained by calling the function __scudo_get_stack_depot_addr() in the
// pointer.
void __scudo_get_error_info(struct scudo_error_info *error_info,
uintptr_t fault_addr, const char *stack_depot,
- const char *region_info, const char *memory,
- const char *memory_tags, uintptr_t memory_addr,
- size_t memory_size);
+ const char *region_info, const char *ring_buffer,
+ const char *memory, const char *memory_tags,
+ uintptr_t memory_addr, size_t memory_size);
enum scudo_error_type {
UNKNOWN,
const char *__scudo_get_region_info_addr();
size_t __scudo_get_region_info_size();
+const char *__scudo_get_ring_buffer_addr();
+size_t __scudo_get_ring_buffer_size();
+
+#ifndef M_DECAY_TIME
+#define M_DECAY_TIME -100
+#endif
+
+#ifndef M_PURGE
+#define M_PURGE -101
+#endif
+
+// Tune the allocator's choice of memory tags to make it more likely that
+// a certain class of memory errors will be detected. The value argument should
+// be one of the M_MEMTAG_TUNING_* constants below.
+#ifndef M_MEMTAG_TUNING
+#define M_MEMTAG_TUNING -102
+#endif
+
+// Per-thread memory initialization tuning. The value argument should be one of:
+// 1: Disable automatic heap initialization and, where possible, memory tagging,
+// on this thread.
+// 0: Normal behavior.
+#ifndef M_THREAD_DISABLE_MEM_INIT
+#define M_THREAD_DISABLE_MEM_INIT -103
+#endif
+
+#ifndef M_CACHE_COUNT_MAX
+#define M_CACHE_COUNT_MAX -200
+#endif
+
+#ifndef M_CACHE_SIZE_MAX
+#define M_CACHE_SIZE_MAX -201
+#endif
+
+#ifndef M_TSDS_COUNT_MAX
+#define M_TSDS_COUNT_MAX -202
+#endif
+
+// Tune for buffer overflows.
+#ifndef M_MEMTAG_TUNING_BUFFER_OVERFLOW
+#define M_MEMTAG_TUNING_BUFFER_OVERFLOW 0
+#endif
+
+// Tune for use-after-free.
+#ifndef M_MEMTAG_TUNING_UAF
+#define M_MEMTAG_TUNING_UAF 1
+#endif
+
} // extern "C"
#endif // SCUDO_INTERFACE_H_
#define FORMAT(F, A) __attribute__((format(printf, F, A)))
#define NOINLINE __attribute__((noinline))
#define NORETURN __attribute__((noreturn))
-#define THREADLOCAL __thread
#define LIKELY(X) __builtin_expect(!!(X), 1)
#define UNLIKELY(X) __builtin_expect(!!(X), 0)
#if defined(__i386__) || defined(__x86_64__)
#define USED __attribute__((used))
#define NOEXCEPT noexcept
+// This check is only available on Clang. This is essentially an alias of
+// C++20's 'constinit' specifier which will take care of this when (if?) we can
+// ask all libc's that use Scudo to compile us with C++20. Dynamic
+// initialization is bad; Scudo is designed to be lazy-initializated on the
+// first call to malloc/free (and friends), and this generally happens in the
+// loader somewhere in libdl's init. After the loader is done, control is
+// transferred to libc's initialization, and the dynamic initializers are run.
+// If there's a dynamic initializer for Scudo, then it will clobber the
+// already-initialized Scudo, and re-initialize all its members back to default
+// values, causing various explosions. Unfortunately, marking
+// scudo::Allocator<>'s constructor as 'constexpr' isn't sufficient to prevent
+// dynamic initialization, as default initialization is fine under 'constexpr'
+// (but not 'constinit'). Clang at -O0, and gcc at all opt levels will emit a
+// dynamic initializer for any constant-initialized variables if there is a mix
+// of default-initialized and constant-initialized variables.
+//
+// If you're looking at this because your build failed, you probably introduced
+// a new member to scudo::Allocator<> (possibly transiently) that didn't have an
+// initializer. The fix is easy - just add one.
+#if defined(__has_attribute)
+#if __has_attribute(require_constant_initialization)
+#define SCUDO_REQUIRE_CONSTANT_INITIALIZATION \
+ __attribute__((__require_constant_initialization__))
+#else
+#define SCUDO_REQUIRE_CONSTANT_INITIALIZATION
+#endif
+#endif
+
namespace scudo {
typedef unsigned long uptr;
void NORETURN reportCheckFailed(const char *File, int Line,
const char *Condition, u64 Value1, u64 Value2);
-
#define CHECK_IMPL(C1, Op, C2) \
do { \
- scudo::u64 V1 = (scudo::u64)(C1); \
- scudo::u64 V2 = (scudo::u64)(C2); \
- if (UNLIKELY(!(V1 Op V2))) { \
- scudo::reportCheckFailed(__FILE__, __LINE__, \
- "(" #C1 ") " #Op " (" #C2 ")", V1, V2); \
+ if (UNLIKELY(!(C1 Op C2))) { \
+ scudo::reportCheckFailed(__FILE__, __LINE__, #C1 " " #Op " " #C2, \
+ (scudo::u64)C1, (scudo::u64)C2); \
scudo::die(); \
} \
} while (false)
#define DCHECK_GT(A, B) CHECK_GT(A, B)
#define DCHECK_GE(A, B) CHECK_GE(A, B)
#else
-#define DCHECK(A)
-#define DCHECK_EQ(A, B)
-#define DCHECK_NE(A, B)
-#define DCHECK_LT(A, B)
-#define DCHECK_LE(A, B)
-#define DCHECK_GT(A, B)
-#define DCHECK_GE(A, B)
+#define DCHECK(A) \
+ do { \
+ } while (false)
+#define DCHECK_EQ(A, B) \
+ do { \
+ } while (false)
+#define DCHECK_NE(A, B) \
+ do { \
+ } while (false)
+#define DCHECK_LT(A, B) \
+ do { \
+ } while (false)
+#define DCHECK_LE(A, B) \
+ do { \
+ } while (false)
+#define DCHECK_GT(A, B) \
+ do { \
+ } while (false)
+#define DCHECK_GE(A, B) \
+ do { \
+ } while (false)
#endif
// The superfluous die() call effectively makes this macro NORETURN.
#define ANDROID_PR_SET_VMA_ANON_NAME 0
#endif
-#ifdef ANDROID_EXPERIMENTAL_MTE
-#include <bionic/mte_kernel.h>
-#endif
-
namespace scudo {
uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
MmapProt = PROT_NONE;
} else {
MmapProt = PROT_READ | PROT_WRITE;
-#if defined(__aarch64__) && defined(ANDROID_EXPERIMENTAL_MTE)
- if (Flags & MAP_MEMTAG)
- MmapProt |= PROT_MTE;
-#endif
}
- if (Addr) {
- // Currently no scenario for a noaccess mapping with a fixed address.
- DCHECK_EQ(Flags & MAP_NOACCESS, 0);
+#if defined(__aarch64__)
+#ifndef PROT_MTE
+#define PROT_MTE 0x20
+#endif
+ if (Flags & MAP_MEMTAG)
+ MmapProt |= PROT_MTE;
+#endif
+ if (Addr)
MmapFlags |= MAP_FIXED;
- }
void *P = mmap(Addr, Size, MmapProt, MmapFlags, -1, 0);
if (P == MAP_FAILED) {
if (!(Flags & MAP_ALLOWNOMEM) || errno != ENOMEM)
- dieOnMapUnmapError(errno == ENOMEM);
+ dieOnMapUnmapError(errno == ENOMEM ? Size : 0);
return nullptr;
}
#if SCUDO_ANDROID
- if (!(Flags & MAP_NOACCESS))
+ if (Name)
prctl(ANDROID_PR_SET_VMA, ANDROID_PR_SET_VMA_ANON_NAME, P, Size, Name);
#endif
return P;
dieOnMapUnmapError();
}
+void setMemoryPermission(uptr Addr, uptr Size, uptr Flags,
+ UNUSED MapPlatformData *Data) {
+ int Prot = (Flags & MAP_NOACCESS) ? PROT_NONE : (PROT_READ | PROT_WRITE);
+ if (mprotect(reinterpret_cast<void *>(Addr), Size, Prot) != 0)
+ dieOnMapUnmapError();
+}
+
void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
UNUSED MapPlatformData *Data) {
void *Addr = reinterpret_cast<void *>(BaseAddress + Offset);
+
while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) {
}
}
}
async_safe_write_log(AndroidLogInfo, "scudo", Buffer);
} else {
- write(2, Buffer, strlen(Buffer));
+ (void)write(2, Buffer, strlen(Buffer));
}
}
// MapPlatformData is unused on Linux, define it as a minimally sized structure.
struct MapPlatformData {};
-#if SCUDO_ANDROID
-
-#if defined(__aarch64__)
-#define __get_tls() \
- ({ \
- void **__v; \
- __asm__("mrs %0, tpidr_el0" : "=r"(__v)); \
- __v; \
- })
-#elif defined(__arm__)
-#define __get_tls() \
- ({ \
- void **__v; \
- __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r"(__v)); \
- __v; \
- })
-#elif defined(__i386__)
-#define __get_tls() \
- ({ \
- void **__v; \
- __asm__("movl %%gs:0, %0" : "=r"(__v)); \
- __v; \
- })
-#elif defined(__x86_64__)
-#define __get_tls() \
- ({ \
- void **__v; \
- __asm__("mov %%fs:0, %0" : "=r"(__v)); \
- __v; \
- })
-#else
-#error "Unsupported architecture."
-#endif
-
-// The Android Bionic team has allocated a TLS slot for sanitizers starting
-// with Q, given that Android currently doesn't support ELF TLS. It is used to
-// store sanitizer thread specific data.
-static const int TLS_SLOT_SANITIZER = 6;
-
-ALWAYS_INLINE uptr *getAndroidTlsPtr() {
- return reinterpret_cast<uptr *>(&__get_tls()[TLS_SLOT_SANITIZER]);
-}
-
-#endif // SCUDO_ANDROID
-
} // namespace scudo
#endif // SCUDO_LINUX
void checkConsistency() const;
protected:
- uptr Size;
- T *First;
- T *Last;
+ uptr Size = 0;
+ T *First = nullptr;
+ T *Last = nullptr;
};
template <class T> void IntrusiveList<T>::checkConsistency() const {
template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
typedef typename SizeClassAllocator::SizeClassMap SizeClassMap;
+ typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
struct TransferBatch {
static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint;
- void setFromArray(void **Array, u32 N) {
+ void setFromArray(CompactPtrT *Array, u32 N) {
DCHECK_LE(N, MaxNumCached);
Count = N;
- memcpy(Batch, Array, sizeof(void *) * Count);
+ memcpy(Batch, Array, sizeof(Batch[0]) * Count);
}
void clear() { Count = 0; }
- void add(void *P) {
+ void add(CompactPtrT P) {
DCHECK_LT(Count, MaxNumCached);
Batch[Count++] = P;
}
- void copyToArray(void **Array) const {
- memcpy(Array, Batch, sizeof(void *) * Count);
+ void copyToArray(CompactPtrT *Array) const {
+ memcpy(Array, Batch, sizeof(Batch[0]) * Count);
}
u32 getCount() const { return Count; }
- void *get(u32 I) const {
+ CompactPtrT get(u32 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
private:
u32 Count;
- void *Batch[MaxNumCached];
+ CompactPtrT Batch[MaxNumCached];
};
- void initLinkerInitialized(GlobalStats *S, SizeClassAllocator *A) {
- Stats.initLinkerInitialized();
+ void init(GlobalStats *S, SizeClassAllocator *A) {
+ DCHECK(isEmpty());
+ Stats.init();
if (LIKELY(S))
S->link(&Stats);
Allocator = A;
}
- void init(GlobalStats *S, SizeClassAllocator *A) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(S, A);
- }
-
void destroy(GlobalStats *S) {
drain();
if (LIKELY(S))
// Count, while Chunks might be further off (depending on Count). That keeps
// the memory accesses in close quarters.
const uptr ClassSize = C->ClassSize;
- void *P = C->Chunks[--C->Count];
- // The jury is still out as to whether any kind of PREFETCH here increases
- // performance. It definitely decreases performance on Android though.
- // if (!SCUDO_ANDROID) PREFETCH(P);
+ CompactPtrT CompactP = C->Chunks[--C->Count];
Stats.add(StatAllocated, ClassSize);
Stats.sub(StatFree, ClassSize);
- return P;
+ return Allocator->decompactPtr(ClassId, CompactP);
}
void deallocate(uptr ClassId, void *P) {
drain(C, ClassId);
// See comment in allocate() about memory accesses.
const uptr ClassSize = C->ClassSize;
- C->Chunks[C->Count++] = P;
+ C->Chunks[C->Count++] =
+ Allocator->compactPtr(ClassId, reinterpret_cast<uptr>(P));
Stats.sub(StatAllocated, ClassSize);
Stats.add(StatFree, ClassSize);
}
+ bool isEmpty() const {
+ for (uptr I = 0; I < NumClasses; ++I)
+ if (PerClassArray[I].Count)
+ return false;
+ return true;
+ }
+
void drain() {
- for (uptr I = 0; I < NumClasses; I++) {
- PerClass *C = &PerClassArray[I];
- while (C->Count > 0)
- drain(C, I);
+ // Drain BatchClassId last as createBatch can refill it.
+ for (uptr I = 0; I < NumClasses; ++I) {
+ if (I == BatchClassId)
+ continue;
+ while (PerClassArray[I].Count > 0)
+ drain(&PerClassArray[I], I);
}
+ while (PerClassArray[BatchClassId].Count > 0)
+ drain(&PerClassArray[BatchClassId], BatchClassId);
+ DCHECK(isEmpty());
}
TransferBatch *createBatch(uptr ClassId, void *B) {
- if (ClassId != SizeClassMap::BatchClassId)
- B = allocate(SizeClassMap::BatchClassId);
+ if (ClassId != BatchClassId)
+ B = allocate(BatchClassId);
return reinterpret_cast<TransferBatch *>(B);
}
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
+ static const uptr BatchClassId = SizeClassMap::BatchClassId;
struct PerClass {
u32 Count;
u32 MaxCount;
+ // Note: ClassSize is zero for the transfer batch.
uptr ClassSize;
- void *Chunks[2 * TransferBatch::MaxNumCached];
+ CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
};
- PerClass PerClassArray[NumClasses];
+ PerClass PerClassArray[NumClasses] = {};
LocalStats Stats;
- SizeClassAllocator *Allocator;
+ SizeClassAllocator *Allocator = nullptr;
ALWAYS_INLINE void initCacheMaybe(PerClass *C) {
if (LIKELY(C->MaxCount))
PerClass *P = &PerClassArray[I];
const uptr Size = SizeClassAllocator::getSizeByClassId(I);
P->MaxCount = 2 * TransferBatch::getMaxCached(Size);
- P->ClassSize = Size;
+ if (I != BatchClassId) {
+ P->ClassSize = Size;
+ } else {
+ // ClassSize in this struct is only used for malloc/free stats, which
+ // should only track user allocations, not internal movements.
+ P->ClassSize = 0;
+ }
}
}
void destroyBatch(uptr ClassId, void *B) {
- if (ClassId != SizeClassMap::BatchClassId)
- deallocate(SizeClassMap::BatchClassId, B);
+ if (ClassId != BatchClassId)
+ deallocate(BatchClassId, B);
}
NOINLINE bool refill(PerClass *C, uptr ClassId) {
DCHECK_GT(B->getCount(), 0);
C->Count = B->getCount();
B->copyToArray(C->Chunks);
+ B->clear();
destroyBatch(ClassId, B);
return true;
}
NOINLINE void drain(PerClass *C, uptr ClassId) {
const u32 Count = Min(C->MaxCount / 2, C->Count);
- TransferBatch *B = createBatch(ClassId, C->Chunks[0]);
+ TransferBatch *B =
+ createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0]));
if (UNLIKELY(!B))
- reportOutOfMemory(
- SizeClassAllocator::getSizeByClassId(SizeClassMap::BatchClassId));
+ reportOutOfMemory(SizeClassAllocator::getSizeByClassId(BatchClassId));
B->setFromArray(&C->Chunks[0], Count);
C->Count -= Count;
for (uptr I = 0; I < C->Count; I++)
#if SCUDO_LINUX
#include <sys/auxv.h>
#include <sys/prctl.h>
-#if defined(ANDROID_EXPERIMENTAL_MTE)
-#include <bionic/mte_kernel.h>
-#endif
#endif
namespace scudo {
-#if defined(__aarch64__) || defined(SCUDO_FUZZ)
+#if (__clang_major__ >= 12 && defined(__aarch64__)) || defined(SCUDO_FUZZ)
+// We assume that Top-Byte Ignore is enabled if the architecture supports memory
+// tagging. Not all operating systems enable TBI, so we only claim architectural
+// support for memory tagging if the operating system enables TBI.
+#if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI)
inline constexpr bool archSupportsMemoryTagging() { return true; }
+#else
+inline constexpr bool archSupportsMemoryTagging() { return false; }
+#endif
+
inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
-inline uint8_t extractTag(uptr Ptr) {
- return (Ptr >> 56) & 0xf;
-}
+inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
#else
#endif
-#if defined(__aarch64__)
+#if __clang_major__ >= 12 && defined(__aarch64__)
+
+#if SCUDO_LINUX
inline bool systemSupportsMemoryTagging() {
-#if defined(ANDROID_EXPERIMENTAL_MTE)
- return getauxval(AT_HWCAP2) & HWCAP2_MTE;
-#else
- return false;
+#ifndef HWCAP2_MTE
+#define HWCAP2_MTE (1 << 18)
#endif
+ return getauxval(AT_HWCAP2) & HWCAP2_MTE;
}
inline bool systemDetectsMemoryTagFaultsTestOnly() {
-#if defined(ANDROID_EXPERIMENTAL_MTE)
- return (prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) & PR_MTE_TCF_MASK) !=
- PR_MTE_TCF_NONE;
-#else
- return false;
+#ifndef PR_SET_TAGGED_ADDR_CTRL
+#define PR_SET_TAGGED_ADDR_CTRL 54
+#endif
+#ifndef PR_GET_TAGGED_ADDR_CTRL
+#define PR_GET_TAGGED_ADDR_CTRL 56
+#endif
+#ifndef PR_TAGGED_ADDR_ENABLE
+#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+#endif
+#ifndef PR_MTE_TCF_SHIFT
+#define PR_MTE_TCF_SHIFT 1
+#endif
+#ifndef PR_MTE_TAG_SHIFT
+#define PR_MTE_TAG_SHIFT 3
#endif
+#ifndef PR_MTE_TCF_NONE
+#define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+#endif
+#ifndef PR_MTE_TCF_SYNC
+#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+#endif
+#ifndef PR_MTE_TCF_MASK
+#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
+#endif
+ return (static_cast<unsigned long>(
+ prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &
+ PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
+}
+
+inline void enableSystemMemoryTaggingTestOnly() {
+ prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
+ 0, 0, 0);
}
-inline void disableMemoryTagChecksTestOnly() {
- __asm__ __volatile__(".arch_extension mte; msr tco, #1");
+#else // !SCUDO_LINUX
+
+inline bool systemSupportsMemoryTagging() { return false; }
+
+inline bool systemDetectsMemoryTagFaultsTestOnly() {
+ UNREACHABLE("memory tagging not supported");
}
-inline void enableMemoryTagChecksTestOnly() {
- __asm__ __volatile__(".arch_extension mte; msr tco, #0");
+inline void enableSystemMemoryTaggingTestOnly() {
+ UNREACHABLE("memory tagging not supported");
}
+#endif // SCUDO_LINUX
+
class ScopedDisableMemoryTagChecks {
- size_t PrevTCO;
+ uptr PrevTCO;
- public:
+public:
ScopedDisableMemoryTagChecks() {
- __asm__ __volatile__(".arch_extension mte; mrs %0, tco; msr tco, #1"
- : "=r"(PrevTCO));
+ __asm__ __volatile__(
+ R"(
+ .arch_extension memtag
+ mrs %0, tco
+ msr tco, #1
+ )"
+ : "=r"(PrevTCO));
}
~ScopedDisableMemoryTagChecks() {
- __asm__ __volatile__(".arch_extension mte; msr tco, %0" : : "r"(PrevTCO));
+ __asm__ __volatile__(
+ R"(
+ .arch_extension memtag
+ msr tco, %0
+ )"
+ :
+ : "r"(PrevTCO));
}
};
-inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
- uptr *TaggedBegin, uptr *TaggedEnd) {
- void *End;
+inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
+ ExcludeMask |= 1; // Always exclude Tag 0.
+ uptr TaggedPtr;
__asm__ __volatile__(
R"(
- .arch_extension mte
-
- // Set a random tag for Ptr in TaggedPtr. This needs to happen even if
- // Size = 0 so that TaggedPtr ends up pointing at a valid address.
- irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
- mov %[Cur], %[TaggedPtr]
-
- // Skip the loop if Size = 0. We don't want to do any tagging in this case.
- cbz %[Size], 2f
+ .arch_extension memtag
+ irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
+ )"
+ : [TaggedPtr] "=r"(TaggedPtr)
+ : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
+ return TaggedPtr;
+}
- // Set the memory tag of the region
- // [TaggedPtr, TaggedPtr + roundUpTo(Size, 16))
- // to the pointer tag stored in TaggedPtr.
- add %[End], %[TaggedPtr], %[Size]
+inline uptr addFixedTag(uptr Ptr, uptr Tag) {
+ DCHECK_LT(Tag, 16);
+ DCHECK_EQ(untagPointer(Ptr), Ptr);
+ return Ptr | (Tag << 56);
+}
+inline uptr storeTags(uptr Begin, uptr End) {
+ DCHECK_EQ(0, Begin % 16);
+ uptr LineSize, Next, Tmp;
+ __asm__ __volatile__(
+ R"(
+ .arch_extension memtag
+
+ // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
+ // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
+ // indicates that the DC instructions are unavailable.
+ DCZID .req %[Tmp]
+ mrs DCZID, dczid_el0
+ tbnz DCZID, #4, 3f
+ and DCZID, DCZID, #15
+ mov %[LineSize], #4
+ lsl %[LineSize], %[LineSize], DCZID
+ .unreq DCZID
+
+ // Our main loop doesn't handle the case where we don't need to perform any
+ // DC GZVA operations. If the size of our tagged region is less than
+ // twice the cache line size, bail out to the slow path since it's not
+ // guaranteed that we'll be able to do a DC GZVA.
+ Size .req %[Tmp]
+ sub Size, %[End], %[Cur]
+ cmp Size, %[LineSize], lsl #1
+ b.lt 3f
+ .unreq Size
+
+ LineMask .req %[Tmp]
+ sub LineMask, %[LineSize], #1
+
+ // STZG until the start of the next cache line.
+ orr %[Next], %[Cur], LineMask
1:
stzg %[Cur], [%[Cur]], #16
- cmp %[Cur], %[End]
+ cmp %[Cur], %[Next]
b.lt 1b
+ // DC GZVA cache lines until we have no more full cache lines.
+ bic %[Next], %[End], LineMask
+ .unreq LineMask
2:
+ dc gzva, %[Cur]
+ add %[Cur], %[Cur], %[LineSize]
+ cmp %[Cur], %[Next]
+ b.lt 2b
+
+ // STZG until the end of the tagged region. This loop is also used to handle
+ // slow path cases.
+ 3:
+ cmp %[Cur], %[End]
+ b.ge 4f
+ stzg %[Cur], [%[Cur]], #16
+ b 3b
+
+ 4:
)"
- :
- [TaggedPtr] "=&r"(*TaggedBegin), [Cur] "=&r"(*TaggedEnd), [End] "=&r"(End)
- : [Ptr] "r"(Ptr), [Size] "r"(Size), [ExcludeMask] "r"(ExcludeMask)
+ : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
+ [Tmp] "=&r"(Tmp)
+ : [End] "r"(End)
: "memory");
+ DCHECK_EQ(0, Begin % 16);
+ return Begin;
}
-inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr BlockEnd) {
- // Prepare the granule before the chunk to store the chunk header by setting
- // its tag to 0. Normally its tag will already be 0, but in the case where a
- // chunk holding a low alignment allocation is reused for a higher alignment
- // allocation, the chunk may already have a non-zero tag from the previous
- // allocation.
- __asm__ __volatile__(".arch_extension mte; stg %0, [%0, #-16]"
- :
- : "r"(Ptr)
- : "memory");
-
- uptr TaggedBegin, TaggedEnd;
- setRandomTag(Ptr, Size, 0, &TaggedBegin, &TaggedEnd);
-
- // Finally, set the tag of the granule past the end of the allocation to 0,
- // to catch linear overflows even if a previous larger allocation used the
- // same block and tag. Only do this if the granule past the end is in our
- // block, because this would otherwise lead to a SEGV if the allocation
- // covers the entire block and our block is at the end of a mapping. The tag
- // of the next block's header granule will be set to 0, so it will serve the
- // purpose of catching linear overflows in this case.
- uptr UntaggedEnd = untagPointer(TaggedEnd);
- if (UntaggedEnd != BlockEnd)
- __asm__ __volatile__(".arch_extension mte; stg %0, [%0]"
- :
- : "r"(UntaggedEnd)
- : "memory");
- return reinterpret_cast<void *>(TaggedBegin);
-}
-
-inline void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr BlockEnd) {
- uptr RoundOldPtr = roundUpTo(OldPtr, 16);
- if (RoundOldPtr >= NewPtr) {
- // If the allocation is shrinking we just need to set the tag past the end
- // of the allocation to 0. See explanation in prepareTaggedChunk above.
- uptr RoundNewPtr = untagPointer(roundUpTo(NewPtr, 16));
- if (RoundNewPtr != BlockEnd)
- __asm__ __volatile__(".arch_extension mte; stg %0, [%0]"
- :
- : "r"(RoundNewPtr)
- : "memory");
- return;
- }
-
+inline void storeTag(uptr Ptr) {
+ DCHECK_EQ(0, Ptr % 16);
__asm__ __volatile__(R"(
- .arch_extension mte
-
- // Set the memory tag of the region
- // [roundUpTo(OldPtr, 16), roundUpTo(NewPtr, 16))
- // to the pointer tag stored in OldPtr.
- 1:
- stzg %[Cur], [%[Cur]], #16
- cmp %[Cur], %[End]
- b.lt 1b
-
- // Finally, set the tag of the granule past the end of the allocation to 0.
- and %[Cur], %[Cur], #(1 << 56) - 1
- cmp %[Cur], %[BlockEnd]
- b.eq 2f
- stg %[Cur], [%[Cur]]
-
- 2:
+ .arch_extension memtag
+ stg %0, [%0]
)"
- : [ Cur ] "+&r"(RoundOldPtr), [ End ] "+&r"(NewPtr)
- : [ BlockEnd ] "r"(BlockEnd)
+ :
+ : "r"(Ptr)
: "memory");
}
inline uptr loadTag(uptr Ptr) {
+ DCHECK_EQ(0, Ptr % 16);
uptr TaggedPtr = Ptr;
- __asm__ __volatile__(".arch_extension mte; ldg %0, [%0]"
- : "+r"(TaggedPtr)
- :
- : "memory");
+ __asm__ __volatile__(
+ R"(
+ .arch_extension memtag
+ ldg %0, [%0]
+ )"
+ : "+r"(TaggedPtr)
+ :
+ : "memory");
return TaggedPtr;
}
UNREACHABLE("memory tagging not supported");
}
-inline void disableMemoryTagChecksTestOnly() {
- UNREACHABLE("memory tagging not supported");
-}
-
-inline void enableMemoryTagChecksTestOnly() {
+inline void enableSystemMemoryTaggingTestOnly() {
UNREACHABLE("memory tagging not supported");
}
ScopedDisableMemoryTagChecks() {}
};
-inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
- uptr *TaggedBegin, uptr *TaggedEnd) {
+inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
(void)Ptr;
- (void)Size;
(void)ExcludeMask;
- (void)TaggedBegin;
- (void)TaggedEnd;
UNREACHABLE("memory tagging not supported");
}
-inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr BlockEnd) {
+inline uptr addFixedTag(uptr Ptr, uptr Tag) {
(void)Ptr;
- (void)Size;
- (void)BlockEnd;
+ (void)Tag;
UNREACHABLE("memory tagging not supported");
}
-inline void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr BlockEnd) {
- (void)OldPtr;
- (void)NewPtr;
- (void)BlockEnd;
+inline uptr storeTags(uptr Begin, uptr End) {
+ (void)Begin;
+ (void)End;
+ UNREACHABLE("memory tagging not supported");
+}
+
+inline void storeTag(uptr Ptr) {
+ (void)Ptr;
UNREACHABLE("memory tagging not supported");
}
#endif
+inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
+ uptr *TaggedBegin, uptr *TaggedEnd) {
+ *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
+ *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
+}
+
+inline void *untagPointer(void *Ptr) {
+ return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
+}
+
+inline void *loadTag(void *Ptr) {
+ return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
+}
+
+inline void *addFixedTag(void *Ptr, uptr Tag) {
+ return reinterpret_cast<void *>(
+ addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
+}
+
+template <typename Config>
+inline constexpr bool allocatorSupportsMemoryTagging() {
+ return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging &&
+ (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
+}
+
} // namespace scudo
#endif
class HybridMutex {
public:
- void init() { M = {}; }
bool tryLock();
NOINLINE void lock() {
if (LIKELY(tryLock()))
static constexpr u8 NumberOfYields = 8U;
#if SCUDO_LINUX
- atomic_u32 M;
+ atomic_u32 M = {};
#elif SCUDO_FUCHSIA
- sync_mutex_t M;
+ sync_mutex_t M = {};
#endif
void lockSlow();
--- /dev/null
+//===-- options.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_OPTIONS_H_
+#define SCUDO_OPTIONS_H_
+
+#include "atomic_helpers.h"
+#include "common.h"
+#include "memtag.h"
+
+namespace scudo {
+
+enum class OptionBit {
+ MayReturnNull,
+ FillContents0of2,
+ FillContents1of2,
+ DeallocTypeMismatch,
+ DeleteSizeMismatch,
+ TrackAllocationStacks,
+ UseOddEvenTags,
+ UseMemoryTagging,
+ AddLargeAllocationSlack,
+};
+
+struct Options {
+ u32 Val;
+
+ bool get(OptionBit Opt) const { return Val & (1U << static_cast<u32>(Opt)); }
+
+ FillContentsMode getFillContentsMode() const {
+ return static_cast<FillContentsMode>(
+ (Val >> static_cast<u32>(OptionBit::FillContents0of2)) & 3);
+ }
+};
+
+template <typename Config> bool useMemoryTagging(Options Options) {
+ return allocatorSupportsMemoryTagging<Config>() &&
+ Options.get(OptionBit::UseMemoryTagging);
+}
+
+struct AtomicOptions {
+ atomic_u32 Val = {};
+
+ Options load() const { return Options{atomic_load_relaxed(&Val)}; }
+
+ void clear(OptionBit Opt) {
+ atomic_fetch_and(&Val, ~(1U << static_cast<u32>(Opt)),
+ memory_order_relaxed);
+ }
+
+ void set(OptionBit Opt) {
+ atomic_fetch_or(&Val, 1U << static_cast<u32>(Opt), memory_order_relaxed);
+ }
+
+ void setFillContentsMode(FillContentsMode FillContents) {
+ u32 Opts = atomic_load_relaxed(&Val), NewOpts;
+ do {
+ NewOpts = Opts;
+ NewOpts &= ~(3U << static_cast<u32>(OptionBit::FillContents0of2));
+ NewOpts |= static_cast<u32>(FillContents)
+ << static_cast<u32>(OptionBit::FillContents0of2);
+ } while (!atomic_compare_exchange_strong(&Val, &Opts, NewOpts,
+ memory_order_relaxed));
+ }
+};
+
+} // namespace scudo
+
+#endif // SCUDO_OPTIONS_H_
// Transitive includes of stdint.h specify some of the defines checked below.
#include <stdint.h>
-#if defined(__linux__)
+#if defined(__linux__) && !defined(__TRUSTY__)
#define SCUDO_LINUX 1
#else
#define SCUDO_LINUX 0
#define SCUDO_FUCHSIA 0
#endif
+#if defined(__TRUSTY__)
+#define SCUDO_TRUSTY 1
+#else
+#define SCUDO_TRUSTY 0
+#endif
+
#if __LP64__
#define SCUDO_WORDSIZE 64U
#else
#include "common.h"
#include "list.h"
#include "local_cache.h"
+#include "options.h"
#include "release.h"
#include "report.h"
#include "stats.h"
// Memory used by this allocator is never unmapped but can be partially
// reclaimed if the platform allows for it.
-template <class SizeClassMapT, uptr RegionSizeLog,
- s32 MinReleaseToOsIntervalMs = INT32_MIN,
- s32 MaxReleaseToOsIntervalMs = INT32_MAX>
-class SizeClassAllocator32 {
+template <typename Config> class SizeClassAllocator32 {
public:
- typedef SizeClassMapT SizeClassMap;
+ typedef typename Config::PrimaryCompactPtrT CompactPtrT;
+ typedef typename Config::SizeClassMap SizeClassMap;
// The bytemap can only track UINT8_MAX - 1 classes.
static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), "");
// Regions should be large enough to hold the largest Block.
- static_assert((1UL << RegionSizeLog) >= SizeClassMap::MaxSize, "");
- typedef SizeClassAllocator32<SizeClassMapT, RegionSizeLog,
- MinReleaseToOsIntervalMs,
- MaxReleaseToOsIntervalMs>
- ThisT;
+ static_assert((1UL << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize,
+ "");
+ typedef SizeClassAllocator32<Config> ThisT;
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
typedef typename CacheT::TransferBatch TransferBatch;
- static const bool SupportsMemoryTagging = false;
static uptr getSizeByClassId(uptr ClassId) {
return (ClassId == SizeClassMap::BatchClassId)
static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
+ void init(s32 ReleaseToOsInterval) {
if (SCUDO_FUCHSIA)
reportError("SizeClassAllocator32 is not supported on Fuchsia");
- PossibleRegions.initLinkerInitialized();
- MinRegionIndex = NumRegions; // MaxRegionIndex is already initialized to 0.
+ if (SCUDO_TRUSTY)
+ reportError("SizeClassAllocator32 is not supported on Trusty");
+ DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
+ PossibleRegions.init();
u32 Seed;
const u64 Time = getMonotonicTime();
- if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed))))
+ if (!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))
Seed = static_cast<u32>(
Time ^ (reinterpret_cast<uptr>(SizeClassInfoArray) >> 6));
- const uptr PageSize = getPageSizeCached();
for (uptr I = 0; I < NumClasses; I++) {
SizeClassInfo *Sci = getSizeClassInfo(I);
Sci->RandState = getRandomU32(&Seed);
- // See comment in the 64-bit primary about releasing smaller size classes.
- Sci->CanRelease = (I != SizeClassMap::BatchClassId) &&
- (getSizeByClassId(I) >= (PageSize / 32));
- if (Sci->CanRelease)
- Sci->ReleaseInfo.LastReleaseAtNs = Time;
+ // Sci->MaxRegionIndex is already initialized to 0.
+ Sci->MinRegionIndex = NumRegions;
+ Sci->ReleaseInfo.LastReleaseAtNs = Time;
}
- setReleaseToOsIntervalMs(ReleaseToOsInterval);
- }
- void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
+ setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
void unmapTestOnly() {
while (NumberOfStashedRegions > 0)
unmap(reinterpret_cast<void *>(RegionsStash[--NumberOfStashedRegions]),
RegionSize);
- for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++)
+ uptr MinRegionIndex = NumRegions, MaxRegionIndex = 0;
+ for (uptr I = 0; I < NumClasses; I++) {
+ SizeClassInfo *Sci = getSizeClassInfo(I);
+ if (Sci->MinRegionIndex < MinRegionIndex)
+ MinRegionIndex = Sci->MinRegionIndex;
+ if (Sci->MaxRegionIndex > MaxRegionIndex)
+ MaxRegionIndex = Sci->MaxRegionIndex;
+ *Sci = {};
+ }
+ for (uptr I = MinRegionIndex; I < MaxRegionIndex; I++)
if (PossibleRegions[I])
unmap(reinterpret_cast<void *>(I * RegionSize), RegionSize);
PossibleRegions.unmapTestOnly();
}
+ CompactPtrT compactPtr(UNUSED uptr ClassId, uptr Ptr) const {
+ return static_cast<CompactPtrT>(Ptr);
+ }
+
+ void *decompactPtr(UNUSED uptr ClassId, CompactPtrT CompactPtr) const {
+ return reinterpret_cast<void *>(static_cast<uptr>(CompactPtr));
+ }
+
TransferBatch *popBatch(CacheT *C, uptr ClassId) {
DCHECK_LT(ClassId, NumClasses);
SizeClassInfo *Sci = getSizeClassInfo(ClassId);
ScopedLock L(Sci->Mutex);
Sci->FreeList.push_front(B);
Sci->Stats.PushedBlocks += B->getCount();
- if (Sci->CanRelease)
+ if (ClassId != SizeClassMap::BatchClassId)
releaseToOSMaybe(Sci, ClassId);
}
}
template <typename F> void iterateOverBlocks(F Callback) {
+ uptr MinRegionIndex = NumRegions, MaxRegionIndex = 0;
+ for (uptr I = 0; I < NumClasses; I++) {
+ SizeClassInfo *Sci = getSizeClassInfo(I);
+ if (Sci->MinRegionIndex < MinRegionIndex)
+ MinRegionIndex = Sci->MinRegionIndex;
+ if (Sci->MaxRegionIndex > MaxRegionIndex)
+ MaxRegionIndex = Sci->MaxRegionIndex;
+ }
for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++)
if (PossibleRegions[I] &&
(PossibleRegions[I] - 1U) != SizeClassMap::BatchClassId) {
getStats(Str, I, 0);
}
- void setReleaseToOsIntervalMs(s32 Interval) {
- if (Interval >= MaxReleaseToOsIntervalMs) {
- Interval = MaxReleaseToOsIntervalMs;
- } else if (Interval <= MinReleaseToOsIntervalMs) {
- Interval = MinReleaseToOsIntervalMs;
+ bool setOption(Option O, sptr Value) {
+ if (O == Option::ReleaseInterval) {
+ const s32 Interval = Max(
+ Min(static_cast<s32>(Value), Config::PrimaryMaxReleaseToOsIntervalMs),
+ Config::PrimaryMinReleaseToOsIntervalMs);
+ atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
+ return true;
}
- atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+ // Not supported by the Primary, but not an error either.
+ return true;
}
uptr releaseToOS() {
uptr TotalReleasedBytes = 0;
for (uptr I = 0; I < NumClasses; I++) {
+ if (I == SizeClassMap::BatchClassId)
+ continue;
SizeClassInfo *Sci = getSizeClassInfo(I);
ScopedLock L(Sci->Mutex);
TotalReleasedBytes += releaseToOSMaybe(Sci, I, /*Force=*/true);
return TotalReleasedBytes;
}
- bool useMemoryTagging() { return false; }
- void disableMemoryTagging() {}
-
const char *getRegionInfoArrayAddress() const { return nullptr; }
static uptr getRegionInfoArraySize() { return 0; }
- static BlockInfo findNearestBlock(const char *RegionInfoData, uptr Ptr) {
- (void)RegionInfoData;
- (void)Ptr;
+ static BlockInfo findNearestBlock(UNUSED const char *RegionInfoData,
+ UNUSED uptr Ptr) {
return {};
}
+ AtomicOptions Options;
+
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
- static const uptr RegionSize = 1UL << RegionSizeLog;
- static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog;
+ static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog;
+ static const uptr NumRegions =
+ SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog;
static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
typedef FlatByteMap<NumRegions> ByteMap;
uptr CurrentRegion;
uptr CurrentRegionAllocated;
SizeClassStats Stats;
- bool CanRelease;
u32 RandState;
uptr AllocatedUser;
+ // Lowest & highest region index allocated for this size class, to avoid
+ // looping through the whole NumRegions.
+ uptr MinRegionIndex;
+ uptr MaxRegionIndex;
ReleaseToOsInfo ReleaseInfo;
};
static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
uptr computeRegionId(uptr Mem) {
- const uptr Id = Mem >> RegionSizeLog;
+ const uptr Id = Mem >> Config::PrimaryRegionSizeLog;
CHECK_LT(Id, NumRegions);
return Id;
}
uptr MapSize = 2 * RegionSize;
const uptr MapBase = reinterpret_cast<uptr>(
map(nullptr, MapSize, "scudo:primary", MAP_ALLOWNOMEM));
- if (UNLIKELY(!MapBase))
+ if (!MapBase)
return 0;
const uptr MapEnd = MapBase + MapSize;
uptr Region = MapBase;
return Region;
}
- uptr allocateRegion(uptr ClassId) {
+ uptr allocateRegion(SizeClassInfo *Sci, uptr ClassId) {
DCHECK_LT(ClassId, NumClasses);
uptr Region = 0;
{
if (!Region)
Region = allocateRegionSlow();
if (LIKELY(Region)) {
+ // Sci->Mutex is held by the caller, updating the Min/Max is safe.
const uptr RegionIndex = computeRegionId(Region);
- if (RegionIndex < MinRegionIndex)
- MinRegionIndex = RegionIndex;
- if (RegionIndex > MaxRegionIndex)
- MaxRegionIndex = RegionIndex;
+ if (RegionIndex < Sci->MinRegionIndex)
+ Sci->MinRegionIndex = RegionIndex;
+ if (RegionIndex > Sci->MaxRegionIndex)
+ Sci->MaxRegionIndex = RegionIndex;
PossibleRegions.set(RegionIndex, static_cast<u8>(ClassId + 1U));
}
return Region;
return &SizeClassInfoArray[ClassId];
}
- bool populateBatches(CacheT *C, SizeClassInfo *Sci, uptr ClassId,
- TransferBatch **CurrentBatch, u32 MaxCount,
- void **PointersArray, u32 Count) {
- if (ClassId != SizeClassMap::BatchClassId)
- shuffle(PointersArray, Count, &Sci->RandState);
- TransferBatch *B = *CurrentBatch;
- for (uptr I = 0; I < Count; I++) {
- if (B && B->getCount() == MaxCount) {
- Sci->FreeList.push_back(B);
- B = nullptr;
- }
- if (!B) {
- B = C->createBatch(ClassId, PointersArray[I]);
- if (UNLIKELY(!B))
- return false;
- B->clear();
- }
- B->add(PointersArray[I]);
- }
- *CurrentBatch = B;
- return true;
- }
-
NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
SizeClassInfo *Sci) {
uptr Region;
Offset = Sci->CurrentRegionAllocated;
} else {
DCHECK_EQ(Sci->CurrentRegionAllocated, 0U);
- Region = allocateRegion(ClassId);
+ Region = allocateRegion(Sci, ClassId);
if (UNLIKELY(!Region))
return nullptr;
C->getStats().add(StatMapped, RegionSize);
static_cast<u32>((RegionSize - Offset) / Size));
DCHECK_GT(NumberOfBlocks, 0U);
- TransferBatch *B = nullptr;
constexpr u32 ShuffleArraySize =
MaxNumBatches * TransferBatch::MaxNumCached;
// Fill the transfer batches and put them in the size-class freelist. We
// need to randomize the blocks for security purposes, so we first fill a
// local array that we then shuffle before populating the batches.
- void *ShuffleArray[ShuffleArraySize];
- u32 Count = 0;
- const uptr AllocatedUser = Size * NumberOfBlocks;
- for (uptr I = Region + Offset; I < Region + Offset + AllocatedUser;
- I += Size) {
- ShuffleArray[Count++] = reinterpret_cast<void *>(I);
- if (Count == ShuffleArraySize) {
- if (UNLIKELY(!populateBatches(C, Sci, ClassId, &B, MaxCount,
- ShuffleArray, Count)))
- return nullptr;
- Count = 0;
- }
- }
- if (Count) {
- if (UNLIKELY(!populateBatches(C, Sci, ClassId, &B, MaxCount, ShuffleArray,
- Count)))
+ CompactPtrT ShuffleArray[ShuffleArraySize];
+ DCHECK_LE(NumberOfBlocks, ShuffleArraySize);
+
+ uptr P = Region + Offset;
+ for (u32 I = 0; I < NumberOfBlocks; I++, P += Size)
+ ShuffleArray[I] = reinterpret_cast<CompactPtrT>(P);
+ // No need to shuffle the batches size class.
+ if (ClassId != SizeClassMap::BatchClassId)
+ shuffle(ShuffleArray, NumberOfBlocks, &Sci->RandState);
+ for (u32 I = 0; I < NumberOfBlocks;) {
+ TransferBatch *B =
+ C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I]));
+ if (UNLIKELY(!B))
return nullptr;
- }
- DCHECK(B);
- if (!Sci->FreeList.empty()) {
+ const u32 N = Min(MaxCount, NumberOfBlocks - I);
+ B->setFromArray(&ShuffleArray[I], N);
Sci->FreeList.push_back(B);
- B = Sci->FreeList.front();
- Sci->FreeList.pop_front();
+ I += N;
}
+ TransferBatch *B = Sci->FreeList.front();
+ Sci->FreeList.pop_front();
+ DCHECK(B);
DCHECK_GT(B->getCount(), 0);
+ const uptr AllocatedUser = Size * NumberOfBlocks;
C->getStats().add(StatFree, AllocatedUser);
DCHECK_LE(Sci->CurrentRegionAllocated + AllocatedUser, RegionSize);
// If there is not enough room in the region currently associated to fit
AvailableChunks, Rss >> 10, Sci->ReleaseInfo.RangesReleased);
}
- s32 getReleaseToOsIntervalMs() {
- return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
- }
-
NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
bool Force = false) {
const uptr BlockSize = getSizeByClassId(ClassId);
const uptr PageSize = getPageSizeCached();
- CHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks);
+ DCHECK_GE(Sci->Stats.PoppedBlocks, Sci->Stats.PushedBlocks);
const uptr BytesInFreeList =
Sci->AllocatedUser -
(Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks) * BlockSize;
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
+ // Releasing smaller blocks is expensive, so we want to make sure that a
+ // significant amount of bytes are free, and that there has been a good
+ // amount of batches pushed to the freelist before attempting to release.
+ if (BlockSize < PageSize / 16U) {
+ if (!Force && BytesPushed < Sci->AllocatedUser / 16U)
+ return 0;
+ // We want 8x% to 9x% free bytes (the larger the block, the lower the %).
+ if ((BytesInFreeList * 100U) / Sci->AllocatedUser <
+ (100U - 1U - BlockSize / 16U))
+ return 0;
+ }
+
if (!Force) {
- const s32 IntervalMs = getReleaseToOsIntervalMs();
+ const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
if (IntervalMs < 0)
return 0;
if (Sci->ReleaseInfo.LastReleaseAtNs +
}
}
- // TODO(kostyak): currently not ideal as we loop over all regions and
- // iterate multiple times over the same freelist if a ClassId spans multiple
- // regions. But it will have to do for now.
+ const uptr First = Sci->MinRegionIndex;
+ const uptr Last = Sci->MaxRegionIndex;
+ DCHECK_NE(Last, 0U);
+ DCHECK_LE(First, Last);
uptr TotalReleasedBytes = 0;
- const uptr MaxSize = (RegionSize / BlockSize) * BlockSize;
- for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) {
- if (PossibleRegions[I] - 1U == ClassId) {
- const uptr Region = I * RegionSize;
- // If the region is the one currently associated to the size-class, we
- // only need to release up to CurrentRegionAllocated, MaxSize otherwise.
- const uptr Size = (Region == Sci->CurrentRegion)
- ? Sci->CurrentRegionAllocated
- : MaxSize;
- ReleaseRecorder Recorder(Region);
- releaseFreeMemoryToOS(Sci->FreeList, Region, Size, BlockSize,
- &Recorder);
- if (Recorder.getReleasedRangesCount() > 0) {
- Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
- Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
- Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
- TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes;
- }
- }
+ const uptr Base = First * RegionSize;
+ const uptr NumberOfRegions = Last - First + 1U;
+ ReleaseRecorder Recorder(Base);
+ auto SkipRegion = [this, First, ClassId](uptr RegionIndex) {
+ return (PossibleRegions[First + RegionIndex] - 1U) != ClassId;
+ };
+ auto DecompactPtr = [](CompactPtrT CompactPtr) {
+ return reinterpret_cast<uptr>(CompactPtr);
+ };
+ releaseFreeMemoryToOS(Sci->FreeList, RegionSize, NumberOfRegions, BlockSize,
+ &Recorder, DecompactPtr, SkipRegion);
+ if (Recorder.getReleasedRangesCount() > 0) {
+ Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks;
+ Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount();
+ Sci->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes();
+ TotalReleasedBytes += Sci->ReleaseInfo.LastReleasedBytes;
}
Sci->ReleaseInfo.LastReleaseAtNs = getMonotonicTime();
+
return TotalReleasedBytes;
}
- SizeClassInfo SizeClassInfoArray[NumClasses];
+ SizeClassInfo SizeClassInfoArray[NumClasses] = {};
// Track the regions in use, 0 is unused, otherwise store ClassId + 1.
- ByteMap PossibleRegions;
- // Keep track of the lowest & highest regions allocated to avoid looping
- // through the whole NumRegions.
- uptr MinRegionIndex;
- uptr MaxRegionIndex;
- atomic_s32 ReleaseToOsIntervalMs;
+ ByteMap PossibleRegions = {};
+ atomic_s32 ReleaseToOsIntervalMs = {};
// Unless several threads request regions simultaneously from different size
// classes, the stash rarely contains more than 1 entry.
static constexpr uptr MaxStashedRegions = 4;
HybridMutex RegionsStashMutex;
- uptr NumberOfStashedRegions;
- uptr RegionsStash[MaxStashedRegions];
+ uptr NumberOfStashedRegions = 0;
+ uptr RegionsStash[MaxStashedRegions] = {};
};
} // namespace scudo
#include "list.h"
#include "local_cache.h"
#include "memtag.h"
+#include "options.h"
#include "release.h"
#include "stats.h"
#include "string_utils.h"
//
// It starts by reserving NumClasses * 2^RegionSizeLog bytes, equally divided in
// Regions, specific to each size class. Note that the base of that mapping is
-// random (based to the platform specific map() capabilities), and that each
-// Region actually starts at a random offset from its base.
+// random (based to the platform specific map() capabilities). If
+// PrimaryEnableRandomOffset is set, each Region actually starts at a random
+// offset from its base.
//
// Regions are mapped incrementally on demand to fulfill allocation requests,
// those mappings being split into equally sized Blocks based on the size class
// The memory used by this allocator is never unmapped, but can be partially
// released if the platform allows for it.
-template <class SizeClassMapT, uptr RegionSizeLog,
- s32 MinReleaseToOsIntervalMs = INT32_MIN,
- s32 MaxReleaseToOsIntervalMs = INT32_MAX,
- bool MaySupportMemoryTagging = false>
-class SizeClassAllocator64 {
+template <typename Config> class SizeClassAllocator64 {
public:
- typedef SizeClassMapT SizeClassMap;
- typedef SizeClassAllocator64<
- SizeClassMap, RegionSizeLog, MinReleaseToOsIntervalMs,
- MaxReleaseToOsIntervalMs, MaySupportMemoryTagging>
- ThisT;
+ typedef typename Config::PrimaryCompactPtrT CompactPtrT;
+ static const uptr CompactPtrScale = Config::PrimaryCompactPtrScale;
+ typedef typename Config::SizeClassMap SizeClassMap;
+ typedef SizeClassAllocator64<Config> ThisT;
typedef SizeClassAllocatorLocalCache<ThisT> CacheT;
typedef typename CacheT::TransferBatch TransferBatch;
- static const bool SupportsMemoryTagging =
- MaySupportMemoryTagging && archSupportsMemoryTagging();
static uptr getSizeByClassId(uptr ClassId) {
return (ClassId == SizeClassMap::BatchClassId)
- ? sizeof(TransferBatch)
+ ? roundUpTo(sizeof(TransferBatch), 1U << CompactPtrScale)
: SizeClassMap::getSizeByClassId(ClassId);
}
static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
+ void init(s32 ReleaseToOsInterval) {
+ DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
+ DCHECK_EQ(PrimaryBase, 0U);
// Reserve the space required for the Primary.
PrimaryBase = reinterpret_cast<uptr>(
- map(nullptr, PrimarySize, "scudo:primary", MAP_NOACCESS, &Data));
+ map(nullptr, PrimarySize, nullptr, MAP_NOACCESS, &Data));
u32 Seed;
const u64 Time = getMonotonicTime();
- if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed))))
+ if (!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))
Seed = static_cast<u32>(Time ^ (PrimaryBase >> 12));
const uptr PageSize = getPageSizeCached();
for (uptr I = 0; I < NumClasses; I++) {
RegionInfo *Region = getRegionInfo(I);
- // The actual start of a region is offseted by a random number of pages.
- Region->RegionBeg =
- getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize;
+ // The actual start of a region is offset by a random number of pages
+ // when PrimaryEnableRandomOffset is set.
+ Region->RegionBeg = getRegionBaseByClassId(I) +
+ (Config::PrimaryEnableRandomOffset
+ ? ((getRandomModN(&Seed, 16) + 1) * PageSize)
+ : 0);
Region->RandState = getRandomU32(&Seed);
- // Releasing smaller size classes doesn't necessarily yield to a
- // meaningful RSS impact: there are more blocks per page, they are
- // randomized around, and thus pages are less likely to be entirely empty.
- // On top of this, attempting to release those require more iterations and
- // memory accesses which ends up being fairly costly. The current lower
- // limit is mostly arbitrary and based on empirical observations.
- // TODO(kostyak): make the lower limit a runtime option
- Region->CanRelease = (I != SizeClassMap::BatchClassId) &&
- (getSizeByClassId(I) >= (PageSize / 32));
- if (Region->CanRelease)
- Region->ReleaseInfo.LastReleaseAtNs = Time;
+ Region->ReleaseInfo.LastReleaseAtNs = Time;
}
- setReleaseToOsIntervalMs(ReleaseToOsInterval);
-
- if (SupportsMemoryTagging)
- UseMemoryTagging = systemSupportsMemoryTagging();
- }
- void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
+ setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
void unmapTestOnly() {
+ for (uptr I = 0; I < NumClasses; I++) {
+ RegionInfo *Region = getRegionInfo(I);
+ *Region = {};
+ }
unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data);
+ PrimaryBase = 0U;
}
TransferBatch *popBatch(CacheT *C, uptr ClassId) {
ScopedLock L(Region->Mutex);
Region->FreeList.push_front(B);
Region->Stats.PushedBlocks += B->getCount();
- if (Region->CanRelease)
+ if (ClassId != SizeClassMap::BatchClassId)
releaseToOSMaybe(Region, ClassId);
}
getStats(Str, I, 0);
}
- void setReleaseToOsIntervalMs(s32 Interval) {
- if (Interval >= MaxReleaseToOsIntervalMs) {
- Interval = MaxReleaseToOsIntervalMs;
- } else if (Interval <= MinReleaseToOsIntervalMs) {
- Interval = MinReleaseToOsIntervalMs;
+ bool setOption(Option O, sptr Value) {
+ if (O == Option::ReleaseInterval) {
+ const s32 Interval = Max(
+ Min(static_cast<s32>(Value), Config::PrimaryMaxReleaseToOsIntervalMs),
+ Config::PrimaryMinReleaseToOsIntervalMs);
+ atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
+ return true;
}
- atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+ // Not supported by the Primary, but not an error either.
+ return true;
}
uptr releaseToOS() {
uptr TotalReleasedBytes = 0;
for (uptr I = 0; I < NumClasses; I++) {
+ if (I == SizeClassMap::BatchClassId)
+ continue;
RegionInfo *Region = getRegionInfo(I);
ScopedLock L(Region->Mutex);
TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true);
return TotalReleasedBytes;
}
- bool useMemoryTagging() const {
- return SupportsMemoryTagging && UseMemoryTagging;
- }
- void disableMemoryTagging() { UseMemoryTagging = false; }
-
const char *getRegionInfoArrayAddress() const {
return reinterpret_cast<const char *>(RegionInfoArray);
}
- static uptr getRegionInfoArraySize() {
- return sizeof(RegionInfoArray);
+ static uptr getRegionInfoArraySize() { return sizeof(RegionInfoArray); }
+
+ uptr getCompactPtrBaseByClassId(uptr ClassId) {
+ // If we are not compacting pointers, base everything off of 0.
+ if (sizeof(CompactPtrT) == sizeof(uptr) && CompactPtrScale == 0)
+ return 0;
+ return getRegionInfo(ClassId)->RegionBeg;
+ }
+
+ CompactPtrT compactPtr(uptr ClassId, uptr Ptr) {
+ DCHECK_LE(ClassId, SizeClassMap::LargestClassId);
+ return compactPtrInternal(getCompactPtrBaseByClassId(ClassId), Ptr);
+ }
+
+ void *decompactPtr(uptr ClassId, CompactPtrT CompactPtr) {
+ DCHECK_LE(ClassId, SizeClassMap::LargestClassId);
+ return reinterpret_cast<void *>(
+ decompactPtrInternal(getCompactPtrBaseByClassId(ClassId), CompactPtr));
}
static BlockInfo findNearestBlock(const char *RegionInfoData, uptr Ptr) {
return B;
}
+ AtomicOptions Options;
+
private:
- static const uptr RegionSize = 1UL << RegionSizeLog;
+ static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog;
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr PrimarySize = RegionSize * NumClasses;
- // Call map for user memory with at least this size.
- static const uptr MapSizeIncrement = 1UL << 18;
+ static const uptr MapSizeIncrement = Config::PrimaryMapSizeIncrement;
// Fill at most this number of batches from the newly map'd memory.
static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
struct UnpaddedRegionInfo {
HybridMutex Mutex;
SinglyLinkedList<TransferBatch> FreeList;
- RegionStats Stats;
- bool CanRelease;
- bool Exhausted;
- u32 RandState;
- uptr RegionBeg;
- uptr MappedUser; // Bytes mapped for user memory.
- uptr AllocatedUser; // Bytes allocated for user memory.
- MapPlatformData Data;
- ReleaseToOsInfo ReleaseInfo;
+ uptr RegionBeg = 0;
+ RegionStats Stats = {};
+ u32 RandState = 0;
+ uptr MappedUser = 0; // Bytes mapped for user memory.
+ uptr AllocatedUser = 0; // Bytes allocated for user memory.
+ MapPlatformData Data = {};
+ ReleaseToOsInfo ReleaseInfo = {};
+ bool Exhausted = false;
};
struct RegionInfo : UnpaddedRegionInfo {
char Padding[SCUDO_CACHE_LINE_SIZE -
- (sizeof(UnpaddedRegionInfo) % SCUDO_CACHE_LINE_SIZE)];
+ (sizeof(UnpaddedRegionInfo) % SCUDO_CACHE_LINE_SIZE)] = {};
};
static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, "");
- uptr PrimaryBase;
- MapPlatformData Data;
- atomic_s32 ReleaseToOsIntervalMs;
- bool UseMemoryTagging;
+ uptr PrimaryBase = 0;
+ MapPlatformData Data = {};
+ atomic_s32 ReleaseToOsIntervalMs = {};
alignas(SCUDO_CACHE_LINE_SIZE) RegionInfo RegionInfoArray[NumClasses];
RegionInfo *getRegionInfo(uptr ClassId) {
}
uptr getRegionBaseByClassId(uptr ClassId) const {
- return PrimaryBase + (ClassId << RegionSizeLog);
+ return PrimaryBase + (ClassId << Config::PrimaryRegionSizeLog);
}
- bool populateBatches(CacheT *C, RegionInfo *Region, uptr ClassId,
- TransferBatch **CurrentBatch, u32 MaxCount,
- void **PointersArray, u32 Count) {
- // No need to shuffle the batches size class.
- if (ClassId != SizeClassMap::BatchClassId)
- shuffle(PointersArray, Count, &Region->RandState);
- TransferBatch *B = *CurrentBatch;
- for (uptr I = 0; I < Count; I++) {
- if (B && B->getCount() == MaxCount) {
- Region->FreeList.push_back(B);
- B = nullptr;
- }
- if (!B) {
- B = C->createBatch(ClassId, PointersArray[I]);
- if (UNLIKELY(!B))
- return false;
- B->clear();
- }
- B->add(PointersArray[I]);
- }
- *CurrentBatch = B;
- return true;
+ static CompactPtrT compactPtrInternal(uptr Base, uptr Ptr) {
+ return static_cast<CompactPtrT>((Ptr - Base) >> CompactPtrScale);
+ }
+
+ static uptr decompactPtrInternal(uptr Base, CompactPtrT CompactPtr) {
+ return Base + (static_cast<uptr>(CompactPtr) << CompactPtrScale);
}
NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
// Map more space for blocks, if necessary.
if (TotalUserBytes > MappedUser) {
// Do the mmap for the user memory.
- const uptr UserMapSize =
+ const uptr MapSize =
roundUpTo(TotalUserBytes - MappedUser, MapSizeIncrement);
const uptr RegionBase = RegionBeg - getRegionBaseByClassId(ClassId);
- if (UNLIKELY(RegionBase + MappedUser + UserMapSize > RegionSize)) {
+ if (UNLIKELY(RegionBase + MappedUser + MapSize > RegionSize)) {
if (!Region->Exhausted) {
Region->Exhausted = true;
- ScopedString Str(1024);
+ ScopedString Str;
getStats(&Str);
Str.append(
- "Scudo OOM: The process has Exhausted %zuM for size class %zu.\n",
+ "Scudo OOM: The process has exhausted %zuM for size class %zu.\n",
RegionSize >> 20, Size);
Str.output();
}
return nullptr;
}
- if (UNLIKELY(MappedUser == 0))
+ if (MappedUser == 0)
Region->Data = Data;
- if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser),
- UserMapSize, "scudo:primary",
- MAP_ALLOWNOMEM | MAP_RESIZABLE |
- (useMemoryTagging() ? MAP_MEMTAG : 0),
- &Region->Data)))
+ if (UNLIKELY(!map(
+ reinterpret_cast<void *>(RegionBeg + MappedUser), MapSize,
+ "scudo:primary",
+ MAP_ALLOWNOMEM | MAP_RESIZABLE |
+ (useMemoryTagging<Config>(Options.load()) ? MAP_MEMTAG : 0),
+ &Region->Data)))
return nullptr;
- Region->MappedUser += UserMapSize;
- C->getStats().add(StatMapped, UserMapSize);
+ Region->MappedUser += MapSize;
+ C->getStats().add(StatMapped, MapSize);
}
const u32 NumberOfBlocks = Min(
static_cast<u32>((Region->MappedUser - Region->AllocatedUser) / Size));
DCHECK_GT(NumberOfBlocks, 0);
- TransferBatch *B = nullptr;
constexpr u32 ShuffleArraySize =
MaxNumBatches * TransferBatch::MaxNumCached;
- void *ShuffleArray[ShuffleArraySize];
- u32 Count = 0;
- const uptr P = RegionBeg + Region->AllocatedUser;
- const uptr AllocatedUser = Size * NumberOfBlocks;
- for (uptr I = P; I < P + AllocatedUser; I += Size) {
- ShuffleArray[Count++] = reinterpret_cast<void *>(I);
- if (Count == ShuffleArraySize) {
- if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount,
- ShuffleArray, Count)))
- return nullptr;
- Count = 0;
- }
- }
- if (Count) {
- if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount,
- ShuffleArray, Count)))
+ CompactPtrT ShuffleArray[ShuffleArraySize];
+ DCHECK_LE(NumberOfBlocks, ShuffleArraySize);
+
+ const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId);
+ uptr P = RegionBeg + Region->AllocatedUser;
+ for (u32 I = 0; I < NumberOfBlocks; I++, P += Size)
+ ShuffleArray[I] = compactPtrInternal(CompactPtrBase, P);
+ // No need to shuffle the batches size class.
+ if (ClassId != SizeClassMap::BatchClassId)
+ shuffle(ShuffleArray, NumberOfBlocks, &Region->RandState);
+ for (u32 I = 0; I < NumberOfBlocks;) {
+ TransferBatch *B =
+ C->createBatch(ClassId, reinterpret_cast<void *>(decompactPtrInternal(
+ CompactPtrBase, ShuffleArray[I])));
+ if (UNLIKELY(!B))
return nullptr;
- }
- DCHECK(B);
- if (!Region->FreeList.empty()) {
+ const u32 N = Min(MaxCount, NumberOfBlocks - I);
+ B->setFromArray(&ShuffleArray[I], N);
Region->FreeList.push_back(B);
- B = Region->FreeList.front();
- Region->FreeList.pop_front();
+ I += N;
}
+ TransferBatch *B = Region->FreeList.front();
+ Region->FreeList.pop_front();
+ DCHECK(B);
DCHECK_GT(B->getCount(), 0);
+ const uptr AllocatedUser = Size * NumberOfBlocks;
C->getStats().add(StatFree, AllocatedUser);
Region->AllocatedUser += AllocatedUser;
- Region->Exhausted = false;
return B;
}
getRegionBaseByClassId(ClassId));
}
- s32 getReleaseToOsIntervalMs() {
- return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
- }
-
NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
bool Force = false) {
const uptr BlockSize = getSizeByClassId(ClassId);
const uptr PageSize = getPageSizeCached();
- CHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks);
+ DCHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks);
const uptr BytesInFreeList =
Region->AllocatedUser -
(Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks) * BlockSize;
if (BytesPushed < PageSize)
return 0; // Nothing new to release.
+ // Releasing smaller blocks is expensive, so we want to make sure that a
+ // significant amount of bytes are free, and that there has been a good
+ // amount of batches pushed to the freelist before attempting to release.
+ if (BlockSize < PageSize / 16U) {
+ if (!Force && BytesPushed < Region->AllocatedUser / 16U)
+ return 0;
+ // We want 8x% to 9x% free bytes (the larger the block, the lower the %).
+ if ((BytesInFreeList * 100U) / Region->AllocatedUser <
+ (100U - 1U - BlockSize / 16U))
+ return 0;
+ }
+
if (!Force) {
- const s32 IntervalMs = getReleaseToOsIntervalMs();
+ const s32 IntervalMs = atomic_load_relaxed(&ReleaseToOsIntervalMs);
if (IntervalMs < 0)
return 0;
if (Region->ReleaseInfo.LastReleaseAtNs +
}
ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data);
- releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg,
- Region->AllocatedUser, BlockSize, &Recorder);
+ const uptr CompactPtrBase = getCompactPtrBaseByClassId(ClassId);
+ auto DecompactPtr = [CompactPtrBase](CompactPtrT CompactPtr) {
+ return decompactPtrInternal(CompactPtrBase, CompactPtr);
+ };
+ auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
+ releaseFreeMemoryToOS(Region->FreeList, Region->AllocatedUser, 1U,
+ BlockSize, &Recorder, DecompactPtr, SkipRegion);
if (Recorder.getReleasedRangesCount() > 0) {
Region->ReleaseInfo.PushedBlocksAtLastRelease =
// Per-thread cache of memory blocks.
template <typename Callback> class QuarantineCache {
public:
- void initLinkerInitialized() {}
- void init() {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized();
- }
+ void init() { DCHECK_EQ(atomic_load_relaxed(&Size), 0U); }
// Total memory used, including internal accounting.
uptr getSize() const { return atomic_load_relaxed(&Size); }
private:
SinglyLinkedList<QuarantineBatch> List;
- atomic_uptr Size;
+ atomic_uptr Size = {};
void addToSize(uptr add) { atomic_store_relaxed(&Size, getSize() + add); }
void subFromSize(uptr sub) { atomic_store_relaxed(&Size, getSize() - sub); }
template <typename Callback, typename Node> class GlobalQuarantine {
public:
typedef QuarantineCache<Callback> CacheT;
+ using ThisT = GlobalQuarantine<Callback, Node>;
- void initLinkerInitialized(uptr Size, uptr CacheSize) {
+ void init(uptr Size, uptr CacheSize) {
+ DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
+ DCHECK_EQ(atomic_load_relaxed(&MaxSize), 0U);
+ DCHECK_EQ(atomic_load_relaxed(&MinSize), 0U);
+ DCHECK_EQ(atomic_load_relaxed(&MaxCacheSize), 0U);
// Thread local quarantine size can be zero only when global quarantine size
// is zero (it allows us to perform just one atomic read per put() call).
CHECK((Size == 0 && CacheSize == 0) || CacheSize != 0);
atomic_store_relaxed(&MinSize, Size / 10 * 9); // 90% of max size.
atomic_store_relaxed(&MaxCacheSize, CacheSize);
- Cache.initLinkerInitialized();
- }
- void init(uptr Size, uptr CacheSize) {
- CacheMutex.init();
Cache.init();
- RecycleMutex.init();
- MinSize = {};
- MaxSize = {};
- MaxCacheSize = {};
- initLinkerInitialized(Size, CacheSize);
}
uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); }
alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex CacheMutex;
CacheT Cache;
alignas(SCUDO_CACHE_LINE_SIZE) HybridMutex RecycleMutex;
- atomic_uptr MinSize;
- atomic_uptr MaxSize;
- alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize;
+ atomic_uptr MinSize = {};
+ atomic_uptr MaxSize = {};
+ alignas(SCUDO_CACHE_LINE_SIZE) atomic_uptr MaxCacheSize = {};
void NOINLINE recycle(uptr MinSize, Callback Cb) {
CacheT Tmp;
namespace scudo {
HybridMutex PackedCounterArray::Mutex = {};
-uptr PackedCounterArray::StaticBuffer[1024];
+uptr PackedCounterArray::StaticBuffer[PackedCounterArray::StaticBufferCount];
} // namespace scudo
class ReleaseRecorder {
public:
- ReleaseRecorder(uptr BaseAddress, MapPlatformData *Data = nullptr)
- : BaseAddress(BaseAddress), Data(Data) {}
+ ReleaseRecorder(uptr Base, MapPlatformData *Data = nullptr)
+ : Base(Base), Data(Data) {}
uptr getReleasedRangesCount() const { return ReleasedRangesCount; }
uptr getReleasedBytes() const { return ReleasedBytes; }
+ uptr getBase() const { return Base; }
+
// Releases [From, To) range of pages back to OS.
void releasePageRangeToOS(uptr From, uptr To) {
const uptr Size = To - From;
- releasePagesToOS(BaseAddress, From, Size, Data);
+ releasePagesToOS(Base, From, Size, Data);
ReleasedRangesCount++;
ReleasedBytes += Size;
}
private:
uptr ReleasedRangesCount = 0;
uptr ReleasedBytes = 0;
- uptr BaseAddress = 0;
+ uptr Base = 0;
MapPlatformData *Data = nullptr;
};
// incremented past MaxValue.
class PackedCounterArray {
public:
- PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) {
- CHECK_GT(NumCounters, 0);
- CHECK_GT(MaxValue, 0);
+ PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion,
+ uptr MaxValue)
+ : Regions(NumberOfRegions), NumCounters(CountersPerRegion) {
+ DCHECK_GT(Regions, 0);
+ DCHECK_GT(NumCounters, 0);
+ DCHECK_GT(MaxValue, 0);
constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
// Rounding counter storage size up to the power of two allows for using
// bit shifts calculating particular counter's Index and offset.
const uptr CounterSizeBits =
roundUpToPowerOfTwo(getMostSignificantSetBitIndex(MaxValue) + 1);
- CHECK_LE(CounterSizeBits, MaxCounterBits);
+ DCHECK_LE(CounterSizeBits, MaxCounterBits);
CounterSizeBitsLog = getLog2(CounterSizeBits);
CounterMask = ~(static_cast<uptr>(0)) >> (MaxCounterBits - CounterSizeBits);
const uptr PackingRatio = MaxCounterBits >> CounterSizeBitsLog;
- CHECK_GT(PackingRatio, 0);
+ DCHECK_GT(PackingRatio, 0);
PackingRatioLog = getLog2(PackingRatio);
BitOffsetMask = PackingRatio - 1;
- BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >>
- PackingRatioLog) *
- sizeof(*Buffer);
- if (BufferSize <= StaticBufferSize && Mutex.tryLock()) {
+ SizePerRegion =
+ roundUpTo(NumCounters, static_cast<uptr>(1U) << PackingRatioLog) >>
+ PackingRatioLog;
+ BufferSize = SizePerRegion * sizeof(*Buffer) * Regions;
+ if (BufferSize <= (StaticBufferCount * sizeof(Buffer[0])) &&
+ Mutex.tryLock()) {
Buffer = &StaticBuffer[0];
memset(Buffer, 0, BufferSize);
} else {
Buffer = reinterpret_cast<uptr *>(
- map(nullptr, BufferSize, "scudo:counters", MAP_ALLOWNOMEM));
+ map(nullptr, roundUpTo(BufferSize, getPageSizeCached()),
+ "scudo:counters", MAP_ALLOWNOMEM));
}
}
~PackedCounterArray() {
if (Buffer == &StaticBuffer[0])
Mutex.unlock();
else
- unmap(reinterpret_cast<void *>(Buffer), BufferSize);
+ unmap(reinterpret_cast<void *>(Buffer),
+ roundUpTo(BufferSize, getPageSizeCached()));
}
bool isAllocated() const { return !!Buffer; }
- uptr getCount() const { return N; }
+ uptr getCount() const { return NumCounters; }
- uptr get(uptr I) const {
- DCHECK_LT(I, N);
+ uptr get(uptr Region, uptr I) const {
+ DCHECK_LT(Region, Regions);
+ DCHECK_LT(I, NumCounters);
const uptr Index = I >> PackingRatioLog;
const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
- return (Buffer[Index] >> BitOffset) & CounterMask;
+ return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask;
}
- void inc(uptr I) const {
- DCHECK_LT(get(I), CounterMask);
+ void inc(uptr Region, uptr I) const {
+ DCHECK_LT(get(Region, I), CounterMask);
const uptr Index = I >> PackingRatioLog;
const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
- Buffer[Index] += static_cast<uptr>(1U) << BitOffset;
+ Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U)
+ << BitOffset;
}
- void incRange(uptr From, uptr To) const {
+ void incRange(uptr Region, uptr From, uptr To) const {
DCHECK_LE(From, To);
- const uptr Top = Min(To + 1, N);
+ const uptr Top = Min(To + 1, NumCounters);
for (uptr I = From; I < Top; I++)
- inc(I);
+ inc(Region, I);
}
uptr getBufferSize() const { return BufferSize; }
+ static const uptr StaticBufferCount = 2048U;
+
private:
- const uptr N;
+ const uptr Regions;
+ const uptr NumCounters;
uptr CounterSizeBitsLog;
uptr CounterMask;
uptr PackingRatioLog;
uptr BitOffsetMask;
+ uptr SizePerRegion;
uptr BufferSize;
uptr *Buffer;
static HybridMutex Mutex;
- static const uptr StaticBufferSize = 1024U;
- static uptr StaticBuffer[StaticBufferSize];
+ static uptr StaticBuffer[StaticBufferCount];
};
template <class ReleaseRecorderT> class FreePagesRangeTracker {
CurrentPage++;
}
+ void skipPages(uptr N) {
+ closeOpenedRange();
+ CurrentPage += N;
+ }
+
void finish() { closeOpenedRange(); }
private:
uptr CurrentRangeStatePage = 0;
};
-template <class TransferBatchT, class ReleaseRecorderT>
+template <class TransferBatchT, class ReleaseRecorderT, typename DecompactPtrT,
+ typename SkipRegionT>
NOINLINE void
-releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
- uptr Size, uptr BlockSize, ReleaseRecorderT *Recorder) {
+releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
+ uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
+ ReleaseRecorderT *Recorder, DecompactPtrT DecompactPtr,
+ SkipRegionT SkipRegion) {
const uptr PageSize = getPageSizeCached();
// Figure out the number of chunks per page and whether we can take a fast
}
}
- const uptr PagesCount = roundUpTo(Size, PageSize) / PageSize;
- PackedCounterArray Counters(PagesCount, FullPagesBlockCountMax);
+ const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
+ PackedCounterArray Counters(NumberOfRegions, PagesCount,
+ FullPagesBlockCountMax);
if (!Counters.isAllocated())
return;
const uptr PageSizeLog = getLog2(PageSize);
- const uptr RoundedSize = PagesCount << PageSizeLog;
+ const uptr RoundedRegionSize = PagesCount << PageSizeLog;
+ const uptr RoundedSize = NumberOfRegions * RoundedRegionSize;
// Iterate over free chunks and count how many free chunks affect each
// allocated page.
if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
// Each chunk affects one page only.
for (const auto &It : FreeList) {
- // If dealing with a TransferBatch, the first pointer of the batch will
- // point to the batch itself, we do not want to mark this for release as
- // the batch is in use, so skip the first entry.
- const bool IsTransferBatch =
- (It.getCount() != 0) &&
- (reinterpret_cast<uptr>(It.get(0)) == reinterpret_cast<uptr>(&It));
- for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
- const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
- // This takes care of P < Base and P >= Base + RoundedSize.
- if (P < RoundedSize)
- Counters.inc(P >> PageSizeLog);
+ for (u32 I = 0; I < It.getCount(); I++) {
+ const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
+ if (P >= RoundedSize)
+ continue;
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ const uptr PInRegion = P - RegionIndex * RegionSize;
+ Counters.inc(RegionIndex, PInRegion >> PageSizeLog);
}
}
- for (uptr P = Size; P < RoundedSize; P += BlockSize)
- Counters.inc(P >> PageSizeLog);
} else {
// In all other cases chunks might affect more than one page.
+ DCHECK_GE(RegionSize, BlockSize);
+ const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
for (const auto &It : FreeList) {
- // See TransferBatch comment above.
- const bool IsTransferBatch =
- (It.getCount() != 0) &&
- (reinterpret_cast<uptr>(It.get(0)) == reinterpret_cast<uptr>(&It));
- for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
- const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
- // This takes care of P < Base and P >= Base + RoundedSize.
- if (P < RoundedSize)
- Counters.incRange(P >> PageSizeLog,
- (P + BlockSize - 1) >> PageSizeLog);
+ for (u32 I = 0; I < It.getCount(); I++) {
+ const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
+ if (P >= RoundedSize)
+ continue;
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ uptr PInRegion = P - RegionIndex * RegionSize;
+ Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+ (PInRegion + BlockSize - 1) >> PageSizeLog);
+ // The last block in a region might straddle a page, so if it's
+ // free, we mark the following "pretend" memory block(s) as free.
+ if (PInRegion == LastBlockInRegion) {
+ PInRegion += BlockSize;
+ while (PInRegion < RoundedRegionSize) {
+ Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+ (PInRegion + BlockSize - 1) >> PageSizeLog);
+ PInRegion += BlockSize;
+ }
+ }
}
}
- for (uptr P = Size; P < RoundedSize; P += BlockSize)
- Counters.incRange(P >> PageSizeLog, (P + BlockSize - 1) >> PageSizeLog);
}
// Iterate over pages detecting ranges of pages with chunk Counters equal
FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder);
if (SameBlockCountPerPage) {
// Fast path, every page has the same number of chunks affecting it.
- for (uptr I = 0; I < Counters.getCount(); I++)
- RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax);
+ for (uptr I = 0; I < NumberOfRegions; I++) {
+ if (SkipRegion(I)) {
+ RangeTracker.skipPages(PagesCount);
+ continue;
+ }
+ for (uptr J = 0; J < PagesCount; J++)
+ RangeTracker.processNextPage(Counters.get(I, J) ==
+ FullPagesBlockCountMax);
+ }
} else {
// Slow path, go through the pages keeping count how many chunks affect
// each page.
// except the first and the last one) and then the last chunk size, adding
// up the number of chunks on the current page and checking on every step
// whether the page boundary was crossed.
- uptr PrevPageBoundary = 0;
- uptr CurrentBoundary = 0;
- for (uptr I = 0; I < Counters.getCount(); I++) {
- const uptr PageBoundary = PrevPageBoundary + PageSize;
- uptr BlocksPerPage = Pn;
- if (CurrentBoundary < PageBoundary) {
- if (CurrentBoundary > PrevPageBoundary)
- BlocksPerPage++;
- CurrentBoundary += Pnc;
+ for (uptr I = 0; I < NumberOfRegions; I++) {
+ if (SkipRegion(I)) {
+ RangeTracker.skipPages(PagesCount);
+ continue;
+ }
+ uptr PrevPageBoundary = 0;
+ uptr CurrentBoundary = 0;
+ for (uptr J = 0; J < PagesCount; J++) {
+ const uptr PageBoundary = PrevPageBoundary + PageSize;
+ uptr BlocksPerPage = Pn;
if (CurrentBoundary < PageBoundary) {
- BlocksPerPage++;
- CurrentBoundary += BlockSize;
+ if (CurrentBoundary > PrevPageBoundary)
+ BlocksPerPage++;
+ CurrentBoundary += Pnc;
+ if (CurrentBoundary < PageBoundary) {
+ BlocksPerPage++;
+ CurrentBoundary += BlockSize;
+ }
}
+ PrevPageBoundary = PageBoundary;
+ RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage);
}
- PrevPageBoundary = PageBoundary;
-
- RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage);
}
}
RangeTracker.finish();
class ScopedErrorReport {
public:
- ScopedErrorReport() : Message(512) { Message.append("Scudo ERROR: "); }
+ ScopedErrorReport() : Message() { Message.append("Scudo ERROR: "); }
void append(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
trap();
}
ScopedErrorReport Report;
- Report.append("CHECK failed @ %s:%d %s (%llu, %llu)\n", File, Line, Condition,
- Value1, Value2);
+ Report.append("CHECK failed @ %s:%d %s ((u64)op1=%llu, (u64)op2=%llu)\n",
+ File, Line, Condition, Value1, Value2);
}
// Generic string fatal error message.
#ifndef SCUDO_SECONDARY_H_
#define SCUDO_SECONDARY_H_
+#include "chunk.h"
#include "common.h"
#include "list.h"
+#include "memtag.h"
#include "mutex.h"
+#include "options.h"
#include "stats.h"
#include "string_utils.h"
namespace LargeBlock {
-struct Header {
+struct alignas(Max<uptr>(archSupportsMemoryTagging()
+ ? archMemoryTagGranuleSize()
+ : 1,
+ 1U << SCUDO_MIN_ALIGNMENT_LOG)) Header {
LargeBlock::Header *Prev;
LargeBlock::Header *Next;
- uptr BlockEnd;
+ uptr CommitBase;
+ uptr CommitSize;
uptr MapBase;
uptr MapSize;
- MapPlatformData Data;
+ [[no_unique_address]] MapPlatformData Data;
};
-constexpr uptr getHeaderSize() {
- return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG);
+static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, "");
+static_assert(!archSupportsMemoryTagging() ||
+ sizeof(Header) % archMemoryTagGranuleSize() == 0,
+ "");
+
+constexpr uptr getHeaderSize() { return sizeof(Header); }
+
+template <typename Config> static uptr addHeaderTag(uptr Ptr) {
+ if (allocatorSupportsMemoryTagging<Config>())
+ return addFixedTag(Ptr, 1);
+ return Ptr;
}
-static Header *getHeader(uptr Ptr) {
- return reinterpret_cast<Header *>(Ptr - getHeaderSize());
+template <typename Config> static Header *getHeader(uptr Ptr) {
+ return reinterpret_cast<Header *>(addHeaderTag<Config>(Ptr)) - 1;
}
-static Header *getHeader(const void *Ptr) {
- return getHeader(reinterpret_cast<uptr>(Ptr));
+template <typename Config> static Header *getHeader(const void *Ptr) {
+ return getHeader<Config>(reinterpret_cast<uptr>(Ptr));
}
} // namespace LargeBlock
+static void unmap(LargeBlock::Header *H) {
+ MapPlatformData Data = H->Data;
+ unmap(reinterpret_cast<void *>(H->MapBase), H->MapSize, UNMAP_ALL, &Data);
+}
+
class MapAllocatorNoCache {
public:
- void initLinkerInitialized(UNUSED s32 ReleaseToOsInterval) {}
void init(UNUSED s32 ReleaseToOsInterval) {}
- bool retrieve(UNUSED uptr Size, UNUSED LargeBlock::Header **H) {
+ bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment,
+ UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) {
return false;
}
- bool store(UNUSED LargeBlock::Header *H) { return false; }
- static bool canCache(UNUSED uptr Size) { return false; }
+ void store(UNUSED Options Options, LargeBlock::Header *H) { unmap(H); }
+ bool canCache(UNUSED uptr Size) { return false; }
void disable() {}
void enable() {}
void releaseToOS() {}
- void setReleaseToOsIntervalMs(UNUSED s32 Interval) {}
+ void disableMemoryTagging() {}
+ void unmapTestOnly() {}
+ bool setOption(Option O, UNUSED sptr Value) {
+ if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount ||
+ O == Option::MaxCacheEntrySize)
+ return false;
+ // Not supported by the Secondary Cache, but not an error either.
+ return true;
+ }
};
-template <uptr MaxEntriesCount = 32U, uptr MaxEntrySize = 1UL << 19,
- s32 MinReleaseToOsIntervalMs = INT32_MIN,
- s32 MaxReleaseToOsIntervalMs = INT32_MAX>
-class MapAllocatorCache {
+static const uptr MaxUnusedCachePages = 4U;
+
+template <typename Config>
+void mapSecondary(Options Options, uptr CommitBase, uptr CommitSize,
+ uptr AllocPos, uptr Flags, MapPlatformData *Data) {
+ const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * getPageSizeCached();
+ if (useMemoryTagging<Config>(Options) && CommitSize > MaxUnusedCacheBytes) {
+ const uptr UntaggedPos = Max(AllocPos, CommitBase + MaxUnusedCacheBytes);
+ map(reinterpret_cast<void *>(CommitBase), UntaggedPos - CommitBase,
+ "scudo:secondary", MAP_RESIZABLE | MAP_MEMTAG | Flags, Data);
+ map(reinterpret_cast<void *>(UntaggedPos),
+ CommitBase + CommitSize - UntaggedPos, "scudo:secondary",
+ MAP_RESIZABLE | Flags, Data);
+ } else {
+ map(reinterpret_cast<void *>(CommitBase), CommitSize, "scudo:secondary",
+ MAP_RESIZABLE | (useMemoryTagging<Config>(Options) ? MAP_MEMTAG : 0) |
+ Flags,
+ Data);
+ }
+}
+
+template <typename Config> class MapAllocatorCache {
public:
- // Fuchsia doesn't allow releasing Secondary blocks yet. Note that 0 length
- // arrays are an extension for some compilers.
- // FIXME(kostyak): support (partially) the cache on Fuchsia.
- static_assert(!SCUDO_FUCHSIA || MaxEntriesCount == 0U, "");
+ // Ensure the default maximum specified fits the array.
+ static_assert(Config::SecondaryCacheDefaultMaxEntriesCount <=
+ Config::SecondaryCacheEntriesArraySize,
+ "");
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
- setReleaseToOsIntervalMs(ReleaseToOsInterval);
- }
void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
+ DCHECK_EQ(EntriesCount, 0U);
+ setOption(Option::MaxCacheEntriesCount,
+ static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntriesCount));
+ setOption(Option::MaxCacheEntrySize,
+ static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntrySize));
+ setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
- bool store(LargeBlock::Header *H) {
+ void store(Options Options, LargeBlock::Header *H) {
+ if (!canCache(H->CommitSize))
+ return unmap(H);
+
bool EntryCached = false;
bool EmptyCache = false;
+ const s32 Interval = atomic_load_relaxed(&ReleaseToOsIntervalMs);
const u64 Time = getMonotonicTime();
- {
+ const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
+ CachedBlock Entry;
+ Entry.CommitBase = H->CommitBase;
+ Entry.CommitSize = H->CommitSize;
+ Entry.MapBase = H->MapBase;
+ Entry.MapSize = H->MapSize;
+ Entry.BlockBegin = reinterpret_cast<uptr>(H + 1);
+ Entry.Data = H->Data;
+ Entry.Time = Time;
+ if (useMemoryTagging<Config>(Options)) {
+ if (Interval == 0 && !SCUDO_FUCHSIA) {
+ // Release the memory and make it inaccessible at the same time by
+ // creating a new MAP_NOACCESS mapping on top of the existing mapping.
+ // Fuchsia does not support replacing mappings by creating a new mapping
+ // on top so we just do the two syscalls there.
+ Entry.Time = 0;
+ mapSecondary<Config>(Options, Entry.CommitBase, Entry.CommitSize,
+ Entry.CommitBase, MAP_NOACCESS, &Entry.Data);
+ } else {
+ setMemoryPermission(Entry.CommitBase, Entry.CommitSize, MAP_NOACCESS,
+ &Entry.Data);
+ }
+ } else if (Interval == 0) {
+ releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
+ Entry.Time = 0;
+ }
+ do {
ScopedLock L(Mutex);
- if (EntriesCount == MaxEntriesCount) {
+ if (useMemoryTagging<Config>(Options) && QuarantinePos == -1U) {
+ // If we get here then memory tagging was disabled in between when we
+ // read Options and when we locked Mutex. We can't insert our entry into
+ // the quarantine or the cache because the permissions would be wrong so
+ // just unmap it.
+ break;
+ }
+ if (Config::SecondaryCacheQuarantineSize &&
+ useMemoryTagging<Config>(Options)) {
+ QuarantinePos =
+ (QuarantinePos + 1) % Max(Config::SecondaryCacheQuarantineSize, 1u);
+ if (!Quarantine[QuarantinePos].CommitBase) {
+ Quarantine[QuarantinePos] = Entry;
+ return;
+ }
+ CachedBlock PrevEntry = Quarantine[QuarantinePos];
+ Quarantine[QuarantinePos] = Entry;
+ if (OldestTime == 0)
+ OldestTime = Entry.Time;
+ Entry = PrevEntry;
+ }
+ if (EntriesCount >= MaxCount) {
if (IsFullEvents++ == 4U)
EmptyCache = true;
} else {
- for (uptr I = 0; I < MaxEntriesCount; I++) {
- if (Entries[I].Block)
+ for (u32 I = 0; I < MaxCount; I++) {
+ if (Entries[I].CommitBase)
continue;
if (I != 0)
Entries[I] = Entries[0];
- Entries[0].Block = reinterpret_cast<uptr>(H);
- Entries[0].BlockEnd = H->BlockEnd;
- Entries[0].MapBase = H->MapBase;
- Entries[0].MapSize = H->MapSize;
- Entries[0].Data = H->Data;
- Entries[0].Time = Time;
+ Entries[0] = Entry;
EntriesCount++;
+ if (OldestTime == 0)
+ OldestTime = Entry.Time;
EntryCached = true;
break;
}
}
- }
- s32 Interval;
+ } while (0);
if (EmptyCache)
empty();
- else if ((Interval = getReleaseToOsIntervalMs()) >= 0)
+ else if (Interval >= 0)
releaseOlderThan(Time - static_cast<u64>(Interval) * 1000000);
- return EntryCached;
+ if (!EntryCached)
+ unmap(reinterpret_cast<void *>(Entry.MapBase), Entry.MapSize, UNMAP_ALL,
+ &Entry.Data);
}
- bool retrieve(uptr Size, LargeBlock::Header **H) {
+ bool retrieve(Options Options, uptr Size, uptr Alignment,
+ LargeBlock::Header **H, bool *Zeroed) {
const uptr PageSize = getPageSizeCached();
- ScopedLock L(Mutex);
- if (EntriesCount == 0)
- return false;
- for (uptr I = 0; I < MaxEntriesCount; I++) {
- if (!Entries[I].Block)
- continue;
- const uptr BlockSize = Entries[I].BlockEnd - Entries[I].Block;
- if (Size > BlockSize)
- continue;
- if (Size < BlockSize - PageSize * 4U)
- continue;
- *H = reinterpret_cast<LargeBlock::Header *>(Entries[I].Block);
- Entries[I].Block = 0;
- (*H)->BlockEnd = Entries[I].BlockEnd;
- (*H)->MapBase = Entries[I].MapBase;
- (*H)->MapSize = Entries[I].MapSize;
- (*H)->Data = Entries[I].Data;
+ const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
+ bool Found = false;
+ CachedBlock Entry;
+ uptr HeaderPos;
+ {
+ ScopedLock L(Mutex);
+ if (EntriesCount == 0)
+ return false;
+ for (u32 I = 0; I < MaxCount; I++) {
+ const uptr CommitBase = Entries[I].CommitBase;
+ if (!CommitBase)
+ continue;
+ const uptr CommitSize = Entries[I].CommitSize;
+ const uptr AllocPos =
+ roundDownTo(CommitBase + CommitSize - Size, Alignment);
+ HeaderPos =
+ AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
+ if (HeaderPos > CommitBase + CommitSize)
+ continue;
+ if (HeaderPos < CommitBase ||
+ AllocPos > CommitBase + PageSize * MaxUnusedCachePages)
+ continue;
+ Found = true;
+ Entry = Entries[I];
+ Entries[I].CommitBase = 0;
+ break;
+ }
+ }
+ if (Found) {
+ *H = reinterpret_cast<LargeBlock::Header *>(
+ LargeBlock::addHeaderTag<Config>(HeaderPos));
+ *Zeroed = Entry.Time == 0;
+ if (useMemoryTagging<Config>(Options))
+ setMemoryPermission(Entry.CommitBase, Entry.CommitSize, 0, &Entry.Data);
+ uptr NewBlockBegin = reinterpret_cast<uptr>(*H + 1);
+ if (useMemoryTagging<Config>(Options)) {
+ if (*Zeroed)
+ storeTags(LargeBlock::addHeaderTag<Config>(Entry.CommitBase),
+ NewBlockBegin);
+ else if (Entry.BlockBegin < NewBlockBegin)
+ storeTags(Entry.BlockBegin, NewBlockBegin);
+ else
+ storeTags(untagPointer(NewBlockBegin),
+ untagPointer(Entry.BlockBegin));
+ }
+ (*H)->CommitBase = Entry.CommitBase;
+ (*H)->CommitSize = Entry.CommitSize;
+ (*H)->MapBase = Entry.MapBase;
+ (*H)->MapSize = Entry.MapSize;
+ (*H)->Data = Entry.Data;
EntriesCount--;
- return true;
}
- return false;
+ return Found;
}
- static bool canCache(uptr Size) {
- return MaxEntriesCount != 0U && Size <= MaxEntrySize;
+ bool canCache(uptr Size) {
+ return atomic_load_relaxed(&MaxEntriesCount) != 0U &&
+ Size <= atomic_load_relaxed(&MaxEntrySize);
}
- void setReleaseToOsIntervalMs(s32 Interval) {
- if (Interval >= MaxReleaseToOsIntervalMs) {
- Interval = MaxReleaseToOsIntervalMs;
- } else if (Interval <= MinReleaseToOsIntervalMs) {
- Interval = MinReleaseToOsIntervalMs;
+ bool setOption(Option O, sptr Value) {
+ if (O == Option::ReleaseInterval) {
+ const s32 Interval =
+ Max(Min(static_cast<s32>(Value),
+ Config::SecondaryCacheMaxReleaseToOsIntervalMs),
+ Config::SecondaryCacheMinReleaseToOsIntervalMs);
+ atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval);
+ return true;
+ }
+ if (O == Option::MaxCacheEntriesCount) {
+ const u32 MaxCount = static_cast<u32>(Value);
+ if (MaxCount > Config::SecondaryCacheEntriesArraySize)
+ return false;
+ atomic_store_relaxed(&MaxEntriesCount, MaxCount);
+ return true;
}
- atomic_store(&ReleaseToOsIntervalMs, Interval, memory_order_relaxed);
+ if (O == Option::MaxCacheEntrySize) {
+ atomic_store_relaxed(&MaxEntrySize, static_cast<uptr>(Value));
+ return true;
+ }
+ // Not supported by the Secondary Cache, but not an error either.
+ return true;
}
void releaseToOS() { releaseOlderThan(UINT64_MAX); }
+ void disableMemoryTagging() {
+ ScopedLock L(Mutex);
+ for (u32 I = 0; I != Config::SecondaryCacheQuarantineSize; ++I) {
+ if (Quarantine[I].CommitBase) {
+ unmap(reinterpret_cast<void *>(Quarantine[I].MapBase),
+ Quarantine[I].MapSize, UNMAP_ALL, &Quarantine[I].Data);
+ Quarantine[I].CommitBase = 0;
+ }
+ }
+ const u32 MaxCount = atomic_load_relaxed(&MaxEntriesCount);
+ for (u32 I = 0; I < MaxCount; I++)
+ if (Entries[I].CommitBase)
+ setMemoryPermission(Entries[I].CommitBase, Entries[I].CommitSize, 0,
+ &Entries[I].Data);
+ QuarantinePos = -1U;
+ }
+
void disable() { Mutex.lock(); }
void enable() { Mutex.unlock(); }
+ void unmapTestOnly() { empty(); }
+
private:
void empty() {
struct {
void *MapBase;
uptr MapSize;
MapPlatformData Data;
- } MapInfo[MaxEntriesCount];
+ } MapInfo[Config::SecondaryCacheEntriesArraySize];
uptr N = 0;
{
ScopedLock L(Mutex);
- for (uptr I = 0; I < MaxEntriesCount; I++) {
- if (!Entries[I].Block)
+ for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++) {
+ if (!Entries[I].CommitBase)
continue;
MapInfo[N].MapBase = reinterpret_cast<void *>(Entries[I].MapBase);
MapInfo[N].MapSize = Entries[I].MapSize;
MapInfo[N].Data = Entries[I].Data;
- Entries[I].Block = 0;
+ Entries[I].CommitBase = 0;
N++;
}
EntriesCount = 0;
&MapInfo[I].Data);
}
- void releaseOlderThan(u64 Time) {
- ScopedLock L(Mutex);
- if (!EntriesCount)
- return;
- for (uptr I = 0; I < MaxEntriesCount; I++) {
- if (!Entries[I].Block || !Entries[I].Time || Entries[I].Time > Time)
- continue;
- releasePagesToOS(Entries[I].Block, 0,
- Entries[I].BlockEnd - Entries[I].Block,
- &Entries[I].Data);
- Entries[I].Time = 0;
- }
- }
-
- s32 getReleaseToOsIntervalMs() {
- return atomic_load(&ReleaseToOsIntervalMs, memory_order_relaxed);
- }
-
struct CachedBlock {
- uptr Block;
- uptr BlockEnd;
+ uptr CommitBase;
+ uptr CommitSize;
uptr MapBase;
uptr MapSize;
- MapPlatformData Data;
+ uptr BlockBegin;
+ [[no_unique_address]] MapPlatformData Data;
u64 Time;
};
+ void releaseIfOlderThan(CachedBlock &Entry, u64 Time) {
+ if (!Entry.CommitBase || !Entry.Time)
+ return;
+ if (Entry.Time > Time) {
+ if (OldestTime == 0 || Entry.Time < OldestTime)
+ OldestTime = Entry.Time;
+ return;
+ }
+ releasePagesToOS(Entry.CommitBase, 0, Entry.CommitSize, &Entry.Data);
+ Entry.Time = 0;
+ }
+
+ void releaseOlderThan(u64 Time) {
+ ScopedLock L(Mutex);
+ if (!EntriesCount || OldestTime == 0 || OldestTime > Time)
+ return;
+ OldestTime = 0;
+ for (uptr I = 0; I < Config::SecondaryCacheQuarantineSize; I++)
+ releaseIfOlderThan(Quarantine[I], Time);
+ for (uptr I = 0; I < Config::SecondaryCacheEntriesArraySize; I++)
+ releaseIfOlderThan(Entries[I], Time);
+ }
+
HybridMutex Mutex;
- CachedBlock Entries[MaxEntriesCount];
- u32 EntriesCount;
- uptr LargestSize;
- u32 IsFullEvents;
- atomic_s32 ReleaseToOsIntervalMs;
+ u32 EntriesCount = 0;
+ u32 QuarantinePos = 0;
+ atomic_u32 MaxEntriesCount = {};
+ atomic_uptr MaxEntrySize = {};
+ u64 OldestTime = 0;
+ u32 IsFullEvents = 0;
+ atomic_s32 ReleaseToOsIntervalMs = {};
+
+ CachedBlock Entries[Config::SecondaryCacheEntriesArraySize] = {};
+ CachedBlock Quarantine[Config::SecondaryCacheQuarantineSize] = {};
};
-template <class CacheT> class MapAllocator {
+template <typename Config> class MapAllocator {
public:
- void initLinkerInitialized(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
- Cache.initLinkerInitialized(ReleaseToOsInterval);
- Stats.initLinkerInitialized();
+ void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
+ DCHECK_EQ(AllocatedBytes, 0U);
+ DCHECK_EQ(FreedBytes, 0U);
+ Cache.init(ReleaseToOsInterval);
+ Stats.init();
if (LIKELY(S))
S->link(&Stats);
}
- void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(S, ReleaseToOsInterval);
- }
- void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr,
+ void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0,
+ uptr *BlockEnd = nullptr,
FillContentsMode FillContents = NoFill);
- void deallocate(void *Ptr);
+ void deallocate(Options Options, void *Ptr);
static uptr getBlockEnd(void *Ptr) {
- return LargeBlock::getHeader(Ptr)->BlockEnd;
+ auto *B = LargeBlock::getHeader<Config>(Ptr);
+ return B->CommitBase + B->CommitSize;
}
static uptr getBlockSize(void *Ptr) {
}
template <typename F> void iterateOverBlocks(F Callback) const {
- for (const auto &H : InUseBlocks)
- Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize());
+ for (const auto &H : InUseBlocks) {
+ uptr Ptr = reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize();
+ if (allocatorSupportsMemoryTagging<Config>())
+ Ptr = untagPointer(Ptr);
+ Callback(Ptr);
+ }
}
- static uptr canCache(uptr Size) { return CacheT::canCache(Size); }
+ uptr canCache(uptr Size) { return Cache.canCache(Size); }
- void setReleaseToOsIntervalMs(s32 Interval) {
- Cache.setReleaseToOsIntervalMs(Interval);
- }
+ bool setOption(Option O, sptr Value) { return Cache.setOption(O, Value); }
void releaseToOS() { Cache.releaseToOS(); }
+ void disableMemoryTagging() { Cache.disableMemoryTagging(); }
+
+ void unmapTestOnly() { Cache.unmapTestOnly(); }
+
private:
- CacheT Cache;
+ typename Config::SecondaryCache Cache;
HybridMutex Mutex;
DoublyLinkedList<LargeBlock::Header> InUseBlocks;
- uptr AllocatedBytes;
- uptr FreedBytes;
- uptr LargestSize;
- u32 NumberOfAllocs;
- u32 NumberOfFrees;
+ uptr AllocatedBytes = 0;
+ uptr FreedBytes = 0;
+ uptr LargestSize = 0;
+ u32 NumberOfAllocs = 0;
+ u32 NumberOfFrees = 0;
LocalStats Stats;
};
// For allocations requested with an alignment greater than or equal to a page,
// the committed memory will amount to something close to Size - AlignmentHint
// (pending rounding and headers).
-template <class CacheT>
-void *MapAllocator<CacheT>::allocate(uptr Size, uptr AlignmentHint,
- uptr *BlockEnd,
+template <typename Config>
+void *MapAllocator<Config>::allocate(Options Options, uptr Size, uptr Alignment,
+ uptr *BlockEndPtr,
FillContentsMode FillContents) {
- DCHECK_GE(Size, AlignmentHint);
+ if (Options.get(OptionBit::AddLargeAllocationSlack))
+ Size += 1UL << SCUDO_MIN_ALIGNMENT_LOG;
+ Alignment = Max(Alignment, 1UL << SCUDO_MIN_ALIGNMENT_LOG);
const uptr PageSize = getPageSizeCached();
- const uptr RoundedSize =
- roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize);
-
- if (AlignmentHint < PageSize && CacheT::canCache(RoundedSize)) {
+ uptr RoundedSize =
+ roundUpTo(roundUpTo(Size, Alignment) + LargeBlock::getHeaderSize() +
+ Chunk::getHeaderSize(),
+ PageSize);
+ if (Alignment > PageSize)
+ RoundedSize += Alignment - PageSize;
+
+ if (Alignment < PageSize && Cache.canCache(RoundedSize)) {
LargeBlock::Header *H;
- if (Cache.retrieve(RoundedSize, &H)) {
- if (BlockEnd)
- *BlockEnd = H->BlockEnd;
- void *Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(H) +
- LargeBlock::getHeaderSize());
- if (FillContents)
+ bool Zeroed;
+ if (Cache.retrieve(Options, Size, Alignment, &H, &Zeroed)) {
+ const uptr BlockEnd = H->CommitBase + H->CommitSize;
+ if (BlockEndPtr)
+ *BlockEndPtr = BlockEnd;
+ uptr HInt = reinterpret_cast<uptr>(H);
+ if (allocatorSupportsMemoryTagging<Config>())
+ HInt = untagPointer(HInt);
+ const uptr PtrInt = HInt + LargeBlock::getHeaderSize();
+ void *Ptr = reinterpret_cast<void *>(PtrInt);
+ if (FillContents && !Zeroed)
memset(Ptr, FillContents == ZeroFill ? 0 : PatternFillByte,
- H->BlockEnd - reinterpret_cast<uptr>(Ptr));
- const uptr BlockSize = H->BlockEnd - reinterpret_cast<uptr>(H);
+ BlockEnd - PtrInt);
+ const uptr BlockSize = BlockEnd - HInt;
{
ScopedLock L(Mutex);
InUseBlocks.push_back(H);
MapPlatformData Data = {};
const uptr MapSize = RoundedSize + 2 * PageSize;
- uptr MapBase =
- reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary",
- MAP_NOACCESS | MAP_ALLOWNOMEM, &Data));
+ uptr MapBase = reinterpret_cast<uptr>(
+ map(nullptr, MapSize, nullptr, MAP_NOACCESS | MAP_ALLOWNOMEM, &Data));
if (UNLIKELY(!MapBase))
return nullptr;
uptr CommitBase = MapBase + PageSize;
// In the unlikely event of alignments larger than a page, adjust the amount
// of memory we want to commit, and trim the extra memory.
- if (UNLIKELY(AlignmentHint >= PageSize)) {
+ if (UNLIKELY(Alignment >= PageSize)) {
// For alignments greater than or equal to a page, the user pointer (eg: the
// pointer that is returned by the C or C++ allocation APIs) ends up on a
// page boundary , and our headers will live in the preceding page.
- CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize;
+ CommitBase = roundUpTo(MapBase + PageSize + 1, Alignment) - PageSize;
const uptr NewMapBase = CommitBase - PageSize;
DCHECK_GE(NewMapBase, MapBase);
// We only trim the extra memory on 32-bit platforms: 64-bit platforms
unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data);
MapBase = NewMapBase;
}
- const uptr NewMapEnd = CommitBase + PageSize +
- roundUpTo((Size - AlignmentHint), PageSize) +
- PageSize;
+ const uptr NewMapEnd =
+ CommitBase + PageSize + roundUpTo(Size, PageSize) + PageSize;
DCHECK_LE(NewMapEnd, MapEnd);
if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) {
unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data);
}
const uptr CommitSize = MapEnd - PageSize - CommitBase;
- const uptr Ptr =
- reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase),
- CommitSize, "scudo:secondary", 0, &Data));
- LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr);
+ const uptr AllocPos = roundDownTo(CommitBase + CommitSize - Size, Alignment);
+ mapSecondary<Config>(Options, CommitBase, CommitSize, AllocPos, 0, &Data);
+ const uptr HeaderPos =
+ AllocPos - Chunk::getHeaderSize() - LargeBlock::getHeaderSize();
+ LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(
+ LargeBlock::addHeaderTag<Config>(HeaderPos));
+ if (useMemoryTagging<Config>(Options))
+ storeTags(LargeBlock::addHeaderTag<Config>(CommitBase),
+ reinterpret_cast<uptr>(H + 1));
H->MapBase = MapBase;
H->MapSize = MapEnd - MapBase;
- H->BlockEnd = CommitBase + CommitSize;
+ H->CommitBase = CommitBase;
+ H->CommitSize = CommitSize;
H->Data = Data;
- if (BlockEnd)
- *BlockEnd = CommitBase + CommitSize;
+ if (BlockEndPtr)
+ *BlockEndPtr = CommitBase + CommitSize;
{
ScopedLock L(Mutex);
InUseBlocks.push_back(H);
Stats.add(StatAllocated, CommitSize);
Stats.add(StatMapped, H->MapSize);
}
- return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize());
+ return reinterpret_cast<void *>(HeaderPos + LargeBlock::getHeaderSize());
}
-template <class CacheT> void MapAllocator<CacheT>::deallocate(void *Ptr) {
- LargeBlock::Header *H = LargeBlock::getHeader(Ptr);
- const uptr Block = reinterpret_cast<uptr>(H);
- const uptr CommitSize = H->BlockEnd - Block;
+template <typename Config>
+void MapAllocator<Config>::deallocate(Options Options, void *Ptr) {
+ LargeBlock::Header *H = LargeBlock::getHeader<Config>(Ptr);
+ const uptr CommitSize = H->CommitSize;
{
ScopedLock L(Mutex);
InUseBlocks.remove(H);
Stats.sub(StatAllocated, CommitSize);
Stats.sub(StatMapped, H->MapSize);
}
- if (CacheT::canCache(CommitSize) && Cache.store(H))
- return;
- void *Addr = reinterpret_cast<void *>(H->MapBase);
- const uptr Size = H->MapSize;
- MapPlatformData Data = H->Data;
- unmap(Addr, Size, UNMAP_ALL, &Data);
+ Cache.store(Options, H);
}
-template <class CacheT>
-void MapAllocator<CacheT>::getStats(ScopedString *Str) const {
+template <typename Config>
+void MapAllocator<Config>::getStats(ScopedString *Str) const {
Str->append(
"Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times "
"(%zuK), remains %zu (%zuK) max %zuM\n",
static const u8 S = Config::NumBits - 1;
static const uptr M = (1UL << S) - 1;
- static const uptr SizeDelta = Chunk::getHeaderSize();
-
public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
- static const uptr MaxSize = (1UL << Config::MaxSizeLog) + SizeDelta;
+ static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses =
MidClass + ((Config::MaxSizeLog - Config::MidSizeLog) << S) + 1;
static_assert(NumClasses <= 256, "");
static uptr getSizeByClassId(uptr ClassId) {
DCHECK_NE(ClassId, BatchClassId);
if (ClassId <= MidClass)
- return (ClassId << Config::MinSizeLog) + SizeDelta;
+ return (ClassId << Config::MinSizeLog) + Config::SizeDelta;
ClassId -= MidClass;
const uptr T = MidSize << (ClassId >> S);
- return T + (T >> S) * (ClassId & M) + SizeDelta;
+ return T + (T >> S) * (ClassId & M) + Config::SizeDelta;
+ }
+
+ static u8 getSizeLSBByClassId(uptr ClassId) {
+ return u8(getLeastSignificantSetBitIndex(getSizeByClassId(ClassId)));
}
+ static constexpr bool usesCompressedLSBFormat() { return false; }
+
static uptr getClassIdBySize(uptr Size) {
- if (Size <= SizeDelta + (1 << Config::MinSizeLog))
+ if (Size <= Config::SizeDelta + (1 << Config::MinSizeLog))
return 1;
- Size -= SizeDelta;
+ Size -= Config::SizeDelta;
DCHECK_LE(Size, MaxSize);
if (Size <= MidSize)
return (Size + MinSize - 1) >> Config::MinSizeLog;
u8 Tab[getTableSize()] = {};
};
- static constexpr SizeTable Table = {};
+ static constexpr SizeTable SzTable = {};
+
+ struct LSBTable {
+ constexpr LSBTable() {
+ u8 Min = 255, Max = 0;
+ for (uptr I = 0; I != ClassesSize; ++I) {
+ for (u8 Bit = 0; Bit != 64; ++Bit) {
+ if (Config::Classes[I] & (1 << Bit)) {
+ Tab[I] = Bit;
+ if (Bit < Min)
+ Min = Bit;
+ if (Bit > Max)
+ Max = Bit;
+ break;
+ }
+ }
+ }
+
+ if (Max - Min > 3 || ClassesSize > 32)
+ return;
+
+ UseCompressedFormat = true;
+ CompressedMin = Min;
+ for (uptr I = 0; I != ClassesSize; ++I)
+ CompressedValue |= u64(Tab[I] - Min) << (I * 2);
+ }
+
+ u8 Tab[ClassesSize] = {};
+
+ bool UseCompressedFormat = false;
+ u8 CompressedMin = 0;
+ u64 CompressedValue = 0;
+ };
+
+ static constexpr LSBTable LTable = {};
public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
return Config::Classes[ClassId - 1];
}
+ static u8 getSizeLSBByClassId(uptr ClassId) {
+ if (LTable.UseCompressedFormat)
+ return ((LTable.CompressedValue >> ((ClassId - 1) * 2)) & 3) +
+ LTable.CompressedMin;
+ else
+ return LTable.Tab[ClassId - 1];
+ }
+
+ static constexpr bool usesCompressedLSBFormat() {
+ return LTable.UseCompressedFormat;
+ }
+
static uptr getClassIdBySize(uptr Size) {
if (Size <= Config::Classes[0])
return 1;
DCHECK_LE(Size, MaxSize);
if (Size <= (1 << Config::MidSizeLog))
return ((Size - 1) >> Config::MinSizeLog) + 1;
- return Table.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
+ return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
}
static u32 getMaxCachedHint(uptr Size) {
}
};
+struct DefaultSizeClassConfig {
+ static const uptr NumBits = 3;
+ static const uptr MinSizeLog = 5;
+ static const uptr MidSizeLog = 8;
+ static const uptr MaxSizeLog = 17;
+ static const u32 MaxNumCachedHint = 14;
+ static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = 0;
+};
+
+typedef FixedSizeClassMap<DefaultSizeClassConfig> DefaultSizeClassMap;
+
+struct FuchsiaSizeClassConfig {
+ static const uptr NumBits = 3;
+ static const uptr MinSizeLog = 5;
+ static const uptr MidSizeLog = 8;
+ static const uptr MaxSizeLog = 17;
+ static const u32 MaxNumCachedHint = 10;
+ static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
+};
+
+typedef FixedSizeClassMap<FuchsiaSizeClassConfig> FuchsiaSizeClassMap;
+
struct AndroidSizeClassConfig {
#if SCUDO_WORDSIZE == 64U
static const uptr NumBits = 7;
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 6;
static const uptr MaxSizeLog = 16;
- static const u32 MaxNumCachedHint = 14;
+ static const u32 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
typedef TableSizeClassMap<AndroidSizeClassConfig> AndroidSizeClassMap;
-struct DefaultSizeClassConfig {
- static const uptr NumBits = 3;
- static const uptr MinSizeLog = 5;
- static const uptr MidSizeLog = 8;
- static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 8;
- static const uptr MaxBytesCachedLog = 10;
-};
-
-typedef FixedSizeClassMap<DefaultSizeClassConfig> DefaultSizeClassMap;
+#if SCUDO_WORDSIZE == 64U && defined(__clang__)
+static_assert(AndroidSizeClassMap::usesCompressedLSBFormat(), "");
+#endif
struct SvelteSizeClassConfig {
#if SCUDO_WORDSIZE == 64U
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 4;
+ static const u32 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
#else
static const uptr NumBits = 4;
static const uptr MinSizeLog = 3;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 14;
- static const u32 MaxNumCachedHint = 5;
+ static const u32 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
#endif
};
typedef FixedSizeClassMap<SvelteSizeClassConfig> SvelteSizeClassMap;
+// Trusty is configured to only have one region containing blocks of size
+// 2^7 bytes.
+struct TrustySizeClassConfig {
+ static const uptr NumBits = 1;
+ static const uptr MinSizeLog = 7;
+ static const uptr MidSizeLog = 7;
+ static const uptr MaxSizeLog = 7;
+ static const u32 MaxNumCachedHint = 8;
+ static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = 0;
+};
+
+typedef FixedSizeClassMap<TrustySizeClassConfig> TrustySizeClassMap;
+
template <typename SCMap> inline void printMap() {
- ScopedString Buffer(1024);
+ ScopedString Buffer;
uptr PrevS = 0;
uptr TotalCached = 0;
for (uptr I = 0; I < SCMap::NumClasses; I++) {
static const u32 R = 24;
u32 H;
- public:
+public:
explicit MurMur2HashBuilder(u32 Init = 0) { H = Seed ^ Init; }
void add(u32 K) {
K *= M;
class StackDepot {
HybridMutex RingEndMu;
- u32 RingEnd;
+ u32 RingEnd = 0;
// This data structure stores a stack trace for each allocation and
// deallocation when stack trace recording is enabled, that may be looked up
#endif
static const uptr TabSize = 1 << TabBits;
static const uptr TabMask = TabSize - 1;
- atomic_u32 Tab[TabSize];
+ atomic_u32 Tab[TabSize] = {};
#ifdef SCUDO_FUZZ
static const uptr RingBits = 4;
#endif
static const uptr RingSize = 1 << RingBits;
static const uptr RingMask = RingSize - 1;
- atomic_u64 Ring[RingSize];
+ atomic_u64 Ring[RingSize] = {};
public:
// Insert hash of the stack trace [Begin, End) into the stack depot, and
// LocalStats::add'ing, this is OK, we will still get a meaningful number.
class LocalStats {
public:
- void initLinkerInitialized() {}
- void init() { memset(this, 0, sizeof(*this)); }
+ void init() {
+ for (uptr I = 0; I < StatCount; I++)
+ DCHECK_EQ(get(static_cast<StatType>(I)), 0U);
+ }
void add(StatType I, uptr V) {
V += atomic_load_relaxed(&StatsArray[I]);
uptr get(StatType I) const { return atomic_load_relaxed(&StatsArray[I]); }
- LocalStats *Next;
- LocalStats *Prev;
+ LocalStats *Next = nullptr;
+ LocalStats *Prev = nullptr;
private:
- atomic_uptr StatsArray[StatCount];
+ atomic_uptr StatsArray[StatCount] = {};
};
// Global stats, used for aggregation and querying.
class GlobalStats : public LocalStats {
public:
- void initLinkerInitialized() {}
- void init() {
- LocalStats::init();
- Mutex.init();
- StatsList = {};
- initLinkerInitialized();
- }
+ void init() { LocalStats::init(); }
void link(LocalStats *S) {
ScopedLock L(Mutex);
S[I] = static_cast<sptr>(S[I]) >= 0 ? S[I] : 0;
}
- void disable() { Mutex.lock(); }
- void enable() { Mutex.unlock(); }
+ void lock() { Mutex.lock(); }
+ void unlock() { Mutex.unlock(); }
+
+ void disable() { lock(); }
+ void enable() { unlock(); }
private:
mutable HybridMutex Mutex;
static int appendSignedDecimal(char **Buffer, const char *BufferEnd, s64 Num,
u8 MinNumberLength, bool PadWithZero) {
const bool Negative = (Num < 0);
- return appendNumber(Buffer, BufferEnd,
- static_cast<u64>(Negative ? -Num : Num), 10,
- MinNumberLength, PadWithZero, Negative,
- /*Upper=*/false);
+ const u64 UnsignedNum = (Num == INT64_MIN)
+ ? static_cast<u64>(INT64_MAX) + 1
+ : static_cast<u64>(Negative ? -Num : Num);
+ return appendNumber(Buffer, BufferEnd, UnsignedNum, 10, MinNumberLength,
+ PadWithZero, Negative, /*Upper=*/false);
}
// Use the fact that explicitly requesting 0 Width (%0s) results in UB and
return Res;
}
-int formatString(char *Buffer, uptr BufferLength, const char *Format,
- va_list Args) {
+static int formatString(char *Buffer, uptr BufferLength, const char *Format,
+ va_list Args) {
static const char *PrintfFormatsHelp =
"Supported formatString formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; "
"%[-]([0-9]*)?(\\.\\*)?s; %c\n";
CHECK(!((Precision >= 0 || LeftJustified) && *Cur != 's'));
switch (*Cur) {
case 'd': {
- DVal = HaveLL ? va_arg(Args, s64)
- : HaveZ ? va_arg(Args, sptr) : va_arg(Args, int);
+ DVal = HaveLL ? va_arg(Args, s64)
+ : HaveZ ? va_arg(Args, sptr)
+ : va_arg(Args, int);
Res += appendSignedDecimal(&Buffer, BufferEnd, DVal, Width, PadWithZero);
break;
}
case 'u':
case 'x':
case 'X': {
- UVal = HaveLL ? va_arg(Args, u64)
- : HaveZ ? va_arg(Args, uptr) : va_arg(Args, unsigned);
+ UVal = HaveLL ? va_arg(Args, u64)
+ : HaveZ ? va_arg(Args, uptr)
+ : va_arg(Args, unsigned);
const bool Upper = (*Cur == 'X');
Res += appendUnsigned(&Buffer, BufferEnd, UVal, (*Cur == 'u') ? 10 : 16,
Width, PadWithZero, Upper);
return Res;
}
+int formatString(char *Buffer, uptr BufferLength, const char *Format, ...) {
+ va_list Args;
+ va_start(Args, Format);
+ int Res = formatString(Buffer, BufferLength, Format, Args);
+ va_end(Args);
+ return Res;
+}
+
void ScopedString::append(const char *Format, va_list Args) {
- DCHECK_LT(Length, String.size());
va_list ArgsCopy;
va_copy(ArgsCopy, Args);
// formatString doesn't currently support a null buffer or zero buffer length,
char C[1];
const uptr AdditionalLength =
static_cast<uptr>(formatString(C, sizeof(C), Format, Args)) + 1;
+ const uptr Length = length();
String.resize(Length + AdditionalLength);
- formatString(String.data() + Length, AdditionalLength, Format, ArgsCopy);
- Length = strlen(String.data());
- CHECK_LT(Length, String.size());
+ const uptr FormattedLength = static_cast<uptr>(formatString(
+ String.data() + Length, String.size() - Length, Format, ArgsCopy));
+ RAW_CHECK(data()[length()] == '\0');
+ RAW_CHECK(FormattedLength + 1 == AdditionalLength);
+ va_end(ArgsCopy);
}
FORMAT(2, 3)
void Printf(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
- ScopedString Msg(1024);
+ ScopedString Msg;
Msg.append(Format, Args);
outputRaw(Msg.data());
va_end(Args);
class ScopedString {
public:
- explicit ScopedString(uptr MaxLength) : String(MaxLength), Length(0) {
- String[0] = '\0';
- }
- uptr length() { return Length; }
+ explicit ScopedString() { String.push_back('\0'); }
+ uptr length() { return String.size() - 1; }
const char *data() { return String.data(); }
void clear() {
- String[0] = '\0';
- Length = 0;
+ String.clear();
+ String.push_back('\0');
}
void append(const char *Format, va_list Args);
void append(const char *Format, ...);
private:
Vector<char> String;
- uptr Length;
};
+int formatString(char *Buffer, uptr BufferLength, const char *Format, ...);
void Printf(const char *Format, ...);
} // namespace scudo
-I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone
-I${COMPILER_RT_SOURCE_DIR}/lib/scudo/standalone/include
-DGTEST_HAS_RTTI=0
- -DSCUDO_DEBUG=1
+ -g
# Extra flags for the C++ tests
# TODO(kostyak): find a way to make -fsized-deallocation work
-Wno-mismatched-new-delete)
+if(COMPILER_RT_DEBUG)
+ list(APPEND SCUDO_UNITTEST_CFLAGS -DSCUDO_DEBUG=1)
+endif()
+
if(ANDROID)
list(APPEND SCUDO_UNITTEST_CFLAGS -fno-emulated-tls)
endif()
if (COMPILER_RT_HAS_GWP_ASAN)
- list(APPEND SCUDO_UNITTEST_CFLAGS -DGWP_ASAN_HOOKS)
+ list(APPEND SCUDO_UNITTEST_CFLAGS -DGWP_ASAN_HOOKS -fno-omit-frame-pointer
+ -mno-omit-leaf-frame-pointer)
endif()
set(SCUDO_TEST_ARCH ${SCUDO_STANDALONE_SUPPORTED_ARCH})
endforeach()
list(APPEND LINK_FLAGS -pthread)
# Linking against libatomic is required with some compilers
-list(APPEND LINK_FLAGS -latomic)
+check_library_exists(atomic __atomic_load_8 "" COMPILER_RT_HAS_LIBATOMIC)
+if (COMPILER_RT_HAS_LIBATOMIC)
+ list(APPEND LINK_FLAGS -latomic)
+endif()
-set(SCUDO_TEST_HEADERS)
+set(SCUDO_TEST_HEADERS
+ scudo_unit_test.h
+ )
foreach (header ${SCUDO_HEADERS})
list(APPEND SCUDO_TEST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header})
endforeach()
checksum_test.cpp
chunk_test.cpp
combined_test.cpp
+ common_test.cpp
flags_test.cpp
list_test.cpp
map_test.cpp
+ memtag_test.cpp
mutex_test.cpp
primary_test.cpp
quarantine_test.cpp
template <typename T> void checkAtomicCompareExchange() {
typedef typename T::Type Type;
- {
- Type OldVal = 42;
- Type NewVal = 24;
- Type V = OldVal;
- EXPECT_TRUE(atomic_compare_exchange_strong(
- reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed));
- EXPECT_FALSE(atomic_compare_exchange_strong(
- reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed));
- EXPECT_EQ(NewVal, OldVal);
- }
- {
- Type OldVal = 42;
- Type NewVal = 24;
- Type V = OldVal;
- EXPECT_TRUE(atomic_compare_exchange_weak(reinterpret_cast<T *>(&V), &OldVal,
+ Type OldVal = 42;
+ Type NewVal = 24;
+ Type V = OldVal;
+ EXPECT_TRUE(atomic_compare_exchange_strong(reinterpret_cast<T *>(&V), &OldVal,
NewVal, memory_order_relaxed));
- EXPECT_FALSE(atomic_compare_exchange_weak(
- reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed));
- EXPECT_EQ(NewVal, OldVal);
- }
+ EXPECT_FALSE(atomic_compare_exchange_strong(
+ reinterpret_cast<T *>(&V), &OldVal, NewVal, memory_order_relaxed));
+ EXPECT_EQ(NewVal, OldVal);
}
TEST(ScudoAtomicTest, AtomicCompareExchangeTest) {
scudo::u8 IdenticalChecksums = 0;
for (scudo::uptr I = 0; I < ArraySize; I++) {
for (scudo::uptr J = 0; J < SCUDO_WORDSIZE; J++) {
- Array[I] ^= 1U << J;
+ Array[I] ^= scudo::uptr{1} << J;
if (F(Seed, Array, ArraySize) == Reference)
IdenticalChecksums++;
- Array[I] ^= 1U << J;
+ Array[I] ^= scudo::uptr{1} << J;
}
}
// Allow for a couple of identical checksums over the whole set of flips.
scudo::HashAlgorithm = scudo::Checksum::HardwareCRC32;
}
-TEST(ScudoChunkTest, ChunkBasic) {
+TEST(ScudoChunkDeathTest, ChunkBasic) {
initChecksum();
const scudo::uptr Size = 0x100U;
scudo::Chunk::UnpackedHeader Header = {};
initChecksum();
const scudo::uptr Size = 0x100U;
scudo::Chunk::UnpackedHeader OldHeader = {};
- OldHeader.Origin = scudo::Chunk::Origin::Malloc;
+ OldHeader.OriginOrWasZeroed = scudo::Chunk::Origin::Malloc;
OldHeader.ClassId = 0x42U;
OldHeader.SizeOrUnusedBytes = Size;
OldHeader.State = scudo::Chunk::State::Allocated;
free(Block);
}
-TEST(ScudoChunkTest, CorruptHeader) {
+TEST(ScudoChunkDeathTest, CorruptHeader) {
initChecksum();
const scudo::uptr Size = 0x100U;
scudo::Chunk::UnpackedHeader Header = {};
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
#include "allocator_config.h"
#include "combined.h"
#include <condition_variable>
+#include <memory>
#include <mutex>
+#include <set>
+#include <stdlib.h>
#include <thread>
#include <vector>
-static std::mutex Mutex;
-static std::condition_variable Cv;
-static bool Ready = false;
-
static constexpr scudo::Chunk::Origin Origin = scudo::Chunk::Origin::Malloc;
+static constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);
-static void disableDebuggerdMaybe() {
+// Fuchsia complains that the function is not used.
+UNUSED static void disableDebuggerdMaybe() {
#if SCUDO_ANDROID
// Disable the debuggerd signal handler on Android, without this we can end
// up spending a significant amount of time creating tombstones.
}
template <class AllocatorT>
-bool isTaggedAllocation(AllocatorT *Allocator, scudo::uptr Size,
- scudo::uptr Alignment) {
- if (!Allocator->useMemoryTagging() ||
- !scudo::systemDetectsMemoryTagFaultsTestOnly())
- return false;
-
+bool isPrimaryAllocation(scudo::uptr Size, scudo::uptr Alignment) {
const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
if (Alignment < MinAlignment)
Alignment = MinAlignment;
template <class AllocatorT>
void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size,
scudo::uptr Alignment) {
- if (!isTaggedAllocation(Allocator, Size, Alignment))
- return;
-
- Size = scudo::roundUpTo(Size, scudo::archMemoryTagGranuleSize());
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- reinterpret_cast<char *>(P)[-1] = 0xaa;
- },
- "");
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- reinterpret_cast<char *>(P)[Size] = 0xaa;
- },
- "");
+ const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
+ Size = scudo::roundUpTo(Size, MinAlignment);
+ if (Allocator->useMemoryTaggingTestOnly())
+ EXPECT_DEATH(
+ {
+ disableDebuggerdMaybe();
+ reinterpret_cast<char *>(P)[-1] = 0xaa;
+ },
+ "");
+ if (isPrimaryAllocation<AllocatorT>(Size, Alignment)
+ ? Allocator->useMemoryTaggingTestOnly()
+ : Alignment == MinAlignment) {
+ EXPECT_DEATH(
+ {
+ disableDebuggerdMaybe();
+ reinterpret_cast<char *>(P)[Size] = 0xaa;
+ },
+ "");
+ }
}
-template <class Config> static void testAllocator() {
- using AllocatorT = scudo::Allocator<Config>;
- auto Deleter = [](AllocatorT *A) {
- A->unmapTestOnly();
- delete A;
- };
- std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
- Deleter);
- Allocator->reset();
+template <typename Config> struct TestAllocator : scudo::Allocator<Config> {
+ TestAllocator() {
+ this->initThreadMaybe();
+ if (scudo::archSupportsMemoryTagging() &&
+ !scudo::systemDetectsMemoryTagFaultsTestOnly())
+ this->disableMemoryTagging();
+ }
+ ~TestAllocator() { this->unmapTestOnly(); }
- EXPECT_FALSE(Allocator->isOwned(&Mutex));
- EXPECT_FALSE(Allocator->isOwned(&Allocator));
- scudo::u64 StackVariable = 0x42424242U;
- EXPECT_FALSE(Allocator->isOwned(&StackVariable));
- EXPECT_EQ(StackVariable, 0x42424242U);
+ void *operator new(size_t size) {
+ void *p = nullptr;
+ EXPECT_EQ(0, posix_memalign(&p, alignof(TestAllocator), size));
+ return p;
+ }
- constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);
+ void operator delete(void *ptr) { free(ptr); }
+};
+
+template <class TypeParam> struct ScudoCombinedTest : public Test {
+ ScudoCombinedTest() {
+ UseQuarantine = std::is_same<TypeParam, scudo::AndroidConfig>::value;
+ Allocator = std::make_unique<AllocatorT>();
+ }
+ ~ScudoCombinedTest() {
+ Allocator->releaseToOS();
+ UseQuarantine = true;
+ }
+
+ void RunTest();
+
+ void BasicTest(scudo::uptr SizeLog);
+
+ using AllocatorT = TestAllocator<TypeParam>;
+ std::unique_ptr<AllocatorT> Allocator;
+};
+
+template <typename T> using ScudoCombinedDeathTest = ScudoCombinedTest<T>;
+
+#if SCUDO_FUCHSIA
+#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, FuchsiaConfig)
+#else
+#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidSvelteConfig) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig)
+#endif
+
+#define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \
+ using FIXTURE##NAME##_##TYPE = FIXTURE##NAME<scudo::TYPE>; \
+ TEST_F(FIXTURE##NAME##_##TYPE, NAME) { Run(); }
+
+#define SCUDO_TYPED_TEST(FIXTURE, NAME) \
+ template <class TypeParam> \
+ struct FIXTURE##NAME : public FIXTURE<TypeParam> { \
+ void Run(); \
+ }; \
+ SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ template <class TypeParam> void FIXTURE##NAME<TypeParam>::Run()
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, IsOwned) {
+ auto *Allocator = this->Allocator.get();
+ static scudo::u8 StaticBuffer[scudo::Chunk::getHeaderSize() + 1];
+ EXPECT_FALSE(
+ Allocator->isOwned(&StaticBuffer[scudo::Chunk::getHeaderSize()]));
+
+ scudo::u8 StackBuffer[scudo::Chunk::getHeaderSize() + 1];
+ for (scudo::uptr I = 0; I < sizeof(StackBuffer); I++)
+ StackBuffer[I] = 0x42U;
+ EXPECT_FALSE(Allocator->isOwned(&StackBuffer[scudo::Chunk::getHeaderSize()]));
+ for (scudo::uptr I = 0; I < sizeof(StackBuffer); I++)
+ EXPECT_EQ(StackBuffer[I], 0x42U);
+}
+
+template <class Config>
+void ScudoCombinedTest<Config>::BasicTest(scudo::uptr SizeLog) {
+ auto *Allocator = this->Allocator.get();
// This allocates and deallocates a bunch of chunks, with a wide range of
// sizes and alignments, with a focus on sizes that could trigger weird
// behaviors (plus or minus a small delta of a power of two for example).
- for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
- for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
- const scudo::uptr Align = 1U << AlignLog;
- for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
- if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0)
- continue;
- const scudo::uptr Size = (1U << SizeLog) + Delta;
- void *P = Allocator->allocate(Size, Origin, Align);
- EXPECT_NE(P, nullptr);
- EXPECT_TRUE(Allocator->isOwned(P));
- EXPECT_TRUE(scudo::isAligned(reinterpret_cast<scudo::uptr>(P), Align));
- EXPECT_LE(Size, Allocator->getUsableSize(P));
- memset(P, 0xaa, Size);
- checkMemoryTaggingMaybe(Allocator.get(), P, Size, Align);
- Allocator->deallocate(P, Origin, Size);
- }
+ for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
+ const scudo::uptr Align = 1U << AlignLog;
+ for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
+ if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0)
+ continue;
+ const scudo::uptr Size = (1U << SizeLog) + Delta;
+ void *P = Allocator->allocate(Size, Origin, Align);
+ EXPECT_NE(P, nullptr);
+ EXPECT_TRUE(Allocator->isOwned(P));
+ EXPECT_TRUE(scudo::isAligned(reinterpret_cast<scudo::uptr>(P), Align));
+ EXPECT_LE(Size, Allocator->getUsableSize(P));
+ memset(P, 0xaa, Size);
+ checkMemoryTaggingMaybe(Allocator, P, Size, Align);
+ Allocator->deallocate(P, Origin, Size);
}
}
- Allocator->releaseToOS();
+}
+
+#define SCUDO_MAKE_BASIC_TEST(SizeLog) \
+ SCUDO_TYPED_TEST(ScudoCombinedDeathTest, BasicCombined##SizeLog) { \
+ this->BasicTest(SizeLog); \
+ }
+
+SCUDO_MAKE_BASIC_TEST(0)
+SCUDO_MAKE_BASIC_TEST(1)
+SCUDO_MAKE_BASIC_TEST(2)
+SCUDO_MAKE_BASIC_TEST(3)
+SCUDO_MAKE_BASIC_TEST(4)
+SCUDO_MAKE_BASIC_TEST(5)
+SCUDO_MAKE_BASIC_TEST(6)
+SCUDO_MAKE_BASIC_TEST(7)
+SCUDO_MAKE_BASIC_TEST(8)
+SCUDO_MAKE_BASIC_TEST(9)
+SCUDO_MAKE_BASIC_TEST(10)
+SCUDO_MAKE_BASIC_TEST(11)
+SCUDO_MAKE_BASIC_TEST(12)
+SCUDO_MAKE_BASIC_TEST(13)
+SCUDO_MAKE_BASIC_TEST(14)
+SCUDO_MAKE_BASIC_TEST(15)
+SCUDO_MAKE_BASIC_TEST(16)
+SCUDO_MAKE_BASIC_TEST(17)
+SCUDO_MAKE_BASIC_TEST(18)
+SCUDO_MAKE_BASIC_TEST(19)
+SCUDO_MAKE_BASIC_TEST(20)
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroContents) {
+ auto *Allocator = this->Allocator.get();
// Ensure that specifying ZeroContents returns a zero'd out block.
for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
Allocator->deallocate(P, Origin, Size);
}
}
- Allocator->releaseToOS();
+}
- // Ensure that specifying ZeroContents returns a zero'd out block.
+SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroFill) {
+ auto *Allocator = this->Allocator.get();
+
+ // Ensure that specifying ZeroFill returns a zero'd out block.
Allocator->setFillContents(scudo::ZeroFill);
for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
Allocator->deallocate(P, Origin, Size);
}
}
- Allocator->releaseToOS();
+}
- // Ensure that specifying PatternOrZeroFill returns a pattern-filled block in
- // the primary allocator, and either pattern or zero filled block in the
- // secondary.
+SCUDO_TYPED_TEST(ScudoCombinedTest, PatternOrZeroFill) {
+ auto *Allocator = this->Allocator.get();
+
+ // Ensure that specifying PatternOrZeroFill returns a pattern or zero filled
+ // block. The primary allocator only produces pattern filled blocks if MTE
+ // is disabled, so we only require pattern filled blocks in that case.
Allocator->setFillContents(scudo::PatternOrZeroFill);
for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
EXPECT_NE(P, nullptr);
for (scudo::uptr I = 0; I < Size; I++) {
unsigned char V = (reinterpret_cast<unsigned char *>(P))[I];
- if (AllocatorT::PrimaryT::canAllocate(Size))
+ if (isPrimaryAllocation<TestAllocator<TypeParam>>(Size,
+ 1U << MinAlignLog) &&
+ !Allocator->useMemoryTaggingTestOnly())
ASSERT_EQ(V, scudo::PatternFillByte);
else
ASSERT_TRUE(V == scudo::PatternFillByte || V == 0);
Allocator->deallocate(P, Origin, Size);
}
}
- Allocator->releaseToOS();
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, BlockReuse) {
+ auto *Allocator = this->Allocator.get();
// Verify that a chunk will end up being reused, at some point.
const scudo::uptr NeedleSize = 1024U;
bool Found = false;
for (scudo::uptr I = 0; I < 1024U && !Found; I++) {
void *P = Allocator->allocate(NeedleSize, Origin);
- if (Allocator->untagPointerMaybe(P) ==
- Allocator->untagPointerMaybe(NeedleP))
+ if (Allocator->getHeaderTaggedPointer(P) ==
+ Allocator->getHeaderTaggedPointer(NeedleP))
Found = true;
Allocator->deallocate(P, Origin);
}
EXPECT_TRUE(Found);
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeIncreasing) {
+ auto *Allocator = this->Allocator.get();
+
+ // Reallocate a chunk all the way up to a secondary allocation, verifying that
+ // we preserve the data in the process.
+ scudo::uptr Size = 16;
+ void *P = Allocator->allocate(Size, Origin);
+ const char Marker = 0xab;
+ memset(P, Marker, Size);
+ while (Size < TypeParam::Primary::SizeClassMap::MaxSize * 4) {
+ void *NewP = Allocator->reallocate(P, Size * 2);
+ EXPECT_NE(NewP, nullptr);
+ for (scudo::uptr J = 0; J < Size; J++)
+ EXPECT_EQ((reinterpret_cast<char *>(NewP))[J], Marker);
+ memset(reinterpret_cast<char *>(NewP) + Size, Marker, Size);
+ Size *= 2U;
+ P = NewP;
+ }
+ Allocator->deallocate(P, Origin);
+}
- constexpr scudo::uptr MaxSize = Config::Primary::SizeClassMap::MaxSize;
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeDecreasing) {
+ auto *Allocator = this->Allocator.get();
// Reallocate a large chunk all the way down to a byte, verifying that we
// preserve the data in the process.
- scudo::uptr Size = MaxSize * 2;
+ scudo::uptr Size = TypeParam::Primary::SizeClassMap::MaxSize * 2;
const scudo::uptr DataSize = 2048U;
void *P = Allocator->allocate(Size, Origin);
const char Marker = 0xab;
P = NewP;
}
Allocator->deallocate(P, Origin);
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedDeathTest, ReallocateSame) {
+ auto *Allocator = this->Allocator.get();
// Check that reallocating a chunk to a slightly smaller or larger size
// returns the same chunk. This requires that all the sizes we iterate on use
// the same block size, but that should be the case for MaxSize - 64 with our
// default class size maps.
- constexpr scudo::uptr ReallocSize = MaxSize - 64;
- P = Allocator->allocate(ReallocSize, Origin);
+ constexpr scudo::uptr ReallocSize =
+ TypeParam::Primary::SizeClassMap::MaxSize - 64;
+ void *P = Allocator->allocate(ReallocSize, Origin);
+ const char Marker = 0xab;
memset(P, Marker, ReallocSize);
for (scudo::sptr Delta = -32; Delta < 32; Delta += 8) {
const scudo::uptr NewSize = ReallocSize + Delta;
EXPECT_EQ(NewP, P);
for (scudo::uptr I = 0; I < ReallocSize - 32; I++)
EXPECT_EQ((reinterpret_cast<char *>(NewP))[I], Marker);
- checkMemoryTaggingMaybe(Allocator.get(), NewP, NewSize, 0);
+ checkMemoryTaggingMaybe(Allocator, NewP, NewSize, 0);
}
Allocator->deallocate(P, Origin);
+}
+SCUDO_TYPED_TEST(ScudoCombinedTest, IterateOverChunks) {
+ auto *Allocator = this->Allocator.get();
// Allocates a bunch of chunks, then iterate over all the chunks, ensuring
// they are the ones we allocated. This requires the allocator to not have any
// other allocated chunk at this point (eg: won't work with the Quarantine).
+ // FIXME: Make it work with UseQuarantine and tagging enabled. Internals of
+ // iterateOverChunks reads header by tagged and non-tagger pointers so one of
+ // them will fail.
if (!UseQuarantine) {
std::vector<void *> V;
for (scudo::uptr I = 0; I < 64U; I++)
- V.push_back(Allocator->allocate(rand() % (MaxSize / 2U), Origin));
+ V.push_back(Allocator->allocate(
+ rand() % (TypeParam::Primary::SizeClassMap::MaxSize / 2U), Origin));
Allocator->disable();
Allocator->iterateOverChunks(
0U, static_cast<scudo::uptr>(SCUDO_MMAP_RANGE_SIZE - 1),
},
reinterpret_cast<void *>(&V));
Allocator->enable();
- while (!V.empty()) {
- Allocator->deallocate(V.back(), Origin);
- V.pop_back();
- }
+ for (auto P : V)
+ Allocator->deallocate(P, Origin);
}
+}
- Allocator->releaseToOS();
+SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) {
+ auto *Allocator = this->Allocator.get();
- if (Allocator->useMemoryTagging() &&
- scudo::systemDetectsMemoryTagFaultsTestOnly()) {
- // Check that use-after-free is detected.
- for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
- const scudo::uptr Size = 1U << SizeLog;
- if (!isTaggedAllocation(Allocator.get(), Size, 1))
- continue;
- // UAF detection is probabilistic, so we repeat the test up to 256 times
- // if necessary. With 15 possible tags this means a 1 in 15^256 chance of
- // a false positive.
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- for (unsigned I = 0; I != 256; ++I) {
- void *P = Allocator->allocate(Size, Origin);
- Allocator->deallocate(P, Origin);
- reinterpret_cast<char *>(P)[0] = 0xaa;
- }
- },
- "");
- EXPECT_DEATH(
- {
- disableDebuggerdMaybe();
- for (unsigned I = 0; I != 256; ++I) {
- void *P = Allocator->allocate(Size, Origin);
- Allocator->deallocate(P, Origin);
- reinterpret_cast<char *>(P)[Size - 1] = 0xaa;
- }
- },
- "");
- }
+ // Check that use-after-free is detected.
+ for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
+ const scudo::uptr Size = 1U << SizeLog;
+ if (!Allocator->useMemoryTaggingTestOnly())
+ continue;
+ EXPECT_DEATH(
+ {
+ disableDebuggerdMaybe();
+ void *P = Allocator->allocate(Size, Origin);
+ Allocator->deallocate(P, Origin);
+ reinterpret_cast<char *>(P)[0] = 0xaa;
+ },
+ "");
+ EXPECT_DEATH(
+ {
+ disableDebuggerdMaybe();
+ void *P = Allocator->allocate(Size, Origin);
+ Allocator->deallocate(P, Origin);
+ reinterpret_cast<char *>(P)[Size - 1] = 0xaa;
+ },
+ "");
+ }
+}
+SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) {
+ auto *Allocator = this->Allocator.get();
+
+ if (Allocator->useMemoryTaggingTestOnly()) {
// Check that disabling memory tagging works correctly.
void *P = Allocator->allocate(2048, Origin);
EXPECT_DEATH(reinterpret_cast<char *>(P)[2048] = 0xaa, "");
- scudo::disableMemoryTagChecksTestOnly();
+ scudo::ScopedDisableMemoryTagChecks NoTagChecks;
Allocator->disableMemoryTagging();
reinterpret_cast<char *>(P)[2048] = 0xaa;
Allocator->deallocate(P, Origin);
P = Allocator->allocate(2048, Origin);
- EXPECT_EQ(Allocator->untagPointerMaybe(P), P);
+ EXPECT_EQ(scudo::untagPointer(P), P);
reinterpret_cast<char *>(P)[2048] = 0xaa;
Allocator->deallocate(P, Origin);
Allocator->releaseToOS();
-
- // Disabling memory tag checks may interfere with subsequent tests.
- // Re-enable them now.
- scudo::enableMemoryTagChecksTestOnly();
}
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, Stats) {
+ auto *Allocator = this->Allocator.get();
scudo::uptr BufferSize = 8192;
std::vector<char> Buffer(BufferSize);
EXPECT_NE(Stats.find("Stats: Quarantine"), std::string::npos);
}
-// Test that multiple instantiations of the allocator have not messed up the
-// process's signal handlers (GWP-ASan used to do this).
-void testSEGV() {
- const scudo::uptr Size = 4 * scudo::getPageSizeCached();
- scudo::MapPlatformData Data = {};
- void *P = scudo::map(nullptr, Size, "testSEGV", MAP_NOACCESS, &Data);
- EXPECT_NE(P, nullptr);
- EXPECT_DEATH(memset(P, 0xaa, Size), "");
- scudo::unmap(P, Size, UNMAP_ALL, &Data);
-}
+SCUDO_TYPED_TEST(ScudoCombinedTest, CacheDrain) {
+ auto *Allocator = this->Allocator.get();
-TEST(ScudoCombinedTest, BasicCombined) {
- UseQuarantine = false;
- testAllocator<scudo::AndroidSvelteConfig>();
-#if SCUDO_FUCHSIA
- testAllocator<scudo::FuchsiaConfig>();
-#else
- testAllocator<scudo::DefaultConfig>();
- UseQuarantine = true;
- testAllocator<scudo::AndroidConfig>();
- testSEGV();
-#endif
-}
+ std::vector<void *> V;
+ for (scudo::uptr I = 0; I < 64U; I++)
+ V.push_back(Allocator->allocate(
+ rand() % (TypeParam::Primary::SizeClassMap::MaxSize / 2U), Origin));
+ for (auto P : V)
+ Allocator->deallocate(P, Origin);
-template <typename AllocatorT> static void stressAllocator(AllocatorT *A) {
- {
- std::unique_lock<std::mutex> Lock(Mutex);
- while (!Ready)
- Cv.wait(Lock);
- }
- std::vector<std::pair<void *, scudo::uptr>> V;
- for (scudo::uptr I = 0; I < 256U; I++) {
- const scudo::uptr Size = std::rand() % 4096U;
- void *P = A->allocate(Size, Origin);
- // A region could have ran out of memory, resulting in a null P.
- if (P)
- V.push_back(std::make_pair(P, Size));
- }
- while (!V.empty()) {
- auto Pair = V.back();
- A->deallocate(Pair.first, Origin, Pair.second);
- V.pop_back();
- }
+ bool UnlockRequired;
+ auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired);
+ EXPECT_TRUE(!TSD->Cache.isEmpty());
+ TSD->Cache.drain();
+ EXPECT_TRUE(TSD->Cache.isEmpty());
+ if (UnlockRequired)
+ TSD->unlock();
}
-template <class Config> static void testAllocatorThreaded() {
- using AllocatorT = scudo::Allocator<Config>;
- auto Deleter = [](AllocatorT *A) {
- A->unmapTestOnly();
- delete A;
- };
- std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
- Deleter);
- Allocator->reset();
+SCUDO_TYPED_TEST(ScudoCombinedTest, ThreadedCombined) {
+ std::mutex Mutex;
+ std::condition_variable Cv;
+ bool Ready = false;
+ auto *Allocator = this->Allocator.get();
std::thread Threads[32];
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
- Threads[I] = std::thread(stressAllocator<AllocatorT>, Allocator.get());
+ Threads[I] = std::thread([&]() {
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ while (!Ready)
+ Cv.wait(Lock);
+ }
+ std::vector<std::pair<void *, scudo::uptr>> V;
+ for (scudo::uptr I = 0; I < 256U; I++) {
+ const scudo::uptr Size = std::rand() % 4096U;
+ void *P = Allocator->allocate(Size, Origin);
+ // A region could have ran out of memory, resulting in a null P.
+ if (P)
+ V.push_back(std::make_pair(P, Size));
+ }
+ while (!V.empty()) {
+ auto Pair = V.back();
+ Allocator->deallocate(Pair.first, Origin, Pair.second);
+ V.pop_back();
+ }
+ });
{
std::unique_lock<std::mutex> Lock(Mutex);
Ready = true;
Allocator->releaseToOS();
}
-TEST(ScudoCombinedTest, ThreadedCombined) {
- UseQuarantine = false;
- testAllocatorThreaded<scudo::AndroidSvelteConfig>();
-#if SCUDO_FUCHSIA
- testAllocatorThreaded<scudo::FuchsiaConfig>();
-#else
- testAllocatorThreaded<scudo::DefaultConfig>();
- UseQuarantine = true;
- testAllocatorThreaded<scudo::AndroidConfig>();
-#endif
+// Test that multiple instantiations of the allocator have not messed up the
+// process's signal handlers (GWP-ASan used to do this).
+TEST(ScudoCombinedDeathTest, SKIP_ON_FUCHSIA(testSEGV)) {
+ const scudo::uptr Size = 4 * scudo::getPageSizeCached();
+ scudo::MapPlatformData Data = {};
+ void *P = scudo::map(nullptr, Size, "testSEGV", MAP_NOACCESS, &Data);
+ EXPECT_NE(P, nullptr);
+ EXPECT_DEATH(memset(P, 0xaa, Size), "");
+ scudo::unmap(P, Size, UNMAP_ALL, &Data);
}
struct DeathSizeClassConfig {
static const scudo::uptr MaxSizeLog = 13;
static const scudo::u32 MaxNumCachedHint = 4;
static const scudo::uptr MaxBytesCachedLog = 12;
+ static const scudo::uptr SizeDelta = 0;
};
static const scudo::uptr DeathRegionSizeLog = 20U;
struct DeathConfig {
+ static const bool MaySupportMemoryTagging = false;
+
// Tiny allocator, its Primary only serves chunks of four sizes.
- using DeathSizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>;
- typedef scudo::SizeClassAllocator64<DeathSizeClassMap, DeathRegionSizeLog>
- Primary;
- typedef scudo::MapAllocator<scudo::MapAllocatorNoCache> Secondary;
- template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U>;
+ using SizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>;
+ typedef scudo::SizeClassAllocator64<DeathConfig> Primary;
+ static const scudo::uptr PrimaryRegionSizeLog = DeathRegionSizeLog;
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ typedef scudo::uptr PrimaryCompactPtrT;
+ static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+
+ typedef scudo::MapAllocatorNoCache SecondaryCache;
+ template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;
};
-TEST(ScudoCombinedTest, DeathCombined) {
- using AllocatorT = scudo::Allocator<DeathConfig>;
- auto Deleter = [](AllocatorT *A) {
- A->unmapTestOnly();
- delete A;
- };
- std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
- Deleter);
- Allocator->reset();
+TEST(ScudoCombinedDeathTest, DeathCombined) {
+ using AllocatorT = TestAllocator<DeathConfig>;
+ auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
const scudo::uptr Size = 1000U;
void *P = Allocator->allocate(Size, Origin);
EXPECT_DEATH(Allocator->getUsableSize(P), "");
}
-// Ensure that releaseToOS can be called prior to any other allocator
-// operation without issue.
-TEST(ScudoCombinedTest, ReleaseToOS) {
- using AllocatorT = scudo::Allocator<DeathConfig>;
- auto Deleter = [](AllocatorT *A) {
- A->unmapTestOnly();
- delete A;
- };
- std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
- Deleter);
- Allocator->reset();
-
- Allocator->releaseToOS();
-}
-
// Verify that when a region gets full, the allocator will still manage to
// fulfill the allocation through a larger size class.
TEST(ScudoCombinedTest, FullRegion) {
- using AllocatorT = scudo::Allocator<DeathConfig>;
- auto Deleter = [](AllocatorT *A) {
- A->unmapTestOnly();
- delete A;
- };
- std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
- Deleter);
- Allocator->reset();
+ using AllocatorT = TestAllocator<DeathConfig>;
+ auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
std::vector<void *> V;
scudo::uptr FailedAllocationsCount = 0;
for (scudo::uptr ClassId = 1U;
- ClassId <= DeathConfig::DeathSizeClassMap::LargestClassId; ClassId++) {
+ ClassId <= DeathConfig::SizeClassMap::LargestClassId; ClassId++) {
const scudo::uptr Size =
- DeathConfig::DeathSizeClassMap::getSizeByClassId(ClassId);
+ DeathConfig::SizeClassMap::getSizeByClassId(ClassId);
// Allocate enough to fill all of the regions above this one.
const scudo::uptr MaxNumberOfChunks =
((1U << DeathRegionSizeLog) / Size) *
- (DeathConfig::DeathSizeClassMap::LargestClassId - ClassId + 1);
+ (DeathConfig::SizeClassMap::LargestClassId - ClassId + 1);
void *P;
for (scudo::uptr I = 0; I <= MaxNumberOfChunks; I++) {
P = Allocator->allocate(Size - 64U, Origin);
}
EXPECT_EQ(FailedAllocationsCount, 0U);
}
+
+// Ensure that releaseToOS can be called prior to any other allocator
+// operation without issue.
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) {
+ auto *Allocator = this->Allocator.get();
+ Allocator->releaseToOS();
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) {
+ auto *Allocator = this->Allocator.get();
+
+ if (!Allocator->useMemoryTaggingTestOnly())
+ return;
+
+ auto CheckOddEven = [](scudo::uptr P1, scudo::uptr P2) {
+ scudo::uptr Tag1 = scudo::extractTag(scudo::loadTag(P1));
+ scudo::uptr Tag2 = scudo::extractTag(scudo::loadTag(P2));
+ EXPECT_NE(Tag1 % 2, Tag2 % 2);
+ };
+
+ using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
+ for (scudo::uptr ClassId = 1U; ClassId <= SizeClassMap::LargestClassId;
+ ClassId++) {
+ const scudo::uptr Size = SizeClassMap::getSizeByClassId(ClassId);
+
+ std::set<scudo::uptr> Ptrs;
+ bool Found = false;
+ for (unsigned I = 0; I != 65536; ++I) {
+ scudo::uptr P = scudo::untagPointer(reinterpret_cast<scudo::uptr>(
+ Allocator->allocate(Size - scudo::Chunk::getHeaderSize(), Origin)));
+ if (Ptrs.count(P - Size)) {
+ Found = true;
+ CheckOddEven(P, P - Size);
+ break;
+ }
+ if (Ptrs.count(P + Size)) {
+ Found = true;
+ CheckOddEven(P, P + Size);
+ break;
+ }
+ Ptrs.insert(P);
+ }
+ EXPECT_TRUE(Found);
+ }
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) {
+ auto *Allocator = this->Allocator.get();
+
+ std::vector<void *> Ptrs(65536, nullptr);
+
+ Allocator->setOption(scudo::Option::ThreadDisableMemInit, 1);
+
+ constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);
+
+ // Test that if mem-init is disabled on a thread, calloc should still work as
+ // expected. This is tricky to ensure when MTE is enabled, so this test tries
+ // to exercise the relevant code on our MTE path.
+ for (scudo::uptr ClassId = 1U; ClassId <= 8; ClassId++) {
+ using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
+ const scudo::uptr Size =
+ SizeClassMap::getSizeByClassId(ClassId) - scudo::Chunk::getHeaderSize();
+ if (Size < 8)
+ continue;
+ for (unsigned I = 0; I != Ptrs.size(); ++I) {
+ Ptrs[I] = Allocator->allocate(Size, Origin);
+ memset(Ptrs[I], 0xaa, Size);
+ }
+ for (unsigned I = 0; I != Ptrs.size(); ++I)
+ Allocator->deallocate(Ptrs[I], Origin, Size);
+ for (unsigned I = 0; I != Ptrs.size(); ++I) {
+ Ptrs[I] = Allocator->allocate(Size - 8, Origin);
+ memset(Ptrs[I], 0xbb, Size - 8);
+ }
+ for (unsigned I = 0; I != Ptrs.size(); ++I)
+ Allocator->deallocate(Ptrs[I], Origin, Size - 8);
+ for (unsigned I = 0; I != Ptrs.size(); ++I) {
+ Ptrs[I] = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true);
+ for (scudo::uptr J = 0; J < Size; ++J)
+ ASSERT_EQ((reinterpret_cast<char *>(Ptrs[I]))[J], 0);
+ }
+ }
+
+ Allocator->setOption(scudo::Option::ThreadDisableMemInit, 0);
+}
--- /dev/null
+//===-- common_test.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "internal_defs.h"
+#include "tests/scudo_unit_test.h"
+
+#include "common.h"
+#include <algorithm>
+#include <fstream>
+
+namespace scudo {
+
+static uptr getResidentMemorySize() {
+ if (!SCUDO_LINUX)
+ UNREACHABLE("Not implemented!");
+ uptr Size;
+ uptr Resident;
+ std::ifstream IFS("/proc/self/statm");
+ IFS >> Size;
+ IFS >> Resident;
+ return Resident * getPageSizeCached();
+}
+
+// Fuchsia needs getResidentMemorySize implementation.
+TEST(ScudoCommonTest, SKIP_ON_FUCHSIA(ResidentMemorySize)) {
+ uptr OnStart = getResidentMemorySize();
+ EXPECT_GT(OnStart, 0UL);
+
+ const uptr Size = 1ull << 30;
+ const uptr Threshold = Size >> 3;
+
+ MapPlatformData Data = {};
+ void *P = map(nullptr, Size, "ResidentMemorySize", 0, &Data);
+ ASSERT_NE(nullptr, P);
+ EXPECT_LT(getResidentMemorySize(), OnStart + Threshold);
+
+ memset(P, 1, Size);
+ EXPECT_GT(getResidentMemorySize(), OnStart + Size - Threshold);
+
+ releasePagesToOS((uptr)P, 0, Size, &Data);
+ EXPECT_LT(getResidentMemorySize(), OnStart + Threshold);
+
+ memset(P, 1, Size);
+ EXPECT_GT(getResidentMemorySize(), OnStart + Size - Threshold);
+
+ unmap(P, Size, 0, &Data);
+}
+
+TEST(ScudoCommonTest, Zeros) {
+ const uptr Size = 1ull << 20;
+
+ MapPlatformData Data = {};
+ uptr *P = reinterpret_cast<uptr *>(map(nullptr, Size, "Zeros", 0, &Data));
+ const ptrdiff_t N = Size / sizeof(*P);
+ ASSERT_NE(nullptr, P);
+ EXPECT_EQ(std::count(P, P + N, 0), N);
+
+ memset(P, 1, Size);
+ EXPECT_EQ(std::count(P, P + N, 0), 0);
+
+ releasePagesToOS((uptr)P, 0, Size, &Data);
+ EXPECT_EQ(std::count(P, P + N, 0), N);
+
+ unmap(P, Size, 0, &Data);
+}
+
+} // namespace scudo
static_cast<scudo::uptr>(getpagesize()));
}
-TEST(ScudoMapTest, MapNoAccessUnmap) {
+TEST(ScudoMapDeathTest, MapNoAccessUnmap) {
const scudo::uptr Size = 4 * scudo::getPageSizeCached();
scudo::MapPlatformData Data = {};
void *P = scudo::map(nullptr, Size, MappingName, MAP_NOACCESS, &Data);
scudo::unmap(P, Size, UNMAP_ALL, &Data);
}
-TEST(ScudoMapTest, MapUnmap) {
+TEST(ScudoMapDeathTest, MapUnmap) {
const scudo::uptr Size = 4 * scudo::getPageSizeCached();
- void *P = scudo::map(nullptr, Size, MappingName, 0, nullptr);
- EXPECT_NE(P, nullptr);
- memset(P, 0xaa, Size);
- scudo::unmap(P, Size, 0, nullptr);
- EXPECT_DEATH(memset(P, 0xbb, Size), "");
+ EXPECT_DEATH(
+ {
+ // Repeat few time to avoid missing crash if it's mmaped by unrelated
+ // code.
+ for (int i = 0; i < 10; ++i) {
+ void *P = scudo::map(nullptr, Size, MappingName, 0, nullptr);
+ if (!P)
+ continue;
+ scudo::unmap(P, Size, 0, nullptr);
+ memset(P, 0xbb, Size);
+ }
+ },
+ "");
}
-TEST(ScudoMapTest, MapWithGuardUnmap) {
+TEST(ScudoMapDeathTest, MapWithGuardUnmap) {
const scudo::uptr PageSize = scudo::getPageSizeCached();
const scudo::uptr Size = 4 * PageSize;
scudo::MapPlatformData Data = {};
--- /dev/null
+//===-- memtag_test.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "common.h"
+#include "memtag.h"
+#include "platform.h"
+#include "tests/scudo_unit_test.h"
+
+#if SCUDO_LINUX
+namespace scudo {
+
+TEST(MemtagBasicDeathTest, Unsupported) {
+ if (archSupportsMemoryTagging())
+ GTEST_SKIP();
+
+ EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
+ EXPECT_DEATH(untagPointer((uptr)0), "not supported");
+ EXPECT_DEATH(extractTag((uptr)0), "not supported");
+
+ EXPECT_DEATH(systemSupportsMemoryTagging(), "not supported");
+ EXPECT_DEATH(systemDetectsMemoryTagFaultsTestOnly(), "not supported");
+ EXPECT_DEATH(enableSystemMemoryTaggingTestOnly(), "not supported");
+
+ EXPECT_DEATH(selectRandomTag((uptr)0, 0), "not supported");
+ EXPECT_DEATH(addFixedTag((uptr)0, 1), "not supported");
+ EXPECT_DEATH(storeTags((uptr)0, (uptr)0 + sizeof(0)), "not supported");
+ EXPECT_DEATH(storeTag((uptr)0), "not supported");
+ EXPECT_DEATH(loadTag((uptr)0), "not supported");
+
+ EXPECT_DEATH(setRandomTag(nullptr, 64, 0, nullptr, nullptr), "not supported");
+ EXPECT_DEATH(untagPointer(nullptr), "not supported");
+ EXPECT_DEATH(loadTag(nullptr), "not supported");
+ EXPECT_DEATH(addFixedTag(nullptr, 0), "not supported");
+}
+
+class MemtagTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ if (!archSupportsMemoryTagging() || !systemDetectsMemoryTagFaultsTestOnly())
+ GTEST_SKIP() << "Memory tagging is not supported";
+
+ BufferSize = getPageSizeCached();
+ Buffer = reinterpret_cast<u8 *>(
+ map(nullptr, BufferSize, "MemtagTest", MAP_MEMTAG, &Data));
+ Addr = reinterpret_cast<uptr>(Buffer);
+ EXPECT_TRUE(isAligned(Addr, archMemoryTagGranuleSize()));
+ EXPECT_EQ(Addr, untagPointer(Addr));
+ }
+
+ void TearDown() override {
+ if (Buffer)
+ unmap(Buffer, BufferSize, 0, &Data);
+ }
+
+ uptr BufferSize = 0;
+ MapPlatformData Data = {};
+ u8 *Buffer = nullptr;
+ uptr Addr = 0;
+};
+
+using MemtagDeathTest = MemtagTest;
+
+TEST_F(MemtagTest, ArchMemoryTagGranuleSize) {
+ EXPECT_GT(archMemoryTagGranuleSize(), 1u);
+ EXPECT_TRUE(isPowerOfTwo(archMemoryTagGranuleSize()));
+}
+
+TEST_F(MemtagTest, ExtractTag) {
+ uptr Tags = 0;
+ // Try all value for the top byte and check the tags values are in the
+ // expected range.
+ for (u64 Top = 0; Top < 0x100; ++Top)
+ Tags = Tags | (1u << extractTag(Addr | (Top << 56)));
+ EXPECT_EQ(0xffffull, Tags);
+}
+
+TEST_F(MemtagDeathTest, AddFixedTag) {
+ for (uptr Tag = 0; Tag < 0x10; ++Tag)
+ EXPECT_EQ(Tag, extractTag(addFixedTag(Addr, Tag)));
+ if (SCUDO_DEBUG) {
+ EXPECT_DEBUG_DEATH(addFixedTag(Addr, 16), "");
+ EXPECT_DEBUG_DEATH(addFixedTag(~Addr, 0), "");
+ }
+}
+
+TEST_F(MemtagTest, UntagPointer) {
+ uptr UnTagMask = untagPointer(~uptr(0));
+ for (u64 Top = 0; Top < 0x100; ++Top) {
+ uptr Ptr = (Addr | (Top << 56)) & UnTagMask;
+ EXPECT_EQ(addFixedTag(Ptr, 0), untagPointer(Ptr));
+ }
+}
+
+TEST_F(MemtagDeathTest, ScopedDisableMemoryTagChecks) {
+ u8 *P = reinterpret_cast<u8 *>(addFixedTag(Addr, 1));
+ EXPECT_NE(P, Buffer);
+
+ EXPECT_DEATH(*P = 20, "");
+ ScopedDisableMemoryTagChecks Disable;
+ *P = 10;
+}
+
+TEST_F(MemtagTest, SelectRandomTag) {
+ for (uptr SrcTag = 0; SrcTag < 0x10; ++SrcTag) {
+ uptr Ptr = addFixedTag(Addr, SrcTag);
+ uptr Tags = 0;
+ for (uptr I = 0; I < 100000; ++I)
+ Tags = Tags | (1u << extractTag(selectRandomTag(Ptr, 0)));
+ EXPECT_EQ(0xfffeull, Tags);
+ }
+}
+
+TEST_F(MemtagTest, SelectRandomTagWithMask) {
+ for (uptr j = 0; j < 32; ++j) {
+ for (uptr i = 0; i < 1000; ++i)
+ EXPECT_NE(j, extractTag(selectRandomTag(Addr, 1ull << j)));
+ }
+}
+
+TEST_F(MemtagDeathTest, SKIP_NO_DEBUG(LoadStoreTagUnaligned)) {
+ for (uptr P = Addr; P < Addr + 4 * archMemoryTagGranuleSize(); ++P) {
+ if (P % archMemoryTagGranuleSize() == 0)
+ continue;
+ EXPECT_DEBUG_DEATH(loadTag(P), "");
+ EXPECT_DEBUG_DEATH(storeTag(P), "");
+ }
+}
+
+TEST_F(MemtagTest, LoadStoreTag) {
+ uptr Base = Addr + 0x100;
+ uptr Tagged = addFixedTag(Base, 7);
+ storeTag(Tagged);
+
+ EXPECT_EQ(Base - archMemoryTagGranuleSize(),
+ loadTag(Base - archMemoryTagGranuleSize()));
+ EXPECT_EQ(Tagged, loadTag(Base));
+ EXPECT_EQ(Base + archMemoryTagGranuleSize(),
+ loadTag(Base + archMemoryTagGranuleSize()));
+}
+
+TEST_F(MemtagDeathTest, SKIP_NO_DEBUG(StoreTagsUnaligned)) {
+ for (uptr P = Addr; P < Addr + 4 * archMemoryTagGranuleSize(); ++P) {
+ uptr Tagged = addFixedTag(P, 5);
+ if (Tagged % archMemoryTagGranuleSize() == 0)
+ continue;
+ EXPECT_DEBUG_DEATH(storeTags(Tagged, Tagged), "");
+ }
+}
+
+TEST_F(MemtagTest, StoreTags) {
+ const uptr MaxTaggedSize = 4 * archMemoryTagGranuleSize();
+ for (uptr Size = 0; Size <= MaxTaggedSize; ++Size) {
+ uptr NoTagBegin = Addr + archMemoryTagGranuleSize();
+ uptr NoTagEnd = NoTagBegin + Size;
+
+ u8 Tag = 5;
+
+ uptr TaggedBegin = addFixedTag(NoTagBegin, Tag);
+ uptr TaggedEnd = addFixedTag(NoTagEnd, Tag);
+
+ EXPECT_EQ(roundUpTo(TaggedEnd, archMemoryTagGranuleSize()),
+ storeTags(TaggedBegin, TaggedEnd));
+
+ uptr LoadPtr = Addr;
+ // Untagged left granule.
+ EXPECT_EQ(LoadPtr, loadTag(LoadPtr));
+
+ for (LoadPtr += archMemoryTagGranuleSize(); LoadPtr < NoTagEnd;
+ LoadPtr += archMemoryTagGranuleSize()) {
+ EXPECT_EQ(addFixedTag(LoadPtr, 5), loadTag(LoadPtr));
+ }
+
+ // Untagged right granule.
+ EXPECT_EQ(LoadPtr, loadTag(LoadPtr));
+
+ // Reset tags without using StoreTags.
+ releasePagesToOS(Addr, 0, BufferSize, &Data);
+ }
+}
+
+} // namespace scudo
+
+#endif
TEST(ScudoMutexTest, Mutex) {
scudo::HybridMutex M;
- M.init();
TestData Data(M);
pthread_t Threads[NumberOfThreads];
for (scudo::u32 I = 0; I < NumberOfThreads; I++)
TEST(ScudoMutexTest, MutexTry) {
scudo::HybridMutex M;
- M.init();
TestData Data(M);
pthread_t Threads[NumberOfThreads];
for (scudo::u32 I = 0; I < NumberOfThreads; I++)
#include <condition_variable>
#include <mutex>
+#include <stdlib.h>
#include <thread>
#include <vector>
// 32-bit architectures. It's not something we want to encourage, but we still
// should ensure the tests pass.
-template <typename Primary> static void testPrimary() {
- const scudo::uptr NumberOfAllocations = 32U;
- auto Deleter = [](Primary *P) {
- P->unmapTestOnly();
- delete P;
- };
- std::unique_ptr<Primary, decltype(Deleter)> Allocator(new Primary, Deleter);
+struct TestConfig1 {
+ static const scudo::uptr PrimaryRegionSizeLog = 18U;
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ static const bool MaySupportMemoryTagging = false;
+ typedef scudo::uptr PrimaryCompactPtrT;
+ static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+};
+
+struct TestConfig2 {
+#if defined(__mips__)
+ // Unable to allocate greater size on QEMU-user.
+ static const scudo::uptr PrimaryRegionSizeLog = 23U;
+#else
+ static const scudo::uptr PrimaryRegionSizeLog = 24U;
+#endif
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ static const bool MaySupportMemoryTagging = false;
+ typedef scudo::uptr PrimaryCompactPtrT;
+ static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+};
+
+struct TestConfig3 {
+#if defined(__mips__)
+ // Unable to allocate greater size on QEMU-user.
+ static const scudo::uptr PrimaryRegionSizeLog = 23U;
+#else
+ static const scudo::uptr PrimaryRegionSizeLog = 24U;
+#endif
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ static const bool MaySupportMemoryTagging = true;
+ typedef scudo::uptr PrimaryCompactPtrT;
+ static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+};
+
+template <typename BaseConfig, typename SizeClassMapT>
+struct Config : public BaseConfig {
+ using SizeClassMap = SizeClassMapT;
+};
+
+template <typename BaseConfig, typename SizeClassMapT>
+struct SizeClassAllocator
+ : public scudo::SizeClassAllocator64<Config<BaseConfig, SizeClassMapT>> {};
+template <typename SizeClassMapT>
+struct SizeClassAllocator<TestConfig1, SizeClassMapT>
+ : public scudo::SizeClassAllocator32<Config<TestConfig1, SizeClassMapT>> {};
+
+template <typename BaseConfig, typename SizeClassMapT>
+struct TestAllocator : public SizeClassAllocator<BaseConfig, SizeClassMapT> {
+ ~TestAllocator() { this->unmapTestOnly(); }
+
+ void *operator new(size_t size) {
+ void *p = nullptr;
+ EXPECT_EQ(0, posix_memalign(&p, alignof(TestAllocator), size));
+ return p;
+ }
+
+ void operator delete(void *ptr) { free(ptr); }
+};
+
+template <class BaseConfig> struct ScudoPrimaryTest : public Test {};
+
+#if SCUDO_FUCHSIA
+#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig2) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig3)
+#else
+#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig1) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig2) \
+ SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConfig3)
+#endif
+
+#define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE) \
+ using FIXTURE##NAME##_##TYPE = FIXTURE##NAME<TYPE>; \
+ TEST_F(FIXTURE##NAME##_##TYPE, NAME) { Run(); }
+
+#define SCUDO_TYPED_TEST(FIXTURE, NAME) \
+ template <class TypeParam> \
+ struct FIXTURE##NAME : public FIXTURE<TypeParam> { \
+ void Run(); \
+ }; \
+ SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME) \
+ template <class TypeParam> void FIXTURE##NAME<TypeParam>::Run()
+
+SCUDO_TYPED_TEST(ScudoPrimaryTest, BasicPrimary) {
+ using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
+ std::unique_ptr<Primary> Allocator(new Primary);
Allocator->init(/*ReleaseToOsInterval=*/-1);
typename Primary::CacheT Cache;
Cache.init(nullptr, Allocator.get());
+ const scudo::uptr NumberOfAllocations = 32U;
for (scudo::uptr I = 0; I <= 16U; I++) {
const scudo::uptr Size = 1UL << I;
if (!Primary::canAllocate(Size))
}
Cache.destroy(nullptr);
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
-TEST(ScudoPrimaryTest, BasicPrimary) {
+struct SmallRegionsConfig {
using SizeClassMap = scudo::DefaultSizeClassMap;
-#if !SCUDO_FUCHSIA
- testPrimary<scudo::SizeClassAllocator32<SizeClassMap, 18U>>();
-#endif
- testPrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U>>();
- testPrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U, true>>();
-}
+ static const scudo::uptr PrimaryRegionSizeLog = 20U;
+ static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+ static const bool MaySupportMemoryTagging = false;
+ typedef scudo::uptr PrimaryCompactPtrT;
+ static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
+};
// The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes.
// For the 32-bit one, it requires actually exhausting memory, so we skip it.
TEST(ScudoPrimaryTest, Primary64OOM) {
- using Primary = scudo::SizeClassAllocator64<scudo::DefaultSizeClassMap, 20U>;
+ using Primary = scudo::SizeClassAllocator64<SmallRegionsConfig>;
using TransferBatch = Primary::CacheT::TransferBatch;
Primary Allocator;
Allocator.init(/*ReleaseToOsInterval=*/-1);
break;
}
for (scudo::u32 J = 0; J < B->getCount(); J++)
- memset(B->get(J), 'B', Size);
+ memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
Batches.push_back(B);
}
while (!Batches.empty()) {
}
Cache.destroy(nullptr);
Allocator.releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator.getStats(&Str);
Str.output();
EXPECT_EQ(AllocationFailed, true);
Allocator.unmapTestOnly();
}
-template <typename Primary> static void testIteratePrimary() {
- auto Deleter = [](Primary *P) {
- P->unmapTestOnly();
- delete P;
- };
- std::unique_ptr<Primary, decltype(Deleter)> Allocator(new Primary, Deleter);
+SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryIterate) {
+ using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
+ std::unique_ptr<Primary> Allocator(new Primary);
Allocator->init(/*ReleaseToOsInterval=*/-1);
typename Primary::CacheT Cache;
Cache.init(nullptr, Allocator.get());
}
Cache.destroy(nullptr);
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
-TEST(ScudoPrimaryTest, PrimaryIterate) {
- using SizeClassMap = scudo::DefaultSizeClassMap;
-#if !SCUDO_FUCHSIA
- testIteratePrimary<scudo::SizeClassAllocator32<SizeClassMap, 18U>>();
-#endif
- testIteratePrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U>>();
- testIteratePrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U, true>>();
-}
-
-static std::mutex Mutex;
-static std::condition_variable Cv;
-static bool Ready = false;
-
-template <typename Primary> static void performAllocations(Primary *Allocator) {
- static THREADLOCAL typename Primary::CacheT Cache;
- Cache.init(nullptr, Allocator);
- std::vector<std::pair<scudo::uptr, void *>> V;
- {
- std::unique_lock<std::mutex> Lock(Mutex);
- while (!Ready)
- Cv.wait(Lock);
- }
- for (scudo::uptr I = 0; I < 256U; I++) {
- const scudo::uptr Size = std::rand() % Primary::SizeClassMap::MaxSize / 4;
- const scudo::uptr ClassId = Primary::SizeClassMap::getClassIdBySize(Size);
- void *P = Cache.allocate(ClassId);
- if (P)
- V.push_back(std::make_pair(ClassId, P));
- }
- while (!V.empty()) {
- auto Pair = V.back();
- Cache.deallocate(Pair.first, Pair.second);
- V.pop_back();
- }
- Cache.destroy(nullptr);
-}
-
-template <typename Primary> static void testPrimaryThreaded() {
- auto Deleter = [](Primary *P) {
- P->unmapTestOnly();
- delete P;
- };
- std::unique_ptr<Primary, decltype(Deleter)> Allocator(new Primary, Deleter);
+SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
+ using Primary = TestAllocator<TypeParam, scudo::SvelteSizeClassMap>;
+ std::unique_ptr<Primary> Allocator(new Primary);
Allocator->init(/*ReleaseToOsInterval=*/-1);
+ std::mutex Mutex;
+ std::condition_variable Cv;
+ bool Ready = false;
std::thread Threads[32];
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
- Threads[I] = std::thread(performAllocations<Primary>, Allocator.get());
+ Threads[I] = std::thread([&]() {
+ static thread_local typename Primary::CacheT Cache;
+ Cache.init(nullptr, Allocator.get());
+ std::vector<std::pair<scudo::uptr, void *>> V;
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ while (!Ready)
+ Cv.wait(Lock);
+ }
+ for (scudo::uptr I = 0; I < 256U; I++) {
+ const scudo::uptr Size =
+ std::rand() % Primary::SizeClassMap::MaxSize / 4;
+ const scudo::uptr ClassId =
+ Primary::SizeClassMap::getClassIdBySize(Size);
+ void *P = Cache.allocate(ClassId);
+ if (P)
+ V.push_back(std::make_pair(ClassId, P));
+ }
+ while (!V.empty()) {
+ auto Pair = V.back();
+ Cache.deallocate(Pair.first, Pair.second);
+ V.pop_back();
+ }
+ Cache.destroy(nullptr);
+ });
{
std::unique_lock<std::mutex> Lock(Mutex);
Ready = true;
for (auto &T : Threads)
T.join();
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
-TEST(ScudoPrimaryTest, PrimaryThreaded) {
- using SizeClassMap = scudo::SvelteSizeClassMap;
-#if !SCUDO_FUCHSIA
- testPrimaryThreaded<scudo::SizeClassAllocator32<SizeClassMap, 18U>>();
-#endif
- testPrimaryThreaded<scudo::SizeClassAllocator64<SizeClassMap, 24U>>();
- testPrimaryThreaded<scudo::SizeClassAllocator64<SizeClassMap, 24U, true>>();
-}
-
// Through a simple allocation that spans two pages, verify that releaseToOS
// actually releases some bytes (at least one page worth). This is a regression
// test for an error in how the release criteria were computed.
-template <typename Primary> static void testReleaseToOS() {
- auto Deleter = [](Primary *P) {
- P->unmapTestOnly();
- delete P;
- };
- std::unique_ptr<Primary, decltype(Deleter)> Allocator(new Primary, Deleter);
+SCUDO_TYPED_TEST(ScudoPrimaryTest, ReleaseToOS) {
+ using Primary = TestAllocator<TypeParam, scudo::DefaultSizeClassMap>;
+ std::unique_ptr<Primary> Allocator(new Primary);
Allocator->init(/*ReleaseToOsInterval=*/-1);
typename Primary::CacheT Cache;
Cache.init(nullptr, Allocator.get());
Cache.destroy(nullptr);
EXPECT_GT(Allocator->releaseToOS(), 0U);
}
-
-TEST(ScudoPrimaryTest, ReleaseToOS) {
- using SizeClassMap = scudo::DefaultSizeClassMap;
-#if !SCUDO_FUCHSIA
- testReleaseToOS<scudo::SizeClassAllocator32<SizeClassMap, 18U>>();
-#endif
- testReleaseToOS<scudo::SizeClassAllocator64<SizeClassMap, 24U>>();
- testReleaseToOS<scudo::SizeClassAllocator64<SizeClassMap, 24U, true>>();
-}
Quarantine.drainAndRecycle(&Cache, Cb);
EXPECT_EQ(Cache.getSize(), 0UL);
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Quarantine.getStats(&Str);
Str.output();
}
-void *populateQuarantine(void *Param) {
+struct PopulateQuarantineThread {
+ pthread_t Thread;
+ QuarantineT *Quarantine;
CacheT Cache;
- Cache.init();
- QuarantineT *Quarantine = reinterpret_cast<QuarantineT *>(Param);
+};
+
+void *populateQuarantine(void *Param) {
+ PopulateQuarantineThread *P = static_cast<PopulateQuarantineThread *>(Param);
+ P->Cache.init();
for (scudo::uptr I = 0; I < 128UL; I++)
- Quarantine->put(&Cache, Cb, FakePtr, LargeBlockSize);
+ P->Quarantine->put(&P->Cache, Cb, FakePtr, LargeBlockSize);
return 0;
}
Quarantine.init(MaxQuarantineSize, MaxCacheSize);
const scudo::uptr NumberOfThreads = 32U;
- pthread_t T[NumberOfThreads];
- for (scudo::uptr I = 0; I < NumberOfThreads; I++)
- pthread_create(&T[I], 0, populateQuarantine, &Quarantine);
+ PopulateQuarantineThread T[NumberOfThreads];
+ for (scudo::uptr I = 0; I < NumberOfThreads; I++) {
+ T[I].Quarantine = &Quarantine;
+ pthread_create(&T[I].Thread, 0, populateQuarantine, &T[I]);
+ }
for (scudo::uptr I = 0; I < NumberOfThreads; I++)
- pthread_join(T[I], 0);
+ pthread_join(T[I].Thread, 0);
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Quarantine.getStats(&Str);
Str.output();
+
+ for (scudo::uptr I = 0; I < NumberOfThreads; I++)
+ Quarantine.drainAndRecycle(&T[I].Cache, Cb);
}
TEST(ScudoReleaseTest, PackedCounterArray) {
for (scudo::uptr I = 0; I < SCUDO_WORDSIZE; I++) {
// Various valid counter's max values packed into one word.
- scudo::PackedCounterArray Counters2N(1, 1UL << I);
+ scudo::PackedCounterArray Counters2N(1U, 1U, 1UL << I);
EXPECT_EQ(sizeof(scudo::uptr), Counters2N.getBufferSize());
// Check the "all bit set" values too.
- scudo::PackedCounterArray Counters2N1_1(1, ~0UL >> I);
+ scudo::PackedCounterArray Counters2N1_1(1U, 1U, ~0UL >> I);
EXPECT_EQ(sizeof(scudo::uptr), Counters2N1_1.getBufferSize());
// Verify the packing ratio, the counter is Expected to be packed into the
// closest power of 2 bits.
- scudo::PackedCounterArray Counters(SCUDO_WORDSIZE, 1UL << I);
+ scudo::PackedCounterArray Counters(1U, SCUDO_WORDSIZE, 1UL << I);
EXPECT_EQ(sizeof(scudo::uptr) * scudo::roundUpToPowerOfTwo(I + 1),
Counters.getBufferSize());
}
// Make sure counters request one memory page for the buffer.
const scudo::uptr NumCounters =
(scudo::getPageSizeCached() / 8) * (SCUDO_WORDSIZE >> I);
- scudo::PackedCounterArray Counters(NumCounters, 1UL << ((1UL << I) - 1));
- Counters.inc(0);
+ scudo::PackedCounterArray Counters(1U, NumCounters,
+ 1UL << ((1UL << I) - 1));
+ Counters.inc(0U, 0U);
for (scudo::uptr C = 1; C < NumCounters - 1; C++) {
- EXPECT_EQ(0UL, Counters.get(C));
- Counters.inc(C);
- EXPECT_EQ(1UL, Counters.get(C - 1));
+ EXPECT_EQ(0UL, Counters.get(0U, C));
+ Counters.inc(0U, C);
+ EXPECT_EQ(1UL, Counters.get(0U, C - 1));
}
- EXPECT_EQ(0UL, Counters.get(NumCounters - 1));
- Counters.inc(NumCounters - 1);
+ EXPECT_EQ(0UL, Counters.get(0U, NumCounters - 1));
+ Counters.inc(0U, NumCounters - 1);
if (I > 0) {
- Counters.incRange(0, NumCounters - 1);
+ Counters.incRange(0u, 0U, NumCounters - 1);
for (scudo::uptr C = 0; C < NumCounters; C++)
- EXPECT_EQ(2UL, Counters.get(C));
+ EXPECT_EQ(2UL, Counters.get(0U, C));
}
}
}
for (scudo::uptr I = From; I < To; I += PageSize)
ReportedPages.insert(I);
}
+
+ scudo::uptr getBase() const { return 0; }
};
// Simplified version of a TransferBatch.
}
// Release the memory.
+ auto SkipRegion = [](UNUSED scudo::uptr RegionIndex) { return false; };
+ auto DecompactPtr = [](scudo::uptr P) { return P; };
ReleasedPagesRecorder Recorder;
- releaseFreeMemoryToOS(FreeList, 0, MaxBlocks * BlockSize, BlockSize,
- &Recorder);
+ releaseFreeMemoryToOS(FreeList, MaxBlocks * BlockSize, 1U, BlockSize,
+ &Recorder, DecompactPtr, SkipRegion);
// Verify that there are no released pages touched by used chunks and all
// ranges of free chunks big enough to contain the entire memory pages had
#include "report.h"
-TEST(ScudoReportTest, Generic) {
+TEST(ScudoReportDeathTest, Check) {
+ CHECK_LT(-1, 1);
+ EXPECT_DEATH(CHECK_GT(-1, 1),
+ "\\(-1\\) > \\(1\\) \\(\\(u64\\)op1=18446744073709551615, "
+ "\\(u64\\)op2=1");
+}
+
+TEST(ScudoReportDeathTest, Generic) {
// Potentially unused if EXPECT_DEATH isn't defined.
UNUSED void *P = reinterpret_cast<void *>(0x42424242U);
EXPECT_DEATH(scudo::reportError("TEST123"), "Scudo ERROR.*TEST123");
"Scudo ERROR.*42424242.*123.*456");
}
-TEST(ScudoReportTest, CSpecific) {
+TEST(ScudoReportDeathTest, CSpecific) {
EXPECT_DEATH(scudo::reportAlignmentNotPowerOfTwo(123), "Scudo ERROR.*123");
EXPECT_DEATH(scudo::reportCallocOverflow(123, 456), "Scudo ERROR.*123.*456");
EXPECT_DEATH(scudo::reportInvalidPosixMemalignAlignment(789),
#if SCUDO_FUCHSIA
#include <zxtest/zxtest.h>
+using Test = ::zxtest::Test;
#else
#include "gtest/gtest.h"
+using Test = ::testing::Test;
#endif
// If EXPECT_DEATH isn't defined, make it a no-op.
#ifndef EXPECT_DEATH
+// If ASSERT_DEATH is defined, make EXPECT_DEATH a wrapper to it.
+#ifdef ASSERT_DEATH
+#define EXPECT_DEATH(X, Y) ASSERT_DEATH(([&] { X; }), "")
+#else
#define EXPECT_DEATH(X, Y) \
do { \
} while (0)
-#endif
+#endif // ASSERT_DEATH
+#endif // EXPECT_DEATH
// If EXPECT_STREQ isn't defined, define our own simple one.
#ifndef EXPECT_STREQ
#define EXPECT_STREQ(X, Y) EXPECT_EQ(strcmp(X, Y), 0)
#endif
+#if SCUDO_FUCHSIA
+#define SKIP_ON_FUCHSIA(T) DISABLED_##T
+#else
+#define SKIP_ON_FUCHSIA(T) T
+#endif
+
+#if SCUDO_DEBUG
+#define SKIP_NO_DEBUG(T) T
+#else
+#define SKIP_NO_DEBUG(T) DISABLED_##T
+#endif
+
extern bool UseQuarantine;
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
// Match Android's default configuration, which disables Scudo's mismatch
#define DEALLOC_TYPE_MISMATCH "true"
#endif
+static void EnableMemoryTaggingIfSupported() {
+ if (!scudo::archSupportsMemoryTagging())
+ return;
+ static bool Done = []() {
+ if (!scudo::systemDetectsMemoryTagFaultsTestOnly())
+ scudo::enableSystemMemoryTaggingTestOnly();
+ return true;
+ }();
+ (void)Done;
+}
+
// This allows us to turn on/off a Quarantine for specific tests. The Quarantine
// parameters are on the low end, to avoid having to loop excessively in some
// tests.
bool UseQuarantine = true;
extern "C" __attribute__((visibility("default"))) const char *
__scudo_default_options() {
+ // The wrapper tests initialize the global allocator early, before main(). We
+ // need to have Memory Tagging enabled before that happens or the allocator
+ // will disable the feature entirely.
+ EnableMemoryTaggingIfSupported();
if (!UseQuarantine)
return "dealloc_type_mismatch=" DEALLOC_TYPE_MISMATCH;
return "quarantine_size_kb=256:thread_local_quarantine_size_kb=128:"
"dealloc_type_mismatch=" DEALLOC_TYPE_MISMATCH;
}
-int main(int argc, char **argv) {
+// The zxtest library provides a default main function that does the same thing
+// for Fuchsia builds.
#if !SCUDO_FUCHSIA
+int main(int argc, char **argv) {
+ EnableMemoryTaggingIfSupported();
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
-#else
- return RUN_ALL_TESTS(argc, argv);
-#endif
}
+#endif
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
+#include "allocator_config.h"
#include "secondary.h"
-#include <stdio.h>
-
#include <condition_variable>
+#include <memory>
#include <mutex>
#include <random>
+#include <stdio.h>
#include <thread>
#include <vector>
-template <class SecondaryT> static void testSecondaryBasic(void) {
+template <typename Config> static scudo::Options getOptionsForConfig() {
+ if (!Config::MaySupportMemoryTagging || !scudo::archSupportsMemoryTagging() ||
+ !scudo::systemSupportsMemoryTagging())
+ return {};
+ scudo::AtomicOptions AO;
+ AO.set(scudo::OptionBit::UseMemoryTagging);
+ return AO.load();
+}
+
+template <typename Config> static void testSecondaryBasic(void) {
+ using SecondaryT = scudo::MapAllocator<Config>;
+ scudo::Options Options = getOptionsForConfig<Config>();
+
scudo::GlobalStats S;
S.init();
- SecondaryT *L = new SecondaryT;
+ std::unique_ptr<SecondaryT> L(new SecondaryT);
L->init(&S);
const scudo::uptr Size = 1U << 16;
- void *P = L->allocate(Size);
+ void *P = L->allocate(Options, Size);
EXPECT_NE(P, nullptr);
memset(P, 'A', Size);
EXPECT_GE(SecondaryT::getBlockSize(P), Size);
- L->deallocate(P);
+ L->deallocate(Options, P);
+
// If the Secondary can't cache that pointer, it will be unmapped.
- if (!SecondaryT::canCache(Size))
- EXPECT_DEATH(memset(P, 'A', Size), "");
+ if (!L->canCache(Size)) {
+ EXPECT_DEATH(
+ {
+ // Repeat few time to avoid missing crash if it's mmaped by unrelated
+ // code.
+ for (int i = 0; i < 10; ++i) {
+ P = L->allocate(Options, Size);
+ L->deallocate(Options, P);
+ memset(P, 'A', Size);
+ }
+ },
+ "");
+ }
const scudo::uptr Align = 1U << 16;
- P = L->allocate(Size + Align, Align);
+ P = L->allocate(Options, Size + Align, Align);
EXPECT_NE(P, nullptr);
void *AlignedP = reinterpret_cast<void *>(
scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align));
memset(AlignedP, 'A', Size);
- L->deallocate(P);
+ L->deallocate(Options, P);
std::vector<void *> V;
for (scudo::uptr I = 0; I < 32U; I++)
- V.push_back(L->allocate(Size));
+ V.push_back(L->allocate(Options, Size));
std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()()));
while (!V.empty()) {
- L->deallocate(V.back());
+ L->deallocate(Options, V.back());
V.pop_back();
}
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
L->getStats(&Str);
Str.output();
+ L->unmapTestOnly();
}
+struct NoCacheConfig {
+ typedef scudo::MapAllocatorNoCache SecondaryCache;
+ static const bool MaySupportMemoryTagging = false;
+};
+
+struct TestConfig {
+ typedef scudo::MapAllocatorCache<TestConfig> SecondaryCache;
+ static const bool MaySupportMemoryTagging = false;
+ static const scudo::u32 SecondaryCacheEntriesArraySize = 128U;
+ static const scudo::u32 SecondaryCacheQuarantineSize = 0U;
+ static const scudo::u32 SecondaryCacheDefaultMaxEntriesCount = 64U;
+ static const scudo::uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 20;
+ static const scudo::s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN;
+ static const scudo::s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX;
+};
+
TEST(ScudoSecondaryTest, SecondaryBasic) {
- testSecondaryBasic<scudo::MapAllocator<scudo::MapAllocatorNoCache>>();
-#if !SCUDO_FUCHSIA
- testSecondaryBasic<scudo::MapAllocator<scudo::MapAllocatorCache<>>>();
- testSecondaryBasic<
- scudo::MapAllocator<scudo::MapAllocatorCache<64U, 1UL << 20>>>();
-#endif
+ testSecondaryBasic<NoCacheConfig>();
+ testSecondaryBasic<scudo::DefaultConfig>();
+ testSecondaryBasic<TestConfig>();
}
-#if SCUDO_FUCHSIA
-using LargeAllocator = scudo::MapAllocator<scudo::MapAllocatorNoCache>;
-#else
-using LargeAllocator = scudo::MapAllocator<scudo::MapAllocatorCache<>>;
-#endif
+struct MapAllocatorTest : public Test {
+ using Config = scudo::DefaultConfig;
+ using LargeAllocator = scudo::MapAllocator<Config>;
+
+ void SetUp() override { Allocator->init(nullptr); }
+
+ void TearDown() override { Allocator->unmapTestOnly(); }
+
+ std::unique_ptr<LargeAllocator> Allocator =
+ std::make_unique<LargeAllocator>();
+ scudo::Options Options = getOptionsForConfig<Config>();
+};
// This exercises a variety of combinations of size and alignment for the
// MapAllocator. The size computation done here mimic the ones done by the
// combined allocator.
-TEST(ScudoSecondaryTest, SecondaryCombinations) {
+TEST_F(MapAllocatorTest, SecondaryCombinations) {
constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16);
constexpr scudo::uptr HeaderSize = scudo::roundUpTo(8, MinAlign);
- LargeAllocator *L = new LargeAllocator;
- L->init(nullptr);
for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) {
for (scudo::uptr AlignLog = FIRST_32_SECOND_64(3, 4); AlignLog <= 16;
AlignLog++) {
scudo::roundUpTo((1U << SizeLog) + Delta, MinAlign);
const scudo::uptr Size =
HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0);
- void *P = L->allocate(Size, Align);
+ void *P = Allocator->allocate(Options, Size, Align);
EXPECT_NE(P, nullptr);
void *AlignedP = reinterpret_cast<void *>(
scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align));
memset(AlignedP, 0xff, UserSize);
- L->deallocate(P);
+ Allocator->deallocate(Options, P);
}
}
}
- scudo::ScopedString Str(1024);
- L->getStats(&Str);
+ scudo::ScopedString Str;
+ Allocator->getStats(&Str);
Str.output();
}
-TEST(ScudoSecondaryTest, SecondaryIterate) {
- LargeAllocator *L = new LargeAllocator;
- L->init(nullptr);
+TEST_F(MapAllocatorTest, SecondaryIterate) {
std::vector<void *> V;
const scudo::uptr PageSize = scudo::getPageSizeCached();
for (scudo::uptr I = 0; I < 32U; I++)
- V.push_back(L->allocate((std::rand() % 16) * PageSize));
+ V.push_back(Allocator->allocate(Options, (std::rand() % 16) * PageSize));
auto Lambda = [V](scudo::uptr Block) {
EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)),
V.end());
};
- L->disable();
- L->iterateOverBlocks(Lambda);
- L->enable();
+ Allocator->disable();
+ Allocator->iterateOverBlocks(Lambda);
+ Allocator->enable();
while (!V.empty()) {
- L->deallocate(V.back());
+ Allocator->deallocate(Options, V.back());
V.pop_back();
}
- scudo::ScopedString Str(1024);
- L->getStats(&Str);
+ scudo::ScopedString Str;
+ Allocator->getStats(&Str);
Str.output();
}
-static std::mutex Mutex;
-static std::condition_variable Cv;
-static bool Ready = false;
-
-static void performAllocations(LargeAllocator *L) {
- std::vector<void *> V;
- const scudo::uptr PageSize = scudo::getPageSizeCached();
- {
- std::unique_lock<std::mutex> Lock(Mutex);
- while (!Ready)
- Cv.wait(Lock);
- }
- for (scudo::uptr I = 0; I < 128U; I++) {
- // Deallocate 75% of the blocks.
- const bool Deallocate = (rand() & 3) != 0;
- void *P = L->allocate((std::rand() % 16) * PageSize);
- if (Deallocate)
- L->deallocate(P);
- else
- V.push_back(P);
- }
- while (!V.empty()) {
- L->deallocate(V.back());
- V.pop_back();
+TEST_F(MapAllocatorTest, SecondaryOptions) {
+ // Attempt to set a maximum number of entries higher than the array size.
+ EXPECT_FALSE(
+ Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4096U));
+ // A negative number will be cast to a scudo::u32, and fail.
+ EXPECT_FALSE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, -1));
+ if (Allocator->canCache(0U)) {
+ // Various valid combinations.
+ EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+ EXPECT_TRUE(
+ Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+ EXPECT_TRUE(Allocator->canCache(1UL << 18));
+ EXPECT_TRUE(
+ Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17));
+ EXPECT_FALSE(Allocator->canCache(1UL << 18));
+ EXPECT_TRUE(Allocator->canCache(1UL << 16));
+ EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 0U));
+ EXPECT_FALSE(Allocator->canCache(1UL << 16));
+ EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U));
+ EXPECT_TRUE(
+ Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
+ EXPECT_TRUE(Allocator->canCache(1UL << 16));
}
}
-TEST(ScudoSecondaryTest, SecondaryThreadsRace) {
- LargeAllocator *L = new LargeAllocator;
- L->init(nullptr, /*ReleaseToOsInterval=*/0);
+struct MapAllocatorWithReleaseTest : public MapAllocatorTest {
+ void SetUp() override { Allocator->init(nullptr, /*ReleaseToOsInterval=*/0); }
+
+ void performAllocations() {
+ std::vector<void *> V;
+ const scudo::uptr PageSize = scudo::getPageSizeCached();
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ while (!Ready)
+ Cv.wait(Lock);
+ }
+ for (scudo::uptr I = 0; I < 128U; I++) {
+ // Deallocate 75% of the blocks.
+ const bool Deallocate = (rand() & 3) != 0;
+ void *P = Allocator->allocate(Options, (std::rand() % 16) * PageSize);
+ if (Deallocate)
+ Allocator->deallocate(Options, P);
+ else
+ V.push_back(P);
+ }
+ while (!V.empty()) {
+ Allocator->deallocate(Options, V.back());
+ V.pop_back();
+ }
+ }
+
+ std::mutex Mutex;
+ std::condition_variable Cv;
+ bool Ready = false;
+};
+
+TEST_F(MapAllocatorWithReleaseTest, SecondaryThreadsRace) {
std::thread Threads[16];
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
- Threads[I] = std::thread(performAllocations, L);
+ Threads[I] =
+ std::thread(&MapAllocatorWithReleaseTest::performAllocations, this);
{
std::unique_lock<std::mutex> Lock(Mutex);
Ready = true;
}
for (auto &T : Threads)
T.join();
- scudo::ScopedString Str(1024);
- L->getStats(&Str);
+ scudo::ScopedString Str;
+ Allocator->getStats(&Str);
Str.output();
}
static const scudo::uptr MaxSizeLog = 5;
static const scudo::u32 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0;
+ static const scudo::uptr SizeDelta = 0;
};
TEST(ScudoSizeClassMapTest, OneClassSizeClassMap) {
static const scudo::uptr MaxSizeLog = 63;
static const scudo::u32 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16;
+ static const scudo::uptr SizeDelta = 0;
};
TEST(ScudoSizeClassMapTest, LargeMaxSizeClassMap) {
#include <limits.h>
+TEST(ScudoStringsTest, Constructor) {
+ scudo::ScopedString Str;
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
TEST(ScudoStringsTest, Basic) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append("a%db%zdc%ue%zuf%xh%zxq%pe%sr", static_cast<int>(-1),
static_cast<scudo::uptr>(-2), static_cast<unsigned>(-4),
static_cast<scudo::uptr>(5), static_cast<unsigned>(10),
EXPECT_STREQ(expectedString.c_str(), Str.data());
}
+TEST(ScudoStringsTest, Clear) {
+ scudo::ScopedString Str;
+ Str.append("123");
+ Str.clear();
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
+TEST(ScudoStringsTest, ClearLarge) {
+ scudo::ScopedString Str;
+ for (int i = 0; i < 10000; ++i)
+ Str.append("123");
+ Str.clear();
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
TEST(ScudoStringsTest, Precision) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append("%.*s", 3, "12345");
EXPECT_EQ(Str.length(), strlen(Str.data()));
EXPECT_STREQ("123", Str.data());
// Use a ScopedString that spans a page, and attempt to write past the end
// of it with variations of append. The expectation is for nothing to crash.
const scudo::uptr PageSize = scudo::getPageSizeCached();
- scudo::ScopedString Str(PageSize);
+ scudo::ScopedString Str;
Str.clear();
fillString(Str, 2 * PageSize);
Str.clear();
template <typename T>
static void testAgainstLibc(const char *Format, T Arg1, T Arg2) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append(Format, Arg1, Arg2);
char Buffer[128];
snprintf(Buffer, sizeof(Buffer), Format, Arg1, Arg2);
#include "tsd_exclusive.h"
#include "tsd_shared.h"
+#include <stdlib.h>
+
#include <condition_variable>
#include <mutex>
+#include <set>
#include <thread>
// We mock out an allocator with a TSD registry, mostly using empty stubs. The
using CacheT = struct MockCache { volatile scudo::uptr Canary; };
using QuarantineCacheT = struct MockQuarantine {};
- void initLinkerInitialized() {
+ void init() {
// This should only be called once by the registry.
EXPECT_FALSE(Initialized);
Initialized = true;
}
- void reset() { memset(this, 0, sizeof(*this)); }
- void unmapTestOnly() { TSDRegistry.unmapTestOnly(); }
- void initCache(CacheT *Cache) { memset(Cache, 0, sizeof(*Cache)); }
+ void unmapTestOnly() { TSDRegistry.unmapTestOnly(this); }
+ void initCache(CacheT *Cache) { *Cache = {}; }
void commitBack(scudo::TSD<MockAllocator> *TSD) {}
TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
void callPostInitCallback() {}
bool isInitialized() { return Initialized; }
+ void *operator new(size_t Size) {
+ void *P = nullptr;
+ EXPECT_EQ(0, posix_memalign(&P, alignof(ThisT), Size));
+ return P;
+ }
+ void operator delete(void *P) { free(P); }
+
private:
- bool Initialized;
+ bool Initialized = false;
TSDRegistryT TSDRegistry;
};
struct OneCache {
template <class Allocator>
- using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 1U>;
+ using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 1U, 1U>;
};
struct SharedCaches {
template <class Allocator>
- using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 16U>;
+ using TSDRegistryT = scudo::TSDRegistrySharedT<Allocator, 16U, 8U>;
};
struct ExclusiveCaches {
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
EXPECT_FALSE(Allocator->isInitialized());
auto Registry = Allocator->getTSDRegistry();
- Registry->initLinkerInitialized(Allocator.get());
+ Registry->init(Allocator.get());
EXPECT_TRUE(Allocator->isInitialized());
}
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
EXPECT_FALSE(Allocator->isInitialized());
auto Registry = Allocator->getTSDRegistry();
static std::mutex Mutex;
static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
template <typename AllocatorT> static void stressCache(AllocatorT *Allocator) {
auto Registry = Allocator->getTSDRegistry();
}
template <class AllocatorT> static void testRegistryThreaded() {
+ Ready = false;
auto Deleter = [](AllocatorT *A) {
A->unmapTestOnly();
delete A;
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
std::thread Threads[32];
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
Threads[I] = std::thread(stressCache<AllocatorT>, Allocator.get());
testRegistryThreaded<MockAllocator<ExclusiveCaches>>();
#endif
}
+
+static std::set<void *> Pointers;
+
+static void stressSharedRegistry(MockAllocator<SharedCaches> *Allocator) {
+ std::set<void *> Set;
+ auto Registry = Allocator->getTSDRegistry();
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ while (!Ready)
+ Cv.wait(Lock);
+ }
+ Registry->initThreadMaybe(Allocator, /*MinimalInit=*/false);
+ bool UnlockRequired;
+ for (scudo::uptr I = 0; I < 4096U; I++) {
+ auto TSD = Registry->getTSDAndLock(&UnlockRequired);
+ EXPECT_NE(TSD, nullptr);
+ Set.insert(reinterpret_cast<void *>(TSD));
+ if (UnlockRequired)
+ TSD->unlock();
+ }
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ Pointers.insert(Set.begin(), Set.end());
+ }
+}
+
+TEST(ScudoTSDTest, TSDRegistryTSDsCount) {
+ Ready = false;
+ Pointers.clear();
+ using AllocatorT = MockAllocator<SharedCaches>;
+ auto Deleter = [](AllocatorT *A) {
+ A->unmapTestOnly();
+ delete A;
+ };
+ std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
+ Deleter);
+ // We attempt to use as many TSDs as the shared cache offers by creating a
+ // decent amount of threads that will be run concurrently and attempt to get
+ // and lock TSDs. We put them all in a set and count the number of entries
+ // after we are done.
+ std::thread Threads[32];
+ for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
+ Threads[I] = std::thread(stressSharedRegistry, Allocator.get());
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ Ready = true;
+ Cv.notify_all();
+ }
+ for (auto &T : Threads)
+ T.join();
+ // The initial number of TSDs we get will be the minimum of the default count
+ // and the number of CPUs.
+ EXPECT_LE(Pointers.size(), 8U);
+ Pointers.clear();
+ auto Registry = Allocator->getTSDRegistry();
+ // Increase the number of TSDs to 16.
+ Registry->setOption(scudo::Option::MaxTSDsCount, 16);
+ Ready = false;
+ for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
+ Threads[I] = std::thread(stressSharedRegistry, Allocator.get());
+ {
+ std::unique_lock<std::mutex> Lock(Mutex);
+ Ready = true;
+ Cv.notify_all();
+ }
+ for (auto &T : Threads)
+ T.join();
+ // We should get 16 distinct TSDs back.
+ EXPECT_EQ(Pointers.size(), 16U);
+}
}
TEST(ScudoVectorTest, Stride) {
- scudo::Vector<int> V;
- for (int i = 0; i < 1000; i++) {
- V.push_back(i);
- EXPECT_EQ(V.size(), i + 1U);
- EXPECT_EQ(V[i], i);
+ scudo::Vector<scudo::uptr> V;
+ for (scudo::uptr I = 0; I < 1000; I++) {
+ V.push_back(I);
+ EXPECT_EQ(V.size(), I + 1U);
+ EXPECT_EQ(V[I], I);
}
- for (int i = 0; i < 1000; i++)
- EXPECT_EQ(V[i], i);
+ for (scudo::uptr I = 0; I < 1000; I++)
+ EXPECT_EQ(V[I], I);
}
TEST(ScudoVectorTest, ResizeReduction) {
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
+#include "scudo/interface.h"
#include "tests/scudo_unit_test.h"
#include <errno.h>
static const size_t Size = 100U;
-TEST(ScudoWrappersCTest, Malloc) {
+TEST(ScudoWrappersCDeathTest, Malloc) {
void *P = malloc(Size);
EXPECT_NE(P, nullptr);
EXPECT_LE(Size, malloc_usable_size(P));
EXPECT_EQ(reinterpret_cast<uintptr_t>(P) % FIRST_32_SECOND_64(8U, 16U), 0U);
+
+ // An update to this warning in Clang now triggers in this line, but it's ok
+ // because the check is expecting a bad pointer and should fail.
+#if defined(__has_warning) && __has_warning("-Wfree-nonheap-object")
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
+#endif
EXPECT_DEATH(
free(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(P) | 1U)), "");
+#if defined(__has_warning) && __has_warning("-Wfree-nonheap-object")
+#pragma GCC diagnostic pop
+#endif
+
free(P);
EXPECT_DEATH(free(P), "");
EXPECT_EQ(errno, ENOMEM);
}
+TEST(ScudoWrappersCTest, SmallAlign) {
+ void *P;
+ for (size_t Size = 1; Size <= 0x10000; Size <<= 1) {
+ for (size_t Align = 1; Align <= 0x10000; Align <<= 1) {
+ for (size_t Count = 0; Count < 3; ++Count) {
+ P = memalign(Align, Size);
+ EXPECT_TRUE(reinterpret_cast<uintptr_t>(P) % Align == 0);
+ }
+ }
+ }
+}
+
TEST(ScudoWrappersCTest, Memalign) {
void *P;
for (size_t I = FIRST_32_SECOND_64(2U, 3U); I <= 18U; I++) {
EXPECT_EQ(errno, EINVAL);
}
-TEST(ScudoWrappersCTest, Realloc) {
+TEST(ScudoWrappersCDeathTest, Realloc) {
// realloc(nullptr, N) is malloc(N)
void *P = realloc(nullptr, 0U);
EXPECT_NE(P, nullptr);
}
}
-#ifndef M_DECAY_TIME
-#define M_DECAY_TIME -100
-#endif
-
-#ifndef M_PURGE
-#define M_PURGE -101
-#endif
-
#if !SCUDO_FUCHSIA
TEST(ScudoWrappersCTest, MallOpt) {
errno = 0;
EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
EXPECT_EQ(mallopt(M_DECAY_TIME, 1), 1);
EXPECT_EQ(mallopt(M_DECAY_TIME, 0), 1);
+
+ if (SCUDO_ANDROID) {
+ EXPECT_EQ(mallopt(M_CACHE_COUNT_MAX, 100), 1);
+ EXPECT_EQ(mallopt(M_CACHE_SIZE_MAX, 1024 * 1024 * 2), 1);
+ EXPECT_EQ(mallopt(M_TSDS_COUNT_MAX, 10), 1);
+ }
}
#endif
static size_t Count;
static void callback(uintptr_t Base, size_t Size, void *Arg) {
+ if (scudo::archSupportsMemoryTagging()) {
+ Base = scudo::untagPointer(Base);
+ BoundaryP = scudo::untagPointer(BoundaryP);
+ }
if (Base == BoundaryP)
Count++;
}
}
}
-// We expect heap operations within a disable/enable scope to deadlock.
-TEST(ScudoWrappersCTest, MallocDisableDeadlock) {
+// Fuchsia doesn't have alarm, fork or malloc_info.
+#if !SCUDO_FUCHSIA
+TEST(ScudoWrappersCDeathTest, MallocDisableDeadlock) {
+ // We expect heap operations within a disable/enable scope to deadlock.
EXPECT_DEATH(
{
void *P = malloc(Size);
"");
}
-// Fuchsia doesn't have fork or malloc_info.
-#if !SCUDO_FUCHSIA
-
TEST(ScudoWrappersCTest, MallocInfo) {
// Use volatile so that the allocations don't get optimized away.
void *volatile P1 = malloc(1234);
free(P2);
}
-TEST(ScudoWrappersCTest, Fork) {
+TEST(ScudoWrappersCDeathTest, Fork) {
void *P;
pid_t Pid = fork();
- EXPECT_GE(Pid, 0);
+ EXPECT_GE(Pid, 0) << strerror(errno);
if (Pid == 0) {
P = malloc(Size);
EXPECT_NE(P, nullptr);
TEST(ScudoWrappersCTest, DisableForkEnable) {
pthread_t ThreadId;
+ Ready = false;
EXPECT_EQ(pthread_create(&ThreadId, nullptr, &enableMalloc, nullptr), 0);
// Wait for the thread to be warmed up.
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
#include <atomic>
#include <condition_variable>
+#include <memory>
#include <mutex>
#include <thread>
#include <vector>
Color C = Color::Red;
};
-TEST(ScudoWrappersCppTest, New) {
+TEST(ScudoWrappersCppDeathTest, New) {
+ if (getenv("SKIP_TYPE_MISMATCH")) {
+ printf("Skipped type mismatch tests.\n");
+ return;
+ }
testCxxNew<bool>();
testCxxNew<uint8_t>();
testCxxNew<uint16_t>();
static std::mutex Mutex;
static std::condition_variable Cv;
-static bool Ready = false;
+static bool Ready;
static void stressNew() {
std::vector<uintptr_t *> V;
}
TEST(ScudoWrappersCppTest, ThreadedNew) {
+ // TODO: Investigate why libc sometimes crashes with tag missmatch in
+ // __pthread_clockjoin_ex.
+ std::unique_ptr<scudo::ScopedDisableMemoryTagChecks> NoTags;
+ if (!SCUDO_ANDROID && scudo::archSupportsMemoryTagging() &&
+ scudo::systemSupportsMemoryTagging())
+ NoTags = std::make_unique<scudo::ScopedDisableMemoryTagChecks>();
+
+ Ready = false;
std::thread Threads[32];
for (size_t I = 0U; I < sizeof(Threads) / sizeof(Threads[0]); I++)
Threads[I] = std::thread(stressNew);
};
size_t measureWastage(const std::vector<Alloc> &allocs,
- const std::vector<size_t> &classes,
- size_t pageSize,
- size_t headerSize) {
+ const std::vector<size_t> &classes, size_t pageSize,
+ size_t headerSize) {
size_t totalWastage = 0;
for (auto &a : allocs) {
size_t sizePlusHeader = a.size + headerSize;
}
Alloc a;
- while (fscanf(f, "<alloc size=\"%zu\" count=\"%zu\"/>\n", &a.size, &a.count) == 2)
+ while (fscanf(f, "<alloc size=\"%zu\" count=\"%zu\"/>\n", &a.size,
+ &a.count) == 2)
allocs.push_back(a);
fclose(f);
}
};
static const uptr SizeDelta = %zu;
};
-)", headerSize);
+)",
+ headerSize);
}
--- /dev/null
+//===-- trusty.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+#if SCUDO_TRUSTY
+
+#include "common.h"
+#include "mutex.h"
+#include "string_utils.h"
+#include "trusty.h"
+
+#include <errno.h> // for errno
+#include <stdio.h> // for printf()
+#include <stdlib.h> // for getenv()
+#include <sys/auxv.h> // for getauxval()
+#include <time.h> // for clock_gettime()
+#include <trusty_syscalls.h> // for _trusty_brk()
+
+#define SBRK_ALIGN 32
+
+namespace scudo {
+
+uptr getPageSize() { return getauxval(AT_PAGESZ); }
+
+void NORETURN die() { abort(); }
+
+void *map(UNUSED void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
+ UNUSED MapPlatformData *Data) {
+ // Calling _trusty_brk(0) returns the current program break.
+ uptr ProgramBreak = reinterpret_cast<uptr>(_trusty_brk(0));
+ uptr Start;
+ uptr End;
+
+ Start = roundUpTo(ProgramBreak, SBRK_ALIGN);
+ // Don't actually extend the heap if MAP_NOACCESS flag is set since this is
+ // the case where Scudo tries to reserve a memory region without mapping
+ // physical pages.
+ if (Flags & MAP_NOACCESS)
+ return reinterpret_cast<void *>(Start);
+
+ // Attempt to extend the heap by Size bytes using _trusty_brk.
+ End = roundUpTo(Start + Size, SBRK_ALIGN);
+ ProgramBreak =
+ reinterpret_cast<uptr>(_trusty_brk(reinterpret_cast<void *>(End)));
+ if (ProgramBreak < End) {
+ errno = ENOMEM;
+ dieOnMapUnmapError(Size);
+ return nullptr;
+ }
+ return reinterpret_cast<void *>(Start); // Base of new reserved region.
+}
+
+// Unmap is a no-op since Trusty uses sbrk instead of memory mapping.
+void unmap(UNUSED void *Addr, UNUSED uptr Size, UNUSED uptr Flags,
+ UNUSED MapPlatformData *Data) {}
+
+void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags,
+ UNUSED MapPlatformData *Data) {}
+
+void releasePagesToOS(UNUSED uptr BaseAddress, UNUSED uptr Offset,
+ UNUSED uptr Size, UNUSED MapPlatformData *Data) {}
+
+const char *getEnv(const char *Name) { return getenv(Name); }
+
+// All mutex operations are a no-op since Trusty doesn't currently support
+// threads.
+bool HybridMutex::tryLock() { return true; }
+
+void HybridMutex::lockSlow() {}
+
+void HybridMutex::unlock() {}
+
+u64 getMonotonicTime() {
+ timespec TS;
+ clock_gettime(CLOCK_MONOTONIC, &TS);
+ return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) +
+ static_cast<u64>(TS.tv_nsec);
+}
+
+u32 getNumberOfCPUs() { return 0; }
+
+u32 getThreadID() { return 0; }
+
+bool getRandom(UNUSED void *Buffer, UNUSED uptr Length, UNUSED bool Blocking) {
+ return false;
+}
+
+void outputRaw(const char *Buffer) { printf("%s", Buffer); }
+
+void setAbortMessage(UNUSED const char *Message) {}
+
+} // namespace scudo
+
+#endif // SCUDO_TRUSTY
--- /dev/null
+//===-- trusty.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TRUSTY_H_
+#define SCUDO_TRUSTY_H_
+
+#include "platform.h"
+
+#if SCUDO_TRUSTY
+
+namespace scudo {
+// MapPlatformData is unused on Trusty, define it as a minimially sized
+// structure.
+struct MapPlatformData {};
+} // namespace scudo
+
+#endif // SCUDO_TRUSTY
+
+#endif // SCUDO_TRUSTY_H_
template <class Allocator> struct alignas(SCUDO_CACHE_LINE_SIZE) TSD {
typename Allocator::CacheT Cache;
typename Allocator::QuarantineCacheT QuarantineCache;
- u8 DestructorIterations;
+ using ThisT = TSD<Allocator>;
+ u8 DestructorIterations = 0;
- void initLinkerInitialized(Allocator *Instance) {
+ void init(Allocator *Instance) {
+ DCHECK_EQ(DestructorIterations, 0U);
+ DCHECK(isAligned(reinterpret_cast<uptr>(this), alignof(ThisT)));
Instance->initCache(&Cache);
DestructorIterations = PTHREAD_DESTRUCTOR_ITERATIONS;
}
- void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
- }
void commitBack(Allocator *Instance) { Instance->commitBack(this); }
private:
HybridMutex Mutex;
- atomic_uptr Precedence;
+ atomic_uptr Precedence = {};
};
} // namespace scudo
namespace scudo {
-enum class ThreadState : u8 {
- NotInitialized = 0,
- Initialized,
- TornDown,
+struct ThreadState {
+ bool DisableMemInit : 1;
+ enum {
+ NotInitialized = 0,
+ Initialized,
+ TornDown,
+ } InitState : 2;
};
template <class Allocator> void teardownThread(void *Ptr);
template <class Allocator> struct TSDRegistryExT {
- void initLinkerInitialized(Allocator *Instance) {
- Instance->initLinkerInitialized();
+ void init(Allocator *Instance) {
+ DCHECK(!Initialized);
+ Instance->init();
CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
- FallbackTSD.initLinkerInitialized(Instance);
+ FallbackTSD.init(Instance);
Initialized = true;
}
- void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
+
+ void initOnceMaybe(Allocator *Instance) {
+ ScopedLock L(Mutex);
+ if (LIKELY(Initialized))
+ return;
+ init(Instance); // Sets Initialized.
}
- void unmapTestOnly() {}
+ void unmapTestOnly(Allocator *Instance) {
+ DCHECK(Instance);
+ if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) {
+ DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)),
+ Instance);
+ ThreadTSD.commitBack(Instance);
+ ThreadTSD = {};
+ }
+ CHECK_EQ(pthread_key_delete(PThreadKey), 0);
+ PThreadKey = {};
+ FallbackTSD.commitBack(Instance);
+ FallbackTSD = {};
+ State = {};
+ Initialized = false;
+ }
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
- if (LIKELY(State != ThreadState::NotInitialized))
+ if (LIKELY(State.InitState != ThreadState::NotInitialized))
return;
initThread(Instance, MinimalInit);
}
ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) {
- if (LIKELY(State == ThreadState::Initialized &&
+ if (LIKELY(State.InitState == ThreadState::Initialized &&
!atomic_load(&Disabled, memory_order_acquire))) {
*UnlockRequired = false;
return &ThreadTSD;
Mutex.unlock();
}
-private:
- void initOnceMaybe(Allocator *Instance) {
- ScopedLock L(Mutex);
- if (LIKELY(Initialized))
- return;
- initLinkerInitialized(Instance); // Sets Initialized.
+ bool setOption(Option O, UNUSED sptr Value) {
+ if (O == Option::ThreadDisableMemInit)
+ State.DisableMemInit = Value;
+ if (O == Option::MaxTSDsCount)
+ return false;
+ return true;
}
+ bool getDisableMemInit() { return State.DisableMemInit; }
+
+private:
// Using minimal initialization allows for global initialization while keeping
// the thread specific structure untouched. The fallback structure will be
// used instead.
return;
CHECK_EQ(
pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
- ThreadTSD.initLinkerInitialized(Instance);
- State = ThreadState::Initialized;
+ ThreadTSD.init(Instance);
+ State.InitState = ThreadState::Initialized;
Instance->callPostInitCallback();
}
- pthread_key_t PThreadKey;
- bool Initialized;
- atomic_u8 Disabled;
+ pthread_key_t PThreadKey = {};
+ bool Initialized = false;
+ atomic_u8 Disabled = {};
TSD<Allocator> FallbackTSD;
HybridMutex Mutex;
- static THREADLOCAL ThreadState State;
- static
THREADLOCAL TSD<Allocator> ThreadTSD;
+ static thread_local ThreadState State;
+ static
thread_local TSD<Allocator> ThreadTSD;
friend void teardownThread<Allocator>(void *Ptr);
};
template <class Allocator>
-
THREADLOCAL TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
+
thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
template <class Allocator>
-
THREADLOCAL ThreadState TSDRegistryExT<Allocator>::State;
+
thread_local ThreadState TSDRegistryExT<Allocator>::State;
template <class Allocator> void teardownThread(void *Ptr) {
typedef TSDRegistryExT<Allocator> TSDRegistryT;
return;
}
TSDRegistryT::ThreadTSD.commitBack(Instance);
- TSDRegistryT::State = ThreadState::TornDown;
+ TSDRegistryT::State.InitState = ThreadState::TornDown;
}
} // namespace scudo
#ifndef SCUDO_TSD_SHARED_H_
#define SCUDO_TSD_SHARED_H_
-#include "linux.h" // for getAndroidTlsPtr()
#include "tsd.h"
+#if SCUDO_HAS_PLATFORM_TLS_SLOT
+// This is a platform-provided header that needs to be on the include path when
+// Scudo is compiled. It must declare a function with the prototype:
+// uintptr_t *getPlatformAllocatorTlsSlot()
+// that returns the address of a thread-local word of storage reserved for
+// Scudo, that must be zero-initialized in newly created threads.
+#include "scudo_platform_tls_slot.h"
+#endif
+
namespace scudo {
-template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
- void initLinkerInitialized(Allocator *Instance) {
- Instance->initLinkerInitialized();
- CHECK_EQ(pthread_key_create(&PThreadKey, nullptr), 0); // For non-TLS
+template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
+struct TSDRegistrySharedT {
+ void init(Allocator *Instance) {
+ DCHECK(!Initialized);
+ Instance->init();
+ for (u32 I = 0; I < TSDsArraySize; I++)
+ TSDs[I].init(Instance);
const u32 NumberOfCPUs = getNumberOfCPUs();
- NumberOfTSDs = (SCUDO_ANDROID || NumberOfCPUs == 0)
- ? MaxTSDCount
- : Min(NumberOfCPUs, MaxTSDCount);
- for (u32 I = 0; I < NumberOfTSDs; I++)
- TSDs[I].initLinkerInitialized(Instance);
- // Compute all the coprimes of NumberOfTSDs. This will be used to walk the
- // array of TSDs in a random order. For details, see:
- // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
- for (u32 I = 0; I < NumberOfTSDs; I++) {
- u32 A = I + 1;
- u32 B = NumberOfTSDs;
- // Find the GCD between I + 1 and NumberOfTSDs. If 1, they are coprimes.
- while (B != 0) {
- const u32 T = A;
- A = B;
- B = T % B;
- }
- if (A == 1)
- CoPrimes[NumberOfCoPrimes++] = I + 1;
- }
+ setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount
+ : Min(NumberOfCPUs, DefaultTSDCount));
Initialized = true;
}
- void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
+
+ void initOnceMaybe(Allocator *Instance) {
+ ScopedLock L(Mutex);
+ if (LIKELY(Initialized))
+ return;
+ init(Instance); // Sets Initialized.
}
- void unmapTestOnly() {
+ void unmapTestOnly(Allocator *Instance) {
+ for (u32 I = 0; I < TSDsArraySize; I++) {
+ TSDs[I].commitBack(Instance);
+ TSDs[I] = {};
+ }
setCurrentTSD(nullptr);
- pthread_key_delete(PThreadKey);
+ Initialized = false;
}
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,
if (TSD->tryLock())
return TSD;
// If that fails, go down the slow path.
+ if (TSDsArraySize == 1U) {
+ // Only 1 TSD, not need to go any further.
+ // The compiler will optimize this one way or the other.
+ TSD->lock();
+ return TSD;
+ }
return getTSDAndLockSlow(TSD);
}
void disable() {
Mutex.lock();
- for (u32 I = 0; I < NumberOfTSDs; I++)
+ for (u32 I = 0; I < TSDsArraySize; I++)
TSDs[I].lock();
}
void enable() {
- for (s32 I = static_cast<s32>(NumberOfTSDs - 1); I >= 0; I--)
+ for (s32 I = static_cast<s32>(TSDsArraySize - 1); I >= 0; I--)
TSDs[I].unlock();
Mutex.unlock();
}
+ bool setOption(Option O, sptr Value) {
+ if (O == Option::MaxTSDsCount)
+ return setNumberOfTSDs(static_cast<u32>(Value));
+ if (O == Option::ThreadDisableMemInit)
+ setDisableMemInit(Value);
+ // Not supported by the TSD Registry, but not an error either.
+ return true;
+ }
+
+ bool getDisableMemInit() const { return *getTlsPtr() & 1; }
+
private:
- ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
-#if _BIONIC
- *getAndroidTlsPtr() = reinterpret_cast<uptr>(CurrentTSD);
-#elif SCUDO_LINUX
- ThreadTSD = CurrentTSD;
+ ALWAYS_INLINE uptr *getTlsPtr() const {
+#if SCUDO_HAS_PLATFORM_TLS_SLOT
+ return reinterpret_cast<uptr *>(getPlatformAllocatorTlsSlot());
#else
- CHECK_EQ(
- pthread_setspecific(PThreadKey, reinterpret_cast<void *>(CurrentTSD)),
- 0);
+ static thread_local uptr ThreadTSD;
+ return &ThreadTSD;
#endif
}
+ static_assert(alignof(TSD<Allocator>) >= 2, "");
+
+ ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
+ *getTlsPtr() &= 1;
+ *getTlsPtr() |= reinterpret_cast<uptr>(CurrentTSD);
+ }
+
ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() {
-#if _BIONIC
- return reinterpret_cast<TSD<Allocator> *>(*getAndroidTlsPtr());
-#elif SCUDO_LINUX
- return ThreadTSD;
-#else
- return reinterpret_cast<TSD<Allocator> *>(pthread_getspecific(PThreadKey));
-#endif
+ return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr() & ~1ULL);
}
- void initOnceMaybe(Allocator *Instance) {
- ScopedLock L(Mutex);
- if (LIKELY(Initialized))
- return;
- initLinkerInitialized(Instance); // Sets Initialized.
+ bool setNumberOfTSDs(u32 N) {
+ ScopedLock L(MutexTSDs);
+ if (N < NumberOfTSDs)
+ return false;
+ if (N > TSDsArraySize)
+ N = TSDsArraySize;
+ NumberOfTSDs = N;
+ NumberOfCoPrimes = 0;
+ // Compute all the coprimes of NumberOfTSDs. This will be used to walk the
+ // array of TSDs in a random order. For details, see:
+ // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
+ for (u32 I = 0; I < N; I++) {
+ u32 A = I + 1;
+ u32 B = N;
+ // Find the GCD between I + 1 and N. If 1, they are coprimes.
+ while (B != 0) {
+ const u32 T = A;
+ A = B;
+ B = T % B;
+ }
+ if (A == 1)
+ CoPrimes[NumberOfCoPrimes++] = I + 1;
+ }
+ return true;
+ }
+
+ void setDisableMemInit(bool B) {
+ *getTlsPtr() &= ~1ULL;
+ *getTlsPtr() |= B;
}
NOINLINE void initThread(Allocator *Instance) {
}
NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD) {
- if (MaxTSDCount > 1U && NumberOfTSDs > 1U) {
- // Use the Precedence of the current TSD as our random seed. Since we are
- // in the slow path, it means that tryLock failed, and as a result it's
- // very likely that said Precedence is non-zero.
- const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
- const u32 Inc = CoPrimes[R % NumberOfCoPrimes];
- u32 Index = R % NumberOfTSDs;
+ // Use the Precedence of the current TSD as our random seed. Since we are
+ // in the slow path, it means that tryLock failed, and as a result it's
+ // very likely that said Precedence is non-zero.
+ const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
+ u32 N, Inc;
+ {
+ ScopedLock L(MutexTSDs);
+ N = NumberOfTSDs;
+ DCHECK_NE(NumberOfCoPrimes, 0U);
+ Inc = CoPrimes[R % NumberOfCoPrimes];
+ }
+ if (N > 1U) {
+ u32 Index = R % N;
uptr LowestPrecedence = UINTPTR_MAX;
TSD<Allocator> *CandidateTSD = nullptr;
// Go randomly through at most 4 contexts and find a candidate.
- for (u32 I = 0; I < Min(4U, NumberOfTSDs); I++) {
+ for (u32 I = 0; I < Min(4U, N); I++) {
if (TSDs[Index].tryLock()) {
setCurrentTSD(&TSDs[Index]);
return &TSDs[Index];
LowestPrecedence = Precedence;
}
Index += Inc;
- if (Index >= NumberOfTSDs)
- Index -= NumberOfTSDs;
+ if (Index >= N)
+ Index -= N;
}
if (CandidateTSD) {
CandidateTSD->lock();
return CurrentTSD;
}
- pthread_key_t PThreadKey;
- atomic_u32 CurrentIndex;
- u32 NumberOfTSDs;
- u32 NumberOfCoPrimes;
- u32 CoPrimes[MaxTSDCount];
- bool Initialized;
+ atomic_u32 CurrentIndex = {};
+ u32 NumberOfTSDs = 0;
+ u32 NumberOfCoPrimes = 0;
+ u32 CoPrimes[TSDsArraySize] = {};
+ bool Initialized = false;
HybridMutex Mutex;
- TSD<Allocator> TSDs[MaxTSDCount];
-#if SCUDO_LINUX && !_BIONIC
- static THREADLOCAL TSD<Allocator> *ThreadTSD;
-#endif
+ HybridMutex MutexTSDs;
+ TSD<Allocator> TSDs[TSDsArraySize];
};
-#if SCUDO_LINUX && !_BIONIC
-template <class Allocator, u32 MaxTSDCount>
-THREADLOCAL TSD<Allocator>
- *TSDRegistrySharedT<Allocator, MaxTSDCount>::ThreadTSD;
-#endif
-
} // namespace scudo
#endif // SCUDO_TSD_SHARED_H_
// small vectors. The current implementation supports only POD types.
template <typename T> class VectorNoCtor {
public:
- void init(uptr InitialCapacity) {
- CapacityBytes = 0;
- Size = 0;
- Data = nullptr;
+ void init(uptr InitialCapacity = 0) {
+ Data = reinterpret_cast<T *>(&LocalData[0]);
+ CapacityBytes = sizeof(LocalData);
reserve(InitialCapacity);
}
void destroy() {
- if (Data)
+ if (Data != reinterpret_cast<T *>(&LocalData[0]))
unmap(Data, CapacityBytes);
}
T &operator[](uptr I) {
void reallocate(uptr NewCapacity) {
DCHECK_GT(NewCapacity, 0);
DCHECK_LE(Size, NewCapacity);
- const uptr NewCapacityBytes =
- roundUpTo(NewCapacity * sizeof(T), getPageSizeCached());
+ NewCapacity = roundUpTo(NewCapacity * sizeof(T), getPageSizeCached());
T *NewData =
- reinterpret_cast<T *>(map(nullptr, NewCapacityBytes, "scudo:vector"));
- if (Data) {
- memcpy(NewData, Data, Size * sizeof(T));
- unmap(Data, CapacityBytes);
- }
+ reinterpret_cast<T *>(map(nullptr, NewCapacity, "scudo:vector"));
+ memcpy(NewData, Data, Size * sizeof(T));
+ destroy();
Data = NewData;
- CapacityBytes = NewCapacityBytes;
+ CapacityBytes = NewCapacity;
}
- T *Data;
- uptr CapacityBytes;
- uptr Size;
+ T *Data = nullptr;
+ u8 LocalData[256] = {};
+ uptr CapacityBytes = 0;
+ uptr Size = 0;
};
template <typename T> class Vector : public VectorNoCtor<T> {
public:
- Vector() { VectorNoCtor<T>::init(1); }
+ Vector() { VectorNoCtor<T>::init(); }
explicit Vector(uptr Count) {
VectorNoCtor<T>::init(Count);
this->resize(Count);
// Export the static allocator so that the C++ wrappers can access it.
// Technically we could have a completely separated heap for C & C++ but in
// reality the amount of cross pollination between the two is staggering.
+SCUDO_REQUIRE_CONSTANT_INITIALIZATION
scudo::Allocator<scudo::Config, SCUDO_PREFIX(malloc_postinit)> SCUDO_ALLOCATOR;
#include "wrappers_c.inc"
#define SCUDO_MALLINFO __scudo_mallinfo
#endif
-#ifndef M_DECAY_TIME
-#define M_DECAY_TIME -100
-#endif
-
-#ifndef M_PURGE
-#define M_PURGE -101
-#endif
-
#endif // SCUDO_WRAPPERS_C_H_
SCUDO_PREFIX(malloc_enable));
}
-INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, UNUSED int value) {
+INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {
if (param == M_DECAY_TIME) {
if (SCUDO_ANDROID) {
if (value == 0) {
} else if (param == M_PURGE) {
SCUDO_ALLOCATOR.releaseToOS();
return 1;
+ } else {
+ scudo::Option option;
+ switch (param) {
+ case M_MEMTAG_TUNING:
+ option = scudo::Option::MemtagTuning;
+ break;
+ case M_THREAD_DISABLE_MEM_INIT:
+ option = scudo::Option::ThreadDisableMemInit;
+ break;
+ case M_CACHE_COUNT_MAX:
+ option = scudo::Option::MaxCacheEntriesCount;
+ break;
+ case M_CACHE_SIZE_MAX:
+ option = scudo::Option::MaxCacheEntrySize;
+ break;
+ case M_TSDS_COUNT_MAX:
+ option = scudo::Option::MaxTSDsCount;
+ break;
+ default:
+ return 0;
+ }
+ return SCUDO_ALLOCATOR.setOption(option, static_cast<scudo::sptr>(value));
}
- return 0;
}
INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,
// Disable memory tagging for the heap. The caller must disable memory tag
// checks globally (e.g. by clearing TCF0 on aarch64) before calling this
-// function, and may not re-enable them after calling the function. The program
-// must be single threaded at the point when the function is called.
+// function, and may not re-enable them after calling the function.
INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {
SCUDO_ALLOCATOR.disableMemoryTagging();
}
// Sets whether scudo records stack traces and other metadata for allocations
// and deallocations. This function only has an effect if the allocator and
-// hardware support memory tagging. The program must be single threaded at the
-// point when the function is called.
+// hardware support memory tagging.
INTERFACE WEAK void
SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {
SCUDO_ALLOCATOR.setTrackAllocationStacks(track);
}
-// Sets whether scudo zero-initializes all allocated memory. The program must
-// be single threaded at the point when the function is called.
+// Sets whether scudo zero-initializes all allocated memory.
INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {
SCUDO_ALLOCATOR.setFillContents(zero_contents ? scudo::ZeroFill
: scudo::NoFill);
}
-// Sets whether scudo pattern-initializes all allocated memory. The program must
-// be single threaded at the point when the function is called.
+// Sets whether scudo pattern-initializes all allocated memory.
INTERFACE WEAK void
SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {
SCUDO_ALLOCATOR.setFillContents(
pattern_fill_contents ? scudo::PatternOrZeroFill : scudo::NoFill);
}
+// Sets whether scudo adds a small amount of slack at the end of large
+// allocations, before the guard page. This can be enabled to work around buggy
+// applications that read a few bytes past the end of their allocation.
+INTERFACE WEAK void
+SCUDO_PREFIX(malloc_set_add_large_allocation_slack)(int add_slack) {
+ SCUDO_ALLOCATOR.setAddLargeAllocationSlack(add_slack);
+}
+
} // extern "C"
#define SCUDO_ALLOCATOR Allocator
extern "C" void SCUDO_PREFIX(malloc_postinit)();
+SCUDO_REQUIRE_CONSTANT_INITIALIZATION
static scudo::Allocator<scudo::AndroidConfig, SCUDO_PREFIX(malloc_postinit)>
SCUDO_ALLOCATOR;
#define SCUDO_ALLOCATOR SvelteAllocator
extern "C" void SCUDO_PREFIX(malloc_postinit)();
+SCUDO_REQUIRE_CONSTANT_INITIALIZATION
static scudo::Allocator<scudo::AndroidSvelteConfig,
SCUDO_PREFIX(malloc_postinit)>
SCUDO_ALLOCATOR;
// TODO(kostyak): support both allocators.
INTERFACE void __scudo_print_stats(void) { Allocator.printStats(); }
-INTERFACE void __scudo_get_error_info(
- struct scudo_error_info *error_info, uintptr_t fault_addr,
- const char *stack_depot, const char *region_info, const char *memory,
- const char *memory_tags, uintptr_t memory_addr, size_t memory_size) {
+INTERFACE void
+__scudo_get_error_info(struct scudo_error_info *error_info,
+ uintptr_t fault_addr, const char *stack_depot,
+ const char *region_info, const char *ring_buffer,
+ const char *memory, const char *memory_tags,
+ uintptr_t memory_addr, size_t memory_size) {
Allocator.getErrorInfo(error_info, fault_addr, stack_depot, region_info,
- memory, memory_tags, memory_addr, memory_size);
+ ring_buffer, memory, memory_tags, memory_addr,
+ memory_size);
}
INTERFACE const char *__scudo_get_stack_depot_addr() {
return Allocator.getRegionInfoArraySize();
}
+INTERFACE const char *__scudo_get_ring_buffer_addr() {
+ return Allocator.getRingBufferAddress();
+}
+
+INTERFACE size_t __scudo_get_ring_buffer_size() {
+ return Allocator.getRingBufferSize();
+}
+
#endif // SCUDO_ANDROID && _BIONIC
BasedOnStyle: Google
AllowShortIfStatementsOnASingleLine: false
+IndentPPDirectives: AfterHash
if(COMPILER_RT_TSAN_DEBUG_OUTPUT)
# Add extra debug information to TSan runtime. This configuration is rarely
# used, but we need to support it so that debug output will not bitrot.
- list(APPEND TSAN_CFLAGS -DTSAN_COLLECT_STATS=1
- -DTSAN_DEBUG_OUTPUT=2)
+ list(APPEND TSAN_CFLAGS -DTSAN_DEBUG_OUTPUT=2)
endif()
set(TSAN_RTL_CFLAGS ${TSAN_CFLAGS})
rtl/tsan_malloc_mac.cpp
rtl/tsan_md5.cpp
rtl/tsan_mman.cpp
- rtl/tsan_mutex.cpp
rtl/tsan_mutexset.cpp
rtl/tsan_preinit.cpp
rtl/tsan_report.cpp
rtl/tsan_rtl_report.cpp
rtl/tsan_rtl_thread.cpp
rtl/tsan_stack_trace.cpp
- rtl/tsan_stat.cpp
rtl/tsan_suppressions.cpp
rtl/tsan_symbolize.cpp
rtl/tsan_sync.cpp
rtl/tsan_interface_inl.h
rtl/tsan_interface_java.h
rtl/tsan_mman.h
- rtl/tsan_mutex.h
rtl/tsan_mutexset.h
rtl/tsan_platform.h
rtl/tsan_ppc_regs.h
rtl/tsan_report.h
rtl/tsan_rtl.h
rtl/tsan_stack_trace.h
- rtl/tsan_stat.h
rtl/tsan_suppressions.h
rtl/tsan_symbolize.h
rtl/tsan_sync.h
rtl/tsan_trace.h
- rtl/tsan_update_shadow_word_inl.h)
+ rtl/tsan_update_shadow_word_inl.h
+ )
set(TSAN_RUNTIME_LIBRARIES)
add_compiler_rt_component(tsan)
+if("${CMAKE_C_FLAGS}" MATCHES "-Wno-(error=)?unused-command-line-argument")
+ set(EXTRA_CFLAGS "-Wno-error=unused-command-line-argument ${EXTRA_CFLAGS}")
+endif()
+
if(APPLE)
# Ideally we would check the SDK version for the actual platform we are
# building for here. To make our lifes easier we assume the host SDK setup is
message(FATAL_ERROR "Building the TSan runtime requires at least macOS SDK 10.12 (or aligned SDK on other platforms)")
endif()
- add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S rtl/tsan_rtl_aarch64.S)
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_amd64.S
+ rtl/tsan_rtl_aarch64.S
+ )
set(TSAN_LINK_LIBS ${SANITIZER_COMMON_LINK_LIBS})
set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
add_custom_target(GotsanRuntimeCheck
COMMAND env "CC=${CMAKE_C_COMPILER} ${OSX_SYSROOT_FLAG}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
DEPENDS tsan ${BUILDGO_SCRIPT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
else()
foreach(arch ${TSAN_SUPPORTED_ARCH})
if(arch STREQUAL "x86_64")
- add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S)
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_amd64.S
+ )
# Sanity check for Go runtime.
set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
add_custom_target(GotsanRuntimeCheck
COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
COMMENT "Checking TSan Go runtime..."
VERBATIM)
elseif(arch STREQUAL "aarch64")
- add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_aarch64.S)
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_aarch64.S
+ )
# Sanity check for Go runtime.
set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
add_custom_target(GotsanRuntimeCheck
COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
COMMENT "Checking TSan Go runtime..."
VERBATIM)
elseif(arch MATCHES "powerpc64|powerpc64le")
- add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_ppc64.S)
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_ppc64.S
+ )
# Sanity check for Go runtime.
set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
add_custom_target(GotsanRuntimeCheck
COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
COMMENT "Checking TSan Go runtime..."
VERBATIM)
elseif(arch MATCHES "mips64|mips64le")
- add_asm_sources(TSAN_ASM_SOURCES rtl/tsan_rtl_mips64.S)
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_mips64.S
+ )
+ elseif(arch MATCHES "s390x")
+ add_asm_sources(TSAN_ASM_SOURCES
+ rtl/tsan_rtl_s390x.S
+ )
+ # Sanity check for Go runtime.
+ set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh)
+ add_custom_target(GotsanRuntimeCheck
+ COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}"
+ EXTRA_CFLAGS=${EXTRA_CFLAGS}
+ IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT}
+ DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go
+ COMMENT "Checking TSan Go runtime..."
+ VERBATIM)
else()
set(TSAN_ASM_SOURCES)
endif()
endforeach()
endif()
-# Make sure that non-platform-specific files don't include any system headers.
-# FreeBSD/NetBSD do not install a number of Clang-provided headers for the
-# compiler in the base system due to incompatibilities between FreeBSD/NetBSD's
-# and Clang's versions. As a workaround do not use --sysroot=. on FreeBSD/NetBSD
-# until this is addressed.
-if(COMPILER_RT_HAS_SYSROOT_FLAG AND NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD"
- AND NOT CMAKE_SYSTEM_NAME MATCHES "NetBSD")
- file(GLOB _tsan_generic_sources rtl/tsan*)
- file(GLOB _tsan_platform_sources rtl/tsan*posix* rtl/tsan*mac*
- rtl/tsan*linux*)
- list(REMOVE_ITEM _tsan_generic_sources ${_tsan_platform_sources})
- set_source_files_properties(${_tsan_generic_sources}
- PROPERTIES COMPILE_FLAGS "--sysroot=.")
-endif()
-
# Build libcxx instrumented with TSan.
if(COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH AND
//
//===----------------------------------------------------------------------===//
+#include <pthread.h>
+
#include "dd_rtl.h"
#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_procmaps.h"
-#include <pthread.h>
-#include <stdlib.h>
using namespace __dsan;
uptr cond = atomic_load(p, memory_order_acquire);
if (!force && cond != 0)
return (pthread_cond_t*)cond;
- void *newcond = malloc(sizeof(pthread_cond_t));
+ void *newcond = InternalAlloc(sizeof(pthread_cond_t));
internal_memset(newcond, 0, sizeof(pthread_cond_t));
if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
memory_order_acq_rel))
return (pthread_cond_t*)newcond;
- free(newcond);
+ InternalFree(newcond);
return (pthread_cond_t*)cond;
}
InitThread();
pthread_cond_t *cond = init_cond(c);
int res = REAL(pthread_cond_destroy)(cond);
- free(cond);
+ InternalFree(cond);
atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed);
return res;
}
bool ignore_interceptors;
};
-struct Callback : DDCallback {
+struct Callback final : public DDCallback {
Thread *thr;
Callback(Thread *thr);
..\rtl\tsan_clock.cpp ^
..\rtl\tsan_flags.cpp ^
..\rtl\tsan_md5.cpp ^
- ..\rtl\tsan_mutex.cpp ^
..\rtl\tsan_report.cpp ^
..\rtl\tsan_rtl.cpp ^
..\rtl\tsan_rtl_mutex.cpp ^
..\rtl\tsan_rtl_report.cpp ^
..\rtl\tsan_rtl_thread.cpp ^
..\rtl\tsan_rtl_proc.cpp ^
- ..\rtl\tsan_stat.cpp ^
..\rtl\tsan_suppressions.cpp ^
..\rtl\tsan_sync.cpp ^
..\rtl\tsan_stack_trace.cpp ^
..\..\sanitizer_common\sanitizer_termination.cpp ^
..\..\sanitizer_common\sanitizer_file.cpp ^
..\..\sanitizer_common\sanitizer_symbolizer_report.cpp ^
+ ..\..\sanitizer_common\sanitizer_mutex.cpp ^
..\rtl\tsan_external.cpp ^
> gotsan.cpp
../rtl/tsan_flags.cpp
../rtl/tsan_interface_atomic.cpp
../rtl/tsan_md5.cpp
- ../rtl/tsan_mutex.cpp
../rtl/tsan_report.cpp
../rtl/tsan_rtl.cpp
../rtl/tsan_rtl_mutex.cpp
../rtl/tsan_rtl_thread.cpp
../rtl/tsan_rtl_proc.cpp
../rtl/tsan_stack_trace.cpp
- ../rtl/tsan_stat.cpp
../rtl/tsan_suppressions.cpp
../rtl/tsan_sync.cpp
../../sanitizer_common/sanitizer_allocator.cpp
../../sanitizer_common/sanitizer_flag_parser.cpp
../../sanitizer_common/sanitizer_flags.cpp
../../sanitizer_common/sanitizer_libc.cpp
+ ../../sanitizer_common/sanitizer_mutex.cpp
../../sanitizer_common/sanitizer_persistent_allocator.cpp
../../sanitizer_common/sanitizer_printf.cpp
../../sanitizer_common/sanitizer_suppressions.cpp
"
if [ "`uname -a | grep ppc64le`" != "" ]; then
SUFFIX="linux_ppc64le"
- ARCHCFLAGS="-m64"
+ ARCHCFLAGS="-m64 -mcpu=power8 -fno-function-sections"
elif [ "`uname -a | grep x86_64`" != "" ]; then
SUFFIX="linux_amd64"
- ARCHCFLAGS="-m64"
+ ARCHCFLAGS="-m64 -msse3"
OSCFLAGS="$OSCFLAGS -ffreestanding -Wno-unused-const-variable -Werror -Wno-unknown-warning-option"
elif [ "`uname -a | grep aarch64`" != "" ]; then
SUFFIX="linux_arm64"
ARCHCFLAGS=""
+ elif [ "`uname -a | grep -i mips64`" != "" ]; then
+ if [ "`lscpu | grep -i Little`" != "" ]; then
+ SUFFIX="linux_mips64le"
+ ARCHCFLAGS="-mips64 -EL"
+ else
+ SUFFIX="linux_mips64"
+ ARCHCFLAGS="-mips64 -EB"
+ fi
+ elif [ "`uname -a | grep s390x`" != "" ]; then
+ SRCS="$SRCS ../../sanitizer_common/sanitizer_linux_s390.cpp"
+ SUFFIX="linux_s390x"
+ ARCHCFLAGS=""
fi
elif [ "`uname -a | grep FreeBSD`" != "" ]; then
# The resulting object still depends on libc.
../../sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
../../sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp
"
-elif [ "`uname -a | grep OpenBSD`" != "" ]; then
- # The resulting object still depends on libc.
- # We removed this dependency for Go runtime for other OSes,
- # and we should remove it for OpenBSD as well, but there is no pressing need.
- DEPENDS_ON_LIBC=1
- SUFFIX="openbsd_amd64"
- OSCFLAGS="-fno-strict-aliasing -fPIC -Werror"
- ARCHCFLAGS="-m64"
- OSLDFLAGS="-pthread -fPIC -fpie"
- SRCS="
- $SRCS
- ../rtl/tsan_platform_linux.cpp
- ../../sanitizer_common/sanitizer_posix.cpp
- ../../sanitizer_common/sanitizer_posix_libcdep.cpp
- ../../sanitizer_common/sanitizer_procmaps_bsd.cpp
- ../../sanitizer_common/sanitizer_procmaps_common.cpp
- ../../sanitizer_common/sanitizer_linux.cpp
- ../../sanitizer_common/sanitizer_linux_libcdep.cpp
- ../../sanitizer_common/sanitizer_openbsd.cpp
- ../../sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
- "
elif [ "`uname -a | grep Darwin`" != "" ]; then
- SUFFIX="darwin_amd64"
OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -mmacosx-version-min=10.7"
- ARCHCFLAGS="-m64"
OSLDFLAGS="-lpthread -fPIC -fpie -mmacosx-version-min=10.7"
SRCS="
$SRCS
../rtl/tsan_platform_mac.cpp
../../sanitizer_common/sanitizer_mac.cpp
+ ../../sanitizer_common/sanitizer_mac_libcdep.cpp
../../sanitizer_common/sanitizer_posix.cpp
../../sanitizer_common/sanitizer_posix_libcdep.cpp
../../sanitizer_common/sanitizer_procmaps_mac.cpp
"
+ if [ "`uname -a | grep x86_64`" != "" ]; then
+ SUFFIX="darwin_amd64"
+ ARCHCFLAGS="-m64"
+ elif [ "`uname -a | grep arm64`" != "" ]; then
+ SUFFIX="darwin_arm64"
+ ARCHCFLAGS=""
+ fi
elif [ "`uname -a | grep MINGW`" != "" ]; then
SUFFIX="windows_amd64"
OSCFLAGS="-Wno-error=attributes -Wno-attributes -Wno-unused-const-variable -Wno-unknown-warning-option"
fi
SRCS="$SRCS $ADD_SRCS"
-
-rm -f $DIR/gotsan.cpp
for F in $SRCS; do
- cat $F >> $DIR/gotsan.cpp
-done
+ echo "#line 1 \"$F\""
+ cat $F
+done > $DIR/gotsan.cpp
-FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++14 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS"
+FLAGS=" -I../rtl -I../.. -I../../sanitizer_common -I../../../include -std=c++14 -Wall -fno-exceptions -fno-rtti -DSANITIZER_GO=1 -DSANITIZER_DEADLOCK_DETECTOR_VERSION=2 $OSCFLAGS $ARCHCFLAGS $EXTRA_CFLAGS"
DEBUG_FLAGS="$FLAGS -DSANITIZER_DEBUG=1 -g"
FLAGS="$FLAGS -DSANITIZER_DEBUG=0 -O3 -fomit-frame-pointer"
-if [ "$SUFFIX" = "linux_ppc64le" ]; then
- FLAGS="$FLAGS -mcpu=power8 -fno-function-sections"
-elif [ "$SUFFIX" = "linux_amd64" ]; then
- FLAGS="$FLAGS -msse3"
-fi
if [ "$DEBUG" = "" ]; then
# Do a build test with debug flags.
// release-store operation by the thread with release_store_tid_ index.
// release_store_reused_ - reuse count of release_store_tid_.
-// We don't have ThreadState in these methods, so this is an ugly hack that
-// works only in C++.
-#if !SANITIZER_GO
-# define CPP_STAT_INC(typ) StatInc(cur_thread(), typ)
-#else
-# define CPP_STAT_INC(typ) (void)0
-#endif
-
namespace __tsan {
static atomic_uint32_t *ref_ptr(ClockBlock *cb) {
void ThreadClock::acquire(ClockCache *c, SyncClock *src) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(src->size_, kMaxTid);
- CPP_STAT_INC(StatClockAcquire);
// Check if it's empty -> no need to do anything.
const uptr nclk = src->size_;
- if (nclk == 0) {
- CPP_STAT_INC(StatClockAcquireEmpty);
+ if (nclk == 0)
return;
- }
bool acquired = false;
for (unsigned i = 0; i < kDirtyTids; i++) {
SyncClock::Dirty dirty = src->dirty_[i];
- unsigned tid = dirty.tid;
+ unsigned tid = dirty.tid();
if (tid != kInvalidTid) {
if (clk_[tid] < dirty.epoch) {
clk_[tid] = dirty.epoch;
// Check if we've already acquired src after the last release operation on src
if (tid_ >= nclk || src->elem(tid_).reused != reused_) {
// O(N) acquire.
- CPP_STAT_INC(StatClockAcquireFull);
nclk_ = max(nclk_, nclk);
u64 *dst_pos = &clk_[0];
for (ClockElem &src_elem : *src) {
}
if (acquired) {
- CPP_STAT_INC(StatClockAcquiredSomething);
last_acquire_ = clk_[tid_];
ResetCached(c);
}
sc->release_store_reused_ = 0;
if (acquired) {
- CPP_STAT_INC(StatClockAcquiredSomething);
last_acquire_ = clk_[tid_];
ResetCached(c);
}
return;
}
- CPP_STAT_INC(StatClockRelease);
// Check if we need to resize dst.
if (dst->size_ < nclk_)
dst->Resize(c, nclk_);
}
// O(N) release.
- CPP_STAT_INC(StatClockReleaseFull);
dst->Unshare(c);
// First, remember whether we've acquired dst.
bool acquired = IsAlreadyAcquired(dst);
- if (acquired)
- CPP_STAT_INC(StatClockReleaseAcquired);
// Update dst->clk_.
dst->FlushDirty();
uptr i = 0;
i++;
}
// Clear 'acquired' flag in the remaining elements.
- if (nclk_ < dst->size_)
- CPP_STAT_INC(StatClockReleaseClearTail);
dst->release_store_tid_ = kInvalidTid;
dst->release_store_reused_ = 0;
// If we've acquired dst, remember this fact,
void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(dst->size_, kMaxTid);
- CPP_STAT_INC(StatClockStore);
if (dst->size_ == 0 && cached_idx_ != 0) {
// Reuse the cached clock.
dst->tab_idx_ = cached_idx_;
dst->size_ = cached_size_;
dst->blocks_ = cached_blocks_;
- CHECK_EQ(dst->dirty_[0].tid, kInvalidTid);
+ CHECK_EQ(dst->dirty_[0].tid(), kInvalidTid);
// The cached clock is shared (immutable),
// so this is where we store the current clock.
- dst->dirty_[0].tid = tid_;
+ dst->dirty_[0].set_tid(tid_);
dst->dirty_[0].epoch = clk_[tid_];
dst->release_store_tid_ = tid_;
dst->release_store_reused_ = reused_;
if (dst->release_store_tid_ == tid_ &&
dst->release_store_reused_ == reused_ &&
!HasAcquiredAfterRelease(dst)) {
- CPP_STAT_INC(StatClockStoreFast);
UpdateCurrentThread(c, dst);
return;
}
// O(N) release-store.
- CPP_STAT_INC(StatClockStoreFull);
dst->Unshare(c);
// Note: dst can be larger than this ThreadClock.
// This is fine since clk_ beyond size is all zeros.
ce.reused = 0;
i++;
}
- for (uptr i = 0; i < kDirtyTids; i++)
- dst->dirty_[i].tid = kInvalidTid;
+ for (uptr i = 0; i < kDirtyTids; i++) dst->dirty_[i].set_tid(kInvalidTid);
dst->release_store_tid_ = tid_;
dst->release_store_reused_ = reused_;
// Rememeber that we don't need to acquire it in future.
}
void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) {
- CPP_STAT_INC(StatClockAcquireRelease);
acquire(c, dst);
ReleaseStore(c, dst);
}
// Update the threads time, but preserve 'acquired' flag.
for (unsigned i = 0; i < kDirtyTids; i++) {
SyncClock::Dirty *dirty = &dst->dirty_[i];
- const unsigned tid = dirty->tid;
+ const unsigned tid = dirty->tid();
if (tid == tid_ || tid == kInvalidTid) {
- CPP_STAT_INC(StatClockReleaseFast);
- dirty->tid = tid_;
+ dirty->set_tid(tid_);
dirty->epoch = clk_[tid_];
return;
}
// Reset all 'acquired' flags, O(N).
// We are going to touch dst elements, so we need to unshare it.
dst->Unshare(c);
- CPP_STAT_INC(StatClockReleaseSlow);
dst->elem(tid_).epoch = clk_[tid_];
for (uptr i = 0; i < dst->size_; i++)
dst->elem(i).reused = 0;
return false;
for (unsigned i = 0; i < kDirtyTids; i++) {
SyncClock::Dirty dirty = src->dirty_[i];
- if (dirty.tid != kInvalidTid) {
- if (clk_[dirty.tid] < dirty.epoch)
+ if (dirty.tid() != kInvalidTid) {
+ if (clk_[dirty.tid()] < dirty.epoch)
return false;
}
}
blocks_ = 0;
release_store_tid_ = kInvalidTid;
release_store_reused_ = 0;
- for (uptr i = 0; i < kDirtyTids; i++)
- dirty_[i].tid = kInvalidTid;
+ for (uptr i = 0; i < kDirtyTids; i++) dirty_[i].set_tid(kInvalidTid);
}
void SyncClock::Resize(ClockCache *c, uptr nclk) {
- CPP_STAT_INC(StatClockReleaseResize);
Unshare(c);
if (nclk <= capacity()) {
// Memory is already allocated, just increase the size.
void SyncClock::FlushDirty() {
for (unsigned i = 0; i < kDirtyTids; i++) {
Dirty *dirty = &dirty_[i];
- if (dirty->tid != kInvalidTid) {
- CHECK_LT(dirty->tid, size_);
- elem(dirty->tid).epoch = dirty->epoch;
- dirty->tid = kInvalidTid;
+ if (dirty->tid() != kInvalidTid) {
+ CHECK_LT(dirty->tid(), size_);
+ elem(dirty->tid()).epoch = dirty->epoch;
+ dirty->set_tid(kInvalidTid);
}
}
}
if (size_ == 0)
return false;
for (unsigned i = 0; i < kDirtyTids; i++) {
- if (dirty_[i].tid != kInvalidTid)
+ if (dirty_[i].tid() != kInvalidTid)
return false;
}
return atomic_load_relaxed(ref_ptr(tab_)) == 1;
u64 SyncClock::get(unsigned tid) const {
for (unsigned i = 0; i < kDirtyTids; i++) {
Dirty dirty = dirty_[i];
- if (dirty.tid == tid)
+ if (dirty.tid() == tid)
return dirty.epoch;
}
return elem(tid).epoch;
for (uptr i = 0; i < size_; i++)
printf("%s%llu", i == 0 ? "" : ",", elem(i).reused);
printf("] release_store_tid=%d/%d dirty_tids=%d[%llu]/%d[%llu]",
- release_store_tid_, release_store_reused_,
- dirty_[0].tid, dirty_[0].epoch,
- dirty_[1].tid, dirty_[1].epoch);
+ release_store_tid_, release_store_reused_, dirty_[0].tid(),
+ dirty_[0].epoch, dirty_[1].tid(), dirty_[1].epoch);
}
void SyncClock::Iter::Next() {
namespace __tsan {
-typedef DenseSlabAlloc<ClockBlock, 1<<16, 1<<10> ClockAlloc;
+typedef DenseSlabAlloc<ClockBlock, 1 << 22, 1 << 10> ClockAlloc;
typedef DenseSlabAllocCache ClockCache;
// The clock that lives in sync variables (mutexes, atomics, etc).
static const uptr kDirtyTids = 2;
struct Dirty {
- u64 epoch : kClkBits;
- u64 tid : 64 - kClkBits; // kInvalidId if not active
+ u32 tid() const { return tid_ == kShortInvalidTid ? kInvalidTid : tid_; }
+ void set_tid(u32 tid) {
+ tid_ = tid == kInvalidTid ? kShortInvalidTid : tid;
+ }
+ u64 epoch : kClkBits;
+
+ private:
+ // Full kInvalidTid won't fit into Dirty::tid.
+ static const u64 kShortInvalidTid = (1ull << (64 - kClkBits)) - 1;
+ u64 tid_ : 64 - kClkBits; // kInvalidId if not active
};
+ static_assert(sizeof(Dirty) == 8, "Dirty is not 64bit");
+
unsigned release_store_tid_;
unsigned release_store_reused_;
Dirty dirty_[kDirtyTids];
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include "tsan_stat.h"
+#include "sanitizer_common/sanitizer_mutex.h"
#include "ubsan/ubsan_platform.h"
// Setup defaults for compile definitions.
# define TSAN_NO_HISTORY 0
#endif
-#ifndef TSAN_COLLECT_STATS
-# define TSAN_COLLECT_STATS 0
-#endif
-
#ifndef TSAN_CONTAINS_UBSAN
# if CAN_SANITIZE_UB && !SANITIZER_GO
# define TSAN_CONTAINS_UBSAN 1
const bool kCollectHistory = true;
#endif
-const u16 kInvalidTid = kMaxTid + 1;
-
// The following "build consistency" machinery ensures that all source files
// are built in the same configuration. Inconsistent builds lead to
// hard to debug crashes.
void build_consistency_release();
#endif
-#if TSAN_COLLECT_STATS
-void build_consistency_stats();
-#else
-void build_consistency_nostats();
-#endif
-
static inline void USED build_consistency() {
#if SANITIZER_DEBUG
build_consistency_debug();
#else
build_consistency_release();
#endif
-#if TSAN_COLLECT_STATS
- build_consistency_stats();
-#else
- build_consistency_nostats();
-#endif
}
template<typename T>
// as 16-bit values, see tsan_defs.h.
};
+enum MutexType {
+ MutexTypeTrace = MutexLastCommon,
+ MutexTypeReport,
+ MutexTypeSyncVar,
+ MutexTypeAnnotations,
+ MutexTypeAtExit,
+ MutexTypeFired,
+ MutexTypeRacy,
+ MutexTypeGlobalProc,
+};
+
} // namespace __tsan
#endif // TSAN_DEFS_H
#include "sanitizer_common/sanitizer_common.h"
#include "tsan_defs.h"
-#include "tsan_mutex.h"
namespace __tsan {
typedef u32 IndexT;
uptr pos;
IndexT cache[kSize];
- template<typename T, uptr kL1Size, uptr kL2Size> friend class DenseSlabAlloc;
+ template <typename, uptr, uptr, u64>
+ friend class DenseSlabAlloc;
};
-template<typename T, uptr kL1Size, uptr kL2Size>
+template <typename T, uptr kL1Size, uptr kL2Size, u64 kReserved = 0>
class DenseSlabAlloc {
public:
typedef DenseSlabAllocCache Cache;
typedef typename Cache::IndexT IndexT;
- explicit DenseSlabAlloc(const char *name) {
- // Check that kL1Size and kL2Size are sane.
- CHECK_EQ(kL1Size & (kL1Size - 1), 0);
- CHECK_EQ(kL2Size & (kL2Size - 1), 0);
- CHECK_GE(1ull << (sizeof(IndexT) * 8), kL1Size * kL2Size);
- // Check that it makes sense to use the dense alloc.
- CHECK_GE(sizeof(T), sizeof(IndexT));
- internal_memset(map_, 0, sizeof(map_));
+ static_assert((kL1Size & (kL1Size - 1)) == 0,
+ "kL1Size must be a power-of-two");
+ static_assert((kL2Size & (kL2Size - 1)) == 0,
+ "kL2Size must be a power-of-two");
+ static_assert((kL1Size * kL2Size) <= (1ull << (sizeof(IndexT) * 8)),
+ "kL1Size/kL2Size are too large");
+ static_assert(((kL1Size * kL2Size - 1) & kReserved) == 0,
+ "reserved bits don't fit");
+ static_assert(sizeof(T) > sizeof(IndexT),
+ "it doesn't make sense to use dense alloc");
+
+ explicit DenseSlabAlloc(LinkerInitialized, const char *name) {
freelist_ = 0;
fillpos_ = 0;
name_ = name;
}
+ explicit DenseSlabAlloc(const char *name)
+ : DenseSlabAlloc(LINKER_INITIALIZED, name) {
+ // It can be very large.
+ // Don't page it in for linker initialized objects.
+ internal_memset(map_, 0, sizeof(map_));
+ }
+
~DenseSlabAlloc() {
for (uptr i = 0; i < kL1Size; i++) {
if (map_[i] != 0)
#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap
#if __has_attribute(noescape)
- #define DISPATCH_NOESCAPE __attribute__((__noescape__))
+# define DISPATCH_NOESCAPE __attribute__((__noescape__))
#else
- #define DISPATCH_NOESCAPE
+# define DISPATCH_NOESCAPE
#endif
+#if SANITIZER_MAC
+# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
+#else
+# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
+#endif
+
+
// Data types used in dispatch APIs
typedef unsigned long size_t;
typedef unsigned long uintptr_t;
//===----------------------------------------------------------------------===//
#include "tsan_rtl.h"
#include "tsan_interceptors.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
namespace __tsan {
#if !SANITIZER_GO
typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
-void ExternalAccess(void *addr, void *caller_pc, void *tag, AccessFunc access) {
+void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessFunc access) {
CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
ThreadState *thr = cur_thread();
- if (caller_pc) FuncEntry(thr, (uptr)caller_pc);
+ if (caller_pc) FuncEntry(thr, caller_pc);
InsertShadowStackFrameForTag(thr, (uptr)tag);
bool in_ignored_lib;
- if (!caller_pc || !libignore()->IsIgnored((uptr)caller_pc, &in_ignored_lib)) {
+ if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib)) {
access(thr, CALLERPC, (uptr)addr, kSizeLog1);
}
FuncExit(thr);
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
- ExternalAccess(addr, caller_pc, tag, MemoryRead);
+ ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryRead);
}
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
- ExternalAccess(addr, caller_pc, tag, MemoryWrite);
+ ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryWrite);
}
} // extern "C"
// Let a frontend override.
parser.ParseString(__tsan_default_options());
#if TSAN_CONTAINS_UBSAN
- const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
+ const char *ubsan_default_options = __ubsan_default_options();
ubsan_parser.ParseString(ubsan_default_options);
#endif
// Override from command line.
LibIgnore *libignore();
#if !SANITIZER_GO
-INLINE bool in_symbolizer() {
+inline bool in_symbolizer() {
cur_thread_init();
return UNLIKELY(cur_thread()->in_symbolizer);
}
} // namespace __tsan
-#define SCOPED_INTERCEPTOR_RAW(func, ...) \
- cur_thread_init(); \
- ThreadState *thr = cur_thread(); \
- const uptr caller_pc = GET_CALLER_PC(); \
- ScopedInterceptor si(thr, #func, caller_pc); \
- const uptr pc = StackTrace::GetCurrentPc(); \
- (void)pc; \
-/**/
+#define SCOPED_INTERCEPTOR_RAW(func, ...) \
+ cur_thread_init(); \
+ ThreadState *thr = cur_thread(); \
+ const uptr caller_pc = GET_CALLER_PC(); \
+ ScopedInterceptor si(thr, #func, caller_pc); \
+ const uptr pc = GET_CURRENT_PC(); \
+ (void)pc; \
+ /**/
#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
#include "BlocksRuntime/Block.h"
#include "tsan_dispatch_defs.h"
+#if SANITIZER_MAC
+# include <Availability.h>
+#endif
+
namespace __tsan {
typedef u16 uint16_t;
DISPATCH_INTERCEPT(dispatch, false)
DISPATCH_INTERCEPT(dispatch_barrier, true)
+// dispatch_async_and_wait() and friends were introduced in macOS 10.14.
+// Linking of these interceptors fails when using an older SDK.
+#if !SANITIZER_MAC || defined(__MAC_10_14)
+// macOS 10.14 is greater than our minimal deployment target. To ensure we
+// generate a weak reference so the TSan dylib continues to work on older
+// systems, we need to forward declare the intercepted functions as "weak
+// imports". Note that this file is multi-platform, so we cannot include the
+// actual header file (#include <dispatch/dispatch.h>).
+SANITIZER_WEAK_IMPORT void dispatch_async_and_wait(
+ dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
+SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f(
+ dispatch_queue_t queue, void *context, dispatch_function_t work);
+SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait(
+ dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);
+SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f(
+ dispatch_queue_t queue, void *context, dispatch_function_t work);
+
+DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false)
+DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false)
+DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true)
+DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true)
+#endif
+
+
DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
dispatch_queue_t queue, void *context, dispatch_function_t work)
INTERCEPT_FUNCTION(dispatch_barrier_async_f);
INTERCEPT_FUNCTION(dispatch_barrier_sync);
INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
+ INTERCEPT_FUNCTION(dispatch_async_and_wait);
+ INTERCEPT_FUNCTION(dispatch_async_and_wait_f);
+ INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait);
+ INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f);
INTERCEPT_FUNCTION(dispatch_after);
INTERCEPT_FUNCTION(dispatch_after_f);
INTERCEPT_FUNCTION(dispatch_once);
// actually aliases of each other, and we cannot have different interceptors for
// them, because they're actually the same function. Thus, we have to stay
// conservative and treat the non-barrier versions as mo_acq_rel.
-static const morder kMacOrderBarrier = mo_acq_rel;
-static const morder kMacOrderNonBarrier = mo_acq_rel;
+static constexpr morder kMacOrderBarrier = mo_acq_rel;
+static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
+static constexpr morder kMacFailureOrder = mo_relaxed;
#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
- kMacOrderNonBarrier, kMacOrderNonBarrier); \
+ kMacOrderNonBarrier, kMacFailureOrder); \
} \
\
TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
return tsan_atomic_f##_compare_exchange_strong( \
(volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
- kMacOrderBarrier, kMacOrderNonBarrier); \
+ kMacOrderBarrier, kMacFailureOrder); \
}
OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
virtual void on_zero_shared() = 0;
virtual void _unused_0x18() = 0;
virtual void on_zero_shared_weak() = 0;
+ virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor
};
} // namespace
namespace __tsan {
-static bool intersects_with_shadow(mach_vm_address_t *address,
+static bool intersects_with_shadow(mach_vm_address_t address,
mach_vm_size_t size, int flags) {
// VM_FLAGS_FIXED is 0x0, so we have to test for VM_FLAGS_ANYWHERE.
if (flags & VM_FLAGS_ANYWHERE) return false;
- uptr ptr = *address;
- return !IsAppMem(ptr) || !IsAppMem(ptr + size - 1);
+ return !IsAppMem(address) || !IsAppMem(address + size - 1);
}
TSAN_INTERCEPTOR(kern_return_t, mach_vm_allocate, vm_map_t target,
SCOPED_TSAN_INTERCEPTOR(mach_vm_allocate, target, address, size, flags);
if (target != mach_task_self())
return REAL(mach_vm_allocate)(target, address, size, flags);
- if (intersects_with_shadow(address, size, flags))
+ if (address && intersects_with_shadow(*address, size, flags))
return KERN_NO_SPACE;
- kern_return_t res = REAL(mach_vm_allocate)(target, address, size, flags);
- if (res == KERN_SUCCESS)
+ kern_return_t kr = REAL(mach_vm_allocate)(target, address, size, flags);
+ if (kr == KERN_SUCCESS)
MemoryRangeImitateWriteOrResetRange(thr, pc, *address, size);
- return res;
+ return kr;
}
TSAN_INTERCEPTOR(kern_return_t, mach_vm_deallocate, vm_map_t target,
SCOPED_TSAN_INTERCEPTOR(mach_vm_deallocate, target, address, size);
if (target != mach_task_self())
return REAL(mach_vm_deallocate)(target, address, size);
- UnmapShadow(thr, address, size);
- return REAL(mach_vm_deallocate)(target, address, size);
+ kern_return_t kr = REAL(mach_vm_deallocate)(target, address, size);
+ if (kr == KERN_SUCCESS && address)
+ UnmapShadow(thr, address, size);
+ return kr;
}
} // namespace __tsan
#include "tsan_mman.h"
#include "tsan_fd.h"
+#include <stdarg.h>
+
using namespace __tsan;
#if SANITIZER_FREEBSD || SANITIZER_MAC
#define vfork __vfork14
#endif
-#if SANITIZER_ANDROID
-#define mallopt(a, b)
-#endif
-
#ifdef __mips__
const int kSigCount = 129;
#else
};
#endif
-#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1
+#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
+ defined(__s390x__)
#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
#elif defined(__aarch64__) || SANITIZER_PPC64V2
#define PTHREAD_ABI_BASE "GLIBC_2.17"
extern "C" int pthread_attr_destroy(void *attr);
DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
+extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void),
+ void (*child)(void));
extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
extern "C" int pthread_setspecific(unsigned key, const void *v);
DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *)
extern "C" int fileno_unlocked(void *stream);
extern "C" int dirfd(void *dirp);
#endif
-#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD
+#if SANITIZER_GLIBC
extern "C" int mallopt(int param, int value);
#endif
#if SANITIZER_NETBSD
#endif
const int MAP_FIXED = 0x10;
typedef long long_t;
+typedef __sanitizer::u16 mode_t;
// From /usr/include/unistd.h
# define F_ULOCK 0 /* Unlock a previously locked region. */
unsigned finalize_key;
#endif
- BlockingMutex atexit_mu;
+ Mutex atexit_mu;
Vector<struct AtExitCtx *> AtExitStack;
- InterceptorContext()
- : libignore(LINKER_INITIALIZED), AtExitStack() {
- }
+ InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {}
};
static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)];
if (!thr_->ignore_interceptors) {
ProcessPendingSignals(thr_);
FuncExit(thr_);
- CheckNoLocks(thr_);
+ CheckedMutex::CheckNoLocks();
}
}
AtExitCtx *ctx;
{
// Ensure thread-safety.
- BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+ Lock l(&interceptor_ctx()->atexit_mu);
// Pop AtExitCtx from the top of the stack of callback functions
uptr element = interceptor_ctx()->AtExitStack.Size() - 1;
// Store ctx in a local stack-like structure
// Ensure thread-safety.
- BlockingMutexLock l(&interceptor_ctx()->atexit_mu);
+ Lock l(&interceptor_ctx()->atexit_mu);
+ // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail
+ // due to atexit_mu held on exit from the calloc interceptor.
+ ScopedIgnoreInterceptors ignore;
res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0);
// Push AtExitCtx on the top of the stack of callback functions
return p;
}
+// In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept
+// __libc_memalign so that (1) we can detect races (2) free will not be called
+// on libc internally allocated blocks.
TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
- SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
+ SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz);
return user_memalign(thr, pc, align, sz);
}
if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED;
void *res = real_mmap(addr, sz, prot, flags, fd, off);
if (res != MAP_FAILED) {
+ if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) {
+ Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n",
+ addr, (void*)sz, res);
+ Die();
+ }
if (fd > 0) FdAccess(thr, pc, fd);
MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz);
}
return (void*)cond;
}
+namespace {
+
+template <class Fn>
struct CondMutexUnlockCtx {
ScopedInterceptor *si;
ThreadState *thr;
uptr pc;
void *m;
+ void *c;
+ const Fn &fn;
+
+ int Cancel() const { return fn(); }
+ void Unlock() const;
};
-static void cond_mutex_unlock(CondMutexUnlockCtx *arg) {
+template <class Fn>
+void CondMutexUnlockCtx<Fn>::Unlock() const {
// pthread_cond_wait interceptor has enabled async signal delivery
// (see BlockingCall below). Disable async signals since we are running
// tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
// since the thread is cancelled, so we have to manually execute them
// (the thread still can run some user code due to pthread_cleanup_push).
- ThreadSignalContext *ctx = SigCtx(arg->thr);
+ ThreadSignalContext *ctx = SigCtx(thr);
CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
- MutexPostLock(arg->thr, arg->pc, (uptr)arg->m, MutexFlagDoPreLockOnPostLock);
+ MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
// Undo BlockingCall ctor effects.
- arg->thr->ignore_interceptors--;
- arg->si->~ScopedInterceptor();
+ thr->ignore_interceptors--;
+ si->~ScopedInterceptor();
}
+} // namespace
INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
void *cond = init_cond(c, true);
return REAL(pthread_cond_init)(cond, a);
}
-static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si,
- int (*fn)(void *c, void *m, void *abstime), void *c,
- void *m, void *t) {
+template <class Fn>
+int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn,
+ void *c, void *m) {
MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false);
MutexUnlock(thr, pc, (uptr)m);
- CondMutexUnlockCtx arg = {si, thr, pc, m};
int res = 0;
// This ensures that we handle mutex lock even in case of pthread_cancel.
// See test/tsan/cond_cancel.cpp.
{
// Enable signal delivery while the thread is blocked.
BlockingCall bc(thr);
+ CondMutexUnlockCtx<Fn> arg = {si, thr, pc, m, c, fn};
res = call_pthread_cancel_with_cleanup(
- fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg);
+ [](void *arg) -> int {
+ return ((const CondMutexUnlockCtx<Fn> *)arg)->Cancel();
+ },
+ [](void *arg) { ((const CondMutexUnlockCtx<Fn> *)arg)->Unlock(); },
+ &arg);
}
if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m);
MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
void *cond = init_cond(c);
SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m);
- return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL(
- pthread_cond_wait),
- cond, m, 0);
+ return cond_wait(
+ thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond,
+ m);
}
INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
void *cond = init_cond(c);
SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime);
- return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m,
- abstime);
+ return cond_wait(
+ thr, pc, &si,
+ [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond,
+ m);
}
+#if SANITIZER_LINUX
+INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m,
+ __sanitizer_clockid_t clock, void *abstime) {
+ void *cond = init_cond(c);
+ SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime);
+ return cond_wait(
+ thr, pc, &si,
+ [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); },
+ cond, m);
+}
+#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait)
+#else
+#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
+#endif
+
#if SANITIZER_MAC
INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m,
void *reltime) {
void *cond = init_cond(c);
SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime);
- return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond,
- m, reltime);
+ return cond_wait(
+ thr, pc, &si,
+ [=]() {
+ return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime);
+ },
+ cond, m);
}
#endif
#define TSAN_MAYBE_INTERCEPT_FSTAT64
#endif
-TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
- SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
+TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
+ va_list ap;
+ va_start(ap, oflag);
+ mode_t mode = va_arg(ap, int);
+ va_end(ap);
+ SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
READ_STRING(thr, pc, name, 0);
- int fd = REAL(open)(name, flags, mode);
+ int fd = REAL(open)(name, oflag, mode);
if (fd >= 0)
FdFileCreate(thr, pc, fd);
return fd;
}
#if SANITIZER_LINUX
-TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
- SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
+TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) {
+ va_list ap;
+ va_start(ap, oflag);
+ mode_t mode = va_arg(ap, int);
+ va_end(ap);
+ SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode);
READ_STRING(thr, pc, name, 0);
- int fd = REAL(open64)(name, flags, mode);
+ int fd = REAL(open64)(name, oflag, mode);
if (fd >= 0)
FdFileCreate(thr, pc, fd);
return fd;
// because in async signal processing case (when handler is called directly
// from rtl_generic_sighandler) we have not yet received the reraised
// signal; and it looks too fragile to intercept all ways to reraise a signal.
- if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
+ if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
+ errno != 99) {
VarSizeStackTrace stack;
// StackTrace::GetNestInstructionPc(pc) is used because return address is
// expected, OutputReport() will undo this.
if (in_symbolizer())
return REAL(fork)(fake);
SCOPED_INTERCEPTOR_RAW(fork, fake);
+ return REAL(fork)(fake);
+}
+
+void atfork_prepare() {
+ if (in_symbolizer())
+ return;
+ ThreadState *thr = cur_thread();
+ const uptr pc = StackTrace::GetCurrentPc();
ForkBefore(thr, pc);
- int pid;
- {
- // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
- // we'll assert in CheckNoLocks() unless we ignore interceptors.
- ScopedIgnoreInterceptors ignore;
- pid = REAL(fork)(fake);
- }
- if (pid == 0) {
- // child
- ForkChildAfter(thr, pc);
- FdOnFork(thr, pc);
- } else if (pid > 0) {
- // parent
- ForkParentAfter(thr, pc);
- } else {
- // error
- ForkParentAfter(thr, pc);
- }
- return pid;
+}
+
+void atfork_parent() {
+ if (in_symbolizer())
+ return;
+ ThreadState *thr = cur_thread();
+ const uptr pc = StackTrace::GetCurrentPc();
+ ForkParentAfter(thr, pc);
+}
+
+void atfork_child() {
+ if (in_symbolizer())
+ return;
+ ThreadState *thr = cur_thread();
+ const uptr pc = StackTrace::GetCurrentPc();
+ ForkChildAfter(thr, pc);
+ FdOnFork(thr, pc);
}
TSAN_INTERCEPTOR(int, vfork, int fake) {
#define NEED_TLS_GET_ADDR
#endif
#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
+#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
INTERCEPT_FUNCTION_VER(name, ver)
+#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
+ (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name))
#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
// the signal handler through rtl_sigaction, very bad things will happen.
// The handler will run synchronously and corrupt tsan per-thread state.
SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
+ if (sig <= 0 || sig >= kSigCount) {
+ errno = errno_EINVAL;
+ return -1;
+ }
__sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
__sanitizer_sigaction old_stored;
if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored));
MemoryAccessRange(thr, pc, p, s, write);
}
-static void syscall_acquire(uptr pc, uptr addr) {
+static USED void syscall_acquire(uptr pc, uptr addr) {
TSAN_SYSCALL();
Acquire(thr, pc, addr);
DPrintf("syscall_acquire(%p)\n", addr);
}
-static void syscall_release(uptr pc, uptr addr) {
+static USED void syscall_release(uptr pc, uptr addr) {
TSAN_SYSCALL();
DPrintf("syscall_release(%p)\n", addr);
Release(thr, pc, addr);
FdRelease(thr, pc, fd);
}
-static void syscall_pre_fork(uptr pc) {
- TSAN_SYSCALL();
- ForkBefore(thr, pc);
-}
+static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); }
static void syscall_post_fork(uptr pc, int pid) {
- TSAN_SYSCALL();
+ ThreadState *thr = cur_thread();
if (pid == 0) {
// child
ForkChildAfter(thr, pc);
#include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
#ifdef NEED_TLS_GET_ADDR
+
+static void handle_tls_addr(void *arg, void *res) {
+ ThreadState *thr = cur_thread();
+ if (!thr)
+ return;
+ DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
+ thr->tls_addr + thr->tls_size);
+ if (!dtv)
+ return;
+ // New DTLS block has been allocated.
+ MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+}
+
+#if !SANITIZER_S390
// Define own interceptor instead of sanitizer_common's for three reasons:
// 1. It must not process pending signals.
// Signal handlers may contain MOVDQA instruction (see below).
// execute MOVDQA with stack addresses.
TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) {
void *res = REAL(__tls_get_addr)(arg);
- ThreadState *thr = cur_thread();
- if (!thr)
- return res;
- DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr,
- thr->tls_addr + thr->tls_size);
- if (!dtv)
- return res;
- // New DTLS block has been allocated.
- MemoryResetRange(thr, 0, dtv->beg, dtv->size);
+ handle_tls_addr(arg, res);
return res;
}
+#else // SANITIZER_S390
+TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
+ uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset));
+ char *tp = static_cast<char *>(__builtin_thread_pointer());
+ handle_tls_addr(arg, res + tp);
+ return res;
+}
+#endif
#endif
#if SANITIZER_NETBSD
#endif
// Instruct libc malloc to consume less memory.
-#if SANITIZER_LINUX
+#if SANITIZER_GLIBC
mallopt(1, 0); // M_MXFAST
mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
#endif
TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
+ TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT;
+
TSAN_INTERCEPT(pthread_mutex_init);
TSAN_INTERCEPT(pthread_mutex_destroy);
TSAN_INTERCEPT(pthread_mutex_trylock);
TSAN_INTERCEPT(_exit);
#ifdef NEED_TLS_GET_ADDR
+#if !SANITIZER_S390
TSAN_INTERCEPT(__tls_get_addr);
+#else
+ TSAN_INTERCEPT(__tls_get_addr_internal);
+ TSAN_INTERCEPT(__tls_get_offset);
+#endif
#endif
TSAN_MAYBE_INTERCEPT__LWP_EXIT;
Printf("ThreadSanitizer: failed to setup atexit callback\n");
Die();
}
+ if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) {
+ Printf("ThreadSanitizer: failed to setup atfork callbacks\n");
+ Die();
+ }
#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
#include "tsan_interface_ann.h"
#include "tsan_rtl.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
#define CALLERPC ((uptr)__builtin_return_address(0))
using namespace __tsan;
-typedef u16 uint16_t;
-typedef u32 uint32_t;
-typedef u64 uint64_t;
-
void __tsan_init() {
cur_thread_init();
Initialize(cur_thread());
}
void __tsan_read16_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr + 8, kSizeLog8);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
}
void __tsan_write16_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr + 8, kSizeLog8);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
}
// __tsan_unaligned_read/write calls are emitted by compiler.
typedef unsigned int a32;
typedef unsigned long long a64;
#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
- || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64)
+ || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
+ !defined(__mips64) && !defined(__s390x__)
__extension__ typedef __int128 a128;
# define __TSAN_HAS_INT128 1
#else
#endif
// Part of ABI, do not change.
-// https://github.com/llvm/llvm-project/blob/master/libcxx/include/atomic
+// https://github.com/llvm/llvm-project/blob/main/libcxx/include/atomic
typedef enum {
mo_relaxed,
mo_consume,
SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
u8 *a);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_on_initialize();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+int __tsan_on_finalize(int failed);
+
} // extern "C"
} // namespace __tsan
#include "sanitizer_common/sanitizer_stacktrace.h"
#include "sanitizer_common/sanitizer_vector.h"
#include "tsan_interface_ann.h"
-#include "tsan_mutex.h"
#include "tsan_report.h"
#include "tsan_rtl.h"
#include "tsan_mman.h"
~ScopedAnnotation() {
FuncExit(thr_);
- CheckNoLocks(thr_);
+ CheckedMutex::CheckNoLocks();
}
private:
ThreadState *const thr_;
return ret; \
ThreadState *thr = cur_thread(); \
const uptr caller_pc = (uptr)__builtin_return_address(0); \
- StatInc(thr, StatAnnotation); \
- StatInc(thr, Stat##typ); \
ScopedAnnotation sa(thr, __func__, caller_pc); \
const uptr pc = StackTrace::GetCurrentPc(); \
(void)pc; \
ExpectRace expect;
ExpectRace benign;
- DynamicAnnContext()
- : mtx(MutexTypeAnnotations, StatMtxAnnotations) {
- }
+ DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
};
static DynamicAnnContext *dyn_ann_ctx;
}
#endif
-template<typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
+template <typename T>
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
+ morder mo) NO_THREAD_SAFETY_ANALYSIS {
CHECK(IsLoadOrder(mo));
// This fast-path is critical for performance.
// Assume the access is atomic.
}
#endif
-template<typename T>
+template <typename T>
static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
- morder mo) {
+ morder mo) NO_THREAD_SAFETY_ANALYSIS {
CHECK(IsStoreOrder(mo));
MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
// This fast-path is critical for performance.
s->mtx.Unlock();
}
-template<typename T, T (*F)(volatile T *v, T op)>
-static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
+template <typename T, T (*F)(volatile T *v, T op)>
+static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v,
+ morder mo) NO_THREAD_SAFETY_ANALYSIS {
MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
SyncVar *s = 0;
if (mo != mo_relaxed) {
return c;
}
-template<typename T>
-static bool AtomicCAS(ThreadState *thr, uptr pc,
- volatile T *a, T *c, T v, morder mo, morder fmo) {
- (void)fmo; // Unused because llvm does not pass it yet.
+template <typename T>
+static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo,
+ morder fmo) NO_THREAD_SAFETY_ANALYSIS {
+ // 31.7.2.18: "The failure argument shall not be memory_order_release
+ // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
+ // (mo_relaxed) when those are used.
+ CHECK(IsLoadOrder(fmo));
+
MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
SyncVar *s = 0;
- bool write_lock = mo != mo_acquire && mo != mo_consume;
- if (mo != mo_relaxed) {
+ bool write_lock = IsReleaseOrder(mo);
+
+ if (mo != mo_relaxed || fmo != mo_relaxed)
s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
+
+ T cc = *c;
+ T pr = func_cas(a, cc, v);
+ bool success = pr == cc;
+ if (!success) {
+ *c = pr;
+ mo = fmo;
+ }
+
+ if (s) {
thr->fast_state.IncrementEpoch();
// Can't increment epoch w/o writing to the trace as well.
TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
- if (IsAcqRelOrder(mo))
+
+ if (success && IsAcqRelOrder(mo))
AcquireReleaseImpl(thr, pc, &s->clock);
- else if (IsReleaseOrder(mo))
+ else if (success && IsReleaseOrder(mo))
ReleaseImpl(thr, pc, &s->clock);
else if (IsAcquireOrder(mo))
AcquireImpl(thr, pc, &s->clock);
- }
- T cc = *c;
- T pr = func_cas(a, cc, v);
- if (s) {
+
if (write_lock)
s->mtx.Unlock();
else
s->mtx.ReadUnlock();
}
- if (pr == cc)
- return true;
- *c = pr;
- return false;
+
+ return success;
}
template<typename T>
const uptr callpc = (uptr)__builtin_return_address(0); \
uptr pc = StackTrace::GetCurrentPc(); \
mo = convert_morder(mo); \
- AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
ScopedAtomic sa(thr, callpc, a, mo, __func__); \
return Atomic##func(thr, pc, __VA_ARGS__); \
/**/
ThreadState *thr_;
};
-static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
- StatInc(thr, StatAtomic);
- StatInc(thr, t);
- StatInc(thr, size == 1 ? StatAtomic1
- : size == 2 ? StatAtomic2
- : size == 4 ? StatAtomic4
- : size == 8 ? StatAtomic8
- : StatAtomic16);
- StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
- : mo == mo_consume ? StatAtomicConsume
- : mo == mo_acquire ? StatAtomicAcquire
- : mo == mo_release ? StatAtomicRelease
- : mo == mo_acq_rel ? StatAtomicAcq_Rel
- : StatAtomicSeq_Cst);
-}
-
extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
#include "tsan_interface.h"
#include "tsan_rtl.h"
+#include "sanitizer_common/sanitizer_ptrauth.h"
#define CALLERPC ((uptr)__builtin_return_address(0))
}
void __tsan_read1_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
}
void __tsan_read2_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
}
void __tsan_read4_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
}
void __tsan_read8_pc(void *addr, void *pc) {
- MemoryRead(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+ MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
}
void __tsan_write1_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog1);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
}
void __tsan_write2_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog2);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
}
void __tsan_write4_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog4);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
}
void __tsan_write8_pc(void *addr, void *pc) {
- MemoryWrite(cur_thread(), (uptr)pc, (uptr)addr, kSizeLog8);
+ MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
}
void __tsan_vptr_update(void **vptr_p, void *new_val) {
}
void __tsan_func_entry(void *pc) {
- FuncEntry(cur_thread(), (uptr)pc);
+ FuncEntry(cur_thread(), STRIP_PAC_PC(pc));
}
void __tsan_func_exit() {
}
void __tsan_read_range_pc(void *addr, uptr size, void *pc) {
- MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, false);
+ MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
}
void __tsan_write_range_pc(void *addr, uptr size, void *pc) {
- MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, true);
+ MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
}
#include "tsan_interface_java.h"
#include "tsan_rtl.h"
-#include "tsan_mutex.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_placement_new.h"
Mutex mtx;
Processor *proc;
- GlobalProc()
- : mtx(MutexTypeGlobalProc, StatMtxGlobalProc)
- , proc(ProcCreate()) {
- }
+ GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
};
static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
if (atomic_load_relaxed(&thr->in_signal_handler) == 0 ||
- !flags()->report_signal_unsafe)
+ !ShouldReport(thr, ReportTypeSignalUnsafe))
return;
VarSizeStackTrace stack;
ObtainCurrentStack(thr, pc, &stack);
namespace __tsan {
+#if defined(__x86_64__)
+#define HAS_48_BIT_ADDRESS_SPACE 1
+#elif SANITIZER_IOSSIM // arm64 iOS simulators (order of #if matters)
+#define HAS_48_BIT_ADDRESS_SPACE 1
+#elif SANITIZER_IOS // arm64 iOS devices (order of #if matters)
+#define HAS_48_BIT_ADDRESS_SPACE 0
+#elif SANITIZER_MAC // arm64 macOS (order of #if matters)
+#define HAS_48_BIT_ADDRESS_SPACE 1
+#else
+#define HAS_48_BIT_ADDRESS_SPACE 0
+#endif
+
#if !SANITIZER_GO
-#if defined(__x86_64__)
+#if HAS_48_BIT_ADDRESS_SPACE
/*
C/C++ on linux/x86_64 and freebsd/x86_64
0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
ff00 0000 00 - ff80 0000 00: - (2 GB)
ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
*/
-struct Mapping {
+struct Mapping40 {
static const uptr kMetaShadowBeg = 0x4000000000ull;
static const uptr kMetaShadowEnd = 0x5000000000ull;
static const uptr kTraceMemBeg = 0xb000000000ull;
};
#define TSAN_MID_APP_RANGE 1
+#define TSAN_RUNTIME_VMA 1
#elif defined(__aarch64__) && defined(__APPLE__)
/*
C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
static const uptr kVdsoBeg = 0x7000000000000000ull;
};
-#elif defined(__aarch64__)
+#elif defined(__aarch64__) && !defined(__APPLE__)
// AArch64 supports multiple VMA which leads to multiple address transformation
// functions. To support these multiple VMAS transformations and mappings TSAN
// runtime for AArch64 uses an external memory read (vmaSize) to select which
// Indicates the runtime will define the memory regions at runtime.
#define TSAN_RUNTIME_VMA 1
+#elif defined(__s390x__)
+/*
+C/C++ on linux/s390x
+While the kernel provides a 64-bit address space, we have to restrict ourselves
+to 48 bits due to how e.g. SyncVar::GetId() works.
+0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
+0e00 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+b000 0000 0000 - be00 0000 0000: -
+be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
+*/
+struct Mapping {
+ static const uptr kMetaShadowBeg = 0x900000000000ull;
+ static const uptr kMetaShadowEnd = 0x980000000000ull;
+ static const uptr kTraceMemBeg = 0xa00000000000ull;
+ static const uptr kTraceMemEnd = 0xb00000000000ull;
+ static const uptr kShadowBeg = 0x400000000000ull;
+ static const uptr kShadowEnd = 0x800000000000ull;
+ static const uptr kHeapMemBeg = 0xbe0000000000ull;
+ static const uptr kHeapMemEnd = 0xc00000000000ull;
+ static const uptr kLoAppMemBeg = 0x000000001000ull;
+ static const uptr kLoAppMemEnd = 0x0e0000000000ull;
+ static const uptr kHiAppMemBeg = 0xc00000004000ull;
+ static const uptr kHiAppMemEnd = 0xc00000004000ull;
+ static const uptr kAppMemMsk = 0xb00000000000ull;
+ static const uptr kAppMemXor = 0x100000000000ull;
+ static const uptr kVdsoBeg = 0xfffffffff000ull;
+};
#endif
-#elif SANITIZER_GO && !SANITIZER_WINDOWS && defined(__x86_64__)
+#elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE
/* Go on linux, darwin and freebsd on x86_64
0000 0000 1000 - 0000 1000 0000: executable
#elif SANITIZER_GO && defined(__aarch64__)
-/* Go on linux/aarch64 (48-bit VMA)
+/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
0000 0000 1000 - 0000 1000 0000: executable
0000 1000 0000 - 00c0 0000 0000: -
00c0 0000 0000 - 00e0 0000 0000: heap
// Indicates the runtime will define the memory regions at runtime.
#define TSAN_RUNTIME_VMA 1
+#elif SANITIZER_GO && defined(__mips64)
+/*
+Go on linux/mips64 (47-bit VMA)
+0000 0000 1000 - 0000 1000 0000: executable
+0000 1000 0000 - 00c0 0000 0000: -
+00c0 0000 0000 - 00e0 0000 0000: heap
+00e0 0000 0000 - 2000 0000 0000: -
+2000 0000 0000 - 3000 0000 0000: shadow
+3000 0000 0000 - 3000 0000 0000: -
+3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
+4000 0000 0000 - 6000 0000 0000: -
+6000 0000 0000 - 6200 0000 0000: traces
+6200 0000 0000 - 8000 0000 0000: -
+*/
+struct Mapping47 {
+ static const uptr kMetaShadowBeg = 0x300000000000ull;
+ static const uptr kMetaShadowEnd = 0x400000000000ull;
+ static const uptr kTraceMemBeg = 0x600000000000ull;
+ static const uptr kTraceMemEnd = 0x620000000000ull;
+ static const uptr kShadowBeg = 0x200000000000ull;
+ static const uptr kShadowEnd = 0x300000000000ull;
+ static const uptr kAppMemBeg = 0x000000001000ull;
+ static const uptr kAppMemEnd = 0x00e000000000ull;
+};
+
+#define TSAN_RUNTIME_VMA 1
+
+#elif SANITIZER_GO && defined(__s390x__)
+/*
+Go on linux/s390x
+0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
+1000 0000 0000 - 4000 0000 0000: -
+4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
+8000 0000 0000 - 9000 0000 0000: -
+9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
+9800 0000 0000 - a000 0000 0000: -
+a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
+*/
+struct Mapping {
+ static const uptr kMetaShadowBeg = 0x900000000000ull;
+ static const uptr kMetaShadowEnd = 0x980000000000ull;
+ static const uptr kTraceMemBeg = 0xa00000000000ull;
+ static const uptr kTraceMemEnd = 0xb00000000000ull;
+ static const uptr kShadowBeg = 0x400000000000ull;
+ static const uptr kShadowEnd = 0x800000000000ull;
+ static const uptr kAppMemBeg = 0x000000001000ull;
+ static const uptr kAppMemEnd = 0x100000000000ull;
+};
+
#else
# error "Unknown platform"
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return MappingImpl<Mapping40, Type>();
+#else
+ case 47: return MappingImpl<Mapping47, Type>();
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return MappingImpl<Mapping, Type>();
#endif
}
DCHECK(0);
return false;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return IsAppMemImpl<Mapping40>(mem);
+#else
+ case 47: return IsAppMemImpl<Mapping47>(mem);
+#endif
+ }
+ DCHECK(0);
+ return false;
#else
return IsAppMemImpl<Mapping>(mem);
#endif
}
DCHECK(0);
return false;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return IsShadowMemImpl<Mapping40>(mem);
+#else
+ case 47: return IsShadowMemImpl<Mapping47>(mem);
+#endif
+ }
+ DCHECK(0);
+ return false;
#else
return IsShadowMemImpl<Mapping>(mem);
#endif
}
DCHECK(0);
return false;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return IsMetaMemImpl<Mapping40>(mem);
+#else
+ case 47: return IsMetaMemImpl<Mapping47>(mem);
+#endif
+ }
+ DCHECK(0);
+ return false;
#else
return IsMetaMemImpl<Mapping>(mem);
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return MemToShadowImpl<Mapping40>(x);
+#else
+ case 47: return MemToShadowImpl<Mapping47>(x);
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return MemToShadowImpl<Mapping>(x);
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return MemToMetaImpl<Mapping40>(x);
+#else
+ case 47: return MemToMetaImpl<Mapping47>(x);
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return MemToMetaImpl<Mapping>(x);
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return ShadowToMemImpl<Mapping40>(s);
+#else
+ case 47: return ShadowToMemImpl<Mapping47>(s);
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return ShadowToMemImpl<Mapping>(s);
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return GetThreadTraceImpl<Mapping40>(tid);
+#else
+ case 47: return GetThreadTraceImpl<Mapping47>(tid);
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return GetThreadTraceImpl<Mapping>(tid);
#endif
}
DCHECK(0);
return 0;
+#elif defined(__mips64)
+ switch (vmaSize) {
+#if !SANITIZER_GO
+ case 40: return GetThreadTraceHeaderImpl<Mapping40>(tid);
+#else
+ case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
+#endif
+ }
+ DCHECK(0);
+ return 0;
#else
return GetThreadTraceHeaderImpl<Mapping>(tid);
#endif
uptr ExtractLongJmpSp(uptr *env);
void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
-int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
- void *abstime), void *c, void *m, void *abstime,
- void(*cleanup)(void *arg), void *arg);
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+ void (*cleanup)(void *arg), void *arg);
void DestroyThreadState();
void PlatformCleanUpThreadState(ThreadState *thr);
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \
- SANITIZER_OPENBSD
+#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
-#include "sanitizer_common/sanitizer_platform_limits_openbsd.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
#include "sanitizer_common/sanitizer_posix.h"
#include "sanitizer_common/sanitizer_procmaps.h"
Die();
}
# endif
+#elif defined(__mips64)
+# if !SANITIZER_GO
+ if (vmaSize != 40) {
+ Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+ Printf("FATAL: Found %zd - Supported 40\n", vmaSize);
+ Die();
+ }
+# else
+ if (vmaSize != 47) {
+ Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+ Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
+ Die();
+ }
+# endif
#endif
#endif
}
return mangled_sp ^ xor_key;
#elif defined(__mips__)
return mangled_sp;
+#elif defined(__s390x__)
+ // tcbhead_t.stack_guard
+ uptr xor_key = ((uptr *)__builtin_thread_pointer())[5];
+ return mangled_sp ^ xor_key;
#else
#error "Unknown platform"
#endif
}
-#ifdef __powerpc__
+#if SANITIZER_NETBSD
+# ifdef __x86_64__
+# define LONG_JMP_SP_ENV_SLOT 6
+# else
+# error unsupported
+# endif
+#elif defined(__powerpc__)
# define LONG_JMP_SP_ENV_SLOT 0
#elif SANITIZER_FREEBSD
# define LONG_JMP_SP_ENV_SLOT 2
-#elif SANITIZER_NETBSD
-# define LONG_JMP_SP_ENV_SLOT 6
#elif SANITIZER_LINUX
# ifdef __aarch64__
# define LONG_JMP_SP_ENV_SLOT 13
# elif defined(__mips64)
# define LONG_JMP_SP_ENV_SLOT 1
+# elif defined(__s390x__)
+# define LONG_JMP_SP_ENV_SLOT 9
# else
# define LONG_JMP_SP_ENV_SLOT 6
# endif
// Note: this function runs with async signals enabled,
// so it must not touch any tsan state.
-int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
- void *abstime), void *c, void *m, void *abstime,
- void(*cleanup)(void *arg), void *arg) {
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+ void (*cleanup)(void *arg), void *arg) {
// pthread_cleanup_push/pop are hardcore macros mess.
// We can't intercept nor call them w/o including pthread.h.
int res;
pthread_cleanup_push(cleanup, arg);
- res = fn(c, m, abstime);
+ res = fn(arg);
pthread_cleanup_pop(0);
return res;
}
dead_thread_state->fast_state.SetIgnoreBit();
dead_thread_state->ignore_interceptors = 1;
dead_thread_state->is_dead = true;
- *const_cast<int*>(&dead_thread_state->tid) = -1;
+ *const_cast<u32*>(&dead_thread_state->tid) = -1;
CHECK_EQ(0, internal_mprotect(dead_thread_state, sizeof(ThreadState),
PROT_READ));
}
} // namespace __tsan
-#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
- // SANITIZER_OPENBSD
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
#endif
void InitializePlatformEarly() {
-#if defined(__aarch64__)
+#if !SANITIZER_GO && !HAS_48_BIT_ADDRESS_SPACE
uptr max_vm = GetMaxUserVirtualAddress() + 1;
if (max_vm != Mapping::kHiAppMemEnd) {
Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
#if !SANITIZER_GO
// Note: this function runs with async signals enabled,
// so it must not touch any tsan state.
-int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
- void *abstime), void *c, void *m, void *abstime,
- void(*cleanup)(void *arg), void *arg) {
+int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
+ void (*cleanup)(void *arg), void *arg) {
// pthread_cleanup_push/pop are hardcore macros mess.
// We can't intercept nor call them w/o including pthread.h.
int res;
pthread_cleanup_push(cleanup, arg);
- res = fn(c, m, abstime);
+ res = fn(arg);
pthread_cleanup_pop(0);
return res;
}
"HINT: if %s is not supported in your environment, you may set "
"TSAN_OPTIONS=%s=0\n";
-static void NoHugePagesInShadow(uptr addr, uptr size) {
- SetShadowRegionHugePageMode(addr, size);
-}
-
static void DontDumpShadow(uptr addr, uptr size) {
if (common_flags()->use_madv_dontdump)
if (!DontDumpShadowMemory(addr, size)) {
#if !SANITIZER_GO
void InitializeShadowMemory() {
// Map memory shadow.
- if (!MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), "shadow")) {
+ if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
+ "shadow")) {
Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
Die();
// Frequently a thread uses only a small part of stack and similarly
// a program uses a small part of large mmap. On some programs
// we see 20% memory usage reduction without huge pages for this range.
- // FIXME: don't use constants here.
-#if defined(__x86_64__)
- const uptr kMadviseRangeBeg = 0x7f0000000000ull;
- const uptr kMadviseRangeSize = 0x010000000000ull;
-#elif defined(__mips64)
- const uptr kMadviseRangeBeg = 0xff00000000ull;
- const uptr kMadviseRangeSize = 0x0100000000ull;
-#elif defined(__aarch64__) && defined(__APPLE__)
- uptr kMadviseRangeBeg = LoAppMemBeg();
- uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
-#elif defined(__aarch64__)
- uptr kMadviseRangeBeg = 0;
- uptr kMadviseRangeSize = 0;
- if (vmaSize == 39) {
- kMadviseRangeBeg = 0x7d00000000ull;
- kMadviseRangeSize = 0x0300000000ull;
- } else if (vmaSize == 42) {
- kMadviseRangeBeg = 0x3f000000000ull;
- kMadviseRangeSize = 0x01000000000ull;
- } else {
- DCHECK(0);
- }
-#elif defined(__powerpc64__)
- uptr kMadviseRangeBeg = 0;
- uptr kMadviseRangeSize = 0;
- if (vmaSize == 44) {
- kMadviseRangeBeg = 0x0f60000000ull;
- kMadviseRangeSize = 0x0010000000ull;
- } else if (vmaSize == 46) {
- kMadviseRangeBeg = 0x3f0000000000ull;
- kMadviseRangeSize = 0x010000000000ull;
- } else {
- DCHECK(0);
- }
-#endif
- NoHugePagesInShadow(MemToShadow(kMadviseRangeBeg),
- kMadviseRangeSize * kShadowMultiplier);
DontDumpShadow(ShadowBeg(), ShadowEnd() - ShadowBeg());
DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
ShadowBeg(), ShadowEnd(),
// Map meta shadow.
const uptr meta = MetaShadowBeg();
const uptr meta_size = MetaShadowEnd() - meta;
- if (!MmapFixedNoReserve(meta, meta_size, "meta shadow")) {
+ if (!MmapFixedSuperNoReserve(meta, meta_size, "meta shadow")) {
Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n");
Printf("FATAL: Make sure to compile with -fPIE and to link with -pie.\n");
Die();
}
- NoHugePagesInShadow(meta, meta_size);
DontDumpShadow(meta, meta_size);
DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
meta, meta + meta_size, meta_size >> 30);
InitializeShadowMemoryPlatform();
}
-static void ProtectRange(uptr beg, uptr end) {
+static bool TryProtectRange(uptr beg, uptr end) {
CHECK_LE(beg, end);
if (beg == end)
- return;
- if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) {
+ return true;
+ return beg == (uptr)MmapFixedNoAccess(beg, end - beg);
+}
+
+static void ProtectRange(uptr beg, uptr end) {
+ if (!TryProtectRange(beg, end)) {
Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end);
Printf("FATAL: Make sure you are not using unlimited stack\n");
Die();
Die();
}
-#if defined(__aarch64__) && defined(__APPLE__)
+#if defined(__aarch64__) && defined(__APPLE__) && !HAS_48_BIT_ADDRESS_SPACE
ProtectRange(HeapMemEnd(), ShadowBeg());
ProtectRange(ShadowEnd(), MetaShadowBeg());
ProtectRange(MetaShadowEnd(), TraceMemBeg());
ProtectRange(TraceMemEnd(), HeapMemBeg());
ProtectRange(HeapEnd(), HiAppMemBeg());
#endif
+
+#if defined(__s390x__)
+ // Protect the rest of the address space.
+ const uptr user_addr_max_l4 = 0x0020000000000000ull;
+ const uptr user_addr_max_l5 = 0xfffffffffffff000ull;
+ // All the maintained s390x kernels support at least 4-level page tables.
+ ProtectRange(HiAppMemEnd(), user_addr_max_l4);
+ // Older s390x kernels may not support 5-level page tables.
+ TryProtectRange(user_addr_max_l4, user_addr_max_l5);
+#endif
}
#endif
const int kThreadBufSize = 32;
const char *thread_name(char *buf, int tid) {
- if (tid == 0)
+ if (tid == kMainTid)
return "main thread";
internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
return buf;
}
SymbolizedStack *frame = ent->frames;
for (int i = 0; frame && frame->info.address; frame = frame->next, i++) {
- InternalScopedString res(2 * GetPageSizeCached());
- RenderFrame(&res, common_flags()->stack_trace_format, i, frame->info,
+ InternalScopedString res;
+ RenderFrame(&res, common_flags()->stack_trace_format, i,
+ frame->info.address, &frame->info,
common_flags()->symbolize_vs_style,
common_flags()->strip_path_prefix, kInterposedFunctionPrefix);
Printf("%s\n", res.data());
static void PrintThread(const ReportThread *rt) {
Decorator d;
- if (rt->id == 0) // Little sense in describing the main thread.
+ if (rt->id == kMainTid) // Little sense in describing the main thread.
return;
Printf("%s", d.ThreadDescription());
Printf(" Thread T%d", rt->id);
ReportErrorSummary(rep_typ_str, frame->info);
}
- if (common_flags()->print_module_map == 2) PrintModuleMap();
+ if (common_flags()->print_module_map == 2)
+ DumpProcessMap();
Printf("==================\n");
}
#else // #if !SANITIZER_GO
-const int kMainThreadId = 1;
+const u32 kMainGoroutineId = 1;
void PrintStack(const ReportStack *ent) {
if (ent == 0 || ent->frames == 0) {
Printf("%s at %p by ",
(first ? (mop->write ? "Write" : "Read")
: (mop->write ? "Previous write" : "Previous read")), mop->addr);
- if (mop->tid == kMainThreadId)
+ if (mop->tid == kMainGoroutineId)
Printf("main goroutine:\n");
else
Printf("goroutine %d:\n", mop->tid);
Printf("\n");
Printf("Heap block of size %zu at %p allocated by ",
loc->heap_chunk_size, loc->heap_chunk_start);
- if (loc->tid == kMainThreadId)
+ if (loc->tid == kMainGoroutineId)
Printf("main goroutine:\n");
else
Printf("goroutine %d:\n", loc->tid);
}
static void PrintThread(const ReportThread *rt) {
- if (rt->id == kMainThreadId)
+ if (rt->id == kMainGoroutineId)
return;
Printf("\n");
Printf("Goroutine %d (%s) created at:\n",
// Main file (entry points) for the TSan run-time.
//===----------------------------------------------------------------------===//
+#include "tsan_rtl.h"
+
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_file.h"
#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_placement_new.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
#include "tsan_defs.h"
-#include "tsan_platform.h"
-#include "tsan_rtl.h"
+#include "tsan_interface.h"
#include "tsan_mman.h"
+#include "tsan_platform.h"
#include "tsan_suppressions.h"
#include "tsan_symbolize.h"
#include "ubsan/ubsan_init.h"
bool OnFinalize(bool failed);
void OnInitialize();
#else
+#include <dlfcn.h>
SANITIZER_WEAK_CXX_DEFAULT_IMPL
bool OnFinalize(bool failed) {
+#if !SANITIZER_GO
+ if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize"))
+ return reinterpret_cast<decltype(&__tsan_on_finalize)>(ptr)(failed);
+#endif
return failed;
}
SANITIZER_WEAK_CXX_DEFAULT_IMPL
-void OnInitialize() {}
+void OnInitialize() {
+#if !SANITIZER_GO
+ if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) {
+ return reinterpret_cast<decltype(&__tsan_on_initialize)>(ptr)();
+ }
+#endif
+}
#endif
-static char thread_registry_placeholder[sizeof(ThreadRegistry)];
+static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
static ThreadContextBase *CreateThreadContext(u32 tid) {
// Map thread trace when context is created.
new((void*)hdr) Trace();
// We are going to use only a small part of the trace with the default
// value of history_size. However, the constructor writes to the whole trace.
- // Unmap the unused part.
+ // Release the unused part.
uptr hdr_end = hdr + sizeof(Trace);
hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
hdr_end = RoundUp(hdr_end, GetPageSizeCached());
- if (hdr_end < hdr + sizeof(Trace))
- UnmapOrDie((void*)hdr_end, hdr + sizeof(Trace) - hdr_end);
+ if (hdr_end < hdr + sizeof(Trace)) {
+ ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));
+ uptr unused = hdr + sizeof(Trace) - hdr_end;
+ if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {
+ Report("ThreadSanitizer: failed to mprotect(%p, %p)\n",
+ hdr_end, unused);
+ CHECK("unable to mprotect" && 0);
+ }
+ }
void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
return new(mem) ThreadContext(tid);
}
#endif
Context::Context()
- : initialized()
- , report_mtx(MutexTypeReport, StatMtxReport)
- , nreported()
- , nmissed_expected()
- , thread_registry(new(thread_registry_placeholder) ThreadRegistry(
- CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse))
- , racy_mtx(MutexTypeRacy, StatMtxRacy)
- , racy_stacks()
- , racy_addresses()
- , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
- , clock_alloc("clock allocator") {
+ : initialized(),
+ report_mtx(MutexTypeReport),
+ nreported(),
+ nmissed_expected(),
+ thread_registry(new (thread_registry_placeholder) ThreadRegistry(
+ CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),
+ racy_mtx(MutexTypeRacy),
+ racy_stacks(),
+ racy_addresses(),
+ fired_suppressions_mtx(MutexTypeFired),
+ clock_alloc(LINKER_INITIALIZED, "clock allocator") {
fired_suppressions.reserve(8);
}
// The objects are allocated in TLS, so one may rely on zero-initialization.
-ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
- unsigned reuse_count,
- uptr stk_addr, uptr stk_size,
+ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
+ unsigned reuse_count, uptr stk_addr, uptr stk_size,
uptr tls_addr, uptr tls_size)
- : fast_state(tid, epoch)
- // Do not touch these, rely on zero initialization,
- // they may be accessed before the ctor.
- // , ignore_reads_and_writes()
- // , ignore_interceptors()
- , clock(tid, reuse_count)
+ : fast_state(tid, epoch)
+ // Do not touch these, rely on zero initialization,
+ // they may be accessed before the ctor.
+ // , ignore_reads_and_writes()
+ // , ignore_interceptors()
+ ,
+ clock(tid, reuse_count)
#if !SANITIZER_GO
- , jmp_bufs()
+ ,
+ jmp_bufs()
#endif
- , tid(tid)
- , unique_id(unique_id)
- , stk_addr(stk_addr)
- , stk_size(stk_size)
- , tls_addr(tls_addr)
- , tls_size(tls_size)
+ ,
+ tid(tid),
+ unique_id(unique_id),
+ stk_addr(stk_addr),
+ stk_size(stk_size),
+ tls_addr(tls_addr),
+ tls_size(tls_size)
#if !SANITIZER_GO
- , last_sleep_clock(tid)
+ ,
+ last_sleep_clock(tid)
#endif
{
}
} else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
mprof_fd = 2;
} else {
- InternalScopedString filename(kMaxPathLength);
+ InternalScopedString filename;
filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
fd_t fd = OpenFile(filename.data(), WrOnly);
if (fd == kInvalidFd) {
Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
- &filename[0]);
+ filename.data());
} else {
mprof_fd = fd;
}
const uptr kPageSize = GetPageSizeCached();
uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
- if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
+ if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
+ "shadow"))
Die();
// Meta shadow is 2:1, so tread carefully.
if (!data_mapped) {
// First call maps data+bss.
data_mapped = true;
- if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+ if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+ "meta shadow"))
Die();
} else {
// Mapping continous heap.
return;
if (meta_begin < mapped_meta_end)
meta_begin = mapped_meta_end;
- if (!MmapFixedNoReserve(meta_begin, meta_end - meta_begin, "meta shadow"))
+ if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
+ "meta shadow"))
Die();
mapped_meta_end = meta_end;
}
CHECK_GE(addr, TraceMemBeg());
CHECK_LE(addr + size, TraceMemEnd());
CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
- if (!MmapFixedNoReserve(addr, size, name)) {
+ if (!MmapFixedSuperNoReserve(addr, size, name)) {
Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
addr, size);
Die();
}
#endif
+void CheckUnwind() {
+ // There is high probability that interceptors will check-fail as well,
+ // on the other hand there is no sense in processing interceptors
+ // since we are going to die soon.
+ ScopedIgnoreInterceptors ignore;
+#if !SANITIZER_GO
+ cur_thread()->ignore_sync++;
+ cur_thread()->ignore_reads_and_writes++;
+#endif
+ PrintCurrentStackSlow(StackTrace::GetCurrentPc());
+}
+
void Initialize(ThreadState *thr) {
// Thread safe because done before all threads exist.
static bool is_initialized = false;
ScopedIgnoreInterceptors ignore;
SanitizerToolName = "ThreadSanitizer";
// Install tool-specific callbacks in sanitizer_common.
- SetCheckFailedCallback(TsanCheckFailed);
+ SetCheckUnwindCallback(CheckUnwind);
ctx = new(ctx_placeholder) Context;
const char *env_name = SANITIZER_GO ? "GORACE" : "TSAN_OPTIONS";
InitializeInterceptors();
CheckShadowMapping();
InitializePlatform();
- InitializeMutex();
InitializeDynamicAnnotations();
#if !SANITIZER_GO
InitializeShadowMemory();
int Finalize(ThreadState *thr) {
bool failed = false;
- if (common_flags()->print_module_map == 1) PrintModuleMap();
+ if (common_flags()->print_module_map == 1)
+ DumpProcessMap();
if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
SleepForMillis(flags()->atexit_sleep_ms);
failed = OnFinalize(failed);
-#if TSAN_COLLECT_STATS
- StatAggregate(ctx->stat, thr->stat);
- StatOutput(ctx->stat);
-#endif
-
return failed ? common_flags()->exitcode : 0;
}
#if !SANITIZER_GO
-void ForkBefore(ThreadState *thr, uptr pc) {
+void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
ctx->thread_registry->Lock();
ctx->report_mtx.Lock();
- // Ignore memory accesses in the pthread_atfork callbacks.
- // If any of them triggers a data race we will deadlock
- // on the report_mtx.
- // We could ignore interceptors and sync operations as well,
+ ScopedErrorReportLock::Lock();
+ // Suppress all reports in the pthread_atfork callbacks.
+ // Reports will deadlock on the report_mtx.
+ // We could ignore sync operations as well,
// but so far it's unclear if it will do more good or harm.
// Unnecessarily ignoring things can lead to false positives later.
- ThreadIgnoreBegin(thr, pc);
+ thr->suppress_reports++;
+ // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
+ // we'll assert in CheckNoLocks() unless we ignore interceptors.
+ thr->ignore_interceptors++;
}
-void ForkParentAfter(ThreadState *thr, uptr pc) {
- ThreadIgnoreEnd(thr, pc); // Begin is in ForkBefore.
+void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+ thr->suppress_reports--; // Enabled in ForkBefore.
+ thr->ignore_interceptors--;
+ ScopedErrorReportLock::Unlock();
ctx->report_mtx.Unlock();
ctx->thread_registry->Unlock();
}
-void ForkChildAfter(ThreadState *thr, uptr pc) {
- ThreadIgnoreEnd(thr, pc); // Begin is in ForkBefore.
+void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+ thr->suppress_reports--; // Enabled in ForkBefore.
+ thr->ignore_interceptors--;
+ ScopedErrorReportLock::Unlock();
ctx->report_mtx.Unlock();
ctx->thread_registry->Unlock();
void MemoryAccessImpl1(ThreadState *thr, uptr addr,
int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
u64 *shadow_mem, Shadow cur) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
// This potentially can live in an MMX/SSE scratch register.
// The required intrinsics are:
return;
// choose a random candidate slot and replace it
StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
- StatInc(thr, StatShadowReplace);
return;
RACE:
HandleRace(thr, shadow_mem, cur, old);
if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
// Access to .rodata section, no races here.
// Measurements show that it can be 10-20% of all memory accesses.
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopRodata);
return;
}
FastState fast_state = thr->fast_state;
if (UNLIKELY(fast_state.GetIgnoreBit())) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopIgnored);
return;
}
if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
thr->fast_synch_epoch, kAccessIsWrite))) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopSame);
return;
}
u64 *shadow_mem, Shadow cur) {
if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
thr->fast_synch_epoch, kAccessIsWrite))) {
- StatInc(thr, StatMop);
- StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
- StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
- StatInc(thr, StatMopSame);
return;
}
// Reset middle part.
u64 *p1 = p;
p = RoundDown(end, kPageSize);
- UnmapOrDie((void*)p1, (uptr)p - (uptr)p1);
- if (!MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+ if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
Die();
// Set the ending.
while (p < end) {
ALWAYS_INLINE USED
void FuncEntry(ThreadState *thr, uptr pc) {
- StatInc(thr, StatFuncEnter);
DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
if (kCollectHistory) {
thr->fast_state.IncrementEpoch();
ALWAYS_INLINE USED
void FuncExit(ThreadState *thr) {
- StatInc(thr, StatFuncExit);
DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
if (kCollectHistory) {
thr->fast_state.IncrementEpoch();
void build_consistency_release() {}
#endif
-#if TSAN_COLLECT_STATS
-void build_consistency_stats() {}
-#else
-void build_consistency_nostats() {}
-#endif
-
} // namespace __tsan
+#if SANITIZER_CHECK_DEADLOCKS
+namespace __sanitizer {
+using namespace __tsan;
+MutexMeta mutex_meta[] = {
+ {MutexInvalid, "Invalid", {}},
+ {MutexThreadRegistry, "ThreadRegistry", {}},
+ {MutexTypeTrace, "Trace", {MutexLeaf}},
+ {MutexTypeReport, "Report", {MutexTypeSyncVar}},
+ {MutexTypeSyncVar, "SyncVar", {}},
+ {MutexTypeAnnotations, "Annotations", {}},
+ {MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
+ {MutexTypeFired, "Fired", {MutexLeaf}},
+ {MutexTypeRacy, "Racy", {MutexLeaf}},
+ {MutexTypeGlobalProc, "GlobalProc", {}},
+ {},
+};
+
+void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
+} // namespace __sanitizer
+#endif
+
#if !SANITIZER_GO
// Must be included in this file to make sure everything is inlined.
-#include "tsan_interface_inl.h"
+# include "tsan_interface_inl.h"
#endif
Allocator *allocator();
#endif
-void TsanCheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2);
-
const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
// FastState (from most significant bit):
Vector<JmpBuf> jmp_bufs;
int ignore_interceptors;
#endif
-#if TSAN_COLLECT_STATS
- u64 stat[StatCnt];
-#endif
- const int tid;
+ const u32 tid;
const int unique_id;
bool in_symbolizer;
bool in_ignored_lib;
const uptr tls_size;
ThreadContext *tctx;
-#if SANITIZER_DEBUG && !SANITIZER_GO
- InternalDeadlockDetector internal_deadlock_detector;
-#endif
DDLogicalThread *dd_lt;
// Current wired Processor, or nullptr. Required to handle any events.
const ReportDesc *current_report;
- explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch,
- unsigned reuse_count,
- uptr stk_addr, uptr stk_size,
+ explicit ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
+ unsigned reuse_count, uptr stk_addr, uptr stk_size,
uptr tls_addr, uptr tls_size);
};
ThreadState *cur_thread();
void set_cur_thread(ThreadState *thr);
void cur_thread_finalize();
-INLINE void cur_thread_init() { }
+inline void cur_thread_init() { }
#else
__attribute__((tls_model("initial-exec")))
extern THREADLOCAL char cur_thread_placeholder[];
-INLINE ThreadState *cur_thread() {
+inline ThreadState *cur_thread() {
return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
}
-INLINE void cur_thread_init() {
+inline void cur_thread_init() {
ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
if (UNLIKELY(!thr->current))
thr->current = thr;
}
-INLINE void set_cur_thread(ThreadState *thr) {
+inline void set_cur_thread(ThreadState *thr) {
reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
}
-INLINE void cur_thread_finalize() { }
+inline void cur_thread_finalize() { }
#endif // SANITIZER_MAC || SANITIZER_ANDROID
#endif // SANITIZER_GO
-class ThreadContext : public ThreadContextBase {
+class ThreadContext final : public ThreadContextBase {
public:
explicit ThreadContext(int tid);
~ThreadContext();
Flags flags;
- u64 stat[StatCnt];
u64 int_alloc_cnt[MBlockTypeCount];
u64 int_alloc_siz[MBlockTypeCount];
};
ScopedErrorReportLock lock_;
};
+bool ShouldReport(ThreadState *thr, ReportType typ);
ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack);
void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
MutexSet *mset, uptr *tag = nullptr);
ObtainCurrentStack(thr, pc, &stack); \
stack.ReverseOrder();
-#if TSAN_COLLECT_STATS
-void StatAggregate(u64 *dst, u64 *src);
-void StatOutput(u64 *stat);
-#endif
-
-void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) {
-#if TSAN_COLLECT_STATS
- thr->stat[typ] += n;
-#endif
-}
-void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) {
-#if TSAN_COLLECT_STATS
- thr->stat[typ] = n;
-#endif
-}
-
void MapShadow(uptr addr, uptr size);
void MapThreadTrace(uptr addr, uptr size, const char *name);
void DontNeedShadowFor(uptr addr, uptr size);
DCHECK_GE((int)typ, 0);
DCHECK_LE((int)typ, 7);
DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
- StatInc(thr, StatEvents);
u64 pos = fs.GetTracePos();
if (UNLIKELY((pos % kTracePartSize) == 0)) {
#if !SANITIZER_GO
void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r);
-struct Callback : DDCallback {
+struct Callback final : public DDCallback {
ThreadState *thr;
uptr pc;
// or false positives (e.g. unlock in a different thread).
if (SANITIZER_GO)
return;
+ if (!ShouldReport(thr, typ))
+ return;
ThreadRegistryLock l(ctx->thread_registry);
ScopedReport rep(typ);
rep.AddMutex(mid);
OutputReport(thr, rep);
}
-void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
- StatInc(thr, StatMutexCreate);
if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
CHECK(!thr->is_freeing);
thr->is_freeing = true;
s->mtx.Unlock();
}
-void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
- StatInc(thr, StatMutexDestroy);
SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
if (s == 0)
return;
ctx->dd->MutexInit(&cb, &s->dd);
}
bool unlock_locked = false;
- if (flags()->report_destroy_locked
- && s->owner_tid != SyncVar::kInvalidTid
- && !s->IsFlagSet(MutexFlagBroken)) {
+ if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
+ !s->IsFlagSet(MutexFlagBroken)) {
s->SetFlags(MutexFlagBroken);
unlock_locked = true;
}
if (!unlock_locked)
s->Reset(thr->proc()); // must not reset it before the report is printed
s->mtx.Unlock();
- if (unlock_locked) {
+ if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {
ThreadRegistryLock l(ctx->thread_registry);
ScopedReport rep(ReportTypeMutexDestroyLocked);
rep.AddMutex(mid);
// s will be destroyed and freed in MetaMap::FreeBlock.
}
-void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
}
}
-void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
+void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz,
+ int rec) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
thr->tid, addr, flagz, rec);
if (flagz & MutexFlagRecursiveLock)
thr->fast_state.IncrementEpoch();
TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
bool report_double_lock = false;
- if (s->owner_tid == SyncVar::kInvalidTid) {
+ if (s->owner_tid == kInvalidTid) {
CHECK_EQ(s->recursion, 0);
s->owner_tid = thr->tid;
s->last_lock = thr->fast_state.raw();
const bool first = s->recursion == 0;
s->recursion += rec;
if (first) {
- StatInc(thr, StatMutexLock);
AcquireImpl(thr, pc, &s->clock);
AcquireImpl(thr, pc, &s->read_clock);
} else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
- StatInc(thr, StatMutexRecLock);
}
thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
bool pre_lock = false;
}
}
-int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
s->recursion -= rec;
if (s->recursion == 0) {
- StatInc(thr, StatMutexUnlock);
- s->owner_tid = SyncVar::kInvalidTid;
+ s->owner_tid = kInvalidTid;
ReleaseStoreImpl(thr, pc, &s->clock);
} else {
- StatInc(thr, StatMutexRecUnlock);
}
}
thr->mset.Del(s->GetId(), true);
return rec;
}
-void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
}
}
-void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
+void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
- StatInc(thr, StatMutexReadLock);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
thr->fast_state.IncrementEpoch();
TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
bool report_bad_lock = false;
- if (s->owner_tid != SyncVar::kInvalidTid) {
+ if (s->owner_tid != kInvalidTid) {
if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
s->SetFlags(MutexFlagBroken);
report_bad_lock = true;
}
}
-void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
+void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
- StatInc(thr, StatMutexReadUnlock);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
thr->fast_state.IncrementEpoch();
TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
bool report_bad_unlock = false;
- if (s->owner_tid != SyncVar::kInvalidTid) {
+ if (s->owner_tid != kInvalidTid) {
if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
s->SetFlags(MutexFlagBroken);
report_bad_unlock = true;
}
}
-void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
+void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
if (IsAppMem(addr))
MemoryReadAtomic(thr, pc, addr, kSizeLog1);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
bool write = true;
bool report_bad_unlock = false;
- if (s->owner_tid == SyncVar::kInvalidTid) {
+ if (s->owner_tid == kInvalidTid) {
// Seems to be read unlock.
write = false;
- StatInc(thr, StatMutexReadUnlock);
thr->fast_state.IncrementEpoch();
TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
ReleaseImpl(thr, pc, &s->read_clock);
CHECK_GT(s->recursion, 0);
s->recursion--;
if (s->recursion == 0) {
- StatInc(thr, StatMutexUnlock);
- s->owner_tid = SyncVar::kInvalidTid;
+ s->owner_tid = kInvalidTid;
ReleaseStoreImpl(thr, pc, &s->clock);
} else {
- StatInc(thr, StatMutexRecUnlock);
}
} else if (!s->IsFlagSet(MutexFlagBroken)) {
s->SetFlags(MutexFlagBroken);
}
}
-void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
+void MutexRepair(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
- s->owner_tid = SyncVar::kInvalidTid;
+ s->owner_tid = kInvalidTid;
s->recursion = 0;
s->mtx.Unlock();
}
-void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
+void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
u64 mid = s->GetId();
ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid);
}
-void Acquire(ThreadState *thr, uptr pc, uptr addr) {
+void Acquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
UpdateClockCallback, thr);
}
-void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
+void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
s->mtx.Unlock();
}
-void Release(ThreadState *thr, uptr pc, uptr addr) {
+void Release(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: Release %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
s->mtx.Unlock();
}
-void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
+void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
if (thr->ignore_sync)
return;
return;
thr->clock.set(thr->fast_state.epoch());
thr->clock.acquire(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncAcquire);
}
void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncReleaseStoreAcquire);
}
void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.release(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncRelease);
}
void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncRelease);
}
void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) {
thr->clock.set(thr->fast_state.epoch());
thr->fast_synch_epoch = thr->fast_state.epoch();
thr->clock.acq_rel(&thr->proc()->clock_cache, c);
- StatInc(thr, StatSyncAcquire);
- StatInc(thr, StatSyncRelease);
}
void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
- if (r == 0)
+ if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
return;
ThreadRegistryLock l(ctx->thread_registry);
ScopedReport rep(ReportTypeDeadlock);
static ReportStack *SymbolizeStack(StackTrace trace);
-void TsanCheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2) {
- // There is high probability that interceptors will check-fail as well,
- // on the other hand there is no sense in processing interceptors
- // since we are going to die soon.
- ScopedIgnoreInterceptors ignore;
-#if !SANITIZER_GO
- cur_thread()->ignore_sync++;
- cur_thread()->ignore_reads_and_writes++;
-#endif
- Printf("FATAL: ThreadSanitizer CHECK failed: "
- "%s:%d \"%s\" (0x%zx, 0x%zx)\n",
- file, line, cond, (uptr)v1, (uptr)v2);
- PrintCurrentStackSlow(StackTrace::GetCurrentPc());
- Die();
-}
-
// Can be overriden by an application/test to intercept reports.
#ifdef TSAN_EXTERNAL_HOOKS
bool OnReport(const ReportDesc *rep, bool suppressed);
return stack;
}
+bool ShouldReport(ThreadState *thr, ReportType typ) {
+ // We set thr->suppress_reports in the fork context.
+ // Taking any locking in the fork context can lead to deadlocks.
+ // If any locks are already taken, it's too late to do this check.
+ CheckedMutex::CheckNoLocks();
+ // For the same reason check we didn't lock thread_registry yet.
+ if (SANITIZER_DEBUG)
+ ThreadRegistryLock l(ctx->thread_registry);
+ if (!flags()->report_bugs || thr->suppress_reports)
+ return false;
+ switch (typ) {
+ case ReportTypeSignalUnsafe:
+ return flags()->report_signal_unsafe;
+ case ReportTypeThreadLeak:
+#if !SANITIZER_GO
+ // It's impossible to join phantom threads
+ // in the child after fork.
+ if (ctx->after_multithreaded_fork)
+ return false;
+#endif
+ return flags()->report_thread_leaks;
+ case ReportTypeMutexDestroyLocked:
+ return flags()->report_destroy_locked;
+ default:
+ return true;
+ }
+}
+
ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
ctx->thread_registry->CheckLocked();
void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
rm->stack = SymbolizeStackId(s->creation_stack_id);
}
-u64 ScopedReportBase::AddMutex(u64 id) {
+u64 ScopedReportBase::AddMutex(u64 id) NO_THREAD_SAFETY_ANALYSIS {
u64 uid = 0;
u64 mid = id;
uptr addr = SyncVar::SplitId(id, &uid);
}
bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
- if (!flags()->report_bugs || thr->suppress_reports)
- return false;
+ // These should have been checked in ShouldReport.
+ // It's too late to check them here, we have already taken locks.
+ CHECK(flags()->report_bugs);
+ CHECK(!thr->suppress_reports);
atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
const ReportDesc *rep = srep.GetReport();
CHECK_EQ(thr->current_report, nullptr);
}
void ReportRace(ThreadState *thr) {
- CheckNoLocks(thr);
+ CheckedMutex::CheckNoLocks();
// Symbolizer makes lots of intercepted calls. If we try to process them,
// at best it will cause deadlocks on internal mutexes.
ScopedIgnoreInterceptors ignore;
- if (!flags()->report_bugs)
+ if (!ShouldReport(thr, ReportTypeRace))
return;
if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
return;
}
#endif
- if (!OutputReport(thr, rep))
- return;
-
+ OutputReport(thr, rep);
}
void PrintCurrentStack(ThreadState *thr, uptr pc) {
// However, this solution is not reliable enough, please see dvyukov's comment
// http://reviews.llvm.org/D19148#406208
// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
-ALWAYS_INLINE
-void PrintCurrentStackSlow(uptr pc) {
+ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
#if !SANITIZER_GO
uptr bp = GET_CURRENT_FRAME();
BufferedStackTrace *ptrace =
--- /dev/null
+#include "sanitizer_common/sanitizer_asm.h"
+
+#define CFA_OFFSET 160
+#define R2_REL_OFFSET 16
+#define R3_REL_OFFSET 24
+#define R14_REL_OFFSET 112
+#define R15_REL_OFFSET 120
+#define FRAME_SIZE 160
+
+.text
+
+ASM_HIDDEN(__tsan_setjmp)
+
+.macro intercept symbol, real
+.comm \real, 8, 8
+.globl ASM_SYMBOL_INTERCEPTOR(\symbol)
+ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
+ASM_SYMBOL_INTERCEPTOR(\symbol):
+ CFI_STARTPROC
+ stmg %r2, %r3, R2_REL_OFFSET(%r15)
+ CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
+ CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
+ stmg %r14, %r15, R14_REL_OFFSET(%r15)
+ CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
+ CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
+ aghi %r15, -FRAME_SIZE
+ CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
+ la %r2, FRAME_SIZE(%r15)
+ brasl %r14, ASM_SYMBOL(__tsan_setjmp)
+ lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
+ CFI_RESTORE(%r14)
+ CFI_RESTORE(%r15)
+ CFI_DEF_CFA_OFFSET(CFA_OFFSET)
+ lmg %r2, %r3, R2_REL_OFFSET(%r15)
+ CFI_RESTORE(%r2)
+ CFI_RESTORE(%r3)
+ larl %r1, \real
+ lg %r1, 0(%r1)
+ br %r1
+ CFI_ENDPROC
+ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
+.endm
+
+intercept setjmp, _ZN14__interception11real_setjmpE
+intercept _setjmp, _ZN14__interception12real__setjmpE
+intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
+intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
void ThreadContext::OnCreated(void *arg) {
thr = 0;
- if (tid == 0)
+ if (tid == kMainTid)
return;
OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg);
if (!args->thr) // GCD workers don't have a parent thread.
TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
ReleaseImpl(args->thr, 0, &sync);
creation_stack_id = CurrentStackId(args->thr, args->pc);
- if (reuse_count == 0)
- StatInc(args->thr, StatThreadMaxTid);
}
void ThreadContext::OnReset() {
thr->fast_synch_epoch = epoch0;
AcquireImpl(thr, 0, &sync);
- StatInc(thr, StatSyncAcquire);
sync.Reset(&thr->proc()->clock_cache);
thr->is_inited = true;
DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
PlatformCleanUpThreadState(thr);
#endif
thr->~ThreadState();
-#if TSAN_COLLECT_STATS
- StatAggregate(ctx->stat, thr->stat);
-#endif
thr = 0;
}
#if !SANITIZER_GO
static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) {
- if (tctx->tid == 0) {
+ if (tctx->tid == kMainTid) {
Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
} else {
Printf("ThreadSanitizer: thread T%d %s finished with ignores enabled,"
void ThreadFinalize(ThreadState *thr) {
ThreadCheckIgnore(thr);
#if !SANITIZER_GO
- if (!flags()->report_thread_leaks)
+ if (!ShouldReport(thr, ReportTypeThreadLeak))
return;
ThreadRegistryLock l(ctx->thread_registry);
Vector<ThreadLeak> leaks;
}
int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
- StatInc(thr, StatThreadCreate);
OnCreatedArgs args = { thr, pc };
u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers.
int tid =
ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);
DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
- StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads());
return tid;
}
uptr tls_size = 0;
#if !SANITIZER_GO
if (thread_type != ThreadType::Fiber)
- GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size);
+ GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
+ &tls_size);
- if (tid) {
+ if (tid != kMainTid) {
if (stk_addr && stk_size)
MemoryRangeImitateWrite(thr, /*pc=*/ 1, stk_addr, stk_size);
void ThreadFinish(ThreadState *thr) {
ThreadCheckIgnore(thr);
- StatInc(thr, StatThreadFinish);
if (thr->stk_addr && thr->stk_size)
DontNeedShadowFor(thr->stk_addr, thr->stk_size);
if (thr->tls_addr && thr->tls_size)
int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
ConsumeThreadContext findCtx = {uid, nullptr};
ctx->thread_registry->FindThread(ConsumeThreadByUid, &findCtx);
- int tid = findCtx.tctx ? findCtx.tctx->tid : ThreadRegistry::kUnknownTid;
+ int tid = findCtx.tctx ? findCtx.tctx->tid : kInvalidTid;
DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, tid);
return tid;
}
}
#endif
- StatInc(thr, StatMopRange);
-
if (*shadow_mem == kShadowRodata) {
DCHECK(!is_write);
// Access to .rodata section, no races here.
// Measurements show that it can be 10-20% of all memory accesses.
- StatInc(thr, StatMopRangeRodata);
return;
}
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
uptr top = 0;
uptr bottom = 0;
- if (StackTrace::WillUseFastUnwind(request_fast)) {
- GetThreadStackTopAndBottom(false, &top, &bottom);
- Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
- } else
- Unwind(max_depth, pc, 0, context, 0, 0, false);
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, top, bottom, fast);
}
#endif // SANITIZER_GO
void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);
-SyncVar::SyncVar()
- : mtx(MutexTypeSyncVar, StatMtxSyncVar) {
- Reset(0);
-}
+SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
this->addr = addr;
}
MetaMap::MetaMap()
- : block_alloc_("heap block allocator")
- , sync_alloc_("sync allocator") {
+ : block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
+ sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
atomic_store(&uid_gen_, 0, memory_order_relaxed);
}
uptr metap = (uptr)MemToMeta(p0);
uptr metasz = sz0 / kMetaRatio;
UnmapOrDie((void*)metap, metasz);
- if (!MmapFixedNoReserve(metap, metasz))
+ if (!MmapFixedSuperNoReserve(metap, metasz))
Die();
}
return GetAndLock(0, 0, addr, write_lock, false);
}
-SyncVar* MetaMap::GetAndLock(ThreadState *thr, uptr pc,
- uptr addr, bool write_lock, bool create) {
+SyncVar *MetaMap::GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,
+ bool create) NO_THREAD_SAFETY_ANALYSIS {
u32 *meta = MemToMeta(addr);
u32 idx0 = *meta;
u32 myidx = 0;
#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
#include "tsan_defs.h"
#include "tsan_clock.h"
-#include "tsan_mutex.h"
#include "tsan_dense_alloc.h"
namespace __tsan {
struct SyncVar {
SyncVar();
- static const int kInvalidTid = -1;
-
uptr addr; // overwritten by DenseSlabAlloc freelist
Mutex mtx;
u64 uid; // Globally unique id.
u32 creation_stack_id;
- int owner_tid; // Set only by exclusive owners.
+ u32 owner_tid; // Set only by exclusive owners.
u64 last_lock;
int recursion;
atomic_uint32_t flags;
static const u32 kFlagMask = 3u << 30;
static const u32 kFlagBlock = 1u << 30;
static const u32 kFlagSync = 2u << 30;
- typedef DenseSlabAlloc<MBlock, 1<<16, 1<<12> BlockAlloc;
- typedef DenseSlabAlloc<SyncVar, 1<<16, 1<<10> SyncAlloc;
+ typedef DenseSlabAlloc<MBlock, 1 << 18, 1 << 12, kFlagMask> BlockAlloc;
+ typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
BlockAlloc block_alloc_;
SyncAlloc sync_alloc_;
atomic_uint64_t uid_gen_;
#define TSAN_TRACE_H
#include "tsan_defs.h"
-#include "tsan_mutex.h"
#include "tsan_stack_trace.h"
#include "tsan_mutexset.h"
// CreateThreadContext.
TraceHeader headers[kTraceParts];
- Trace()
- : mtx(MutexTypeTrace, StatMtxTrace) {
- }
+ Trace() : mtx(MutexTypeTrace) {}
};
} // namespace __tsan
// produce sligtly less efficient code.
//===----------------------------------------------------------------------===//
do {
- StatInc(thr, StatShadowProcessed);
const unsigned kAccessSize = 1 << kAccessSizeLog;
u64 *sp = &shadow_mem[idx];
old = LoadShadow(sp);
if (LIKELY(old.IsZero())) {
- StatInc(thr, StatShadowZero);
if (!stored) {
StoreIfNotYetStored(sp, &store_word);
stored = true;
}
// is the memory access equal to the previous?
if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
- StatInc(thr, StatShadowSameSize);
// same thread?
if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
- StatInc(thr, StatShadowSameThread);
if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
StoreIfNotYetStored(sp, &store_word);
stored = true;
}
break;
}
- StatInc(thr, StatShadowAnotherThread);
if (HappensBefore(old, thr)) {
if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
StoreIfNotYetStored(sp, &store_word);
}
// Do the memory access intersect?
if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
- StatInc(thr, StatShadowIntersect);
- if (Shadow::TidsAreEqual(old, cur)) {
- StatInc(thr, StatShadowSameThread);
+ if (Shadow::TidsAreEqual(old, cur))
break;
- }
- StatInc(thr, StatShadowAnotherThread);
if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
break;
if (LIKELY(HappensBefore(old, thr)))
goto RACE;
}
// The accesses do not intersect.
- StatInc(thr, StatShadowNotIntersect);
break;
} while (0);
FOLDER "Compiler-RT Tests")
set(TSAN_UNITTEST_CFLAGS
- ${TSAN_CFLAGS}
${COMPILER_RT_UNITTEST_CFLAGS}
${COMPILER_RT_GTEST_CFLAGS}
-I${COMPILER_RT_SOURCE_DIR}/include
-I${COMPILER_RT_SOURCE_DIR}/lib
-I${COMPILER_RT_SOURCE_DIR}/lib/tsan/rtl
- -DGTEST_HAS_RTTI=0)
+ -DGTEST_HAS_RTTI=0
+ -fno-rtti
+)
+
+if(COMPILER_RT_TSAN_DEBUG_OUTPUT)
+ # Need to match these flags with the runtime.
+ list(APPEND TSAN_UNITTEST_CFLAGS -DTSAN_COLLECT_STATS=1
+ -DTSAN_DEBUG_OUTPUT=2)
+endif()
set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH})
list(APPEND TSAN_RTL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../${header})
endforeach()
+set(TSAN_DEPS gtest tsan)
+# TSan uses C++ standard library headers.
+if (TARGET cxx-headers OR HAVE_LIBCXX)
+ set(TSAN_DEPS cxx-headers)
+endif()
+
# add_tsan_unittest(<name>
# SOURCES <sources list>
# HEADERS <extra headers list>)
SOURCES ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}
RUNTIME ${TSAN_TEST_RUNTIME}
COMPILE_DEPS ${TEST_HEADERS} ${TSAN_RTL_HEADERS}
- DEPS gtest tsan
+ DEPS ${TSAN_DEPS}
CFLAGS ${TSAN_UNITTEST_CFLAGS}
LINK_FLAGS ${LINK_FLAGS})
endforeach()
}
TEST(DISABLED_BENCH, MutexLocal) {
- Mutex m;
+ UserMutex m;
ScopedThread().Create(m);
for (int i = 0; i < 50; i++) {
ScopedThread t;
}
TEST(ThreadSanitizer, WriteThenLockedRead) {
- Mutex m(Mutex::RW);
+ UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
MemLoc l;
}
TEST(ThreadSanitizer, LockedWriteThenRead) {
- Mutex m(Mutex::RW);
+ UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
MemLoc l;
TEST(ThreadSanitizer, BasicMutex) {
ScopedThread t;
- Mutex m;
+ UserMutex m;
t.Create(m);
t.Lock(m);
TEST(ThreadSanitizer, BasicSpinMutex) {
ScopedThread t;
- Mutex m(Mutex::Spin);
+ UserMutex m(UserMutex::Spin);
t.Create(m);
t.Lock(m);
TEST(ThreadSanitizer, BasicRwMutex) {
ScopedThread t;
- Mutex m(Mutex::RW);
+ UserMutex m(UserMutex::RW);
t.Create(m);
t.Lock(m);
}
TEST(ThreadSanitizer, Mutex) {
- Mutex m;
+ UserMutex m;
MainThread t0;
t0.Create(m);
}
TEST(ThreadSanitizer, SpinMutex) {
- Mutex m(Mutex::Spin);
+ UserMutex m(UserMutex::Spin);
MainThread t0;
t0.Create(m);
}
TEST(ThreadSanitizer, RwMutex) {
- Mutex m(Mutex::RW);
+ UserMutex m(UserMutex::RW);
MainThread t0;
t0.Create(m);
TEST(ThreadSanitizer, StaticMutex) {
// Emulates statically initialized mutex.
- Mutex m;
+ UserMutex m;
m.StaticInit();
{
ScopedThread t1, t2;
void operator = (const MemLoc&);
};
-class Mutex {
+class UserMutex {
public:
enum Type {
Normal,
#endif
};
- explicit Mutex(Type type = Normal);
- ~Mutex();
+ explicit UserMutex(Type type = Normal);
+ ~UserMutex();
void Init();
void StaticInit(); // Emulates static initialization (tsan invisible).
bool alive_;
const Type type_;
- Mutex(const Mutex&);
- void operator = (const Mutex&);
+ UserMutex(const UserMutex &);
+ void operator=(const UserMutex &);
};
// A thread is started in CTOR and joined in DTOR.
void Call(void(*pc)());
void Return();
- void Create(const Mutex &m);
- void Destroy(const Mutex &m);
- void Lock(const Mutex &m);
- bool TryLock(const Mutex &m);
- void Unlock(const Mutex &m);
- void ReadLock(const Mutex &m);
- bool TryReadLock(const Mutex &m);
- void ReadUnlock(const Mutex &m);
+ void Create(const UserMutex &m);
+ void Destroy(const UserMutex &m);
+ void Lock(const UserMutex &m);
+ bool TryLock(const UserMutex &m);
+ void Unlock(const UserMutex &m);
+ void ReadLock(const UserMutex &m);
+ bool TryReadLock(const UserMutex &m);
+ void ReadUnlock(const UserMutex &m);
void Memcpy(void *dst, const void *src, int size, bool expect_race = false);
void Memset(void *dst, int val, int size, bool expect_race = false);
MemLoc::~MemLoc() {
}
-Mutex::Mutex(Type type)
- : alive_()
- , type_(type) {
-}
+UserMutex::UserMutex(Type type) : alive_(), type_(type) {}
-Mutex::~Mutex() {
- CHECK(!alive_);
-}
+UserMutex::~UserMutex() { CHECK(!alive_); }
-void Mutex::Init() {
+void UserMutex::Init() {
CHECK(!alive_);
alive_ = true;
if (type_ == Normal)
CHECK(0);
}
-void Mutex::StaticInit() {
+void UserMutex::StaticInit() {
CHECK(!alive_);
CHECK(type_ == Normal);
alive_ = true;
memcpy(mtx_, &tmp, sizeof(tmp));
}
-void Mutex::Destroy() {
+void UserMutex::Destroy() {
CHECK(alive_);
alive_ = false;
if (type_ == Normal)
CHECK_EQ(__interceptor_pthread_rwlock_destroy((pthread_rwlock_t*)mtx_), 0);
}
-void Mutex::Lock() {
+void UserMutex::Lock() {
CHECK(alive_);
if (type_ == Normal)
CHECK_EQ(__interceptor_pthread_mutex_lock((pthread_mutex_t*)mtx_), 0);
CHECK_EQ(__interceptor_pthread_rwlock_wrlock((pthread_rwlock_t*)mtx_), 0);
}
-bool Mutex::TryLock() {
+bool UserMutex::TryLock() {
CHECK(alive_);
if (type_ == Normal)
return __interceptor_pthread_mutex_trylock((pthread_mutex_t*)mtx_) == 0;
return false;
}
-void Mutex::Unlock() {
+void UserMutex::Unlock() {
CHECK(alive_);
if (type_ == Normal)
CHECK_EQ(__interceptor_pthread_mutex_unlock((pthread_mutex_t*)mtx_), 0);
CHECK_EQ(__interceptor_pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0);
}
-void Mutex::ReadLock() {
+void UserMutex::ReadLock() {
CHECK(alive_);
CHECK(type_ == RW);
CHECK_EQ(__interceptor_pthread_rwlock_rdlock((pthread_rwlock_t*)mtx_), 0);
}
-bool Mutex::TryReadLock() {
+bool UserMutex::TryReadLock() {
CHECK(alive_);
CHECK(type_ == RW);
return __interceptor_pthread_rwlock_tryrdlock((pthread_rwlock_t*)mtx_) == 0;
}
-void Mutex::ReadUnlock() {
+void UserMutex::ReadUnlock() {
CHECK(alive_);
CHECK(type_ == RW);
CHECK_EQ(__interceptor_pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0);
__tsan_func_exit();
break;
case Event::MUTEX_CREATE:
- static_cast<Mutex*>(ev->ptr)->Init();
+ static_cast<UserMutex *>(ev->ptr)->Init();
break;
case Event::MUTEX_DESTROY:
- static_cast<Mutex*>(ev->ptr)->Destroy();
+ static_cast<UserMutex *>(ev->ptr)->Destroy();
break;
case Event::MUTEX_LOCK:
- static_cast<Mutex*>(ev->ptr)->Lock();
+ static_cast<UserMutex *>(ev->ptr)->Lock();
break;
case Event::MUTEX_TRYLOCK:
- ev->res = static_cast<Mutex*>(ev->ptr)->TryLock();
+ ev->res = static_cast<UserMutex *>(ev->ptr)->TryLock();
break;
case Event::MUTEX_UNLOCK:
- static_cast<Mutex*>(ev->ptr)->Unlock();
+ static_cast<UserMutex *>(ev->ptr)->Unlock();
break;
case Event::MUTEX_READLOCK:
- static_cast<Mutex*>(ev->ptr)->ReadLock();
+ static_cast<UserMutex *>(ev->ptr)->ReadLock();
break;
case Event::MUTEX_TRYREADLOCK:
- ev->res = static_cast<Mutex*>(ev->ptr)->TryReadLock();
+ ev->res = static_cast<UserMutex *>(ev->ptr)->TryReadLock();
break;
case Event::MUTEX_READUNLOCK:
- static_cast<Mutex*>(ev->ptr)->ReadUnlock();
+ static_cast<UserMutex *>(ev->ptr)->ReadUnlock();
break;
case Event::MEMCPY:
__interceptor_memcpy(ev->ptr, (void*)ev->arg, ev->arg2);
impl_->send(&event);
}
-void ScopedThread::Create(const Mutex &m) {
+void ScopedThread::Create(const UserMutex &m) {
Event event(Event::MUTEX_CREATE, &m);
impl_->send(&event);
}
-void ScopedThread::Destroy(const Mutex &m) {
+void ScopedThread::Destroy(const UserMutex &m) {
Event event(Event::MUTEX_DESTROY, &m);
impl_->send(&event);
}
-void ScopedThread::Lock(const Mutex &m) {
+void ScopedThread::Lock(const UserMutex &m) {
Event event(Event::MUTEX_LOCK, &m);
impl_->send(&event);
}
-bool ScopedThread::TryLock(const Mutex &m) {
+bool ScopedThread::TryLock(const UserMutex &m) {
Event event(Event::MUTEX_TRYLOCK, &m);
impl_->send(&event);
return event.res;
}
-void ScopedThread::Unlock(const Mutex &m) {
+void ScopedThread::Unlock(const UserMutex &m) {
Event event(Event::MUTEX_UNLOCK, &m);
impl_->send(&event);
}
-void ScopedThread::ReadLock(const Mutex &m) {
+void ScopedThread::ReadLock(const UserMutex &m) {
Event event(Event::MUTEX_READLOCK, &m);
impl_->send(&event);
}
-bool ScopedThread::TryReadLock(const Mutex &m) {
+bool ScopedThread::TryReadLock(const UserMutex &m) {
Event event(Event::MUTEX_TRYREADLOCK, &m);
impl_->send(&event);
return event.res;
}
-void ScopedThread::ReadUnlock(const Mutex &m) {
+void ScopedThread::ReadUnlock(const UserMutex &m) {
Event event(Event::MUTEX_READUNLOCK, &m);
impl_->send(&event);
}
set(TSAN_UNIT_TEST_SOURCES
tsan_clock_test.cpp
+ tsan_dense_alloc_test.cpp
tsan_flags_test.cpp
tsan_mman_test.cpp
- tsan_mutex_test.cpp
tsan_shadow_test.cpp
tsan_stack_test.cpp
tsan_sync_test.cpp
namespace __tsan {
TEST(DenseSlabAlloc, Basic) {
- typedef DenseSlabAlloc<int, 128, 128> Alloc;
+ typedef u64 T;
+ typedef DenseSlabAlloc<T, 128, 128> Alloc;
typedef Alloc::Cache Cache;
typedef Alloc::IndexT IndexT;
- const int N = 1000;
+ const T N = 1000;
- Alloc alloc;
+ Alloc alloc("test");
Cache cache;
alloc.InitCache(&cache);
IndexT blocks[N];
for (int ntry = 0; ntry < 3; ntry++) {
- for (int i = 0; i < N; i++) {
+ for (T i = 0; i < N; i++) {
IndexT idx = alloc.Alloc(&cache);
blocks[i] = idx;
EXPECT_NE(idx, 0U);
- int *v = alloc.Map(idx);
+ T *v = alloc.Map(idx);
*v = i;
}
- for (int i = 0; i < N; i++) {
+ for (T i = 0; i < N; i++) {
IndexT idx = blocks[i];
- int *v = alloc.Map(idx);
+ T *v = alloc.Map(idx);
EXPECT_EQ(*v, i);
alloc.Free(&cache, idx);
}
EXPECT_EQ(mb2, (MBlock*)0);
}
-TEST(MetaMap, Sync) {
+TEST(MetaMap, Sync) NO_THREAD_SAFETY_ANALYSIS {
+ // EXPECT can call memset/etc. Disable interceptors to prevent
+ // them from detecting that we exit runtime with mutexes held.
+ ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
MetaMap *m = &ctx->metamap;
u64 block[4] = {}; // fake malloc block
m->OnProcIdle(thr->proc());
}
-TEST(MetaMap, MoveMemory) {
+TEST(MetaMap, MoveMemory) NO_THREAD_SAFETY_ANALYSIS {
+ ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
MetaMap *m = &ctx->metamap;
u64 block1[4] = {}; // fake malloc block
m->FreeRange(thr->proc(), (uptr)&block2[0], 4 * sizeof(u64));
}
-TEST(MetaMap, ResetSync) {
+TEST(MetaMap, ResetSync) NO_THREAD_SAFETY_ANALYSIS {
+ ScopedIgnoreInterceptors ignore;
ThreadState *thr = cur_thread();
MetaMap *m = &ctx->metamap;
u64 block[1] = {}; // fake malloc block
CFLAGS ${UBSAN_CXXFLAGS}
PARENT_TARGET ubsan)
- if (FUCHSIA OR UNIX)
+ if (COMPILER_RT_HAS_VERSION_SCRIPT)
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
add_compiler_rt_object_libraries(RTUbsan_dynamic_version_script_dummy
ARCHS ${UBSAN_SUPPORTED_ARCH}
-Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.ubsan_standalone-dynamic-${arch}.vers)
# The Solaris 11.4 linker supports a subset of GNU ld version scripts,
# but requires a special option to enable it.
- if (OS_NAME MATCHES "SunOS")
+ if (COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
list(APPEND VERSION_SCRIPT_FLAG -Wl,-z,gnu-version-script-compat)
endif()
set_property(SOURCE
}
// Emit data.
- InternalScopedString Buffer(1024);
+ InternalScopedString Buffer;
for (uptr P = Min; P != Max; ++P) {
unsigned char C = *reinterpret_cast<const unsigned char*>(P);
Buffer.append("%s%02x", (P % 8 == 0) ? " " : " ", C);
// All diagnostics should be printed under report mutex.
ScopedReport::CheckLocked();
Decorator Decor;
- InternalScopedString Buffer(1024);
+ InternalScopedString Buffer;
// Prepare a report that a monitor process can inspect.
if (Level == DL_Error) {
ScopedReport::~ScopedReport() {
MaybePrintStackTrace(Opts.pc, Opts.bp);
MaybeReportErrorSummary(SummaryLoc, Type);
+
+ if (common_flags()->print_module_map >= 2)
+ DumpProcessMap();
+
if (flags()->halt_on_error)
Die();
}
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
uptr top = 0;
uptr bottom = 0;
- if (StackTrace::WillUseFastUnwind(request_fast)) {
- GetThreadStackTopAndBottom(false, &top, &bottom);
- Unwind(max_depth, pc, bp, nullptr, top, bottom, true);
- } else
- Unwind(max_depth, pc, bp, context, 0, 0, false);
+ GetThreadStackTopAndBottom(false, &top, &bottom);
+ bool fast = StackTrace::WillUseFastUnwind(request_fast);
+ Unwind(max_depth, pc, bp, context, top, bottom, fast);
}
extern "C" {
namespace __ubsan {
-const char *MaybeCallUbsanDefaultOptions() {
- return (&__ubsan_default_options) ? __ubsan_default_options() : "";
-}
-
static const char *GetFlag(const char *flag) {
// We cannot call getenv() from inside a preinit array initializer
if (SANITIZER_CAN_USE_PREINIT_ARRAY) {
RegisterUbsanFlags(&parser, f);
// Override from user-specified string.
- parser.ParseString(MaybeCallUbsanDefaultOptions());
+ parser.ParseString(__ubsan_default_options());
// Override from environment variable.
parser.ParseStringFromEnv("UBSAN_OPTIONS");
InitializeCommonFlags();
void InitializeFlags();
void RegisterUbsanFlags(FlagParser *parser, Flags *f);
-const char *MaybeCallUbsanDefaultOptions();
-
} // namespace __ubsan
extern "C" {
InitializeSuppressions();
}
+static void UbsanDie() {
+ if (common_flags()->print_module_map >= 1)
+ DumpProcessMap();
+}
+
static void CommonStandaloneInit() {
SanitizerToolName = GetSanititizerToolName();
CacheBinaryName();
AndroidLogInit();
InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
CommonInit();
+
+ // Only add die callback when running in standalone mode to avoid printing
+ // the same information from multiple sanitizers' output
+ AddDieCallback(UbsanDie);
Symbolizer::LateInitialize();
}
UndefinedBehaviorReport::UndefinedBehaviorReport(const char *IssueKind,
Location &Loc,
InternalScopedString &Msg)
- : IssueKind(IssueKind), Loc(Loc), Buffer(Msg.length() + 1) {
+ : IssueKind(IssueKind), Loc(Loc) {
// We have the common sanitizer reporting lock, so it's safe to register a
// new UB report.
RegisterUndefinedBehaviorReport(this);
// Ensure that the first character of the diagnostic text can't start with a
// lowercase letter.
- char FirstChar = Buf.data()[0];
+ char FirstChar = *Buf.data();
if (FirstChar >= 'a' && FirstChar <= 'z')
- Buf.data()[0] = FirstChar - 'a' + 'A';
+ *Buf.data() += 'A' - 'a';
*OutIssueKind = CurrentUBR->IssueKind;
*OutMessage = Buf.data();
// Other platforms should be easy to add, and probably work as-is.
#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \
- defined(__NetBSD__) || defined(__OpenBSD__) || \
- (defined(__sun__) && defined(__svr4__)) || \
- defined(_WIN32) || defined(__Fuchsia__) || defined(__rtems__)
-# define CAN_SANITIZE_UB 1
+ defined(__NetBSD__) || defined(__DragonFly__) || \
+ (defined(__sun__) && defined(__svr4__)) || defined(_WIN32) || \
+ defined(__Fuchsia__)
+#define CAN_SANITIZE_UB 1
#else
# define CAN_SANITIZE_UB 0
#endif
#include "sanitizer_common/sanitizer_platform.h"
#include "ubsan_platform.h"
-#if CAN_SANITIZE_UB && !SANITIZER_WINDOWS
+#if CAN_SANITIZE_UB && !defined(_MSC_VER)
#include "ubsan_type_hash.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_platform.h"
#include "ubsan_platform.h"
-#if CAN_SANITIZE_UB && SANITIZER_WINDOWS
+#if CAN_SANITIZE_UB && defined(_MSC_VER)
#include "ubsan_type_hash.h"
#include "sanitizer_common/sanitizer_common.h"
// to SIntMax.
const unsigned ExtraBits =
sizeof(SIntMax) * 8 - getType().getIntegerBitWidth();
- return SIntMax(Val) << ExtraBits >> ExtraBits;
+ return SIntMax(UIntMax(Val) << ExtraBits) >> ExtraBits;
}
if (getType().getIntegerBitWidth() == 64)
return *reinterpret_cast<s64*>(Val);
static void message(const char *msg) { ubsan_message(msg); }
#else
static void message(const char *msg) {
- write(2, msg, strlen(msg));
+ (void)write(2, msg, strlen(msg));
}
#endif
include_directories(..)
include_directories(../../include)
-set(XRAY_CFLAGS ${SANITIZER_COMMON_CFLAGS})
+set(XRAY_CFLAGS ${COMPILER_RT_COMMON_CFLAGS})
set(XRAY_COMMON_DEFINITIONS XRAY_HAS_EXCEPTIONS=1)
# We don't need RTTI in XRay, so turn that off.
RTSanitizerCommon
RTSanitizerCommonLibc)
+# XRay uses C++ standard library headers.
if (TARGET cxx-headers OR HAVE_LIBCXX)
set(XRAY_DEPS cxx-headers)
endif()
if (NOT APPLE)
# Needed by LLVMSupport.
append_list_if(
- COMPILER_RT_HAS_TERMINFO
+ LLVM_ENABLE_TERMINFO
-l${COMPILER_RT_TERMINFO_LIB} XRAY_UNITTEST_LINK_FLAGS)
if (COMPILER_RT_STANDALONE_BUILD)
#include <fcntl.h>
#include <pthread.h>
#include <sys/stat.h>
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
#include <sys/syscall.h>
#endif
#include <sys/types.h>
return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
}
- s32 Result = XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
- if (!atomic_compare_exchange_strong(&LogFlushStatus, &Result,
- XRayLogFlushStatus::XRAY_LOG_FLUSHING,
- memory_order_release)) {
+ if (atomic_exchange(&LogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHING,
+ memory_order_release) ==
+ XRayLogFlushStatus::XRAY_LOG_FLUSHING) {
if (Verbosity())
- Report("Not flushing log, implementation is still finalizing.\n");
- return static_cast<XRayLogFlushStatus>(Result);
+ Report("Not flushing log, implementation is still flushing.\n");
+ return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
}
if (BQ == nullptr) {
// When |Enable|==false, we set back the first instruction in the sled to be
// B #44
+ uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
if (Enable) {
uint32_t LoTracingHookAddr =
reinterpret_cast<int32_t>(TracingHook) & 0xffff;
(reinterpret_cast<int32_t>(TracingHook) >> 16) & 0xffff;
uint32_t LoFunctionID = FuncId & 0xffff;
uint32_t HiFunctionID = (FuncId >> 16) & 0xffff;
- *reinterpret_cast<uint32_t *>(Sled.Address + 8) = encodeInstruction(
- PatchOpcodes::PO_SW, RegNum::RN_SP, RegNum::RN_RA, 0x4);
- *reinterpret_cast<uint32_t *>(Sled.Address + 12) = encodeInstruction(
- PatchOpcodes::PO_SW, RegNum::RN_SP, RegNum::RN_T9, 0x0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 16) = encodeInstruction(
- PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T9, HiTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 20) = encodeInstruction(
- PatchOpcodes::PO_ORI, RegNum::RN_T9, RegNum::RN_T9, LoTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 24) = encodeInstruction(
- PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T0, HiFunctionID);
- *reinterpret_cast<uint32_t *>(Sled.Address + 28) = encodeSpecialInstruction(
- PatchOpcodes::PO_JALR, RegNum::RN_T9, 0x0, RegNum::RN_RA, 0X0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 32) = encodeInstruction(
- PatchOpcodes::PO_ORI, RegNum::RN_T0, RegNum::RN_T0, LoFunctionID);
- *reinterpret_cast<uint32_t *>(Sled.Address + 36) = encodeInstruction(
- PatchOpcodes::PO_LW, RegNum::RN_SP, RegNum::RN_T9, 0x0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 40) = encodeInstruction(
- PatchOpcodes::PO_LW, RegNum::RN_SP, RegNum::RN_RA, 0x4);
- *reinterpret_cast<uint32_t *>(Sled.Address + 44) = encodeInstruction(
- PatchOpcodes::PO_ADDIU, RegNum::RN_SP, RegNum::RN_SP, 0x8);
+ Address[2] = encodeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_RA, 0x4);
+ Address[3] = encodeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_T9, 0x0);
+ Address[4] = encodeInstruction(PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T9,
+ HiTracingHookAddr);
+ Address[5] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T9,
+ RegNum::RN_T9, LoTracingHookAddr);
+ Address[6] = encodeInstruction(PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T0,
+ HiFunctionID);
+ Address[7] = encodeSpecialInstruction(PatchOpcodes::PO_JALR, RegNum::RN_T9,
+ 0x0, RegNum::RN_RA, 0X0);
+ Address[8] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T0,
+ RegNum::RN_T0, LoFunctionID);
+ Address[9] = encodeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_T9, 0x0);
+ Address[10] = encodeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_RA, 0x4);
+ Address[11] = encodeInstruction(PatchOpcodes::PO_ADDIU, RegNum::RN_SP,
+ RegNum::RN_SP, 0x8);
uint32_t CreateStackSpaceInstr = encodeInstruction(
PatchOpcodes::PO_ADDIU, RegNum::RN_SP, RegNum::RN_SP, 0xFFF8);
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint32_t> *>(Sled.Address),
+ reinterpret_cast<std::atomic<uint32_t> *>(Address),
uint32_t(CreateStackSpaceInstr), std::memory_order_release);
} else {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint32_t> *>(Sled.Address),
+ reinterpret_cast<std::atomic<uint32_t> *>(Address),
uint32_t(PatchOpcodes::PO_B44), std::memory_order_release);
}
return true;
// When |Enable|==false, we set back the first instruction in the sled to be
// B #60
+ uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
if (Enable) {
uint32_t LoTracingHookAddr =
reinterpret_cast<int64_t>(TracingHook) & 0xffff;
(reinterpret_cast<int64_t>(TracingHook) >> 48) & 0xffff;
uint32_t LoFunctionID = FuncId & 0xffff;
uint32_t HiFunctionID = (FuncId >> 16) & 0xffff;
- *reinterpret_cast<uint32_t *>(Sled.Address + 8) = encodeInstruction(
- PatchOpcodes::PO_SD, RegNum::RN_SP, RegNum::RN_RA, 0x8);
- *reinterpret_cast<uint32_t *>(Sled.Address + 12) = encodeInstruction(
- PatchOpcodes::PO_SD, RegNum::RN_SP, RegNum::RN_T9, 0x0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 16) = encodeInstruction(
- PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T9, HighestTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 20) =
- encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T9, RegNum::RN_T9,
- HigherTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 24) = encodeSpecialInstruction(
- PatchOpcodes::PO_DSLL, 0x0, RegNum::RN_T9, RegNum::RN_T9, 0x10);
- *reinterpret_cast<uint32_t *>(Sled.Address + 28) = encodeInstruction(
- PatchOpcodes::PO_ORI, RegNum::RN_T9, RegNum::RN_T9, HiTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 32) = encodeSpecialInstruction(
- PatchOpcodes::PO_DSLL, 0x0, RegNum::RN_T9, RegNum::RN_T9, 0x10);
- *reinterpret_cast<uint32_t *>(Sled.Address + 36) = encodeInstruction(
- PatchOpcodes::PO_ORI, RegNum::RN_T9, RegNum::RN_T9, LoTracingHookAddr);
- *reinterpret_cast<uint32_t *>(Sled.Address + 40) = encodeInstruction(
- PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T0, HiFunctionID);
- *reinterpret_cast<uint32_t *>(Sled.Address + 44) = encodeSpecialInstruction(
- PatchOpcodes::PO_JALR, RegNum::RN_T9, 0x0, RegNum::RN_RA, 0X0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 48) = encodeInstruction(
- PatchOpcodes::PO_ORI, RegNum::RN_T0, RegNum::RN_T0, LoFunctionID);
- *reinterpret_cast<uint32_t *>(Sled.Address + 52) = encodeInstruction(
- PatchOpcodes::PO_LD, RegNum::RN_SP, RegNum::RN_T9, 0x0);
- *reinterpret_cast<uint32_t *>(Sled.Address + 56) = encodeInstruction(
- PatchOpcodes::PO_LD, RegNum::RN_SP, RegNum::RN_RA, 0x8);
- *reinterpret_cast<uint32_t *>(Sled.Address + 60) = encodeInstruction(
- PatchOpcodes::PO_DADDIU, RegNum::RN_SP, RegNum::RN_SP, 0x10);
+ Address[2] = encodeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_RA, 0x8);
+ Address[3] = encodeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_T9, 0x0);
+ Address[4] = encodeInstruction(PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T9,
+ HighestTracingHookAddr);
+ Address[5] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T9,
+ RegNum::RN_T9, HigherTracingHookAddr);
+ Address[6] = encodeSpecialInstruction(PatchOpcodes::PO_DSLL, 0x0,
+ RegNum::RN_T9, RegNum::RN_T9, 0x10);
+ Address[7] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T9,
+ RegNum::RN_T9, HiTracingHookAddr);
+ Address[8] = encodeSpecialInstruction(PatchOpcodes::PO_DSLL, 0x0,
+ RegNum::RN_T9, RegNum::RN_T9, 0x10);
+ Address[9] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T9,
+ RegNum::RN_T9, LoTracingHookAddr);
+ Address[10] = encodeInstruction(PatchOpcodes::PO_LUI, 0x0, RegNum::RN_T0,
+ HiFunctionID);
+ Address[11] = encodeSpecialInstruction(PatchOpcodes::PO_JALR, RegNum::RN_T9,
+ 0x0, RegNum::RN_RA, 0X0);
+ Address[12] = encodeInstruction(PatchOpcodes::PO_ORI, RegNum::RN_T0,
+ RegNum::RN_T0, LoFunctionID);
+ Address[13] = encodeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_T9, 0x0);
+ Address[14] = encodeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_RA, 0x8);
+ Address[15] = encodeInstruction(PatchOpcodes::PO_DADDIU, RegNum::RN_SP,
+ RegNum::RN_SP, 0x10);
uint32_t CreateStackSpace = encodeInstruction(
PatchOpcodes::PO_DADDIU, RegNum::RN_SP, RegNum::RN_SP, 0xfff0);
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint32_t> *>(Sled.Address),
- CreateStackSpace, std::memory_order_release);
+ reinterpret_cast<std::atomic<uint32_t> *>(Address), CreateStackSpace,
+ std::memory_order_release);
} else {
std::atomic_store_explicit(
- reinterpret_cast<std::atomic<uint32_t> *>(Sled.Address),
+ reinterpret_cast<std::atomic<uint32_t> *>(Address),
uint32_t(PatchOpcodes::PO_B60), std::memory_order_release);
}
return true;
#include "../builtins/assembly.h"
#include "../sanitizer_common/sanitizer_asm.h"
+// XRay trampolines which are not produced by intrinsics are not System V AMD64
+// ABI compliant because they are called with a stack that is always misaligned
+// by 8 bytes with respect to a 16 bytes alignment. This is because they are
+// called immediately after the call to, or immediately before returning from,
+// the function being instrumented. This saves space in the patch point, but
+// misaligns the stack by 8 bytes.
+
+.macro ALIGN_STACK_16B
+#if defined(__APPLE__)
+ subq $$8, %rsp
+#else
+ subq $8, %rsp
+#endif
+ CFI_ADJUST_CFA_OFFSET(8)
+.endm
+.macro RESTORE_STACK_ALIGNMENT
+#if defined(__APPLE__)
+ addq $$8, %rsp
+#else
+ addq $8, %rsp
+#endif
+ CFI_ADJUST_CFA_OFFSET(-8)
+.endm
+// This macro should keep the stack aligned to 16 bytes.
.macro SAVE_REGISTERS
pushfq
+ CFI_ADJUST_CFA_OFFSET(8)
subq $240, %rsp
- CFI_DEF_CFA_OFFSET(248)
+ CFI_ADJUST_CFA_OFFSET(240)
movq %rbp, 232(%rsp)
movupd %xmm0, 216(%rsp)
movupd %xmm1, 200(%rsp)
movq %r15, 0(%rsp)
.endm
+// This macro should keep the stack aligned to 16 bytes.
.macro RESTORE_REGISTERS
movq 232(%rsp), %rbp
movupd 216(%rsp), %xmm0
movq 8(%rsp), %r14
movq 0(%rsp), %r15
addq $240, %rsp
+ CFI_ADJUST_CFA_OFFSET(-240)
popfq
- CFI_DEF_CFA_OFFSET(8)
-.endm
-
-.macro ALIGNED_CALL_RAX
- // Call the logging handler, after aligning the stack to a 16-byte boundary.
- // The approach we're taking here uses additional stack space to stash the
- // stack pointer twice before aligning the pointer to 16-bytes. If the stack
- // was 8-byte aligned, it will become 16-byte aligned -- when restoring the
- // pointer, we can always look -8 bytes from the current position to get
- // either of the values we've stashed in the first place.
- pushq %rsp
- pushq (%rsp)
- andq $-0x10, %rsp
- callq *%rax
- movq 8(%rsp), %rsp
+ CFI_ADJUST_CFA_OFFSET(-8)
.endm
.text
# LLVM-MCA-BEGIN __xray_FunctionEntry
ASM_SYMBOL(__xray_FunctionEntry):
CFI_STARTPROC
+ ALIGN_STACK_16B
SAVE_REGISTERS
// This load has to be atomic, it's concurrent with __xray_patch().
// The patched function prologue puts its xray_instr_map index into %r10d.
movl %r10d, %edi
xor %esi,%esi
- ALIGNED_CALL_RAX
+ callq *%rax
.Ltmp0:
RESTORE_REGISTERS
+ RESTORE_STACK_ALIGNMENT
retq
# LLVM-MCA-END
ASM_SIZE(__xray_FunctionEntry)
# LLVM-MCA-BEGIN __xray_FunctionExit
ASM_SYMBOL(__xray_FunctionExit):
CFI_STARTPROC
+ ALIGN_STACK_16B
+
// Save the important registers first. Since we're assuming that this
// function is only jumped into, we only preserve the registers for
// returning.
- subq $56, %rsp
- CFI_DEF_CFA_OFFSET(64)
+ subq $64, %rsp
+ CFI_ADJUST_CFA_OFFSET(64)
movq %rbp, 48(%rsp)
movupd %xmm0, 32(%rsp)
movupd %xmm1, 16(%rsp)
movl %r10d, %edi
movl $1, %esi
- ALIGNED_CALL_RAX
+ callq *%rax
.Ltmp2:
// Restore the important registers.
movupd 16(%rsp), %xmm1
movq 8(%rsp), %rax
movq 0(%rsp), %rdx
- addq $56, %rsp
- CFI_DEF_CFA_OFFSET(8)
+ addq $64, %rsp
+ CFI_ADJUST_CFA_OFFSET(-64)
+
+ RESTORE_STACK_ALIGNMENT
retq
# LLVM-MCA-END
ASM_SIZE(__xray_FunctionExit)
# LLVM-MCA-BEGIN __xray_FunctionTailExit
ASM_SYMBOL(__xray_FunctionTailExit):
CFI_STARTPROC
+ ALIGN_STACK_16B
SAVE_REGISTERS
movq ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)(%rip), %rax
movl %r10d, %edi
movl $2, %esi
-
- ALIGNED_CALL_RAX
+ callq *%rax
.Ltmp4:
RESTORE_REGISTERS
+ RESTORE_STACK_ALIGNMENT
retq
# LLVM-MCA-END
ASM_SIZE(__xray_FunctionTailExit)
# LLVM-MCA-BEGIN __xray_ArgLoggerEntry
ASM_SYMBOL(__xray_ArgLoggerEntry):
CFI_STARTPROC
+ ALIGN_STACK_16B
SAVE_REGISTERS
// Again, these function pointer loads must be atomic; MOV is fine.
// 32-bit function ID becomes the first
movl %r10d, %edi
- ALIGNED_CALL_RAX
+
+ callq *%rax
.Larg1entryFail:
RESTORE_REGISTERS
+ RESTORE_STACK_ALIGNMENT
retq
# LLVM-MCA-END
ASM_SIZE(__xray_ArgLoggerEntry)
testq %rax,%rax
je .LcustomEventCleanup
- ALIGNED_CALL_RAX
+ callq *%rax
.LcustomEventCleanup:
RESTORE_REGISTERS
testq %rax,%rax
je .LtypedEventCleanup
- ALIGNED_CALL_RAX
+ callq *%rax
.LtypedEventCleanup:
RESTORE_REGISTERS
#include <errno.h>
#include <fcntl.h>
#include <iterator>
+#include <new>
#include <stdlib.h>
#include <sys/types.h>
#include <tuple>
#include "xray_defs.h"
#include "xray_interface_internal.h"
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
#include <sys/types.h>
-#if SANITIZER_OPENBSD
-#include <sys/time.h>
-#include <machine/cpu.h>
-#endif
#include <sys/sysctl.h>
#elif SANITIZER_FUCHSIA
#include <zircon/syscalls.h>
}
return TSCFrequency == -1 ? 0 : static_cast<uint64_t>(TSCFrequency);
}
-#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_OPENBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
long long TSCFrequency = -1;
size_t tscfreqsz = sizeof(TSCFrequency);
-#if SANITIZER_OPENBSD
- int Mib[2] = { CTL_MACHDEP, CPU_TSCFREQ };
- if (internal_sysctl(Mib, 2, &TSCFrequency, &tscfreqsz, NULL, 0) != -1) {
-#elif SANITIZER_MAC
+#if SANITIZER_MAC
if (internal_sysctlbyname("machdep.tsc.frequency", &TSCFrequency,
&tscfreqsz, NULL, 0) != -1) {
//===----------------------------------------------------------------------===//
#include <cstdint>
-#include <x86intrin.h>
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "xray_defs.h"
target_include_directories(
stack_trace_compressor_fuzzer PRIVATE ../../lib/)
+ add_executable(options_parser_fuzzer
+ ../../lib/gwp_asan/optional/options_parser.cpp
+ ../../lib/gwp_asan/optional/options_parser.h
+ options_parser_fuzzer.cpp)
+ set_target_properties(
+ options_parser_fuzzer PROPERTIES FOLDER "Fuzzers")
+ target_compile_options(
+ options_parser_fuzzer PRIVATE -fsanitize=fuzzer-no-link)
+ set_target_properties(
+ options_parser_fuzzer PROPERTIES LINK_FLAGS -fsanitize=fuzzer)
+ target_include_directories(
+ options_parser_fuzzer PRIVATE ../../lib/)
+
if (TARGET gwp_asan)
- add_dependencies(gwp_asan stack_trace_compressor_fuzzer)
+ add_dependencies(gwp_asan stack_trace_compressor_fuzzer options_parser_fuzzer)
endif()
endif()
--- /dev/null
+#include <cstddef>
+#include <cstdint>
+
+#include <fuzzer/FuzzedDataProvider.h>
+
+#include "gwp_asan/optional/options_parser.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+ FuzzedDataProvider Fdp(Data, Size);
+ gwp_asan::options::initOptions(Fdp.ConsumeRemainingBytesAsString().c_str());
+ return 0;
+}
import lit.formats
+# Copied from libcxx's config.py
+def get_lit_conf(name, default=None):
+ # Allow overriding on the command line using --param=<name>=<val>
+ val = lit_config.params.get(name, None)
+ if val is None:
+ val = getattr(config, name, None)
+ if val is None:
+ val = default
+ return val
+
+emulator = get_lit_conf('emulator', None)
+
# Setup test format
llvm_build_mode = getattr(config, "llvm_build_mode", "Debug")
-config.test_format = lit.formats.GoogleTest(llvm_build_mode, "Test")
+config.test_format = lit.formats.GoogleTest(llvm_build_mode, "Test", emulator)
# Setup test suffixes.
config.suffixes = []
config.host_os = "@HOST_OS@"
config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
config.gwp_asan = @COMPILER_RT_HAS_GWP_ASAN_PYBOOL@
+config.emulator = "@COMPILER_RT_EMULATOR@"
# LLVM tools dir and build mode can be passed in lit parameters,
# so try to apply substitution.
pcmd("/* TODO */")
} else if (syscall == "dup2") {
pcmd("/* Nothing to do */")
+ } else if (syscall == "getrandom") {
+ pcmd("/* TODO */")
} else if (syscall == "fcntl") {
pcmd("/* Nothing to do */")
} else if (syscall == "compat_50_select") {
pcmd("/* TODO */")
} else if (syscall == "sysarch") {
pcmd("/* TODO */")
+ } else if (syscall == "__futex") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__futex_set_robust_list") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__futex_get_robust_list") {
+ pcmd("/* TODO */")
} else if (syscall == "compat_10_osemsys") {
pcmd("/* TODO */")
} else if (syscall == "compat_10_omsgsys") {
pcmd(" PRE_READ(fhp_, fh_size_);")
pcmd("}")
}
+ } else if (syscall == "__acl_get_link") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_set_link") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_delete_link") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_aclcheck_link") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_get_file") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_set_file") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_get_fd") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_set_fd") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_delete_file") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_delete_fd") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_aclcheck_file") {
+ pcmd("/* TODO */")
+ } else if (syscall == "__acl_aclcheck_fd") {
+ pcmd("/* TODO */")
+ } else if (syscall == "lpathconf") {
+ pcmd("/* TODO */")
} else {
print "Unrecognized syscall: " syscall
abnormal_exit = 1
<a href="http://lists.llvm.org/mailman/listinfo/llvm-dev">llvm-dev</a>
<a href="http://lists.llvm.org/mailman/listinfo/llvm-commits">llvm-commits</a>
<a href="http://llvm.org/bugs/">Bug Reports</a>
- <a href="https://github.com/llvm/llvm-project/tree/master/compiler-rt/">Browse Sources</a>
+ <a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/">Browse Sources</a>
</div>
</div>
projects / openbsd / commitdiff