Skip to content

Commit

Permalink
add fuzz tests
Browse files Browse the repository at this point in the history
  • Loading branch information
biojppm committed May 25, 2024
1 parent f38ef06 commit 90dcf62
Show file tree
Hide file tree
Showing 8 changed files with 335 additions and 8 deletions.
2 changes: 1 addition & 1 deletion ext/c4core
Submodule c4core updated 1 files
+1 −1 cmake
134 changes: 127 additions & 7 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -189,20 +189,17 @@ endif()
option(RYML_TEST_SUITE "Enable cases from yaml-test-suite, https://github.com/yaml/yaml-test-suite." ON)

if(RYML_TEST_SUITE)
set(ed ${CMAKE_CURRENT_BINARY_DIR}/subprojects) # casual ryml extern dir (these projects are not part of ryml and are downloaded and compiled on the fly)

c4_require_subproject(c4log REMOTE
GIT_REPOSITORY https://github.com/biojppm/c4log
GIT_TAG master)

set(tsdir ${ed}/yaml-test-suite)
c4_download_remote_proj(yaml-test-suite suite_dir
GIT_REPOSITORY https://github.com/yaml/yaml-test-suite
GIT_TAG data-2022-01-17)
if(NOT EXISTS ${suite_dir}/229Q)
c4_err("cannot find yaml-test-suite at ${suite_dir} -- was there an error downloading the project?")
endif()

c4_require_subproject(c4log REMOTE
GIT_REPOSITORY https://github.com/biojppm/c4log
GIT_TAG master)

c4_add_executable(ryml-test-suite
SOURCES
test_suite.cpp
Expand Down Expand Up @@ -265,3 +262,126 @@ if(RYML_TEST_SUITE)
ryml_add_test_from_suite(${case})
endforeach()
endif(RYML_TEST_SUITE)


#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------

string(TOUPPER "${CMAKE_BUILD_TYPE}" upper_build_type)
if((upper_build_type STREQUAL FUZZ) OR (upper_build_type STREQUAL COVERAGE))
c4_download_remote_proj(rapidyaml-data rapidyaml_data_dir
GIT_REPOSITORY https://github.com/biojppm/rapidyaml-data
GIT_TAG master)
if(NOT EXISTS ${rapidyaml_data_dir}/fuzz/yaml.dict)
c4_err("cannot find rapidyaml-data at ${rapidyaml_data_dir} -- was there an error downloading the project?")
endif()
#
set(corpus_suite_dir ${rapidyaml_data_dir}/fuzz/yaml_test_suite)
set(corpus_generated_dir ${rapidyaml_data_dir}/fuzz/yaml_generated)
set(corpus_artifacts_dir ${rapidyaml_data_dir}/fuzz/yaml_artifacts)
set(corpus_merged_dir ${rapidyaml_data_dir}/fuzz/yaml_merged)
set(yaml_dict ${rapidyaml_data_dir}/fuzz/yaml.dict)
file(GLOB_RECURSE fuzz_files RELATIVE "${corpus_artifacts_dir}" "${corpus_artifacts_dir}/*")
file(GLOB_RECURSE suite_files RELATIVE "${corpus_suite_dir}" "${corpus_suite_dir}/*")
#
function(ryml_add_fuzz_test name)
c4_add_executable(ryml-test-fuzz-${name}
SOURCES
test_fuzz/test_fuzz_common.hpp
test_fuzz/test_fuzz_${name}.cpp
test_fuzz/test_fuzz_main.cpp
${ARGN}
INC_DIRS ${CMAKE_CURRENT_LIST_DIR}
LIBS ryml c4fs
FOLDER test/fuzz)
function(ryml_add_fuzz_test_file name_ dir file)
string(REPLACE "/" "_" fuzz_name "${file}")
#add_test(NAME ryml-test-fuzz-${name_}-${fuzz_name}
# COMMAND $<TARGET_FILE:ryml-test-fuzz-${name_}> ${dir}/${file})
endfunction()
foreach(fuzz_file ${fuzz_files})
ryml_add_fuzz_test_file(${name} ${corpus_artifacts_dir} ${fuzz_file})
endforeach()
foreach(fuzz_file ${suite_files})
ryml_add_fuzz_test_file(${name} ${corpus_suite_dir} ${fuzz_file})
endforeach()
if(RYML_DBG)
target_compile_definitions(ryml-test-fuzz-${name} PUBLIC RYML_DBG)
endif()
add_dependencies(ryml-test-build ryml-test-fuzz-${name})
endfunction()
ryml_add_fuzz_test(parse_emit)
ryml_add_fuzz_test(events
../test/test_suite/test_suite_event_handler.hpp
../test/test_suite/test_suite_event_handler.cpp)
#
#
# fuzzing libraries:
# https://llvm.org/docs/LibFuzzer.html
# http://lcamtuf.coredump.cx/afl/
# https://github.com/AFLplusplus/AFLplusplus
# https://gitlab.com/akihe/radamsa
#
# actions:
# https://google.github.io/clusterfuzzlite/
# https://github.com/google/oss-fuzz
#
#
# libfuzzer: https://llvm.org/docs/LibFuzzer.html
if((upper_build_type STREQUAL FUZZ) AND ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang"))
option(RYML_FUZZ_LIBFUZZER_MERGE OFF "merge fuzz corpus")
option(RYML_FUZZ_LIBFUZZER_MERGE_RESUME ON "resume merge")
option(RYML_FUZZ_LIBFUZZER_DICT ON "use a yaml dict")
option(RYML_FUZZ_LIBFUZZER_FIXED_SEED ON "use a fixed seed")
set(RYML_FUZZ_LIBFUZZER_OPTIONS "-timeout=5" CACHE STRING "options for libfuzzer https://llvm.org/docs/LibFuzzer.html#id16")
function(ryml_add_libfuzzer_test name)
c4_add_executable(ryml-test-libfuzzer-${name}
SOURCES
test_fuzz/test_fuzz_common.hpp
test_fuzz/test_fuzz_${name}.cpp
${ARGN}
CFLAGS -fsanitize=fuzzer
INC_DIRS ${CMAKE_CURRENT_LIST_DIR}
LIBS ryml -fsanitize=fuzzer
FOLDER test/fuzz)
if(RYML_DBG)
target_compile_definitions(ryml-test-libfuzzer-${name} PUBLIC RYML_DBG)
endif()
add_dependencies(ryml-test-build ryml-test-libfuzzer-${name})
set(corpus_dirs
${corpus_generated_dir} # generated inputs go here
${corpus_artifacts_dir} # corpus with crash/timeout artifacts
${corpus_suite_dir} # corpus with yaml test suite
)
set(opts)
if(RYML_FUZZ_LIBFUZZER_DICT)
list(APPEND opts "-dict=${yaml_dict}")
endif()
file(MAKE_DIRECTORY ${corpus_merged_dir})
if(RYML_FUZZ_LIBFUZZER_MERGE)
list(APPEND opts "-merge=1")
if(RYML_FUZZ_LIBFUZZER_MERGE_RESUME)
list(APPEND opts --merge_control_file=${CMAKE_CURRENT_BINARY_DIR}/fuzz_merge_control_file)
endif()
list(PREPEND corpus_dirs ${corpus_merged_dir})
else()
list(APPEND opts "-merge=0")
endif()
set(cmd $<TARGET_FILE:ryml-test-libfuzzer-${name}> ${opts} ${RYML_FUZZ_LIBFUZZER_OPTIONS} ${corpus_dirs})
add_custom_target(ryml-test-libfuzzer-${name}-run
COMMAND ${cmd}
COMMENT "cd\ ${corpus_artifacts_dir}\ ;\ ${cmd}"
WORKING_DIRECTORY ${corpus_artifacts_dir}) # setting the workdir to this will collect the artifacts in there
if(RYML_FUZZ_LIBFUZZER_MERGE)
add_test(NAME ryml-test-fuzz-libfuzzer-${name}
COMMAND ${cmd}
WORKING_DIRECTORY ${corpus_artifacts_dir}) # setting the workdir to this will collect the artifacts in there
endif()
endfunction()
ryml_add_libfuzzer_test(parse_emit)
ryml_add_libfuzzer_test(events
../test/test_suite/test_suite_event_handler.hpp
../test/test_suite/test_suite_event_handler.cpp)
endif()
endif()
134 changes: 134 additions & 0 deletions test/test_fuzz/test_fuzz_common.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#pragma once
#ifndef TEST_FUZZ_COMMON_H
#define TEST_FUZZ_COMMON_H

#ifdef RYML_SINGLE_HEADER
#include <ryml_all.hpp>
#else
#include <c4/yml/std/std.hpp>
#include <c4/yml/parse.hpp>
#include <c4/yml/emit.hpp>
#include <c4/yml/event_handler_tree.hpp>
#include <c4/yml/parse_engine.def.hpp>
#endif
#include <test_suite/test_suite_events.hpp>
#include <test_suite/test_suite_event_handler.hpp>
#include <cstdio>

#ifdef C4_EXCEPTIONS
#include <stdexcept>
#else
#include <csetjmp>
std::jmp_buf jmp_env = {};
c4::csubstr jmp_msg = {};
#endif


#ifdef RYML_DBG
#define _if_dbg(...) __VA_ARGS__
bool report_errors = true;
#else
#define _if_dbg(...)
bool report_errors = false;
#endif

inline void report_error(const char* msg, size_t length, c4::yml::Location loc, FILE *f)
{
if(!report_errors)
return;
if(!loc.name.empty())
{
fwrite(loc.name.str, 1, loc.name.len, f);
fputc(':', f);
}
fprintf(f, "%zu:", loc.line);
if(loc.col)
fprintf(f, "%zu:", loc.col);
if(loc.offset)
fprintf(f, " (%zuB):", loc.offset);
fputc(' ', f);
fprintf(f, "%.*s\n", static_cast<int>(length), msg);
fflush(f);
}

inline C4_NORETURN void errcallback(const char *msg, size_t msg_len, c4::yml::Location location, void *)
{
report_error(msg, msg_len, location, stderr);
C4_IF_EXCEPTIONS(
throw std::runtime_error({msg, msg_len});
,
jmp_msg.assign(msg, msg_len);
std::longjmp(jmp_env, 1);
);
};

inline c4::yml::Callbacks create_custom_callbacks()
{
c4::set_error_flags(c4::ON_ERROR_CALLBACK);
c4::set_error_callback([](const char *msg, size_t msg_len){
errcallback(msg, msg_len, {}, nullptr);
});
c4::yml::Callbacks callbacks = {};
callbacks.m_error = errcallback;
return callbacks;
}

namespace c4 {
namespace yml {

inline int fuzztest_parse_emit(uint32_t case_number, csubstr src)
{
C4_UNUSED(case_number);
set_callbacks(create_custom_callbacks());
Tree tree(create_custom_callbacks());
bool parse_success = false;
C4_IF_EXCEPTIONS_(try, if(setjmp(jmp_env) == 0))
{
RYML_ASSERT(tree.empty());
_if_dbg(_dbg_printf("in[{}]: [{}]~~~\n{}\n~~~\n", case_number, src.len, src); fflush(NULL));
parse_in_arena(src, &tree);
parse_success = true;
_if_dbg(print_tree("parsed tree", tree));
_if_dbg(_dbg_printf("in[{}]: [{}]~~~\n{}\n~~~\n", case_number, src.len, src); fflush(NULL));
std::string dst = emitrs_yaml<std::string>(tree);
_if_dbg(_dbg_printf("emitted[{}]: [{}]~~~\n{}\n~~~\n", case_number, dst.size(), to_csubstr(dst)); fflush(NULL));
C4_DONT_OPTIMIZE(dst);
C4_DONT_OPTIMIZE(parse_success);
}
C4_IF_EXCEPTIONS_(catch(std::exception const&), else)
{
// if an exception leaks from here, it is likely because of a greedy noexcept
_if_dbg(if(parse_success) print_tree("parsed tree", tree));
return 1;
}
return 0;
}

inline int fuzztest_yaml_events(uint32_t case_number, csubstr src)
{
C4_UNUSED(case_number);
set_callbacks(create_custom_callbacks());
EventHandlerYamlStd::EventSink sink = {};
EventHandlerYamlStd handler(&sink, create_custom_callbacks());
ParseEngine<EventHandlerYamlStd> parser(&handler);
std::string str(src.begin(), src.end());
C4_IF_EXCEPTIONS_(try, if(setjmp(jmp_env) == 0))
{
_if_dbg(_dbg_printf("in[{}]: [{}]~~~\n{}\n~~~\n", case_number, src.len, src); fflush(NULL));
parser.parse_in_place_ev("input", c4::to_substr(str));
_if_dbg(_dbg_printf("evts[{}]: ~~~\n{}\n~~~\n", case_number, sink.result.c_str()); fflush(NULL));
C4_DONT_OPTIMIZE(sink.result);
}
C4_IF_EXCEPTIONS_(catch(std::exception const&), else)
{
// if an exception leaks from here, it is likely because of a greedy noexcept
_if_dbg(fprintf(stdout, "err\n"); fflush(NULL));
return 1;
}
return 0;
}

} // namespace yml
} // namespace c4

#endif /* TEST_FUZZ_COMMON_H */
9 changes: 9 additions & 0 deletions test/test_fuzz/test_fuzz_events.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#include "./test_fuzz_common.hpp"
#include <atomic>

extern "C" int LLVMFuzzerTestOneInput(const uint8_t *str, size_t len)
{
static std::atomic_uint32_t case_number{0};
c4::csubstr src = {reinterpret_cast<const char*>(str), len};
return c4::yml::fuzztest_yaml_events(case_number++, src);
}
16 changes: 16 additions & 0 deletions test/test_fuzz/test_fuzz_main.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#include <stdio.h>
#include <c4/fs/fs.hpp>

extern "C" int LLVMFuzzerTestOneInput(const uint8_t *str, size_t len);

int main(int argc, const char *argv[])
{
if(argc < 2)
return 1;
const char *filename = argv[1];
if(!c4::fs::file_exists(filename))
return 1;
std::string file = c4::fs::file_get_contents<std::string>(filename);
(void)LLVMFuzzerTestOneInput(reinterpret_cast<const uint8_t*>(&file[0]), file.size());
return 0;
}
9 changes: 9 additions & 0 deletions test/test_fuzz/test_fuzz_parse_emit.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#include "./test_fuzz_common.hpp"
#include <atomic>

extern "C" int LLVMFuzzerTestOneInput(const uint8_t *str, size_t len)
{
static std::atomic_uint32_t case_number{0};
c4::csubstr src = {reinterpret_cast<const char*>(str), len};
return c4::yml::fuzztest_parse_emit(case_number++, src);
}
26 changes: 26 additions & 0 deletions test/test_parse_engine_6_qmrk.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,32 @@ ENGINE_TEST(Qmrk3,
___(ps.end_stream());
}

ENGINE_TEST(Qmrk4_0,
("[?baz:,]",
"[{?baz: }]"),
"+STR\n"
"+DOC\n"
"+SEQ []\n"
"+MAP {}\n"
"=VAL :?baz\n"
"=VAL :\n"
"-MAP\n"
"-SEQ\n"
"-DOC\n"
"-STR\n")
{
___(ps.begin_stream());
___(ps.begin_doc());
___(ps.begin_seq_val_flow());
___(ps.begin_map_val_flow());
___(ps.set_key_scalar_plain("?baz"));
___(ps.set_val_scalar_plain({}));
___(ps.end_map());
___(ps.end_seq());
___(ps.end_doc());
___(ps.end_stream());
}

ENGINE_TEST(Qmrk4,
("[ ? an explicit key, ? foo,? bar,?baz:,?bat]",
"[{an explicit key: },{foo: },{bar: },{?baz: },?bat]"),
Expand Down
13 changes: 13 additions & 0 deletions test/test_tag_property.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1041,6 +1041,19 @@ TEST(tags, EHF6)
}
}

TEST(tags, fuzzcrash0)
{
Tree tree;
ExpectError::do_check(&tree, [&]{
parse_in_arena("%TAG !! " "\n"
"})" "\n"
"" "\n"
"!!<" "\n"
,
&tree);
});
}


//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
Expand Down

0 comments on commit 90dcf62

Please sign in to comment.