Patchwork D7564: fuzz: use a more standard approach to allow local builds of fuzzers

login
register
mail settings
Submitter phabricator
Date Dec. 7, 2019, 4:04 a.m.
Message ID <f00c71b101b9dd0adfa5a87f57aa4e3f@localhost.localdomain>
Download mbox | patch
Permalink /patch/43629/
State Not Applicable
Headers show

Comments

phabricator - Dec. 7, 2019, 4:04 a.m.
Closed by commit rHG28a91a581fd9: fuzz: use a more standard approach to allow local builds of fuzzers (authored by durin42).
This revision was automatically updated to reflect the committed changes.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7564?vs=18502&id=18515

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7564/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7564

AFFECTED FILES
  contrib/fuzz/FuzzedDataProvider.h
  contrib/fuzz/Makefile
  contrib/fuzz/bdiff.cc
  contrib/fuzz/jsonescapeu8fast.cc
  contrib/fuzz/standalone_fuzz_target_runner.cc
  contrib/fuzz/xdiff.cc
  tests/test-check-code.t

CHANGE DETAILS




To: durin42, #hg-reviewers, indygreg
Cc: mercurial-devel

Patch

diff --git a/tests/test-check-code.t b/tests/test-check-code.t
--- a/tests/test-check-code.t
+++ b/tests/test-check-code.t
@@ -21,6 +21,8 @@ 
   Skipping contrib/automation/hgautomation/try_server.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/windows.py it has no-che?k-code (glob)
   Skipping contrib/automation/hgautomation/winrm.py it has no-che?k-code (glob)
+  Skipping contrib/fuzz/FuzzedDataProvider.h it has no-che?k-code (glob)
+  Skipping contrib/fuzz/standalone_fuzz_target_runner.cc it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/cli.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/downloads.py it has no-che?k-code (glob)
   Skipping contrib/packaging/hgpackaging/inno.py it has no-che?k-code (glob)
diff --git a/contrib/fuzz/xdiff.cc b/contrib/fuzz/xdiff.cc
--- a/contrib/fuzz/xdiff.cc
+++ b/contrib/fuzz/xdiff.cc
@@ -10,7 +10,7 @@ 
 #include <inttypes.h>
 #include <stdlib.h>
 
-#include <fuzzer/FuzzedDataProvider.h>
+#include "FuzzedDataProvider.h"
 
 extern "C" {
 
diff --git a/contrib/fuzz/standalone_fuzz_target_runner.cc b/contrib/fuzz/standalone_fuzz_target_runner.cc
new file mode 100644
--- /dev/null
+++ b/contrib/fuzz/standalone_fuzz_target_runner.cc
@@ -0,0 +1,41 @@ 
+// Copyright 2017 Google Inc. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+
+// Example of a standalone runner for "fuzz targets".
+// It reads all files passed as parameters and feeds their contents
+// one by one into the fuzz target (LLVMFuzzerTestOneInput).
+// This runner does not do any fuzzing, but allows us to run the fuzz target
+// on the test corpus (e.g. "do_stuff_test_data") or on a single file,
+// e.g. the one that comes from a bug report.
+
+#include <cassert>
+#include <fstream>
+#include <iostream>
+#include <vector>
+
+// Forward declare the "fuzz target" interface.
+// We deliberately keep this inteface simple and header-free.
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+int main(int argc, char **argv)
+{
+	for (int i = 1; i < argc; i++) {
+		std::ifstream in(argv[i]);
+		in.seekg(0, in.end);
+		size_t length = in.tellg();
+		in.seekg(0, in.beg);
+		std::cout << "Reading " << length << " bytes from " << argv[i]
+		          << std::endl;
+		// Allocate exactly length bytes so that we reliably catch
+		// buffer overflows.
+		std::vector<char> bytes(length);
+		in.read(bytes.data(), bytes.size());
+		assert(in);
+		LLVMFuzzerTestOneInput(
+		    reinterpret_cast<const uint8_t *>(bytes.data()),
+		    bytes.size());
+		std::cout << "Execution successful" << std::endl;
+	}
+	return 0;
+}
+// no-check-code since this is from a third party
diff --git a/contrib/fuzz/jsonescapeu8fast.cc b/contrib/fuzz/jsonescapeu8fast.cc
--- a/contrib/fuzz/jsonescapeu8fast.cc
+++ b/contrib/fuzz/jsonescapeu8fast.cc
@@ -5,9 +5,9 @@ 
 
 #include "pyutil.h"
 
-#include <fuzzer/FuzzedDataProvider.h>
 #include <iostream>
 #include <string>
+#include "FuzzedDataProvider.h"
 
 extern "C" {
 
diff --git a/contrib/fuzz/bdiff.cc b/contrib/fuzz/bdiff.cc
--- a/contrib/fuzz/bdiff.cc
+++ b/contrib/fuzz/bdiff.cc
@@ -9,7 +9,7 @@ 
 #include <memory>
 #include <stdlib.h>
 
-#include <fuzzer/FuzzedDataProvider.h>
+#include "FuzzedDataProvider.h"
 
 extern "C" {
 #include "bdiff.h"
diff --git a/contrib/fuzz/Makefile b/contrib/fuzz/Makefile
--- a/contrib/fuzz/Makefile
+++ b/contrib/fuzz/Makefile
@@ -1,7 +1,14 @@ 
 CC = clang
 CXX = clang++
 
-LIB_FUZZING_ENGINE ?= -lFuzzingEngine
+# By default, use our own standalone_fuzz_target_runner.
+# This runner does no fuzzing, but simply executes the inputs
+# provided via parameters.
+# Run e.g. "make all LIB_FUZZING_ENGINE=/path/to/libFuzzer.a"
+# to link the fuzzer(s) against a real fuzzing engine.
+#
+# OSS-Fuzz will define its own value for LIB_FUZZING_ENGINE.
+LIB_FUZZING_ENGINE ?= standalone_fuzz_target_runner.o
 
 PYTHON_CONFIG ?= $$OUT/sanpy/bin/python-config
 
@@ -9,6 +16,8 @@ 
 
 all: bdiff mpatch xdiff
 
+standalone_fuzz_target_runner.o: standalone_fuzz_target_runner.cc
+
 pyutil.o: pyutil.cc pyutil.h
 	$(CXX) $(CXXFLAGS) -g -O1 \
 	  `$(PYTHON_CONFIG) --cflags` \
diff --git a/contrib/fuzz/FuzzedDataProvider.h b/contrib/fuzz/FuzzedDataProvider.h
new file mode 100644
--- /dev/null
+++ b/contrib/fuzz/FuzzedDataProvider.h
@@ -0,0 +1,368 @@ 
+//===- FuzzedDataProvider.h - Utility header for fuzz targets ---*- C++ -* ===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// A single header library providing an utility class to break up an array of
+// bytes. Whenever run on the same input, provides the same output, as long as
+// its methods are called in the same order, with the same arguments.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
+#define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
+
+#include <algorithm>
+#include <climits>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+// In addition to the comments below, the API is also briefly documented at
+// https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider
+class FuzzedDataProvider
+{
+      public:
+	// |data| is an array of length |size| that the FuzzedDataProvider wraps
+	// to provide more granular access. |data| must outlive the
+	// FuzzedDataProvider.
+	FuzzedDataProvider(const uint8_t *data, size_t size)
+	    : data_ptr_(data), remaining_bytes_(size)
+	{
+	}
+	~FuzzedDataProvider() = default;
+
+	// Returns a std::vector containing |num_bytes| of input data. If fewer
+	// than |num_bytes| of data remain, returns a shorter std::vector
+	// containing all of the data that's left. Can be used with any byte
+	// sized type, such as char, unsigned char, uint8_t, etc.
+	template <typename T> std::vector<T> ConsumeBytes(size_t num_bytes)
+	{
+		num_bytes = std::min(num_bytes, remaining_bytes_);
+		return ConsumeBytes<T>(num_bytes, num_bytes);
+	}
+
+	// Similar to |ConsumeBytes|, but also appends the terminator value at
+	// the end of the resulting vector. Useful, when a mutable
+	// null-terminated C-string is needed, for example. But that is a rare
+	// case. Better avoid it, if possible, and prefer using |ConsumeBytes|
+	// or |ConsumeBytesAsString| methods.
+	template <typename T>
+	std::vector<T> ConsumeBytesWithTerminator(size_t num_bytes,
+	                                          T terminator = 0)
+	{
+		num_bytes = std::min(num_bytes, remaining_bytes_);
+		std::vector<T> result =
+		    ConsumeBytes<T>(num_bytes + 1, num_bytes);
+		result.back() = terminator;
+		return result;
+	}
+
+	// Returns a std::string containing |num_bytes| of input data. Using
+	// this and
+	// |.c_str()| on the resulting string is the best way to get an
+	// immutable null-terminated C string. If fewer than |num_bytes| of data
+	// remain, returns a shorter std::string containing all of the data
+	// that's left.
+	std::string ConsumeBytesAsString(size_t num_bytes)
+	{
+		static_assert(sizeof(std::string::value_type) ==
+		                  sizeof(uint8_t),
+		              "ConsumeBytesAsString cannot convert the data to "
+		              "a string.");
+
+		num_bytes = std::min(num_bytes, remaining_bytes_);
+		std::string result(
+		    reinterpret_cast<const std::string::value_type *>(
+		        data_ptr_),
+		    num_bytes);
+		Advance(num_bytes);
+		return result;
+	}
+
+	// Returns a number in the range [min, max] by consuming bytes from the
+	// input data. The value might not be uniformly distributed in the given
+	// range. If there's no input data left, always returns |min|. |min|
+	// must be less than or equal to |max|.
+	template <typename T> T ConsumeIntegralInRange(T min, T max)
+	{
+		static_assert(std::is_integral<T>::value,
+		              "An integral type is required.");
+		static_assert(sizeof(T) <= sizeof(uint64_t),
+		              "Unsupported integral type.");
+
+		if (min > max)
+			abort();
+
+		// Use the biggest type possible to hold the range and the
+		// result.
+		uint64_t range = static_cast<uint64_t>(max) - min;
+		uint64_t result = 0;
+		size_t offset = 0;
+
+		while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 &&
+		       remaining_bytes_ != 0) {
+			// Pull bytes off the end of the seed data.
+			// Experimentally, this seems to allow the fuzzer to
+			// more easily explore the input space. This makes
+			// sense, since it works by modifying inputs that caused
+			// new code to run, and this data is often used to
+			// encode length of data read by |ConsumeBytes|.
+			// Separating out read lengths makes it easier modify
+			// the contents of the data that is actually read.
+			--remaining_bytes_;
+			result =
+			    (result << CHAR_BIT) | data_ptr_[remaining_bytes_];
+			offset += CHAR_BIT;
+		}
+
+		// Avoid division by 0, in case |range + 1| results in overflow.
+		if (range != std::numeric_limits<decltype(range)>::max())
+			result = result % (range + 1);
+
+		return static_cast<T>(min + result);
+	}
+
+	// Returns a std::string of length from 0 to |max_length|. When it runs
+	// out of input data, returns what remains of the input. Designed to be
+	// more stable with respect to a fuzzer inserting characters than just
+	// picking a random length and then consuming that many bytes with
+	// |ConsumeBytes|.
+	std::string ConsumeRandomLengthString(size_t max_length)
+	{
+		// Reads bytes from the start of |data_ptr_|. Maps "\\" to "\",
+		// and maps "\" followed by anything else to the end of the
+		// string. As a result of this logic, a fuzzer can insert
+		// characters into the string, and the string will be lengthened
+		// to include those new characters, resulting in a more stable
+		// fuzzer than picking the length of a string independently from
+		// picking its contents.
+		std::string result;
+
+		// Reserve the anticipated capaticity to prevent several
+		// reallocations.
+		result.reserve(std::min(max_length, remaining_bytes_));
+		for (size_t i = 0; i < max_length && remaining_bytes_ != 0;
+		     ++i) {
+			char next = ConvertUnsignedToSigned<char>(data_ptr_[0]);
+			Advance(1);
+			if (next == '\\' && remaining_bytes_ != 0) {
+				next =
+				    ConvertUnsignedToSigned<char>(data_ptr_[0]);
+				Advance(1);
+				if (next != '\\')
+					break;
+			}
+			result += next;
+		}
+
+		result.shrink_to_fit();
+		return result;
+	}
+
+	// Returns a std::vector containing all remaining bytes of the input
+	// data.
+	template <typename T> std::vector<T> ConsumeRemainingBytes()
+	{
+		return ConsumeBytes<T>(remaining_bytes_);
+	}
+
+	// Returns a std::string containing all remaining bytes of the input
+	// data. Prefer using |ConsumeRemainingBytes| unless you actually need a
+	// std::string object.
+	std::string ConsumeRemainingBytesAsString()
+	{
+		return ConsumeBytesAsString(remaining_bytes_);
+	}
+
+	// Returns a number in the range [Type's min, Type's max]. The value
+	// might not be uniformly distributed in the given range. If there's no
+	// input data left, always returns |min|.
+	template <typename T> T ConsumeIntegral()
+	{
+		return ConsumeIntegralInRange(std::numeric_limits<T>::min(),
+		                              std::numeric_limits<T>::max());
+	}
+
+	// Reads one byte and returns a bool, or false when no data remains.
+	bool ConsumeBool()
+	{
+		return 1 & ConsumeIntegral<uint8_t>();
+	}
+
+	// Returns a copy of the value selected from the given fixed-size
+	// |array|.
+	template <typename T, size_t size>
+	T PickValueInArray(const T (&array)[size])
+	{
+		static_assert(size > 0, "The array must be non empty.");
+		return array[ConsumeIntegralInRange<size_t>(0, size - 1)];
+	}
+
+	template <typename T>
+	T PickValueInArray(std::initializer_list<const T> list)
+	{
+		// TODO(Dor1s): switch to static_assert once C++14 is allowed.
+		if (!list.size())
+			abort();
+
+		return *(list.begin() +
+		         ConsumeIntegralInRange<size_t>(0, list.size() - 1));
+	}
+
+	// Returns an enum value. The enum must start at 0 and be contiguous. It
+	// must also contain |kMaxValue| aliased to its largest (inclusive)
+	// value. Such as: enum class Foo { SomeValue, OtherValue, kMaxValue =
+	// OtherValue };
+	template <typename T> T ConsumeEnum()
+	{
+		static_assert(std::is_enum<T>::value,
+		              "|T| must be an enum type.");
+		return static_cast<T>(ConsumeIntegralInRange<uint32_t>(
+		    0, static_cast<uint32_t>(T::kMaxValue)));
+	}
+
+	// Returns a floating point number in the range [0.0, 1.0]. If there's
+	// no input data left, always returns 0.
+	template <typename T> T ConsumeProbability()
+	{
+		static_assert(std::is_floating_point<T>::value,
+		              "A floating point type is required.");
+
+		// Use different integral types for different floating point
+		// types in order to provide better density of the resulting
+		// values.
+		using IntegralType =
+		    typename std::conditional<(sizeof(T) <= sizeof(uint32_t)),
+		                              uint32_t, uint64_t>::type;
+
+		T result = static_cast<T>(ConsumeIntegral<IntegralType>());
+		result /=
+		    static_cast<T>(std::numeric_limits<IntegralType>::max());
+		return result;
+	}
+
+	// Returns a floating point value in the range [Type's lowest, Type's
+	// max] by consuming bytes from the input data. If there's no input data
+	// left, always returns approximately 0.
+	template <typename T> T ConsumeFloatingPoint()
+	{
+		return ConsumeFloatingPointInRange<T>(
+		    std::numeric_limits<T>::lowest(),
+		    std::numeric_limits<T>::max());
+	}
+
+	// Returns a floating point value in the given range by consuming bytes
+	// from the input data. If there's no input data left, returns |min|.
+	// Note that |min| must be less than or equal to |max|.
+	template <typename T> T ConsumeFloatingPointInRange(T min, T max)
+	{
+		if (min > max)
+			abort();
+
+		T range = .0;
+		T result = min;
+		constexpr T zero(.0);
+		if (max > zero && min < zero &&
+		    max > min + std::numeric_limits<T>::max()) {
+			// The diff |max - min| would overflow the given
+			// floating point type. Use the half of the diff as the
+			// range and consume a bool to decide whether the result
+			// is in the first of the second part of the diff.
+			range = (max / 2.0) - (min / 2.0);
+			if (ConsumeBool()) {
+				result += range;
+			}
+		} else {
+			range = max - min;
+		}
+
+		return result + range * ConsumeProbability<T>();
+	}
+
+	// Reports the remaining bytes available for fuzzed input.
+	size_t remaining_bytes()
+	{
+		return remaining_bytes_;
+	}
+
+      private:
+	FuzzedDataProvider(const FuzzedDataProvider &) = delete;
+	FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete;
+
+	void Advance(size_t num_bytes)
+	{
+		if (num_bytes > remaining_bytes_)
+			abort();
+
+		data_ptr_ += num_bytes;
+		remaining_bytes_ -= num_bytes;
+	}
+
+	template <typename T>
+	std::vector<T> ConsumeBytes(size_t size, size_t num_bytes_to_consume)
+	{
+		static_assert(sizeof(T) == sizeof(uint8_t),
+		              "Incompatible data type.");
+
+		// The point of using the size-based constructor below is to
+		// increase the odds of having a vector object with capacity
+		// being equal to the length. That part is always implementation
+		// specific, but at least both libc++ and libstdc++ allocate the
+		// requested number of bytes in that constructor, which seems to
+		// be a natural choice for other implementations as well. To
+		// increase the odds even more, we also call |shrink_to_fit|
+		// below.
+		std::vector<T> result(size);
+		if (size == 0) {
+			if (num_bytes_to_consume != 0)
+				abort();
+			return result;
+		}
+
+		std::memcpy(result.data(), data_ptr_, num_bytes_to_consume);
+		Advance(num_bytes_to_consume);
+
+		// Even though |shrink_to_fit| is also implementation specific,
+		// we expect it to provide an additional assurance in case
+		// vector's constructor allocated a buffer which is larger than
+		// the actual amount of data we put inside it.
+		result.shrink_to_fit();
+		return result;
+	}
+
+	template <typename TS, typename TU> TS ConvertUnsignedToSigned(TU value)
+	{
+		static_assert(sizeof(TS) == sizeof(TU),
+		              "Incompatible data types.");
+		static_assert(!std::numeric_limits<TU>::is_signed,
+		              "Source type must be unsigned.");
+
+		// TODO(Dor1s): change to `if constexpr` once C++17 becomes
+		// mainstream.
+		if (std::numeric_limits<TS>::is_modulo)
+			return static_cast<TS>(value);
+
+		// Avoid using implementation-defined unsigned to signer
+		// conversions. To learn more, see
+		// https://stackoverflow.com/questions/13150449.
+		if (value <= std::numeric_limits<TS>::max()) {
+			return static_cast<TS>(value);
+		} else {
+			constexpr auto TS_min = std::numeric_limits<TS>::min();
+			return TS_min + static_cast<char>(value - TS_min);
+		}
+	}
+
+	const uint8_t *data_ptr_;
+	size_t remaining_bytes_;
+};
+
+#endif // LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
+// no-check-code since this is from a third party