Merge pull request #103759 from Ivorforce/zero-constructible

Optimize `Array.resize` by using `memset` (through new `is_zero_constructible` type trait)

Author: Repiteo

core/math/aabb.h

@@ -495,3 +495,6 @@ AABB AABB::quantized(real_t p_unit) const {
 	ret.quantize(p_unit);
 	return ret;
 }
+
+template <>
+struct is_zero_constructible<AABB> : std::true_type {};

core/math/audio_frame.h

@@ -168,3 +168,6 @@ _ALWAYS_INLINE_ AudioFrame operator*(int32_t p_scalar, const AudioFrame &p_frame
 _ALWAYS_INLINE_ AudioFrame operator*(int64_t p_scalar, const AudioFrame &p_frame) {
 	return AudioFrame(p_frame.left * p_scalar, p_frame.right * p_scalar);
 }
+
+template <>
+struct is_zero_constructible<AudioFrame> : std::true_type {};

core/math/face3.h

@@ -236,3 +236,6 @@ bool Face3::intersects_aabb2(const AABB &p_aabb) const {
 	}
 	return true;
 }
+
+template <>
+struct is_zero_constructible<Face3> : std::true_type {};

core/math/plane.h

@@ -132,3 +132,6 @@ bool Plane::operator==(const Plane &p_plane) const {
 bool Plane::operator!=(const Plane &p_plane) const {
 	return normal != p_plane.normal || d != p_plane.d;
 }
+
+template <>
+struct is_zero_constructible<Plane> : std::true_type {};

core/math/rect2.h

@@ -371,3 +371,6 @@ struct [[nodiscard]] Rect2 {
 			size(p_size) {
 	}
 };
+
+template <>
+struct is_zero_constructible<Rect2> : std::true_type {};

core/math/rect2i.h

@@ -236,3 +236,6 @@ struct [[nodiscard]] Rect2i {
 			size(p_size) {
 	}
 };
+
+template <>
+struct is_zero_constructible<Rect2i> : std::true_type {};

core/math/vector2.h

@@ -326,3 +326,6 @@ _FORCE_INLINE_ Vector2 operator*(int64_t p_scalar, const Vector2 &p_vec) {
 
 typedef Vector2 Size2;
 typedef Vector2 Point2;
+
+template <>
+struct is_zero_constructible<Vector2> : std::true_type {};

core/math/vector2i.h

@@ -168,3 +168,6 @@ _FORCE_INLINE_ Vector2i operator*(double p_scalar, const Vector2i &p_vector) {
 
 typedef Vector2i Size2i;
 typedef Vector2i Point2i;
+
+template <>
+struct is_zero_constructible<Vector2i> : std::true_type {};

core/math/vector3.h

@@ -549,3 +549,6 @@ Vector3 Vector3::reflect(const Vector3 &p_normal) const {
 #endif
 	return 2.0f * p_normal * dot(p_normal) - *this;
 }
+
+template <>
+struct is_zero_constructible<Vector3> : std::true_type {};

core/math/vector3i.h

@@ -334,3 +334,6 @@ bool Vector3i::operator>=(const Vector3i &p_v) const {
 void Vector3i::zero() {
 	x = y = z = 0;
 }
+
+template <>
+struct is_zero_constructible<Vector3i> : std::true_type {};

core/math/vector4.h

@@ -302,3 +302,6 @@ _FORCE_INLINE_ Vector4 operator*(int32_t p_scalar, const Vector4 &p_vec) {
 _FORCE_INLINE_ Vector4 operator*(int64_t p_scalar, const Vector4 &p_vec) {
 	return p_vec * p_scalar;
 }
+
+template <>
+struct is_zero_constructible<Vector4> : std::true_type {};

core/math/vector4i.h

@@ -362,3 +362,6 @@ bool Vector4i::operator>=(const Vector4i &p_v) const {
 void Vector4i::zero() {
 	x = y = z = w = 0;
 }
+
+template <>
+struct is_zero_constructible<Vector4i> : std::true_type {};

core/os/memory.h

@@ -34,6 +34,7 @@
 #include "core/templates/safe_refcount.h"
 
 #include <stddef.h>
+#include <cstring>
 #include <new>
 #include <type_traits>
 
@@ -195,6 +196,22 @@ T *memnew_arr_template(size_t p_elements) {
 	return (T *)mem;
 }
 
+// Fast alternative to a loop constructor pattern.
+template <bool p_ensure_zero = false, typename T>
+_FORCE_INLINE_ void memnew_arr_placement(T *p_start, size_t p_num) {
+	if constexpr (std::is_trivially_constructible_v<T> && !p_ensure_zero) {
+		// Don't need to do anything :)
+	} else if constexpr (is_zero_constructible_v<T>) {
+		// Can optimize with memset.
+		memset(static_cast<void *>(p_start), 0, p_num * sizeof(T));
+	} else {
+		// Need to use a for loop.
+		for (size_t i = 0; i < p_num; i++) {
+			memnew_placement(p_start + i, T);
+		}
+	}
+}
+
 /**
  * Wonders of having own array functions, you can actually check the length of
  * an allocated-with memnew_arr() array

core/string/ustring.h

@@ -652,6 +652,10 @@ class String {
 	}
 };
 
+// Zero-constructing String initializes _cowdata.ptr() to nullptr and thus empty.
+template <>
+struct is_zero_constructible<String> : std::true_type {};
+
 bool operator==(const char *p_chr, const String &p_str);
 bool operator==(const wchar_t *p_chr, const String &p_str);
 bool operator!=(const char *p_chr, const String &p_str);

core/templates/cowdata.h

@@ -389,14 +389,7 @@ Error CowData<T>::resize(Size p_size) {
 		}
 
 		// construct the newly created elements
-
-		if constexpr (!std::is_trivially_constructible_v<T>) {
-			for (Size i = *_get_size(); i < p_size; i++) {
-				memnew_placement(&_ptr[i], T);
-			}
-		} else if (p_ensure_zero) {
-			memset((void *)(_ptr + current_size), 0, (p_size - current_size) * sizeof(T));
-		}
+		memnew_arr_placement<p_ensure_zero>(_ptr + current_size, p_size - current_size);
 
 		*_get_size() = p_size;
 
@@ -523,3 +516,7 @@ CowData<T>::CowData(std::initializer_list<T> p_init) {
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
+
+// Zero-constructing CowData initializes _ptr to nullptr (and thus empty).
+template <typename T>
+struct is_zero_constructible<CowData<T>> : std::true_type {};

core/templates/local_vector.h

@@ -160,9 +160,7 @@ class LocalVector {
 				CRASH_COND_MSG(!data, "Out of memory");
 			}
 			if constexpr (!std::is_trivially_constructible_v<T> && !force_trivial) {
-				for (U i = count; i < p_size; i++) {
-					memnew_placement(&data[i], T);
-				}
+				memnew_arr_placement(data + count, p_size - count);
 			}
 			count = p_size;
 		}
@@ -382,3 +380,7 @@ class LocalVector {
 
 template <typename T, typename U = uint32_t, bool force_trivial = false>
 using TightLocalVector = LocalVector<T, U, force_trivial, true>;
+
+// Zero-constructing LocalVector initializes count, capacity and data to 0 and thus empty.
+template <typename T, typename U, bool force_trivial, bool tight>
+struct is_zero_constructible<LocalVector<T, U, force_trivial, tight>> : std::true_type {};

core/templates/vector.h

@@ -335,3 +335,7 @@ void Vector<T>::fill(T p_elem) {
 		p[i] = p_elem;
 	}
 }
+
+// Zero-constructing Vector initializes CowData.ptr() to nullptr and thus empty.
+template <typename T>
+struct is_zero_constructible<Vector<T>> : std::true_type {};

core/typedefs.h

@@ -325,3 +325,21 @@ struct BuildIndexSequence<0, Is...> : IndexSequence<Is...> {};
 #define ____gd_is_defined(arg1_or_junk) __gd_take_second_arg(arg1_or_junk true, false)
 #define ___gd_is_defined(val) ____gd_is_defined(__GDARG_PLACEHOLDER_##val)
 #define GD_IS_DEFINED(x) ___gd_is_defined(x)
+
+// Whether the default value of a type is just all-0 bytes.
+// This can most commonly be exploited by using memset for these types instead of loop-construct.
+// Trivially constructible types are also zero-constructible.
+template <typename T>
+struct is_zero_constructible : std::is_trivially_constructible<T> {};
+
+template <typename T>
+struct is_zero_constructible<const T> : is_zero_constructible<T> {};
+
+template <typename T>
+struct is_zero_constructible<volatile T> : is_zero_constructible<T> {};
+
+template <typename T>
+struct is_zero_constructible<const volatile T> : is_zero_constructible<T> {};
+
+template <typename T>
+inline constexpr bool is_zero_constructible_v = is_zero_constructible<T>::value;

core/variant/variant.h

@@ -1021,3 +1021,7 @@ Array::ConstIterator &Array::ConstIterator::operator--() {
 	element_ptr--;
 	return *this;
 }
+
+// Zero-constructing Variant results in NULL.
+template <>
+struct is_zero_constructible<Variant> : std::true_type {};