diff --git a/runtime-core/core-types/decl/array_decl.inl b/runtime-core/core-types/decl/array_decl.inl index 007b536113..e85963cf64 100644 --- a/runtime-core/core-types/decl/array_decl.inl +++ b/runtime-core/core-types/decl/array_decl.inl @@ -26,13 +26,6 @@ struct array_size { inline array_size &min(const array_size &other) noexcept; }; -#if defined(__clang__) || (defined(__GNUC__) && __GNUC__ < 10) -// clang complains about 'flexible array member x of type T[] with non-trivial destruction' -#define KPHP_ARRAY_TAIL_SIZE 0 -#else -// gcc10 complains about out-of-bounds access to an array of zero size -#define KPHP_ARRAY_TAIL_SIZE -#endif namespace dl { template @@ -95,15 +88,17 @@ private: uint32_t string_size{0}; }; - struct array_inner : array_inner_control { + struct alignas(array_bucket) array_inner : array_inner_control { static constexpr uint32_t MAX_HASHTABLE_SIZE = (1 << 26); //empty hash_entry identified by (next == EMPTY_POINTER) static constexpr entry_pointer_type EMPTY_POINTER = 0; - - array_bucket entries[KPHP_ARRAY_TAIL_SIZE]; + static constexpr size_t ENTRIES_OFFSET = sizeof(array_inner); inline bool is_vector() const noexcept __attribute__ ((always_inline)); + inline array_bucket* entries() noexcept __attribute__ ((always_inline)); + inline const array_bucket* entries() const noexcept __attribute__ ((always_inline)); + inline list_hash_entry *get_entry(entry_pointer_type pointer) const __attribute__ ((always_inline)); inline entry_pointer_type get_pointer(list_hash_entry *entry) const __attribute__ ((always_inline)); @@ -111,7 +106,6 @@ private: inline const array_bucket *next(const array_bucket *ptr) const __attribute__ ((always_inline)) ubsan_supp("alignment"); inline const array_bucket *prev(const array_bucket *ptr) const __attribute__ ((always_inline)) ubsan_supp("alignment"); inline const array_bucket *end() const __attribute__ ((always_inline)) ubsan_supp("alignment"); - inline array_bucket *begin() __attribute__ ((always_inline)) ubsan_supp("alignment"); inline array_bucket *next(array_bucket *ptr) __attribute__ ((always_inline)) ubsan_supp("alignment"); inline array_bucket *prev(array_bucket *ptr) __attribute__ ((always_inline)) ubsan_supp("alignment"); diff --git a/runtime-core/core-types/decl/array_iterator.h b/runtime-core/core-types/decl/array_iterator.h index b03b59af9c..f4c6050a9f 100644 --- a/runtime-core/core-types/decl/array_iterator.h +++ b/runtime-core/core-types/decl/array_iterator.h @@ -49,7 +49,7 @@ class array_iterator { inline key_type get_key() const noexcept __attribute__ ((always_inline)) { if (self_->is_vector()) { - return key_type{static_cast(reinterpret_cast(entry_) - reinterpret_cast(self_->entries))}; + return key_type{static_cast(reinterpret_cast(entry_) - reinterpret_cast(self_->entries()))}; } if (is_string_key()) { @@ -112,7 +112,7 @@ class array_iterator { static inline array_iterator make_begin(std::add_const_t &arr) noexcept __attribute__ ((always_inline)) { static_assert(std::is_const{}, "expected to be const"); return arr.is_vector() - ? array_iterator{arr.p, arr.p->entries} + ? array_iterator{arr.p, arr.p->entries()} : array_iterator{arr.p, arr.p->begin()}; } @@ -120,7 +120,7 @@ class array_iterator { static_assert(!std::is_const{}, "expected to be mutable"); if (arr.is_vector()) { arr.mutate_if_vector_shared(); - return array_iterator{arr.p, arr.p->entries}; + return array_iterator{arr.p, arr.p->entries()}; } arr.mutate_if_map_shared(); @@ -129,7 +129,7 @@ class array_iterator { static inline array_iterator make_end(array_type &arr) noexcept __attribute__ ((always_inline)) { return arr.is_vector() - ? array_iterator{arr.p, reinterpret_cast(reinterpret_cast(arr.p->entries) + arr.p->size)} + ? array_iterator{arr.p, reinterpret_cast(reinterpret_cast(arr.p->entries()) + arr.p->size)} : array_iterator{arr.p, arr.p->end()}; } @@ -147,7 +147,7 @@ class array_iterator { return make_end(arr); } - return array_iterator{arr.p, reinterpret_cast(reinterpret_cast(arr.p->entries) + n)}; + return array_iterator{arr.p, reinterpret_cast(reinterpret_cast(arr.p->entries()) + n)}; } if (n < -l / 2) { diff --git a/runtime-core/core-types/definition/array.inl b/runtime-core/core-types/definition/array.inl index 9c8e08f744..55d871568a 100644 --- a/runtime-core/core-types/definition/array.inl +++ b/runtime-core/core-types/definition/array.inl @@ -105,12 +105,16 @@ bool array::is_int_key(const typename array::key_type &key) { template<> inline typename array::array_inner *array::array_inner::empty_array() { - static array_inner_control empty_array{ - true, ExtraRefCnt::for_global_const, -1, + // need this hack because gcc10 and newer complains about + // "array subscript is outside array bounds of array::array_inner" + static array_inner_control empty_array[1]{{ + true, ExtraRefCnt::for_global_const, + -1, {0, 0}, - 0, 2, - }; - return static_cast::array_inner *>(&empty_array); + 0, + 2, + }}; + return static_cast::array_inner *>(&empty_array[0]); } template @@ -133,6 +137,15 @@ bool array::array_inner::is_vector() const noexcept { return is_vector_internal; } +template +typename array::array_bucket *array::array_inner::entries() noexcept { + return reinterpret_cast(reinterpret_cast(this) + ENTRIES_OFFSET); +} + +template +const typename array::array_bucket *array::array_inner::entries() const noexcept { + return reinterpret_cast(reinterpret_cast(this) + ENTRIES_OFFSET); +} template typename array::list_hash_entry *array::array_inner::get_entry(entry_pointer_type pointer) const { @@ -204,12 +217,15 @@ const typename array::array_inner_fields_for_map &array::array_inner::fiel template size_t array::array_inner::sizeof_vector(uint32_t int_size) noexcept { - return sizeof(array_inner) + int_size * sizeof(T); + return sizeof(array_inner) + + int_size * sizeof(T); } template size_t array::array_inner::sizeof_map(uint32_t int_size) noexcept { - return sizeof(array_inner_fields_for_map) + sizeof(array_inner) + int_size * sizeof(array_bucket); + return sizeof(array_inner_fields_for_map) + + sizeof(array_inner) + + int_size * sizeof(array_bucket); } template @@ -272,7 +288,7 @@ void array::array_inner::dispose() { if (ref_cnt <= -1) { if (is_vector()) { for (uint32_t i = 0; i < size; i++) { - ((T *)entries)[i].~T(); + ((T *)entries())[i].~T(); } RuntimeAllocator::current().free_script_memory((void *)this, sizeof_vector(buf_size)); @@ -287,7 +303,7 @@ void array::array_inner::dispose() { } php_assert(this != empty_array()); - auto shifted_this = reinterpret_cast(this) - sizeof(array_inner_fields_for_map); + auto shifted_this = std::launder(reinterpret_cast(this)) - sizeof(array_inner_fields_for_map); RuntimeAllocator::current().free_script_memory(shifted_this, sizeof_map(buf_size)); } } @@ -307,10 +323,10 @@ template inline T &array::array_inner::emplace_back_vector_value(Args &&... args) noexcept { static_assert(std::is_constructible{}, "should be constructible"); php_assert (size < buf_size); - new(&((T *)entries)[size]) T(std::forward(args)...); + new(&((T *)entries())[size]) T(std::forward(args)...); max_key++; size++; - return reinterpret_cast(entries)[max_key]; + return reinterpret_cast(entries())[max_key]; } template @@ -320,19 +336,19 @@ T &array::array_inner::push_back_vector_value(const T &v) { template T &array::array_inner::get_vector_value(int64_t int_key) { - return reinterpret_cast(entries)[int_key]; + return reinterpret_cast(entries())[int_key]; } template const T &array::array_inner::get_vector_value(int64_t int_key) const { - return reinterpret_cast(entries)[int_key]; + return reinterpret_cast(entries())[int_key]; } template template T &array::array_inner::emplace_vector_value(int64_t int_key, Args &&... args) noexcept { static_assert(std::is_constructible{}, "should be constructible"); - reinterpret_cast(entries)[int_key] = T(std::forward(args)...); + reinterpret_cast(entries())[int_key] = T(std::forward(args)...); return get_vector_value(int_key); } @@ -346,24 +362,24 @@ template T &array::array_inner::emplace_int_key_map_value(overwrite_element policy, int64_t int_key, Args &&... args) noexcept { static_assert(std::is_constructible{}, "should be constructible"); uint32_t bucket = choose_bucket(int_key); - while (entries[bucket].next != EMPTY_POINTER && - (entries[bucket].int_key != int_key || !entries[bucket].string_key.is_dummy_string())) { + while (entries()[bucket].next != EMPTY_POINTER && + (entries()[bucket].int_key != int_key || !entries()[bucket].string_key.is_dummy_string())) { if (unlikely(++bucket == buf_size)) { bucket = 0; } } - if (entries[bucket].next == EMPTY_POINTER) { - entries[bucket].int_key = int_key; - new(&entries[bucket].string_key) string{ArrayBucketDummyStrTag{}}; + if (entries()[bucket].next == EMPTY_POINTER) { + entries()[bucket].int_key = int_key; + new(&entries()[bucket].string_key) string{ArrayBucketDummyStrTag{}}; - entries[bucket].prev = end()->prev; - get_entry(end()->prev)->next = get_pointer(&entries[bucket]); + entries()[bucket].prev = end()->prev; + get_entry(end()->prev)->next = get_pointer(&entries()[bucket]); - entries[bucket].next = get_pointer(end()); - end()->prev = get_pointer(&entries[bucket]); + entries()[bucket].next = get_pointer(end()); + end()->prev = get_pointer(&entries()[bucket]); - new(&entries[bucket].value) T(std::forward(args)...); + new(&entries()[bucket].value) T(std::forward(args)...); size++; @@ -371,10 +387,10 @@ T &array::array_inner::emplace_int_key_map_value(overwrite_element policy, in max_key = int_key; } } else if (policy == overwrite_element::YES) { - entries[bucket].value = T(std::forward(args)...); + entries()[bucket].value = T(std::forward(args)...); } - return entries[bucket].value; + return entries()[bucket].value; } template @@ -385,30 +401,30 @@ T &array::array_inner::set_map_value(overwrite_element policy, int64_t int_ke template T array::array_inner::unset_vector_value() { --size; - T res = std::move(reinterpret_cast(entries)[max_key--]); + T res = std::move(reinterpret_cast(entries())[max_key--]); return res; } template T array::array_inner::unset_map_value(int64_t int_key) { uint32_t bucket = choose_bucket(int_key); - while (entries[bucket].next != EMPTY_POINTER && - (entries[bucket].int_key != int_key || !entries[bucket].string_key.is_dummy_string())) { + while (entries()[bucket].next != EMPTY_POINTER && + (entries()[bucket].int_key != int_key || !entries()[bucket].string_key.is_dummy_string())) { if (unlikely (++bucket == buf_size)) { bucket = 0; } } - if (entries[bucket].next != EMPTY_POINTER) { - entries[bucket].int_key = 0; + if (entries()[bucket].next != EMPTY_POINTER) { + entries()[bucket].int_key = 0; - get_entry(entries[bucket].prev)->next = entries[bucket].next; - get_entry(entries[bucket].next)->prev = entries[bucket].prev; + get_entry(entries()[bucket].prev)->next = entries()[bucket].next; + get_entry(entries()[bucket].next)->prev = entries()[bucket].prev; - entries[bucket].next = EMPTY_POINTER; - entries[bucket].prev = EMPTY_POINTER; + entries()[bucket].next = EMPTY_POINTER; + entries()[bucket].prev = EMPTY_POINTER; - T res = std::move(entries[bucket].value); + T res = std::move(entries()[bucket].value); size--; @@ -417,15 +433,15 @@ T array::array_inner::unset_map_value(int64_t int_key) { uint32_t j, rj, ri = bucket; for (j = bucket + 1; 1; j++) { rj = FIXD(j); - if (entries[rj].next == EMPTY_POINTER) { + if (entries()[rj].next == EMPTY_POINTER) { break; } - uint32_t bucket_j = choose_bucket(entries[rj].int_key); + uint32_t bucket_j = choose_bucket(entries()[rj].int_key); uint32_t wnt = FIXU(bucket_j, bucket); if (wnt > j || wnt <= bucket) { - list_hash_entry *ei = entries + ri, *ej = entries + rj; + list_hash_entry *ei = entries() + ri, *ej = entries() + rj; memcpy(ei, ej, sizeof(array_bucket)); ej->next = EMPTY_POINTER; @@ -447,14 +463,14 @@ template template auto &array::array_inner::find_map_entry(S &self, int64_t int_key) noexcept { uint32_t bucket = self.choose_bucket(int_key); - while (self.entries[bucket].next != EMPTY_POINTER && - (self.entries[bucket].int_key != int_key || !self.entries[bucket].string_key.is_dummy_string())) { + while (self.entries()[bucket].next != EMPTY_POINTER && + (self.entries()[bucket].int_key != int_key || !self.entries()[bucket].string_key.is_dummy_string())) { if (unlikely (++bucket == self.buf_size)) { bucket = 0; } } - return self.entries[bucket]; + return self.entries()[bucket]; } template @@ -469,7 +485,7 @@ auto &array::array_inner::find_map_entry(S &self, const char *key, string::si static const auto str_not_eq = [](const string &lhs, const char *rhs, string::size_type rhs_size) { return lhs.size() != rhs_size || string::compare(lhs, rhs, rhs_size) != 0; }; - auto *string_entries = self.entries; + auto *string_entries = self.entries(); uint32_t bucket = self.choose_bucket(precomputed_hash); while (string_entries[bucket].next != EMPTY_POINTER && (string_entries[bucket].int_key != precomputed_hash || string_entries[bucket].string_key.is_dummy_string() || str_not_eq(string_entries[bucket].string_key, key, key_size))) { @@ -503,7 +519,7 @@ template std::pair array::array_inner::emplace_string_key_map_value(overwrite_element policy, int64_t int_key, STRING &&string_key, Args &&... args) noexcept { static_assert(std::is_same, string>::value, "string_key should be string"); - array_bucket *string_entries = entries; + array_bucket *string_entries = entries(); auto &fields = fields_for_map(); uint32_t bucket = choose_bucket(fields, int_key); while (string_entries[bucket].next != EMPTY_POINTER && @@ -544,7 +560,7 @@ T &array::array_inner::set_map_value(overwrite_element policy, int64_t int_ke template T array::array_inner::unset_map_value(const string &string_key, int64_t precomputed_hash) { - array_bucket *string_entries = entries; + array_bucket *string_entries = entries(); auto &fields = fields_for_map(); uint32_t bucket = choose_bucket(fields, precomputed_hash); while (string_entries[bucket].next != EMPTY_POINTER && @@ -663,7 +679,7 @@ bool array::mutate_to_size_if_vector_shared(int64_t int_size) { array_inner *new_array = array_inner::create(int_size, true); const auto size = static_cast(p->size); - T *it = (T *)p->entries; + T *it = (T *)p->entries(); for (uint32_t i = 0; i < size; i++) { new_array->push_back_vector_value(it[i]); @@ -766,7 +782,7 @@ void array::reserve(int64_t int_size, bool make_vector_if_possible) { if (is_vector()) { for (uint32_t it = 0; it != p->size; it++) { - new_array->set_map_value(overwrite_element::YES, it, ((T *)p->entries)[it]); + new_array->set_map_value(overwrite_element::YES, it, ((T *)p->entries())[it]); } php_assert (new_array->max_key == p->max_key); } else { @@ -840,7 +856,7 @@ template void array::convert_to_map() { array_inner *new_array = array_inner::create(p->size + 4, false); - T *elements = reinterpret_cast(p->entries); + T *elements = reinterpret_cast(p->entries()); const bool move_values = p->ref_cnt == 0; if (move_values) { for (uint32_t it = 0; it != p->size; it++) { @@ -870,7 +886,7 @@ void array::copy_from(const array &other) { if (new_array->is_vector()) { uint32_t size = other.p->size; - T1 *it = reinterpret_cast(other.p->entries); + T1 *it = reinterpret_cast(other.p->entries()); for (uint32_t i = 0; i < size; i++) { new_array->push_back_vector_value(convert_to::convert(it[i])); } @@ -907,7 +923,7 @@ void array::move_from(array &&other) noexcept { if (new_array->is_vector()) { uint32_t size = other.p->size; - T1 *it = reinterpret_cast(other.p->entries); + T1 *it = reinterpret_cast(other.p->entries()); for (uint32_t i = 0; i < size; i++) { new_array->emplace_back_vector_value(convert_to::convert(std::move(it[i]))); } @@ -1114,7 +1130,7 @@ T &array::operator[](double double_key) { template T &array::operator[](const const_iterator &it) noexcept { if (it.self_->is_vector()) { - const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries)); + const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries())); return operator[](key); } auto *entry = reinterpret_cast(it.entry_); @@ -1281,7 +1297,7 @@ void array::set_value(const Optional &key, const T &value) noexcep template void array::set_value(const const_iterator &it) noexcept { if (it.self_->is_vector()) { - const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries)); + const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries())); emplace_value(key, *reinterpret_cast(it.entry_)); return; } @@ -1358,7 +1374,7 @@ const T *array::find_value(double double_key) const noexcept { template const T *array::find_value(const const_iterator &it) const noexcept { if (it.self_->is_vector()) { - const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries)); + const auto key = static_cast(reinterpret_cast(it.entry_) - reinterpret_cast(it.self_->entries())); return find_value(key); } else { auto *entry = reinterpret_cast(it.entry_); @@ -1594,7 +1610,7 @@ const array array::operator+(const array &other) const { if (is_vector()) { uint32_t size = p->size; - T *it = (T *)p->entries; + T *it = (T *)p->entries(); if (result.is_vector()) { for (uint32_t i = 0; i < size; i++) { @@ -1617,7 +1633,7 @@ const array array::operator+(const array &other) const { if (other.is_vector()) { uint32_t size = other.p->size; - T *it = (T *)other.p->entries; + T *it = (T *)other.p->entries(); if (result.is_vector()) { for (uint32_t i = p->size; i < size; i++) { @@ -1649,11 +1665,11 @@ array &array::operator+=(const array &other) { if (is_vector()) { if (other.is_vector()) { uint32_t size = other.p->size; - T *it = (T *)other.p->entries; + T *it = (T *)other.p->entries(); if (p->ref_cnt > 0) { uint32_t my_size = p->size; - T *my_it = (T *)p->entries; + T *my_it = (T *)p->entries(); array_inner *new_array = array_inner::create(max(size, my_size), true); @@ -1680,7 +1696,7 @@ array &array::operator+=(const array &other) { return *this; } else { array_inner *new_array = array_inner::create(p->size + other.p->size + 4, false); - T *it = (T *)p->entries; + T *it = (T *)p->entries(); for (uint32_t i = 0; i != p->size; i++) { new_array->set_map_value(overwrite_element::YES, i, it[i]); @@ -1714,7 +1730,7 @@ array &array::operator+=(const array &other) { if (other.is_vector()) { uint32_t size = other.p->size; - T *it = (T *)other.p->entries; + T *it = (T *)other.p->entries(); for (uint32_t i = 0; i < size; i++) { p->set_map_value(overwrite_element::NO, i, it[i]); @@ -1826,7 +1842,7 @@ void array::swap_int_keys(int64_t idx1, int64_t idx2) noexcept { // this function is supposed to be used for vector optimization, else branch is just to be on the safe side if (is_vector() && idx1 >= 0 && idx2 >= 0 && idx1 < p->size && idx2 < p->size) { mutate_if_vector_shared(); - std::swap(reinterpret_cast(p->entries)[idx1], reinterpret_cast(p->entries)[idx2]); + std::swap(reinterpret_cast(p->entries())[idx1], reinterpret_cast(p->entries())[idx2]); } else { if (auto *v1 = find_value(idx1)) { if (auto *v2 = find_value(idx2)) { @@ -1842,7 +1858,7 @@ template void array::fill_vector(int64_t num, const T &value) { php_assert(is_vector() && p->size == 0 && num <= p->buf_size); - std::uninitialized_fill((T *)p->entries, (T *)p->entries + num, value); + std::uninitialized_fill((T *)p->entries(), (T *)p->entries() + num, value); p->max_key = num - 1; p->size = static_cast(num); } @@ -1853,7 +1869,7 @@ void array::memcpy_vector(int64_t num __attribute__((unused)), const void *sr php_assert(is_vector() && p->size == 0 && num <= p->buf_size); mutate_if_vector_shared(); - memcpy(reinterpret_cast(p->entries), src_buf, num * sizeof(T)); + memcpy(reinterpret_cast(p->entries()), src_buf, num * sizeof(T)); p->max_key = num - 1; p->size = static_cast(num); } else { @@ -1894,7 +1910,7 @@ void array::sort(const T1 &compare, bool renumber) { [&compare](const T &lhs, const T &rhs) { return compare(lhs, rhs) > 0; }; - T *begin = reinterpret_cast(p->entries); + T *begin = reinterpret_cast(p->entries()); dl::sort(begin, begin + n, elements_cmp); return; } @@ -1954,7 +1970,7 @@ void array::ksort(const T1 &compare) { keys.p->push_back_vector_value(it->get_key()); } - key_type *keysp = (key_type *)keys.p->entries; + key_type *keysp = (key_type *)keys.p->entries(); dl::sort(keysp, keysp + n, compare); list_hash_entry *prev = (list_hash_entry *)p->end(); @@ -1963,16 +1979,16 @@ void array::ksort(const T1 &compare) { if (is_int_key(keysp[j])) { int64_t int_key = keysp[j].to_int(); uint32_t bucket = p->choose_bucket(int_key); - while (p->entries[bucket].int_key != int_key || !p->entries[bucket].string_key.is_dummy_string()) { + while (p->entries()[bucket].int_key != int_key || !p->entries()[bucket].string_key.is_dummy_string()) { if (unlikely (++bucket == p->buf_size)) { bucket = 0; } } - cur = (list_hash_entry * ) & p->entries[bucket]; + cur = (list_hash_entry * ) &p->entries()[bucket]; } else { string string_key = keysp[j].to_string(); int64_t int_key = string_key.hash(); - array_bucket *string_entries = p->entries; + array_bucket *string_entries = p->entries(); uint32_t bucket = p->choose_bucket(int_key); while ((string_entries[bucket].int_key != int_key || string_entries[bucket].string_key.is_dummy_string() || string_entries[bucket].string_key != string_key)) { if (unlikely (++bucket == p->buf_size)) { @@ -2033,7 +2049,7 @@ T array::shift() { if (is_vector()) { mutate_if_vector_shared(); - T *it = (T *)p->entries; + T *it = (T *)p->entries(); T res = *it; it->~T(); @@ -2076,7 +2092,7 @@ int64_t array::unshift(const T &val) { if (is_vector()) { mutate_if_vector_needs_space(); - T *it = (T *)p->entries; + T *it = (T *)p->entries(); memmove((void *)(it + 1), it, p->size++ * sizeof(T)); p->max_key++; new(it) T(val); @@ -2162,7 +2178,7 @@ void array::force_destroy(ExtraRefCnt expected_ref_cnt) noexcept { template typename array::iterator array::begin_no_mutate() { if (is_vector()) { - return typename array::iterator(p, p->entries); + return typename array::iterator(p, p->entries()); } return typename array::iterator(p, p->begin()); }