| ... | ... | @@ -11,15 +11,10 @@ |
| 11 | 11 | |
| 12 | 12 | #include <__algorithm/comp.h> |
| 13 | 13 | #include <__algorithm/comp_ref_type.h> |
| 14 | | #include <__algorithm/iter_swap.h> |
| 15 | 14 | #include <__algorithm/iterator_operations.h> |
| 16 | 15 | #include <__algorithm/min_element.h> |
| 17 | 16 | #include <__algorithm/partial_sort.h> |
| 18 | 17 | #include <__algorithm/unwrap_iter.h> |
| 19 | | #include <__assert> |
| 20 | | #include <__bit/blsr.h> |
| 21 | | #include <__bit/countl.h> |
| 22 | | #include <__bit/countr.h> |
| 23 | 18 | #include <__config> |
| 24 | 19 | #include <__debug> |
| 25 | 20 | #include <__debug_utils/randomize_range.h> |
| ... | ... | @@ -28,10 +23,11 @@ |
| 28 | 23 | #include <__iterator/iterator_traits.h> |
| 29 | 24 | #include <__memory/destruct_n.h> |
| 30 | 25 | #include <__memory/unique_ptr.h> |
| 31 | | #include <__type_traits/conditional.h> |
| 32 | 26 | #include <__type_traits/is_arithmetic.h> |
| 27 | #include <__type_traits/is_trivially_copy_assignable.h> |
| 28 | #include <__type_traits/is_trivially_copy_constructible.h> |
| 33 | 29 | #include <__utility/move.h> |
| 34 | | #include <__utility/pair.h> |
| 30 | #include <bit> |
| 35 | 31 | #include <climits> |
| 36 | 32 | #include <cstdint> |
| 37 | 33 | |
| ... | ... | @@ -132,7 +128,8 @@ template <class _AlgPolicy, class _Compare, class _ForwardIterator> |
| 132 | 128 | _LIBCPP_HIDE_FROM_ABI |
| 133 | 129 | unsigned __sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3, _ForwardIterator __x4, |
| 134 | 130 | _Compare __c) { |
| 135 | | using _Ops = _IterOps<_AlgPolicy>; |
| 131 | using _Ops = _IterOps<_AlgPolicy>; |
| 132 | |
| 136 | 133 | unsigned __r = std::__sort3<_AlgPolicy, _Compare>(__x1, __x2, __x3, __c); |
| 137 | 134 | if (__c(*__x4, *__x3)) { |
| 138 | 135 | _Ops::iter_swap(__x3, __x4); |
| ... | ... | @@ -187,7 +184,7 @@ _LIBCPP_HIDE_FROM_ABI unsigned __sort5_wrap_policy( |
| 187 | 184 | _Compare __c) { |
| 188 | 185 | using _WrappedComp = typename _WrapAlgPolicy<_AlgPolicy, _Compare>::type; |
| 189 | 186 | _WrappedComp __wrapped_comp(__c); |
| 190 | | return std::__sort5<_WrappedComp, _ForwardIterator>( |
| 187 | return std::__sort5<_WrappedComp>( |
| 191 | 188 | std::move(__x1), std::move(__x2), std::move(__x3), std::move(__x4), std::move(__x5), __wrapped_comp); |
| 192 | 189 | } |
| 193 | 190 | |
| ... | ... | @@ -212,13 +209,6 @@ using __use_branchless_sort = |
| 212 | 209 | integral_constant<bool, __is_cpp17_contiguous_iterator<_Iter>::value && sizeof(_Tp) <= sizeof(void*) && |
| 213 | 210 | is_arithmetic<_Tp>::value && __is_simple_comparator<_Compare>::value>; |
| 214 | 211 | |
| 215 | | namespace __detail { |
| 216 | | |
| 217 | | // Size in bits for the bitset in use. |
| 218 | | enum { __block_size = sizeof(uint64_t) * 8 }; |
| 219 | | |
| 220 | | } // namespace __detail |
| 221 | | |
| 222 | 212 | // Ensures that __c(*__x, *__y) is true by swapping *__x and *__y if necessary. |
| 223 | 213 | template <class _Compare, class _RandomAccessIterator> |
| 224 | 214 | inline _LIBCPP_HIDE_FROM_ABI void __cond_swap(_RandomAccessIterator __x, _RandomAccessIterator __y, _Compare __c) { |
| ... | ... | @@ -278,15 +268,10 @@ __sort4_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, |
| 278 | 268 | std::__sort4<_AlgPolicy, _Compare>(__x1, __x2, __x3, __x4, __c); |
| 279 | 269 | } |
| 280 | 270 | |
| 281 | | template <class _AlgPolicy, class _Compare, class _RandomAccessIterator> |
| 271 | template <class, class _Compare, class _RandomAccessIterator> |
| 282 | 272 | inline _LIBCPP_HIDE_FROM_ABI __enable_if_t<__use_branchless_sort<_Compare, _RandomAccessIterator>::value, void> |
| 283 | | __sort5_maybe_branchless( |
| 284 | | _RandomAccessIterator __x1, |
| 285 | | _RandomAccessIterator __x2, |
| 286 | | _RandomAccessIterator __x3, |
| 287 | | _RandomAccessIterator __x4, |
| 288 | | _RandomAccessIterator __x5, |
| 289 | | _Compare __c) { |
| 273 | __sort5_maybe_branchless(_RandomAccessIterator __x1, _RandomAccessIterator __x2, _RandomAccessIterator __x3, |
| 274 | _RandomAccessIterator __x4, _RandomAccessIterator __x5, _Compare __c) { |
| 290 | 275 | std::__cond_swap<_Compare>(__x1, __x2, __c); |
| 291 | 276 | std::__cond_swap<_Compare>(__x4, __x5, __c); |
| 292 | 277 | std::__partially_sorted_swap<_Compare>(__x3, __x4, __x5, __c); |
| ... | ... | @@ -315,48 +300,34 @@ _LIBCPP_CONSTEXPR_SINCE_CXX14 void __selection_sort(_BidirectionalIterator __fir |
| 315 | 300 | } |
| 316 | 301 | } |
| 317 | 302 | |
| 318 | | // Sort the iterator range [__first, __last) using the comparator __comp using |
| 319 | | // the insertion sort algorithm. |
| 320 | 303 | template <class _AlgPolicy, class _Compare, class _BidirectionalIterator> |
| 321 | 304 | _LIBCPP_HIDE_FROM_ABI |
| 322 | 305 | void __insertion_sort(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp) { |
| 323 | 306 | using _Ops = _IterOps<_AlgPolicy>; |
| 324 | 307 | |
| 325 | 308 | typedef typename iterator_traits<_BidirectionalIterator>::value_type value_type; |
| 326 | | if (__first == __last) |
| 327 | | return; |
| 328 | | _BidirectionalIterator __i = __first; |
| 329 | | for (++__i; __i != __last; ++__i) { |
| 330 | | _BidirectionalIterator __j = __i; |
| 331 | | --__j; |
| 332 | | if (__comp(*__i, *__j)) { |
| 333 | | value_type __t(_Ops::__iter_move(__i)); |
| 334 | | _BidirectionalIterator __k = __j; |
| 335 | | __j = __i; |
| 336 | | do { |
| 309 | if (__first != __last) { |
| 310 | _BidirectionalIterator __i = __first; |
| 311 | for (++__i; __i != __last; ++__i) { |
| 312 | _BidirectionalIterator __j = __i; |
| 313 | value_type __t(_Ops::__iter_move(__j)); |
| 314 | for (_BidirectionalIterator __k = __i; __k != __first && __comp(__t, *--__k); --__j) |
| 337 | 315 | *__j = _Ops::__iter_move(__k); |
| 338 | | __j = __k; |
| 339 | | } while (__j != __first && __comp(__t, *--__k)); |
| 340 | 316 | *__j = std::move(__t); |
| 341 | 317 | } |
| 342 | 318 | } |
| 343 | 319 | } |
| 344 | 320 | |
| 345 | | // Sort the iterator range [__first, __last) using the comparator __comp using |
| 346 | | // the insertion sort algorithm. Insertion sort has two loops, outer and inner. |
| 347 | | // The implementation below has not bounds check (unguarded) for the inner loop. |
| 348 | | // Assumes that there is an element in the position (__first - 1) and that each |
| 349 | | // element in the input range is greater or equal to the element at __first - 1. |
| 350 | 321 | template <class _AlgPolicy, class _Compare, class _RandomAccessIterator> |
| 351 | | _LIBCPP_HIDE_FROM_ABI void |
| 352 | | __insertion_sort_unguarded(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { |
| 322 | _LIBCPP_HIDE_FROM_ABI |
| 323 | void __insertion_sort_3(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { |
| 353 | 324 | using _Ops = _IterOps<_AlgPolicy>; |
| 325 | |
| 354 | 326 | typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 355 | 327 | typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; |
| 356 | | if (__first == __last) |
| 357 | | return; |
| 358 | | for (_RandomAccessIterator __i = __first + difference_type(1); __i != __last; ++__i) { |
| 359 | | _RandomAccessIterator __j = __i - difference_type(1); |
| 328 | _RandomAccessIterator __j = __first + difference_type(2); |
| 329 | std::__sort3_maybe_branchless<_AlgPolicy, _Compare>(__first, __first + difference_type(1), __j, __comp); |
| 330 | for (_RandomAccessIterator __i = __j + difference_type(1); __i != __last; ++__i) { |
| 360 | 331 | if (__comp(*__i, *__j)) { |
| 361 | 332 | value_type __t(_Ops::__iter_move(__i)); |
| 362 | 333 | _RandomAccessIterator __k = __j; |
| ... | ... | @@ -364,9 +335,10 @@ __insertion_sort_unguarded(_RandomAccessIterator __first, _RandomAccessIterator |
| 364 | 335 | do { |
| 365 | 336 | *__j = _Ops::__iter_move(__k); |
| 366 | 337 | __j = __k; |
| 367 | | } while (__comp(__t, *--__k)); // No need for bounds check due to the assumption stated above. |
| 338 | } while (__j != __first && __comp(__t, *--__k)); |
| 368 | 339 | *__j = std::move(__t); |
| 369 | 340 | } |
| 341 | __j = __i; |
| 370 | 342 | } |
| 371 | 343 | } |
| 372 | 344 | |
| ... | ... | @@ -387,7 +359,7 @@ _LIBCPP_HIDDEN bool __insertion_sort_incomplete( |
| 387 | 359 | return true; |
| 388 | 360 | case 2: |
| 389 | 361 | if (__comp(*--__last, *__first)) |
| 390 | | _Ops::iter_swap(__first, __last); |
| 362 | _IterOps<_AlgPolicy>::iter_swap(__first, __last); |
| 391 | 363 | return true; |
| 392 | 364 | case 3: |
| 393 | 365 | std::__sort3_maybe_branchless<_AlgPolicy, _Compare>(__first, __first + difference_type(1), --__last, __comp); |
| ... | ... | @@ -456,336 +428,17 @@ void __insertion_sort_move(_BidirectionalIterator __first1, _BidirectionalIterat |
| 456 | 428 | } |
| 457 | 429 | } |
| 458 | 430 | |
| 459 | | template <class _AlgPolicy, class _RandomAccessIterator> |
| 460 | | inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos( |
| 461 | | _RandomAccessIterator __first, _RandomAccessIterator __last, uint64_t& __left_bitset, uint64_t& __right_bitset) { |
| 462 | | using _Ops = _IterOps<_AlgPolicy>; |
| 463 | | typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 464 | | // Swap one pair on each iteration as long as both bitsets have at least one |
| 465 | | // element for swapping. |
| 466 | | while (__left_bitset != 0 && __right_bitset != 0) { |
| 467 | | difference_type tz_left = __libcpp_ctz(__left_bitset); |
| 468 | | __left_bitset = __libcpp_blsr(__left_bitset); |
| 469 | | difference_type tz_right = __libcpp_ctz(__right_bitset); |
| 470 | | __right_bitset = __libcpp_blsr(__right_bitset); |
| 471 | | _Ops::iter_swap(__first + tz_left, __last - tz_right); |
| 472 | | } |
| 473 | | } |
| 474 | | |
| 475 | | template <class _Compare, |
| 476 | | class _RandomAccessIterator, |
| 477 | | class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type> |
| 478 | | inline _LIBCPP_HIDE_FROM_ABI void |
| 479 | | __populate_left_bitset(_RandomAccessIterator __first, _Compare __comp, _ValueType& __pivot, uint64_t& __left_bitset) { |
| 480 | | // Possible vectorization. With a proper "-march" flag, the following loop |
| 481 | | // will be compiled into a set of SIMD instructions. |
| 482 | | _RandomAccessIterator __iter = __first; |
| 483 | | for (int __j = 0; __j < __detail::__block_size;) { |
| 484 | | bool __comp_result = !__comp(*__iter, __pivot); |
| 485 | | __left_bitset |= (static_cast<uint64_t>(__comp_result) << __j); |
| 486 | | __j++; |
| 487 | | ++__iter; |
| 488 | | } |
| 489 | | } |
| 490 | | |
| 491 | | template <class _Compare, |
| 492 | | class _RandomAccessIterator, |
| 493 | | class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type> |
| 494 | | inline _LIBCPP_HIDE_FROM_ABI void |
| 495 | | __populate_right_bitset(_RandomAccessIterator __lm1, _Compare __comp, _ValueType& __pivot, uint64_t& __right_bitset) { |
| 496 | | // Possible vectorization. With a proper "-march" flag, the following loop |
| 497 | | // will be compiled into a set of SIMD instructions. |
| 498 | | _RandomAccessIterator __iter = __lm1; |
| 499 | | for (int __j = 0; __j < __detail::__block_size;) { |
| 500 | | bool __comp_result = __comp(*__iter, __pivot); |
| 501 | | __right_bitset |= (static_cast<uint64_t>(__comp_result) << __j); |
| 502 | | __j++; |
| 503 | | --__iter; |
| 504 | | } |
| 505 | | } |
| 506 | | |
| 507 | | template <class _AlgPolicy, |
| 508 | | class _Compare, |
| 509 | | class _RandomAccessIterator, |
| 510 | | class _ValueType = typename iterator_traits<_RandomAccessIterator>::value_type> |
| 511 | | inline _LIBCPP_HIDE_FROM_ABI void __bitset_partition_partial_blocks( |
| 512 | | _RandomAccessIterator& __first, |
| 513 | | _RandomAccessIterator& __lm1, |
| 514 | | _Compare __comp, |
| 515 | | _ValueType& __pivot, |
| 516 | | uint64_t& __left_bitset, |
| 517 | | uint64_t& __right_bitset) { |
| 518 | | typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 519 | | difference_type __remaining_len = __lm1 - __first + 1; |
| 520 | | difference_type __l_size; |
| 521 | | difference_type __r_size; |
| 522 | | if (__left_bitset == 0 && __right_bitset == 0) { |
| 523 | | __l_size = __remaining_len / 2; |
| 524 | | __r_size = __remaining_len - __l_size; |
| 525 | | } else if (__left_bitset == 0) { |
| 526 | | // We know at least one side is a full block. |
| 527 | | __l_size = __remaining_len - __detail::__block_size; |
| 528 | | __r_size = __detail::__block_size; |
| 529 | | } else { // if (__right_bitset == 0) |
| 530 | | __l_size = __detail::__block_size; |
| 531 | | __r_size = __remaining_len - __detail::__block_size; |
| 532 | | } |
| 533 | | // Record the comparison outcomes for the elements currently on the left side. |
| 534 | | if (__left_bitset == 0) { |
| 535 | | _RandomAccessIterator __iter = __first; |
| 536 | | for (int j = 0; j < __l_size; j++) { |
| 537 | | bool __comp_result = !__comp(*__iter, __pivot); |
| 538 | | __left_bitset |= (static_cast<uint64_t>(__comp_result) << j); |
| 539 | | ++__iter; |
| 540 | | } |
| 541 | | } |
| 542 | | // Record the comparison outcomes for the elements currently on the right |
| 543 | | // side. |
| 544 | | if (__right_bitset == 0) { |
| 545 | | _RandomAccessIterator __iter = __lm1; |
| 546 | | for (int j = 0; j < __r_size; j++) { |
| 547 | | bool __comp_result = __comp(*__iter, __pivot); |
| 548 | | __right_bitset |= (static_cast<uint64_t>(__comp_result) << j); |
| 549 | | --__iter; |
| 550 | | } |
| 551 | | } |
| 552 | | std::__swap_bitmap_pos<_AlgPolicy, _RandomAccessIterator>(__first, __lm1, __left_bitset, __right_bitset); |
| 553 | | __first += (__left_bitset == 0) ? __l_size : 0; |
| 554 | | __lm1 -= (__right_bitset == 0) ? __r_size : 0; |
| 555 | | } |
| 556 | | |
| 557 | | template <class _AlgPolicy, class _RandomAccessIterator> |
| 558 | | inline _LIBCPP_HIDE_FROM_ABI void __swap_bitmap_pos_within( |
| 559 | | _RandomAccessIterator& __first, _RandomAccessIterator& __lm1, uint64_t& __left_bitset, uint64_t& __right_bitset) { |
| 560 | | using _Ops = _IterOps<_AlgPolicy>; |
| 561 | | typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 562 | | if (__left_bitset) { |
| 563 | | // Swap within the left side. Need to find set positions in the reverse |
| 564 | | // order. |
| 565 | | while (__left_bitset != 0) { |
| 566 | | difference_type __tz_left = __detail::__block_size - 1 - __libcpp_clz(__left_bitset); |
| 567 | | __left_bitset &= (static_cast<uint64_t>(1) << __tz_left) - 1; |
| 568 | | _RandomAccessIterator it = __first + __tz_left; |
| 569 | | if (it != __lm1) { |
| 570 | | _Ops::iter_swap(it, __lm1); |
| 571 | | } |
| 572 | | --__lm1; |
| 573 | | } |
| 574 | | __first = __lm1 + difference_type(1); |
| 575 | | } else if (__right_bitset) { |
| 576 | | // Swap within the right side. Need to find set positions in the reverse |
| 577 | | // order. |
| 578 | | while (__right_bitset != 0) { |
| 579 | | difference_type __tz_right = __detail::__block_size - 1 - __libcpp_clz(__right_bitset); |
| 580 | | __right_bitset &= (static_cast<uint64_t>(1) << __tz_right) - 1; |
| 581 | | _RandomAccessIterator it = __lm1 - __tz_right; |
| 582 | | if (it != __first) { |
| 583 | | _Ops::iter_swap(it, __first); |
| 584 | | } |
| 585 | | ++__first; |
| 586 | | } |
| 587 | | } |
| 588 | | } |
| 589 | | |
| 590 | | // Partition [__first, __last) using the comparator __comp. *__first has the |
| 591 | | // chosen pivot. Elements that are equivalent are kept to the left of the |
| 592 | | // pivot. Returns the iterator for the pivot and a bool value which is true if |
| 593 | | // the provided range is already sorted, false otherwise. We assume that the |
| 594 | | // length of the range is at least three elements. |
| 595 | | // |
| 596 | | // __bitset_partition uses bitsets for storing outcomes of the comparisons |
| 597 | | // between the pivot and other elements. |
| 598 | | template <class _AlgPolicy, class _RandomAccessIterator, class _Compare> |
| 599 | | _LIBCPP_HIDE_FROM_ABI std::pair<_RandomAccessIterator, bool> |
| 600 | | __bitset_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { |
| 601 | | using _Ops = _IterOps<_AlgPolicy>; |
| 602 | | typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; |
| 603 | | typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 604 | | _LIBCPP_ASSERT(__last - __first >= difference_type(3), ""); |
| 605 | | |
| 606 | | _RandomAccessIterator __begin = __first; |
| 607 | | value_type __pivot(_Ops::__iter_move(__first)); |
| 608 | | // Find the first element greater than the pivot. |
| 609 | | if (__comp(__pivot, *(__last - difference_type(1)))) { |
| 610 | | // Not guarded since we know the last element is greater than the pivot. |
| 611 | | while (!__comp(__pivot, *++__first)) { |
| 612 | | } |
| 613 | | } else { |
| 614 | | while (++__first < __last && !__comp(__pivot, *__first)) { |
| 615 | | } |
| 616 | | } |
| 617 | | // Find the last element less than or equal to the pivot. |
| 618 | | if (__first < __last) { |
| 619 | | // It will be always guarded because __introsort will do the median-of-three |
| 620 | | // before calling this. |
| 621 | | while (__comp(__pivot, *--__last)) { |
| 622 | | } |
| 623 | | } |
| 624 | | // If the first element greater than the pivot is at or after the |
| 625 | | // last element less than or equal to the pivot, then we have covered the |
| 626 | | // entire range without swapping elements. This implies the range is already |
| 627 | | // partitioned. |
| 628 | | bool __already_partitioned = __first >= __last; |
| 629 | | if (!__already_partitioned) { |
| 630 | | _Ops::iter_swap(__first, __last); |
| 631 | | ++__first; |
| 632 | | } |
| 633 | | |
| 634 | | // In [__first, __last) __last is not inclusive. From now on, it uses last |
| 635 | | // minus one to be inclusive on both sides. |
| 636 | | _RandomAccessIterator __lm1 = __last - difference_type(1); |
| 637 | | uint64_t __left_bitset = 0; |
| 638 | | uint64_t __right_bitset = 0; |
| 639 | | |
| 640 | | // Reminder: length = __lm1 - __first + 1. |
| 641 | | while (__lm1 - __first >= 2 * __detail::__block_size - 1) { |
| 642 | | // Record the comparison outcomes for the elements currently on the left |
| 643 | | // side. |
| 644 | | if (__left_bitset == 0) |
| 645 | | std::__populate_left_bitset<_Compare>(__first, __comp, __pivot, __left_bitset); |
| 646 | | // Record the comparison outcomes for the elements currently on the right |
| 647 | | // side. |
| 648 | | if (__right_bitset == 0) |
| 649 | | std::__populate_right_bitset<_Compare>(__lm1, __comp, __pivot, __right_bitset); |
| 650 | | // Swap the elements recorded to be the candidates for swapping in the |
| 651 | | // bitsets. |
| 652 | | std::__swap_bitmap_pos<_AlgPolicy, _RandomAccessIterator>(__first, __lm1, __left_bitset, __right_bitset); |
| 653 | | // Only advance the iterator if all the elements that need to be moved to |
| 654 | | // other side were moved. |
| 655 | | __first += (__left_bitset == 0) ? difference_type(__detail::__block_size) : difference_type(0); |
| 656 | | __lm1 -= (__right_bitset == 0) ? difference_type(__detail::__block_size) : difference_type(0); |
| 657 | | } |
| 658 | | // Now, we have a less-than a block worth of elements on at least one of the |
| 659 | | // sides. |
| 660 | | std::__bitset_partition_partial_blocks<_AlgPolicy, _Compare>( |
| 661 | | __first, __lm1, __comp, __pivot, __left_bitset, __right_bitset); |
| 662 | | // At least one the bitsets would be empty. For the non-empty one, we need to |
| 663 | | // properly partition the elements that appear within that bitset. |
| 664 | | std::__swap_bitmap_pos_within<_AlgPolicy>(__first, __lm1, __left_bitset, __right_bitset); |
| 665 | | |
| 666 | | // Move the pivot to its correct position. |
| 667 | | _RandomAccessIterator __pivot_pos = __first - difference_type(1); |
| 668 | | if (__begin != __pivot_pos) { |
| 669 | | *__begin = _Ops::__iter_move(__pivot_pos); |
| 670 | | } |
| 671 | | *__pivot_pos = std::move(__pivot); |
| 672 | | return std::make_pair(__pivot_pos, __already_partitioned); |
| 673 | | } |
| 674 | | |
| 675 | | // Partition [__first, __last) using the comparator __comp. *__first has the |
| 676 | | // chosen pivot. Elements that are equivalent are kept to the right of the |
| 677 | | // pivot. Returns the iterator for the pivot and a bool value which is true if |
| 678 | | // the provided range is already sorted, false otherwise. We assume that the |
| 679 | | // length of the range is at least three elements. |
| 680 | | template <class _AlgPolicy, class _RandomAccessIterator, class _Compare> |
| 681 | | _LIBCPP_HIDE_FROM_ABI std::pair<_RandomAccessIterator, bool> |
| 682 | | __partition_with_equals_on_right(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { |
| 683 | | using _Ops = _IterOps<_AlgPolicy>; |
| 684 | | typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 685 | | typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; |
| 686 | | _LIBCPP_ASSERT(__last - __first >= difference_type(3), ""); |
| 687 | | _RandomAccessIterator __begin = __first; |
| 688 | | value_type __pivot(_Ops::__iter_move(__first)); |
| 689 | | // Find the first element greater or equal to the pivot. It will be always |
| 690 | | // guarded because __introsort will do the median-of-three before calling |
| 691 | | // this. |
| 692 | | while (__comp(*++__first, __pivot)) |
| 693 | | ; |
| 694 | | |
| 695 | | // Find the last element less than the pivot. |
| 696 | | if (__begin == __first - difference_type(1)) { |
| 697 | | while (__first < __last && !__comp(*--__last, __pivot)) |
| 698 | | ; |
| 699 | | } else { |
| 700 | | // Guarded. |
| 701 | | while (!__comp(*--__last, __pivot)) |
| 702 | | ; |
| 703 | | } |
| 704 | | |
| 705 | | // If the first element greater than or equal to the pivot is at or after the |
| 706 | | // last element less than the pivot, then we have covered the entire range |
| 707 | | // without swapping elements. This implies the range is already partitioned. |
| 708 | | bool __already_partitioned = __first >= __last; |
| 709 | | // Go through the remaining elements. Swap pairs of elements (one to the |
| 710 | | // right of the pivot and the other to left of the pivot) that are not on the |
| 711 | | // correct side of the pivot. |
| 712 | | while (__first < __last) { |
| 713 | | _Ops::iter_swap(__first, __last); |
| 714 | | while (__comp(*++__first, __pivot)) |
| 715 | | ; |
| 716 | | while (!__comp(*--__last, __pivot)) |
| 717 | | ; |
| 718 | | } |
| 719 | | // Move the pivot to its correct position. |
| 720 | | _RandomAccessIterator __pivot_pos = __first - difference_type(1); |
| 721 | | if (__begin != __pivot_pos) { |
| 722 | | *__begin = _Ops::__iter_move(__pivot_pos); |
| 723 | | } |
| 724 | | *__pivot_pos = std::move(__pivot); |
| 725 | | return std::make_pair(__pivot_pos, __already_partitioned); |
| 726 | | } |
| 727 | | |
| 728 | | // Similar to the above function. Elements equivalent to the pivot are put to |
| 729 | | // the left of the pivot. Returns the iterator to the pivot element. |
| 730 | | template <class _AlgPolicy, class _RandomAccessIterator, class _Compare> |
| 731 | | _LIBCPP_HIDE_FROM_ABI _RandomAccessIterator |
| 732 | | __partition_with_equals_on_left(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp) { |
| 431 | template <class _AlgPolicy, class _Compare, class _RandomAccessIterator> |
| 432 | void __introsort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, |
| 433 | typename iterator_traits<_RandomAccessIterator>::difference_type __depth) { |
| 733 | 434 | using _Ops = _IterOps<_AlgPolicy>; |
| 734 | | typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 735 | | typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type; |
| 736 | | _RandomAccessIterator __begin = __first; |
| 737 | | value_type __pivot(_Ops::__iter_move(__first)); |
| 738 | | if (__comp(__pivot, *(__last - difference_type(1)))) { |
| 739 | | // Guarded. |
| 740 | | while (!__comp(__pivot, *++__first)) { |
| 741 | | } |
| 742 | | } else { |
| 743 | | while (++__first < __last && !__comp(__pivot, *__first)) { |
| 744 | | } |
| 745 | | } |
| 746 | | |
| 747 | | if (__first < __last) { |
| 748 | | // It will be always guarded because __introsort will do the |
| 749 | | // median-of-three before calling this. |
| 750 | | while (__comp(__pivot, *--__last)) { |
| 751 | | } |
| 752 | | } |
| 753 | | while (__first < __last) { |
| 754 | | _Ops::iter_swap(__first, __last); |
| 755 | | while (!__comp(__pivot, *++__first)) |
| 756 | | ; |
| 757 | | while (__comp(__pivot, *--__last)) |
| 758 | | ; |
| 759 | | } |
| 760 | | _RandomAccessIterator __pivot_pos = __first - difference_type(1); |
| 761 | | if (__begin != __pivot_pos) { |
| 762 | | *__begin = _Ops::__iter_move(__pivot_pos); |
| 763 | | } |
| 764 | | *__pivot_pos = std::move(__pivot); |
| 765 | | return __first; |
| 766 | | } |
| 767 | 435 | |
| 768 | | // The main sorting function. Implements introsort combined with other ideas: |
| 769 | | // - option of using block quick sort for partitioning, |
| 770 | | // - guarded and unguarded insertion sort for small lengths, |
| 771 | | // - Tuckey's ninther technique for computing the pivot, |
| 772 | | // - check on whether partition was not required. |
| 773 | | // The implementation is partly based on Orson Peters' pattern-defeating |
| 774 | | // quicksort, published at: <https://github.com/orlp/pdqsort>. |
| 775 | | template <class _AlgPolicy, class _Compare, class _RandomAccessIterator, bool _UseBitSetPartition> |
| 776 | | void __introsort(_RandomAccessIterator __first, |
| 777 | | _RandomAccessIterator __last, |
| 778 | | _Compare __comp, |
| 779 | | typename iterator_traits<_RandomAccessIterator>::difference_type __depth, |
| 780 | | bool __leftmost = true) { |
| 781 | | using _Ops = _IterOps<_AlgPolicy>; |
| 782 | 436 | typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type; |
| 783 | | using _Comp_ref = __comp_ref_type<_Compare>; |
| 784 | | // Upper bound for using insertion sort for sorting. |
| 785 | | _LIBCPP_CONSTEXPR difference_type __limit = 24; |
| 786 | | // Lower bound for using Tuckey's ninther technique for median computation. |
| 787 | | _LIBCPP_CONSTEXPR difference_type __ninther_threshold = 128; |
| 437 | typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type; |
| 438 | const difference_type __limit = |
| 439 | is_trivially_copy_constructible<value_type>::value && is_trivially_copy_assignable<value_type>::value ? 30 : 6; |
| 788 | 440 | while (true) { |
| 441 | __restart: |
| 789 | 442 | difference_type __len = __last - __first; |
| 790 | 443 | switch (__len) { |
| 791 | 444 | case 0: |
| ... | ... | @@ -793,7 +446,7 @@ void __introsort(_RandomAccessIterator __first, |
| 793 | 446 | return; |
| 794 | 447 | case 2: |
| 795 | 448 | if (__comp(*--__last, *__first)) |
| 796 | | _Ops::iter_swap(__first, __last); |
| 449 | _IterOps<_AlgPolicy>::iter_swap(__first, __last); |
| 797 | 450 | return; |
| 798 | 451 | case 3: |
| 799 | 452 | std::__sort3_maybe_branchless<_AlgPolicy, _Compare>(__first, __first + difference_type(1), --__last, __comp); |
| ... | ... | @@ -808,60 +461,127 @@ void __introsort(_RandomAccessIterator __first, |
| 808 | 461 | --__last, __comp); |
| 809 | 462 | return; |
| 810 | 463 | } |
| 811 | | // Use insertion sort if the length of the range is below the specified limit. |
| 812 | | if (__len < __limit) { |
| 813 | | if (__leftmost) { |
| 814 | | std::__insertion_sort<_AlgPolicy, _Compare>(__first, __last, __comp); |
| 815 | | } else { |
| 816 | | std::__insertion_sort_unguarded<_AlgPolicy, _Compare>(__first, __last, __comp); |
| 817 | | } |
| 464 | if (__len <= __limit) { |
| 465 | std::__insertion_sort_3<_AlgPolicy, _Compare>(__first, __last, __comp); |
| 818 | 466 | return; |
| 819 | 467 | } |
| 468 | // __len > 5 |
| 820 | 469 | if (__depth == 0) { |
| 821 | 470 | // Fallback to heap sort as Introsort suggests. |
| 822 | 471 | std::__partial_sort<_AlgPolicy, _Compare>(__first, __last, __last, __comp); |
| 823 | 472 | return; |
| 824 | 473 | } |
| 825 | 474 | --__depth; |
| 475 | _RandomAccessIterator __m = __first; |
| 476 | _RandomAccessIterator __lm1 = __last; |
| 477 | --__lm1; |
| 478 | unsigned __n_swaps; |
| 826 | 479 | { |
| 827 | | difference_type __half_len = __len / 2; |
| 828 | | // Use Tuckey's ninther technique or median of 3 for pivot selection |
| 829 | | // depending on the length of the range being sorted. |
| 830 | | if (__len > __ninther_threshold) { |
| 831 | | std::__sort3<_AlgPolicy, _Compare>(__first, __first + __half_len, __last - difference_type(1), __comp); |
| 832 | | std::__sort3<_AlgPolicy, _Compare>( |
| 833 | | __first + difference_type(1), __first + (__half_len - 1), __last - difference_type(2), __comp); |
| 834 | | std::__sort3<_AlgPolicy, _Compare>( |
| 835 | | __first + difference_type(2), __first + (__half_len + 1), __last - difference_type(3), __comp); |
| 836 | | std::__sort3<_AlgPolicy, _Compare>( |
| 837 | | __first + (__half_len - 1), __first + __half_len, __first + (__half_len + 1), __comp); |
| 838 | | _Ops::iter_swap(__first, __first + __half_len); |
| 480 | difference_type __delta; |
| 481 | if (__len >= 1000) { |
| 482 | __delta = __len / 2; |
| 483 | __m += __delta; |
| 484 | __delta /= 2; |
| 485 | __n_swaps = std::__sort5_wrap_policy<_AlgPolicy, _Compare>( |
| 486 | __first, __first + __delta, __m, __m + __delta, __lm1, __comp); |
| 839 | 487 | } else { |
| 840 | | std::__sort3<_AlgPolicy, _Compare>(__first + __half_len, __first, __last - difference_type(1), __comp); |
| 488 | __delta = __len / 2; |
| 489 | __m += __delta; |
| 490 | __n_swaps = std::__sort3<_AlgPolicy, _Compare>(__first, __m, __lm1, __comp); |
| 841 | 491 | } |
| 842 | 492 | } |
| 843 | | // The elements to the left of the current iterator range are already |
| 844 | | // sorted. If the current iterator range to be sorted is not the |
| 845 | | // leftmost part of the entire iterator range and the pivot is same as |
| 846 | | // the highest element in the range to the left, then we know that all |
| 847 | | // the elements in the range [first, pivot] would be equal to the pivot, |
| 848 | | // assuming the equal elements are put on the left side when |
| 849 | | // partitioned. This also means that we do not need to sort the left |
| 850 | | // side of the partition. |
| 851 | | if (!__leftmost && !__comp(*(__first - difference_type(1)), *__first)) { |
| 852 | | __first = std::__partition_with_equals_on_left<_AlgPolicy, _RandomAccessIterator, _Comp_ref>( |
| 853 | | __first, __last, _Comp_ref(__comp)); |
| 854 | | continue; |
| 493 | // *__m is median |
| 494 | // partition [__first, __m) < *__m and *__m <= [__m, __last) |
| 495 | // (this inhibits tossing elements equivalent to __m around unnecessarily) |
| 496 | _RandomAccessIterator __i = __first; |
| 497 | _RandomAccessIterator __j = __lm1; |
| 498 | // j points beyond range to be tested, *__m is known to be <= *__lm1 |
| 499 | // The search going up is known to be guarded but the search coming down isn't. |
| 500 | // Prime the downward search with a guard. |
| 501 | if (!__comp(*__i, *__m)) // if *__first == *__m |
| 502 | { |
| 503 | // *__first == *__m, *__first doesn't go in first part |
| 504 | // manually guard downward moving __j against __i |
| 505 | while (true) { |
| 506 | if (__i == --__j) { |
| 507 | // *__first == *__m, *__m <= all other elements |
| 508 | // Parition instead into [__first, __i) == *__first and *__first < [__i, __last) |
| 509 | ++__i; // __first + 1 |
| 510 | __j = __last; |
| 511 | if (!__comp(*__first, *--__j)) // we need a guard if *__first == *(__last-1) |
| 512 | { |
| 513 | while (true) { |
| 514 | if (__i == __j) |
| 515 | return; // [__first, __last) all equivalent elements |
| 516 | if (__comp(*__first, *__i)) { |
| 517 | _Ops::iter_swap(__i, __j); |
| 518 | ++__n_swaps; |
| 519 | ++__i; |
| 520 | break; |
| 521 | } |
| 522 | ++__i; |
| 523 | } |
| 524 | } |
| 525 | // [__first, __i) == *__first and *__first < [__j, __last) and __j == __last - 1 |
| 526 | if (__i == __j) |
| 527 | return; |
| 528 | while (true) { |
| 529 | while (!__comp(*__first, *__i)) |
| 530 | ++__i; |
| 531 | while (__comp(*__first, *--__j)) |
| 532 | ; |
| 533 | if (__i >= __j) |
| 534 | break; |
| 535 | _Ops::iter_swap(__i, __j); |
| 536 | ++__n_swaps; |
| 537 | ++__i; |
| 538 | } |
| 539 | // [__first, __i) == *__first and *__first < [__i, __last) |
| 540 | // The first part is sorted, sort the second part |
| 541 | // std::__sort<_Compare>(__i, __last, __comp); |
| 542 | __first = __i; |
| 543 | goto __restart; |
| 544 | } |
| 545 | if (__comp(*__j, *__m)) { |
| 546 | _Ops::iter_swap(__i, __j); |
| 547 | ++__n_swaps; |
| 548 | break; // found guard for downward moving __j, now use unguarded partition |
| 549 | } |
| 550 | } |
| 551 | } |
| 552 | // It is known that *__i < *__m |
| 553 | ++__i; |
| 554 | // j points beyond range to be tested, *__m is known to be <= *__lm1 |
| 555 | // if not yet partitioned... |
| 556 | if (__i < __j) { |
| 557 | // known that *(__i - 1) < *__m |
| 558 | // known that __i <= __m |
| 559 | while (true) { |
| 560 | // __m still guards upward moving __i |
| 561 | while (__comp(*__i, *__m)) |
| 562 | ++__i; |
| 563 | // It is now known that a guard exists for downward moving __j |
| 564 | while (!__comp(*--__j, *__m)) |
| 565 | ; |
| 566 | if (__i > __j) |
| 567 | break; |
| 568 | _Ops::iter_swap(__i, __j); |
| 569 | ++__n_swaps; |
| 570 | // It is known that __m != __j |
| 571 | // If __m just moved, follow it |
| 572 | if (__m == __i) |
| 573 | __m = __j; |
| 574 | ++__i; |
| 575 | } |
| 576 | } |
| 577 | // [__first, __i) < *__m and *__m <= [__i, __last) |
| 578 | if (__i != __m && __comp(*__m, *__i)) { |
| 579 | _Ops::iter_swap(__i, __m); |
| 580 | ++__n_swaps; |
| 855 | 581 | } |
| 856 | | // Use bitset partition only if asked for. |
| 857 | | auto __ret = |
| 858 | | _UseBitSetPartition |
| 859 | | ? std::__bitset_partition<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp) |
| 860 | | : std::__partition_with_equals_on_right<_AlgPolicy, _RandomAccessIterator, _Compare>(__first, __last, __comp); |
| 861 | | _RandomAccessIterator __i = __ret.first; |
| 862 | 582 | // [__first, __i) < *__i and *__i <= [__i+1, __last) |
| 863 | 583 | // If we were given a perfect partition, see if insertion sort is quick... |
| 864 | | if (__ret.second) { |
| 584 | if (__n_swaps == 0) { |
| 865 | 585 | using _WrappedComp = typename _WrapAlgPolicy<_AlgPolicy, _Compare>::type; |
| 866 | 586 | _WrappedComp __wrapped_comp(__comp); |
| 867 | 587 | bool __fs = std::__insertion_sort_incomplete<_WrappedComp>(__first, __i, __wrapped_comp); |
| ... | ... | @@ -877,11 +597,14 @@ void __introsort(_RandomAccessIterator __first, |
| 877 | 597 | } |
| 878 | 598 | } |
| 879 | 599 | } |
| 880 | | // Sort the left partiton recursively and the right partition with tail recursion elimination. |
| 881 | | std::__introsort<_AlgPolicy, _Compare, _RandomAccessIterator, _UseBitSetPartition>( |
| 882 | | __first, __i, __comp, __depth, __leftmost); |
| 883 | | __leftmost = false; |
| 884 | | __first = ++__i; |
| 600 | // sort smaller range with recursive call and larger with tail recursion elimination |
| 601 | if (__i - __first < __last - __i) { |
| 602 | std::__introsort<_AlgPolicy, _Compare>(__first, __i, __comp, __depth); |
| 603 | __first = ++__i; |
| 604 | } else { |
| 605 | std::__introsort<_AlgPolicy, _Compare>(__i + difference_type(1), __last, __comp, __depth); |
| 606 | __last = __i; |
| 607 | } |
| 885 | 608 | } |
| 886 | 609 | } |
| 887 | 610 | |
| ... | ... | @@ -913,14 +636,7 @@ _LIBCPP_HIDDEN void __sort(_RandomAccessIterator __first, _RandomAccessIterator |
| 913 | 636 | using _AlgPolicy = typename _Unwrap::_AlgPolicy; |
| 914 | 637 | using _Compare = typename _Unwrap::_Comp; |
| 915 | 638 | _Compare __comp = _Unwrap::__get_comp(__wrapped_comp); |
| 916 | | // Only use bitset partitioning for arithmetic types. We should also check |
| 917 | | // that the default comparator is in use so that we are sure that there are no |
| 918 | | // branches in the comparator. |
| 919 | | std::__introsort<_AlgPolicy, |
| 920 | | _Compare, |
| 921 | | _RandomAccessIterator, |
| 922 | | __use_branchless_sort<_Compare, _RandomAccessIterator>::value>( |
| 923 | | __first, __last, __comp, __depth_limit); |
| 639 | std::__introsort<_AlgPolicy, _Compare>(__first, __last, __comp, __depth_limit); |
| 924 | 640 | } |
| 925 | 641 | |
| 926 | 642 | template <class _Compare, class _Tp> |
Andrew Kelley <andrew@ziglang.org> 2023-04-05 23:22:29-07:00
lib