Commit a83aad152b502ab205f4f6f6c4f70a25b2fa955d | zig

author	Alex Rønne Petersen <alex@alexrp.com> 2026-04-26 07:03:38+02:00
committer	Alex Rønne Petersen <alex@alexrp.com> 2026-04-26 07:03:38+02:00
log	a83aad152b502ab205f4f6f6c4f70a25b2fa955d
tree	6e073a56aa5955e5da24ade26508fa60a3a4f78a
parent	f0a7e3ccae45ce7d09038204c16155d5b98ad560
parent	8443e2ab43000a4b196d3a6014655c77f054d893

Merge pull request 'LLVM 22' (#32013) from llvm22 into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/32013

730 files changed, 34010 insertions(+), 24951 deletions(-)

CMakeLists.txt
build.zig
ci
cmake
lib
src
- Compilation.zig
- Package
  - Module.zig
- clang_options.zon
- codegen
  - c.zig
  - llvm.zig
  - llvm
    FuncGen.zig
    bindings.zig
- libs
  - libcxx.zig
- link
  - Lld.zig
  - Wasm.zig
- main.zig
- target.zig
- zig_clang_cc1_main.cpp
- zig_clang_cc1as_main.cpp
- zig_clang_driver.cpp
- zig_llvm-ar.cpp
- zig_llvm.cpp
stage1
- zig.h
- zig1.wasm
test
tools
- update_clang_options.zig
- update_cpu_features.zig

CMakeLists.txt+3-3

...	...	@@ -133,9 +133,9 @@ else()
133	133	set(ZIG_SYSTEM_LIBCXX "stdc++" CACHE STRING "system libcxx name for build.zig")
134	134	endif()
135	135
136		find_package(llvm 21)
137		find_package(clang 21)
138		find_package(lld 21)
	136	find_package(llvm 22)
	137	find_package(clang 22)
	138	find_package(lld 22)
139	139
140	140	if(ZIG_STATIC_ZLIB)
141	141	if (MSVC)

build.zig+14-6

...	...	@@ -1206,13 +1206,16 @@ const zig_cpp_sources = [_][]const u8{
1206	1206	const clang_libs = [_][]const u8{
1207	1207	"clangFrontendTool",
1208	1208	"clangCodeGen",
	1209	"clangStaticAnalyzerFrontend",
	1210	"clangStaticAnalyzerCheckers",
	1211	"clangStaticAnalyzerCore",
	1212	"clangCrossTU",
1209	1213	"clangFrontend",
1210	1214	"clangDriver",
	1215	"clangOptions",
1211	1216	"clangSerialization",
1212	1217	"clangSema",
1213		"clangStaticAnalyzerFrontend",
1214		"clangStaticAnalyzerCheckers",
1215		"clangStaticAnalyzerCore",
	1218	"clangAnalysisLifetimeSafety",
1216	1219	"clangAnalysis",
1217	1220	"clangASTMatchers",
1218	1221	"clangAST",
...	...	@@ -1224,8 +1227,9 @@ const clang_libs = [_][]const u8{
1224	1227	"clangLex",
1225	1228	"clangRewriteFrontend",
1226	1229	"clangRewrite",
1227		"clangCrossTU",
1228	1230	"clangIndex",
	1231	"clangFormat",
	1232	"clangToolingInclusions",
1229	1233	"clangToolingCore",
1230	1234	"clangExtractAPI",
1231	1235	"clangSupport",
...	...	@@ -1373,11 +1377,12 @@ const llvm_libs = [_][]const u8{
1373	1377	"LLVMObjCopy",
1374	1378	"LLVMMCA",
1375	1379	"LLVMMCDisassembler",
	1380	"LLVMDTLTO",
1376	1381	"LLVMLTO",
1377	1382	"LLVMFrontendOpenACC",
1378		"LLVMFrontendHLSL",
1379	1383	"LLVMFrontendDriver",
1380	1384	"LLVMExtensions",
	1385	"LLVMPlugins",
1381	1386	"LLVMPasses",
1382	1387	"LLVMHipStdPar",
1383	1388	"LLVMCoroutines",
...	...	@@ -1404,6 +1409,7 @@ const llvm_libs = [_][]const u8{
1404	1409	"LLVMObjCARCOpts",
1405	1410	"LLVMCodeGenTypes",
1406	1411	"LLVMCGData",
	1412	"LLVMCAS",
1407	1413	"LLVMIRPrinter",
1408	1414	"LLVMInterfaceStub",
1409	1415	"LLVMFileCheck",
...	...	@@ -1422,15 +1428,17 @@ const llvm_libs = [_][]const u8{
1422	1428	"LLVMDebugInfoCodeView",
1423	1429	"LLVMDebugInfoGSYM",
1424	1430	"LLVMDebugInfoDWARF",
1425		"LLVMDebugInfoDWARFLowLevel",
1426	1431	"LLVMObject",
1427	1432	"LLVMTextAPI",
1428	1433	"LLVMMCParser",
1429	1434	"LLVMIRReader",
1430	1435	"LLVMAsmParser",
1431	1436	"LLVMMC",
	1437	"LLVMDebugInfoDWARFLowLevel",
1432	1438	"LLVMBitReader",
	1439	"LLVMFrontendHLSL",
1433	1440	"LLVMFuzzerCLI",
	1441	"LLVMABI",
1434	1442	"LLVMCore",
1435	1443	"LLVMRemarks",
1436	1444	"LLVMBitstreamReader",

ci/aarch64-freebsd-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-freebsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-freebsd-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-freebsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-macos-debug.sh+1-1

...	...	@@ -8,7 +8,7 @@ set -e
8	8	ZIGDIR="$PWD"
9	9	TARGET="aarch64-macos-none"
10	10	MCPU="baseline"
11		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	11	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
12	12	PREFIX="$HOME/$CACHE_BASENAME"
13	13	ZIG="$PREFIX/bin/zig"
14	14

ci/aarch64-macos-release.sh+1-1

...	...	@@ -8,7 +8,7 @@ set -e
8	8	ZIGDIR="$PWD"
9	9	TARGET="aarch64-macos-none"
10	10	MCPU="baseline"
11		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	11	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
12	12	PREFIX="$HOME/$CACHE_BASENAME"
13	13	ZIG="$PREFIX/bin/zig"
14	14

ci/aarch64-netbsd-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-netbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-netbsd-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="aarch64-netbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/aarch64-windows.ps1+1-1

...	...	@@ -1,5 +1,5 @@
1	1	$TARGET = "aarch64-windows-gnu"
2		$ZIG_LLVM_CLANG_LLD_NAME = "zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	2	$ZIG_LLVM_CLANG_LLD_NAME = "zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
3	3	$MCPU = "baseline"
4	4	$ZIG_LLVM_CLANG_LLD_URL = "https://ziglang.org/deps/$ZIG_LLVM_CLANG_LLD_NAME.zip"
5	5	$PREFIX_PATH = "$(Get-Location)\..\$ZIG_LLVM_CLANG_LLD_NAME"

ci/loongarch64-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="loongarch64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.157+7fdd60df1"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/loongarch64-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="loongarch64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.157+7fdd60df1"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/powerpc64le-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="powerpc64le-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1594+9fa433d71"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/powerpc64le-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="powerpc64le-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1594+9fa433d71"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/riscv64-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="riscv64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-riscv64-linux-musl-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/riscv64-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="riscv64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-riscv64-linux-musl-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/s390x-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="s390x-linux-musl"
9	9	MCPU="z15"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1354+94e98bfe8"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/s390x-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="s390x-linux-musl"
9	9	MCPU="z15"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1354+94e98bfe8"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-freebsd-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-freebsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.312+164c598cd"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-freebsd-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-freebsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.312+164c598cd"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-linux-debug-llvm.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-linux-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-linux-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-linux-musl"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-netbsd-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-netbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-netbsd-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-netbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2287+eb3f16db5"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-openbsd-debug.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-openbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2051+28b83e3b0"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-openbsd-release.sh+1-1

...	...	@@ -7,7 +7,7 @@ set -e
7	7
8	8	TARGET="x86_64-openbsd-none"
9	9	MCPU="baseline"
10		CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.2051+28b83e3b0"
	10	CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
11	11	PREFIX="$HOME/deps/$CACHE_BASENAME"
12	12	ZIG="$PREFIX/bin/zig"
13	13

ci/x86_64-windows-debug.ps1+1-1

...	...	@@ -1,6 +1,6 @@
1	1	$TARGET = "x86_64-windows-gnu"
2	2	$MCPU = "baseline"
3		$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	3	$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
4	4	$ZIG = "$PREFIX_PATH\bin\zig.exe"
5	5	$ZIG_LIB_DIR = "$(Get-Location)\lib"
6	6	$ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 }

ci/x86_64-windows-release.ps1+1-1

...	...	@@ -1,6 +1,6 @@
1	1	$TARGET = "x86_64-windows-gnu"
2	2	$MCPU = "baseline"
3		$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.16.0-dev.104+689461e31"
	3	$PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523"
4	4	$ZIG = "$PREFIX_PATH\bin\zig.exe"
5	5	$ZIG_LIB_DIR = "$(Get-Location)\lib"
6	6	$ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 }

cmake/Findclang.cmake+12-8

...	...	@@ -17,10 +17,10 @@ find_path(CLANG_INCLUDE_DIRS NAMES clang/Frontend/ASTUnit.h
17	17	if(${LLVM_LINK_MODE} STREQUAL "shared")
18	18	find_library(CLANG_LIBRARIES
19	19	NAMES
20		libclang-cpp.so.21
21		libclang-cpp.so.21.1
22		clang-cpp-21.0
23		clang-cpp210
	20	libclang-cpp.so.22
	21	libclang-cpp.so.22.1
	22	clang-cpp-22.0
	23	clang-cpp220
24	24	clang-cpp
25	25	NAMES_PER_DIR
26	26	HINTS "${LLVM_LIBDIRS}"
...	...	@@ -44,13 +44,16 @@ else()
44	44
45	45	FIND_AND_ADD_CLANG_LIB(clangFrontendTool)
46	46	FIND_AND_ADD_CLANG_LIB(clangCodeGen)
	47	FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerFrontend)
	48	FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerCheckers)
	49	FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerCore)
	50	FIND_AND_ADD_CLANG_LIB(clangCrossTU)
47	51	FIND_AND_ADD_CLANG_LIB(clangFrontend)
48	52	FIND_AND_ADD_CLANG_LIB(clangDriver)
	53	FIND_AND_ADD_CLANG_LIB(clangOptions)
49	54	FIND_AND_ADD_CLANG_LIB(clangSerialization)
50	55	FIND_AND_ADD_CLANG_LIB(clangSema)
51		FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerFrontend)
52		FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerCheckers)
53		FIND_AND_ADD_CLANG_LIB(clangStaticAnalyzerCore)
	56	FIND_AND_ADD_CLANG_LIB(clangAnalysisLifetimeSafety)
54	57	FIND_AND_ADD_CLANG_LIB(clangAnalysis)
55	58	FIND_AND_ADD_CLANG_LIB(clangASTMatchers)
56	59	FIND_AND_ADD_CLANG_LIB(clangAST)
...	...	@@ -62,8 +65,9 @@ else()
62	65	FIND_AND_ADD_CLANG_LIB(clangLex)
63	66	FIND_AND_ADD_CLANG_LIB(clangRewriteFrontend)
64	67	FIND_AND_ADD_CLANG_LIB(clangRewrite)
65		FIND_AND_ADD_CLANG_LIB(clangCrossTU)
66	68	FIND_AND_ADD_CLANG_LIB(clangIndex)
	69	FIND_AND_ADD_CLANG_LIB(clangFormat)
	70	FIND_AND_ADD_CLANG_LIB(clangToolingInclusions)
67	71	FIND_AND_ADD_CLANG_LIB(clangToolingCore)
68	72	FIND_AND_ADD_CLANG_LIB(clangExtractAPI)
69	73	FIND_AND_ADD_CLANG_LIB(clangSupport)

cmake/Findlld.cmake+19-19

...	...	@@ -9,23 +9,23 @@
9	9	find_path(LLD_INCLUDE_DIRS NAMES lld/Common/Driver.h
10	10	HINTS ${LLVM_INCLUDE_DIRS}
11	11	PATHS
12		/usr/lib/llvm-21/include
13		/usr/local/llvm210/include
14		/usr/local/llvm21/include
15		/usr/local/opt/lld@21/include
16		/opt/homebrew/opt/lld@21/include
17		/home/linuxbrew/.linuxbrew/opt/lld@21/include
	12	/usr/lib/llvm-22/include
	13	/usr/local/llvm220/include
	14	/usr/local/llvm22/include
	15	/usr/local/opt/lld@22/include
	16	/opt/homebrew/opt/lld@22/include
	17	/home/linuxbrew/.linuxbrew/opt/lld@22/include
18	18	/mingw64/include)
19	19
20		find_library(LLD_LIBRARY NAMES lld-21.0 lld210 lld NAMES_PER_DIR
	20	find_library(LLD_LIBRARY NAMES lld-22.0 lld220 lld NAMES_PER_DIR
21	21	HINTS ${LLVM_LIBDIRS}
22	22	PATHS
23		/usr/lib/llvm-21/lib
24		/usr/local/llvm210/lib
25		/usr/local/llvm21/lib
26		/usr/local/opt/lld@21/lib
27		/opt/homebrew/opt/lld@21/lib
28		/home/linuxbrew/.linuxbrew/opt/lld@21/lib
	23	/usr/lib/llvm-22/lib
	24	/usr/local/llvm220/lib
	25	/usr/local/llvm22/lib
	26	/usr/local/opt/lld@22/lib
	27	/opt/homebrew/opt/lld@22/lib
	28	/home/linuxbrew/.linuxbrew/opt/lld@22/lib
29	29	)
30	30	if(EXISTS ${LLD_LIBRARY})
31	31	set(LLD_LIBRARIES ${LLD_LIBRARY})
...	...	@@ -36,12 +36,12 @@ else()
36	36	HINTS ${LLVM_LIBDIRS}
37	37	PATHS
38	38	${LLD_LIBDIRS}
39		/usr/lib/llvm-21/lib
40		/usr/local/llvm210/lib
41		/usr/local/llvm21/lib
42		/usr/local/opt/lld@21/lib
43		/opt/homebrew/opt/lld@21/lib
44		/home/linuxbrew/.linuxbrew/opt/lld@21/lib
	39	/usr/lib/llvm-22/lib
	40	/usr/local/llvm220/lib
	41	/usr/local/llvm22/lib
	42	/usr/local/opt/lld@22/lib
	43	/opt/homebrew/opt/lld@22/lib
	44	/home/linuxbrew/.linuxbrew/opt/lld@22/lib
45	45	/mingw64/lib
46	46	/c/msys64/mingw64/lib
47	47	c:/msys64/mingw64/lib)

cmake/Findllvm.cmake+12-8

...	...	@@ -17,12 +17,12 @@ if(ZIG_USE_LLVM_CONFIG)
17	17	# terminate when the right LLVM version is not found.
18	18	unset(LLVM_CONFIG_EXE CACHE)
19	19	find_program(LLVM_CONFIG_EXE
20		NAMES llvm-config-21 llvm-config-21.0 llvm-config210 llvm-config21 llvm-config NAMES_PER_DIR
	20	NAMES llvm-config-22 llvm-config-22.0 llvm-config220 llvm-config22 llvm-config NAMES_PER_DIR
21	21	PATHS
22	22	"/mingw64/bin"
23	23	"/c/msys64/mingw64/bin"
24	24	"c:/msys64/mingw64/bin"
25		"C:/Libraries/llvm-21.0.0/bin")
	25	"C:/Libraries/llvm-22.0.0/bin")
26	26
27	27	if ("${LLVM_CONFIG_EXE}" STREQUAL "LLVM_CONFIG_EXE-NOTFOUND")
28	28	if (NOT LLVM_CONFIG_ERROR_MESSAGES STREQUAL "")
...	...	@@ -40,9 +40,9 @@ if(ZIG_USE_LLVM_CONFIG)
40	40	OUTPUT_STRIP_TRAILING_WHITESPACE)
41	41
42	42	get_filename_component(LLVM_CONFIG_DIR "${LLVM_CONFIG_EXE}" DIRECTORY)
43		if("${LLVM_CONFIG_VERSION}" VERSION_LESS 21 OR "${LLVM_CONFIG_VERSION}" VERSION_EQUAL 22 OR "${LLVM_CONFIG_VERSION}" VERSION_GREATER 22)
	43	if("${LLVM_CONFIG_VERSION}" VERSION_LESS 22 OR "${LLVM_CONFIG_VERSION}" VERSION_EQUAL 23 OR "${LLVM_CONFIG_VERSION}" VERSION_GREATER 23)
44	44	# Save the error message, in case this is the last llvm-config we find
45		list(APPEND LLVM_CONFIG_ERROR_MESSAGES "expected LLVM 21.x but found ${LLVM_CONFIG_VERSION} using ${LLVM_CONFIG_EXE}")
	45	list(APPEND LLVM_CONFIG_ERROR_MESSAGES "expected LLVM 22.x but found ${LLVM_CONFIG_VERSION} using ${LLVM_CONFIG_EXE}")
46	46
47	47	# Ignore this directory and try the search again
48	48	list(APPEND CMAKE_IGNORE_PATH "${LLVM_CONFIG_DIR}")
...	...	@@ -66,9 +66,9 @@ if(ZIG_USE_LLVM_CONFIG)
66	66	if (LLVM_CONFIG_ERROR)
67	67	# Save the error message, in case this is the last llvm-config we find
68	68	if (ZIG_SHARED_LLVM)
69		list(APPEND LLVM_CONFIG_ERROR_MESSAGES "LLVM 21.x found at ${LLVM_CONFIG_EXE} does not support linking as a shared library")
	69	list(APPEND LLVM_CONFIG_ERROR_MESSAGES "LLVM 22.x found at ${LLVM_CONFIG_EXE} does not support linking as a shared library")
70	70	else()
71		list(APPEND LLVM_CONFIG_ERROR_MESSAGES "LLVM 21.x found at ${LLVM_CONFIG_EXE} does not support linking as a static library")
	71	list(APPEND LLVM_CONFIG_ERROR_MESSAGES "LLVM 22.x found at ${LLVM_CONFIG_EXE} does not support linking as a static library")
72	72	endif()
73	73
74	74	# Ignore this directory and try the search again
...	...	@@ -321,11 +321,12 @@ else()
321	321	FIND_AND_ADD_LLVM_LIB(LLVMObjCopy)
322	322	FIND_AND_ADD_LLVM_LIB(LLVMMCA)
323	323	FIND_AND_ADD_LLVM_LIB(LLVMMCDisassembler)
	324	FIND_AND_ADD_LLVM_LIB(LLVMDTLTO)
324	325	FIND_AND_ADD_LLVM_LIB(LLVMLTO)
325	326	FIND_AND_ADD_LLVM_LIB(LLVMFrontendOpenACC)
326		FIND_AND_ADD_LLVM_LIB(LLVMFrontendHLSL)
327	327	FIND_AND_ADD_LLVM_LIB(LLVMFrontendDriver)
328	328	FIND_AND_ADD_LLVM_LIB(LLVMExtensions)
	329	FIND_AND_ADD_LLVM_LIB(LLVMPlugins)
329	330	FIND_AND_ADD_LLVM_LIB(LLVMPasses)
330	331	FIND_AND_ADD_LLVM_LIB(LLVMHipStdPar)
331	332	FIND_AND_ADD_LLVM_LIB(LLVMCoroutines)
...	...	@@ -352,6 +353,7 @@ else()
352	353	FIND_AND_ADD_LLVM_LIB(LLVMObjCARCOpts)
353	354	FIND_AND_ADD_LLVM_LIB(LLVMCodeGenTypes)
354	355	FIND_AND_ADD_LLVM_LIB(LLVMCGData)
	356	FIND_AND_ADD_LLVM_LIB(LLVMCAS)
355	357	FIND_AND_ADD_LLVM_LIB(LLVMIRPrinter)
356	358	FIND_AND_ADD_LLVM_LIB(LLVMInterfaceStub)
357	359	FIND_AND_ADD_LLVM_LIB(LLVMFileCheck)
...	...	@@ -370,15 +372,17 @@ else()
370	372	FIND_AND_ADD_LLVM_LIB(LLVMDebugInfoCodeView)
371	373	FIND_AND_ADD_LLVM_LIB(LLVMDebugInfoGSYM)
372	374	FIND_AND_ADD_LLVM_LIB(LLVMDebugInfoDWARF)
373		FIND_AND_ADD_LLVM_LIB(LLVMDebugInfoDWARFLowLevel)
374	375	FIND_AND_ADD_LLVM_LIB(LLVMObject)
375	376	FIND_AND_ADD_LLVM_LIB(LLVMTextAPI)
376	377	FIND_AND_ADD_LLVM_LIB(LLVMMCParser)
377	378	FIND_AND_ADD_LLVM_LIB(LLVMIRReader)
378	379	FIND_AND_ADD_LLVM_LIB(LLVMAsmParser)
379	380	FIND_AND_ADD_LLVM_LIB(LLVMMC)
	381	FIND_AND_ADD_LLVM_LIB(LLVMDebugInfoDWARFLowLevel)
380	382	FIND_AND_ADD_LLVM_LIB(LLVMBitReader)
	383	FIND_AND_ADD_LLVM_LIB(LLVMFrontendHLSL)
381	384	FIND_AND_ADD_LLVM_LIB(LLVMFuzzerCLI)
	385	FIND_AND_ADD_LLVM_LIB(LLVMABI)
382	386	FIND_AND_ADD_LLVM_LIB(LLVMCore)
383	387	FIND_AND_ADD_LLVM_LIB(LLVMRemarks)
384	388	FIND_AND_ADD_LLVM_LIB(LLVMBitstreamReader)

lib/compiler/aro/aro/Compilation.zig-3

...	...	@@ -498,7 +498,6 @@ fn generateSystemDefines(comp: Compilation, w: Io.Writer) !void {
498	498	.{ .fma, "__FMA__" },
499	499	.{ .f16c, "__F16C__" },
500	500	.{ .gfni, "__GFNI__" },
501		.{ .evex512, "__EVEX512__" },
502	501
503	502	.{ .avx10_1, "__AVX10_1__" },
504	503	.{ .avx10_1, "__AVX10_1_512__" },
...	...	@@ -560,7 +559,6 @@ fn generateSystemDefines(comp: Compilation, w: Io.Writer) !void {
560	559	.{ .amx_complex, "__AMX_COMPLEX__" },
561	560	.{ .amx_fp8, "__AMX_FP8__" },
562	561	.{ .amx_movrs, "__AMX_MOVRS__" },
563		.{ .amx_transpose, "__AMX_TRANSPOSE__" },
564	562	.{ .amx_avx512, "__AMX_AVX512__" },
565	563	.{ .amx_tf32, "__AMX_TF32__" },
566	564	.{ .cmpccxadd, "__CMPCCXADD__" },
...	...	@@ -798,7 +796,6 @@ fn generateSystemDefines(comp: Compilation, w: Io.Writer) !void {
798	796	.{ .fullfp16, "FP16_SCALAR_ARITHMETIC" },
799	797	.{ .dotprod, "DOTPROD" },
800	798	.{ .mte, "MEMORY_TAGGING" },
801		.{ .tme, "TME" },
802	799	.{ .i8mm, "MATMUL_INT8" },
803	800	.{ .lse, "ATOMICS" },
804	801	.{ .f64mm, "SVE_MATMUL_FP64" },

lib/compiler_rt.zig+1-22

...	...	@@ -447,29 +447,8 @@ pub const gnu_f16_abi = switch (builtin.cpu.arch) {
447	447
448	448	pub const want_sparc_abi = builtin.cpu.arch.isSPARC();
449	449
450		/// This seems to mostly correspond to `clang::TargetInfo::HasFloat16`.
451	450	pub fn F16T(comptime OtherType: type) type {
452	451	return switch (builtin.cpu.arch) {
453		.amdgcn,
454		.arm,
455		.armeb,
456		.thumb,
457		.thumbeb,
458		.aarch64,
459		.aarch64_be,
460		.hexagon,
461		.loongarch32,
462		.loongarch64,
463		.nvptx,
464		.nvptx64,
465		.riscv32,
466		.riscv32be,
467		.riscv64,
468		.riscv64be,
469		.s390x,
470		.spirv32,
471		.spirv64,
472		=> f16,
473	452	.x86, .x86_64 => if (builtin.target.os.tag.isDarwin()) switch (OtherType) {
474	453	// Starting with LLVM 16, Darwin uses different abi for f16
475	454	// depending on the type of the other return/argument..???
...	...	@@ -477,7 +456,7 @@ pub fn F16T(comptime OtherType: type) type {
477	456	f80, f128 => f16,
478	457	else => unreachable,
479	458	} else f16,
480		else => u16,
	459	else => f16,
481	460	};
482	461	}
483	462

lib/compiler_rt/trunctfhf2.zig+3

...	...	@@ -4,6 +4,9 @@ const truncf = @import("./truncf.zig").truncf;
4	4
5	5	comptime {
6	6	symbol(&__trunctfhf2, "__trunctfhf2");
	7	if (compiler_rt.want_ppc_abi) {
	8	symbol(&__trunctfhf2, "__trunckfhf2");
	9	}
7	10	}
8	11
9	12	pub fn __trunctfhf2(a: f128) callconv(.c) compiler_rt.F16T(f128) {

lib/include/__clang_spirv_builtins.h+12-12

...	...	@@ -52,30 +52,30 @@
52	52	// Builtin IDs and sizes
53	53
54	54	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_num_workgroups) __size_t
55		__spirv_NumWorkgroups(int);
	55	__spirv_BuiltInNumWorkgroups(int);
56	56	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_workgroup_size) __size_t
57		__spirv_WorkgroupSize(int);
	57	__spirv_BuiltInWorkgroupSize(int);
58	58	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_workgroup_id) __size_t
59		__spirv_WorkgroupId(int);
	59	__spirv_BuiltInWorkgroupId(int);
60	60	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_local_invocation_id) __size_t
61		__spirv_LocalInvocationId(int);
	61	__spirv_BuiltInLocalInvocationId(int);
62	62	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_global_invocation_id) __size_t
63		__spirv_GlobalInvocationId(int);
	63	__spirv_BuiltInGlobalInvocationId(int);
64	64
65	65	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_global_size) __size_t
66		__spirv_GlobalSize(int);
	66	__spirv_BuiltInGlobalSize(int);
67	67	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_global_offset) __size_t
68		__spirv_GlobalOffset(int);
	68	__spirv_BuiltInGlobalOffset(int);
69	69	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_subgroup_size) __uint32_t
70		__spirv_SubgroupSize();
	70	__spirv_BuiltInSubgroupSize();
71	71	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_subgroup_max_size) __uint32_t
72		__spirv_SubgroupMaxSize();
	72	__spirv_BuiltInSubgroupMaxSize();
73	73	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_num_subgroups) __uint32_t
74		__spirv_NumSubgroups();
	74	__spirv_BuiltInNumSubgroups();
75	75	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_subgroup_id) __uint32_t
76		__spirv_SubgroupId();
	76	__spirv_BuiltInSubgroupId();
77	77	extern __SPIRV_BUILTIN_ALIAS(__builtin_spirv_subgroup_local_invocation_id)
78		__uint32_t __spirv_SubgroupLocalInvocationId();
	78	__uint32_t __spirv_BuiltInSubgroupLocalInvocationId();
79	79
80	80	// OpGenericCastToPtrExplicit
81	81

lib/include/__float_float.h created+176

...	...	@@ -0,0 +1,176 @@
	1	/*===---- __float_float.h --------------------------------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __CLANG_FLOAT_FLOAT_H
	11	#define __CLANG_FLOAT_FLOAT_H
	12
	13	#if (defined(__MINGW32__) \|\| defined(_MSC_VER) \|\| defined(_AIX)) && \
	14	__STDC_HOSTED__
	15
	16	/* Undefine anything that we'll be redefining below. */
	17	# undef FLT_EVAL_METHOD
	18	# undef FLT_ROUNDS
	19	# undef FLT_RADIX
	20	# undef FLT_MANT_DIG
	21	# undef DBL_MANT_DIG
	22	# undef LDBL_MANT_DIG
	23	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
	24	!defined(__STRICT_ANSI__) \|\| \
	25	(defined(__cplusplus) && __cplusplus >= 201103L) \|\| \
	26	(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
	27	# undef DECIMAL_DIG
	28	# endif
	29	# undef FLT_DIG
	30	# undef DBL_DIG
	31	# undef LDBL_DIG
	32	# undef FLT_MIN_EXP
	33	# undef DBL_MIN_EXP
	34	# undef LDBL_MIN_EXP
	35	# undef FLT_MIN_10_EXP
	36	# undef DBL_MIN_10_EXP
	37	# undef LDBL_MIN_10_EXP
	38	# undef FLT_MAX_EXP
	39	# undef DBL_MAX_EXP
	40	# undef LDBL_MAX_EXP
	41	# undef FLT_MAX_10_EXP
	42	# undef DBL_MAX_10_EXP
	43	# undef LDBL_MAX_10_EXP
	44	# undef FLT_MAX
	45	# undef DBL_MAX
	46	# undef LDBL_MAX
	47	# undef FLT_EPSILON
	48	# undef DBL_EPSILON
	49	# undef LDBL_EPSILON
	50	# undef FLT_MIN
	51	# undef DBL_MIN
	52	# undef LDBL_MIN
	53	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \|\| \
	54	!defined(__STRICT_ANSI__) \|\| \
	55	(defined(__cplusplus) && __cplusplus >= 201703L) \|\| \
	56	(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
	57	# undef FLT_TRUE_MIN
	58	# undef DBL_TRUE_MIN
	59	# undef LDBL_TRUE_MIN
	60	# undef FLT_DECIMAL_DIG
	61	# undef DBL_DECIMAL_DIG
	62	# undef LDBL_DECIMAL_DIG
	63	# undef FLT_HAS_SUBNORM
	64	# undef DBL_HAS_SUBNORM
	65	# undef LDBL_HAS_SUBNORM
	66	# endif
	67	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
	68	!defined(__STRICT_ANSI__)
	69	# undef FLT_NORM_MAX
	70	# undef DBL_NORM_MAX
	71	# undef LDBL_NORM_MAX
	72	#endif
	73	#endif
	74
	75	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
	76	!defined(__STRICT_ANSI__)
	77	# undef FLT_SNAN
	78	# undef DBL_SNAN
	79	# undef LDBL_SNAN
	80	#endif
	81
	82	/* Characteristics of floating point types, C99 5.2.4.2.2 */
	83
	84	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
	85	(defined(__cplusplus) && __cplusplus >= 201103L)
	86	#define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
	87	#endif
	88	#define FLT_ROUNDS (__builtin_flt_rounds())
	89	#define FLT_RADIX __FLT_RADIX__
	90
	91	#define FLT_MANT_DIG __FLT_MANT_DIG__
	92	#define DBL_MANT_DIG __DBL_MANT_DIG__
	93	#define LDBL_MANT_DIG __LDBL_MANT_DIG__
	94
	95	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
	96	!defined(__STRICT_ANSI__) \|\| \
	97	(defined(__cplusplus) && __cplusplus >= 201103L) \|\| \
	98	(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
	99	# define DECIMAL_DIG __DECIMAL_DIG__
	100	#endif
	101
	102	#define FLT_DIG __FLT_DIG__
	103	#define DBL_DIG __DBL_DIG__
	104	#define LDBL_DIG __LDBL_DIG__
	105
	106	#define FLT_MIN_EXP __FLT_MIN_EXP__
	107	#define DBL_MIN_EXP __DBL_MIN_EXP__
	108	#define LDBL_MIN_EXP __LDBL_MIN_EXP__
	109
	110	#define FLT_MIN_10_EXP __FLT_MIN_10_EXP__
	111	#define DBL_MIN_10_EXP __DBL_MIN_10_EXP__
	112	#define LDBL_MIN_10_EXP __LDBL_MIN_10_EXP__
	113
	114	#define FLT_MAX_EXP __FLT_MAX_EXP__
	115	#define DBL_MAX_EXP __DBL_MAX_EXP__
	116	#define LDBL_MAX_EXP __LDBL_MAX_EXP__
	117
	118	#define FLT_MAX_10_EXP __FLT_MAX_10_EXP__
	119	#define DBL_MAX_10_EXP __DBL_MAX_10_EXP__
	120	#define LDBL_MAX_10_EXP __LDBL_MAX_10_EXP__
	121
	122	#define FLT_MAX __FLT_MAX__
	123	#define DBL_MAX __DBL_MAX__
	124	#define LDBL_MAX __LDBL_MAX__
	125
	126	#define FLT_EPSILON __FLT_EPSILON__
	127	#define DBL_EPSILON __DBL_EPSILON__
	128	#define LDBL_EPSILON __LDBL_EPSILON__
	129
	130	#define FLT_MIN __FLT_MIN__
	131	#define DBL_MIN __DBL_MIN__
	132	#define LDBL_MIN __LDBL_MIN__
	133
	134	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \|\| \
	135	!defined(__STRICT_ANSI__) \|\| \
	136	(defined(__cplusplus) && __cplusplus >= 201703L) \|\| \
	137	(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
	138	# define FLT_TRUE_MIN __FLT_DENORM_MIN__
	139	# define DBL_TRUE_MIN __DBL_DENORM_MIN__
	140	# define LDBL_TRUE_MIN __LDBL_DENORM_MIN__
	141	# define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__
	142	# define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__
	143	# define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__
	144	# define FLT_HAS_SUBNORM __FLT_HAS_DENORM__
	145	# define DBL_HAS_SUBNORM __DBL_HAS_DENORM__
	146	# define LDBL_HAS_SUBNORM __LDBL_HAS_DENORM__
	147	#endif
	148
	149	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
	150	!defined(__STRICT_ANSI__)
	151	/* C23 5.2.5.3.2p28 */
	152	# define FLT_SNAN (__builtin_nansf(""))
	153	# define DBL_SNAN (__builtin_nans(""))
	154	# define LDBL_SNAN (__builtin_nansl(""))
	155
	156	/* C23 5.2.5.3.3p32 */
	157	# define FLT_NORM_MAX __FLT_NORM_MAX__
	158	# define DBL_NORM_MAX __DBL_NORM_MAX__
	159	# define LDBL_NORM_MAX __LDBL_NORM_MAX__
	160	#endif
	161
	162	#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__
	163	# define FLT16_MANT_DIG __FLT16_MANT_DIG__
	164	# define FLT16_DECIMAL_DIG __FLT16_DECIMAL_DIG__
	165	# define FLT16_DIG __FLT16_DIG__
	166	# define FLT16_MIN_EXP __FLT16_MIN_EXP__
	167	# define FLT16_MIN_10_EXP __FLT16_MIN_10_EXP__
	168	# define FLT16_MAX_EXP __FLT16_MAX_EXP__
	169	# define FLT16_MAX_10_EXP __FLT16_MAX_10_EXP__
	170	# define FLT16_MAX __FLT16_MAX__
	171	# define FLT16_EPSILON __FLT16_EPSILON__
	172	# define FLT16_MIN __FLT16_MIN__
	173	# define FLT16_TRUE_MIN __FLT16_TRUE_MIN__
	174	#endif /* __STDC_WANT_IEC_60559_TYPES_EXT__ */
	175
	176	#endif /* __CLANG_FLOAT_FLOAT_H */

lib/include/__float_header_macro.h created+12

...	...	@@ -0,0 +1,12 @@
	1	/*===---- __float_header_macro.h -------------------------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __CLANG_FLOAT_H
	11	#define __CLANG_FLOAT_H
	12	#endif /* __CLANG_FLOAT_H */

lib/include/__float_infinity_nan.h created+20

...	...	@@ -0,0 +1,20 @@
	1	/*===---- __float_infinity_nan.h -------------------------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __CLANG_FLOAT_INFINITY_NAN_H
	11	#define __CLANG_FLOAT_INFINITY_NAN_H
	12
	13	/* C23 5.2.5.3.3p29-30 */
	14	#undef INFINITY
	15	#undef NAN
	16
	17	#define INFINITY (__builtin_inff())
	18	#define NAN (__builtin_nanf(""))
	19
	20	#endif /* __CLANG_FLOAT_INFINITY_NAN_H */

lib/include/amo.h created+131

...	...	@@ -0,0 +1,131 @@
	1	/===---- amo.h - PowerPC Atomic Memory Operations ------------------------===\
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	\===----------------------------------------------------------------------===/
	8
	9	/* This header provides compatibility for GCC's AMO functions.
	10	* The functions here call Clang's underlying AMO builtins.
	11	*/
	12
	13	#ifndef _AMO_H
	14	#define _AMO_H
	15
	16	#include <stdint.h>
	17
	18	#ifdef __cplusplus
	19	extern "C" {
	20	#endif
	21
	22	/* AMO Load Operation Codes (FC values) */
	23	enum {
	24	_AMO_LD_ADD = 0x00, /* Fetch and Add */
	25	_AMO_LD_XOR = 0x01, /* Fetch and XOR */
	26	_AMO_LD_IOR = 0x02, /* Fetch and OR */
	27	_AMO_LD_AND = 0x03, /* Fetch and AND */
	28	_AMO_LD_UMAX = 0x04, /* Fetch and Maximum Unsigned */
	29	_AMO_LD_SMAX = 0x05, /* Fetch and Maximum Signed */
	30	_AMO_LD_UMIN = 0x06, /* Fetch and Minimum Unsigned */
	31	_AMO_LD_SMIN = 0x07, /* Fetch and Minimum Signed */
	32	_AMO_LD_SWAP = 0x08 /* Swap */
	33	};
	34
	35	/* 32-bit unsigned AMO load operations */
	36	static inline uint32_t amo_lwat_add(uint32_t *ptr, uint32_t val) {
	37	return __builtin_amo_lwat(ptr, val, _AMO_LD_ADD);
	38	}
	39
	40	static inline uint32_t amo_lwat_xor(uint32_t *ptr, uint32_t val) {
	41	return __builtin_amo_lwat(ptr, val, _AMO_LD_XOR);
	42	}
	43
	44	static inline uint32_t amo_lwat_ior(uint32_t *ptr, uint32_t val) {
	45	return __builtin_amo_lwat(ptr, val, _AMO_LD_IOR);
	46	}
	47
	48	static inline uint32_t amo_lwat_and(uint32_t *ptr, uint32_t val) {
	49	return __builtin_amo_lwat(ptr, val, _AMO_LD_AND);
	50	}
	51
	52	static inline uint32_t amo_lwat_umax(uint32_t *ptr, uint32_t val) {
	53	return __builtin_amo_lwat(ptr, val, _AMO_LD_UMAX);
	54	}
	55
	56	static inline uint32_t amo_lwat_umin(uint32_t *ptr, uint32_t val) {
	57	return __builtin_amo_lwat(ptr, val, _AMO_LD_UMIN);
	58	}
	59
	60	static inline uint32_t amo_lwat_swap(uint32_t *ptr, uint32_t val) {
	61	return __builtin_amo_lwat(ptr, val, _AMO_LD_SWAP);
	62	}
	63
	64	/* 32-bit signed AMO load operations */
	65	static inline int32_t amo_lwat_sadd(int32_t *ptr, int32_t val) {
	66	return __builtin_amo_lwat_s(ptr, val, _AMO_LD_ADD);
	67	}
	68
	69	static inline int32_t amo_lwat_smax(int32_t *ptr, int32_t val) {
	70	return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SMAX);
	71	}
	72
	73	static inline int32_t amo_lwat_smin(int32_t *ptr, int32_t val) {
	74	return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SMIN);
	75	}
	76
	77	static inline int32_t amo_lwat_sswap(int32_t *ptr, int32_t val) {
	78	return __builtin_amo_lwat_s(ptr, val, _AMO_LD_SWAP);
	79	}
	80
	81	/* 64-bit unsigned AMO load operations */
	82	static inline uint64_t amo_ldat_add(uint64_t *ptr, uint64_t val) {
	83	return __builtin_amo_ldat(ptr, val, _AMO_LD_ADD);
	84	}
	85
	86	static inline uint64_t amo_ldat_xor(uint64_t *ptr, uint64_t val) {
	87	return __builtin_amo_ldat(ptr, val, _AMO_LD_XOR);
	88	}
	89
	90	static inline uint64_t amo_ldat_ior(uint64_t *ptr, uint64_t val) {
	91	return __builtin_amo_ldat(ptr, val, _AMO_LD_IOR);
	92	}
	93
	94	static inline uint64_t amo_ldat_and(uint64_t *ptr, uint64_t val) {
	95	return __builtin_amo_ldat(ptr, val, _AMO_LD_AND);
	96	}
	97
	98	static inline uint64_t amo_ldat_umax(uint64_t *ptr, uint64_t val) {
	99	return __builtin_amo_ldat(ptr, val, _AMO_LD_UMAX);
	100	}
	101
	102	static inline uint64_t amo_ldat_umin(uint64_t *ptr, uint64_t val) {
	103	return __builtin_amo_ldat(ptr, val, _AMO_LD_UMIN);
	104	}
	105
	106	static inline uint64_t amo_ldat_swap(uint64_t *ptr, uint64_t val) {
	107	return __builtin_amo_ldat(ptr, val, _AMO_LD_SWAP);
	108	}
	109
	110	/* 64-bit signed AMO load operations */
	111	static inline int64_t amo_ldat_sadd(int64_t *ptr, int64_t val) {
	112	return __builtin_amo_ldat_s(ptr, val, _AMO_LD_ADD);
	113	}
	114
	115	static inline int64_t amo_ldat_smax(int64_t *ptr, int64_t val) {
	116	return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SMAX);
	117	}
	118
	119	static inline int64_t amo_ldat_smin(int64_t *ptr, int64_t val) {
	120	return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SMIN);
	121	}
	122
	123	static inline int64_t amo_ldat_sswap(int64_t *ptr, int64_t val) {
	124	return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SWAP);
	125	}
	126
	127	#ifdef __cplusplus
	128	}
	129	#endif
	130
	131	#endif /* _AMO_H */

lib/include/amxavx512intrin.h+215-1

...	...	@@ -16,7 +16,7 @@
16	16
17	17	#define __DEFAULT_FN_ATTRS_AVX512 \
18	18	__attribute__((__always_inline__, __nodebug__, \
19		__target__("amx-avx512,avx10.2-512")))
	19	__target__("amx-avx512,avx10.2"), __min_vector_width__(512)))
20	20
21	21	/// Moves a row from a tile register to a zmm destination register, converting
22	22	/// the int32 source elements to fp32. The row of the tile is selected by a
...	...	@@ -52,6 +52,40 @@
52	52	/// The row of the source tile
53	53	#define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row)
54	54
	55	/// Moves a row from a tile register to a zmm destination register, converting
	56	/// the int32 source elements to fp32. The row of the tile is selected by a
	57	/// 8b immediate value.
	58	///
	59	/// \headerfile <x86intrin.h>
	60	///
	61	/// \code
	62	/// __m512i _tile_cvtrowd2psi(__tile tsrc, const unsigned int imm8);
	63	/// \endcode
	64	///
	65	/// \code{.operation}
	66	/// VL := 512
	67	/// VL_bytes := VL >> 3
	68	/// row_index := imm8 & 0x3f
	69	/// row_chunk := (imm8 >> 6) * VL_bytes
	70	/// FOR i := 0 TO (VL_bytes / 4) - 1
	71	/// IF i + row_chunk / 4 >= tsrc.colsb / 4
	72	/// dst.dword[i] := 0
	73	/// ELSE
	74	/// dst.f32[i] := CONVERT_INT32_TO_FP32(tsrc.row[row_index].dword[row_chunk/4+i], RNE)
	75	/// FI
	76	/// ENDFOR
	77	/// dst[MAX_VL-1:VL] := 0
	78	/// zero_tileconfig_start()
	79	/// \endcode
	80	///
	81	/// This intrinsic corresponds to the \c TCVTROWD2PS instruction.
	82	///
	83	/// \param tsrc
	84	/// The source tile. Max size is 1024 Bytes.
	85	/// \param imm8
	86	/// The row of the source tile
	87	#define _tile_cvtrowd2psi(tsrc, imm8) __builtin_ia32_tcvtrowd2psi(tsrc, imm8)
	88
55	89	/// Moves a row from a tile register to a zmm destination register, converting
56	90	/// the fp32 source elements to bf16. It places the resulting bf16 elements
57	91	/// in the high 16 bits within each dword. The row of the tile is selected
...	...	@@ -89,6 +123,43 @@
89	123	#define _tile_cvtrowps2bf16h(tsrc, row) \
90	124	__builtin_ia32_tcvtrowps2bf16h(tsrc, row)
91	125
	126	/// Moves a row from a tile register to a zmm destination register, converting
	127	/// the fp32 source elements to bf16. It places the resulting bf16 elements
	128	/// in the high 16 bits within each dword. The row of the tile is selected
	129	/// by a 8b immediate value.
	130	///
	131	/// \headerfile <x86intrin.h>
	132	///
	133	/// \code
	134	/// __m512i _tile_cvtrowps2bf16hi(__tile tsrc, const unsigned int imm8);
	135	/// \endcode
	136	///
	137	/// \code{.operation}
	138	/// VL := 512
	139	/// VL_bytes := VL >> 3
	140	/// row_index := imm8 & 0x3f
	141	/// row_chunk := (imm8 >> 6) * VL_bytes
	142	/// FOR i := 0 TO (VL_bytes / 4) - 1
	143	/// IF i + row_chunk / 4 >= tsrc.colsb / 4
	144	/// dst.dword[i] := 0
	145	/// ELSE
	146	/// dst.word[2*i+0] := 0
	147	/// dst.bf16[2*i+1] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
	148	/// FI
	149	/// ENDFOR
	150	/// dst[MAX_VL-1:VL] := 0
	151	/// zero_tileconfig_start()
	152	/// \endcode
	153	///
	154	/// This intrinsic corresponds to the \c TCVTROWPS2BF16H instruction.
	155	///
	156	/// \param tsrc
	157	/// The source tile. Max size is 1024 Bytes.
	158	/// \param imm8
	159	/// The the row of the source tile.
	160	#define _tile_cvtrowps2bf16hi(tsrc, imm8) \
	161	__builtin_ia32_tcvtrowps2bf16hi(tsrc, imm8)
	162
92	163	/// Moves a row from a tile register to a zmm destination register, converting
93	164	/// the fp32 source elements to bf16. It places the resulting bf16 elements
94	165	/// in the low 16 bits within each dword. The row of the tile is selected
...	...	@@ -126,6 +197,43 @@
126	197	#define _tile_cvtrowps2bf16l(tsrc, row) \
127	198	__builtin_ia32_tcvtrowps2bf16l(tsrc, row)
128	199
	200	/// Moves a row from a tile register to a zmm destination register, converting
	201	/// the fp32 source elements to bf16. It places the resulting bf16 elements
	202	/// in the low 16 bits within each dword. The row of the tile is selected
	203	/// by a 8b immediate value.
	204	///
	205	/// \headerfile <x86intrin.h>
	206	///
	207	/// \code
	208	/// __m512i _tile_cvtrowps2bf16li(__tile tsrc, const unsigned int imm8);
	209	/// \endcode
	210	///
	211	/// \code{.operation}
	212	/// VL := 512
	213	/// VL_bytes := VL >> 3
	214	/// row_index := imm8 & 0x3f
	215	/// row_chunk := (imm8 >> 6) * VL_bytes
	216	/// FOR i := 0 TO (VL_bytes / 4) - 1
	217	/// IF i + row_chunk / 4 >= tsrc.colsb / 4
	218	/// dst.dword[i] := 0
	219	/// ELSE
	220	/// dst.word[2*i+1] := 0
	221	/// dst.bf16[2*i+0] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
	222	/// FI
	223	/// ENDFOR
	224	/// dst[MAX_VL-1:VL] := 0
	225	/// zero_tileconfig_start()
	226	/// \endcode
	227	///
	228	/// This intrinsic corresponds to the \c TCVTROWPS2BF16L instruction.
	229	///
	230	/// \param tsrc
	231	/// The source tile. Max size is 1024 Bytes.
	232	/// \param imm8
	233	/// The the row of the source tile.
	234	#define _tile_cvtrowps2bf16li(tsrc, imm8) \
	235	__builtin_ia32_tcvtrowps2bf16li(tsrc, imm8)
	236
129	237	/// Moves a row from a tile register to a zmm destination register, converting
130	238	/// the fp32 source elements to fp16. It places the resulting fp16 elements
131	239	/// in the high 16 bits within each dword. The row of the tile is selected
...	...	@@ -162,6 +270,43 @@
162	270	/// The the row of the source tile.
163	271	#define _tile_cvtrowps2phh(tsrc, row) __builtin_ia32_tcvtrowps2phh(tsrc, row)
164	272
	273	/// Moves a row from a tile register to a zmm destination register, converting
	274	/// the fp32 source elements to fp16. It places the resulting fp16 elements
	275	/// in the high 16 bits within each dword. The row of the tile is selected
	276	/// by a 8b immediate value.
	277	///
	278	/// \headerfile <x86intrin.h>
	279	///
	280	/// \code
	281	/// __m512i _tile_cvtrowps2phhi(__tile tsrc, constunsigned int imm8);
	282	/// \endcode
	283	///
	284	/// \code{.operation}
	285	/// VL := 512
	286	/// VL_bytes := VL >> 3
	287	/// row_index := imm8 & 0x3f
	288	/// row_chunk := (imm8 >> 6) * VL_bytes
	289	/// FOR i := 0 TO (VL_bytes / 4) - 1
	290	/// IF i + row_chunk / 4 >= tsrc.colsb / 4
	291	/// dst.dword[i] := 0
	292	/// ELSE
	293	/// dst.word[2*i+0] := 0
	294	/// dst.fp16[2*i+1] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
	295	/// FI
	296	/// ENDFOR
	297	/// dst[MAX_VL-1:VL] := 0
	298	/// zero_tileconfig_start()
	299	/// \endcode
	300	///
	301	/// This intrinsic corresponds to the \c TCVTROWPS2PHH instruction.
	302	///
	303	/// \param tsrc
	304	/// The source tile. Max size is 1024 Bytes.
	305	/// \param imm8
	306	/// The the row of the source tile.
	307	#define _tile_cvtrowps2phhi(tsrc, imm8) \
	308	__builtin_ia32_tcvtrowps2phhi(tsrc, imm8)
	309
165	310	/// Moves a row from a tile register to a zmm destination register, converting
166	311	/// the fp32 source elements to fp16. It places the resulting fp16 elements
167	312	/// in the low 16 bits within each dword. The row of the tile is selected
...	...	@@ -198,6 +343,43 @@
198	343	/// The the row of the source tile.
199	344	#define _tile_cvtrowps2phl(tsrc, row) __builtin_ia32_tcvtrowps2phl(tsrc, row)
200	345
	346	/// Moves a row from a tile register to a zmm destination register, converting
	347	/// the fp32 source elements to fp16. It places the resulting fp16 elements
	348	/// in the low 16 bits within each dword. The row of the tile is selected
	349	/// by a 8b immediate value.
	350	///
	351	/// \headerfile <x86intrin.h>
	352	///
	353	/// \code
	354	/// __m512i _tile_cvtrowps2phli(__tile tsrc, const unsigned int imm8);
	355	/// \endcode
	356	///
	357	/// \code{.operation}
	358	/// VL := 512
	359	/// VL_bytes := VL >> 3
	360	/// row_index := imm8 & 0x3f
	361	/// row_chunk := (imm8 >> 6) * VL_bytes
	362	/// FOR i := 0 TO (VL_bytes / 4) - 1
	363	/// IF i + row_chunk / 4 >= tsrc.colsb / 4
	364	/// dst.dword[i] := 0
	365	/// ELSE
	366	/// dst.word[2*i+1] := 0
	367	/// dst.fp16[2*i+0] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
	368	/// FI
	369	/// ENDFOR
	370	/// dst[MAX_VL-1:VL] := 0
	371	/// zero_tileconfig_start()
	372	/// \endcode
	373	///
	374	/// This intrinsic corresponds to the \c TCVTROWPS2PHL instruction.
	375	///
	376	/// \param tsrc
	377	/// The source tile. Max size is 1024 Bytes.
	378	/// \param imm8
	379	/// The the row of the source tile.
	380	#define _tile_cvtrowps2phli(tsrc, imm8) \
	381	__builtin_ia32_tcvtrowps2phli(tsrc, imm8)
	382
201	383	/// Move one row of a tile data to a v16f32 data.
202	384	/// The row of the tile is selected by a 32b GPR.
203	385	///
...	...	@@ -230,6 +412,38 @@
230	412	/// \endcode
231	413	#define _tile_movrow(a, b) ((__m512i)__builtin_ia32_tilemovrow(a, b))
232	414
	415	/// Move one row of a tile data to a v16f32 data.
	416	/// The row of the tile is selected by a 8b immediate value.
	417	///
	418	/// \headerfile <immintrin.h>
	419	///
	420	/// \code
	421	/// __m512 _tile_movrowi(__tile a, const unsigned b);
	422	/// \endcode
	423	///
	424	/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
	425	///
	426	/// \param a
	427	/// The 1st source tile. Max size is 1024 Bytes.
	428	/// \param b
	429	/// The 2nd source r32. Size is 4 Bytes.
	430	/// \returns
	431	/// The destination v16f32 data. Size is 64 Bytes.
	432	///
	433	/// \code{.operation}
	434	/// VL := 512
	435	/// VL_bytes := VL>>3
	436	/// row_index := b&0x3f
	437	/// row_chunk := (b>>6) * VL_bytes
	438	/// FOR i := 0 TO (VL_bytes-1)
	439	/// IF (row_chunk + i >= a.colsb)
	440	/// dst.byte[i] := 0
	441	/// ELSE
	442	/// dst.byte[i] := a.row[row_index].byte[row_chunk+i]
	443	/// ENDFOR
	444	/// \endcode
	445	#define _tile_movrowi(a, b) ((__m512i)__builtin_ia32_tilemovrowi(a, b))
	446
233	447	/// This is internal intrinsic. C/C++ user should avoid calling it directly.
234	448
235	449	static __inline__ __m512 __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowd2ps_internal(

lib/include/amxbf16transposeintrin.h deleted-94

...	...	@@ -1,94 +0,0 @@
1		/*===----- amxbf16transposeintrin.h - AMX-BF16 and AMX-TRANSPOSE ------------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		*===------------------------------------------------------------------------===
8		*/
9
10		#ifndef __IMMINTRIN_H
11		#error \
12		"Never use <amxbf16transposeintrin.h> directly; use <immintrin.h> instead."
13		#endif /* __IMMINTRIN_H */
14
15		#ifndef __AMX_BF16TRANSPOSEINTRIN_H
16		#define __AMX_BF16TRANSPOSEINTRIN_H
17		#ifdef __x86_64__
18
19		/* Define the default attributes for the functions in this file. */
20		#define __DEFAULT_FN_ATTRS \
21		__attribute__((__always_inline__, __nodebug__, \
22		__target__("amx-bf16,amx-transpose")))
23
24		/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
25		/// tiles \a a and \a b, accumulating the intermediate single-precision
26		/// (32-bit) floating-point elements with elements in \a dst, and store the
27		/// 32-bit result back to tile \a dst.
28		///
29		/// \headerfile <immintrin.h>
30		///
31		/// \code
32		/// void _tile_tdpbf16ps (__tile dst, __tile a, __tile b)
33		/// \endcode
34		///
35		/// \code{.operation}
36		/// FOR m := 0 TO dst.rows - 1
37		/// tmp := dst.row[m]
38		/// FOR k := 0 TO (a.colsb / 4) - 1
39		/// FOR n := 0 TO (dst.colsb / 4) - 1
40		/// tmp.bf32[n] += FP32(a.row[m].bf16[2k+0])
41		/// FP32(b.row[k].bf16[2*n+0])
42		/// tmp.bf32[n] += FP32(a.row[m].bf16[2k+1])
43		/// FP32(b.row[k].bf16[2*n+1])
44		/// ENDFOR
45		/// ENDFOR
46		/// write_row_and_zero(dst, m, tmp, dst.colsb)
47		/// ENDFOR
48		/// zero_upper_rows(dst, dst.rows)
49		/// zero_tileconfig_start()
50		/// \endcode
51		///
52		/// This intrinsic corresponds to the \c TTDPBF16PS instruction.
53		///
54		/// \param dst
55		/// The destination tile. Max size is 1024 Bytes.
56		/// \param a
57		/// The 1st source tile. Max size is 1024 Bytes.
58		/// \param b
59		/// The 2nd source tile. Max size is 1024 Bytes.
60		#define _tile_tdpbf16ps(dst, a, b) __builtin_ia32_ttdpbf16ps((dst), (a), (b))
61
62		/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63		static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64		_tile_tdpbf16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65		_tile1024i dst, _tile1024i src1, _tile1024i src2) {
66		return __builtin_ia32_ttdpbf16ps_internal(m, n, k, dst, src1, src2);
67		}
68
69		/// Compute transpose and dot-product of BF16 (16-bit) floating-point pairs in
70		/// tiles src0 and src1, accumulating the intermediate single-precision
71		/// (32-bit) floating-point elements with elements in "dst", and store the
72		/// 32-bit result back to tile "dst".
73		///
74		/// \headerfile <immintrin.h>
75		///
76		/// This intrinsic corresponds to the <c> TTDPBF16PS </c> instruction.
77		///
78		/// \param dst
79		/// The destination tile. Max size is 1024 Bytes.
80		/// \param src0
81		/// The 1st source tile. Max size is 1024 Bytes.
82		/// \param src1
83		/// The 2nd source tile. Max size is 1024 Bytes.
84		__DEFAULT_FN_ATTRS
85		static __inline__ void __tile_tdpbf16ps(__tile1024i *dst, __tile1024i src0,
86		__tile1024i src1) {
87		dst->tile = _tile_tdpbf16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88		src0.tile, src1.tile);
89		}
90
91		#undef __DEFAULT_FN_ATTRS
92
93		#endif /* __x86_64__ */
94		#endif /* __AMX_BF16TRANSPOSEINTRIN_H */

lib/include/amxcomplextransposeintrin.h deleted-303

...	...	@@ -1,303 +0,0 @@
1		/*===----- amxcomplextransposeintrin.h - AMX-COMPLEX and AMX-TRANSPOSE ------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		*===------------------------------------------------------------------------===
8		*/
9
10		#ifndef __IMMINTRIN_H
11		#error \
12		"Never use <amxcomplextransposeintrin.h> directly; include <immintrin.h> instead."
13		#endif // __IMMINTRIN_H
14
15		#ifndef __AMX_COMPLEXTRANSPOSEINTRIN_H
16		#define __AMX_COMPLEXTRANSPOSEINTRIN_H
17		#ifdef __x86_64__
18
19		#define __DEFAULT_FN_ATTRS \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("amx-complex,amx-transpose")))
22
23		/// Perform matrix multiplication of two tiles containing complex elements and
24		/// accumulate the results into a packed single precision tile. Each dword
25		/// element in input tiles \a a and \a b is interpreted as a complex number
26		/// with FP16 real part and FP16 imaginary part.
27		/// Calculates the imaginary part of the result. For each possible combination
28		/// of (transposed column of \a a, column of \a b), it performs a set of
29		/// multiplication and accumulations on all corresponding complex numbers
30		/// (one from \a a and one from \a b). The imaginary part of the \a a element
31		/// is multiplied with the real part of the corresponding \a b element, and
32		/// the real part of the \a a element is multiplied with the imaginary part
33		/// of the corresponding \a b elements. The two accumulated results are
34		/// added, and then accumulated into the corresponding row and column of
35		/// \a dst.
36		///
37		/// \headerfile <x86intrin.h>
38		///
39		/// \code
40		/// void _tile_tcmmimfp16ps(__tile dst, __tile a, __tile b);
41		/// \endcode
42		///
43		/// \code{.operation}
44		/// FOR m := 0 TO dst.rows - 1
45		/// tmp := dst.row[m]
46		/// FOR k := 0 TO a.rows - 1
47		/// FOR n := 0 TO (dst.colsb / 4) - 1
48		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0]) FP32(b.row[k].fp16[2*n+1])
49		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+1]) FP32(b.row[k].fp16[2*n+0])
50		/// ENDFOR
51		/// ENDFOR
52		/// write_row_and_zero(dst, m, tmp, dst.colsb)
53		/// ENDFOR
54		/// zero_upper_rows(dst, dst.rows)
55		/// zero_tileconfig_start()
56		/// \endcode
57		///
58		/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
59		///
60		/// \param dst
61		/// The destination tile. Max size is 1024 Bytes.
62		/// \param a
63		/// The 1st source tile. Max size is 1024 Bytes.
64		/// \param b
65		/// The 2nd source tile. Max size is 1024 Bytes.
66		#define _tile_tcmmimfp16ps(dst, a, b) \
67		__builtin_ia32_ttcmmimfp16ps((dst), (a), (b))
68
69		/// Perform matrix multiplication of two tiles containing complex elements and
70		/// accumulate the results into a packed single precision tile. Each dword
71		/// element in input tiles \a a and \a b is interpreted as a complex number
72		/// with FP16 real part and FP16 imaginary part.
73		/// Calculates the real part of the result. For each possible combination
74		/// of (rtransposed colum of \a a, column of \a b), it performs a set of
75		/// multiplication and accumulations on all corresponding complex numbers
76		/// (one from \a a and one from \a b). The real part of the \a a element is
77		/// multiplied with the real part of the corresponding \a b element, and the
78		/// negated imaginary part of the \a a element is multiplied with the
79		/// imaginary part of the corresponding \a b elements. The two accumulated
80		/// results are added, and then accumulated into the corresponding row and
81		/// column of \a dst.
82		///
83		/// \headerfile <x86intrin.h>
84		///
85		/// \code
86		/// void _tile_tcmmrlfp16ps(__tile dst, __tile a, __tile b);
87		/// \endcode
88		///
89		/// \code{.operation}
90		/// FOR m := 0 TO dst.rows - 1
91		/// tmp := dst.row[m]
92		/// FOR k := 0 TO a.rows - 1
93		/// FOR n := 0 TO (dst.colsb / 4) - 1
94		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0]) FP32(b.row[k].fp16[2*n+0])
95		/// tmp.fp32[n] += FP32(-a.row[m].fp16[2k+1]) FP32(b.row[k].fp16[2*n+1])
96		/// ENDFOR
97		/// ENDFOR
98		/// write_row_and_zero(dst, m, tmp, dst.colsb)
99		/// ENDFOR
100		/// zero_upper_rows(dst, dst.rows)
101		/// zero_tileconfig_start()
102		/// \endcode
103		///
104		/// This intrinsic corresponds to the \c TTCMMIMFP16PS instruction.
105		///
106		/// \param dst
107		/// The destination tile. Max size is 1024 Bytes.
108		/// \param a
109		/// The 1st source tile. Max size is 1024 Bytes.
110		/// \param b
111		/// The 2nd source tile. Max size is 1024 Bytes.
112		#define _tile_tcmmrlfp16ps(dst, a, b) \
113		__builtin_ia32_ttcmmrlfp16ps((dst), (a), (b))
114
115		/// Perform matrix conjugate transpose and multiplication of two tiles
116		/// containing complex elements and accumulate the results into a packed
117		/// single precision tile. Each dword element in input tiles \a a and \a b
118		/// is interpreted as a complex number with FP16 real part and FP16 imaginary
119		/// part.
120		/// Calculates the imaginary part of the result. For each possible combination
121		/// of (transposed column of \a a, column of \a b), it performs a set of
122		/// multiplication and accumulations on all corresponding complex numbers
123		/// (one from \a a and one from \a b). The negated imaginary part of the \a a
124		/// element is multiplied with the real part of the corresponding \a b
125		/// element, and the real part of the \a a element is multiplied with the
126		/// imaginary part of the corresponding \a b elements. The two accumulated
127		/// results are added, and then accumulated into the corresponding row and
128		/// column of \a dst.
129		///
130		/// \headerfile <x86intrin.h>
131		///
132		/// \code
133		/// void _tile_conjtcmmimfp16ps(__tile dst, __tile a, __tile b);
134		/// \endcode
135		///
136		/// \code{.operation}
137		/// FOR m := 0 TO dst.rows - 1
138		/// tmp := dst.row[m]
139		/// FOR k := 0 TO a.rows - 1
140		/// FOR n := 0 TO (dst.colsb / 4) - 1
141		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0]) FP32(b.row[k].fp16[2*n+1])
142		/// tmp.fp32[n] += FP32(-a.row[m].fp16[2k+1]) FP32(b.row[k].fp16[2*n+0])
143		/// ENDFOR
144		/// ENDFOR
145		/// write_row_and_zero(dst, m, tmp, dst.colsb)
146		/// ENDFOR
147		/// zero_upper_rows(dst, dst.rows)
148		/// zero_tileconfig_start()
149		/// \endcode
150		///
151		/// This intrinsic corresponds to the \c TCONJTCMMIMFP16PS instruction.
152		///
153		/// \param dst
154		/// The destination tile. Max size is 1024 Bytes.
155		/// \param a
156		/// The 1st source tile. Max size is 1024 Bytes.
157		/// \param b
158		/// The 2nd source tile. Max size is 1024 Bytes.
159		#define _tile_conjtcmmimfp16ps(dst, a, b) \
160		__builtin_ia32_tconjtcmmimfp16ps((dst), (a), (b))
161
162		/// Perform conjugate transpose of an FP16-pair of complex elements from \a a
163		/// and writes the result to \a dst.
164		///
165		/// \headerfile <x86intrin.h>
166		///
167		/// \code
168		/// void _tile_conjtfp16(__tile dst, __tile a);
169		/// \endcode
170		///
171		/// \code{.operation}
172		/// FOR i := 0 TO dst.rows - 1
173		/// FOR j := 0 TO (dst.colsb / 4) - 1
174		/// tmp.fp16[2j+0] := a.row[j].fp16[2i+0]
175		/// tmp.fp16[2j+1] := -a.row[j].fp16[2i+1]
176		/// ENDFOR
177		/// write_row_and_zero(dst, i, tmp, dst.colsb)
178		/// ENDFOR
179		/// zero_upper_rows(dst, dst.rows)
180		/// zero_tileconfig_start()
181		/// \endcode
182		///
183		/// This intrinsic corresponds to the \c TCONJTFP16 instruction.
184		///
185		/// \param dst
186		/// The destination tile. Max size is 1024 Bytes.
187		/// \param a
188		/// The source tile. Max size is 1024 Bytes.
189		#define _tile_conjtfp16(dst, a) __builtin_ia32_tconjtfp16((dst), (a))
190
191		static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmimfp16ps_internal(
192		unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
193		_tile1024i src1, _tile1024i src2) {
194		return __builtin_ia32_ttcmmimfp16ps_internal(m, n, k, dst, src1, src2);
195		}
196
197		static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_tcmmrlfp16ps_internal(
198		unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
199		_tile1024i src1, _tile1024i src2) {
200		return __builtin_ia32_ttcmmrlfp16ps_internal(m, n, k, dst, src1, src2);
201		}
202
203		static __inline__ _tile1024i __DEFAULT_FN_ATTRS _tile_conjtcmmimfp16ps_internal(
204		unsigned short m, unsigned short n, unsigned short k, _tile1024i dst,
205		_tile1024i src1, _tile1024i src2) {
206		return __builtin_ia32_tconjtcmmimfp16ps_internal(m, n, k, dst, src1, src2);
207		}
208
209		static __inline__ _tile1024i __DEFAULT_FN_ATTRS
210		_tile_conjtfp16_internal(unsigned short m, unsigned short n, _tile1024i src) {
211		return __builtin_ia32_tconjtfp16_internal(m, n, src);
212		}
213
214		/// Perform matrix multiplication of two tiles containing complex elements and
215		/// accumulate the results into a packed single precision tile. Each dword
216		/// element in input tiles src0 and src1 is interpreted as a complex number
217		/// with FP16 real part and FP16 imaginary part.
218		/// This function calculates the imaginary part of the result.
219		///
220		/// \headerfile <immintrin.h>
221		///
222		/// This intrinsic corresponds to the <c> TTCMMIMFP16PS </c> instruction.
223		///
224		/// \param dst
225		/// The destination tile. Max size is 1024 Bytes.
226		/// \param src0
227		/// The 1st source tile. Max size is 1024 Bytes.
228		/// \param src1
229		/// The 2nd source tile. Max size is 1024 Bytes.
230		__DEFAULT_FN_ATTRS
231		static void __tile_tcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
232		__tile1024i src1) {
233		dst->tile = _tile_tcmmimfp16ps_internal(src0.row, src1.col, src0.col,
234		dst->tile, src0.tile, src1.tile);
235		}
236
237		/// Perform matrix multiplication of two tiles containing complex elements and
238		/// accumulate the results into a packed single precision tile. Each dword
239		/// element in input tiles src0 and src1 is interpreted as a complex number
240		/// with FP16 real part and FP16 imaginary part.
241		/// This function calculates the real part of the result.
242		///
243		/// \headerfile <immintrin.h>
244		///
245		/// This intrinsic corresponds to the <c> TTCMMRLFP16PS </c> instruction.
246		///
247		/// \param dst
248		/// The destination tile. Max size is 1024 Bytes.
249		/// \param src0
250		/// The 1st source tile. Max size is 1024 Bytes.
251		/// \param src1
252		/// The 2nd source tile. Max size is 1024 Bytes.
253		__DEFAULT_FN_ATTRS
254		static void __tile_tcmmrlfp16ps(__tile1024i *dst, __tile1024i src0,
255		__tile1024i src1) {
256		dst->tile = _tile_tcmmrlfp16ps_internal(src0.row, src1.col, src0.col,
257		dst->tile, src0.tile, src1.tile);
258		}
259
260		/// Perform matrix conjugate transpose and multiplication of two tiles
261		/// containing complex elements and accumulate the results into a packed
262		/// single precision tile. Each dword element in input tiles src0 and src1
263		/// is interpreted as a complex number with FP16 real part and FP16 imaginary
264		/// part.
265		/// This function calculates the imaginary part of the result.
266		///
267		/// \headerfile <immintrin.h>
268		///
269		/// This intrinsic corresponds to the <c> TCONJTCMMIMFP16PS </c> instruction.
270		///
271		/// \param dst
272		/// The destination tile. Max size is 1024 Bytes.
273		/// \param src0
274		/// The 1st source tile. Max size is 1024 Bytes.
275		/// \param src1
276		/// The 2nd source tile. Max size is 1024 Bytes.
277		__DEFAULT_FN_ATTRS
278		static void __tile_conjtcmmimfp16ps(__tile1024i *dst, __tile1024i src0,
279		__tile1024i src1) {
280		dst->tile = _tile_conjtcmmimfp16ps_internal(src0.row, src1.col, src0.col,
281		dst->tile, src0.tile, src1.tile);
282		}
283
284		/// Perform conjugate transpose of an FP16-pair of complex elements from src and
285		/// writes the result to dst.
286		///
287		/// \headerfile <immintrin.h>
288		///
289		/// This intrinsic corresponds to the <c> TCONJTFP16 </c> instruction.
290		///
291		/// \param dst
292		/// The destination tile. Max size is 1024 Bytes.
293		/// \param src
294		/// The source tile. Max size is 1024 Bytes.
295		__DEFAULT_FN_ATTRS
296		static void __tile_conjtfp16(__tile1024i *dst, __tile1024i src) {
297		dst->tile = _tile_conjtfp16_internal(src.row, src.col, src.tile);
298		}
299
300		#undef __DEFAULT_FN_ATTRS
301
302		#endif // __x86_64__
303		#endif // __AMX_COMPLEXTRANSPOSEINTRIN_H

lib/include/amxfp16transposeintrin.h deleted-94

...	...	@@ -1,94 +0,0 @@
1		/*===----- amxfp16transposeintrin.h - AMX-FP16 and AMX-TRANSPOSE ------------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		*===------------------------------------------------------------------------===
8		*/
9
10		#ifndef __IMMINTRIN_H
11		#error \
12		"Never use <amxfp16transposeintrin.h> directly; use <immintrin.h> instead."
13		#endif /* __IMMINTRIN_H */
14
15		#ifndef __AMX_FP16TRANSPOSEINTRIN_H
16		#define __AMX_FP16TRANSPOSEINTRIN_H
17		#ifdef __x86_64__
18
19		/* Define the default attributes for the functions in this file. */
20		#define __DEFAULT_FN_ATTRS \
21		__attribute__((__always_inline__, __nodebug__, \
22		__target__("amx-fp16,amx-transpose")))
23
24		/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
25		/// tiles \a a and \a b, accumulating the intermediate single-precision
26		/// (32-bit) floating-point elements with elements in \a dst, and store the
27		/// 32-bit result back to tile \a dst.
28		///
29		/// \headerfile <immintrin.h>
30		///
31		/// \code
32		/// void _tile_tdpfp16ps (__tile dst, __tile a, __tile b)
33		/// \endcode
34		///
35		/// \code{.operation}
36		/// FOR m := 0 TO dst.rows - 1
37		/// tmp := dst.row[m]
38		/// FOR k := 0 TO (a.colsb / 4) - 1
39		/// FOR n := 0 TO (dst.colsb / 4) - 1
40		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+0])
41		/// FP32(b.row[k].fp16[2*n+0])
42		/// tmp.fp32[n] += FP32(a.row[m].fp16[2k+1])
43		/// FP32(b.row[k].fp16[2*n+1])
44		/// ENDFOR
45		/// ENDFOR
46		/// write_row_and_zero(dst, m, tmp, dst.colsb)
47		/// ENDFOR
48		/// zero_upper_rows(dst, dst.rows)
49		/// zero_tileconfig_start()
50		/// \endcode
51		///
52		/// This intrinsic corresponds to the \c TTDPFP16PS instruction.
53		///
54		/// \param dst
55		/// The destination tile. Max size is 1024 Bytes.
56		/// \param a
57		/// The 1st source tile. Max size is 1024 Bytes.
58		/// \param b
59		/// The 2nd source tile. Max size is 1024 Bytes.
60		#define _tile_tdpfp16ps(dst, a, b) __builtin_ia32_ttdpfp16ps((dst), (a), (b))
61
62		/// This is internal intrinsic. C/C++ user should avoid calling it directly.
63		static __inline__ _tile1024i __DEFAULT_FN_ATTRS
64		_tile_tdpfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
65		_tile1024i dst, _tile1024i src1, _tile1024i src2) {
66		return __builtin_ia32_ttdpfp16ps_internal(m, n, k, dst, src1, src2);
67		}
68
69		/// Compute transpose and dot-product of FP16 (16-bit) floating-point pairs in
70		/// tiles src0 and src1, accumulating the intermediate single-precision
71		/// (32-bit) floating-point elements with elements in "dst", and store the
72		/// 32-bit result back to tile "dst".
73		///
74		/// \headerfile <immintrin.h>
75		///
76		/// This intrinsic corresponds to the <c> TTDPFP16PS </c> instruction.
77		///
78		/// \param dst
79		/// The destination tile. Max size is 1024 Bytes.
80		/// \param src0
81		/// The 1st source tile. Max size is 1024 Bytes.
82		/// \param src1
83		/// The 2nd source tile. Max size is 1024 Bytes.
84		__DEFAULT_FN_ATTRS
85		static __inline__ void __tile_tdpfp16ps(__tile1024i *dst, __tile1024i src0,
86		__tile1024i src1) {
87		dst->tile = _tile_tdpfp16ps_internal(src0.row, src1.col, src0.col, dst->tile,
88		src0.tile, src1.tile);
89		}
90
91		#undef __DEFAULT_FN_ATTRS
92
93		#endif /* __x86_64__ */
94		#endif /* __AMX_FP16TRANSPOSEINTRIN_H */

lib/include/amxintrin.h-2

...	...	@@ -230,8 +230,6 @@ static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {
230	230	/// bytes. Since there is no 2D type in llvm IR, we use vector type to
231	231	/// represent 2D tile and the fixed size is maximum amx tile register size.
232	232	typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
233		typedef int _tile1024i_1024a
234		__attribute__((__vector_size__(1024), __aligned__(1024)));
235	233
236	234	/// This is internal intrinsic. C/C++ user should avoid calling it directly.
237	235	static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TILE

lib/include/amxmovrstransposeintrin.h deleted-200

...	...	@@ -1,200 +0,0 @@
1		/* ===--- amxmovrstransposeintrin.h - AMX_MOVRS_TRANSPOSE intrinsics --------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		* ===-----------------------------------------------------------------------===
8		*/
9
10		#ifndef __IMMINTRIN_H
11		#error \
12		"Never use <amxmovrstransposeintrin.h> directly; use <immintrin.h> instead."
13		#endif /* __IMMINTRIN_H */
14
15		#ifndef __AMX_MOVRS_TRANSPOSEINTRIN_H
16		#define __AMX_MOVRS_TRANSPOSEINTRIN_H
17		#ifdef __x86_64__
18
19		#define __DEFAULT_FN_ATTRS \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("amx-transpose,amx-movrs")))
22
23		#define _tile_2rpntlvwz0rs(tdst, base, stride) \
24		__builtin_ia32_t2rpntlvwz0rs(tdst, base, stride)
25		#define _tile_2rpntlvwz0rst1(tdst, base, stride) \
26		__builtin_ia32_t2rpntlvwz0rst1(tdst, base, stride)
27		#define _tile_2rpntlvwz1rs(tdst, base, stride) \
28		__builtin_ia32_t2rpntlvwz1rs(tdst, base, stride)
29		#define _tile_2rpntlvwz1rst1(tdst, base, stride) \
30		__builtin_ia32_t2rpntlvwz1rst1(tdst, base, stride)
31
32		static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rs_internal(
33		unsigned short row, unsigned short col0, unsigned short col1,
34		_tile1024i dst0, _tile1024i dst1, const void *base,
35		__SIZE_TYPE__ stride) {
36		// Use __tile1024i_1024a* to escape the alignment check in
37		// clang/test/Headers/x86-intrinsics-headers-clean.cpp
38		__builtin_ia32_t2rpntlvwz0rs_internal(
39		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
40		(__SIZE_TYPE__)(stride));
41		}
42
43		static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rst1_internal(
44		unsigned short row, unsigned short col0, unsigned short col1,
45		_tile1024i dst0, _tile1024i dst1, const void *base,
46		__SIZE_TYPE__ stride) {
47		__builtin_ia32_t2rpntlvwz0rst1_internal(
48		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
49		(__SIZE_TYPE__)(stride));
50		}
51
52		static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rs_internal(
53		unsigned short row, unsigned short col0, unsigned short col1,
54		_tile1024i dst0, _tile1024i dst1, const void *base,
55		__SIZE_TYPE__ stride) {
56		__builtin_ia32_t2rpntlvwz1rs_internal(
57		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
58		(__SIZE_TYPE__)(stride));
59		}
60
61		static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rst1_internal(
62		unsigned short row, unsigned short col0, unsigned short col1,
63		_tile1024i dst0, _tile1024i dst1, const void *base,
64		__SIZE_TYPE__ stride) {
65		__builtin_ia32_t2rpntlvwz1rst1_internal(
66		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
67		(__SIZE_TYPE__)(stride));
68		}
69
70		/// Converts a pair of tiles from memory into VNNI format, and places the
71		/// results in a pair of destinations specified by dst. The pair of tiles
72		/// in memory is specified via a tsib; the second tile is after the first
73		/// one, separated by the same stride that separates each row.
74		/// The tile configuration for the destination tiles indicates the amount
75		/// of data to read from memory. The instruction will load a number of rows
76		/// that is equal to twice the number of rows in tmm1. The size of each row
77		/// is equal to the average width of the destination tiles. If the second
78		/// tile is configured with zero rows and columns, only the first tile will
79		/// be written.
80		/// Provides a hint to the implementation that the data will likely become
81		/// read shared in the near future and the data caching can be optimized.
82		///
83		/// \headerfile <immintrin.h>
84		///
85		/// This intrinsic corresponds to the <c> T2RPNTLVWZ0RS </c> instruction.
86		///
87		/// \param dst0
88		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
89		/// \param dst1
90		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
91		/// \param base
92		/// A pointer to base address.
93		/// \param stride
94		/// The stride between the rows' data to be loaded in memory.
95		__DEFAULT_FN_ATTRS
96		static void __tile_2rpntlvwz0rs(__tile1024i dst0, __tile1024i dst1,
97		const void *base, __SIZE_TYPE__ stride) {
98		_tile_2rpntlvwz0rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
99		&dst1->tile, base, stride);
100		}
101
102		/// Converts a pair of tiles from memory into VNNI format, and places the
103		/// results in a pair of destinations specified by dst. The pair of tiles
104		/// in memory is specified via a tsib; the second tile is after the first
105		/// one, separated by the same stride that separates each row.
106		/// The tile configuration for the destination tiles indicates the amount
107		/// of data to read from memory. The instruction will load a number of rows
108		/// that is equal to twice the number of rows in tmm1. The size of each row
109		/// is equal to the average width of the destination tiles. If the second
110		/// tile is configured with zero rows and columns, only the first tile will
111		/// be written.
112		///
113		/// \headerfile <immintrin.h>
114		///
115		/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1RS </c> instruction.
116		///
117		/// \param dst0
118		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
119		/// \param dst1
120		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
121		/// \param base
122		/// A pointer to base address.
123		/// \param stride
124		/// The stride between the rows' data to be loaded in memory.
125		__DEFAULT_FN_ATTRS
126		static void __tile_2rpntlvwz0rst1(__tile1024i dst0, __tile1024i dst1,
127		const void *base, __SIZE_TYPE__ stride) {
128		_tile_2rpntlvwz0rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
129		&dst1->tile, base, stride);
130		}
131
132		/// Converts a pair of tiles from memory into VNNI format, and places the
133		/// results in a pair of destinations specified by dst. The pair of tiles
134		/// in memory is specified via a tsib; the second tile is after the first
135		/// one, separated by the same stride that separates each row.
136		/// The tile configuration for the destination tiles indicates the amount
137		/// of data to read from memory. The instruction will load a number of rows
138		/// that is equal to twice the number of rows in tmm1. The size of each row
139		/// is equal to the average width of the destination tiles. If the second
140		/// tile is configured with zero rows and columns, only the first tile will
141		/// be written. The last row will be not be read from memory but instead
142		/// filled with zeros.
143		/// Provides a hint to the implementation that the data will likely become
144		/// read shared in the near future and the data caching can be optimized.
145		///
146		/// \headerfile <immintrin.h>
147		///
148		/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
149		///
150		/// \param dst0
151		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
152		/// \param dst1
153		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
154		/// \param base
155		/// A pointer to base address.
156		/// \param stride
157		/// The stride between the rows' data to be loaded in memory.
158		__DEFAULT_FN_ATTRS
159		static void __tile_2rpntlvwz1rs(__tile1024i dst0, __tile1024i dst1,
160		const void *base, __SIZE_TYPE__ stride) {
161		_tile_2rpntlvwz1rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
162		&dst1->tile, base, stride);
163		}
164
165		/// Converts a pair of tiles from memory into VNNI format, and places the
166		/// results in a pair of destinations specified by dst. The pair of tiles
167		/// in memory is specified via a tsib; the second tile is after the first
168		/// one, separated by the same stride that separates each row.
169		/// The tile configuration for the destination tiles indicates the amount
170		/// of data to read from memory. The instruction will load a number of rows
171		/// that is equal to twice the number of rows in tmm1. The size of each row
172		/// is equal to the average width of the destination tiles. If the second
173		/// tile is configured with zero rows and columns, only the first tile will
174		/// be written. The last row will be not be read from memory but instead
175		/// filled with zeros.
176		/// Provides a hint to the implementation that the data will likely become
177		/// read shared in the near future and the data caching can be optimized.
178		///
179		/// \headerfile <immintrin.h>
180		///
181		/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1RS </c> instruction.
182		///
183		/// \param dst0
184		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
185		/// \param dst1
186		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
187		/// \param base
188		/// A pointer to base address.
189		/// \param stride
190		/// The stride between the rows' data to be loaded in memory.
191		__DEFAULT_FN_ATTRS
192		static void __tile_2rpntlvwz1rst1(__tile1024i dst0, __tile1024i dst1,
193		const void *base, __SIZE_TYPE__ stride) {
194		_tile_2rpntlvwz1rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
195		&dst1->tile, base, stride);
196		}
197
198		#undef __DEFAULT_FN_ATTRS
199		#endif /* __x86_64__ */
200		#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */

lib/include/amxtf32transposeintrin.h deleted-105

...	...	@@ -1,105 +0,0 @@
1		/*===--------- amxtf32transposeintrin.h - AMX-TF32 and AMX-TRANSPOSE --------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		*===------------------------------------------------------------------------===
8		*/
9		#ifndef __IMMINTRIN_H
10		#error \
11		"Never use <amxtf32transposeintrin.h> directly; include <immintrin.h> instead."
12		#endif // __IMMINTRIN_H
13
14		#ifndef __AMX_TF32TRANSPOSEINTRIN_H
15		#define __AMX_TF32TRANSPOSEINTRIN_H
16		#ifdef __x86_64__
17
18		#define __DEFAULT_FN_ATTRS_TF32_TRANSPOSE \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("amx-tf32,amx-transpose")))
21
22		/// \code
23		/// void _tile_tmmultf32ps(constexpr int srcdst, constexpr int a, \
24		/// constexpr int b);
25		/// \endcode
26		///
27		/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
28		///
29		/// \param srcdst
30		/// The destination tile. Max size is 1024 Bytes.
31		/// \param a
32		/// The 1st source tile. Max size is 1024 Bytes.
33		/// \param b
34		/// The 2nd source tile. Max size is 1024 Bytes.
35		///
36		/// \code{.operation}
37		/// DEFINE zero_lower_mantissa_bits_fp32(x[31:0]) {
38		/// dword[12:0] := 0
39		/// dword[31:13] := x[31:13]
40		/// return dword
41		/// }
42		///
43		/// DEFINE silence_snan_fp32(x[31:0]) {
44		/// IF (x.exponent == 255 and x.fraction != 0 and x.fraction[22] == 0)
45		/// x.fraction[22] := 1
46		/// return x
47		/// }
48		///
49		/// elements_dest:= srcdst.colsb/4
50		///
51		/// FOR m := 0 TO (srcdst.rows-1)
52		/// tmp[511:0] := 0
53		/// FOR k := 0 TO (a.rows-1)
54		/// FOR n := 0 TO (elements_dest-1)
55		/// a1e := silence_snan_fp32(a.row[k].fp32[m])
56		/// a2e := silence_snan_fp32(b.row[k].fp32[n])
57		/// s1e := zero_lower_mantissa_bits_fp32(a1e)
58		/// s2e := zero_lower_mantissa_bits_fp32(a2e)
59		/// tmp.fp32[n] += s1e * s2e
60		/// ENDFOR
61		/// ENDFOR
62		///
63		/// FOR n := 0 TO (elements_dest-1)
64		/// tmp.fp32[n] += srcdst.row[m].fp32[n]
65		/// ENDFOR
66		/// write_row_and_zero(srcdst, m, tmp, srcdst.colsb)
67		///
68		/// ENDFOR
69		///
70		/// zero_upper_rows(srcdst, srcdst.rows)
71		/// zero_tileconfig_start()
72		/// \endcode
73		#define _tile_tmmultf32ps(srcdst, a, b) \
74		__builtin_ia32_ttmmultf32ps((srcdst), (a), (b))
75
76		// dst = m x n (srcdest), src1 = k x m, src2 = k x n
77		static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TF32_TRANSPOSE
78		_tile_tmmultf32ps_internal(unsigned short m, unsigned short n, unsigned short k,
79		_tile1024i dst, _tile1024i src1, _tile1024i src2) {
80		return __builtin_ia32_ttmmultf32ps_internal(m, n, k, dst, src1, src2);
81		}
82
83		/// Compute transpose and do Matrix Multiplication of src0 and src1, and then do
84		/// Matrix Plus with dst. All the calculation is base on float32 but with the
85		/// lower 13-bit set to 0.
86		///
87		/// \headerfile <immintrin.h>
88		///
89		/// This intrinsic corresponds to the <c> TTMMULTF32PS </c> instruction.
90		///
91		/// \param dst
92		/// The destination tile. Max size is 1024 Bytes.
93		/// \param src0
94		/// The 1st source tile. Max size is 1024 Bytes.
95		/// \param src1
96		/// The 2nd source tile. Max size is 1024 Bytes.
97		__DEFAULT_FN_ATTRS_TF32_TRANSPOSE
98		static void __tile_tmmultf32ps(__tile1024i *dst, __tile1024i src0,
99		__tile1024i src1) {
100		dst->tile = _tile_tmmultf32ps_internal(src0.row, src1.col, src0.col,
101		dst->tile, src0.tile, src1.tile);
102		}
103
104		#endif // __x86_64__
105		#endif // __AMX_TF32TRANSPOSEINTRIN_H

lib/include/amxtransposeintrin.h deleted-248

...	...	@@ -1,248 +0,0 @@
1		/* ===--- amxtransposeintrin.h - AMX_TRANSPOSE intrinsics -- C++ ----------===
2		*
3		* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		* See https://llvm.org/LICENSE.txt for license information.
5		* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		*
7		* ===-----------------------------------------------------------------------===
8		*/
9
10		#ifndef __IMMINTRIN_H
11		#error "Never use <amxtransposeintrin.h> directly; use <immintrin.h> instead."
12		#endif /* __IMMINTRIN_H */
13
14		#ifndef __AMX_TRANSPOSEINTRIN_H
15		#define __AMX_TRANSPOSEINTRIN_H
16		#ifdef __x86_64__
17
18		#define __DEFAULT_FN_ATTRS_TRANSPOSE \
19		__attribute__((__always_inline__, __nodebug__, __target__("amx-transpose")))
20
21		#define _tile_2rpntlvwz0(tdst, base, stride) \
22		__builtin_ia32_t2rpntlvwz0(tdst, base, stride)
23		#define _tile_2rpntlvwz0t1(tdst, base, stride) \
24		__builtin_ia32_t2rpntlvwz0t1(tdst, base, stride)
25		#define _tile_2rpntlvwz1(tdst, base, stride) \
26		__builtin_ia32_t2rpntlvwz1(tdst, base, stride)
27		#define _tile_2rpntlvwz1t1(tdst, base, stride) \
28		__builtin_ia32_t2rpntlvwz1t1(tdst, base, stride)
29
30		/// Transpose 32-bit elements from \a src and write the result to \a dst.
31		///
32		/// \headerfile <immintrin.h>
33		///
34		/// \code
35		/// void _tile_transposed(__tile dst, __tile src);
36		/// \endcode
37		///
38		/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
39		///
40		/// \param dst
41		/// The destination tile. Max size is 1024 Bytes.
42		/// \param src
43		/// The source tile. Max size is 1024 Bytes.
44		///
45		/// \code{.operation}
46		///
47		/// FOR i := 0 TO (dst.rows-1)
48		/// tmp[511:0] := 0
49		/// FOR j := 0 TO (dst.colsb/4-1)
50		/// tmp.dword[j] := src.row[j].dword[i]
51		/// ENDFOR
52		/// dst.row[i] := tmp
53		/// ENDFOR
54		///
55		/// zero_upper_rows(dst, dst.rows)
56		/// zero_tileconfig_start()
57		/// \endcode
58		#define _tile_transposed(dst, src) __builtin_ia32_ttransposed(dst, src)
59
60		static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0_internal(
61		unsigned short row, unsigned short col0, unsigned short col1,
62		_tile1024i dst0, _tile1024i dst1, const void *base,
63		__SIZE_TYPE__ stride) {
64		// Use __tile1024i_1024a* to escape the alignment check in
65		// clang/test/Headers/x86-intrinsics-headers-clean.cpp
66		__builtin_ia32_t2rpntlvwz0_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
67		(_tile1024i_1024a *)dst1, base,
68		(__SIZE_TYPE__)(stride));
69		}
70
71		static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz0t1_internal(
72		unsigned short row, unsigned short col0, unsigned short col1,
73		_tile1024i dst0, _tile1024i dst1, const void *base,
74		__SIZE_TYPE__ stride) {
75		__builtin_ia32_t2rpntlvwz0t1_internal(
76		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
77		(__SIZE_TYPE__)(stride));
78		}
79
80		static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1_internal(
81		unsigned short row, unsigned short col0, unsigned short col1,
82		_tile1024i dst0, _tile1024i dst1, const void *base,
83		__SIZE_TYPE__ stride) {
84		__builtin_ia32_t2rpntlvwz1_internal(row, col0, col1, (_tile1024i_1024a *)dst0,
85		(_tile1024i_1024a *)dst1, base,
86		(__SIZE_TYPE__)(stride));
87		}
88
89		static __inline__ void __DEFAULT_FN_ATTRS_TRANSPOSE _tile_2rpntlvwz1t1_internal(
90		unsigned short row, unsigned short col0, unsigned short col1,
91		_tile1024i dst0, _tile1024i dst1, const void *base,
92		__SIZE_TYPE__ stride) {
93		__builtin_ia32_t2rpntlvwz1t1_internal(
94		row, col0, col1, (_tile1024i_1024a )dst0, (_tile1024i_1024a )dst1, base,
95		(__SIZE_TYPE__)(stride));
96		}
97
98		// This is internal intrinsic. C/C++ user should avoid calling it directly.
99		static __inline__ _tile1024i __DEFAULT_FN_ATTRS_TRANSPOSE
100		_tile_transposed_internal(unsigned short m, unsigned short n, _tile1024i src) {
101		return __builtin_ia32_ttransposed_internal(m, n, src);
102		}
103
104		/// Converts a pair of tiles from memory into VNNI format, and places the
105		/// results in a pair of destinations specified by dst. The pair of tiles
106		/// in memory is specified via a tsib; the second tile is after the first
107		/// one, separated by the same stride that separates each row.
108		/// The tile configuration for the destination tiles indicates the amount
109		/// of data to read from memory. The instruction will load a number of rows
110		/// that is equal to twice the number of rows in tmm1. The size of each row
111		/// is equal to the average width of the destination tiles. If the second
112		/// tile is configured with zero rows and columns, only the first tile will
113		/// be written.
114		/// Provides a hint to the implementation that the data will likely not be
115		/// reused in the near future and the data caching can be optimized.
116		///
117		/// \headerfile <immintrin.h>
118		///
119		/// This intrinsic corresponds to the <c> T2RPNTLVWZ0 </c> instruction.
120		///
121		/// \param dst0
122		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
123		/// \param dst1
124		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
125		/// \param base
126		/// A pointer to base address.
127		/// \param stride
128		/// The stride between the rows' data to be loaded in memory.
129		__DEFAULT_FN_ATTRS_TRANSPOSE
130		static void __tile_2rpntlvwz0(__tile1024i dst0, __tile1024i dst1,
131		const void *base, __SIZE_TYPE__ stride) {
132		_tile_2rpntlvwz0_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
133		&dst1->tile, base, stride);
134		}
135
136		/// Converts a pair of tiles from memory into VNNI format, and places the
137		/// results in a pair of destinations specified by dst. The pair of tiles
138		/// in memory is specified via a tsib; the second tile is after the first
139		/// one, separated by the same stride that separates each row.
140		/// The tile configuration for the destination tiles indicates the amount
141		/// of data to read from memory. The instruction will load a number of rows
142		/// that is equal to twice the number of rows in tmm1. The size of each row
143		/// is equal to the average width of the destination tiles. If the second
144		/// tile is configured with zero rows and columns, only the first tile will
145		/// be written.
146		///
147		/// \headerfile <immintrin.h>
148		///
149		/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1 </c> instruction.
150		///
151		/// \param dst0
152		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
153		/// \param dst1
154		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
155		/// \param base
156		/// A pointer to base address.
157		/// \param stride
158		/// The stride between the rows' data to be loaded in memory.
159		__DEFAULT_FN_ATTRS_TRANSPOSE
160		static void __tile_2rpntlvwz0t1(__tile1024i dst0, __tile1024i dst1,
161		const void *base, __SIZE_TYPE__ stride) {
162		_tile_2rpntlvwz0t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
163		&dst1->tile, base, stride);
164		}
165
166		/// Converts a pair of tiles from memory into VNNI format, and places the
167		/// results in a pair of destinations specified by dst. The pair of tiles
168		/// in memory is specified via a tsib; the second tile is after the first
169		/// one, separated by the same stride that separates each row.
170		/// The tile configuration for the destination tiles indicates the amount
171		/// of data to read from memory. The instruction will load a number of rows
172		/// that is equal to twice the number of rows in tmm1. The size of each row
173		/// is equal to the average width of the destination tiles. If the second
174		/// tile is configured with zero rows and columns, only the first tile will
175		/// be written. The last row will be not be read from memory but instead
176		/// filled with zeros.
177		/// Provides a hint to the implementation that the data will likely not be
178		/// reused in the near future and the data caching can be optimized.
179		///
180		/// \headerfile <immintrin.h>
181		///
182		/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
183		///
184		/// \param dst0
185		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
186		/// \param dst1
187		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
188		/// \param base
189		/// A pointer to base address.
190		/// \param stride
191		/// The stride between the rows' data to be loaded in memory.
192		__DEFAULT_FN_ATTRS_TRANSPOSE
193		static void __tile_2rpntlvwz1(__tile1024i dst0, __tile1024i dst1,
194		const void *base, __SIZE_TYPE__ stride) {
195		_tile_2rpntlvwz1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
196		&dst1->tile, base, stride);
197		}
198
199		/// Converts a pair of tiles from memory into VNNI format, and places the
200		/// results in a pair of destinations specified by dst. The pair of tiles
201		/// in memory is specified via a tsib; the second tile is after the first
202		/// one, separated by the same stride that separates each row.
203		/// The tile configuration for the destination tiles indicates the amount
204		/// of data to read from memory. The instruction will load a number of rows
205		/// that is equal to twice the number of rows in tmm1. The size of each row
206		/// is equal to the average width of the destination tiles. If the second
207		/// tile is configured with zero rows and columns, only the first tile will
208		/// be written. The last row will be not be read from memory but instead
209		/// filled with zeros.
210		/// Provides a hint to the implementation that the data will likely not be
211		/// reused in the near future and the data caching can be optimized.
212		///
213		/// \headerfile <immintrin.h>
214		///
215		/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1 </c> instruction.
216		///
217		/// \param dst0
218		/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
219		/// \param dst1
220		/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
221		/// \param base
222		/// A pointer to base address.
223		/// \param stride
224		/// The stride between the rows' data to be loaded in memory.
225		__DEFAULT_FN_ATTRS_TRANSPOSE
226		static void __tile_2rpntlvwz1t1(__tile1024i dst0, __tile1024i dst1,
227		const void *base, __SIZE_TYPE__ stride) {
228		_tile_2rpntlvwz1t1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
229		&dst1->tile, base, stride);
230		}
231
232		/// Transpose 32-bit elements from src and write the result to dst.
233		///
234		/// \headerfile <immintrin.h>
235		///
236		/// This intrinsic corresponds to the <c> TTRANSPOSED </c> instruction.
237		///
238		/// \param dst
239		/// The destination tile. Max size is 1024 Bytes.
240		/// \param src
241		/// The source tile. Max size is 1024 Bytes.
242		__DEFAULT_FN_ATTRS_TRANSPOSE
243		static void __tile_transposed(__tile1024i *dst, __tile1024i src) {
244		dst->tile = _tile_transposed_internal(dst->row, dst->col, src.tile);
245		}
246
247		#endif /* __x86_64__ */
248		#endif /* __AMX_TRANSPOSEINTRIN_H */

lib/include/arm_acle.h+37-27

...	...	@@ -55,11 +55,37 @@ __chkfeat(uint64_t __features) {
55	55	/* 7.5 Swap */
56	56	static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
57	57	__swp(uint32_t __x, volatile uint32_t *__p) {
58		uint32_t v;
59		do
60		v = __builtin_arm_ldrex(__p);
61		while (__builtin_arm_strex(__x, __p));
62		return v;
	58	uint32_t __v;
	59	#if (__ARM_FEATURE_LDREX & 4) \|\| __ARM_ARCH_6M__ \|\| __linux__
	60	/*
	61	* Using this clang builtin is sensible in most situations. Where
	62	* LDREX and STREX are available, it will compile to a loop using
	63	* them. Otherwise it will compile to a libcall, requiring the
	64	* runtime to provide that library function.
	65	*
	66	* That's unavoidable on Armv6-M, which has no atomic instructions
	67	* at all (not even SWP), so in that situation the user will just
	68	* have to provide an implementation of __atomic_exchange_4 (perhaps
	69	* it would temporarily disable interrupts, and then do a separate
	70	* load and store).
	71	*
	72	* We also use the libcall strategy on pre-Armv7 Linux targets, on
	73	* the theory that Linux's runtime support library _will_ provide a
	74	* suitable libcall, and it's better to use that than the SWP
	75	* instruction because then when the same binary is run on a later
	76	* Linux system the libcall implementation will use LDREX instead.
	77	*/
	78	__v = __atomic_exchange_n(__p, __x, __ATOMIC_RELAXED);
	79	#else
	80	/*
	81	* But for older Arm architectures when the target is not Linux, we
	82	* fall back to using the SWP instruction via inline assembler. ACLE
	83	* is clear that we're allowed to do this, but shouldn't do it if we
	84	* have a better alternative.
	85	*/
	86	__asm__("swp %0, %1, [%2]" : "=r"(__v) : "r"(__x), "r"(__p) : "memory");
	87	#endif
	88	return __v;
63	89	}
64	90
65	91	/* 7.6 Memory prefetch intrinsics */
...	...	@@ -72,6 +98,12 @@ __swp(uint32_t __x, volatile uint32_t *__p) {
72	98	#else
73	99	#define __pldx(access_kind, cache_level, retention_policy, addr) \
74	100	__builtin_arm_prefetch(addr, access_kind, cache_level, retention_policy, 1)
	101	#define __pldx_range(access_kind, retention_policy, length, count, stride, \
	102	reuse_distance, addr) \
	103	__builtin_arm_range_prefetch_x(addr, access_kind, retention_policy, length, \
	104	count, stride, reuse_distance)
	105	#define __pld_range(access_kind, retention_policy, metadata, addr) \
	106	__builtin_arm_range_prefetch(addr, access_kind, retention_policy, metadata)
75	107	#endif
76	108
77	109	/* 7.6.2 Instruction prefetch */
...	...	@@ -795,28 +827,6 @@ __arm_st64bv0(void *__addr, data512_t __value) {
795	827
796	828	#endif // __ARM_FEATURE_COPROC
797	829
798		/* 17 Transactional Memory Extension (TME) Intrinsics */
799		#if defined(__ARM_FEATURE_TME) && __ARM_FEATURE_TME
800
801		#define _TMFAILURE_REASON 0x00007fffu
802		#define _TMFAILURE_RTRY 0x00008000u
803		#define _TMFAILURE_CNCL 0x00010000u
804		#define _TMFAILURE_MEM 0x00020000u
805		#define _TMFAILURE_IMP 0x00040000u
806		#define _TMFAILURE_ERR 0x00080000u
807		#define _TMFAILURE_SIZE 0x00100000u
808		#define _TMFAILURE_NEST 0x00200000u
809		#define _TMFAILURE_DBG 0x00400000u
810		#define _TMFAILURE_INT 0x00800000u
811		#define _TMFAILURE_TRIVIAL 0x01000000u
812
813		#define __tstart() __builtin_arm_tstart()
814		#define __tcommit() __builtin_arm_tcommit()
815		#define __tcancel(__arg) __builtin_arm_tcancel(__arg)
816		#define __ttest() __builtin_arm_ttest()
817
818		#endif /* __ARM_FEATURE_TME */
819
820	830	/* 8.7 Armv8.5-A Random number generation intrinsics */
821	831	#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
822	832	static __inline__ int __attribute__((__always_inline__, __nodebug__, target("rand")))

lib/include/arm_neon.h+218-82

...	...	@@ -10562,7 +10562,7 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10562	10562	#define vget_lane_p8(__p0, __p1) __extension__ ({ \
10563	10563	poly8_t __ret; \
10564	10564	poly8x8_t __s0 = __p0; \
10565		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__s0, __p1)); \
	10565	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__builtin_bit_cast(int8x8_t, __s0), __p1)); \
10566	10566	__ret; \
10567	10567	})
10568	10568	#else
...	...	@@ -10570,13 +10570,13 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10570	10570	poly8_t __ret; \
10571	10571	poly8x8_t __s0 = __p0; \
10572	10572	poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_8); \
10573		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__rev0, __p1)); \
	10573	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__builtin_bit_cast(int8x8_t, __rev0), __p1)); \
10574	10574	__ret; \
10575	10575	})
10576	10576	#define __noswap_vget_lane_p8(__p0, __p1) __extension__ ({ \
10577	10577	poly8_t __ret; \
10578	10578	poly8x8_t __s0 = __p0; \
10579		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__s0, __p1)); \
	10579	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vget_lane_i8(__builtin_bit_cast(int8x8_t, __s0), __p1)); \
10580	10580	__ret; \
10581	10581	})
10582	10582	#endif
...	...	@@ -10585,7 +10585,7 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10585	10585	#define vget_lane_p16(__p0, __p1) __extension__ ({ \
10586	10586	poly16_t __ret; \
10587	10587	poly16x4_t __s0 = __p0; \
10588		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__s0, __p1)); \
	10588	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__builtin_bit_cast(int16x4_t, __s0), __p1)); \
10589	10589	__ret; \
10590	10590	})
10591	10591	#else
...	...	@@ -10593,13 +10593,13 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10593	10593	poly16_t __ret; \
10594	10594	poly16x4_t __s0 = __p0; \
10595	10595	poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \
10596		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__rev0, __p1)); \
	10596	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__builtin_bit_cast(int16x4_t, __rev0), __p1)); \
10597	10597	__ret; \
10598	10598	})
10599	10599	#define __noswap_vget_lane_p16(__p0, __p1) __extension__ ({ \
10600	10600	poly16_t __ret; \
10601	10601	poly16x4_t __s0 = __p0; \
10602		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__s0, __p1)); \
	10602	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vget_lane_i16(__builtin_bit_cast(int16x4_t, __s0), __p1)); \
10603	10603	__ret; \
10604	10604	})
10605	10605	#endif
...	...	@@ -10608,7 +10608,7 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10608	10608	#define vgetq_lane_p8(__p0, __p1) __extension__ ({ \
10609	10609	poly8_t __ret; \
10610	10610	poly8x16_t __s0 = __p0; \
10611		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__s0, __p1)); \
	10611	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__builtin_bit_cast(int8x16_t, __s0), __p1)); \
10612	10612	__ret; \
10613	10613	})
10614	10614	#else
...	...	@@ -10616,13 +10616,13 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10616	10616	poly8_t __ret; \
10617	10617	poly8x16_t __s0 = __p0; \
10618	10618	poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_8); \
10619		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__rev0, __p1)); \
	10619	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__builtin_bit_cast(int8x16_t, __rev0), __p1)); \
10620	10620	__ret; \
10621	10621	})
10622	10622	#define __noswap_vgetq_lane_p8(__p0, __p1) __extension__ ({ \
10623	10623	poly8_t __ret; \
10624	10624	poly8x16_t __s0 = __p0; \
10625		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__s0, __p1)); \
	10625	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vgetq_lane_i8(__builtin_bit_cast(int8x16_t, __s0), __p1)); \
10626	10626	__ret; \
10627	10627	})
10628	10628	#endif
...	...	@@ -10631,7 +10631,7 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10631	10631	#define vgetq_lane_p16(__p0, __p1) __extension__ ({ \
10632	10632	poly16_t __ret; \
10633	10633	poly16x8_t __s0 = __p0; \
10634		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__s0, __p1)); \
	10634	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__builtin_bit_cast(int16x8_t, __s0), __p1)); \
10635	10635	__ret; \
10636	10636	})
10637	10637	#else
...	...	@@ -10639,13 +10639,13 @@ __ai __attribute__((target("neon"))) int16x4_t __noswap_vget_high_s16(int16x8_t
10639	10639	poly16_t __ret; \
10640	10640	poly16x8_t __s0 = __p0; \
10641	10641	poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \
10642		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__rev0, __p1)); \
	10642	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__builtin_bit_cast(int16x8_t, __rev0), __p1)); \
10643	10643	__ret; \
10644	10644	})
10645	10645	#define __noswap_vgetq_lane_p16(__p0, __p1) __extension__ ({ \
10646	10646	poly16_t __ret; \
10647	10647	poly16x8_t __s0 = __p0; \
10648		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__s0, __p1)); \
	10648	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vgetq_lane_i16(__builtin_bit_cast(int16x8_t, __s0), __p1)); \
10649	10649	__ret; \
10650	10650	})
10651	10651	#endif
...	...	@@ -27027,7 +27027,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27027	27027	poly8x8_t __ret; \
27028	27028	poly8_t __s0 = __p0; \
27029	27029	poly8x8_t __s1 = __p1; \
27030		__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __s1, __p2)); \
	27030	__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __builtin_bit_cast(int8x8_t, __s1), __p2)); \
27031	27031	__ret; \
27032	27032	})
27033	27033	#else
...	...	@@ -27036,7 +27036,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27036	27036	poly8_t __s0 = __p0; \
27037	27037	poly8x8_t __s1 = __p1; \
27038	27038	poly8x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_8); \
27039		__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __rev1, __p2)); \
	27039	__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __builtin_bit_cast(int8x8_t, __rev1), __p2)); \
27040	27040	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_8); \
27041	27041	__ret; \
27042	27042	})
...	...	@@ -27044,7 +27044,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27044	27044	poly8x8_t __ret; \
27045	27045	poly8_t __s0 = __p0; \
27046	27046	poly8x8_t __s1 = __p1; \
27047		__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __s1, __p2)); \
	27047	__ret = __builtin_bit_cast(poly8x8_t, __builtin_neon_vset_lane_i8(__s0, __builtin_bit_cast(int8x8_t, __s1), __p2)); \
27048	27048	__ret; \
27049	27049	})
27050	27050	#endif
...	...	@@ -27054,7 +27054,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27054	27054	poly16x4_t __ret; \
27055	27055	poly16_t __s0 = __p0; \
27056	27056	poly16x4_t __s1 = __p1; \
27057		__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __s1, __p2)); \
	27057	__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __builtin_bit_cast(int16x4_t, __s1), __p2)); \
27058	27058	__ret; \
27059	27059	})
27060	27060	#else
...	...	@@ -27063,7 +27063,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27063	27063	poly16_t __s0 = __p0; \
27064	27064	poly16x4_t __s1 = __p1; \
27065	27065	poly16x4_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_64_16); \
27066		__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __rev1, __p2)); \
	27066	__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __builtin_bit_cast(int16x4_t, __rev1), __p2)); \
27067	27067	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_64_16); \
27068	27068	__ret; \
27069	27069	})
...	...	@@ -27071,7 +27071,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27071	27071	poly16x4_t __ret; \
27072	27072	poly16_t __s0 = __p0; \
27073	27073	poly16x4_t __s1 = __p1; \
27074		__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __s1, __p2)); \
	27074	__ret = __builtin_bit_cast(poly16x4_t, __builtin_neon_vset_lane_i16(__s0, __builtin_bit_cast(int16x4_t, __s1), __p2)); \
27075	27075	__ret; \
27076	27076	})
27077	27077	#endif
...	...	@@ -27081,7 +27081,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27081	27081	poly8x16_t __ret; \
27082	27082	poly8_t __s0 = __p0; \
27083	27083	poly8x16_t __s1 = __p1; \
27084		__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __s1, __p2)); \
	27084	__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __builtin_bit_cast(int8x16_t, __s1), __p2)); \
27085	27085	__ret; \
27086	27086	})
27087	27087	#else
...	...	@@ -27090,7 +27090,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27090	27090	poly8_t __s0 = __p0; \
27091	27091	poly8x16_t __s1 = __p1; \
27092	27092	poly8x16_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_8); \
27093		__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __rev1, __p2)); \
	27093	__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __builtin_bit_cast(int8x16_t, __rev1), __p2)); \
27094	27094	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_8); \
27095	27095	__ret; \
27096	27096	})
...	...	@@ -27098,7 +27098,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27098	27098	poly8x16_t __ret; \
27099	27099	poly8_t __s0 = __p0; \
27100	27100	poly8x16_t __s1 = __p1; \
27101		__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __s1, __p2)); \
	27101	__ret = __builtin_bit_cast(poly8x16_t, __builtin_neon_vsetq_lane_i8(__s0, __builtin_bit_cast(int8x16_t, __s1), __p2)); \
27102	27102	__ret; \
27103	27103	})
27104	27104	#endif
...	...	@@ -27108,7 +27108,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27108	27108	poly16x8_t __ret; \
27109	27109	poly16_t __s0 = __p0; \
27110	27110	poly16x8_t __s1 = __p1; \
27111		__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __s1, __p2)); \
	27111	__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __builtin_bit_cast(int16x8_t, __s1), __p2)); \
27112	27112	__ret; \
27113	27113	})
27114	27114	#else
...	...	@@ -27117,7 +27117,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27117	27117	poly16_t __s0 = __p0; \
27118	27118	poly16x8_t __s1 = __p1; \
27119	27119	poly16x8_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_16); \
27120		__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __rev1, __p2)); \
	27120	__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __builtin_bit_cast(int16x8_t, __rev1), __p2)); \
27121	27121	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16); \
27122	27122	__ret; \
27123	27123	})
...	...	@@ -27125,7 +27125,7 @@ __ai __attribute__((target("neon"))) int8x8_t __noswap_vrsubhn_s16(int16x8_t __p
27125	27125	poly16x8_t __ret; \
27126	27126	poly16_t __s0 = __p0; \
27127	27127	poly16x8_t __s1 = __p1; \
27128		__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __s1, __p2)); \
	27128	__ret = __builtin_bit_cast(poly16x8_t, __builtin_neon_vsetq_lane_i16(__s0, __builtin_bit_cast(int16x8_t, __s1), __p2)); \
27129	27129	__ret; \
27130	27130	})
27131	27131	#endif
...	...	@@ -41141,6 +41141,42 @@ __ai __attribute__((target("neon"))) float32x2_t vfms_f32(float32x2_t __p0, floa
41141	41141
41142	41142	#endif
41143	41143	#if defined(__aarch64__)
	41144	#ifdef __LITTLE_ENDIAN__
	41145	__ai __attribute__((target("f8f16mm,neon"))) float16x8_t vmmlaq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
	41146	float16x8_t __ret;
	41147	__ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmmlaq_f16_mf8_fpm(__builtin_bit_cast(int8x16_t, __p0), __p1, __p2, __p3));
	41148	return __ret;
	41149	}
	41150	#else
	41151	__ai __attribute__((target("f8f16mm,neon"))) float16x8_t vmmlaq_f16_mf8_fpm(float16x8_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
	41152	float16x8_t __ret;
	41153	float16x8_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_16);
	41154	mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_8);
	41155	mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_128_8);
	41156	__ret = __builtin_bit_cast(float16x8_t, __builtin_neon_vmmlaq_f16_mf8_fpm(__builtin_bit_cast(int8x16_t, __rev0), __rev1, __rev2, __p3));
	41157	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_16);
	41158	return __ret;
	41159	}
	41160	#endif
	41161
	41162	#ifdef __LITTLE_ENDIAN__
	41163	__ai __attribute__((target("f8f32mm,neon"))) float32x4_t vmmlaq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
	41164	float32x4_t __ret;
	41165	__ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vmmlaq_f32_mf8_fpm(__p0, __p1, __p2, __p3));
	41166	return __ret;
	41167	}
	41168	#else
	41169	__ai __attribute__((target("f8f32mm,neon"))) float32x4_t vmmlaq_f32_mf8_fpm(float32x4_t __p0, mfloat8x16_t __p1, mfloat8x16_t __p2, fpm_t __p3) {
	41170	float32x4_t __ret;
	41171	float32x4_t __rev0; __rev0 = __builtin_shufflevector(__p0, __p0, __lane_reverse_128_32);
	41172	mfloat8x16_t __rev1; __rev1 = __builtin_shufflevector(__p1, __p1, __lane_reverse_128_8);
	41173	mfloat8x16_t __rev2; __rev2 = __builtin_shufflevector(__p2, __p2, __lane_reverse_128_8);
	41174	__ret = __builtin_bit_cast(float32x4_t, __builtin_neon_vmmlaq_f32_mf8_fpm(__rev0, __rev1, __rev2, __p3));
	41175	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_32);
	41176	return __ret;
	41177	}
	41178	#endif
	41179
41144	41180	#ifdef __LITTLE_ENDIAN__
41145	41181	__ai __attribute__((target("fp8,neon"))) bfloat16x8_t vcvt1_bf16_mf8_fpm(mfloat8x8_t __p0, fpm_t __p1) {
41146	41182	bfloat16x8_t __ret;
...	...	@@ -49847,6 +49883,16 @@ __ai __attribute__((target("neon"))) int32_t vcvts_s32_f32(float32_t __p0) {
49847	49883	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvts_s32_f32(__p0));
49848	49884	return __ret;
49849	49885	}
	49886	__ai __attribute__((target("neon"))) int32_t vcvtd_s32_f64(float64_t __p0) {
	49887	int32_t __ret;
	49888	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtd_s32_f64(__p0));
	49889	return __ret;
	49890	}
	49891	__ai __attribute__((target("neon"))) int64_t vcvts_s64_f32(float32_t __p0) {
	49892	int64_t __ret;
	49893	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvts_s64_f32(__p0));
	49894	return __ret;
	49895	}
49850	49896	__ai __attribute__((target("neon"))) int64_t vcvtd_s64_f64(float64_t __p0) {
49851	49897	int64_t __ret;
49852	49898	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtd_s64_f64(__p0));
...	...	@@ -49878,6 +49924,16 @@ __ai __attribute__((target("neon"))) uint32_t vcvts_u32_f32(float32_t __p0) {
49878	49924	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvts_u32_f32(__p0));
49879	49925	return __ret;
49880	49926	}
	49927	__ai __attribute__((target("neon"))) uint32_t vcvtd_u32_f64(float64_t __p0) {
	49928	uint32_t __ret;
	49929	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtd_u32_f64(__p0));
	49930	return __ret;
	49931	}
	49932	__ai __attribute__((target("neon"))) uint64_t vcvts_u64_f32(float32_t __p0) {
	49933	uint64_t __ret;
	49934	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvts_u64_f32(__p0));
	49935	return __ret;
	49936	}
49881	49937	__ai __attribute__((target("neon"))) uint64_t vcvtd_u64_f64(float64_t __p0) {
49882	49938	uint64_t __ret;
49883	49939	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtd_u64_f64(__p0));
...	...	@@ -49909,6 +49965,11 @@ __ai __attribute__((target("neon"))) int32_t vcvtas_s32_f32(float32_t __p0) {
49909	49965	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtas_s32_f32(__p0));
49910	49966	return __ret;
49911	49967	}
	49968	__ai __attribute__((target("neon"))) int32_t vcvtad_s32_f64(float64_t __p0) {
	49969	int32_t __ret;
	49970	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtad_s32_f64(__p0));
	49971	return __ret;
	49972	}
49912	49973	#ifdef __LITTLE_ENDIAN__
49913	49974	__ai __attribute__((target("neon"))) int64x2_t vcvtaq_s64_f64(float64x2_t __p0) {
49914	49975	int64x2_t __ret;
...	...	@@ -49930,6 +49991,11 @@ __ai __attribute__((target("neon"))) int64x1_t vcvta_s64_f64(float64x1_t __p0) {
49930	49991	__ret = __builtin_bit_cast(int64x1_t, __builtin_neon_vcvta_s64_v(__builtin_bit_cast(int8x8_t, __p0), 3));
49931	49992	return __ret;
49932	49993	}
	49994	__ai __attribute__((target("neon"))) int64_t vcvtas_s64_f32(float32_t __p0) {
	49995	int64_t __ret;
	49996	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtas_s64_f32(__p0));
	49997	return __ret;
	49998	}
49933	49999	__ai __attribute__((target("neon"))) int64_t vcvtad_s64_f64(float64_t __p0) {
49934	50000	int64_t __ret;
49935	50001	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtad_s64_f64(__p0));
...	...	@@ -49940,6 +50006,11 @@ __ai __attribute__((target("neon"))) uint32_t vcvtas_u32_f32(float32_t __p0) {
49940	50006	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtas_u32_f32(__p0));
49941	50007	return __ret;
49942	50008	}
	50009	__ai __attribute__((target("neon"))) uint32_t vcvtad_u32_f64(float64_t __p0) {
	50010	uint32_t __ret;
	50011	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtad_u32_f64(__p0));
	50012	return __ret;
	50013	}
49943	50014	#ifdef __LITTLE_ENDIAN__
49944	50015	__ai __attribute__((target("neon"))) uint64x2_t vcvtaq_u64_f64(float64x2_t __p0) {
49945	50016	uint64x2_t __ret;
...	...	@@ -49961,6 +50032,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vcvta_u64_f64(float64x1_t __p0)
49961	50032	__ret = __builtin_bit_cast(uint64x1_t, __builtin_neon_vcvta_u64_v(__builtin_bit_cast(int8x8_t, __p0), 19));
49962	50033	return __ret;
49963	50034	}
	50035	__ai __attribute__((target("neon"))) uint64_t vcvtas_u64_f32(float32_t __p0) {
	50036	uint64_t __ret;
	50037	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtas_u64_f32(__p0));
	50038	return __ret;
	50039	}
49964	50040	__ai __attribute__((target("neon"))) uint64_t vcvtad_u64_f64(float64_t __p0) {
49965	50041	uint64_t __ret;
49966	50042	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtad_u64_f64(__p0));
...	...	@@ -49971,6 +50047,11 @@ __ai __attribute__((target("neon"))) int32_t vcvtms_s32_f32(float32_t __p0) {
49971	50047	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtms_s32_f32(__p0));
49972	50048	return __ret;
49973	50049	}
	50050	__ai __attribute__((target("neon"))) int32_t vcvtmd_s32_f64(float64_t __p0) {
	50051	int32_t __ret;
	50052	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtmd_s32_f64(__p0));
	50053	return __ret;
	50054	}
49974	50055	#ifdef __LITTLE_ENDIAN__
49975	50056	__ai __attribute__((target("neon"))) int64x2_t vcvtmq_s64_f64(float64x2_t __p0) {
49976	50057	int64x2_t __ret;
...	...	@@ -49992,6 +50073,11 @@ __ai __attribute__((target("neon"))) int64x1_t vcvtm_s64_f64(float64x1_t __p0) {
49992	50073	__ret = __builtin_bit_cast(int64x1_t, __builtin_neon_vcvtm_s64_v(__builtin_bit_cast(int8x8_t, __p0), 3));
49993	50074	return __ret;
49994	50075	}
	50076	__ai __attribute__((target("neon"))) int64_t vcvtms_s64_f32(float32_t __p0) {
	50077	int64_t __ret;
	50078	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtms_s64_f32(__p0));
	50079	return __ret;
	50080	}
49995	50081	__ai __attribute__((target("neon"))) int64_t vcvtmd_s64_f64(float64_t __p0) {
49996	50082	int64_t __ret;
49997	50083	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtmd_s64_f64(__p0));
...	...	@@ -50002,6 +50088,11 @@ __ai __attribute__((target("neon"))) uint32_t vcvtms_u32_f32(float32_t __p0) {
50002	50088	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtms_u32_f32(__p0));
50003	50089	return __ret;
50004	50090	}
	50091	__ai __attribute__((target("neon"))) uint32_t vcvtmd_u32_f64(float64_t __p0) {
	50092	uint32_t __ret;
	50093	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtmd_u32_f64(__p0));
	50094	return __ret;
	50095	}
50005	50096	#ifdef __LITTLE_ENDIAN__
50006	50097	__ai __attribute__((target("neon"))) uint64x2_t vcvtmq_u64_f64(float64x2_t __p0) {
50007	50098	uint64x2_t __ret;
...	...	@@ -50023,6 +50114,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vcvtm_u64_f64(float64x1_t __p0)
50023	50114	__ret = __builtin_bit_cast(uint64x1_t, __builtin_neon_vcvtm_u64_v(__builtin_bit_cast(int8x8_t, __p0), 19));
50024	50115	return __ret;
50025	50116	}
	50117	__ai __attribute__((target("neon"))) uint64_t vcvtms_u64_f32(float32_t __p0) {
	50118	uint64_t __ret;
	50119	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtms_u64_f32(__p0));
	50120	return __ret;
	50121	}
50026	50122	__ai __attribute__((target("neon"))) uint64_t vcvtmd_u64_f64(float64_t __p0) {
50027	50123	uint64_t __ret;
50028	50124	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtmd_u64_f64(__p0));
...	...	@@ -50033,6 +50129,11 @@ __ai __attribute__((target("neon"))) int32_t vcvtns_s32_f32(float32_t __p0) {
50033	50129	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtns_s32_f32(__p0));
50034	50130	return __ret;
50035	50131	}
	50132	__ai __attribute__((target("neon"))) int32_t vcvtnd_s32_f64(float64_t __p0) {
	50133	int32_t __ret;
	50134	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtnd_s32_f64(__p0));
	50135	return __ret;
	50136	}
50036	50137	#ifdef __LITTLE_ENDIAN__
50037	50138	__ai __attribute__((target("neon"))) int64x2_t vcvtnq_s64_f64(float64x2_t __p0) {
50038	50139	int64x2_t __ret;
...	...	@@ -50054,6 +50155,11 @@ __ai __attribute__((target("neon"))) int64x1_t vcvtn_s64_f64(float64x1_t __p0) {
50054	50155	__ret = __builtin_bit_cast(int64x1_t, __builtin_neon_vcvtn_s64_v(__builtin_bit_cast(int8x8_t, __p0), 3));
50055	50156	return __ret;
50056	50157	}
	50158	__ai __attribute__((target("neon"))) int64_t vcvtns_s64_f32(float32_t __p0) {
	50159	int64_t __ret;
	50160	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtns_s64_f32(__p0));
	50161	return __ret;
	50162	}
50057	50163	__ai __attribute__((target("neon"))) int64_t vcvtnd_s64_f64(float64_t __p0) {
50058	50164	int64_t __ret;
50059	50165	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtnd_s64_f64(__p0));
...	...	@@ -50064,6 +50170,11 @@ __ai __attribute__((target("neon"))) uint32_t vcvtns_u32_f32(float32_t __p0) {
50064	50170	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtns_u32_f32(__p0));
50065	50171	return __ret;
50066	50172	}
	50173	__ai __attribute__((target("neon"))) uint32_t vcvtnd_u32_f64(float64_t __p0) {
	50174	uint32_t __ret;
	50175	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtnd_u32_f64(__p0));
	50176	return __ret;
	50177	}
50067	50178	#ifdef __LITTLE_ENDIAN__
50068	50179	__ai __attribute__((target("neon"))) uint64x2_t vcvtnq_u64_f64(float64x2_t __p0) {
50069	50180	uint64x2_t __ret;
...	...	@@ -50085,6 +50196,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vcvtn_u64_f64(float64x1_t __p0)
50085	50196	__ret = __builtin_bit_cast(uint64x1_t, __builtin_neon_vcvtn_u64_v(__builtin_bit_cast(int8x8_t, __p0), 19));
50086	50197	return __ret;
50087	50198	}
	50199	__ai __attribute__((target("neon"))) uint64_t vcvtns_u64_f32(float32_t __p0) {
	50200	uint64_t __ret;
	50201	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtns_u64_f32(__p0));
	50202	return __ret;
	50203	}
50088	50204	__ai __attribute__((target("neon"))) uint64_t vcvtnd_u64_f64(float64_t __p0) {
50089	50205	uint64_t __ret;
50090	50206	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtnd_u64_f64(__p0));
...	...	@@ -50095,6 +50211,11 @@ __ai __attribute__((target("neon"))) int32_t vcvtps_s32_f32(float32_t __p0) {
50095	50211	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtps_s32_f32(__p0));
50096	50212	return __ret;
50097	50213	}
	50214	__ai __attribute__((target("neon"))) int32_t vcvtpd_s32_f64(float64_t __p0) {
	50215	int32_t __ret;
	50216	__ret = __builtin_bit_cast(int32_t, __builtin_neon_vcvtpd_s32_f64(__p0));
	50217	return __ret;
	50218	}
50098	50219	#ifdef __LITTLE_ENDIAN__
50099	50220	__ai __attribute__((target("neon"))) int64x2_t vcvtpq_s64_f64(float64x2_t __p0) {
50100	50221	int64x2_t __ret;
...	...	@@ -50116,6 +50237,11 @@ __ai __attribute__((target("neon"))) int64x1_t vcvtp_s64_f64(float64x1_t __p0) {
50116	50237	__ret = __builtin_bit_cast(int64x1_t, __builtin_neon_vcvtp_s64_v(__builtin_bit_cast(int8x8_t, __p0), 3));
50117	50238	return __ret;
50118	50239	}
	50240	__ai __attribute__((target("neon"))) int64_t vcvtps_s64_f32(float32_t __p0) {
	50241	int64_t __ret;
	50242	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtps_s64_f32(__p0));
	50243	return __ret;
	50244	}
50119	50245	__ai __attribute__((target("neon"))) int64_t vcvtpd_s64_f64(float64_t __p0) {
50120	50246	int64_t __ret;
50121	50247	__ret = __builtin_bit_cast(int64_t, __builtin_neon_vcvtpd_s64_f64(__p0));
...	...	@@ -50126,6 +50252,11 @@ __ai __attribute__((target("neon"))) uint32_t vcvtps_u32_f32(float32_t __p0) {
50126	50252	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtps_u32_f32(__p0));
50127	50253	return __ret;
50128	50254	}
	50255	__ai __attribute__((target("neon"))) uint32_t vcvtpd_u32_f64(float64_t __p0) {
	50256	uint32_t __ret;
	50257	__ret = __builtin_bit_cast(uint32_t, __builtin_neon_vcvtpd_u32_f64(__p0));
	50258	return __ret;
	50259	}
50129	50260	#ifdef __LITTLE_ENDIAN__
50130	50261	__ai __attribute__((target("neon"))) uint64x2_t vcvtpq_u64_f64(float64x2_t __p0) {
50131	50262	uint64x2_t __ret;
...	...	@@ -50147,6 +50278,11 @@ __ai __attribute__((target("neon"))) uint64x1_t vcvtp_u64_f64(float64x1_t __p0)
50147	50278	__ret = __builtin_bit_cast(uint64x1_t, __builtin_neon_vcvtp_u64_v(__builtin_bit_cast(int8x8_t, __p0), 19));
50148	50279	return __ret;
50149	50280	}
	50281	__ai __attribute__((target("neon"))) uint64_t vcvtps_u64_f32(float32_t __p0) {
	50282	uint64_t __ret;
	50283	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtps_u64_f32(__p0));
	50284	return __ret;
	50285	}
50150	50286	__ai __attribute__((target("neon"))) uint64_t vcvtpd_u64_f64(float64_t __p0) {
50151	50287	uint64_t __ret;
50152	50288	__ret = __builtin_bit_cast(uint64_t, __builtin_neon_vcvtpd_u64_f64(__p0));
...	...	@@ -50255,7 +50391,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50255	50391	#define vdupb_lane_p8(__p0, __p1) __extension__ ({ \
50256	50392	poly8_t __ret; \
50257	50393	poly8x8_t __s0 = __p0; \
50258		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_lane_i8(__s0, __p1)); \
	50394	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_lane_i8(__builtin_bit_cast(int8x8_t, __s0), __p1)); \
50259	50395	__ret; \
50260	50396	})
50261	50397	#else
...	...	@@ -50263,7 +50399,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50263	50399	poly8_t __ret; \
50264	50400	poly8x8_t __s0 = __p0; \
50265	50401	poly8x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_8); \
50266		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_lane_i8(__rev0, __p1)); \
	50402	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_lane_i8(__builtin_bit_cast(int8x8_t, __rev0), __p1)); \
50267	50403	__ret; \
50268	50404	})
50269	50405	#endif
...	...	@@ -50272,7 +50408,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50272	50408	#define vduph_lane_p16(__p0, __p1) __extension__ ({ \
50273	50409	poly16_t __ret; \
50274	50410	poly16x4_t __s0 = __p0; \
50275		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_lane_i16(__s0, __p1)); \
	50411	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_lane_i16(__builtin_bit_cast(int16x4_t, __s0), __p1)); \
50276	50412	__ret; \
50277	50413	})
50278	50414	#else
...	...	@@ -50280,7 +50416,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50280	50416	poly16_t __ret; \
50281	50417	poly16x4_t __s0 = __p0; \
50282	50418	poly16x4_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_64_16); \
50283		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_lane_i16(__rev0, __p1)); \
	50419	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_lane_i16(__builtin_bit_cast(int16x4_t, __rev0), __p1)); \
50284	50420	__ret; \
50285	50421	})
50286	50422	#endif
...	...	@@ -50506,7 +50642,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50506	50642	#define vdupb_laneq_p8(__p0, __p1) __extension__ ({ \
50507	50643	poly8_t __ret; \
50508	50644	poly8x16_t __s0 = __p0; \
50509		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_laneq_i8(__s0, __p1)); \
	50645	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_laneq_i8(__builtin_bit_cast(int8x16_t, __s0), __p1)); \
50510	50646	__ret; \
50511	50647	})
50512	50648	#else
...	...	@@ -50514,7 +50650,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50514	50650	poly8_t __ret; \
50515	50651	poly8x16_t __s0 = __p0; \
50516	50652	poly8x16_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_8); \
50517		__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_laneq_i8(__rev0, __p1)); \
	50653	__ret = __builtin_bit_cast(poly8_t, __builtin_neon_vdupb_laneq_i8(__builtin_bit_cast(int8x16_t, __rev0), __p1)); \
50518	50654	__ret; \
50519	50655	})
50520	50656	#endif
...	...	@@ -50523,7 +50659,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50523	50659	#define vduph_laneq_p16(__p0, __p1) __extension__ ({ \
50524	50660	poly16_t __ret; \
50525	50661	poly16x8_t __s0 = __p0; \
50526		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_laneq_i16(__s0, __p1)); \
	50662	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_laneq_i16(__builtin_bit_cast(int16x8_t, __s0), __p1)); \
50527	50663	__ret; \
50528	50664	})
50529	50665	#else
...	...	@@ -50531,7 +50667,7 @@ __ai __attribute__((target("neon"))) float32x2_t vdiv_f32(float32x2_t __p0, floa
50531	50667	poly16_t __ret; \
50532	50668	poly16x8_t __s0 = __p0; \
50533	50669	poly16x8_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_16); \
50534		__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_laneq_i16(__rev0, __p1)); \
	50670	__ret = __builtin_bit_cast(poly16_t, __builtin_neon_vduph_laneq_i16(__builtin_bit_cast(int16x8_t, __rev0), __p1)); \
50535	50671	__ret; \
50536	50672	})
50537	50673	#endif
...	...	@@ -52105,14 +52241,14 @@ __ai __attribute__((target("neon"))) float64x1_t vget_high_f64(float64x2_t __p0)
52105	52241	#define vget_lane_p64(__p0, __p1) __extension__ ({ \
52106	52242	poly64_t __ret; \
52107	52243	poly64x1_t __s0 = __p0; \
52108		__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vget_lane_i64(__s0, __p1)); \
	52244	__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vget_lane_i64(__builtin_bit_cast(int64x1_t, __s0), __p1)); \
52109	52245	__ret; \
52110	52246	})
52111	52247	#ifdef __LITTLE_ENDIAN__
52112	52248	#define vgetq_lane_p64(__p0, __p1) __extension__ ({ \
52113	52249	poly64_t __ret; \
52114	52250	poly64x2_t __s0 = __p0; \
52115		__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__s0, __p1)); \
	52251	__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__builtin_bit_cast(int64x2_t, __s0), __p1)); \
52116	52252	__ret; \
52117	52253	})
52118	52254	#else
...	...	@@ -52120,13 +52256,13 @@ __ai __attribute__((target("neon"))) float64x1_t vget_high_f64(float64x2_t __p0)
52120	52256	poly64_t __ret; \
52121	52257	poly64x2_t __s0 = __p0; \
52122	52258	poly64x2_t __rev0; __rev0 = __builtin_shufflevector(__s0, __s0, __lane_reverse_128_64); \
52123		__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__rev0, __p1)); \
	52259	__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__builtin_bit_cast(int64x2_t, __rev0), __p1)); \
52124	52260	__ret; \
52125	52261	})
52126	52262	#define __noswap_vgetq_lane_p64(__p0, __p1) __extension__ ({ \
52127	52263	poly64_t __ret; \
52128	52264	poly64x2_t __s0 = __p0; \
52129		__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__s0, __p1)); \
	52265	__ret = __builtin_bit_cast(poly64_t, __builtin_neon_vgetq_lane_i64(__builtin_bit_cast(int64x2_t, __s0), __p1)); \
52130	52266	__ret; \
52131	52267	})
52132	52268	#endif
...	...	@@ -59743,20 +59879,20 @@ __ai __attribute__((target("neon"))) int16_t vqrshlh_s16(int16_t __p0, int16_t _
59743	59879	})
59744	59880	#ifdef __LITTLE_ENDIAN__
59745	59881	#define vqrshrun_high_n_s32(__p0_716, __p1_716, __p2_716) __extension__ ({ \
59746		int16x8_t __ret_716; \
59747		int16x4_t __s0_716 = __p0_716; \
	59882	uint16x8_t __ret_716; \
	59883	uint16x4_t __s0_716 = __p0_716; \
59748	59884	int32x4_t __s1_716 = __p1_716; \
59749		__ret_716 = __builtin_bit_cast(int16x8_t, vcombine_s16(__builtin_bit_cast(int16x4_t, __s0_716), __builtin_bit_cast(int16x4_t, vqrshrun_n_s32(__s1_716, __p2_716)))); \
	59885	__ret_716 = __builtin_bit_cast(uint16x8_t, vcombine_u16(__builtin_bit_cast(uint16x4_t, __s0_716), __builtin_bit_cast(uint16x4_t, vqrshrun_n_s32(__s1_716, __p2_716)))); \
59750	59886	__ret_716; \
59751	59887	})
59752	59888	#else
59753	59889	#define vqrshrun_high_n_s32(__p0_717, __p1_717, __p2_717) __extension__ ({ \
59754		int16x8_t __ret_717; \
59755		int16x4_t __s0_717 = __p0_717; \
	59890	uint16x8_t __ret_717; \
	59891	uint16x4_t __s0_717 = __p0_717; \
59756	59892	int32x4_t __s1_717 = __p1_717; \
59757		int16x4_t __rev0_717; __rev0_717 = __builtin_shufflevector(__s0_717, __s0_717, __lane_reverse_64_16); \
	59893	uint16x4_t __rev0_717; __rev0_717 = __builtin_shufflevector(__s0_717, __s0_717, __lane_reverse_64_16); \
59758	59894	int32x4_t __rev1_717; __rev1_717 = __builtin_shufflevector(__s1_717, __s1_717, __lane_reverse_128_32); \
59759		__ret_717 = __builtin_bit_cast(int16x8_t, __noswap_vcombine_s16(__builtin_bit_cast(int16x4_t, __rev0_717), __builtin_bit_cast(int16x4_t, __noswap_vqrshrun_n_s32(__rev1_717, __p2_717)))); \
	59895	__ret_717 = __builtin_bit_cast(uint16x8_t, __noswap_vcombine_u16(__builtin_bit_cast(uint16x4_t, __rev0_717), __builtin_bit_cast(uint16x4_t, __noswap_vqrshrun_n_s32(__rev1_717, __p2_717)))); \
59760	59896	__ret_717 = __builtin_shufflevector(__ret_717, __ret_717, __lane_reverse_128_16); \
59761	59897	__ret_717; \
59762	59898	})
...	...	@@ -59764,20 +59900,20 @@ __ai __attribute__((target("neon"))) int16_t vqrshlh_s16(int16_t __p0, int16_t _
59764	59900
59765	59901	#ifdef __LITTLE_ENDIAN__
59766	59902	#define vqrshrun_high_n_s64(__p0_718, __p1_718, __p2_718) __extension__ ({ \
59767		int32x4_t __ret_718; \
59768		int32x2_t __s0_718 = __p0_718; \
	59903	uint32x4_t __ret_718; \
	59904	uint32x2_t __s0_718 = __p0_718; \
59769	59905	int64x2_t __s1_718 = __p1_718; \
59770		__ret_718 = __builtin_bit_cast(int32x4_t, vcombine_s32(__builtin_bit_cast(int32x2_t, __s0_718), __builtin_bit_cast(int32x2_t, vqrshrun_n_s64(__s1_718, __p2_718)))); \
	59906	__ret_718 = __builtin_bit_cast(uint32x4_t, vcombine_u32(__builtin_bit_cast(uint32x2_t, __s0_718), __builtin_bit_cast(uint32x2_t, vqrshrun_n_s64(__s1_718, __p2_718)))); \
59771	59907	__ret_718; \
59772	59908	})
59773	59909	#else
59774	59910	#define vqrshrun_high_n_s64(__p0_719, __p1_719, __p2_719) __extension__ ({ \
59775		int32x4_t __ret_719; \
59776		int32x2_t __s0_719 = __p0_719; \
	59911	uint32x4_t __ret_719; \
	59912	uint32x2_t __s0_719 = __p0_719; \
59777	59913	int64x2_t __s1_719 = __p1_719; \
59778		int32x2_t __rev0_719; __rev0_719 = __builtin_shufflevector(__s0_719, __s0_719, __lane_reverse_64_32); \
	59914	uint32x2_t __rev0_719; __rev0_719 = __builtin_shufflevector(__s0_719, __s0_719, __lane_reverse_64_32); \
59779	59915	int64x2_t __rev1_719; __rev1_719 = __builtin_shufflevector(__s1_719, __s1_719, __lane_reverse_128_64); \
59780		__ret_719 = __builtin_bit_cast(int32x4_t, __noswap_vcombine_s32(__builtin_bit_cast(int32x2_t, __rev0_719), __builtin_bit_cast(int32x2_t, __noswap_vqrshrun_n_s64(__rev1_719, __p2_719)))); \
	59916	__ret_719 = __builtin_bit_cast(uint32x4_t, __noswap_vcombine_u32(__builtin_bit_cast(uint32x2_t, __rev0_719), __builtin_bit_cast(uint32x2_t, __noswap_vqrshrun_n_s64(__rev1_719, __p2_719)))); \
59781	59917	__ret_719 = __builtin_shufflevector(__ret_719, __ret_719, __lane_reverse_128_32); \
59782	59918	__ret_719; \
59783	59919	})
...	...	@@ -59785,20 +59921,20 @@ __ai __attribute__((target("neon"))) int16_t vqrshlh_s16(int16_t __p0, int16_t _
59785	59921
59786	59922	#ifdef __LITTLE_ENDIAN__
59787	59923	#define vqrshrun_high_n_s16(__p0_720, __p1_720, __p2_720) __extension__ ({ \
59788		int8x16_t __ret_720; \
59789		int8x8_t __s0_720 = __p0_720; \
	59924	uint8x16_t __ret_720; \
	59925	uint8x8_t __s0_720 = __p0_720; \
59790	59926	int16x8_t __s1_720 = __p1_720; \
59791		__ret_720 = __builtin_bit_cast(int8x16_t, vcombine_s8(__builtin_bit_cast(int8x8_t, __s0_720), __builtin_bit_cast(int8x8_t, vqrshrun_n_s16(__s1_720, __p2_720)))); \
	59927	__ret_720 = __builtin_bit_cast(uint8x16_t, vcombine_u8(__builtin_bit_cast(uint8x8_t, __s0_720), __builtin_bit_cast(uint8x8_t, vqrshrun_n_s16(__s1_720, __p2_720)))); \
59792	59928	__ret_720; \
59793	59929	})
59794	59930	#else
59795	59931	#define vqrshrun_high_n_s16(__p0_721, __p1_721, __p2_721) __extension__ ({ \
59796		int8x16_t __ret_721; \
59797		int8x8_t __s0_721 = __p0_721; \
	59932	uint8x16_t __ret_721; \
	59933	uint8x8_t __s0_721 = __p0_721; \
59798	59934	int16x8_t __s1_721 = __p1_721; \
59799		int8x8_t __rev0_721; __rev0_721 = __builtin_shufflevector(__s0_721, __s0_721, __lane_reverse_64_8); \
	59935	uint8x8_t __rev0_721; __rev0_721 = __builtin_shufflevector(__s0_721, __s0_721, __lane_reverse_64_8); \
59800	59936	int16x8_t __rev1_721; __rev1_721 = __builtin_shufflevector(__s1_721, __s1_721, __lane_reverse_128_16); \
59801		__ret_721 = __builtin_bit_cast(int8x16_t, __noswap_vcombine_s8(__builtin_bit_cast(int8x8_t, __rev0_721), __builtin_bit_cast(int8x8_t, __noswap_vqrshrun_n_s16(__rev1_721, __p2_721)))); \
	59937	__ret_721 = __builtin_bit_cast(uint8x16_t, __noswap_vcombine_u8(__builtin_bit_cast(uint8x8_t, __rev0_721), __builtin_bit_cast(uint8x8_t, __noswap_vqrshrun_n_s16(__rev1_721, __p2_721)))); \
59802	59938	__ret_721 = __builtin_shufflevector(__ret_721, __ret_721, __lane_reverse_128_8); \
59803	59939	__ret_721; \
59804	59940	})
...	...	@@ -60098,20 +60234,20 @@ __ai __attribute__((target("neon"))) int16_t vqshlh_s16(int16_t __p0, int16_t __
60098	60234	})
60099	60235	#ifdef __LITTLE_ENDIAN__
60100	60236	#define vqshrun_high_n_s32(__p0_734, __p1_734, __p2_734) __extension__ ({ \
60101		int16x8_t __ret_734; \
60102		int16x4_t __s0_734 = __p0_734; \
	60237	uint16x8_t __ret_734; \
	60238	uint16x4_t __s0_734 = __p0_734; \
60103	60239	int32x4_t __s1_734 = __p1_734; \
60104		__ret_734 = __builtin_bit_cast(int16x8_t, vcombine_s16(__builtin_bit_cast(int16x4_t, __s0_734), __builtin_bit_cast(int16x4_t, vqshrun_n_s32(__s1_734, __p2_734)))); \
	60240	__ret_734 = __builtin_bit_cast(uint16x8_t, vcombine_u16(__builtin_bit_cast(uint16x4_t, __s0_734), __builtin_bit_cast(uint16x4_t, vqshrun_n_s32(__s1_734, __p2_734)))); \
60105	60241	__ret_734; \
60106	60242	})
60107	60243	#else
60108	60244	#define vqshrun_high_n_s32(__p0_735, __p1_735, __p2_735) __extension__ ({ \
60109		int16x8_t __ret_735; \
60110		int16x4_t __s0_735 = __p0_735; \
	60245	uint16x8_t __ret_735; \
	60246	uint16x4_t __s0_735 = __p0_735; \
60111	60247	int32x4_t __s1_735 = __p1_735; \
60112		int16x4_t __rev0_735; __rev0_735 = __builtin_shufflevector(__s0_735, __s0_735, __lane_reverse_64_16); \
	60248	uint16x4_t __rev0_735; __rev0_735 = __builtin_shufflevector(__s0_735, __s0_735, __lane_reverse_64_16); \
60113	60249	int32x4_t __rev1_735; __rev1_735 = __builtin_shufflevector(__s1_735, __s1_735, __lane_reverse_128_32); \
60114		__ret_735 = __builtin_bit_cast(int16x8_t, __noswap_vcombine_s16(__builtin_bit_cast(int16x4_t, __rev0_735), __builtin_bit_cast(int16x4_t, __noswap_vqshrun_n_s32(__rev1_735, __p2_735)))); \
	60250	__ret_735 = __builtin_bit_cast(uint16x8_t, __noswap_vcombine_u16(__builtin_bit_cast(uint16x4_t, __rev0_735), __builtin_bit_cast(uint16x4_t, __noswap_vqshrun_n_s32(__rev1_735, __p2_735)))); \
60115	60251	__ret_735 = __builtin_shufflevector(__ret_735, __ret_735, __lane_reverse_128_16); \
60116	60252	__ret_735; \
60117	60253	})
...	...	@@ -60119,20 +60255,20 @@ __ai __attribute__((target("neon"))) int16_t vqshlh_s16(int16_t __p0, int16_t __
60119	60255
60120	60256	#ifdef __LITTLE_ENDIAN__
60121	60257	#define vqshrun_high_n_s64(__p0_736, __p1_736, __p2_736) __extension__ ({ \
60122		int32x4_t __ret_736; \
60123		int32x2_t __s0_736 = __p0_736; \
	60258	uint32x4_t __ret_736; \
	60259	uint32x2_t __s0_736 = __p0_736; \
60124	60260	int64x2_t __s1_736 = __p1_736; \
60125		__ret_736 = __builtin_bit_cast(int32x4_t, vcombine_s32(__builtin_bit_cast(int32x2_t, __s0_736), __builtin_bit_cast(int32x2_t, vqshrun_n_s64(__s1_736, __p2_736)))); \
	60261	__ret_736 = __builtin_bit_cast(uint32x4_t, vcombine_u32(__builtin_bit_cast(uint32x2_t, __s0_736), __builtin_bit_cast(uint32x2_t, vqshrun_n_s64(__s1_736, __p2_736)))); \
60126	60262	__ret_736; \
60127	60263	})
60128	60264	#else
60129	60265	#define vqshrun_high_n_s64(__p0_737, __p1_737, __p2_737) __extension__ ({ \
60130		int32x4_t __ret_737; \
60131		int32x2_t __s0_737 = __p0_737; \
	60266	uint32x4_t __ret_737; \
	60267	uint32x2_t __s0_737 = __p0_737; \
60132	60268	int64x2_t __s1_737 = __p1_737; \
60133		int32x2_t __rev0_737; __rev0_737 = __builtin_shufflevector(__s0_737, __s0_737, __lane_reverse_64_32); \
	60269	uint32x2_t __rev0_737; __rev0_737 = __builtin_shufflevector(__s0_737, __s0_737, __lane_reverse_64_32); \
60134	60270	int64x2_t __rev1_737; __rev1_737 = __builtin_shufflevector(__s1_737, __s1_737, __lane_reverse_128_64); \
60135		__ret_737 = __builtin_bit_cast(int32x4_t, __noswap_vcombine_s32(__builtin_bit_cast(int32x2_t, __rev0_737), __builtin_bit_cast(int32x2_t, __noswap_vqshrun_n_s64(__rev1_737, __p2_737)))); \
	60271	__ret_737 = __builtin_bit_cast(uint32x4_t, __noswap_vcombine_u32(__builtin_bit_cast(uint32x2_t, __rev0_737), __builtin_bit_cast(uint32x2_t, __noswap_vqshrun_n_s64(__rev1_737, __p2_737)))); \
60136	60272	__ret_737 = __builtin_shufflevector(__ret_737, __ret_737, __lane_reverse_128_32); \
60137	60273	__ret_737; \
60138	60274	})
...	...	@@ -60140,20 +60276,20 @@ __ai __attribute__((target("neon"))) int16_t vqshlh_s16(int16_t __p0, int16_t __
60140	60276
60141	60277	#ifdef __LITTLE_ENDIAN__
60142	60278	#define vqshrun_high_n_s16(__p0_738, __p1_738, __p2_738) __extension__ ({ \
60143		int8x16_t __ret_738; \
60144		int8x8_t __s0_738 = __p0_738; \
	60279	uint8x16_t __ret_738; \
	60280	uint8x8_t __s0_738 = __p0_738; \
60145	60281	int16x8_t __s1_738 = __p1_738; \
60146		__ret_738 = __builtin_bit_cast(int8x16_t, vcombine_s8(__builtin_bit_cast(int8x8_t, __s0_738), __builtin_bit_cast(int8x8_t, vqshrun_n_s16(__s1_738, __p2_738)))); \
	60282	__ret_738 = __builtin_bit_cast(uint8x16_t, vcombine_u8(__builtin_bit_cast(uint8x8_t, __s0_738), __builtin_bit_cast(uint8x8_t, vqshrun_n_s16(__s1_738, __p2_738)))); \
60147	60283	__ret_738; \
60148	60284	})
60149	60285	#else
60150	60286	#define vqshrun_high_n_s16(__p0_739, __p1_739, __p2_739) __extension__ ({ \
60151		int8x16_t __ret_739; \
60152		int8x8_t __s0_739 = __p0_739; \
	60287	uint8x16_t __ret_739; \
	60288	uint8x8_t __s0_739 = __p0_739; \
60153	60289	int16x8_t __s1_739 = __p1_739; \
60154		int8x8_t __rev0_739; __rev0_739 = __builtin_shufflevector(__s0_739, __s0_739, __lane_reverse_64_8); \
	60290	uint8x8_t __rev0_739; __rev0_739 = __builtin_shufflevector(__s0_739, __s0_739, __lane_reverse_64_8); \
60155	60291	int16x8_t __rev1_739; __rev1_739 = __builtin_shufflevector(__s1_739, __s1_739, __lane_reverse_128_16); \
60156		__ret_739 = __builtin_bit_cast(int8x16_t, __noswap_vcombine_s8(__builtin_bit_cast(int8x8_t, __rev0_739), __builtin_bit_cast(int8x8_t, __noswap_vqshrun_n_s16(__rev1_739, __p2_739)))); \
	60292	__ret_739 = __builtin_bit_cast(uint8x16_t, __noswap_vcombine_u8(__builtin_bit_cast(uint8x8_t, __rev0_739), __builtin_bit_cast(uint8x8_t, __noswap_vqshrun_n_s16(__rev1_739, __p2_739)))); \
60157	60293	__ret_739 = __builtin_shufflevector(__ret_739, __ret_739, __lane_reverse_128_8); \
60158	60294	__ret_739; \
60159	60295	})
...	...	@@ -64345,7 +64481,7 @@ __ai __attribute__((target("neon"))) int8x16_t vrsubhn_high_s16(int8x8_t __p0, i
64345	64481	poly64x1_t __ret; \
64346	64482	poly64_t __s0 = __p0; \
64347	64483	poly64x1_t __s1 = __p1; \
64348		__ret = __builtin_bit_cast(poly64x1_t, __builtin_neon_vset_lane_i64(__s0, __s1, __p2)); \
	64484	__ret = __builtin_bit_cast(poly64x1_t, __builtin_neon_vset_lane_i64(__s0, __builtin_bit_cast(int64x1_t, __s1), __p2)); \
64349	64485	__ret; \
64350	64486	})
64351	64487	#ifdef __LITTLE_ENDIAN__
...	...	@@ -64353,7 +64489,7 @@ __ai __attribute__((target("neon"))) int8x16_t vrsubhn_high_s16(int8x8_t __p0, i
64353	64489	poly64x2_t __ret; \
64354	64490	poly64_t __s0 = __p0; \
64355	64491	poly64x2_t __s1 = __p1; \
64356		__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __s1, __p2)); \
	64492	__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __builtin_bit_cast(int64x2_t, __s1), __p2)); \
64357	64493	__ret; \
64358	64494	})
64359	64495	#else
...	...	@@ -64362,7 +64498,7 @@ __ai __attribute__((target("neon"))) int8x16_t vrsubhn_high_s16(int8x8_t __p0, i
64362	64498	poly64_t __s0 = __p0; \
64363	64499	poly64x2_t __s1 = __p1; \
64364	64500	poly64x2_t __rev1; __rev1 = __builtin_shufflevector(__s1, __s1, __lane_reverse_128_64); \
64365		__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __rev1, __p2)); \
	64501	__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __builtin_bit_cast(int64x2_t, __rev1), __p2)); \
64366	64502	__ret = __builtin_shufflevector(__ret, __ret, __lane_reverse_128_64); \
64367	64503	__ret; \
64368	64504	})
...	...	@@ -64370,7 +64506,7 @@ __ai __attribute__((target("neon"))) int8x16_t vrsubhn_high_s16(int8x8_t __p0, i
64370	64506	poly64x2_t __ret; \
64371	64507	poly64_t __s0 = __p0; \
64372	64508	poly64x2_t __s1 = __p1; \
64373		__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __s1, __p2)); \
	64509	__ret = __builtin_bit_cast(poly64x2_t, __builtin_neon_vsetq_lane_i64(__s0, __builtin_bit_cast(int64x2_t, __s1), __p2)); \
64374	64510	__ret; \
64375	64511	})
64376	64512	#endif

lib/include/arm_sme.h+8-8

...	...	@@ -796,8 +796,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x2
796	796	void svdot_za16_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
797	797	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
798	798	void svdot_za16_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
799		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
800		void svmla_single_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
801	799	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
802	800	void svmla_single_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
803	801	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
...	...	@@ -808,6 +806,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_
808	806	void svmla_lane_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
809	807	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
810	808	void svmla_lane_za16_mf8_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
	809	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x1_fpm)))
	810	void svmla_za16_mf8_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
811	811	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
812	812	void svmla_za16_mf8_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
813	813	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
...	...	@@ -828,8 +828,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x
828	828	void svdot_za16_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
829	829	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za16_mf8_vg1x4_fpm)))
830	830	void svdot_za16_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
831		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x1_fpm)))
832		void svmla_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
833	831	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x2_fpm)))
834	832	void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
835	833	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za16_mf8_vg2x4_fpm)))
...	...	@@ -840,6 +838,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8
840	838	void svmla_lane_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
841	839	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za16_mf8_vg2x4_fpm)))
842	840	void svmla_lane_za16_vg2x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
	841	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x1_fpm)))
	842	void svmla_za16_vg2x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
843	843	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x2_fpm)))
844	844	void svmla_za16_vg2x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
845	845	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za16_mf8_vg2x4_fpm)))
...	...	@@ -860,8 +860,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x2
860	860	void svdot_za32_mf8_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
861	861	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
862	862	void svdot_za32_mf8_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
863		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
864		void svmla_single_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
865	863	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
866	864	void svmla_single_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
867	865	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
...	...	@@ -872,6 +870,8 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_
872	870	void svmla_lane_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
873	871	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
874	872	void svmla_lane_za32_mf8_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
	873	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x1_fpm)))
	874	void svmla_za32_mf8_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
875	875	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
876	876	void svmla_za32_mf8_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
877	877	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))
...	...	@@ -894,8 +894,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x
894	894	void svdot_za32_vg1x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
895	895	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svdot_za32_mf8_vg1x4_fpm)))
896	896	void svdot_za32_vg1x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8x4_t, fpm_t);
897		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x1_fpm)))
898		void svmla_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
899	897	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x2_fpm)))
900	898	void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, fpm_t);
901	899	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_single_za32_mf8_vg4x4_fpm)))
...	...	@@ -906,6 +904,8 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8
906	904	void svmla_lane_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8_t, uint64_t, fpm_t);
907	905	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_lane_za32_mf8_vg4x4_fpm)))
908	906	void svmla_lane_za32_vg4x4_fpm(uint32_t, svmfloat8x4_t, svmfloat8_t, uint64_t, fpm_t);
	907	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x1_fpm)))
	908	void svmla_za32_vg4x1_fpm(uint32_t, svmfloat8_t, svmfloat8_t, fpm_t);
909	909	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x2_fpm)))
910	910	void svmla_za32_vg4x2_fpm(uint32_t, svmfloat8x2_t, svmfloat8x2_t, fpm_t);
911	911	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmla_za32_mf8_vg4x4_fpm)))

lib/include/arm_sve.h+4162-3782

...	...	@@ -4617,6 +4617,86 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x2)))
4617	4617	svbfloat16x2_t svminnm(svbfloat16x2_t, svbfloat16x2_t);
4618	4618	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x4)))
4619	4619	svbfloat16x4_t svminnm(svbfloat16x4_t, svbfloat16x4_t);
	4620	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_bf16_x2)))
	4621	svbfloat16x2_t svmul_single_bf16_x2(svbfloat16x2_t, svbfloat16_t);
	4622	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_bf16_x4)))
	4623	svbfloat16x4_t svmul_single_bf16_x4(svbfloat16x4_t, svbfloat16_t);
	4624	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x2)))
	4625	svbfloat16x2_t svmul_bf16_x2(svbfloat16x2_t, svbfloat16x2_t);
	4626	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x4)))
	4627	svbfloat16x4_t svmul_bf16_x4(svbfloat16x4_t, svbfloat16x4_t);
	4628	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_bf16_x2)))
	4629	svbfloat16x2_t svscale_single_bf16_x2(svbfloat16x2_t, svint16_t);
	4630	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_bf16_x4)))
	4631	svbfloat16x4_t svscale_single_bf16_x4(svbfloat16x4_t, svint16_t);
	4632	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x2)))
	4633	svbfloat16x2_t svscale_bf16_x2(svbfloat16x2_t, svint16x2_t);
	4634	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x4)))
	4635	svbfloat16x4_t svscale_bf16_x4(svbfloat16x4_t, svint16x4_t);
	4636	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_bf16_x2)))
	4637	svbfloat16x2_t svmul(svbfloat16x2_t, svbfloat16_t);
	4638	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_bf16_x4)))
	4639	svbfloat16x4_t svmul(svbfloat16x4_t, svbfloat16_t);
	4640	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x2)))
	4641	svbfloat16x2_t svmul(svbfloat16x2_t, svbfloat16x2_t);
	4642	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x4)))
	4643	svbfloat16x4_t svmul(svbfloat16x4_t, svbfloat16x4_t);
	4644	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_bf16_x2)))
	4645	svbfloat16x2_t svscale(svbfloat16x2_t, svint16_t);
	4646	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_single_bf16_x4)))
	4647	svbfloat16x4_t svscale(svbfloat16x4_t, svint16_t);
	4648	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x2)))
	4649	svbfloat16x2_t svscale(svbfloat16x2_t, svint16x2_t);
	4650	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x4)))
	4651	svbfloat16x4_t svscale(svbfloat16x4_t, svint16x4_t);
	4652	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f64_x2)))
	4653	svfloat64x2_t svmul_single_f64_x2(svfloat64x2_t, svfloat64_t);
	4654	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f32_x2)))
	4655	svfloat32x2_t svmul_single_f32_x2(svfloat32x2_t, svfloat32_t);
	4656	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f16_x2)))
	4657	svfloat16x2_t svmul_single_f16_x2(svfloat16x2_t, svfloat16_t);
	4658	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f64_x4)))
	4659	svfloat64x4_t svmul_single_f64_x4(svfloat64x4_t, svfloat64_t);
	4660	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f32_x4)))
	4661	svfloat32x4_t svmul_single_f32_x4(svfloat32x4_t, svfloat32_t);
	4662	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f16_x4)))
	4663	svfloat16x4_t svmul_single_f16_x4(svfloat16x4_t, svfloat16_t);
	4664	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f64_x2)))
	4665	svfloat64x2_t svmul_f64_x2(svfloat64x2_t, svfloat64x2_t);
	4666	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x2)))
	4667	svfloat32x2_t svmul_f32_x2(svfloat32x2_t, svfloat32x2_t);
	4668	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x2)))
	4669	svfloat16x2_t svmul_f16_x2(svfloat16x2_t, svfloat16x2_t);
	4670	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f64_x4)))
	4671	svfloat64x4_t svmul_f64_x4(svfloat64x4_t, svfloat64x4_t);
	4672	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x4)))
	4673	svfloat32x4_t svmul_f32_x4(svfloat32x4_t, svfloat32x4_t);
	4674	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x4)))
	4675	svfloat16x4_t svmul_f16_x4(svfloat16x4_t, svfloat16x4_t);
	4676	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f64_x2)))
	4677	svfloat64x2_t svmul(svfloat64x2_t, svfloat64_t);
	4678	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f32_x2)))
	4679	svfloat32x2_t svmul(svfloat32x2_t, svfloat32_t);
	4680	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f16_x2)))
	4681	svfloat16x2_t svmul(svfloat16x2_t, svfloat16_t);
	4682	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f64_x4)))
	4683	svfloat64x4_t svmul(svfloat64x4_t, svfloat64_t);
	4684	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f32_x4)))
	4685	svfloat32x4_t svmul(svfloat32x4_t, svfloat32_t);
	4686	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_single_f16_x4)))
	4687	svfloat16x4_t svmul(svfloat16x4_t, svfloat16_t);
	4688	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f64_x2)))
	4689	svfloat64x2_t svmul(svfloat64x2_t, svfloat64x2_t);
	4690	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x2)))
	4691	svfloat32x2_t svmul(svfloat32x2_t, svfloat32x2_t);
	4692	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x2)))
	4693	svfloat16x2_t svmul(svfloat16x2_t, svfloat16x2_t);
	4694	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f64_x4)))
	4695	svfloat64x4_t svmul(svfloat64x4_t, svfloat64x4_t);
	4696	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f32_x4)))
	4697	svfloat32x4_t svmul(svfloat32x4_t, svfloat32x4_t);
	4698	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_f16_x4)))
	4699	svfloat16x4_t svmul(svfloat16x4_t, svfloat16x4_t);
4620	4700	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f64)))
4621	4701	float64_t svadda_f64(svbool_t, float64_t, svfloat64_t);
4622	4702	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadda_f32)))
...	...	@@ -4655,18 +4735,6 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrw_u32base_s32in
4655	4735	svuint32_t svadrw_u32base_s32index(svuint32_t, svint32_t);
4656	4736	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrw_u64base_s64index)))
4657	4737	svuint64_t svadrw_u64base_s64index(svuint64_t, svint64_t);
4658		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u32)))
4659		svuint32_t svcompact_u32(svbool_t, svuint32_t);
4660		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u64)))
4661		svuint64_t svcompact_u64(svbool_t, svuint64_t);
4662		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f64)))
4663		svfloat64_t svcompact_f64(svbool_t, svfloat64_t);
4664		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f32)))
4665		svfloat32_t svcompact_f32(svbool_t, svfloat32_t);
4666		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s32)))
4667		svint32_t svcompact_s32(svbool_t, svint32_t);
4668		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s64)))
4669		svint64_t svcompact_s64(svbool_t, svint64_t);
4670	4738	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_gather_u32base_index_u32)))
4671	4739	svuint32_t svld1_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
4672	4740	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_gather_u64base_index_u64)))
...	...	@@ -5923,18 +5991,6 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrw_u32base_s32i
5923	5991	svuint32_t svadrw_index(svuint32_t, svint32_t);
5924	5992	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadrw_u64base_s64index)))
5925	5993	svuint64_t svadrw_index(svuint64_t, svint64_t);
5926		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u32)))
5927		svuint32_t svcompact(svbool_t, svuint32_t);
5928		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u64)))
5929		svuint64_t svcompact(svbool_t, svuint64_t);
5930		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f64)))
5931		svfloat64_t svcompact(svbool_t, svfloat64_t);
5932		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f32)))
5933		svfloat32_t svcompact(svbool_t, svfloat32_t);
5934		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s32)))
5935		svint32_t svcompact(svbool_t, svint32_t);
5936		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s64)))
5937		svint64_t svcompact(svbool_t, svint64_t);
5938	5994	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_gather_u32base_index_u32)))
5939	5995	svuint32_t svld1_gather_index_u32(svbool_t, svuint32_t, int64_t);
5940	5996	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_gather_u64base_index_u64)))
...	...	@@ -8777,3764 +8833,222 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_mf8)))
8777	8833	svmfloat8_t svzipq2(svmfloat8_t, svmfloat8_t);
8778	8834	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzipq2_s16)))
8779	8835	svint16_t svzipq2(svint16_t, svint16_t);
8780		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmmla_f32)))
8781		svfloat32_t svbfmmla_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
8782		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmmla_f32)))
8783		svfloat32_t svbfmmla(svfloat32_t, svbfloat16_t, svbfloat16_t);
8784		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_n_f32)))
8785		svfloat32_t svbfdot_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
8786		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_f32)))
8787		svfloat32_t svbfdot_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
8788		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_lane_f32)))
8789		svfloat32_t svbfdot_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8790		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_n_f32)))
8791		svfloat32_t svbfmlalb_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
8792		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_f32)))
8793		svfloat32_t svbfmlalb_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
8794		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_lane_f32)))
8795		svfloat32_t svbfmlalb_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8796		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_n_f32)))
8797		svfloat32_t svbfmlalt_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
8798		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_f32)))
8799		svfloat32_t svbfmlalt_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
8800		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_lane_f32)))
8801		svfloat32_t svbfmlalt_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8802		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_m)))
8803		svbfloat16_t svcvt_bf16_f32_m(svbfloat16_t, svbool_t, svfloat32_t);
8804		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_x)))
8805		svbfloat16_t svcvt_bf16_f32_x(svbool_t, svfloat32_t);
8806		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_z)))
8807		svbfloat16_t svcvt_bf16_f32_z(svbool_t, svfloat32_t);
8808		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_m)))
8809		svbfloat16_t svcvtnt_bf16_f32_m(svbfloat16_t, svbool_t, svfloat32_t);
8810		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_n_f32)))
8811		svfloat32_t svbfdot(svfloat32_t, svbfloat16_t, bfloat16_t);
8812		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_f32)))
8813		svfloat32_t svbfdot(svfloat32_t, svbfloat16_t, svbfloat16_t);
8814		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_lane_f32)))
8815		svfloat32_t svbfdot_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8816		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_n_f32)))
8817		svfloat32_t svbfmlalb(svfloat32_t, svbfloat16_t, bfloat16_t);
8818		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_f32)))
8819		svfloat32_t svbfmlalb(svfloat32_t, svbfloat16_t, svbfloat16_t);
8820		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_lane_f32)))
8821		svfloat32_t svbfmlalb_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8822		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_n_f32)))
8823		svfloat32_t svbfmlalt(svfloat32_t, svbfloat16_t, bfloat16_t);
8824		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_f32)))
8825		svfloat32_t svbfmlalt(svfloat32_t, svbfloat16_t, svbfloat16_t);
8826		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_lane_f32)))
8827		svfloat32_t svbfmlalt_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
8828		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_m)))
8829		svbfloat16_t svcvt_bf16_m(svbfloat16_t, svbool_t, svfloat32_t);
8830		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_x)))
8831		svbfloat16_t svcvt_bf16_x(svbool_t, svfloat32_t);
8832		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_z)))
8833		svbfloat16_t svcvt_bf16_z(svbool_t, svfloat32_t);
8834		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_m)))
8835		svbfloat16_t svcvtnt_bf16_m(svbfloat16_t, svbool_t, svfloat32_t);
8836		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32)))
8837		svfloat32_t svmmla_f32(svfloat32_t, svfloat32_t, svfloat32_t);
8838		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32)))
8839		svfloat32_t svmmla(svfloat32_t, svfloat32_t, svfloat32_t);
8840		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u8)))
8841		svuint8_t svld1ro_u8(svbool_t, uint8_t const *);
8842		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u32)))
8843		svuint32_t svld1ro_u32(svbool_t, uint32_t const *);
8844		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u64)))
8845		svuint64_t svld1ro_u64(svbool_t, uint64_t const *);
8846		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u16)))
8847		svuint16_t svld1ro_u16(svbool_t, uint16_t const *);
8848		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_bf16)))
8849		svbfloat16_t svld1ro_bf16(svbool_t, bfloat16_t const *);
8850		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s8)))
8851		svint8_t svld1ro_s8(svbool_t, int8_t const *);
8852		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f64)))
8853		svfloat64_t svld1ro_f64(svbool_t, float64_t const *);
8854		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f32)))
8855		svfloat32_t svld1ro_f32(svbool_t, float32_t const *);
8856		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f16)))
8857		svfloat16_t svld1ro_f16(svbool_t, float16_t const *);
8858		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s32)))
8859		svint32_t svld1ro_s32(svbool_t, int32_t const *);
8860		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s64)))
8861		svint64_t svld1ro_s64(svbool_t, int64_t const *);
8862		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_mf8)))
8863		svmfloat8_t svld1ro_mf8(svbool_t, mfloat8_t const *);
8864		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s16)))
8865		svint16_t svld1ro_s16(svbool_t, int16_t const *);
8866		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f64)))
8867		svfloat64_t svmmla_f64(svfloat64_t, svfloat64_t, svfloat64_t);
8868		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u8)))
8869		svuint8_t svtrn1q_u8(svuint8_t, svuint8_t);
8870		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u32)))
8871		svuint32_t svtrn1q_u32(svuint32_t, svuint32_t);
8872		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u64)))
8873		svuint64_t svtrn1q_u64(svuint64_t, svuint64_t);
8874		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u16)))
8875		svuint16_t svtrn1q_u16(svuint16_t, svuint16_t);
8876		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_bf16)))
8877		svbfloat16_t svtrn1q_bf16(svbfloat16_t, svbfloat16_t);
8878		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s8)))
8879		svint8_t svtrn1q_s8(svint8_t, svint8_t);
8880		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f64)))
8881		svfloat64_t svtrn1q_f64(svfloat64_t, svfloat64_t);
8882		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f32)))
8883		svfloat32_t svtrn1q_f32(svfloat32_t, svfloat32_t);
8884		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f16)))
8885		svfloat16_t svtrn1q_f16(svfloat16_t, svfloat16_t);
8886		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s32)))
8887		svint32_t svtrn1q_s32(svint32_t, svint32_t);
8888		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s64)))
8889		svint64_t svtrn1q_s64(svint64_t, svint64_t);
8890		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s16)))
8891		svint16_t svtrn1q_s16(svint16_t, svint16_t);
8892		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u8)))
8893		svuint8_t svtrn2q_u8(svuint8_t, svuint8_t);
8894		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u32)))
8895		svuint32_t svtrn2q_u32(svuint32_t, svuint32_t);
8896		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u64)))
8897		svuint64_t svtrn2q_u64(svuint64_t, svuint64_t);
8898		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u16)))
8899		svuint16_t svtrn2q_u16(svuint16_t, svuint16_t);
8900		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_bf16)))
8901		svbfloat16_t svtrn2q_bf16(svbfloat16_t, svbfloat16_t);
8902		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s8)))
8903		svint8_t svtrn2q_s8(svint8_t, svint8_t);
8904		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f64)))
8905		svfloat64_t svtrn2q_f64(svfloat64_t, svfloat64_t);
8906		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f32)))
8907		svfloat32_t svtrn2q_f32(svfloat32_t, svfloat32_t);
8908		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f16)))
8909		svfloat16_t svtrn2q_f16(svfloat16_t, svfloat16_t);
8910		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s32)))
8911		svint32_t svtrn2q_s32(svint32_t, svint32_t);
8912		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s64)))
8913		svint64_t svtrn2q_s64(svint64_t, svint64_t);
8914		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s16)))
8915		svint16_t svtrn2q_s16(svint16_t, svint16_t);
8916		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u8)))
8917		svuint8_t svuzp1q_u8(svuint8_t, svuint8_t);
8918		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u32)))
8919		svuint32_t svuzp1q_u32(svuint32_t, svuint32_t);
8920		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u64)))
8921		svuint64_t svuzp1q_u64(svuint64_t, svuint64_t);
8922		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u16)))
8923		svuint16_t svuzp1q_u16(svuint16_t, svuint16_t);
8924		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_bf16)))
8925		svbfloat16_t svuzp1q_bf16(svbfloat16_t, svbfloat16_t);
8926		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s8)))
8927		svint8_t svuzp1q_s8(svint8_t, svint8_t);
8928		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f64)))
8929		svfloat64_t svuzp1q_f64(svfloat64_t, svfloat64_t);
8930		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f32)))
8931		svfloat32_t svuzp1q_f32(svfloat32_t, svfloat32_t);
8932		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f16)))
8933		svfloat16_t svuzp1q_f16(svfloat16_t, svfloat16_t);
8934		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s32)))
8935		svint32_t svuzp1q_s32(svint32_t, svint32_t);
8936		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s64)))
8937		svint64_t svuzp1q_s64(svint64_t, svint64_t);
8938		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s16)))
8939		svint16_t svuzp1q_s16(svint16_t, svint16_t);
8940		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u8)))
8941		svuint8_t svuzp2q_u8(svuint8_t, svuint8_t);
8942		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u32)))
8943		svuint32_t svuzp2q_u32(svuint32_t, svuint32_t);
8944		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u64)))
8945		svuint64_t svuzp2q_u64(svuint64_t, svuint64_t);
8946		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u16)))
8947		svuint16_t svuzp2q_u16(svuint16_t, svuint16_t);
8948		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_bf16)))
8949		svbfloat16_t svuzp2q_bf16(svbfloat16_t, svbfloat16_t);
8950		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s8)))
8951		svint8_t svuzp2q_s8(svint8_t, svint8_t);
8952		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f64)))
8953		svfloat64_t svuzp2q_f64(svfloat64_t, svfloat64_t);
8954		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f32)))
8955		svfloat32_t svuzp2q_f32(svfloat32_t, svfloat32_t);
8956		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f16)))
8957		svfloat16_t svuzp2q_f16(svfloat16_t, svfloat16_t);
8958		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s32)))
8959		svint32_t svuzp2q_s32(svint32_t, svint32_t);
8960		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s64)))
8961		svint64_t svuzp2q_s64(svint64_t, svint64_t);
8962		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s16)))
8963		svint16_t svuzp2q_s16(svint16_t, svint16_t);
8964		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u8)))
8965		svuint8_t svzip1q_u8(svuint8_t, svuint8_t);
8966		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u32)))
8967		svuint32_t svzip1q_u32(svuint32_t, svuint32_t);
8968		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u64)))
8969		svuint64_t svzip1q_u64(svuint64_t, svuint64_t);
8970		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u16)))
8971		svuint16_t svzip1q_u16(svuint16_t, svuint16_t);
8972		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_bf16)))
8973		svbfloat16_t svzip1q_bf16(svbfloat16_t, svbfloat16_t);
8974		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s8)))
8975		svint8_t svzip1q_s8(svint8_t, svint8_t);
8976		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f64)))
8977		svfloat64_t svzip1q_f64(svfloat64_t, svfloat64_t);
8978		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f32)))
8979		svfloat32_t svzip1q_f32(svfloat32_t, svfloat32_t);
8980		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f16)))
8981		svfloat16_t svzip1q_f16(svfloat16_t, svfloat16_t);
8982		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s32)))
8983		svint32_t svzip1q_s32(svint32_t, svint32_t);
8984		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s64)))
8985		svint64_t svzip1q_s64(svint64_t, svint64_t);
8986		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s16)))
8987		svint16_t svzip1q_s16(svint16_t, svint16_t);
8988		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u8)))
8989		svuint8_t svzip2q_u8(svuint8_t, svuint8_t);
8990		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u32)))
8991		svuint32_t svzip2q_u32(svuint32_t, svuint32_t);
8992		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u64)))
8993		svuint64_t svzip2q_u64(svuint64_t, svuint64_t);
8994		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u16)))
8995		svuint16_t svzip2q_u16(svuint16_t, svuint16_t);
8996		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_bf16)))
8997		svbfloat16_t svzip2q_bf16(svbfloat16_t, svbfloat16_t);
8998		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s8)))
8999		svint8_t svzip2q_s8(svint8_t, svint8_t);
9000		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f64)))
9001		svfloat64_t svzip2q_f64(svfloat64_t, svfloat64_t);
9002		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f32)))
9003		svfloat32_t svzip2q_f32(svfloat32_t, svfloat32_t);
9004		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f16)))
9005		svfloat16_t svzip2q_f16(svfloat16_t, svfloat16_t);
9006		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s32)))
9007		svint32_t svzip2q_s32(svint32_t, svint32_t);
9008		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s64)))
9009		svint64_t svzip2q_s64(svint64_t, svint64_t);
9010		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s16)))
9011		svint16_t svzip2q_s16(svint16_t, svint16_t);
9012		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u8)))
9013		svuint8_t svld1ro(svbool_t, uint8_t const *);
9014		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u32)))
9015		svuint32_t svld1ro(svbool_t, uint32_t const *);
9016		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u64)))
9017		svuint64_t svld1ro(svbool_t, uint64_t const *);
9018		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u16)))
9019		svuint16_t svld1ro(svbool_t, uint16_t const *);
9020		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_bf16)))
9021		svbfloat16_t svld1ro(svbool_t, bfloat16_t const *);
9022		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s8)))
9023		svint8_t svld1ro(svbool_t, int8_t const *);
9024		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f64)))
9025		svfloat64_t svld1ro(svbool_t, float64_t const *);
9026		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f32)))
9027		svfloat32_t svld1ro(svbool_t, float32_t const *);
9028		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f16)))
9029		svfloat16_t svld1ro(svbool_t, float16_t const *);
9030		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s32)))
9031		svint32_t svld1ro(svbool_t, int32_t const *);
9032		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s64)))
9033		svint64_t svld1ro(svbool_t, int64_t const *);
9034		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_mf8)))
9035		svmfloat8_t svld1ro(svbool_t, mfloat8_t const *);
9036		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s16)))
9037		svint16_t svld1ro(svbool_t, int16_t const *);
9038		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f64)))
9039		svfloat64_t svmmla(svfloat64_t, svfloat64_t, svfloat64_t);
9040		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u8)))
9041		svuint8_t svtrn1q(svuint8_t, svuint8_t);
9042		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u32)))
9043		svuint32_t svtrn1q(svuint32_t, svuint32_t);
9044		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u64)))
9045		svuint64_t svtrn1q(svuint64_t, svuint64_t);
9046		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u16)))
9047		svuint16_t svtrn1q(svuint16_t, svuint16_t);
9048		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_bf16)))
9049		svbfloat16_t svtrn1q(svbfloat16_t, svbfloat16_t);
9050		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s8)))
9051		svint8_t svtrn1q(svint8_t, svint8_t);
9052		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f64)))
9053		svfloat64_t svtrn1q(svfloat64_t, svfloat64_t);
9054		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f32)))
9055		svfloat32_t svtrn1q(svfloat32_t, svfloat32_t);
9056		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f16)))
9057		svfloat16_t svtrn1q(svfloat16_t, svfloat16_t);
9058		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s32)))
9059		svint32_t svtrn1q(svint32_t, svint32_t);
9060		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s64)))
9061		svint64_t svtrn1q(svint64_t, svint64_t);
9062		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s16)))
9063		svint16_t svtrn1q(svint16_t, svint16_t);
9064		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u8)))
9065		svuint8_t svtrn2q(svuint8_t, svuint8_t);
9066		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u32)))
9067		svuint32_t svtrn2q(svuint32_t, svuint32_t);
9068		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u64)))
9069		svuint64_t svtrn2q(svuint64_t, svuint64_t);
9070		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u16)))
9071		svuint16_t svtrn2q(svuint16_t, svuint16_t);
9072		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_bf16)))
9073		svbfloat16_t svtrn2q(svbfloat16_t, svbfloat16_t);
9074		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s8)))
9075		svint8_t svtrn2q(svint8_t, svint8_t);
9076		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f64)))
9077		svfloat64_t svtrn2q(svfloat64_t, svfloat64_t);
9078		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f32)))
9079		svfloat32_t svtrn2q(svfloat32_t, svfloat32_t);
9080		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f16)))
9081		svfloat16_t svtrn2q(svfloat16_t, svfloat16_t);
9082		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s32)))
9083		svint32_t svtrn2q(svint32_t, svint32_t);
9084		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s64)))
9085		svint64_t svtrn2q(svint64_t, svint64_t);
9086		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s16)))
9087		svint16_t svtrn2q(svint16_t, svint16_t);
9088		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u8)))
9089		svuint8_t svuzp1q(svuint8_t, svuint8_t);
9090		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u32)))
9091		svuint32_t svuzp1q(svuint32_t, svuint32_t);
9092		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u64)))
9093		svuint64_t svuzp1q(svuint64_t, svuint64_t);
9094		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u16)))
9095		svuint16_t svuzp1q(svuint16_t, svuint16_t);
9096		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_bf16)))
9097		svbfloat16_t svuzp1q(svbfloat16_t, svbfloat16_t);
9098		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s8)))
9099		svint8_t svuzp1q(svint8_t, svint8_t);
9100		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f64)))
9101		svfloat64_t svuzp1q(svfloat64_t, svfloat64_t);
9102		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f32)))
9103		svfloat32_t svuzp1q(svfloat32_t, svfloat32_t);
9104		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f16)))
9105		svfloat16_t svuzp1q(svfloat16_t, svfloat16_t);
9106		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s32)))
9107		svint32_t svuzp1q(svint32_t, svint32_t);
9108		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s64)))
9109		svint64_t svuzp1q(svint64_t, svint64_t);
9110		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s16)))
9111		svint16_t svuzp1q(svint16_t, svint16_t);
9112		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u8)))
9113		svuint8_t svuzp2q(svuint8_t, svuint8_t);
9114		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u32)))
9115		svuint32_t svuzp2q(svuint32_t, svuint32_t);
9116		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u64)))
9117		svuint64_t svuzp2q(svuint64_t, svuint64_t);
9118		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u16)))
9119		svuint16_t svuzp2q(svuint16_t, svuint16_t);
9120		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_bf16)))
9121		svbfloat16_t svuzp2q(svbfloat16_t, svbfloat16_t);
9122		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s8)))
9123		svint8_t svuzp2q(svint8_t, svint8_t);
9124		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f64)))
9125		svfloat64_t svuzp2q(svfloat64_t, svfloat64_t);
9126		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f32)))
9127		svfloat32_t svuzp2q(svfloat32_t, svfloat32_t);
9128		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f16)))
9129		svfloat16_t svuzp2q(svfloat16_t, svfloat16_t);
9130		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s32)))
9131		svint32_t svuzp2q(svint32_t, svint32_t);
9132		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s64)))
9133		svint64_t svuzp2q(svint64_t, svint64_t);
9134		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s16)))
9135		svint16_t svuzp2q(svint16_t, svint16_t);
9136		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u8)))
9137		svuint8_t svzip1q(svuint8_t, svuint8_t);
9138		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u32)))
9139		svuint32_t svzip1q(svuint32_t, svuint32_t);
9140		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u64)))
9141		svuint64_t svzip1q(svuint64_t, svuint64_t);
9142		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u16)))
9143		svuint16_t svzip1q(svuint16_t, svuint16_t);
9144		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_bf16)))
9145		svbfloat16_t svzip1q(svbfloat16_t, svbfloat16_t);
9146		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s8)))
9147		svint8_t svzip1q(svint8_t, svint8_t);
9148		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f64)))
9149		svfloat64_t svzip1q(svfloat64_t, svfloat64_t);
9150		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f32)))
9151		svfloat32_t svzip1q(svfloat32_t, svfloat32_t);
9152		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f16)))
9153		svfloat16_t svzip1q(svfloat16_t, svfloat16_t);
9154		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s32)))
9155		svint32_t svzip1q(svint32_t, svint32_t);
9156		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s64)))
9157		svint64_t svzip1q(svint64_t, svint64_t);
9158		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s16)))
9159		svint16_t svzip1q(svint16_t, svint16_t);
9160		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u8)))
9161		svuint8_t svzip2q(svuint8_t, svuint8_t);
9162		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u32)))
9163		svuint32_t svzip2q(svuint32_t, svuint32_t);
9164		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u64)))
9165		svuint64_t svzip2q(svuint64_t, svuint64_t);
9166		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u16)))
9167		svuint16_t svzip2q(svuint16_t, svuint16_t);
9168		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_bf16)))
9169		svbfloat16_t svzip2q(svbfloat16_t, svbfloat16_t);
9170		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s8)))
9171		svint8_t svzip2q(svint8_t, svint8_t);
9172		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f64)))
9173		svfloat64_t svzip2q(svfloat64_t, svfloat64_t);
9174		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f32)))
9175		svfloat32_t svzip2q(svfloat32_t, svfloat32_t);
9176		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f16)))
9177		svfloat16_t svzip2q(svfloat16_t, svfloat16_t);
9178		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s32)))
9179		svint32_t svzip2q(svint32_t, svint32_t);
9180		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s64)))
9181		svint64_t svzip2q(svint64_t, svint64_t);
9182		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s16)))
9183		svint16_t svzip2q(svint16_t, svint16_t);
9184		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_s32)))
9185		svint32_t svmmla_s32(svint32_t, svint8_t, svint8_t);
9186		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_u32)))
9187		svuint32_t svmmla_u32(svuint32_t, svuint8_t, svuint8_t);
9188		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusmmla_s32)))
9189		svint32_t svusmmla_s32(svint32_t, svuint8_t, svint8_t);
9190		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_s32)))
9191		svint32_t svmmla(svint32_t, svint8_t, svint8_t);
9192		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_u32)))
9193		svuint32_t svmmla(svuint32_t, svuint8_t, svuint8_t);
9194		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusmmla_s32)))
9195		svint32_t svusmmla(svint32_t, svuint8_t, svint8_t);
9196		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_n_s32)))
9197		svint32_t svsudot_n_s32(svint32_t, svint8_t, uint8_t);
9198		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_s32)))
9199		svint32_t svsudot_s32(svint32_t, svint8_t, svuint8_t);
9200		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_lane_s32)))
9201		svint32_t svsudot_lane_s32(svint32_t, svint8_t, svuint8_t, uint64_t);
9202		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_n_s32)))
9203		svint32_t svusdot_n_s32(svint32_t, svuint8_t, int8_t);
9204		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_s32)))
9205		svint32_t svusdot_s32(svint32_t, svuint8_t, svint8_t);
9206		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_lane_s32)))
9207		svint32_t svusdot_lane_s32(svint32_t, svuint8_t, svint8_t, uint64_t);
9208		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_n_s32)))
9209		svint32_t svsudot(svint32_t, svint8_t, uint8_t);
9210		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_s32)))
9211		svint32_t svsudot(svint32_t, svint8_t, svuint8_t);
9212		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_lane_s32)))
9213		svint32_t svsudot_lane(svint32_t, svint8_t, svuint8_t, uint64_t);
9214		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_n_s32)))
9215		svint32_t svusdot(svint32_t, svuint8_t, int8_t);
9216		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_s32)))
9217		svint32_t svusdot(svint32_t, svuint8_t, svint8_t);
9218		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_lane_s32)))
9219		svint32_t svusdot_lane(svint32_t, svuint8_t, svint8_t, uint64_t);
9220		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_u64)))
9221		svuint64_t svrax1_u64(svuint64_t, svuint64_t);
9222		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_s64)))
9223		svint64_t svrax1_s64(svint64_t, svint64_t);
9224		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_u64)))
9225		svuint64_t svrax1(svuint64_t, svuint64_t);
9226		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_s64)))
9227		svint64_t svrax1(svint64_t, svint64_t);
9228		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u32_z)))
9229		svuint32_t svhistcnt_u32_z(svbool_t, svuint32_t, svuint32_t);
9230		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u64_z)))
9231		svuint64_t svhistcnt_u64_z(svbool_t, svuint64_t, svuint64_t);
9232		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s32_z)))
9233		svuint32_t svhistcnt_s32_z(svbool_t, svint32_t, svint32_t);
9234		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s64_z)))
9235		svuint64_t svhistcnt_s64_z(svbool_t, svint64_t, svint64_t);
9236		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_u8)))
9237		svuint8_t svhistseg_u8(svuint8_t, svuint8_t);
9238		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_s8)))
9239		svuint8_t svhistseg_s8(svint8_t, svint8_t);
9240		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_u32)))
9241		svuint32_t svldnt1_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
9242		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_u64)))
9243		svuint64_t svldnt1_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
9244		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_f64)))
9245		svfloat64_t svldnt1_gather_u64base_index_f64(svbool_t, svuint64_t, int64_t);
9246		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_f32)))
9247		svfloat32_t svldnt1_gather_u32base_index_f32(svbool_t, svuint32_t, int64_t);
9248		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_s32)))
9249		svint32_t svldnt1_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
9250		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_s64)))
9251		svint64_t svldnt1_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
9252		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_u32)))
9253		svuint32_t svldnt1_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
9254		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_u64)))
9255		svuint64_t svldnt1_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9256		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_f64)))
9257		svfloat64_t svldnt1_gather_u64base_offset_f64(svbool_t, svuint64_t, int64_t);
9258		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_f32)))
9259		svfloat32_t svldnt1_gather_u32base_offset_f32(svbool_t, svuint32_t, int64_t);
9260		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_s32)))
9261		svint32_t svldnt1_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
9262		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_s64)))
9263		svint64_t svldnt1_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9264		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_u32)))
9265		svuint32_t svldnt1_gather_u32base_u32(svbool_t, svuint32_t);
9266		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_u64)))
9267		svuint64_t svldnt1_gather_u64base_u64(svbool_t, svuint64_t);
9268		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_f64)))
9269		svfloat64_t svldnt1_gather_u64base_f64(svbool_t, svuint64_t);
9270		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_f32)))
9271		svfloat32_t svldnt1_gather_u32base_f32(svbool_t, svuint32_t);
9272		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_s32)))
9273		svint32_t svldnt1_gather_u32base_s32(svbool_t, svuint32_t);
9274		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_s64)))
9275		svint64_t svldnt1_gather_u64base_s64(svbool_t, svuint64_t);
9276		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_u64)))
9277		svuint64_t svldnt1_gather_s64index_u64(svbool_t, uint64_t const *, svint64_t);
9278		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_f64)))
9279		svfloat64_t svldnt1_gather_s64index_f64(svbool_t, float64_t const *, svint64_t);
9280		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_s64)))
9281		svint64_t svldnt1_gather_s64index_s64(svbool_t, int64_t const *, svint64_t);
9282		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_u64)))
9283		svuint64_t svldnt1_gather_u64index_u64(svbool_t, uint64_t const *, svuint64_t);
9284		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_f64)))
9285		svfloat64_t svldnt1_gather_u64index_f64(svbool_t, float64_t const *, svuint64_t);
9286		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_s64)))
9287		svint64_t svldnt1_gather_u64index_s64(svbool_t, int64_t const *, svuint64_t);
9288		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_u32)))
9289		svuint32_t svldnt1_gather_u32offset_u32(svbool_t, uint32_t const *, svuint32_t);
9290		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_f32)))
9291		svfloat32_t svldnt1_gather_u32offset_f32(svbool_t, float32_t const *, svuint32_t);
9292		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_s32)))
9293		svint32_t svldnt1_gather_u32offset_s32(svbool_t, int32_t const *, svuint32_t);
9294		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_u64)))
9295		svuint64_t svldnt1_gather_s64offset_u64(svbool_t, uint64_t const *, svint64_t);
9296		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_f64)))
9297		svfloat64_t svldnt1_gather_s64offset_f64(svbool_t, float64_t const *, svint64_t);
9298		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_s64)))
9299		svint64_t svldnt1_gather_s64offset_s64(svbool_t, int64_t const *, svint64_t);
9300		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_u64)))
9301		svuint64_t svldnt1_gather_u64offset_u64(svbool_t, uint64_t const *, svuint64_t);
9302		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_f64)))
9303		svfloat64_t svldnt1_gather_u64offset_f64(svbool_t, float64_t const *, svuint64_t);
9304		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_s64)))
9305		svint64_t svldnt1_gather_u64offset_s64(svbool_t, int64_t const *, svuint64_t);
9306		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_u32)))
9307		svuint32_t svldnt1sb_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
9308		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_u64)))
9309		svuint64_t svldnt1sb_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9310		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_s32)))
9311		svint32_t svldnt1sb_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
9312		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_s64)))
9313		svint64_t svldnt1sb_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9314		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_u32)))
9315		svuint32_t svldnt1sb_gather_u32base_u32(svbool_t, svuint32_t);
9316		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_u64)))
9317		svuint64_t svldnt1sb_gather_u64base_u64(svbool_t, svuint64_t);
9318		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_s32)))
9319		svint32_t svldnt1sb_gather_u32base_s32(svbool_t, svuint32_t);
9320		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_s64)))
9321		svint64_t svldnt1sb_gather_u64base_s64(svbool_t, svuint64_t);
9322		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_u32)))
9323		svuint32_t svldnt1sb_gather_u32offset_u32(svbool_t, int8_t const *, svuint32_t);
9324		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_s32)))
9325		svint32_t svldnt1sb_gather_u32offset_s32(svbool_t, int8_t const *, svuint32_t);
9326		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_u64)))
9327		svuint64_t svldnt1sb_gather_s64offset_u64(svbool_t, int8_t const *, svint64_t);
9328		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_s64)))
9329		svint64_t svldnt1sb_gather_s64offset_s64(svbool_t, int8_t const *, svint64_t);
9330		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_u64)))
9331		svuint64_t svldnt1sb_gather_u64offset_u64(svbool_t, int8_t const *, svuint64_t);
9332		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_s64)))
9333		svint64_t svldnt1sb_gather_u64offset_s64(svbool_t, int8_t const *, svuint64_t);
9334		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_u32)))
9335		svuint32_t svldnt1sh_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
9336		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_u64)))
9337		svuint64_t svldnt1sh_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
9338		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_s32)))
9339		svint32_t svldnt1sh_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
9340		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_s64)))
9341		svint64_t svldnt1sh_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
9342		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_u32)))
9343		svuint32_t svldnt1sh_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
9344		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_u64)))
9345		svuint64_t svldnt1sh_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9346		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_s32)))
9347		svint32_t svldnt1sh_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
9348		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_s64)))
9349		svint64_t svldnt1sh_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9350		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_u32)))
9351		svuint32_t svldnt1sh_gather_u32base_u32(svbool_t, svuint32_t);
9352		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_u64)))
9353		svuint64_t svldnt1sh_gather_u64base_u64(svbool_t, svuint64_t);
9354		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_s32)))
9355		svint32_t svldnt1sh_gather_u32base_s32(svbool_t, svuint32_t);
9356		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_s64)))
9357		svint64_t svldnt1sh_gather_u64base_s64(svbool_t, svuint64_t);
9358		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_u64)))
9359		svuint64_t svldnt1sh_gather_s64index_u64(svbool_t, int16_t const *, svint64_t);
9360		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_s64)))
9361		svint64_t svldnt1sh_gather_s64index_s64(svbool_t, int16_t const *, svint64_t);
9362		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_u64)))
9363		svuint64_t svldnt1sh_gather_u64index_u64(svbool_t, int16_t const *, svuint64_t);
9364		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_s64)))
9365		svint64_t svldnt1sh_gather_u64index_s64(svbool_t, int16_t const *, svuint64_t);
9366		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_u32)))
9367		svuint32_t svldnt1sh_gather_u32offset_u32(svbool_t, int16_t const *, svuint32_t);
9368		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_s32)))
9369		svint32_t svldnt1sh_gather_u32offset_s32(svbool_t, int16_t const *, svuint32_t);
9370		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_u64)))
9371		svuint64_t svldnt1sh_gather_s64offset_u64(svbool_t, int16_t const *, svint64_t);
9372		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_s64)))
9373		svint64_t svldnt1sh_gather_s64offset_s64(svbool_t, int16_t const *, svint64_t);
9374		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_u64)))
9375		svuint64_t svldnt1sh_gather_u64offset_u64(svbool_t, int16_t const *, svuint64_t);
9376		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_s64)))
9377		svint64_t svldnt1sh_gather_u64offset_s64(svbool_t, int16_t const *, svuint64_t);
9378		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_u64)))
9379		svuint64_t svldnt1sw_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
9380		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_s64)))
9381		svint64_t svldnt1sw_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
9382		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_u64)))
9383		svuint64_t svldnt1sw_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9384		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_s64)))
9385		svint64_t svldnt1sw_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9386		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_u64)))
9387		svuint64_t svldnt1sw_gather_u64base_u64(svbool_t, svuint64_t);
9388		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_s64)))
9389		svint64_t svldnt1sw_gather_u64base_s64(svbool_t, svuint64_t);
9390		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_u64)))
9391		svuint64_t svldnt1sw_gather_s64index_u64(svbool_t, int32_t const *, svint64_t);
9392		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_s64)))
9393		svint64_t svldnt1sw_gather_s64index_s64(svbool_t, int32_t const *, svint64_t);
9394		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_u64)))
9395		svuint64_t svldnt1sw_gather_u64index_u64(svbool_t, int32_t const *, svuint64_t);
9396		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_s64)))
9397		svint64_t svldnt1sw_gather_u64index_s64(svbool_t, int32_t const *, svuint64_t);
9398		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_u64)))
9399		svuint64_t svldnt1sw_gather_s64offset_u64(svbool_t, int32_t const *, svint64_t);
9400		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_s64)))
9401		svint64_t svldnt1sw_gather_s64offset_s64(svbool_t, int32_t const *, svint64_t);
9402		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_u64)))
9403		svuint64_t svldnt1sw_gather_u64offset_u64(svbool_t, int32_t const *, svuint64_t);
9404		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_s64)))
9405		svint64_t svldnt1sw_gather_u64offset_s64(svbool_t, int32_t const *, svuint64_t);
9406		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_u32)))
9407		svuint32_t svldnt1ub_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
9408		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_u64)))
9409		svuint64_t svldnt1ub_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9410		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_s32)))
9411		svint32_t svldnt1ub_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
9412		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_s64)))
9413		svint64_t svldnt1ub_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9414		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_u32)))
9415		svuint32_t svldnt1ub_gather_u32base_u32(svbool_t, svuint32_t);
9416		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_u64)))
9417		svuint64_t svldnt1ub_gather_u64base_u64(svbool_t, svuint64_t);
9418		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_s32)))
9419		svint32_t svldnt1ub_gather_u32base_s32(svbool_t, svuint32_t);
9420		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_s64)))
9421		svint64_t svldnt1ub_gather_u64base_s64(svbool_t, svuint64_t);
9422		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_u32)))
9423		svuint32_t svldnt1ub_gather_u32offset_u32(svbool_t, uint8_t const *, svuint32_t);
9424		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_s32)))
9425		svint32_t svldnt1ub_gather_u32offset_s32(svbool_t, uint8_t const *, svuint32_t);
9426		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_u64)))
9427		svuint64_t svldnt1ub_gather_s64offset_u64(svbool_t, uint8_t const *, svint64_t);
9428		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_s64)))
9429		svint64_t svldnt1ub_gather_s64offset_s64(svbool_t, uint8_t const *, svint64_t);
9430		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_u64)))
9431		svuint64_t svldnt1ub_gather_u64offset_u64(svbool_t, uint8_t const *, svuint64_t);
9432		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_s64)))
9433		svint64_t svldnt1ub_gather_u64offset_s64(svbool_t, uint8_t const *, svuint64_t);
9434		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_u32)))
9435		svuint32_t svldnt1uh_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
9436		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_u64)))
9437		svuint64_t svldnt1uh_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
9438		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_s32)))
9439		svint32_t svldnt1uh_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
9440		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_s64)))
9441		svint64_t svldnt1uh_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
9442		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_u32)))
9443		svuint32_t svldnt1uh_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
9444		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_u64)))
9445		svuint64_t svldnt1uh_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9446		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_s32)))
9447		svint32_t svldnt1uh_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
9448		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_s64)))
9449		svint64_t svldnt1uh_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9450		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_u32)))
9451		svuint32_t svldnt1uh_gather_u32base_u32(svbool_t, svuint32_t);
9452		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_u64)))
9453		svuint64_t svldnt1uh_gather_u64base_u64(svbool_t, svuint64_t);
9454		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_s32)))
9455		svint32_t svldnt1uh_gather_u32base_s32(svbool_t, svuint32_t);
9456		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_s64)))
9457		svint64_t svldnt1uh_gather_u64base_s64(svbool_t, svuint64_t);
9458		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_u64)))
9459		svuint64_t svldnt1uh_gather_s64index_u64(svbool_t, uint16_t const *, svint64_t);
9460		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_s64)))
9461		svint64_t svldnt1uh_gather_s64index_s64(svbool_t, uint16_t const *, svint64_t);
9462		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_u64)))
9463		svuint64_t svldnt1uh_gather_u64index_u64(svbool_t, uint16_t const *, svuint64_t);
9464		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_s64)))
9465		svint64_t svldnt1uh_gather_u64index_s64(svbool_t, uint16_t const *, svuint64_t);
9466		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_u32)))
9467		svuint32_t svldnt1uh_gather_u32offset_u32(svbool_t, uint16_t const *, svuint32_t);
9468		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_s32)))
9469		svint32_t svldnt1uh_gather_u32offset_s32(svbool_t, uint16_t const *, svuint32_t);
9470		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_u64)))
9471		svuint64_t svldnt1uh_gather_s64offset_u64(svbool_t, uint16_t const *, svint64_t);
9472		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_s64)))
9473		svint64_t svldnt1uh_gather_s64offset_s64(svbool_t, uint16_t const *, svint64_t);
9474		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_u64)))
9475		svuint64_t svldnt1uh_gather_u64offset_u64(svbool_t, uint16_t const *, svuint64_t);
9476		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_s64)))
9477		svint64_t svldnt1uh_gather_u64offset_s64(svbool_t, uint16_t const *, svuint64_t);
9478		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_u64)))
9479		svuint64_t svldnt1uw_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
9480		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_s64)))
9481		svint64_t svldnt1uw_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
9482		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_u64)))
9483		svuint64_t svldnt1uw_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
9484		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_s64)))
9485		svint64_t svldnt1uw_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
9486		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_u64)))
9487		svuint64_t svldnt1uw_gather_u64base_u64(svbool_t, svuint64_t);
9488		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_s64)))
9489		svint64_t svldnt1uw_gather_u64base_s64(svbool_t, svuint64_t);
9490		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_u64)))
9491		svuint64_t svldnt1uw_gather_s64index_u64(svbool_t, uint32_t const *, svint64_t);
9492		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_s64)))
9493		svint64_t svldnt1uw_gather_s64index_s64(svbool_t, uint32_t const *, svint64_t);
9494		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_u64)))
9495		svuint64_t svldnt1uw_gather_u64index_u64(svbool_t, uint32_t const *, svuint64_t);
9496		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_s64)))
9497		svint64_t svldnt1uw_gather_u64index_s64(svbool_t, uint32_t const *, svuint64_t);
9498		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_u64)))
9499		svuint64_t svldnt1uw_gather_s64offset_u64(svbool_t, uint32_t const *, svint64_t);
9500		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_s64)))
9501		svint64_t svldnt1uw_gather_s64offset_s64(svbool_t, uint32_t const *, svint64_t);
9502		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_u64)))
9503		svuint64_t svldnt1uw_gather_u64offset_u64(svbool_t, uint32_t const *, svuint64_t);
9504		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_s64)))
9505		svint64_t svldnt1uw_gather_u64offset_s64(svbool_t, uint32_t const *, svuint64_t);
9506		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u8)))
9507		svbool_t svmatch_u8(svbool_t, svuint8_t, svuint8_t);
9508		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u16)))
9509		svbool_t svmatch_u16(svbool_t, svuint16_t, svuint16_t);
9510		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s8)))
9511		svbool_t svmatch_s8(svbool_t, svint8_t, svint8_t);
9512		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s16)))
9513		svbool_t svmatch_s16(svbool_t, svint16_t, svint16_t);
9514		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u8)))
9515		svbool_t svnmatch_u8(svbool_t, svuint8_t, svuint8_t);
9516		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u16)))
9517		svbool_t svnmatch_u16(svbool_t, svuint16_t, svuint16_t);
9518		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s8)))
9519		svbool_t svnmatch_s8(svbool_t, svint8_t, svint8_t);
9520		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s16)))
9521		svbool_t svnmatch_s16(svbool_t, svint16_t, svint16_t);
9522		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_u32)))
9523		void svstnt1_scatter_u32base_index_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
9524		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_u64)))
9525		void svstnt1_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9526		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_f64)))
9527		void svstnt1_scatter_u64base_index_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
9528		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_f32)))
9529		void svstnt1_scatter_u32base_index_f32(svbool_t, svuint32_t, int64_t, svfloat32_t);
9530		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_s32)))
9531		void svstnt1_scatter_u32base_index_s32(svbool_t, svuint32_t, int64_t, svint32_t);
9532		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_s64)))
9533		void svstnt1_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9534		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_u32)))
9535		void svstnt1_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
9536		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_u64)))
9537		void svstnt1_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9538		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_f64)))
9539		void svstnt1_scatter_u64base_offset_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
9540		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_f32)))
9541		void svstnt1_scatter_u32base_offset_f32(svbool_t, svuint32_t, int64_t, svfloat32_t);
9542		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_s32)))
9543		void svstnt1_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
9544		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_s64)))
9545		void svstnt1_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9546		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_u32)))
9547		void svstnt1_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
9548		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_u64)))
9549		void svstnt1_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
9550		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_f64)))
9551		void svstnt1_scatter_u64base_f64(svbool_t, svuint64_t, svfloat64_t);
9552		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_f32)))
9553		void svstnt1_scatter_u32base_f32(svbool_t, svuint32_t, svfloat32_t);
9554		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_s32)))
9555		void svstnt1_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
9556		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_s64)))
9557		void svstnt1_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
9558		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_u64)))
9559		void svstnt1_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9560		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_f64)))
9561		void svstnt1_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9562		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_s64)))
9563		void svstnt1_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9564		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_u64)))
9565		void svstnt1_scatter_u64index_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
9566		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_f64)))
9567		void svstnt1_scatter_u64index_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
9568		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_s64)))
9569		void svstnt1_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
9570		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_u32)))
9571		void svstnt1_scatter_u32offset_u32(svbool_t, uint32_t *, svuint32_t, svuint32_t);
9572		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_f32)))
9573		void svstnt1_scatter_u32offset_f32(svbool_t, float32_t *, svuint32_t, svfloat32_t);
9574		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_s32)))
9575		void svstnt1_scatter_u32offset_s32(svbool_t, int32_t *, svuint32_t, svint32_t);
9576		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_u64)))
9577		void svstnt1_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
9578		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_f64)))
9579		void svstnt1_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
9580		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_s64)))
9581		void svstnt1_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
9582		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_u64)))
9583		void svstnt1_scatter_u64offset_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
9584		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_f64)))
9585		void svstnt1_scatter_u64offset_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
9586		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_s64)))
9587		void svstnt1_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
9588		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_u32)))
9589		void svstnt1b_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
9590		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_u64)))
9591		void svstnt1b_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9592		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_s32)))
9593		void svstnt1b_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
9594		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_s64)))
9595		void svstnt1b_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9596		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_u32)))
9597		void svstnt1b_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
9598		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_u64)))
9599		void svstnt1b_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
9600		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_s32)))
9601		void svstnt1b_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
9602		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_s64)))
9603		void svstnt1b_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
9604		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_s32)))
9605		void svstnt1b_scatter_u32offset_s32(svbool_t, int8_t *, svuint32_t, svint32_t);
9606		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_u32)))
9607		void svstnt1b_scatter_u32offset_u32(svbool_t, uint8_t *, svuint32_t, svuint32_t);
9608		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_s64)))
9609		void svstnt1b_scatter_s64offset_s64(svbool_t, int8_t *, svint64_t, svint64_t);
9610		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_u64)))
9611		void svstnt1b_scatter_s64offset_u64(svbool_t, uint8_t *, svint64_t, svuint64_t);
9612		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_s64)))
9613		void svstnt1b_scatter_u64offset_s64(svbool_t, int8_t *, svuint64_t, svint64_t);
9614		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_u64)))
9615		void svstnt1b_scatter_u64offset_u64(svbool_t, uint8_t *, svuint64_t, svuint64_t);
9616		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_u32)))
9617		void svstnt1h_scatter_u32base_index_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
9618		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_u64)))
9619		void svstnt1h_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9620		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_s32)))
9621		void svstnt1h_scatter_u32base_index_s32(svbool_t, svuint32_t, int64_t, svint32_t);
9622		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_s64)))
9623		void svstnt1h_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9624		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_u32)))
9625		void svstnt1h_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
9626		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_u64)))
9627		void svstnt1h_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9628		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_s32)))
9629		void svstnt1h_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
9630		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_s64)))
9631		void svstnt1h_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9632		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_u32)))
9633		void svstnt1h_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
9634		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_u64)))
9635		void svstnt1h_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
9636		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_s32)))
9637		void svstnt1h_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
9638		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_s64)))
9639		void svstnt1h_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
9640		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_s64)))
9641		void svstnt1h_scatter_s64index_s64(svbool_t, int16_t *, svint64_t, svint64_t);
9642		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_u64)))
9643		void svstnt1h_scatter_s64index_u64(svbool_t, uint16_t *, svint64_t, svuint64_t);
9644		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_s64)))
9645		void svstnt1h_scatter_u64index_s64(svbool_t, int16_t *, svuint64_t, svint64_t);
9646		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_u64)))
9647		void svstnt1h_scatter_u64index_u64(svbool_t, uint16_t *, svuint64_t, svuint64_t);
9648		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_s32)))
9649		void svstnt1h_scatter_u32offset_s32(svbool_t, int16_t *, svuint32_t, svint32_t);
9650		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_u32)))
9651		void svstnt1h_scatter_u32offset_u32(svbool_t, uint16_t *, svuint32_t, svuint32_t);
9652		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_s64)))
9653		void svstnt1h_scatter_s64offset_s64(svbool_t, int16_t *, svint64_t, svint64_t);
9654		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_u64)))
9655		void svstnt1h_scatter_s64offset_u64(svbool_t, uint16_t *, svint64_t, svuint64_t);
9656		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_s64)))
9657		void svstnt1h_scatter_u64offset_s64(svbool_t, int16_t *, svuint64_t, svint64_t);
9658		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_u64)))
9659		void svstnt1h_scatter_u64offset_u64(svbool_t, uint16_t *, svuint64_t, svuint64_t);
9660		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_u64)))
9661		void svstnt1w_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9662		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_s64)))
9663		void svstnt1w_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9664		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_u64)))
9665		void svstnt1w_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
9666		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_s64)))
9667		void svstnt1w_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
9668		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_u64)))
9669		void svstnt1w_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
9670		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_s64)))
9671		void svstnt1w_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
9672		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_s64)))
9673		void svstnt1w_scatter_s64index_s64(svbool_t, int32_t *, svint64_t, svint64_t);
9674		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_u64)))
9675		void svstnt1w_scatter_s64index_u64(svbool_t, uint32_t *, svint64_t, svuint64_t);
9676		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_s64)))
9677		void svstnt1w_scatter_u64index_s64(svbool_t, int32_t *, svuint64_t, svint64_t);
9678		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_u64)))
9679		void svstnt1w_scatter_u64index_u64(svbool_t, uint32_t *, svuint64_t, svuint64_t);
9680		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_s64)))
9681		void svstnt1w_scatter_s64offset_s64(svbool_t, int32_t *, svint64_t, svint64_t);
9682		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_u64)))
9683		void svstnt1w_scatter_s64offset_u64(svbool_t, uint32_t *, svint64_t, svuint64_t);
9684		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_s64)))
9685		void svstnt1w_scatter_u64offset_s64(svbool_t, int32_t *, svuint64_t, svint64_t);
9686		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
9687		void svstnt1w_scatter_u64offset_u64(svbool_t, uint32_t *, svuint64_t, svuint64_t);
9688		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u32_z)))
9689		svuint32_t svhistcnt_z(svbool_t, svuint32_t, svuint32_t);
9690		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u64_z)))
9691		svuint64_t svhistcnt_z(svbool_t, svuint64_t, svuint64_t);
9692		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s32_z)))
9693		svuint32_t svhistcnt_z(svbool_t, svint32_t, svint32_t);
9694		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s64_z)))
9695		svuint64_t svhistcnt_z(svbool_t, svint64_t, svint64_t);
9696		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_u8)))
9697		svuint8_t svhistseg(svuint8_t, svuint8_t);
9698		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_s8)))
9699		svuint8_t svhistseg(svint8_t, svint8_t);
9700		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_u32)))
9701		svuint32_t svldnt1_gather_index_u32(svbool_t, svuint32_t, int64_t);
9702		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_u64)))
9703		svuint64_t svldnt1_gather_index_u64(svbool_t, svuint64_t, int64_t);
9704		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_f64)))
9705		svfloat64_t svldnt1_gather_index_f64(svbool_t, svuint64_t, int64_t);
9706		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_f32)))
9707		svfloat32_t svldnt1_gather_index_f32(svbool_t, svuint32_t, int64_t);
9708		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_s32)))
9709		svint32_t svldnt1_gather_index_s32(svbool_t, svuint32_t, int64_t);
9710		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_s64)))
9711		svint64_t svldnt1_gather_index_s64(svbool_t, svuint64_t, int64_t);
9712		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_u32)))
9713		svuint32_t svldnt1_gather_offset_u32(svbool_t, svuint32_t, int64_t);
9714		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_u64)))
9715		svuint64_t svldnt1_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9716		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_f64)))
9717		svfloat64_t svldnt1_gather_offset_f64(svbool_t, svuint64_t, int64_t);
9718		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_f32)))
9719		svfloat32_t svldnt1_gather_offset_f32(svbool_t, svuint32_t, int64_t);
9720		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_s32)))
9721		svint32_t svldnt1_gather_offset_s32(svbool_t, svuint32_t, int64_t);
9722		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_s64)))
9723		svint64_t svldnt1_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9724		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_u32)))
9725		svuint32_t svldnt1_gather_u32(svbool_t, svuint32_t);
9726		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_u64)))
9727		svuint64_t svldnt1_gather_u64(svbool_t, svuint64_t);
9728		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_f64)))
9729		svfloat64_t svldnt1_gather_f64(svbool_t, svuint64_t);
9730		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_f32)))
9731		svfloat32_t svldnt1_gather_f32(svbool_t, svuint32_t);
9732		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_s32)))
9733		svint32_t svldnt1_gather_s32(svbool_t, svuint32_t);
9734		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_s64)))
9735		svint64_t svldnt1_gather_s64(svbool_t, svuint64_t);
9736		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_u64)))
9737		svuint64_t svldnt1_gather_index(svbool_t, uint64_t const *, svint64_t);
9738		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_f64)))
9739		svfloat64_t svldnt1_gather_index(svbool_t, float64_t const *, svint64_t);
9740		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_s64)))
9741		svint64_t svldnt1_gather_index(svbool_t, int64_t const *, svint64_t);
9742		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_u64)))
9743		svuint64_t svldnt1_gather_index(svbool_t, uint64_t const *, svuint64_t);
9744		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_f64)))
9745		svfloat64_t svldnt1_gather_index(svbool_t, float64_t const *, svuint64_t);
9746		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_s64)))
9747		svint64_t svldnt1_gather_index(svbool_t, int64_t const *, svuint64_t);
9748		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_u32)))
9749		svuint32_t svldnt1_gather_offset(svbool_t, uint32_t const *, svuint32_t);
9750		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_f32)))
9751		svfloat32_t svldnt1_gather_offset(svbool_t, float32_t const *, svuint32_t);
9752		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_s32)))
9753		svint32_t svldnt1_gather_offset(svbool_t, int32_t const *, svuint32_t);
9754		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_u64)))
9755		svuint64_t svldnt1_gather_offset(svbool_t, uint64_t const *, svint64_t);
9756		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_f64)))
9757		svfloat64_t svldnt1_gather_offset(svbool_t, float64_t const *, svint64_t);
9758		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_s64)))
9759		svint64_t svldnt1_gather_offset(svbool_t, int64_t const *, svint64_t);
9760		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_u64)))
9761		svuint64_t svldnt1_gather_offset(svbool_t, uint64_t const *, svuint64_t);
9762		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_f64)))
9763		svfloat64_t svldnt1_gather_offset(svbool_t, float64_t const *, svuint64_t);
9764		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_s64)))
9765		svint64_t svldnt1_gather_offset(svbool_t, int64_t const *, svuint64_t);
9766		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_u32)))
9767		svuint32_t svldnt1sb_gather_offset_u32(svbool_t, svuint32_t, int64_t);
9768		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_u64)))
9769		svuint64_t svldnt1sb_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9770		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_s32)))
9771		svint32_t svldnt1sb_gather_offset_s32(svbool_t, svuint32_t, int64_t);
9772		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_s64)))
9773		svint64_t svldnt1sb_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9774		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_u32)))
9775		svuint32_t svldnt1sb_gather_u32(svbool_t, svuint32_t);
9776		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_u64)))
9777		svuint64_t svldnt1sb_gather_u64(svbool_t, svuint64_t);
9778		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_s32)))
9779		svint32_t svldnt1sb_gather_s32(svbool_t, svuint32_t);
9780		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_s64)))
9781		svint64_t svldnt1sb_gather_s64(svbool_t, svuint64_t);
9782		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_u32)))
9783		svuint32_t svldnt1sb_gather_offset_u32(svbool_t, int8_t const *, svuint32_t);
9784		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_s32)))
9785		svint32_t svldnt1sb_gather_offset_s32(svbool_t, int8_t const *, svuint32_t);
9786		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_u64)))
9787		svuint64_t svldnt1sb_gather_offset_u64(svbool_t, int8_t const *, svint64_t);
9788		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_s64)))
9789		svint64_t svldnt1sb_gather_offset_s64(svbool_t, int8_t const *, svint64_t);
9790		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_u64)))
9791		svuint64_t svldnt1sb_gather_offset_u64(svbool_t, int8_t const *, svuint64_t);
9792		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_s64)))
9793		svint64_t svldnt1sb_gather_offset_s64(svbool_t, int8_t const *, svuint64_t);
9794		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_u32)))
9795		svuint32_t svldnt1sh_gather_index_u32(svbool_t, svuint32_t, int64_t);
9796		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_u64)))
9797		svuint64_t svldnt1sh_gather_index_u64(svbool_t, svuint64_t, int64_t);
9798		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_s32)))
9799		svint32_t svldnt1sh_gather_index_s32(svbool_t, svuint32_t, int64_t);
9800		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_s64)))
9801		svint64_t svldnt1sh_gather_index_s64(svbool_t, svuint64_t, int64_t);
9802		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_u32)))
9803		svuint32_t svldnt1sh_gather_offset_u32(svbool_t, svuint32_t, int64_t);
9804		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_u64)))
9805		svuint64_t svldnt1sh_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9806		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_s32)))
9807		svint32_t svldnt1sh_gather_offset_s32(svbool_t, svuint32_t, int64_t);
9808		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_s64)))
9809		svint64_t svldnt1sh_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9810		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_u32)))
9811		svuint32_t svldnt1sh_gather_u32(svbool_t, svuint32_t);
9812		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_u64)))
9813		svuint64_t svldnt1sh_gather_u64(svbool_t, svuint64_t);
9814		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_s32)))
9815		svint32_t svldnt1sh_gather_s32(svbool_t, svuint32_t);
9816		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_s64)))
9817		svint64_t svldnt1sh_gather_s64(svbool_t, svuint64_t);
9818		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_u64)))
9819		svuint64_t svldnt1sh_gather_index_u64(svbool_t, int16_t const *, svint64_t);
9820		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_s64)))
9821		svint64_t svldnt1sh_gather_index_s64(svbool_t, int16_t const *, svint64_t);
9822		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_u64)))
9823		svuint64_t svldnt1sh_gather_index_u64(svbool_t, int16_t const *, svuint64_t);
9824		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_s64)))
9825		svint64_t svldnt1sh_gather_index_s64(svbool_t, int16_t const *, svuint64_t);
9826		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_u32)))
9827		svuint32_t svldnt1sh_gather_offset_u32(svbool_t, int16_t const *, svuint32_t);
9828		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_s32)))
9829		svint32_t svldnt1sh_gather_offset_s32(svbool_t, int16_t const *, svuint32_t);
9830		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_u64)))
9831		svuint64_t svldnt1sh_gather_offset_u64(svbool_t, int16_t const *, svint64_t);
9832		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_s64)))
9833		svint64_t svldnt1sh_gather_offset_s64(svbool_t, int16_t const *, svint64_t);
9834		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_u64)))
9835		svuint64_t svldnt1sh_gather_offset_u64(svbool_t, int16_t const *, svuint64_t);
9836		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_s64)))
9837		svint64_t svldnt1sh_gather_offset_s64(svbool_t, int16_t const *, svuint64_t);
9838		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_u64)))
9839		svuint64_t svldnt1sw_gather_index_u64(svbool_t, svuint64_t, int64_t);
9840		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_s64)))
9841		svint64_t svldnt1sw_gather_index_s64(svbool_t, svuint64_t, int64_t);
9842		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_u64)))
9843		svuint64_t svldnt1sw_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9844		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_s64)))
9845		svint64_t svldnt1sw_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9846		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_u64)))
9847		svuint64_t svldnt1sw_gather_u64(svbool_t, svuint64_t);
9848		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_s64)))
9849		svint64_t svldnt1sw_gather_s64(svbool_t, svuint64_t);
9850		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_u64)))
9851		svuint64_t svldnt1sw_gather_index_u64(svbool_t, int32_t const *, svint64_t);
9852		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_s64)))
9853		svint64_t svldnt1sw_gather_index_s64(svbool_t, int32_t const *, svint64_t);
9854		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_u64)))
9855		svuint64_t svldnt1sw_gather_index_u64(svbool_t, int32_t const *, svuint64_t);
9856		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_s64)))
9857		svint64_t svldnt1sw_gather_index_s64(svbool_t, int32_t const *, svuint64_t);
9858		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_u64)))
9859		svuint64_t svldnt1sw_gather_offset_u64(svbool_t, int32_t const *, svint64_t);
9860		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_s64)))
9861		svint64_t svldnt1sw_gather_offset_s64(svbool_t, int32_t const *, svint64_t);
9862		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_u64)))
9863		svuint64_t svldnt1sw_gather_offset_u64(svbool_t, int32_t const *, svuint64_t);
9864		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_s64)))
9865		svint64_t svldnt1sw_gather_offset_s64(svbool_t, int32_t const *, svuint64_t);
9866		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_u32)))
9867		svuint32_t svldnt1ub_gather_offset_u32(svbool_t, svuint32_t, int64_t);
9868		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_u64)))
9869		svuint64_t svldnt1ub_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9870		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_s32)))
9871		svint32_t svldnt1ub_gather_offset_s32(svbool_t, svuint32_t, int64_t);
9872		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_s64)))
9873		svint64_t svldnt1ub_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9874		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_u32)))
9875		svuint32_t svldnt1ub_gather_u32(svbool_t, svuint32_t);
9876		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_u64)))
9877		svuint64_t svldnt1ub_gather_u64(svbool_t, svuint64_t);
9878		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_s32)))
9879		svint32_t svldnt1ub_gather_s32(svbool_t, svuint32_t);
9880		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_s64)))
9881		svint64_t svldnt1ub_gather_s64(svbool_t, svuint64_t);
9882		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_u32)))
9883		svuint32_t svldnt1ub_gather_offset_u32(svbool_t, uint8_t const *, svuint32_t);
9884		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_s32)))
9885		svint32_t svldnt1ub_gather_offset_s32(svbool_t, uint8_t const *, svuint32_t);
9886		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_u64)))
9887		svuint64_t svldnt1ub_gather_offset_u64(svbool_t, uint8_t const *, svint64_t);
9888		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_s64)))
9889		svint64_t svldnt1ub_gather_offset_s64(svbool_t, uint8_t const *, svint64_t);
9890		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_u64)))
9891		svuint64_t svldnt1ub_gather_offset_u64(svbool_t, uint8_t const *, svuint64_t);
9892		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_s64)))
9893		svint64_t svldnt1ub_gather_offset_s64(svbool_t, uint8_t const *, svuint64_t);
9894		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_u32)))
9895		svuint32_t svldnt1uh_gather_index_u32(svbool_t, svuint32_t, int64_t);
9896		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_u64)))
9897		svuint64_t svldnt1uh_gather_index_u64(svbool_t, svuint64_t, int64_t);
9898		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_s32)))
9899		svint32_t svldnt1uh_gather_index_s32(svbool_t, svuint32_t, int64_t);
9900		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_s64)))
9901		svint64_t svldnt1uh_gather_index_s64(svbool_t, svuint64_t, int64_t);
9902		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_u32)))
9903		svuint32_t svldnt1uh_gather_offset_u32(svbool_t, svuint32_t, int64_t);
9904		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_u64)))
9905		svuint64_t svldnt1uh_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9906		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_s32)))
9907		svint32_t svldnt1uh_gather_offset_s32(svbool_t, svuint32_t, int64_t);
9908		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_s64)))
9909		svint64_t svldnt1uh_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9910		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_u32)))
9911		svuint32_t svldnt1uh_gather_u32(svbool_t, svuint32_t);
9912		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_u64)))
9913		svuint64_t svldnt1uh_gather_u64(svbool_t, svuint64_t);
9914		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_s32)))
9915		svint32_t svldnt1uh_gather_s32(svbool_t, svuint32_t);
9916		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_s64)))
9917		svint64_t svldnt1uh_gather_s64(svbool_t, svuint64_t);
9918		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_u64)))
9919		svuint64_t svldnt1uh_gather_index_u64(svbool_t, uint16_t const *, svint64_t);
9920		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_s64)))
9921		svint64_t svldnt1uh_gather_index_s64(svbool_t, uint16_t const *, svint64_t);
9922		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_u64)))
9923		svuint64_t svldnt1uh_gather_index_u64(svbool_t, uint16_t const *, svuint64_t);
9924		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_s64)))
9925		svint64_t svldnt1uh_gather_index_s64(svbool_t, uint16_t const *, svuint64_t);
9926		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_u32)))
9927		svuint32_t svldnt1uh_gather_offset_u32(svbool_t, uint16_t const *, svuint32_t);
9928		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_s32)))
9929		svint32_t svldnt1uh_gather_offset_s32(svbool_t, uint16_t const *, svuint32_t);
9930		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_u64)))
9931		svuint64_t svldnt1uh_gather_offset_u64(svbool_t, uint16_t const *, svint64_t);
9932		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_s64)))
9933		svint64_t svldnt1uh_gather_offset_s64(svbool_t, uint16_t const *, svint64_t);
9934		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_u64)))
9935		svuint64_t svldnt1uh_gather_offset_u64(svbool_t, uint16_t const *, svuint64_t);
9936		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_s64)))
9937		svint64_t svldnt1uh_gather_offset_s64(svbool_t, uint16_t const *, svuint64_t);
9938		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_u64)))
9939		svuint64_t svldnt1uw_gather_index_u64(svbool_t, svuint64_t, int64_t);
9940		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_s64)))
9941		svint64_t svldnt1uw_gather_index_s64(svbool_t, svuint64_t, int64_t);
9942		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_u64)))
9943		svuint64_t svldnt1uw_gather_offset_u64(svbool_t, svuint64_t, int64_t);
9944		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_s64)))
9945		svint64_t svldnt1uw_gather_offset_s64(svbool_t, svuint64_t, int64_t);
9946		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_u64)))
9947		svuint64_t svldnt1uw_gather_u64(svbool_t, svuint64_t);
9948		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_s64)))
9949		svint64_t svldnt1uw_gather_s64(svbool_t, svuint64_t);
9950		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_u64)))
9951		svuint64_t svldnt1uw_gather_index_u64(svbool_t, uint32_t const *, svint64_t);
9952		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_s64)))
9953		svint64_t svldnt1uw_gather_index_s64(svbool_t, uint32_t const *, svint64_t);
9954		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_u64)))
9955		svuint64_t svldnt1uw_gather_index_u64(svbool_t, uint32_t const *, svuint64_t);
9956		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_s64)))
9957		svint64_t svldnt1uw_gather_index_s64(svbool_t, uint32_t const *, svuint64_t);
9958		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_u64)))
9959		svuint64_t svldnt1uw_gather_offset_u64(svbool_t, uint32_t const *, svint64_t);
9960		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_s64)))
9961		svint64_t svldnt1uw_gather_offset_s64(svbool_t, uint32_t const *, svint64_t);
9962		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_u64)))
9963		svuint64_t svldnt1uw_gather_offset_u64(svbool_t, uint32_t const *, svuint64_t);
9964		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_s64)))
9965		svint64_t svldnt1uw_gather_offset_s64(svbool_t, uint32_t const *, svuint64_t);
9966		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u8)))
9967		svbool_t svmatch(svbool_t, svuint8_t, svuint8_t);
9968		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u16)))
9969		svbool_t svmatch(svbool_t, svuint16_t, svuint16_t);
9970		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s8)))
9971		svbool_t svmatch(svbool_t, svint8_t, svint8_t);
9972		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s16)))
9973		svbool_t svmatch(svbool_t, svint16_t, svint16_t);
9974		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u8)))
9975		svbool_t svnmatch(svbool_t, svuint8_t, svuint8_t);
9976		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u16)))
9977		svbool_t svnmatch(svbool_t, svuint16_t, svuint16_t);
9978		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s8)))
9979		svbool_t svnmatch(svbool_t, svint8_t, svint8_t);
9980		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s16)))
9981		svbool_t svnmatch(svbool_t, svint16_t, svint16_t);
9982		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_u32)))
9983		void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svuint32_t);
9984		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_u64)))
9985		void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
9986		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_f64)))
9987		void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svfloat64_t);
9988		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_f32)))
9989		void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svfloat32_t);
9990		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_s32)))
9991		void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svint32_t);
9992		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_s64)))
9993		void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
9994		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_u32)))
9995		void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
9996		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_u64)))
9997		void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
9998		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_f64)))
9999		void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat64_t);
10000		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_f32)))
10001		void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svfloat32_t);
10002		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_s32)))
10003		void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
10004		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_s64)))
10005		void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
10006		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_u32)))
10007		void svstnt1_scatter(svbool_t, svuint32_t, svuint32_t);
10008		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_u64)))
10009		void svstnt1_scatter(svbool_t, svuint64_t, svuint64_t);
10010		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_f64)))
10011		void svstnt1_scatter(svbool_t, svuint64_t, svfloat64_t);
10012		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_f32)))
10013		void svstnt1_scatter(svbool_t, svuint32_t, svfloat32_t);
10014		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_s32)))
10015		void svstnt1_scatter(svbool_t, svuint32_t, svint32_t);
10016		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_s64)))
10017		void svstnt1_scatter(svbool_t, svuint64_t, svint64_t);
10018		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_u64)))
10019		void svstnt1_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
10020		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_f64)))
10021		void svstnt1_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
10022		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_s64)))
10023		void svstnt1_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
10024		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_u64)))
10025		void svstnt1_scatter_index(svbool_t, uint64_t *, svuint64_t, svuint64_t);
10026		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_f64)))
10027		void svstnt1_scatter_index(svbool_t, float64_t *, svuint64_t, svfloat64_t);
10028		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_s64)))
10029		void svstnt1_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
10030		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_u32)))
10031		void svstnt1_scatter_offset(svbool_t, uint32_t *, svuint32_t, svuint32_t);
10032		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_f32)))
10033		void svstnt1_scatter_offset(svbool_t, float32_t *, svuint32_t, svfloat32_t);
10034		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_s32)))
10035		void svstnt1_scatter_offset(svbool_t, int32_t *, svuint32_t, svint32_t);
10036		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_u64)))
10037		void svstnt1_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
10038		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_f64)))
10039		void svstnt1_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
10040		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_s64)))
10041		void svstnt1_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
10042		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_u64)))
10043		void svstnt1_scatter_offset(svbool_t, uint64_t *, svuint64_t, svuint64_t);
10044		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_f64)))
10045		void svstnt1_scatter_offset(svbool_t, float64_t *, svuint64_t, svfloat64_t);
10046		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_s64)))
10047		void svstnt1_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
10048		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_u32)))
10049		void svstnt1b_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
10050		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_u64)))
10051		void svstnt1b_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
10052		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_s32)))
10053		void svstnt1b_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
10054		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_s64)))
10055		void svstnt1b_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
10056		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_u32)))
10057		void svstnt1b_scatter(svbool_t, svuint32_t, svuint32_t);
10058		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_u64)))
10059		void svstnt1b_scatter(svbool_t, svuint64_t, svuint64_t);
10060		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_s32)))
10061		void svstnt1b_scatter(svbool_t, svuint32_t, svint32_t);
10062		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_s64)))
10063		void svstnt1b_scatter(svbool_t, svuint64_t, svint64_t);
10064		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_s32)))
10065		void svstnt1b_scatter_offset(svbool_t, int8_t *, svuint32_t, svint32_t);
10066		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_u32)))
10067		void svstnt1b_scatter_offset(svbool_t, uint8_t *, svuint32_t, svuint32_t);
10068		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_s64)))
10069		void svstnt1b_scatter_offset(svbool_t, int8_t *, svint64_t, svint64_t);
10070		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_u64)))
10071		void svstnt1b_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint64_t);
10072		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_s64)))
10073		void svstnt1b_scatter_offset(svbool_t, int8_t *, svuint64_t, svint64_t);
10074		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_u64)))
10075		void svstnt1b_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint64_t);
10076		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_u32)))
10077		void svstnt1h_scatter_index(svbool_t, svuint32_t, int64_t, svuint32_t);
10078		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_u64)))
10079		void svstnt1h_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
10080		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_s32)))
10081		void svstnt1h_scatter_index(svbool_t, svuint32_t, int64_t, svint32_t);
10082		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_s64)))
10083		void svstnt1h_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
10084		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_u32)))
10085		void svstnt1h_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
10086		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_u64)))
10087		void svstnt1h_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
10088		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_s32)))
10089		void svstnt1h_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
10090		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_s64)))
10091		void svstnt1h_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
10092		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_u32)))
10093		void svstnt1h_scatter(svbool_t, svuint32_t, svuint32_t);
10094		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_u64)))
10095		void svstnt1h_scatter(svbool_t, svuint64_t, svuint64_t);
10096		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_s32)))
10097		void svstnt1h_scatter(svbool_t, svuint32_t, svint32_t);
10098		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_s64)))
10099		void svstnt1h_scatter(svbool_t, svuint64_t, svint64_t);
10100		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_s64)))
10101		void svstnt1h_scatter_index(svbool_t, int16_t *, svint64_t, svint64_t);
10102		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_u64)))
10103		void svstnt1h_scatter_index(svbool_t, uint16_t *, svint64_t, svuint64_t);
10104		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_s64)))
10105		void svstnt1h_scatter_index(svbool_t, int16_t *, svuint64_t, svint64_t);
10106		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_u64)))
10107		void svstnt1h_scatter_index(svbool_t, uint16_t *, svuint64_t, svuint64_t);
10108		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_s32)))
10109		void svstnt1h_scatter_offset(svbool_t, int16_t *, svuint32_t, svint32_t);
10110		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_u32)))
10111		void svstnt1h_scatter_offset(svbool_t, uint16_t *, svuint32_t, svuint32_t);
10112		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_s64)))
10113		void svstnt1h_scatter_offset(svbool_t, int16_t *, svint64_t, svint64_t);
10114		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_u64)))
10115		void svstnt1h_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint64_t);
10116		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_s64)))
10117		void svstnt1h_scatter_offset(svbool_t, int16_t *, svuint64_t, svint64_t);
10118		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_u64)))
10119		void svstnt1h_scatter_offset(svbool_t, uint16_t *, svuint64_t, svuint64_t);
10120		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_u64)))
10121		void svstnt1w_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
10122		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_s64)))
10123		void svstnt1w_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
10124		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_u64)))
10125		void svstnt1w_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
10126		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_s64)))
10127		void svstnt1w_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
10128		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_u64)))
10129		void svstnt1w_scatter(svbool_t, svuint64_t, svuint64_t);
10130		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_s64)))
10131		void svstnt1w_scatter(svbool_t, svuint64_t, svint64_t);
10132		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_s64)))
10133		void svstnt1w_scatter_index(svbool_t, int32_t *, svint64_t, svint64_t);
10134		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_u64)))
10135		void svstnt1w_scatter_index(svbool_t, uint32_t *, svint64_t, svuint64_t);
10136		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_s64)))
10137		void svstnt1w_scatter_index(svbool_t, int32_t *, svuint64_t, svint64_t);
10138		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_u64)))
10139		void svstnt1w_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint64_t);
10140		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_s64)))
10141		void svstnt1w_scatter_offset(svbool_t, int32_t *, svint64_t, svint64_t);
10142		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_u64)))
10143		void svstnt1w_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint64_t);
10144		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_s64)))
10145		void svstnt1w_scatter_offset(svbool_t, int32_t *, svuint64_t, svint64_t);
10146		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
10147		void svstnt1w_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint64_t);
10148		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
10149		svfloat64_t svamax_n_f64_m(svbool_t, svfloat64_t, float64_t);
10150		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
10151		svfloat32_t svamax_n_f32_m(svbool_t, svfloat32_t, float32_t);
10152		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
10153		svfloat16_t svamax_n_f16_m(svbool_t, svfloat16_t, float16_t);
10154		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
10155		svfloat64_t svamax_n_f64_x(svbool_t, svfloat64_t, float64_t);
10156		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
10157		svfloat32_t svamax_n_f32_x(svbool_t, svfloat32_t, float32_t);
10158		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
10159		svfloat16_t svamax_n_f16_x(svbool_t, svfloat16_t, float16_t);
10160		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
10161		svfloat64_t svamax_n_f64_z(svbool_t, svfloat64_t, float64_t);
10162		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
10163		svfloat32_t svamax_n_f32_z(svbool_t, svfloat32_t, float32_t);
10164		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
10165		svfloat16_t svamax_n_f16_z(svbool_t, svfloat16_t, float16_t);
10166		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
10167		svfloat64_t svamax_f64_m(svbool_t, svfloat64_t, svfloat64_t);
10168		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
10169		svfloat32_t svamax_f32_m(svbool_t, svfloat32_t, svfloat32_t);
10170		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
10171		svfloat16_t svamax_f16_m(svbool_t, svfloat16_t, svfloat16_t);
10172		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
10173		svfloat64_t svamax_f64_x(svbool_t, svfloat64_t, svfloat64_t);
10174		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
10175		svfloat32_t svamax_f32_x(svbool_t, svfloat32_t, svfloat32_t);
10176		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
10177		svfloat16_t svamax_f16_x(svbool_t, svfloat16_t, svfloat16_t);
10178		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
10179		svfloat64_t svamax_f64_z(svbool_t, svfloat64_t, svfloat64_t);
10180		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
10181		svfloat32_t svamax_f32_z(svbool_t, svfloat32_t, svfloat32_t);
10182		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
10183		svfloat16_t svamax_f16_z(svbool_t, svfloat16_t, svfloat16_t);
10184		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
10185		svfloat64_t svamin_n_f64_m(svbool_t, svfloat64_t, float64_t);
10186		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
10187		svfloat32_t svamin_n_f32_m(svbool_t, svfloat32_t, float32_t);
10188		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
10189		svfloat16_t svamin_n_f16_m(svbool_t, svfloat16_t, float16_t);
10190		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
10191		svfloat64_t svamin_n_f64_x(svbool_t, svfloat64_t, float64_t);
10192		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
10193		svfloat32_t svamin_n_f32_x(svbool_t, svfloat32_t, float32_t);
10194		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
10195		svfloat16_t svamin_n_f16_x(svbool_t, svfloat16_t, float16_t);
10196		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
10197		svfloat64_t svamin_n_f64_z(svbool_t, svfloat64_t, float64_t);
10198		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
10199		svfloat32_t svamin_n_f32_z(svbool_t, svfloat32_t, float32_t);
10200		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
10201		svfloat16_t svamin_n_f16_z(svbool_t, svfloat16_t, float16_t);
10202		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
10203		svfloat64_t svamin_f64_m(svbool_t, svfloat64_t, svfloat64_t);
10204		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
10205		svfloat32_t svamin_f32_m(svbool_t, svfloat32_t, svfloat32_t);
10206		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
10207		svfloat16_t svamin_f16_m(svbool_t, svfloat16_t, svfloat16_t);
10208		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
10209		svfloat64_t svamin_f64_x(svbool_t, svfloat64_t, svfloat64_t);
10210		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
10211		svfloat32_t svamin_f32_x(svbool_t, svfloat32_t, svfloat32_t);
10212		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
10213		svfloat16_t svamin_f16_x(svbool_t, svfloat16_t, svfloat16_t);
10214		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
10215		svfloat64_t svamin_f64_z(svbool_t, svfloat64_t, svfloat64_t);
10216		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
10217		svfloat32_t svamin_f32_z(svbool_t, svfloat32_t, svfloat32_t);
10218		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
10219		svfloat16_t svamin_f16_z(svbool_t, svfloat16_t, svfloat16_t);
10220		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
10221		svfloat64_t svamax_m(svbool_t, svfloat64_t, float64_t);
10222		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
10223		svfloat32_t svamax_m(svbool_t, svfloat32_t, float32_t);
10224		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
10225		svfloat16_t svamax_m(svbool_t, svfloat16_t, float16_t);
10226		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
10227		svfloat64_t svamax_x(svbool_t, svfloat64_t, float64_t);
10228		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
10229		svfloat32_t svamax_x(svbool_t, svfloat32_t, float32_t);
10230		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
10231		svfloat16_t svamax_x(svbool_t, svfloat16_t, float16_t);
10232		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
10233		svfloat64_t svamax_z(svbool_t, svfloat64_t, float64_t);
10234		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
10235		svfloat32_t svamax_z(svbool_t, svfloat32_t, float32_t);
10236		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
10237		svfloat16_t svamax_z(svbool_t, svfloat16_t, float16_t);
10238		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
10239		svfloat64_t svamax_m(svbool_t, svfloat64_t, svfloat64_t);
10240		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
10241		svfloat32_t svamax_m(svbool_t, svfloat32_t, svfloat32_t);
10242		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
10243		svfloat16_t svamax_m(svbool_t, svfloat16_t, svfloat16_t);
10244		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
10245		svfloat64_t svamax_x(svbool_t, svfloat64_t, svfloat64_t);
10246		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
10247		svfloat32_t svamax_x(svbool_t, svfloat32_t, svfloat32_t);
10248		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
10249		svfloat16_t svamax_x(svbool_t, svfloat16_t, svfloat16_t);
10250		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
10251		svfloat64_t svamax_z(svbool_t, svfloat64_t, svfloat64_t);
10252		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
10253		svfloat32_t svamax_z(svbool_t, svfloat32_t, svfloat32_t);
10254		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
10255		svfloat16_t svamax_z(svbool_t, svfloat16_t, svfloat16_t);
10256		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
10257		svfloat64_t svamin_m(svbool_t, svfloat64_t, float64_t);
10258		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
10259		svfloat32_t svamin_m(svbool_t, svfloat32_t, float32_t);
10260		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
10261		svfloat16_t svamin_m(svbool_t, svfloat16_t, float16_t);
10262		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
10263		svfloat64_t svamin_x(svbool_t, svfloat64_t, float64_t);
10264		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
10265		svfloat32_t svamin_x(svbool_t, svfloat32_t, float32_t);
10266		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
10267		svfloat16_t svamin_x(svbool_t, svfloat16_t, float16_t);
10268		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
10269		svfloat64_t svamin_z(svbool_t, svfloat64_t, float64_t);
10270		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
10271		svfloat32_t svamin_z(svbool_t, svfloat32_t, float32_t);
10272		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
10273		svfloat16_t svamin_z(svbool_t, svfloat16_t, float16_t);
10274		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
10275		svfloat64_t svamin_m(svbool_t, svfloat64_t, svfloat64_t);
10276		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
10277		svfloat32_t svamin_m(svbool_t, svfloat32_t, svfloat32_t);
10278		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
10279		svfloat16_t svamin_m(svbool_t, svfloat16_t, svfloat16_t);
10280		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
10281		svfloat64_t svamin_x(svbool_t, svfloat64_t, svfloat64_t);
10282		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
10283		svfloat32_t svamin_x(svbool_t, svfloat32_t, svfloat32_t);
10284		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
10285		svfloat16_t svamin_x(svbool_t, svfloat16_t, svfloat16_t);
10286		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
10287		svfloat64_t svamin_z(svbool_t, svfloat64_t, svfloat64_t);
10288		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
10289		svfloat32_t svamin_z(svbool_t, svfloat32_t, svfloat32_t);
10290		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
10291		svfloat16_t svamin_z(svbool_t, svfloat16_t, svfloat16_t);
10292		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
10293		svfloat16_t svdot_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10294		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
10295		svfloat16_t svdot_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10296		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
10297		svfloat16_t svdot_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10298		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
10299		svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10300		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
10301		svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10302		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
10303		svfloat16_t svdot_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10304		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
10305		svfloat32_t svdot_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10306		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
10307		svfloat32_t svdot_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10308		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
10309		svfloat32_t svdot_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10310		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
10311		svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10312		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
10313		svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10314		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
10315		svfloat32_t svdot_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10316		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
10317		svfloat16_t svmlalb_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10318		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
10319		svfloat16_t svmlalb_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10320		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
10321		svfloat16_t svmlalb_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10322		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
10323		svfloat32_t svmlallbb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10324		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
10325		svfloat32_t svmlallbb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10326		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
10327		svfloat32_t svmlallbb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10328		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
10329		svfloat32_t svmlallbt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10330		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
10331		svfloat32_t svmlallbt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10332		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
10333		svfloat32_t svmlallbt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10334		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
10335		svfloat32_t svmlalltb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10336		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
10337		svfloat32_t svmlalltb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10338		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
10339		svfloat32_t svmlalltb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10340		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
10341		svfloat32_t svmlalltt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10342		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
10343		svfloat32_t svmlalltt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10344		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
10345		svfloat32_t svmlalltt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10346		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
10347		svfloat16_t svmlalt_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10348		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
10349		svfloat16_t svmlalt_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10350		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
10351		svfloat16_t svmlalt_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10352		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
10353		svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10354		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
10355		svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10356		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
10357		svfloat16_t svmlalb_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10358		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
10359		svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10360		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
10361		svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10362		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
10363		svfloat32_t svmlallbb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10364		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
10365		svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10366		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
10367		svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10368		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
10369		svfloat32_t svmlallbt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10370		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
10371		svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10372		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
10373		svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10374		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
10375		svfloat32_t svmlalltb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10376		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
10377		svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
10378		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
10379		svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
10380		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
10381		svfloat32_t svmlalltt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10382		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
10383		svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
10384		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
10385		svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
10386		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
10387		svfloat16_t svmlalt_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
10388		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
10389		svbfloat16_t svcvt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
10390		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
10391		svfloat16_t svcvt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
10392		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
10393		svbfloat16_t svcvt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
10394		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
10395		svfloat16_t svcvt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
10396		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
10397		svbfloat16_t svcvtlt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
10398		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
10399		svfloat16_t svcvtlt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
10400		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
10401		svbfloat16_t svcvtlt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
10402		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
10403		svfloat16_t svcvtlt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
10404		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
10405		svmfloat8_t svcvtn_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
10406		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
10407		svmfloat8_t svcvtn_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
10408		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
10409		svmfloat8_t svcvtnb_mf8_f32_x2_fpm(svfloat32x2_t, fpm_t);
10410		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
10411		svmfloat8_t svcvtnt_mf8_f32_x2_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
10412		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
10413		svbfloat16_t svcvt1_bf16_fpm(svmfloat8_t, fpm_t);
10414		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
10415		svfloat16_t svcvt1_f16_fpm(svmfloat8_t, fpm_t);
10416		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
10417		svbfloat16_t svcvt2_bf16_fpm(svmfloat8_t, fpm_t);
10418		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
10419		svfloat16_t svcvt2_f16_fpm(svmfloat8_t, fpm_t);
10420		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
10421		svbfloat16_t svcvtlt1_bf16_fpm(svmfloat8_t, fpm_t);
10422		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
10423		svfloat16_t svcvtlt1_f16_fpm(svmfloat8_t, fpm_t);
10424		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
10425		svbfloat16_t svcvtlt2_bf16_fpm(svmfloat8_t, fpm_t);
10426		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
10427		svfloat16_t svcvtlt2_f16_fpm(svmfloat8_t, fpm_t);
10428		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
10429		svmfloat8_t svcvtn_mf8_fpm(svbfloat16x2_t, fpm_t);
10430		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
10431		svmfloat8_t svcvtn_mf8_fpm(svfloat16x2_t, fpm_t);
10432		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
10433		svmfloat8_t svcvtnb_mf8_fpm(svfloat32x2_t, fpm_t);
10434		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
10435		svmfloat8_t svcvtnt_mf8_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
10436		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
10437		svuint8_t svluti2_lane_u8(svuint8_t, svuint8_t, uint64_t);
10438		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
10439		svint8_t svluti2_lane_s8(svint8_t, svuint8_t, uint64_t);
10440		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
10441		svuint16_t svluti2_lane_u16(svuint16_t, svuint8_t, uint64_t);
10442		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
10443		svbfloat16_t svluti2_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
10444		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
10445		svfloat16_t svluti2_lane_f16(svfloat16_t, svuint8_t, uint64_t);
10446		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
10447		svint16_t svluti2_lane_s16(svint16_t, svuint8_t, uint64_t);
10448		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
10449		svuint8_t svluti4_lane_u8(svuint8_t, svuint8_t, uint64_t);
10450		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
10451		svint8_t svluti4_lane_s8(svint8_t, svuint8_t, uint64_t);
10452		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
10453		svuint16_t svluti4_lane_u16(svuint16_t, svuint8_t, uint64_t);
10454		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
10455		svbfloat16_t svluti4_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
10456		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
10457		svfloat16_t svluti4_lane_f16(svfloat16_t, svuint8_t, uint64_t);
10458		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
10459		svint16_t svluti4_lane_s16(svint16_t, svuint8_t, uint64_t);
10460		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
10461		svuint16_t svluti4_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t);
10462		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
10463		svbfloat16_t svluti4_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t);
10464		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
10465		svfloat16_t svluti4_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t);
10466		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
10467		svint16_t svluti4_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t);
10468		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
10469		svuint8_t svluti2_lane(svuint8_t, svuint8_t, uint64_t);
10470		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
10471		svint8_t svluti2_lane(svint8_t, svuint8_t, uint64_t);
10472		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
10473		svuint16_t svluti2_lane(svuint16_t, svuint8_t, uint64_t);
10474		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
10475		svbfloat16_t svluti2_lane(svbfloat16_t, svuint8_t, uint64_t);
10476		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
10477		svfloat16_t svluti2_lane(svfloat16_t, svuint8_t, uint64_t);
10478		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
10479		svint16_t svluti2_lane(svint16_t, svuint8_t, uint64_t);
10480		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
10481		svuint8_t svluti4_lane(svuint8_t, svuint8_t, uint64_t);
10482		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
10483		svint8_t svluti4_lane(svint8_t, svuint8_t, uint64_t);
10484		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
10485		svuint16_t svluti4_lane(svuint16_t, svuint8_t, uint64_t);
10486		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
10487		svbfloat16_t svluti4_lane(svbfloat16_t, svuint8_t, uint64_t);
10488		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
10489		svfloat16_t svluti4_lane(svfloat16_t, svuint8_t, uint64_t);
10490		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
10491		svint16_t svluti4_lane(svint16_t, svuint8_t, uint64_t);
10492		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
10493		svuint16_t svluti4_lane(svuint16x2_t, svuint8_t, uint64_t);
10494		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
10495		svbfloat16_t svluti4_lane(svbfloat16x2_t, svuint8_t, uint64_t);
10496		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
10497		svfloat16_t svluti4_lane(svfloat16x2_t, svuint8_t, uint64_t);
10498		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
10499		svint16_t svluti4_lane(svint16x2_t, svuint8_t, uint64_t);
10500		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
10501		svuint8_t svaesd_u8(svuint8_t, svuint8_t);
10502		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
10503		svuint8_t svaese_u8(svuint8_t, svuint8_t);
10504		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
10505		svuint8_t svaesimc_u8(svuint8_t);
10506		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
10507		svuint8_t svaesmc_u8(svuint8_t);
10508		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
10509		svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
10510		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
10511		svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
10512		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
10513		svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
10514		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
10515		svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
10516		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
10517		svuint8_t svaesd(svuint8_t, svuint8_t);
10518		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
10519		svuint8_t svaese(svuint8_t, svuint8_t);
10520		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
10521		svuint8_t svaesimc(svuint8_t);
10522		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
10523		svuint8_t svaesmc(svuint8_t);
10524		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
10525		svuint64_t svpmullb_pair(svuint64_t, uint64_t);
10526		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
10527		svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
10528		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
10529		svuint64_t svpmullt_pair(svuint64_t, uint64_t);
10530		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
10531		svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
10532		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
10533		svbfloat16_t svadd_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10534		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
10535		svbfloat16_t svadd_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10536		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_z)))
10537		svbfloat16_t svadd_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10538		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_m)))
10539		svbfloat16_t svadd_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10540		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_x)))
10541		svbfloat16_t svadd_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10542		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_z)))
10543		svbfloat16_t svadd_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10544		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_bf16)))
10545		svbfloat16_t svclamp_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t);
10546		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_m)))
10547		svbfloat16_t svmax_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10548		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_x)))
10549		svbfloat16_t svmax_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10550		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_z)))
10551		svbfloat16_t svmax_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10552		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_m)))
10553		svbfloat16_t svmax_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10554		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_x)))
10555		svbfloat16_t svmax_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10556		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_z)))
10557		svbfloat16_t svmax_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10558		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_m)))
10559		svbfloat16_t svmaxnm_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10560		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_x)))
10561		svbfloat16_t svmaxnm_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10562		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_z)))
10563		svbfloat16_t svmaxnm_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10564		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_m)))
10565		svbfloat16_t svmaxnm_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10566		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_x)))
10567		svbfloat16_t svmaxnm_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10568		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_z)))
10569		svbfloat16_t svmaxnm_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10570		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_m)))
10571		svbfloat16_t svmin_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10572		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_x)))
10573		svbfloat16_t svmin_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10574		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_z)))
10575		svbfloat16_t svmin_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10576		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_m)))
10577		svbfloat16_t svmin_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10578		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_x)))
10579		svbfloat16_t svmin_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10580		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_z)))
10581		svbfloat16_t svmin_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10582		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_m)))
10583		svbfloat16_t svminnm_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10584		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_x)))
10585		svbfloat16_t svminnm_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10586		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_z)))
10587		svbfloat16_t svminnm_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10588		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_m)))
10589		svbfloat16_t svminnm_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10590		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x)))
10591		svbfloat16_t svminnm_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10592		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_z)))
10593		svbfloat16_t svminnm_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10594		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_m)))
10595		svbfloat16_t svmla_n_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10596		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_x)))
10597		svbfloat16_t svmla_n_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10598		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_z)))
10599		svbfloat16_t svmla_n_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10600		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_m)))
10601		svbfloat16_t svmla_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10602		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_x)))
10603		svbfloat16_t svmla_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10604		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_z)))
10605		svbfloat16_t svmla_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10606		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_lane_bf16)))
10607		svbfloat16_t svmla_lane_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
10608		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_m)))
10609		svbfloat16_t svmls_n_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10610		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_x)))
10611		svbfloat16_t svmls_n_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10612		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_z)))
10613		svbfloat16_t svmls_n_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10614		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_m)))
10615		svbfloat16_t svmls_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10616		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_x)))
10617		svbfloat16_t svmls_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10618		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_z)))
10619		svbfloat16_t svmls_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10620		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_lane_bf16)))
10621		svbfloat16_t svmls_lane_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
10622		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_m)))
10623		svbfloat16_t svmul_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10624		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_x)))
10625		svbfloat16_t svmul_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10626		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_z)))
10627		svbfloat16_t svmul_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10628		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_m)))
10629		svbfloat16_t svmul_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10630		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x)))
10631		svbfloat16_t svmul_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10632		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_z)))
10633		svbfloat16_t svmul_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10634		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_lane_bf16)))
10635		svbfloat16_t svmul_lane_bf16(svbfloat16_t, svbfloat16_t, uint64_t);
10636		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_m)))
10637		svbfloat16_t svsub_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
10638		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_x)))
10639		svbfloat16_t svsub_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
10640		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_z)))
10641		svbfloat16_t svsub_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
10642		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_m)))
10643		svbfloat16_t svsub_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
10644		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
10645		svbfloat16_t svsub_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
10646		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
10647		svbfloat16_t svsub_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
10648		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
10649		svbfloat16_t svadd_m(svbool_t, svbfloat16_t, bfloat16_t);
10650		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
10651		svbfloat16_t svadd_x(svbool_t, svbfloat16_t, bfloat16_t);
10652		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_z)))
10653		svbfloat16_t svadd_z(svbool_t, svbfloat16_t, bfloat16_t);
10654		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_m)))
10655		svbfloat16_t svadd_m(svbool_t, svbfloat16_t, svbfloat16_t);
10656		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_x)))
10657		svbfloat16_t svadd_x(svbool_t, svbfloat16_t, svbfloat16_t);
10658		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_z)))
10659		svbfloat16_t svadd_z(svbool_t, svbfloat16_t, svbfloat16_t);
10660		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_bf16)))
10661		svbfloat16_t svclamp(svbfloat16_t, svbfloat16_t, svbfloat16_t);
10662		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_m)))
10663		svbfloat16_t svmax_m(svbool_t, svbfloat16_t, bfloat16_t);
10664		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_x)))
10665		svbfloat16_t svmax_x(svbool_t, svbfloat16_t, bfloat16_t);
10666		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_z)))
10667		svbfloat16_t svmax_z(svbool_t, svbfloat16_t, bfloat16_t);
10668		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_m)))
10669		svbfloat16_t svmax_m(svbool_t, svbfloat16_t, svbfloat16_t);
10670		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_x)))
10671		svbfloat16_t svmax_x(svbool_t, svbfloat16_t, svbfloat16_t);
10672		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_z)))
10673		svbfloat16_t svmax_z(svbool_t, svbfloat16_t, svbfloat16_t);
10674		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_m)))
10675		svbfloat16_t svmaxnm_m(svbool_t, svbfloat16_t, bfloat16_t);
10676		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_x)))
10677		svbfloat16_t svmaxnm_x(svbool_t, svbfloat16_t, bfloat16_t);
10678		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_z)))
10679		svbfloat16_t svmaxnm_z(svbool_t, svbfloat16_t, bfloat16_t);
10680		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_m)))
10681		svbfloat16_t svmaxnm_m(svbool_t, svbfloat16_t, svbfloat16_t);
10682		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_x)))
10683		svbfloat16_t svmaxnm_x(svbool_t, svbfloat16_t, svbfloat16_t);
10684		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_z)))
10685		svbfloat16_t svmaxnm_z(svbool_t, svbfloat16_t, svbfloat16_t);
10686		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_m)))
10687		svbfloat16_t svmin_m(svbool_t, svbfloat16_t, bfloat16_t);
10688		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_x)))
10689		svbfloat16_t svmin_x(svbool_t, svbfloat16_t, bfloat16_t);
10690		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_z)))
10691		svbfloat16_t svmin_z(svbool_t, svbfloat16_t, bfloat16_t);
10692		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_m)))
10693		svbfloat16_t svmin_m(svbool_t, svbfloat16_t, svbfloat16_t);
10694		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_x)))
10695		svbfloat16_t svmin_x(svbool_t, svbfloat16_t, svbfloat16_t);
10696		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_z)))
10697		svbfloat16_t svmin_z(svbool_t, svbfloat16_t, svbfloat16_t);
10698		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_m)))
10699		svbfloat16_t svminnm_m(svbool_t, svbfloat16_t, bfloat16_t);
10700		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_x)))
10701		svbfloat16_t svminnm_x(svbool_t, svbfloat16_t, bfloat16_t);
10702		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_z)))
10703		svbfloat16_t svminnm_z(svbool_t, svbfloat16_t, bfloat16_t);
10704		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_m)))
10705		svbfloat16_t svminnm_m(svbool_t, svbfloat16_t, svbfloat16_t);
10706		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x)))
10707		svbfloat16_t svminnm_x(svbool_t, svbfloat16_t, svbfloat16_t);
10708		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_z)))
10709		svbfloat16_t svminnm_z(svbool_t, svbfloat16_t, svbfloat16_t);
10710		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_m)))
10711		svbfloat16_t svmla_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10712		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_x)))
10713		svbfloat16_t svmla_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10714		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_z)))
10715		svbfloat16_t svmla_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10716		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_m)))
10717		svbfloat16_t svmla_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10718		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_x)))
10719		svbfloat16_t svmla_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10720		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_z)))
10721		svbfloat16_t svmla_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10722		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_lane_bf16)))
10723		svbfloat16_t svmla_lane(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
10724		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_m)))
10725		svbfloat16_t svmls_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10726		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_x)))
10727		svbfloat16_t svmls_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10728		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_z)))
10729		svbfloat16_t svmls_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
10730		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_m)))
10731		svbfloat16_t svmls_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10732		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_x)))
10733		svbfloat16_t svmls_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10734		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_z)))
10735		svbfloat16_t svmls_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
10736		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_lane_bf16)))
10737		svbfloat16_t svmls_lane(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
10738		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_m)))
10739		svbfloat16_t svmul_m(svbool_t, svbfloat16_t, bfloat16_t);
10740		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_x)))
10741		svbfloat16_t svmul_x(svbool_t, svbfloat16_t, bfloat16_t);
10742		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_z)))
10743		svbfloat16_t svmul_z(svbool_t, svbfloat16_t, bfloat16_t);
10744		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_m)))
10745		svbfloat16_t svmul_m(svbool_t, svbfloat16_t, svbfloat16_t);
10746		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x)))
10747		svbfloat16_t svmul_x(svbool_t, svbfloat16_t, svbfloat16_t);
10748		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_z)))
10749		svbfloat16_t svmul_z(svbool_t, svbfloat16_t, svbfloat16_t);
10750		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_lane_bf16)))
10751		svbfloat16_t svmul_lane(svbfloat16_t, svbfloat16_t, uint64_t);
10752		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_m)))
10753		svbfloat16_t svsub_m(svbool_t, svbfloat16_t, bfloat16_t);
10754		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_x)))
10755		svbfloat16_t svsub_x(svbool_t, svbfloat16_t, bfloat16_t);
10756		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_z)))
10757		svbfloat16_t svsub_z(svbool_t, svbfloat16_t, bfloat16_t);
10758		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_m)))
10759		svbfloat16_t svsub_m(svbool_t, svbfloat16_t, svbfloat16_t);
10760		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
10761		svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);
10762		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
10763		svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);
10764		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
10765		svuint8_t svbdep_n_u8(svuint8_t, uint8_t);
10766		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
10767		svuint32_t svbdep_n_u32(svuint32_t, uint32_t);
10768		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u64)))
10769		svuint64_t svbdep_n_u64(svuint64_t, uint64_t);
10770		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u16)))
10771		svuint16_t svbdep_n_u16(svuint16_t, uint16_t);
10772		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u8)))
10773		svuint8_t svbdep_u8(svuint8_t, svuint8_t);
10774		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u32)))
10775		svuint32_t svbdep_u32(svuint32_t, svuint32_t);
10776		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u64)))
10777		svuint64_t svbdep_u64(svuint64_t, svuint64_t);
10778		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u16)))
10779		svuint16_t svbdep_u16(svuint16_t, svuint16_t);
10780		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u8)))
10781		svuint8_t svbext_n_u8(svuint8_t, uint8_t);
10782		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u32)))
10783		svuint32_t svbext_n_u32(svuint32_t, uint32_t);
10784		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u64)))
10785		svuint64_t svbext_n_u64(svuint64_t, uint64_t);
10786		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u16)))
10787		svuint16_t svbext_n_u16(svuint16_t, uint16_t);
10788		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u8)))
10789		svuint8_t svbext_u8(svuint8_t, svuint8_t);
10790		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u32)))
10791		svuint32_t svbext_u32(svuint32_t, svuint32_t);
10792		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u64)))
10793		svuint64_t svbext_u64(svuint64_t, svuint64_t);
10794		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u16)))
10795		svuint16_t svbext_u16(svuint16_t, svuint16_t);
10796		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u8)))
10797		svuint8_t svbgrp_n_u8(svuint8_t, uint8_t);
10798		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u32)))
10799		svuint32_t svbgrp_n_u32(svuint32_t, uint32_t);
10800		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u64)))
10801		svuint64_t svbgrp_n_u64(svuint64_t, uint64_t);
10802		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u16)))
10803		svuint16_t svbgrp_n_u16(svuint16_t, uint16_t);
10804		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u8)))
10805		svuint8_t svbgrp_u8(svuint8_t, svuint8_t);
10806		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u32)))
10807		svuint32_t svbgrp_u32(svuint32_t, svuint32_t);
10808		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u64)))
10809		svuint64_t svbgrp_u64(svuint64_t, svuint64_t);
10810		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u16)))
10811		svuint16_t svbgrp_u16(svuint16_t, svuint16_t);
10812		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
10813		svuint8_t svbdep(svuint8_t, uint8_t);
10814		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
10815		svuint32_t svbdep(svuint32_t, uint32_t);
10816		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u64)))
10817		svuint64_t svbdep(svuint64_t, uint64_t);
10818		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u16)))
10819		svuint16_t svbdep(svuint16_t, uint16_t);
10820		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u8)))
10821		svuint8_t svbdep(svuint8_t, svuint8_t);
10822		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u32)))
10823		svuint32_t svbdep(svuint32_t, svuint32_t);
10824		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u64)))
10825		svuint64_t svbdep(svuint64_t, svuint64_t);
10826		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u16)))
10827		svuint16_t svbdep(svuint16_t, svuint16_t);
10828		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u8)))
10829		svuint8_t svbext(svuint8_t, uint8_t);
10830		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u32)))
10831		svuint32_t svbext(svuint32_t, uint32_t);
10832		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u64)))
10833		svuint64_t svbext(svuint64_t, uint64_t);
10834		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u16)))
10835		svuint16_t svbext(svuint16_t, uint16_t);
10836		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u8)))
10837		svuint8_t svbext(svuint8_t, svuint8_t);
10838		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u32)))
10839		svuint32_t svbext(svuint32_t, svuint32_t);
10840		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u64)))
10841		svuint64_t svbext(svuint64_t, svuint64_t);
10842		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u16)))
10843		svuint16_t svbext(svuint16_t, svuint16_t);
10844		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u8)))
10845		svuint8_t svbgrp(svuint8_t, uint8_t);
10846		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u32)))
10847		svuint32_t svbgrp(svuint32_t, uint32_t);
10848		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u64)))
10849		svuint64_t svbgrp(svuint64_t, uint64_t);
10850		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u16)))
10851		svuint16_t svbgrp(svuint16_t, uint16_t);
10852		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u8)))
10853		svuint8_t svbgrp(svuint8_t, svuint8_t);
10854		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u32)))
10855		svuint32_t svbgrp(svuint32_t, svuint32_t);
10856		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u64)))
10857		svuint64_t svbgrp(svuint64_t, svuint64_t);
10858		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u16)))
10859		svuint16_t svbgrp(svuint16_t, svuint16_t);
10860		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
10861		svuint32_t svsm4e_u32(svuint32_t, svuint32_t);
10862		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
10863		svuint32_t svsm4ekey_u32(svuint32_t, svuint32_t);
10864		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
10865		svuint32_t svsm4e(svuint32_t, svuint32_t);
10866		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
10867		svuint32_t svsm4ekey(svuint32_t, svuint32_t);
10868		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
10869		svuint32_t svld1q_gather_u64base_index_u32(svbool_t, svuint64_t, int64_t);
10870		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u64)))
10871		svuint64_t svld1q_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
10872		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u16)))
10873		svuint16_t svld1q_gather_u64base_index_u16(svbool_t, svuint64_t, int64_t);
10874		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_bf16)))
10875		svbfloat16_t svld1q_gather_u64base_index_bf16(svbool_t, svuint64_t, int64_t);
10876		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f64)))
10877		svfloat64_t svld1q_gather_u64base_index_f64(svbool_t, svuint64_t, int64_t);
10878		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f32)))
10879		svfloat32_t svld1q_gather_u64base_index_f32(svbool_t, svuint64_t, int64_t);
10880		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f16)))
10881		svfloat16_t svld1q_gather_u64base_index_f16(svbool_t, svuint64_t, int64_t);
10882		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s32)))
10883		svint32_t svld1q_gather_u64base_index_s32(svbool_t, svuint64_t, int64_t);
10884		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s64)))
10885		svint64_t svld1q_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
10886		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s16)))
10887		svint16_t svld1q_gather_u64base_index_s16(svbool_t, svuint64_t, int64_t);
10888		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u8)))
10889		svuint8_t svld1q_gather_u64base_offset_u8(svbool_t, svuint64_t, int64_t);
10890		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u32)))
10891		svuint32_t svld1q_gather_u64base_offset_u32(svbool_t, svuint64_t, int64_t);
10892		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u64)))
10893		svuint64_t svld1q_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
10894		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u16)))
10895		svuint16_t svld1q_gather_u64base_offset_u16(svbool_t, svuint64_t, int64_t);
10896		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_bf16)))
10897		svbfloat16_t svld1q_gather_u64base_offset_bf16(svbool_t, svuint64_t, int64_t);
10898		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s8)))
10899		svint8_t svld1q_gather_u64base_offset_s8(svbool_t, svuint64_t, int64_t);
10900		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f64)))
10901		svfloat64_t svld1q_gather_u64base_offset_f64(svbool_t, svuint64_t, int64_t);
10902		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f32)))
10903		svfloat32_t svld1q_gather_u64base_offset_f32(svbool_t, svuint64_t, int64_t);
10904		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f16)))
10905		svfloat16_t svld1q_gather_u64base_offset_f16(svbool_t, svuint64_t, int64_t);
10906		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s32)))
10907		svint32_t svld1q_gather_u64base_offset_s32(svbool_t, svuint64_t, int64_t);
10908		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s64)))
10909		svint64_t svld1q_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
10910		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_mf8)))
10911		svmfloat8_t svld1q_gather_u64base_offset_mf8(svbool_t, svuint64_t, int64_t);
10912		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s16)))
10913		svint16_t svld1q_gather_u64base_offset_s16(svbool_t, svuint64_t, int64_t);
10914		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u8)))
10915		svuint8_t svld1q_gather_u64base_u8(svbool_t, svuint64_t);
10916		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u32)))
10917		svuint32_t svld1q_gather_u64base_u32(svbool_t, svuint64_t);
10918		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u64)))
10919		svuint64_t svld1q_gather_u64base_u64(svbool_t, svuint64_t);
10920		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u16)))
10921		svuint16_t svld1q_gather_u64base_u16(svbool_t, svuint64_t);
10922		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_bf16)))
10923		svbfloat16_t svld1q_gather_u64base_bf16(svbool_t, svuint64_t);
10924		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s8)))
10925		svint8_t svld1q_gather_u64base_s8(svbool_t, svuint64_t);
10926		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f64)))
10927		svfloat64_t svld1q_gather_u64base_f64(svbool_t, svuint64_t);
10928		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f32)))
10929		svfloat32_t svld1q_gather_u64base_f32(svbool_t, svuint64_t);
10930		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f16)))
10931		svfloat16_t svld1q_gather_u64base_f16(svbool_t, svuint64_t);
10932		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s32)))
10933		svint32_t svld1q_gather_u64base_s32(svbool_t, svuint64_t);
10934		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s64)))
10935		svint64_t svld1q_gather_u64base_s64(svbool_t, svuint64_t);
10936		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_mf8)))
10937		svmfloat8_t svld1q_gather_u64base_mf8(svbool_t, svuint64_t);
10938		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s16)))
10939		svint16_t svld1q_gather_u64base_s16(svbool_t, svuint64_t);
10940		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u32)))
10941		svuint32_t svld1q_gather_u64index_u32(svbool_t, uint32_t const *, svuint64_t);
10942		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u64)))
10943		svuint64_t svld1q_gather_u64index_u64(svbool_t, uint64_t const *, svuint64_t);
10944		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u16)))
10945		svuint16_t svld1q_gather_u64index_u16(svbool_t, uint16_t const *, svuint64_t);
10946		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_bf16)))
10947		svbfloat16_t svld1q_gather_u64index_bf16(svbool_t, bfloat16_t const *, svuint64_t);
10948		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f64)))
10949		svfloat64_t svld1q_gather_u64index_f64(svbool_t, float64_t const *, svuint64_t);
10950		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f32)))
10951		svfloat32_t svld1q_gather_u64index_f32(svbool_t, float32_t const *, svuint64_t);
10952		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f16)))
10953		svfloat16_t svld1q_gather_u64index_f16(svbool_t, float16_t const *, svuint64_t);
10954		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s32)))
10955		svint32_t svld1q_gather_u64index_s32(svbool_t, int32_t const *, svuint64_t);
10956		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s64)))
10957		svint64_t svld1q_gather_u64index_s64(svbool_t, int64_t const *, svuint64_t);
10958		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s16)))
10959		svint16_t svld1q_gather_u64index_s16(svbool_t, int16_t const *, svuint64_t);
10960		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u8)))
10961		svuint8_t svld1q_gather_u64offset_u8(svbool_t, uint8_t const *, svuint64_t);
10962		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u32)))
10963		svuint32_t svld1q_gather_u64offset_u32(svbool_t, uint32_t const *, svuint64_t);
10964		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u64)))
10965		svuint64_t svld1q_gather_u64offset_u64(svbool_t, uint64_t const *, svuint64_t);
10966		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u16)))
10967		svuint16_t svld1q_gather_u64offset_u16(svbool_t, uint16_t const *, svuint64_t);
10968		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_bf16)))
10969		svbfloat16_t svld1q_gather_u64offset_bf16(svbool_t, bfloat16_t const *, svuint64_t);
10970		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s8)))
10971		svint8_t svld1q_gather_u64offset_s8(svbool_t, int8_t const *, svuint64_t);
10972		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f64)))
10973		svfloat64_t svld1q_gather_u64offset_f64(svbool_t, float64_t const *, svuint64_t);
10974		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f32)))
10975		svfloat32_t svld1q_gather_u64offset_f32(svbool_t, float32_t const *, svuint64_t);
10976		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f16)))
10977		svfloat16_t svld1q_gather_u64offset_f16(svbool_t, float16_t const *, svuint64_t);
10978		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s32)))
10979		svint32_t svld1q_gather_u64offset_s32(svbool_t, int32_t const *, svuint64_t);
10980		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s64)))
10981		svint64_t svld1q_gather_u64offset_s64(svbool_t, int64_t const *, svuint64_t);
10982		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_mf8)))
10983		svmfloat8_t svld1q_gather_u64offset_mf8(svbool_t, mfloat8_t const *, svuint64_t);
10984		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s16)))
10985		svint16_t svld1q_gather_u64offset_s16(svbool_t, int16_t const *, svuint64_t);
10986		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_u64)))
10987		svuint64_t svld1udq_u64(svbool_t, uint64_t const *);
10988		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_f64)))
10989		svfloat64_t svld1udq_f64(svbool_t, float64_t const *);
10990		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_s64)))
10991		svint64_t svld1udq_s64(svbool_t, int64_t const *);
10992		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_u64)))
10993		svuint64_t svld1udq_vnum_u64(svbool_t, uint64_t const *, int64_t);
10994		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_f64)))
10995		svfloat64_t svld1udq_vnum_f64(svbool_t, float64_t const *, int64_t);
10996		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_s64)))
10997		svint64_t svld1udq_vnum_s64(svbool_t, int64_t const *, int64_t);
10998		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_u32)))
10999		svuint32_t svld1uwq_u32(svbool_t, uint32_t const *);
11000		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_f32)))
11001		svfloat32_t svld1uwq_f32(svbool_t, float32_t const *);
11002		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_s32)))
11003		svint32_t svld1uwq_s32(svbool_t, int32_t const *);
11004		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_u32)))
11005		svuint32_t svld1uwq_vnum_u32(svbool_t, uint32_t const *, int64_t);
11006		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_f32)))
11007		svfloat32_t svld1uwq_vnum_f32(svbool_t, float32_t const *, int64_t);
11008		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_s32)))
11009		svint32_t svld1uwq_vnum_s32(svbool_t, int32_t const *, int64_t);
11010		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
11011		void svst1dq_u64(svbool_t, uint64_t *, svuint64_t);
11012		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
11013		void svst1dq_f64(svbool_t, float64_t *, svfloat64_t);
11014		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
11015		void svst1dq_s64(svbool_t, int64_t *, svint64_t);
11016		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
11017		void svst1dq_vnum_u64(svbool_t, uint64_t *, int64_t, svuint64_t);
11018		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
11019		void svst1dq_vnum_f64(svbool_t, float64_t *, int64_t, svfloat64_t);
11020		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
11021		void svst1dq_vnum_s64(svbool_t, int64_t *, int64_t, svint64_t);
11022		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
11023		void svst1q_scatter_u64base_u8(svbool_t, svuint64_t, svuint8_t);
11024		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
11025		void svst1q_scatter_u64base_u32(svbool_t, svuint64_t, svuint32_t);
11026		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u64)))
11027		void svst1q_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
11028		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u16)))
11029		void svst1q_scatter_u64base_u16(svbool_t, svuint64_t, svuint16_t);
11030		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_bf16)))
11031		void svst1q_scatter_u64base_bf16(svbool_t, svuint64_t, svbfloat16_t);
11032		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s8)))
11033		void svst1q_scatter_u64base_s8(svbool_t, svuint64_t, svint8_t);
11034		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f64)))
11035		void svst1q_scatter_u64base_f64(svbool_t, svuint64_t, svfloat64_t);
11036		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f32)))
11037		void svst1q_scatter_u64base_f32(svbool_t, svuint64_t, svfloat32_t);
11038		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f16)))
11039		void svst1q_scatter_u64base_f16(svbool_t, svuint64_t, svfloat16_t);
11040		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s32)))
11041		void svst1q_scatter_u64base_s32(svbool_t, svuint64_t, svint32_t);
11042		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s64)))
11043		void svst1q_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
11044		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_mf8)))
11045		void svst1q_scatter_u64base_mf8(svbool_t, svuint64_t, svmfloat8_t);
11046		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s16)))
11047		void svst1q_scatter_u64base_s16(svbool_t, svuint64_t, svint16_t);
11048		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u32)))
11049		void svst1q_scatter_u64base_index_u32(svbool_t, svuint64_t, int64_t, svuint32_t);
11050		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u64)))
11051		void svst1q_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
11052		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u16)))
11053		void svst1q_scatter_u64base_index_u16(svbool_t, svuint64_t, int64_t, svuint16_t);
11054		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_bf16)))
11055		void svst1q_scatter_u64base_index_bf16(svbool_t, svuint64_t, int64_t, svbfloat16_t);
11056		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f64)))
11057		void svst1q_scatter_u64base_index_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
11058		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f32)))
11059		void svst1q_scatter_u64base_index_f32(svbool_t, svuint64_t, int64_t, svfloat32_t);
11060		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f16)))
11061		void svst1q_scatter_u64base_index_f16(svbool_t, svuint64_t, int64_t, svfloat16_t);
11062		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s32)))
11063		void svst1q_scatter_u64base_index_s32(svbool_t, svuint64_t, int64_t, svint32_t);
11064		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s64)))
11065		void svst1q_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
11066		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s16)))
11067		void svst1q_scatter_u64base_index_s16(svbool_t, svuint64_t, int64_t, svint16_t);
11068		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u8)))
11069		void svst1q_scatter_u64base_offset_u8(svbool_t, svuint64_t, int64_t, svuint8_t);
11070		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u32)))
11071		void svst1q_scatter_u64base_offset_u32(svbool_t, svuint64_t, int64_t, svuint32_t);
11072		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u64)))
11073		void svst1q_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
11074		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u16)))
11075		void svst1q_scatter_u64base_offset_u16(svbool_t, svuint64_t, int64_t, svuint16_t);
11076		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_bf16)))
11077		void svst1q_scatter_u64base_offset_bf16(svbool_t, svuint64_t, int64_t, svbfloat16_t);
11078		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s8)))
11079		void svst1q_scatter_u64base_offset_s8(svbool_t, svuint64_t, int64_t, svint8_t);
11080		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f64)))
11081		void svst1q_scatter_u64base_offset_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
11082		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f32)))
11083		void svst1q_scatter_u64base_offset_f32(svbool_t, svuint64_t, int64_t, svfloat32_t);
11084		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f16)))
11085		void svst1q_scatter_u64base_offset_f16(svbool_t, svuint64_t, int64_t, svfloat16_t);
11086		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s32)))
11087		void svst1q_scatter_u64base_offset_s32(svbool_t, svuint64_t, int64_t, svint32_t);
11088		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s64)))
11089		void svst1q_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
11090		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_mf8)))
11091		void svst1q_scatter_u64base_offset_mf8(svbool_t, svuint64_t, int64_t, svmfloat8_t);
11092		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
11093		void svst1q_scatter_u64base_offset_s16(svbool_t, svuint64_t, int64_t, svint16_t);
11094		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
11095		void svst1q_scatter_s64index_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
11096		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
11097		void svst1q_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
11098		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
11099		void svst1q_scatter_s64index_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
11100		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
11101		void svst1q_scatter_s64index_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
11102		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
11103		void svst1q_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
11104		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
11105		void svst1q_scatter_s64index_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
11106		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
11107		void svst1q_scatter_s64index_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
11108		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
11109		void svst1q_scatter_s64index_s32(svbool_t, int32_t *, svint64_t, svint32_t);
11110		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
11111		void svst1q_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
11112		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
11113		void svst1q_scatter_s64index_s16(svbool_t, int16_t *, svint64_t, svint16_t);
11114		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
11115		void svst1q_scatter_u64index_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
11116		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
11117		void svst1q_scatter_u64index_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
11118		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u16)))
11119		void svst1q_scatter_u64index_u16(svbool_t, uint16_t *, svuint64_t, svuint16_t);
11120		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_bf16)))
11121		void svst1q_scatter_u64index_bf16(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
11122		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f64)))
11123		void svst1q_scatter_u64index_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
11124		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f32)))
11125		void svst1q_scatter_u64index_f32(svbool_t, float32_t *, svuint64_t, svfloat32_t);
11126		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f16)))
11127		void svst1q_scatter_u64index_f16(svbool_t, float16_t *, svuint64_t, svfloat16_t);
11128		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s32)))
11129		void svst1q_scatter_u64index_s32(svbool_t, int32_t *, svuint64_t, svint32_t);
11130		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s64)))
11131		void svst1q_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
11132		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
11133		void svst1q_scatter_u64index_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
11134		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
11135		void svst1q_scatter_s64offset_u8(svbool_t, uint8_t *, svint64_t, svuint8_t);
11136		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
11137		void svst1q_scatter_s64offset_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
11138		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
11139		void svst1q_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
11140		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
11141		void svst1q_scatter_s64offset_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
11142		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
11143		void svst1q_scatter_s64offset_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
11144		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
11145		void svst1q_scatter_s64offset_s8(svbool_t, int8_t *, svint64_t, svint8_t);
11146		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
11147		void svst1q_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
11148		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
11149		void svst1q_scatter_s64offset_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
11150		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
11151		void svst1q_scatter_s64offset_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
11152		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
11153		void svst1q_scatter_s64offset_s32(svbool_t, int32_t *, svint64_t, svint32_t);
11154		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
11155		void svst1q_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
11156		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_mf8)))
11157		void svst1q_scatter_s64offset_mf8(svbool_t, mfloat8_t *, svint64_t, svmfloat8_t);
11158		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
11159		void svst1q_scatter_s64offset_s16(svbool_t, int16_t *, svint64_t, svint16_t);
11160		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
11161		void svst1q_scatter_u64offset_u8(svbool_t, uint8_t *, svuint64_t, svuint8_t);
11162		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
11163		void svst1q_scatter_u64offset_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
11164		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u64)))
11165		void svst1q_scatter_u64offset_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
11166		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u16)))
11167		void svst1q_scatter_u64offset_u16(svbool_t, uint16_t *, svuint64_t, svuint16_t);
11168		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_bf16)))
11169		void svst1q_scatter_u64offset_bf16(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
11170		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s8)))
11171		void svst1q_scatter_u64offset_s8(svbool_t, int8_t *, svuint64_t, svint8_t);
11172		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f64)))
11173		void svst1q_scatter_u64offset_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
11174		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f32)))
11175		void svst1q_scatter_u64offset_f32(svbool_t, float32_t *, svuint64_t, svfloat32_t);
11176		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f16)))
11177		void svst1q_scatter_u64offset_f16(svbool_t, float16_t *, svuint64_t, svfloat16_t);
11178		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s32)))
11179		void svst1q_scatter_u64offset_s32(svbool_t, int32_t *, svuint64_t, svint32_t);
11180		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s64)))
11181		void svst1q_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
11182		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_mf8)))
11183		void svst1q_scatter_u64offset_mf8(svbool_t, mfloat8_t *, svuint64_t, svmfloat8_t);
11184		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
11185		void svst1q_scatter_u64offset_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
11186		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
11187		void svst1wq_u32(svbool_t, uint32_t *, svuint32_t);
11188		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
11189		void svst1wq_f32(svbool_t, float32_t *, svfloat32_t);
11190		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
11191		void svst1wq_s32(svbool_t, int32_t *, svint32_t);
11192		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
11193		void svst1wq_vnum_u32(svbool_t, uint32_t *, int64_t, svuint32_t);
11194		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
11195		void svst1wq_vnum_f32(svbool_t, float32_t *, int64_t, svfloat32_t);
11196		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
11197		void svst1wq_vnum_s32(svbool_t, int32_t *, int64_t, svint32_t);
11198		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
11199		svuint32_t svld1q_gather_index_u32(svbool_t, svuint64_t, int64_t);
11200		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u64)))
11201		svuint64_t svld1q_gather_index_u64(svbool_t, svuint64_t, int64_t);
11202		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u16)))
11203		svuint16_t svld1q_gather_index_u16(svbool_t, svuint64_t, int64_t);
11204		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_bf16)))
11205		svbfloat16_t svld1q_gather_index_bf16(svbool_t, svuint64_t, int64_t);
11206		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f64)))
11207		svfloat64_t svld1q_gather_index_f64(svbool_t, svuint64_t, int64_t);
11208		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f32)))
11209		svfloat32_t svld1q_gather_index_f32(svbool_t, svuint64_t, int64_t);
11210		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f16)))
11211		svfloat16_t svld1q_gather_index_f16(svbool_t, svuint64_t, int64_t);
11212		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s32)))
11213		svint32_t svld1q_gather_index_s32(svbool_t, svuint64_t, int64_t);
11214		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s64)))
11215		svint64_t svld1q_gather_index_s64(svbool_t, svuint64_t, int64_t);
11216		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s16)))
11217		svint16_t svld1q_gather_index_s16(svbool_t, svuint64_t, int64_t);
11218		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u8)))
11219		svuint8_t svld1q_gather_offset_u8(svbool_t, svuint64_t, int64_t);
11220		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u32)))
11221		svuint32_t svld1q_gather_offset_u32(svbool_t, svuint64_t, int64_t);
11222		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u64)))
11223		svuint64_t svld1q_gather_offset_u64(svbool_t, svuint64_t, int64_t);
11224		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u16)))
11225		svuint16_t svld1q_gather_offset_u16(svbool_t, svuint64_t, int64_t);
11226		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_bf16)))
11227		svbfloat16_t svld1q_gather_offset_bf16(svbool_t, svuint64_t, int64_t);
11228		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s8)))
11229		svint8_t svld1q_gather_offset_s8(svbool_t, svuint64_t, int64_t);
11230		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f64)))
11231		svfloat64_t svld1q_gather_offset_f64(svbool_t, svuint64_t, int64_t);
11232		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f32)))
11233		svfloat32_t svld1q_gather_offset_f32(svbool_t, svuint64_t, int64_t);
11234		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f16)))
11235		svfloat16_t svld1q_gather_offset_f16(svbool_t, svuint64_t, int64_t);
11236		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s32)))
11237		svint32_t svld1q_gather_offset_s32(svbool_t, svuint64_t, int64_t);
11238		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s64)))
11239		svint64_t svld1q_gather_offset_s64(svbool_t, svuint64_t, int64_t);
11240		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_mf8)))
11241		svmfloat8_t svld1q_gather_offset_mf8(svbool_t, svuint64_t, int64_t);
11242		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s16)))
11243		svint16_t svld1q_gather_offset_s16(svbool_t, svuint64_t, int64_t);
11244		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u8)))
11245		svuint8_t svld1q_gather_u8(svbool_t, svuint64_t);
11246		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u32)))
11247		svuint32_t svld1q_gather_u32(svbool_t, svuint64_t);
11248		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u64)))
11249		svuint64_t svld1q_gather_u64(svbool_t, svuint64_t);
11250		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u16)))
11251		svuint16_t svld1q_gather_u16(svbool_t, svuint64_t);
11252		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_bf16)))
11253		svbfloat16_t svld1q_gather_bf16(svbool_t, svuint64_t);
11254		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s8)))
11255		svint8_t svld1q_gather_s8(svbool_t, svuint64_t);
11256		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f64)))
11257		svfloat64_t svld1q_gather_f64(svbool_t, svuint64_t);
11258		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f32)))
11259		svfloat32_t svld1q_gather_f32(svbool_t, svuint64_t);
11260		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f16)))
11261		svfloat16_t svld1q_gather_f16(svbool_t, svuint64_t);
11262		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s32)))
11263		svint32_t svld1q_gather_s32(svbool_t, svuint64_t);
11264		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s64)))
11265		svint64_t svld1q_gather_s64(svbool_t, svuint64_t);
11266		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_mf8)))
11267		svmfloat8_t svld1q_gather_mf8(svbool_t, svuint64_t);
11268		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s16)))
11269		svint16_t svld1q_gather_s16(svbool_t, svuint64_t);
11270		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u32)))
11271		svuint32_t svld1q_gather_index(svbool_t, uint32_t const *, svuint64_t);
11272		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u64)))
11273		svuint64_t svld1q_gather_index(svbool_t, uint64_t const *, svuint64_t);
11274		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u16)))
11275		svuint16_t svld1q_gather_index(svbool_t, uint16_t const *, svuint64_t);
11276		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_bf16)))
11277		svbfloat16_t svld1q_gather_index(svbool_t, bfloat16_t const *, svuint64_t);
11278		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f64)))
11279		svfloat64_t svld1q_gather_index(svbool_t, float64_t const *, svuint64_t);
11280		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f32)))
11281		svfloat32_t svld1q_gather_index(svbool_t, float32_t const *, svuint64_t);
11282		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f16)))
11283		svfloat16_t svld1q_gather_index(svbool_t, float16_t const *, svuint64_t);
11284		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s32)))
11285		svint32_t svld1q_gather_index(svbool_t, int32_t const *, svuint64_t);
11286		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s64)))
11287		svint64_t svld1q_gather_index(svbool_t, int64_t const *, svuint64_t);
11288		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s16)))
11289		svint16_t svld1q_gather_index(svbool_t, int16_t const *, svuint64_t);
11290		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u8)))
11291		svuint8_t svld1q_gather_offset(svbool_t, uint8_t const *, svuint64_t);
11292		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u32)))
11293		svuint32_t svld1q_gather_offset(svbool_t, uint32_t const *, svuint64_t);
11294		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u64)))
11295		svuint64_t svld1q_gather_offset(svbool_t, uint64_t const *, svuint64_t);
11296		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u16)))
11297		svuint16_t svld1q_gather_offset(svbool_t, uint16_t const *, svuint64_t);
11298		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_bf16)))
11299		svbfloat16_t svld1q_gather_offset(svbool_t, bfloat16_t const *, svuint64_t);
11300		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s8)))
11301		svint8_t svld1q_gather_offset(svbool_t, int8_t const *, svuint64_t);
11302		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f64)))
11303		svfloat64_t svld1q_gather_offset(svbool_t, float64_t const *, svuint64_t);
11304		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f32)))
11305		svfloat32_t svld1q_gather_offset(svbool_t, float32_t const *, svuint64_t);
11306		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f16)))
11307		svfloat16_t svld1q_gather_offset(svbool_t, float16_t const *, svuint64_t);
11308		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s32)))
11309		svint32_t svld1q_gather_offset(svbool_t, int32_t const *, svuint64_t);
11310		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s64)))
11311		svint64_t svld1q_gather_offset(svbool_t, int64_t const *, svuint64_t);
11312		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_mf8)))
11313		svmfloat8_t svld1q_gather_offset(svbool_t, mfloat8_t const *, svuint64_t);
11314		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s16)))
11315		svint16_t svld1q_gather_offset(svbool_t, int16_t const *, svuint64_t);
11316		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_u64)))
11317		svuint64_t svld1udq(svbool_t, uint64_t const *);
11318		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_f64)))
11319		svfloat64_t svld1udq(svbool_t, float64_t const *);
11320		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_s64)))
11321		svint64_t svld1udq(svbool_t, int64_t const *);
11322		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_u64)))
11323		svuint64_t svld1udq_vnum(svbool_t, uint64_t const *, int64_t);
11324		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_f64)))
11325		svfloat64_t svld1udq_vnum(svbool_t, float64_t const *, int64_t);
11326		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_s64)))
11327		svint64_t svld1udq_vnum(svbool_t, int64_t const *, int64_t);
11328		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_u32)))
11329		svuint32_t svld1uwq(svbool_t, uint32_t const *);
11330		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_f32)))
11331		svfloat32_t svld1uwq(svbool_t, float32_t const *);
11332		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_s32)))
11333		svint32_t svld1uwq(svbool_t, int32_t const *);
11334		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_u32)))
11335		svuint32_t svld1uwq_vnum(svbool_t, uint32_t const *, int64_t);
11336		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_f32)))
11337		svfloat32_t svld1uwq_vnum(svbool_t, float32_t const *, int64_t);
11338		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_s32)))
11339		svint32_t svld1uwq_vnum(svbool_t, int32_t const *, int64_t);
11340		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
11341		void svst1dq(svbool_t, uint64_t *, svuint64_t);
11342		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
11343		void svst1dq(svbool_t, float64_t *, svfloat64_t);
11344		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
11345		void svst1dq(svbool_t, int64_t *, svint64_t);
11346		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
11347		void svst1dq_vnum(svbool_t, uint64_t *, int64_t, svuint64_t);
11348		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
11349		void svst1dq_vnum(svbool_t, float64_t *, int64_t, svfloat64_t);
11350		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
11351		void svst1dq_vnum(svbool_t, int64_t *, int64_t, svint64_t);
11352		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
11353		void svst1q_scatter(svbool_t, svuint64_t, svuint8_t);
11354		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
11355		void svst1q_scatter(svbool_t, svuint64_t, svuint32_t);
11356		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u64)))
11357		void svst1q_scatter(svbool_t, svuint64_t, svuint64_t);
11358		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u16)))
11359		void svst1q_scatter(svbool_t, svuint64_t, svuint16_t);
11360		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_bf16)))
11361		void svst1q_scatter(svbool_t, svuint64_t, svbfloat16_t);
11362		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s8)))
11363		void svst1q_scatter(svbool_t, svuint64_t, svint8_t);
11364		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f64)))
11365		void svst1q_scatter(svbool_t, svuint64_t, svfloat64_t);
11366		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f32)))
11367		void svst1q_scatter(svbool_t, svuint64_t, svfloat32_t);
11368		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f16)))
11369		void svst1q_scatter(svbool_t, svuint64_t, svfloat16_t);
11370		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s32)))
11371		void svst1q_scatter(svbool_t, svuint64_t, svint32_t);
11372		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s64)))
11373		void svst1q_scatter(svbool_t, svuint64_t, svint64_t);
11374		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_mf8)))
11375		void svst1q_scatter(svbool_t, svuint64_t, svmfloat8_t);
11376		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s16)))
11377		void svst1q_scatter(svbool_t, svuint64_t, svint16_t);
11378		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u32)))
11379		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint32_t);
11380		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u64)))
11381		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
11382		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u16)))
11383		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint16_t);
11384		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_bf16)))
11385		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svbfloat16_t);
11386		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f64)))
11387		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat64_t);
11388		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f32)))
11389		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat32_t);
11390		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f16)))
11391		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat16_t);
11392		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s32)))
11393		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint32_t);
11394		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s64)))
11395		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
11396		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s16)))
11397		void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint16_t);
11398		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u8)))
11399		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint8_t);
11400		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u32)))
11401		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint32_t);
11402		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u64)))
11403		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
11404		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u16)))
11405		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint16_t);
11406		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_bf16)))
11407		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svbfloat16_t);
11408		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s8)))
11409		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint8_t);
11410		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f64)))
11411		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat64_t);
11412		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f32)))
11413		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat32_t);
11414		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f16)))
11415		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat16_t);
11416		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s32)))
11417		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint32_t);
11418		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s64)))
11419		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
11420		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_mf8)))
11421		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svmfloat8_t);
11422		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
11423		void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint16_t);
11424		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
11425		void svst1q_scatter_index(svbool_t, uint32_t *, svint64_t, svuint32_t);
11426		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
11427		void svst1q_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
11428		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
11429		void svst1q_scatter_index(svbool_t, uint16_t *, svint64_t, svuint16_t);
11430		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
11431		void svst1q_scatter_index(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
11432		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
11433		void svst1q_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
11434		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
11435		void svst1q_scatter_index(svbool_t, float32_t *, svint64_t, svfloat32_t);
11436		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
11437		void svst1q_scatter_index(svbool_t, float16_t *, svint64_t, svfloat16_t);
11438		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
11439		void svst1q_scatter_index(svbool_t, int32_t *, svint64_t, svint32_t);
11440		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
11441		void svst1q_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
11442		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
11443		void svst1q_scatter_index(svbool_t, int16_t *, svint64_t, svint16_t);
11444		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
11445		void svst1q_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint32_t);
11446		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
11447		void svst1q_scatter_index(svbool_t, uint64_t *, svuint64_t, svuint64_t);
11448		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u16)))
11449		void svst1q_scatter_index(svbool_t, uint16_t *, svuint64_t, svuint16_t);
11450		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_bf16)))
11451		void svst1q_scatter_index(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
11452		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f64)))
11453		void svst1q_scatter_index(svbool_t, float64_t *, svuint64_t, svfloat64_t);
11454		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f32)))
11455		void svst1q_scatter_index(svbool_t, float32_t *, svuint64_t, svfloat32_t);
11456		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f16)))
11457		void svst1q_scatter_index(svbool_t, float16_t *, svuint64_t, svfloat16_t);
11458		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s32)))
11459		void svst1q_scatter_index(svbool_t, int32_t *, svuint64_t, svint32_t);
11460		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s64)))
11461		void svst1q_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
11462		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
11463		void svst1q_scatter_index(svbool_t, int16_t *, svuint64_t, svint16_t);
11464		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
11465		void svst1q_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint8_t);
11466		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
11467		void svst1q_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint32_t);
11468		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
11469		void svst1q_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
11470		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
11471		void svst1q_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint16_t);
11472		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
11473		void svst1q_scatter_offset(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
11474		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
11475		void svst1q_scatter_offset(svbool_t, int8_t *, svint64_t, svint8_t);
11476		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
11477		void svst1q_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
11478		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
11479		void svst1q_scatter_offset(svbool_t, float32_t *, svint64_t, svfloat32_t);
11480		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
11481		void svst1q_scatter_offset(svbool_t, float16_t *, svint64_t, svfloat16_t);
11482		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
11483		void svst1q_scatter_offset(svbool_t, int32_t *, svint64_t, svint32_t);
11484		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
11485		void svst1q_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
11486		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_mf8)))
11487		void svst1q_scatter_offset(svbool_t, mfloat8_t *, svint64_t, svmfloat8_t);
11488		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
11489		void svst1q_scatter_offset(svbool_t, int16_t *, svint64_t, svint16_t);
11490		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
11491		void svst1q_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint8_t);
11492		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
11493		void svst1q_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint32_t);
11494		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u64)))
11495		void svst1q_scatter_offset(svbool_t, uint64_t *, svuint64_t, svuint64_t);
11496		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u16)))
11497		void svst1q_scatter_offset(svbool_t, uint16_t *, svuint64_t, svuint16_t);
11498		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_bf16)))
11499		void svst1q_scatter_offset(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
11500		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s8)))
11501		void svst1q_scatter_offset(svbool_t, int8_t *, svuint64_t, svint8_t);
11502		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f64)))
11503		void svst1q_scatter_offset(svbool_t, float64_t *, svuint64_t, svfloat64_t);
11504		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f32)))
11505		void svst1q_scatter_offset(svbool_t, float32_t *, svuint64_t, svfloat32_t);
11506		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f16)))
11507		void svst1q_scatter_offset(svbool_t, float16_t *, svuint64_t, svfloat16_t);
11508		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s32)))
11509		void svst1q_scatter_offset(svbool_t, int32_t *, svuint64_t, svint32_t);
11510		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s64)))
11511		void svst1q_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
11512		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_mf8)))
11513		void svst1q_scatter_offset(svbool_t, mfloat8_t *, svuint64_t, svmfloat8_t);
11514		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
11515		void svst1q_scatter_offset(svbool_t, int16_t *, svuint64_t, svint16_t);
11516		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
11517		void svst1wq(svbool_t, uint32_t *, svuint32_t);
11518		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
11519		void svst1wq(svbool_t, float32_t *, svfloat32_t);
11520		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
11521		void svst1wq(svbool_t, int32_t *, svint32_t);
11522		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
11523		void svst1wq_vnum(svbool_t, uint32_t *, int64_t, svuint32_t);
11524		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
11525		void svst1wq_vnum(svbool_t, float32_t *, int64_t, svfloat32_t);
11526		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
11527		void svst1wq_vnum(svbool_t, int32_t *, int64_t, svint32_t);
11528		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c8)))
11529		uint64_t svcntp_c8(svcount_t, uint64_t);
11530		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c32)))
11531		uint64_t svcntp_c32(svcount_t, uint64_t);
11532		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c64)))
11533		uint64_t svcntp_c64(svcount_t, uint64_t);
11534		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c16)))
11535		uint64_t svcntp_c16(svcount_t, uint64_t);
11536		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x2)))
11537		svuint8x2_t svld1_u8_x2(svcount_t, uint8_t const *);
11538		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x2)))
11539		svint8x2_t svld1_s8_x2(svcount_t, int8_t const *);
11540		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x2)))
11541		svmfloat8x2_t svld1_mf8_x2(svcount_t, mfloat8_t const *);
11542		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x2)))
11543		svuint64x2_t svld1_u64_x2(svcount_t, uint64_t const *);
11544		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x2)))
11545		svfloat64x2_t svld1_f64_x2(svcount_t, float64_t const *);
11546		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x2)))
11547		svint64x2_t svld1_s64_x2(svcount_t, int64_t const *);
11548		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x2)))
11549		svuint16x2_t svld1_u16_x2(svcount_t, uint16_t const *);
11550		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x2)))
11551		svbfloat16x2_t svld1_bf16_x2(svcount_t, bfloat16_t const *);
11552		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x2)))
11553		svfloat16x2_t svld1_f16_x2(svcount_t, float16_t const *);
11554		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x2)))
11555		svint16x2_t svld1_s16_x2(svcount_t, int16_t const *);
11556		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x2)))
11557		svuint32x2_t svld1_u32_x2(svcount_t, uint32_t const *);
11558		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x2)))
11559		svfloat32x2_t svld1_f32_x2(svcount_t, float32_t const *);
11560		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x2)))
11561		svint32x2_t svld1_s32_x2(svcount_t, int32_t const *);
11562		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x4)))
11563		svuint8x4_t svld1_u8_x4(svcount_t, uint8_t const *);
11564		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x4)))
11565		svint8x4_t svld1_s8_x4(svcount_t, int8_t const *);
11566		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x4)))
11567		svmfloat8x4_t svld1_mf8_x4(svcount_t, mfloat8_t const *);
11568		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x4)))
11569		svuint64x4_t svld1_u64_x4(svcount_t, uint64_t const *);
11570		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x4)))
11571		svfloat64x4_t svld1_f64_x4(svcount_t, float64_t const *);
11572		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x4)))
11573		svint64x4_t svld1_s64_x4(svcount_t, int64_t const *);
11574		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x4)))
11575		svuint16x4_t svld1_u16_x4(svcount_t, uint16_t const *);
11576		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x4)))
11577		svbfloat16x4_t svld1_bf16_x4(svcount_t, bfloat16_t const *);
11578		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x4)))
11579		svfloat16x4_t svld1_f16_x4(svcount_t, float16_t const *);
11580		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x4)))
11581		svint16x4_t svld1_s16_x4(svcount_t, int16_t const *);
11582		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x4)))
11583		svuint32x4_t svld1_u32_x4(svcount_t, uint32_t const *);
11584		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x4)))
11585		svfloat32x4_t svld1_f32_x4(svcount_t, float32_t const *);
11586		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x4)))
11587		svint32x4_t svld1_s32_x4(svcount_t, int32_t const *);
11588		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x2)))
11589		svuint8x2_t svld1_vnum_u8_x2(svcount_t, uint8_t const *, int64_t);
11590		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x2)))
11591		svint8x2_t svld1_vnum_s8_x2(svcount_t, int8_t const *, int64_t);
11592		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x2)))
11593		svmfloat8x2_t svld1_vnum_mf8_x2(svcount_t, mfloat8_t const *, int64_t);
11594		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x2)))
11595		svuint64x2_t svld1_vnum_u64_x2(svcount_t, uint64_t const *, int64_t);
11596		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x2)))
11597		svfloat64x2_t svld1_vnum_f64_x2(svcount_t, float64_t const *, int64_t);
11598		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x2)))
11599		svint64x2_t svld1_vnum_s64_x2(svcount_t, int64_t const *, int64_t);
11600		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x2)))
11601		svuint16x2_t svld1_vnum_u16_x2(svcount_t, uint16_t const *, int64_t);
11602		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x2)))
11603		svbfloat16x2_t svld1_vnum_bf16_x2(svcount_t, bfloat16_t const *, int64_t);
11604		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x2)))
11605		svfloat16x2_t svld1_vnum_f16_x2(svcount_t, float16_t const *, int64_t);
11606		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x2)))
11607		svint16x2_t svld1_vnum_s16_x2(svcount_t, int16_t const *, int64_t);
11608		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x2)))
11609		svuint32x2_t svld1_vnum_u32_x2(svcount_t, uint32_t const *, int64_t);
11610		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x2)))
11611		svfloat32x2_t svld1_vnum_f32_x2(svcount_t, float32_t const *, int64_t);
11612		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x2)))
11613		svint32x2_t svld1_vnum_s32_x2(svcount_t, int32_t const *, int64_t);
11614		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x4)))
11615		svuint8x4_t svld1_vnum_u8_x4(svcount_t, uint8_t const *, int64_t);
11616		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x4)))
11617		svint8x4_t svld1_vnum_s8_x4(svcount_t, int8_t const *, int64_t);
11618		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x4)))
11619		svmfloat8x4_t svld1_vnum_mf8_x4(svcount_t, mfloat8_t const *, int64_t);
11620		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x4)))
11621		svuint64x4_t svld1_vnum_u64_x4(svcount_t, uint64_t const *, int64_t);
11622		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x4)))
11623		svfloat64x4_t svld1_vnum_f64_x4(svcount_t, float64_t const *, int64_t);
11624		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x4)))
11625		svint64x4_t svld1_vnum_s64_x4(svcount_t, int64_t const *, int64_t);
11626		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x4)))
11627		svuint16x4_t svld1_vnum_u16_x4(svcount_t, uint16_t const *, int64_t);
11628		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x4)))
11629		svbfloat16x4_t svld1_vnum_bf16_x4(svcount_t, bfloat16_t const *, int64_t);
11630		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x4)))
11631		svfloat16x4_t svld1_vnum_f16_x4(svcount_t, float16_t const *, int64_t);
11632		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x4)))
11633		svint16x4_t svld1_vnum_s16_x4(svcount_t, int16_t const *, int64_t);
11634		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x4)))
11635		svuint32x4_t svld1_vnum_u32_x4(svcount_t, uint32_t const *, int64_t);
11636		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x4)))
11637		svfloat32x4_t svld1_vnum_f32_x4(svcount_t, float32_t const *, int64_t);
11638		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x4)))
11639		svint32x4_t svld1_vnum_s32_x4(svcount_t, int32_t const *, int64_t);
11640		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x2)))
11641		svuint8x2_t svldnt1_u8_x2(svcount_t, uint8_t const *);
11642		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x2)))
11643		svint8x2_t svldnt1_s8_x2(svcount_t, int8_t const *);
11644		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x2)))
11645		svmfloat8x2_t svldnt1_mf8_x2(svcount_t, mfloat8_t const *);
11646		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x2)))
11647		svuint64x2_t svldnt1_u64_x2(svcount_t, uint64_t const *);
11648		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x2)))
11649		svfloat64x2_t svldnt1_f64_x2(svcount_t, float64_t const *);
11650		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x2)))
11651		svint64x2_t svldnt1_s64_x2(svcount_t, int64_t const *);
11652		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x2)))
11653		svuint16x2_t svldnt1_u16_x2(svcount_t, uint16_t const *);
11654		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x2)))
11655		svbfloat16x2_t svldnt1_bf16_x2(svcount_t, bfloat16_t const *);
11656		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x2)))
11657		svfloat16x2_t svldnt1_f16_x2(svcount_t, float16_t const *);
11658		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x2)))
11659		svint16x2_t svldnt1_s16_x2(svcount_t, int16_t const *);
11660		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x2)))
11661		svuint32x2_t svldnt1_u32_x2(svcount_t, uint32_t const *);
11662		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x2)))
11663		svfloat32x2_t svldnt1_f32_x2(svcount_t, float32_t const *);
11664		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x2)))
11665		svint32x2_t svldnt1_s32_x2(svcount_t, int32_t const *);
11666		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x4)))
11667		svuint8x4_t svldnt1_u8_x4(svcount_t, uint8_t const *);
11668		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x4)))
11669		svint8x4_t svldnt1_s8_x4(svcount_t, int8_t const *);
11670		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x4)))
11671		svmfloat8x4_t svldnt1_mf8_x4(svcount_t, mfloat8_t const *);
11672		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x4)))
11673		svuint64x4_t svldnt1_u64_x4(svcount_t, uint64_t const *);
11674		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x4)))
11675		svfloat64x4_t svldnt1_f64_x4(svcount_t, float64_t const *);
11676		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x4)))
11677		svint64x4_t svldnt1_s64_x4(svcount_t, int64_t const *);
11678		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x4)))
11679		svuint16x4_t svldnt1_u16_x4(svcount_t, uint16_t const *);
11680		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x4)))
11681		svbfloat16x4_t svldnt1_bf16_x4(svcount_t, bfloat16_t const *);
11682		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x4)))
11683		svfloat16x4_t svldnt1_f16_x4(svcount_t, float16_t const *);
11684		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x4)))
11685		svint16x4_t svldnt1_s16_x4(svcount_t, int16_t const *);
11686		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x4)))
11687		svuint32x4_t svldnt1_u32_x4(svcount_t, uint32_t const *);
11688		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x4)))
11689		svfloat32x4_t svldnt1_f32_x4(svcount_t, float32_t const *);
11690		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x4)))
11691		svint32x4_t svldnt1_s32_x4(svcount_t, int32_t const *);
11692		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x2)))
11693		svuint8x2_t svldnt1_vnum_u8_x2(svcount_t, uint8_t const *, int64_t);
11694		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x2)))
11695		svint8x2_t svldnt1_vnum_s8_x2(svcount_t, int8_t const *, int64_t);
11696		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x2)))
11697		svmfloat8x2_t svldnt1_vnum_mf8_x2(svcount_t, mfloat8_t const *, int64_t);
11698		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x2)))
11699		svuint64x2_t svldnt1_vnum_u64_x2(svcount_t, uint64_t const *, int64_t);
11700		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x2)))
11701		svfloat64x2_t svldnt1_vnum_f64_x2(svcount_t, float64_t const *, int64_t);
11702		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x2)))
11703		svint64x2_t svldnt1_vnum_s64_x2(svcount_t, int64_t const *, int64_t);
11704		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x2)))
11705		svuint16x2_t svldnt1_vnum_u16_x2(svcount_t, uint16_t const *, int64_t);
11706		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x2)))
11707		svbfloat16x2_t svldnt1_vnum_bf16_x2(svcount_t, bfloat16_t const *, int64_t);
11708		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x2)))
11709		svfloat16x2_t svldnt1_vnum_f16_x2(svcount_t, float16_t const *, int64_t);
11710		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x2)))
11711		svint16x2_t svldnt1_vnum_s16_x2(svcount_t, int16_t const *, int64_t);
11712		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x2)))
11713		svuint32x2_t svldnt1_vnum_u32_x2(svcount_t, uint32_t const *, int64_t);
11714		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x2)))
11715		svfloat32x2_t svldnt1_vnum_f32_x2(svcount_t, float32_t const *, int64_t);
11716		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x2)))
11717		svint32x2_t svldnt1_vnum_s32_x2(svcount_t, int32_t const *, int64_t);
11718		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x4)))
11719		svuint8x4_t svldnt1_vnum_u8_x4(svcount_t, uint8_t const *, int64_t);
11720		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x4)))
11721		svint8x4_t svldnt1_vnum_s8_x4(svcount_t, int8_t const *, int64_t);
11722		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x4)))
11723		svmfloat8x4_t svldnt1_vnum_mf8_x4(svcount_t, mfloat8_t const *, int64_t);
11724		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x4)))
11725		svuint64x4_t svldnt1_vnum_u64_x4(svcount_t, uint64_t const *, int64_t);
11726		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x4)))
11727		svfloat64x4_t svldnt1_vnum_f64_x4(svcount_t, float64_t const *, int64_t);
11728		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x4)))
11729		svint64x4_t svldnt1_vnum_s64_x4(svcount_t, int64_t const *, int64_t);
11730		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x4)))
11731		svuint16x4_t svldnt1_vnum_u16_x4(svcount_t, uint16_t const *, int64_t);
11732		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x4)))
11733		svbfloat16x4_t svldnt1_vnum_bf16_x4(svcount_t, bfloat16_t const *, int64_t);
11734		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x4)))
11735		svfloat16x4_t svldnt1_vnum_f16_x4(svcount_t, float16_t const *, int64_t);
11736		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x4)))
11737		svint16x4_t svldnt1_vnum_s16_x4(svcount_t, int16_t const *, int64_t);
11738		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x4)))
11739		svuint32x4_t svldnt1_vnum_u32_x4(svcount_t, uint32_t const *, int64_t);
11740		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x4)))
11741		svfloat32x4_t svldnt1_vnum_f32_x4(svcount_t, float32_t const *, int64_t);
11742		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x4)))
11743		svint32x4_t svldnt1_vnum_s32_x4(svcount_t, int32_t const *, int64_t);
11744		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c8)))
11745		svbool_t svpext_lane_c8(svcount_t, uint64_t);
11746		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c32)))
11747		svbool_t svpext_lane_c32(svcount_t, uint64_t);
11748		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c64)))
11749		svbool_t svpext_lane_c64(svcount_t, uint64_t);
11750		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c16)))
11751		svbool_t svpext_lane_c16(svcount_t, uint64_t);
11752		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c8_x2)))
11753		svboolx2_t svpext_lane_c8_x2(svcount_t, uint64_t);
11754		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c32_x2)))
11755		svboolx2_t svpext_lane_c32_x2(svcount_t, uint64_t);
11756		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c64_x2)))
11757		svboolx2_t svpext_lane_c64_x2(svcount_t, uint64_t);
11758		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c16_x2)))
11759		svboolx2_t svpext_lane_c16_x2(svcount_t, uint64_t);
11760		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpfalse_c)))
11761		svcount_t svpfalse_c(void);
11762		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c16)))
11763		svcount_t svpsel_lane_c16(svcount_t, svbool_t, uint32_t);
11764		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c32)))
11765		svcount_t svpsel_lane_c32(svcount_t, svbool_t, uint32_t);
11766		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c64)))
11767		svcount_t svpsel_lane_c64(svcount_t, svbool_t, uint32_t);
11768		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c8)))
11769		svcount_t svpsel_lane_c8(svcount_t, svbool_t, uint32_t);
11770		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c8)))
11771		svcount_t svptrue_c8(void);
11772		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c32)))
11773		svcount_t svptrue_c32(void);
11774		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c64)))
11775		svcount_t svptrue_c64(void);
11776		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c16)))
11777		svcount_t svptrue_c16(void);
11778		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_b)))
11779		svbool_t svreinterpret_b(svcount_t);
11780		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_c)))
11781		svcount_t svreinterpret_c(svbool_t);
11782		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x2)))
11783		void svst1_u8_x2(svcount_t, uint8_t *, svuint8x2_t);
11784		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x2)))
11785		void svst1_s8_x2(svcount_t, int8_t *, svint8x2_t);
11786		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x2)))
11787		void svst1_mf8_x2(svcount_t, mfloat8_t *, svmfloat8x2_t);
11788		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x2)))
11789		void svst1_u64_x2(svcount_t, uint64_t *, svuint64x2_t);
11790		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x2)))
11791		void svst1_f64_x2(svcount_t, float64_t *, svfloat64x2_t);
11792		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x2)))
11793		void svst1_s64_x2(svcount_t, int64_t *, svint64x2_t);
11794		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x2)))
11795		void svst1_u16_x2(svcount_t, uint16_t *, svuint16x2_t);
11796		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x2)))
11797		void svst1_bf16_x2(svcount_t, bfloat16_t *, svbfloat16x2_t);
11798		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x2)))
11799		void svst1_f16_x2(svcount_t, float16_t *, svfloat16x2_t);
11800		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x2)))
11801		void svst1_s16_x2(svcount_t, int16_t *, svint16x2_t);
11802		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x2)))
11803		void svst1_u32_x2(svcount_t, uint32_t *, svuint32x2_t);
11804		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x2)))
11805		void svst1_f32_x2(svcount_t, float32_t *, svfloat32x2_t);
11806		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x2)))
11807		void svst1_s32_x2(svcount_t, int32_t *, svint32x2_t);
11808		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x4)))
11809		void svst1_u8_x4(svcount_t, uint8_t *, svuint8x4_t);
11810		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x4)))
11811		void svst1_s8_x4(svcount_t, int8_t *, svint8x4_t);
11812		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x4)))
11813		void svst1_mf8_x4(svcount_t, mfloat8_t *, svmfloat8x4_t);
11814		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x4)))
11815		void svst1_u64_x4(svcount_t, uint64_t *, svuint64x4_t);
11816		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x4)))
11817		void svst1_f64_x4(svcount_t, float64_t *, svfloat64x4_t);
11818		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x4)))
11819		void svst1_s64_x4(svcount_t, int64_t *, svint64x4_t);
11820		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x4)))
11821		void svst1_u16_x4(svcount_t, uint16_t *, svuint16x4_t);
11822		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x4)))
11823		void svst1_bf16_x4(svcount_t, bfloat16_t *, svbfloat16x4_t);
11824		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x4)))
11825		void svst1_f16_x4(svcount_t, float16_t *, svfloat16x4_t);
11826		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x4)))
11827		void svst1_s16_x4(svcount_t, int16_t *, svint16x4_t);
11828		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x4)))
11829		void svst1_u32_x4(svcount_t, uint32_t *, svuint32x4_t);
11830		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x4)))
11831		void svst1_f32_x4(svcount_t, float32_t *, svfloat32x4_t);
11832		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x4)))
11833		void svst1_s32_x4(svcount_t, int32_t *, svint32x4_t);
11834		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x2)))
11835		void svst1_vnum_u8_x2(svcount_t, uint8_t *, int64_t, svuint8x2_t);
11836		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x2)))
11837		void svst1_vnum_s8_x2(svcount_t, int8_t *, int64_t, svint8x2_t);
11838		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x2)))
11839		void svst1_vnum_mf8_x2(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
11840		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x2)))
11841		void svst1_vnum_u64_x2(svcount_t, uint64_t *, int64_t, svuint64x2_t);
11842		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x2)))
11843		void svst1_vnum_f64_x2(svcount_t, float64_t *, int64_t, svfloat64x2_t);
11844		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x2)))
11845		void svst1_vnum_s64_x2(svcount_t, int64_t *, int64_t, svint64x2_t);
11846		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x2)))
11847		void svst1_vnum_u16_x2(svcount_t, uint16_t *, int64_t, svuint16x2_t);
11848		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x2)))
11849		void svst1_vnum_bf16_x2(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
11850		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x2)))
11851		void svst1_vnum_f16_x2(svcount_t, float16_t *, int64_t, svfloat16x2_t);
11852		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x2)))
11853		void svst1_vnum_s16_x2(svcount_t, int16_t *, int64_t, svint16x2_t);
11854		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x2)))
11855		void svst1_vnum_u32_x2(svcount_t, uint32_t *, int64_t, svuint32x2_t);
11856		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x2)))
11857		void svst1_vnum_f32_x2(svcount_t, float32_t *, int64_t, svfloat32x2_t);
11858		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x2)))
11859		void svst1_vnum_s32_x2(svcount_t, int32_t *, int64_t, svint32x2_t);
11860		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x4)))
11861		void svst1_vnum_u8_x4(svcount_t, uint8_t *, int64_t, svuint8x4_t);
11862		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x4)))
11863		void svst1_vnum_s8_x4(svcount_t, int8_t *, int64_t, svint8x4_t);
11864		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x4)))
11865		void svst1_vnum_mf8_x4(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
11866		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x4)))
11867		void svst1_vnum_u64_x4(svcount_t, uint64_t *, int64_t, svuint64x4_t);
11868		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x4)))
11869		void svst1_vnum_f64_x4(svcount_t, float64_t *, int64_t, svfloat64x4_t);
11870		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x4)))
11871		void svst1_vnum_s64_x4(svcount_t, int64_t *, int64_t, svint64x4_t);
11872		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x4)))
11873		void svst1_vnum_u16_x4(svcount_t, uint16_t *, int64_t, svuint16x4_t);
11874		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x4)))
11875		void svst1_vnum_bf16_x4(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
11876		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x4)))
11877		void svst1_vnum_f16_x4(svcount_t, float16_t *, int64_t, svfloat16x4_t);
11878		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x4)))
11879		void svst1_vnum_s16_x4(svcount_t, int16_t *, int64_t, svint16x4_t);
11880		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x4)))
11881		void svst1_vnum_u32_x4(svcount_t, uint32_t *, int64_t, svuint32x4_t);
11882		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x4)))
11883		void svst1_vnum_f32_x4(svcount_t, float32_t *, int64_t, svfloat32x4_t);
11884		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x4)))
11885		void svst1_vnum_s32_x4(svcount_t, int32_t *, int64_t, svint32x4_t);
11886		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x2)))
11887		void svstnt1_u8_x2(svcount_t, uint8_t *, svuint8x2_t);
11888		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x2)))
11889		void svstnt1_s8_x2(svcount_t, int8_t *, svint8x2_t);
11890		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x2)))
11891		void svstnt1_mf8_x2(svcount_t, mfloat8_t *, svmfloat8x2_t);
11892		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x2)))
11893		void svstnt1_u64_x2(svcount_t, uint64_t *, svuint64x2_t);
11894		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x2)))
11895		void svstnt1_f64_x2(svcount_t, float64_t *, svfloat64x2_t);
11896		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x2)))
11897		void svstnt1_s64_x2(svcount_t, int64_t *, svint64x2_t);
11898		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x2)))
11899		void svstnt1_u16_x2(svcount_t, uint16_t *, svuint16x2_t);
11900		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x2)))
11901		void svstnt1_bf16_x2(svcount_t, bfloat16_t *, svbfloat16x2_t);
11902		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x2)))
11903		void svstnt1_f16_x2(svcount_t, float16_t *, svfloat16x2_t);
11904		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x2)))
11905		void svstnt1_s16_x2(svcount_t, int16_t *, svint16x2_t);
11906		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x2)))
11907		void svstnt1_u32_x2(svcount_t, uint32_t *, svuint32x2_t);
11908		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x2)))
11909		void svstnt1_f32_x2(svcount_t, float32_t *, svfloat32x2_t);
11910		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x2)))
11911		void svstnt1_s32_x2(svcount_t, int32_t *, svint32x2_t);
11912		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x4)))
11913		void svstnt1_u8_x4(svcount_t, uint8_t *, svuint8x4_t);
11914		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x4)))
11915		void svstnt1_s8_x4(svcount_t, int8_t *, svint8x4_t);
11916		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x4)))
11917		void svstnt1_mf8_x4(svcount_t, mfloat8_t *, svmfloat8x4_t);
11918		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x4)))
11919		void svstnt1_u64_x4(svcount_t, uint64_t *, svuint64x4_t);
11920		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x4)))
11921		void svstnt1_f64_x4(svcount_t, float64_t *, svfloat64x4_t);
11922		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x4)))
11923		void svstnt1_s64_x4(svcount_t, int64_t *, svint64x4_t);
11924		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x4)))
11925		void svstnt1_u16_x4(svcount_t, uint16_t *, svuint16x4_t);
11926		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x4)))
11927		void svstnt1_bf16_x4(svcount_t, bfloat16_t *, svbfloat16x4_t);
11928		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x4)))
11929		void svstnt1_f16_x4(svcount_t, float16_t *, svfloat16x4_t);
11930		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x4)))
11931		void svstnt1_s16_x4(svcount_t, int16_t *, svint16x4_t);
11932		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x4)))
11933		void svstnt1_u32_x4(svcount_t, uint32_t *, svuint32x4_t);
11934		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x4)))
11935		void svstnt1_f32_x4(svcount_t, float32_t *, svfloat32x4_t);
11936		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x4)))
11937		void svstnt1_s32_x4(svcount_t, int32_t *, svint32x4_t);
11938		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x2)))
11939		void svstnt1_vnum_u8_x2(svcount_t, uint8_t *, int64_t, svuint8x2_t);
11940		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x2)))
11941		void svstnt1_vnum_s8_x2(svcount_t, int8_t *, int64_t, svint8x2_t);
11942		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x2)))
11943		void svstnt1_vnum_mf8_x2(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
11944		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x2)))
11945		void svstnt1_vnum_u64_x2(svcount_t, uint64_t *, int64_t, svuint64x2_t);
11946		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x2)))
11947		void svstnt1_vnum_f64_x2(svcount_t, float64_t *, int64_t, svfloat64x2_t);
11948		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x2)))
11949		void svstnt1_vnum_s64_x2(svcount_t, int64_t *, int64_t, svint64x2_t);
11950		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x2)))
11951		void svstnt1_vnum_u16_x2(svcount_t, uint16_t *, int64_t, svuint16x2_t);
11952		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x2)))
11953		void svstnt1_vnum_bf16_x2(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
11954		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x2)))
11955		void svstnt1_vnum_f16_x2(svcount_t, float16_t *, int64_t, svfloat16x2_t);
11956		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x2)))
11957		void svstnt1_vnum_s16_x2(svcount_t, int16_t *, int64_t, svint16x2_t);
11958		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x2)))
11959		void svstnt1_vnum_u32_x2(svcount_t, uint32_t *, int64_t, svuint32x2_t);
11960		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x2)))
11961		void svstnt1_vnum_f32_x2(svcount_t, float32_t *, int64_t, svfloat32x2_t);
11962		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x2)))
11963		void svstnt1_vnum_s32_x2(svcount_t, int32_t *, int64_t, svint32x2_t);
11964		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x4)))
11965		void svstnt1_vnum_u8_x4(svcount_t, uint8_t *, int64_t, svuint8x4_t);
11966		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x4)))
11967		void svstnt1_vnum_s8_x4(svcount_t, int8_t *, int64_t, svint8x4_t);
11968		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x4)))
11969		void svstnt1_vnum_mf8_x4(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
11970		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x4)))
11971		void svstnt1_vnum_u64_x4(svcount_t, uint64_t *, int64_t, svuint64x4_t);
11972		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x4)))
11973		void svstnt1_vnum_f64_x4(svcount_t, float64_t *, int64_t, svfloat64x4_t);
11974		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x4)))
11975		void svstnt1_vnum_s64_x4(svcount_t, int64_t *, int64_t, svint64x4_t);
11976		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x4)))
11977		void svstnt1_vnum_u16_x4(svcount_t, uint16_t *, int64_t, svuint16x4_t);
11978		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x4)))
11979		void svstnt1_vnum_bf16_x4(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
11980		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x4)))
11981		void svstnt1_vnum_f16_x4(svcount_t, float16_t *, int64_t, svfloat16x4_t);
11982		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x4)))
11983		void svstnt1_vnum_s16_x4(svcount_t, int16_t *, int64_t, svint16x4_t);
11984		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x4)))
11985		void svstnt1_vnum_u32_x4(svcount_t, uint32_t *, int64_t, svuint32x4_t);
11986		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x4)))
11987		void svstnt1_vnum_f32_x4(svcount_t, float32_t *, int64_t, svfloat32x4_t);
11988		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x4)))
11989		void svstnt1_vnum_s32_x4(svcount_t, int32_t *, int64_t, svint32x4_t);
11990		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_s64)))
11991		svcount_t svwhilege_c8_s64(int64_t, int64_t, uint64_t);
11992		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_s64)))
11993		svcount_t svwhilege_c32_s64(int64_t, int64_t, uint64_t);
11994		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_s64)))
11995		svcount_t svwhilege_c64_s64(int64_t, int64_t, uint64_t);
11996		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_s64)))
11997		svcount_t svwhilege_c16_s64(int64_t, int64_t, uint64_t);
11998		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_u64)))
11999		svcount_t svwhilege_c8_u64(uint64_t, uint64_t, uint64_t);
12000		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_u64)))
12001		svcount_t svwhilege_c32_u64(uint64_t, uint64_t, uint64_t);
12002		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_u64)))
12003		svcount_t svwhilege_c64_u64(uint64_t, uint64_t, uint64_t);
12004		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_u64)))
12005		svcount_t svwhilege_c16_u64(uint64_t, uint64_t, uint64_t);
12006		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_s64)))
12007		svcount_t svwhilegt_c8_s64(int64_t, int64_t, uint64_t);
12008		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_s64)))
12009		svcount_t svwhilegt_c32_s64(int64_t, int64_t, uint64_t);
12010		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_s64)))
12011		svcount_t svwhilegt_c64_s64(int64_t, int64_t, uint64_t);
12012		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_s64)))
12013		svcount_t svwhilegt_c16_s64(int64_t, int64_t, uint64_t);
12014		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_u64)))
12015		svcount_t svwhilegt_c8_u64(uint64_t, uint64_t, uint64_t);
12016		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_u64)))
12017		svcount_t svwhilegt_c32_u64(uint64_t, uint64_t, uint64_t);
12018		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_u64)))
12019		svcount_t svwhilegt_c64_u64(uint64_t, uint64_t, uint64_t);
12020		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_u64)))
12021		svcount_t svwhilegt_c16_u64(uint64_t, uint64_t, uint64_t);
12022		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_s64)))
12023		svcount_t svwhilele_c8_s64(int64_t, int64_t, uint64_t);
12024		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_s64)))
12025		svcount_t svwhilele_c32_s64(int64_t, int64_t, uint64_t);
12026		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_s64)))
12027		svcount_t svwhilele_c64_s64(int64_t, int64_t, uint64_t);
12028		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_s64)))
12029		svcount_t svwhilele_c16_s64(int64_t, int64_t, uint64_t);
12030		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_u64)))
12031		svcount_t svwhilele_c8_u64(uint64_t, uint64_t, uint64_t);
12032		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_u64)))
12033		svcount_t svwhilele_c32_u64(uint64_t, uint64_t, uint64_t);
12034		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_u64)))
12035		svcount_t svwhilele_c64_u64(uint64_t, uint64_t, uint64_t);
12036		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_u64)))
12037		svcount_t svwhilele_c16_u64(uint64_t, uint64_t, uint64_t);
12038		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_u64)))
12039		svcount_t svwhilelt_c8_u64(uint64_t, uint64_t, uint64_t);
12040		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_u64)))
12041		svcount_t svwhilelt_c32_u64(uint64_t, uint64_t, uint64_t);
12042		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_u64)))
12043		svcount_t svwhilelt_c64_u64(uint64_t, uint64_t, uint64_t);
12044		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_u64)))
12045		svcount_t svwhilelt_c16_u64(uint64_t, uint64_t, uint64_t);
12046		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_s64)))
12047		svcount_t svwhilelt_c8_s64(int64_t, int64_t, uint64_t);
12048		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_s64)))
12049		svcount_t svwhilelt_c32_s64(int64_t, int64_t, uint64_t);
12050		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64)))
12051		svcount_t svwhilelt_c64_s64(int64_t, int64_t, uint64_t);
12052		__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64)))
12053		svcount_t svwhilelt_c16_s64(int64_t, int64_t, uint64_t);
12054		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x2)))
12055		svuint8x2_t svld1_x2(svcount_t, uint8_t const *);
12056		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x2)))
12057		svint8x2_t svld1_x2(svcount_t, int8_t const *);
12058		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x2)))
12059		svmfloat8x2_t svld1_x2(svcount_t, mfloat8_t const *);
12060		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x2)))
12061		svuint64x2_t svld1_x2(svcount_t, uint64_t const *);
12062		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x2)))
12063		svfloat64x2_t svld1_x2(svcount_t, float64_t const *);
12064		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x2)))
12065		svint64x2_t svld1_x2(svcount_t, int64_t const *);
12066		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x2)))
12067		svuint16x2_t svld1_x2(svcount_t, uint16_t const *);
12068		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x2)))
12069		svbfloat16x2_t svld1_x2(svcount_t, bfloat16_t const *);
12070		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x2)))
12071		svfloat16x2_t svld1_x2(svcount_t, float16_t const *);
12072		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x2)))
12073		svint16x2_t svld1_x2(svcount_t, int16_t const *);
12074		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x2)))
12075		svuint32x2_t svld1_x2(svcount_t, uint32_t const *);
12076		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x2)))
12077		svfloat32x2_t svld1_x2(svcount_t, float32_t const *);
12078		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x2)))
12079		svint32x2_t svld1_x2(svcount_t, int32_t const *);
12080		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x4)))
12081		svuint8x4_t svld1_x4(svcount_t, uint8_t const *);
12082		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x4)))
12083		svint8x4_t svld1_x4(svcount_t, int8_t const *);
12084		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x4)))
12085		svmfloat8x4_t svld1_x4(svcount_t, mfloat8_t const *);
12086		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x4)))
12087		svuint64x4_t svld1_x4(svcount_t, uint64_t const *);
12088		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x4)))
12089		svfloat64x4_t svld1_x4(svcount_t, float64_t const *);
12090		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x4)))
12091		svint64x4_t svld1_x4(svcount_t, int64_t const *);
12092		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x4)))
12093		svuint16x4_t svld1_x4(svcount_t, uint16_t const *);
12094		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x4)))
12095		svbfloat16x4_t svld1_x4(svcount_t, bfloat16_t const *);
12096		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x4)))
12097		svfloat16x4_t svld1_x4(svcount_t, float16_t const *);
12098		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x4)))
12099		svint16x4_t svld1_x4(svcount_t, int16_t const *);
12100		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x4)))
12101		svuint32x4_t svld1_x4(svcount_t, uint32_t const *);
12102		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x4)))
12103		svfloat32x4_t svld1_x4(svcount_t, float32_t const *);
12104		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x4)))
12105		svint32x4_t svld1_x4(svcount_t, int32_t const *);
12106		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x2)))
12107		svuint8x2_t svld1_vnum_x2(svcount_t, uint8_t const *, int64_t);
12108		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x2)))
12109		svint8x2_t svld1_vnum_x2(svcount_t, int8_t const *, int64_t);
12110		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x2)))
12111		svmfloat8x2_t svld1_vnum_x2(svcount_t, mfloat8_t const *, int64_t);
12112		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x2)))
12113		svuint64x2_t svld1_vnum_x2(svcount_t, uint64_t const *, int64_t);
12114		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x2)))
12115		svfloat64x2_t svld1_vnum_x2(svcount_t, float64_t const *, int64_t);
12116		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x2)))
12117		svint64x2_t svld1_vnum_x2(svcount_t, int64_t const *, int64_t);
12118		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x2)))
12119		svuint16x2_t svld1_vnum_x2(svcount_t, uint16_t const *, int64_t);
12120		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x2)))
12121		svbfloat16x2_t svld1_vnum_x2(svcount_t, bfloat16_t const *, int64_t);
12122		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x2)))
12123		svfloat16x2_t svld1_vnum_x2(svcount_t, float16_t const *, int64_t);
12124		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x2)))
12125		svint16x2_t svld1_vnum_x2(svcount_t, int16_t const *, int64_t);
12126		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x2)))
12127		svuint32x2_t svld1_vnum_x2(svcount_t, uint32_t const *, int64_t);
12128		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x2)))
12129		svfloat32x2_t svld1_vnum_x2(svcount_t, float32_t const *, int64_t);
12130		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x2)))
12131		svint32x2_t svld1_vnum_x2(svcount_t, int32_t const *, int64_t);
12132		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x4)))
12133		svuint8x4_t svld1_vnum_x4(svcount_t, uint8_t const *, int64_t);
12134		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x4)))
12135		svint8x4_t svld1_vnum_x4(svcount_t, int8_t const *, int64_t);
12136		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x4)))
12137		svmfloat8x4_t svld1_vnum_x4(svcount_t, mfloat8_t const *, int64_t);
12138		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x4)))
12139		svuint64x4_t svld1_vnum_x4(svcount_t, uint64_t const *, int64_t);
12140		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x4)))
12141		svfloat64x4_t svld1_vnum_x4(svcount_t, float64_t const *, int64_t);
12142		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x4)))
12143		svint64x4_t svld1_vnum_x4(svcount_t, int64_t const *, int64_t);
12144		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x4)))
12145		svuint16x4_t svld1_vnum_x4(svcount_t, uint16_t const *, int64_t);
12146		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x4)))
12147		svbfloat16x4_t svld1_vnum_x4(svcount_t, bfloat16_t const *, int64_t);
12148		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x4)))
12149		svfloat16x4_t svld1_vnum_x4(svcount_t, float16_t const *, int64_t);
12150		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x4)))
12151		svint16x4_t svld1_vnum_x4(svcount_t, int16_t const *, int64_t);
12152		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x4)))
12153		svuint32x4_t svld1_vnum_x4(svcount_t, uint32_t const *, int64_t);
12154		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x4)))
12155		svfloat32x4_t svld1_vnum_x4(svcount_t, float32_t const *, int64_t);
12156		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x4)))
12157		svint32x4_t svld1_vnum_x4(svcount_t, int32_t const *, int64_t);
12158		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x2)))
12159		svuint8x2_t svldnt1_x2(svcount_t, uint8_t const *);
12160		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x2)))
12161		svint8x2_t svldnt1_x2(svcount_t, int8_t const *);
12162		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x2)))
12163		svmfloat8x2_t svldnt1_x2(svcount_t, mfloat8_t const *);
12164		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x2)))
12165		svuint64x2_t svldnt1_x2(svcount_t, uint64_t const *);
12166		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x2)))
12167		svfloat64x2_t svldnt1_x2(svcount_t, float64_t const *);
12168		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x2)))
12169		svint64x2_t svldnt1_x2(svcount_t, int64_t const *);
12170		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x2)))
12171		svuint16x2_t svldnt1_x2(svcount_t, uint16_t const *);
12172		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x2)))
12173		svbfloat16x2_t svldnt1_x2(svcount_t, bfloat16_t const *);
12174		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x2)))
12175		svfloat16x2_t svldnt1_x2(svcount_t, float16_t const *);
12176		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x2)))
12177		svint16x2_t svldnt1_x2(svcount_t, int16_t const *);
12178		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x2)))
12179		svuint32x2_t svldnt1_x2(svcount_t, uint32_t const *);
12180		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x2)))
12181		svfloat32x2_t svldnt1_x2(svcount_t, float32_t const *);
12182		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x2)))
12183		svint32x2_t svldnt1_x2(svcount_t, int32_t const *);
12184		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x4)))
12185		svuint8x4_t svldnt1_x4(svcount_t, uint8_t const *);
12186		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x4)))
12187		svint8x4_t svldnt1_x4(svcount_t, int8_t const *);
12188		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x4)))
12189		svmfloat8x4_t svldnt1_x4(svcount_t, mfloat8_t const *);
12190		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x4)))
12191		svuint64x4_t svldnt1_x4(svcount_t, uint64_t const *);
12192		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x4)))
12193		svfloat64x4_t svldnt1_x4(svcount_t, float64_t const *);
12194		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x4)))
12195		svint64x4_t svldnt1_x4(svcount_t, int64_t const *);
12196		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x4)))
12197		svuint16x4_t svldnt1_x4(svcount_t, uint16_t const *);
12198		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x4)))
12199		svbfloat16x4_t svldnt1_x4(svcount_t, bfloat16_t const *);
12200		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x4)))
12201		svfloat16x4_t svldnt1_x4(svcount_t, float16_t const *);
12202		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x4)))
12203		svint16x4_t svldnt1_x4(svcount_t, int16_t const *);
12204		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x4)))
12205		svuint32x4_t svldnt1_x4(svcount_t, uint32_t const *);
12206		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x4)))
12207		svfloat32x4_t svldnt1_x4(svcount_t, float32_t const *);
12208		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x4)))
12209		svint32x4_t svldnt1_x4(svcount_t, int32_t const *);
12210		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x2)))
12211		svuint8x2_t svldnt1_vnum_x2(svcount_t, uint8_t const *, int64_t);
12212		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x2)))
12213		svint8x2_t svldnt1_vnum_x2(svcount_t, int8_t const *, int64_t);
12214		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x2)))
12215		svmfloat8x2_t svldnt1_vnum_x2(svcount_t, mfloat8_t const *, int64_t);
12216		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x2)))
12217		svuint64x2_t svldnt1_vnum_x2(svcount_t, uint64_t const *, int64_t);
12218		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x2)))
12219		svfloat64x2_t svldnt1_vnum_x2(svcount_t, float64_t const *, int64_t);
12220		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x2)))
12221		svint64x2_t svldnt1_vnum_x2(svcount_t, int64_t const *, int64_t);
12222		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x2)))
12223		svuint16x2_t svldnt1_vnum_x2(svcount_t, uint16_t const *, int64_t);
12224		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x2)))
12225		svbfloat16x2_t svldnt1_vnum_x2(svcount_t, bfloat16_t const *, int64_t);
12226		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x2)))
12227		svfloat16x2_t svldnt1_vnum_x2(svcount_t, float16_t const *, int64_t);
12228		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x2)))
12229		svint16x2_t svldnt1_vnum_x2(svcount_t, int16_t const *, int64_t);
12230		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x2)))
12231		svuint32x2_t svldnt1_vnum_x2(svcount_t, uint32_t const *, int64_t);
12232		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x2)))
12233		svfloat32x2_t svldnt1_vnum_x2(svcount_t, float32_t const *, int64_t);
12234		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x2)))
12235		svint32x2_t svldnt1_vnum_x2(svcount_t, int32_t const *, int64_t);
12236		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x4)))
12237		svuint8x4_t svldnt1_vnum_x4(svcount_t, uint8_t const *, int64_t);
12238		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x4)))
12239		svint8x4_t svldnt1_vnum_x4(svcount_t, int8_t const *, int64_t);
12240		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x4)))
12241		svmfloat8x4_t svldnt1_vnum_x4(svcount_t, mfloat8_t const *, int64_t);
12242		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x4)))
12243		svuint64x4_t svldnt1_vnum_x4(svcount_t, uint64_t const *, int64_t);
12244		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x4)))
12245		svfloat64x4_t svldnt1_vnum_x4(svcount_t, float64_t const *, int64_t);
12246		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x4)))
12247		svint64x4_t svldnt1_vnum_x4(svcount_t, int64_t const *, int64_t);
12248		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x4)))
12249		svuint16x4_t svldnt1_vnum_x4(svcount_t, uint16_t const *, int64_t);
12250		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x4)))
12251		svbfloat16x4_t svldnt1_vnum_x4(svcount_t, bfloat16_t const *, int64_t);
12252		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x4)))
12253		svfloat16x4_t svldnt1_vnum_x4(svcount_t, float16_t const *, int64_t);
12254		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x4)))
12255		svint16x4_t svldnt1_vnum_x4(svcount_t, int16_t const *, int64_t);
12256		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x4)))
12257		svuint32x4_t svldnt1_vnum_x4(svcount_t, uint32_t const *, int64_t);
12258		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x4)))
12259		svfloat32x4_t svldnt1_vnum_x4(svcount_t, float32_t const *, int64_t);
12260		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x4)))
12261		svint32x4_t svldnt1_vnum_x4(svcount_t, int32_t const *, int64_t);
12262		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_b)))
12263		svbool_t svreinterpret(svcount_t);
12264		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_c)))
12265		svcount_t svreinterpret(svbool_t);
12266		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x2)))
12267		void svst1(svcount_t, uint8_t *, svuint8x2_t);
12268		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x2)))
12269		void svst1(svcount_t, int8_t *, svint8x2_t);
12270		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x2)))
12271		void svst1(svcount_t, mfloat8_t *, svmfloat8x2_t);
12272		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x2)))
12273		void svst1(svcount_t, uint64_t *, svuint64x2_t);
12274		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x2)))
12275		void svst1(svcount_t, float64_t *, svfloat64x2_t);
12276		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x2)))
12277		void svst1(svcount_t, int64_t *, svint64x2_t);
12278		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x2)))
12279		void svst1(svcount_t, uint16_t *, svuint16x2_t);
12280		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x2)))
12281		void svst1(svcount_t, bfloat16_t *, svbfloat16x2_t);
12282		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x2)))
12283		void svst1(svcount_t, float16_t *, svfloat16x2_t);
12284		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x2)))
12285		void svst1(svcount_t, int16_t *, svint16x2_t);
12286		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x2)))
12287		void svst1(svcount_t, uint32_t *, svuint32x2_t);
12288		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x2)))
12289		void svst1(svcount_t, float32_t *, svfloat32x2_t);
12290		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x2)))
12291		void svst1(svcount_t, int32_t *, svint32x2_t);
12292		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x4)))
12293		void svst1(svcount_t, uint8_t *, svuint8x4_t);
12294		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x4)))
12295		void svst1(svcount_t, int8_t *, svint8x4_t);
12296		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x4)))
12297		void svst1(svcount_t, mfloat8_t *, svmfloat8x4_t);
12298		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x4)))
12299		void svst1(svcount_t, uint64_t *, svuint64x4_t);
12300		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x4)))
12301		void svst1(svcount_t, float64_t *, svfloat64x4_t);
12302		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x4)))
12303		void svst1(svcount_t, int64_t *, svint64x4_t);
12304		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x4)))
12305		void svst1(svcount_t, uint16_t *, svuint16x4_t);
12306		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x4)))
12307		void svst1(svcount_t, bfloat16_t *, svbfloat16x4_t);
12308		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x4)))
12309		void svst1(svcount_t, float16_t *, svfloat16x4_t);
12310		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x4)))
12311		void svst1(svcount_t, int16_t *, svint16x4_t);
12312		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x4)))
12313		void svst1(svcount_t, uint32_t *, svuint32x4_t);
12314		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x4)))
12315		void svst1(svcount_t, float32_t *, svfloat32x4_t);
12316		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x4)))
12317		void svst1(svcount_t, int32_t *, svint32x4_t);
12318		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x2)))
12319		void svst1_vnum(svcount_t, uint8_t *, int64_t, svuint8x2_t);
12320		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x2)))
12321		void svst1_vnum(svcount_t, int8_t *, int64_t, svint8x2_t);
12322		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x2)))
12323		void svst1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
12324		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x2)))
12325		void svst1_vnum(svcount_t, uint64_t *, int64_t, svuint64x2_t);
12326		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x2)))
12327		void svst1_vnum(svcount_t, float64_t *, int64_t, svfloat64x2_t);
12328		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x2)))
12329		void svst1_vnum(svcount_t, int64_t *, int64_t, svint64x2_t);
12330		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x2)))
12331		void svst1_vnum(svcount_t, uint16_t *, int64_t, svuint16x2_t);
12332		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x2)))
12333		void svst1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
12334		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x2)))
12335		void svst1_vnum(svcount_t, float16_t *, int64_t, svfloat16x2_t);
12336		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x2)))
12337		void svst1_vnum(svcount_t, int16_t *, int64_t, svint16x2_t);
12338		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x2)))
12339		void svst1_vnum(svcount_t, uint32_t *, int64_t, svuint32x2_t);
12340		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x2)))
12341		void svst1_vnum(svcount_t, float32_t *, int64_t, svfloat32x2_t);
12342		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x2)))
12343		void svst1_vnum(svcount_t, int32_t *, int64_t, svint32x2_t);
12344		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x4)))
12345		void svst1_vnum(svcount_t, uint8_t *, int64_t, svuint8x4_t);
12346		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x4)))
12347		void svst1_vnum(svcount_t, int8_t *, int64_t, svint8x4_t);
12348		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x4)))
12349		void svst1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
12350		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x4)))
12351		void svst1_vnum(svcount_t, uint64_t *, int64_t, svuint64x4_t);
12352		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x4)))
12353		void svst1_vnum(svcount_t, float64_t *, int64_t, svfloat64x4_t);
12354		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x4)))
12355		void svst1_vnum(svcount_t, int64_t *, int64_t, svint64x4_t);
12356		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x4)))
12357		void svst1_vnum(svcount_t, uint16_t *, int64_t, svuint16x4_t);
12358		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x4)))
12359		void svst1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
12360		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x4)))
12361		void svst1_vnum(svcount_t, float16_t *, int64_t, svfloat16x4_t);
12362		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x4)))
12363		void svst1_vnum(svcount_t, int16_t *, int64_t, svint16x4_t);
12364		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x4)))
12365		void svst1_vnum(svcount_t, uint32_t *, int64_t, svuint32x4_t);
12366		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x4)))
12367		void svst1_vnum(svcount_t, float32_t *, int64_t, svfloat32x4_t);
12368		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x4)))
12369		void svst1_vnum(svcount_t, int32_t *, int64_t, svint32x4_t);
12370		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x2)))
12371		void svstnt1(svcount_t, uint8_t *, svuint8x2_t);
12372		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x2)))
12373		void svstnt1(svcount_t, int8_t *, svint8x2_t);
12374		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x2)))
12375		void svstnt1(svcount_t, mfloat8_t *, svmfloat8x2_t);
12376		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x2)))
12377		void svstnt1(svcount_t, uint64_t *, svuint64x2_t);
12378		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x2)))
12379		void svstnt1(svcount_t, float64_t *, svfloat64x2_t);
12380		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x2)))
12381		void svstnt1(svcount_t, int64_t *, svint64x2_t);
12382		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x2)))
12383		void svstnt1(svcount_t, uint16_t *, svuint16x2_t);
12384		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x2)))
12385		void svstnt1(svcount_t, bfloat16_t *, svbfloat16x2_t);
12386		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x2)))
12387		void svstnt1(svcount_t, float16_t *, svfloat16x2_t);
12388		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x2)))
12389		void svstnt1(svcount_t, int16_t *, svint16x2_t);
12390		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x2)))
12391		void svstnt1(svcount_t, uint32_t *, svuint32x2_t);
12392		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x2)))
12393		void svstnt1(svcount_t, float32_t *, svfloat32x2_t);
12394		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x2)))
12395		void svstnt1(svcount_t, int32_t *, svint32x2_t);
12396		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x4)))
12397		void svstnt1(svcount_t, uint8_t *, svuint8x4_t);
12398		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x4)))
12399		void svstnt1(svcount_t, int8_t *, svint8x4_t);
12400		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x4)))
12401		void svstnt1(svcount_t, mfloat8_t *, svmfloat8x4_t);
12402		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x4)))
12403		void svstnt1(svcount_t, uint64_t *, svuint64x4_t);
12404		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x4)))
12405		void svstnt1(svcount_t, float64_t *, svfloat64x4_t);
12406		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x4)))
12407		void svstnt1(svcount_t, int64_t *, svint64x4_t);
12408		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x4)))
12409		void svstnt1(svcount_t, uint16_t *, svuint16x4_t);
12410		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x4)))
12411		void svstnt1(svcount_t, bfloat16_t *, svbfloat16x4_t);
12412		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x4)))
12413		void svstnt1(svcount_t, float16_t *, svfloat16x4_t);
12414		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x4)))
12415		void svstnt1(svcount_t, int16_t *, svint16x4_t);
12416		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x4)))
12417		void svstnt1(svcount_t, uint32_t *, svuint32x4_t);
12418		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x4)))
12419		void svstnt1(svcount_t, float32_t *, svfloat32x4_t);
12420		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x4)))
12421		void svstnt1(svcount_t, int32_t *, svint32x4_t);
12422		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x2)))
12423		void svstnt1_vnum(svcount_t, uint8_t *, int64_t, svuint8x2_t);
12424		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x2)))
12425		void svstnt1_vnum(svcount_t, int8_t *, int64_t, svint8x2_t);
12426		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x2)))
12427		void svstnt1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
12428		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x2)))
12429		void svstnt1_vnum(svcount_t, uint64_t *, int64_t, svuint64x2_t);
12430		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x2)))
12431		void svstnt1_vnum(svcount_t, float64_t *, int64_t, svfloat64x2_t);
12432		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x2)))
12433		void svstnt1_vnum(svcount_t, int64_t *, int64_t, svint64x2_t);
12434		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x2)))
12435		void svstnt1_vnum(svcount_t, uint16_t *, int64_t, svuint16x2_t);
12436		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x2)))
12437		void svstnt1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
12438		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x2)))
12439		void svstnt1_vnum(svcount_t, float16_t *, int64_t, svfloat16x2_t);
12440		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x2)))
12441		void svstnt1_vnum(svcount_t, int16_t *, int64_t, svint16x2_t);
12442		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x2)))
12443		void svstnt1_vnum(svcount_t, uint32_t *, int64_t, svuint32x2_t);
12444		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x2)))
12445		void svstnt1_vnum(svcount_t, float32_t *, int64_t, svfloat32x2_t);
12446		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x2)))
12447		void svstnt1_vnum(svcount_t, int32_t *, int64_t, svint32x2_t);
12448		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x4)))
12449		void svstnt1_vnum(svcount_t, uint8_t *, int64_t, svuint8x4_t);
12450		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x4)))
12451		void svstnt1_vnum(svcount_t, int8_t *, int64_t, svint8x4_t);
12452		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x4)))
12453		void svstnt1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
12454		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x4)))
12455		void svstnt1_vnum(svcount_t, uint64_t *, int64_t, svuint64x4_t);
12456		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x4)))
12457		void svstnt1_vnum(svcount_t, float64_t *, int64_t, svfloat64x4_t);
12458		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x4)))
12459		void svstnt1_vnum(svcount_t, int64_t *, int64_t, svint64x4_t);
12460		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x4)))
12461		void svstnt1_vnum(svcount_t, uint16_t *, int64_t, svuint16x4_t);
12462		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x4)))
12463		void svstnt1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
12464		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x4)))
12465		void svstnt1_vnum(svcount_t, float16_t *, int64_t, svfloat16x4_t);
12466		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x4)))
12467		void svstnt1_vnum(svcount_t, int16_t *, int64_t, svint16x4_t);
12468		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x4)))
12469		void svstnt1_vnum(svcount_t, uint32_t *, int64_t, svuint32x4_t);
12470		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x4)))
12471		void svstnt1_vnum(svcount_t, float32_t *, int64_t, svfloat32x4_t);
12472		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x4)))
12473		void svstnt1_vnum(svcount_t, int32_t *, int64_t, svint32x4_t);
12474		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_s64)))
12475		svcount_t svwhilege_c8(int64_t, int64_t, uint64_t);
12476		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_s64)))
12477		svcount_t svwhilege_c32(int64_t, int64_t, uint64_t);
12478		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_s64)))
12479		svcount_t svwhilege_c64(int64_t, int64_t, uint64_t);
12480		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_s64)))
12481		svcount_t svwhilege_c16(int64_t, int64_t, uint64_t);
12482		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_u64)))
12483		svcount_t svwhilege_c8(uint64_t, uint64_t, uint64_t);
12484		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_u64)))
12485		svcount_t svwhilege_c32(uint64_t, uint64_t, uint64_t);
12486		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_u64)))
12487		svcount_t svwhilege_c64(uint64_t, uint64_t, uint64_t);
12488		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_u64)))
12489		svcount_t svwhilege_c16(uint64_t, uint64_t, uint64_t);
12490		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_s64)))
12491		svcount_t svwhilegt_c8(int64_t, int64_t, uint64_t);
12492		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_s64)))
12493		svcount_t svwhilegt_c32(int64_t, int64_t, uint64_t);
12494		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_s64)))
12495		svcount_t svwhilegt_c64(int64_t, int64_t, uint64_t);
12496		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_s64)))
12497		svcount_t svwhilegt_c16(int64_t, int64_t, uint64_t);
12498		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_u64)))
12499		svcount_t svwhilegt_c8(uint64_t, uint64_t, uint64_t);
12500		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_u64)))
12501		svcount_t svwhilegt_c32(uint64_t, uint64_t, uint64_t);
12502		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_u64)))
12503		svcount_t svwhilegt_c64(uint64_t, uint64_t, uint64_t);
12504		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_u64)))
12505		svcount_t svwhilegt_c16(uint64_t, uint64_t, uint64_t);
12506		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_s64)))
12507		svcount_t svwhilele_c8(int64_t, int64_t, uint64_t);
12508		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_s64)))
12509		svcount_t svwhilele_c32(int64_t, int64_t, uint64_t);
12510		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_s64)))
12511		svcount_t svwhilele_c64(int64_t, int64_t, uint64_t);
12512		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_s64)))
12513		svcount_t svwhilele_c16(int64_t, int64_t, uint64_t);
12514		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_u64)))
12515		svcount_t svwhilele_c8(uint64_t, uint64_t, uint64_t);
12516		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_u64)))
12517		svcount_t svwhilele_c32(uint64_t, uint64_t, uint64_t);
12518		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_u64)))
12519		svcount_t svwhilele_c64(uint64_t, uint64_t, uint64_t);
12520		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_u64)))
12521		svcount_t svwhilele_c16(uint64_t, uint64_t, uint64_t);
12522		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_u64)))
12523		svcount_t svwhilelt_c8(uint64_t, uint64_t, uint64_t);
12524		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_u64)))
12525		svcount_t svwhilelt_c32(uint64_t, uint64_t, uint64_t);
12526		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_u64)))
12527		svcount_t svwhilelt_c64(uint64_t, uint64_t, uint64_t);
12528		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_u64)))
12529		svcount_t svwhilelt_c16(uint64_t, uint64_t, uint64_t);
12530		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_s64)))
12531		svcount_t svwhilelt_c8(int64_t, int64_t, uint64_t);
12532		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_s64)))
12533		svcount_t svwhilelt_c32(int64_t, int64_t, uint64_t);
12534		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64)))
12535		svcount_t svwhilelt_c64(int64_t, int64_t, uint64_t);
12536		__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64)))
12537		svcount_t svwhilelt_c16(int64_t, int64_t, uint64_t);
	8836	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt_f32_f16_z)))
	8837	svfloat32_t svcvtlt_f32_f16_z(svbool_t, svfloat16_t);
	8838	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt_f64_f32_z)))
	8839	svfloat64_t svcvtlt_f64_f32_z(svbool_t, svfloat32_t);
	8840	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_z)))
	8841	svbfloat16_t svcvtnt_bf16_f32_z(svbfloat16_t, svbool_t, svfloat32_t);
	8842	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_f16_f32_z)))
	8843	svfloat16_t svcvtnt_f16_f32_z(svfloat16_t, svbool_t, svfloat32_t);
	8844	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_f32_f64_z)))
	8845	svfloat32_t svcvtnt_f32_f64_z(svfloat32_t, svbool_t, svfloat64_t);
	8846	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtxnt_f32_f64_z)))
	8847	svfloat32_t svcvtxnt_f32_f64_z(svfloat32_t, svbool_t, svfloat64_t);
	8848	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svfirstp_b8)))
	8849	int64_t svfirstp_b8(svbool_t, svbool_t);
	8850	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svfirstp_b32)))
	8851	int64_t svfirstp_b32(svbool_t, svbool_t);
	8852	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svfirstp_b64)))
	8853	int64_t svfirstp_b64(svbool_t, svbool_t);
	8854	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svfirstp_b16)))
	8855	int64_t svfirstp_b16(svbool_t, svbool_t);
	8856	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svlastp_b8)))
	8857	int64_t svlastp_b8(svbool_t, svbool_t);
	8858	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svlastp_b32)))
	8859	int64_t svlastp_b32(svbool_t, svbool_t);
	8860	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svlastp_b64)))
	8861	int64_t svlastp_b64(svbool_t, svbool_t);
	8862	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svlastp_b16)))
	8863	int64_t svlastp_b16(svbool_t, svbool_t);
	8864	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_m)))
	8865	svfloat64_t svrint32x_f64_m(svfloat64_t, svbool_t, svfloat64_t);
	8866	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_m)))
	8867	svfloat32_t svrint32x_f32_m(svfloat32_t, svbool_t, svfloat32_t);
	8868	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_x)))
	8869	svfloat64_t svrint32x_f64_x(svbool_t, svfloat64_t);
	8870	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_x)))
	8871	svfloat32_t svrint32x_f32_x(svbool_t, svfloat32_t);
	8872	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_z)))
	8873	svfloat64_t svrint32x_f64_z(svbool_t, svfloat64_t);
	8874	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_z)))
	8875	svfloat32_t svrint32x_f32_z(svbool_t, svfloat32_t);
	8876	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_m)))
	8877	svfloat64_t svrint32z_f64_m(svfloat64_t, svbool_t, svfloat64_t);
	8878	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_m)))
	8879	svfloat32_t svrint32z_f32_m(svfloat32_t, svbool_t, svfloat32_t);
	8880	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_x)))
	8881	svfloat64_t svrint32z_f64_x(svbool_t, svfloat64_t);
	8882	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_x)))
	8883	svfloat32_t svrint32z_f32_x(svbool_t, svfloat32_t);
	8884	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_z)))
	8885	svfloat64_t svrint32z_f64_z(svbool_t, svfloat64_t);
	8886	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_z)))
	8887	svfloat32_t svrint32z_f32_z(svbool_t, svfloat32_t);
	8888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_m)))
	8889	svfloat64_t svrint64x_f64_m(svfloat64_t, svbool_t, svfloat64_t);
	8890	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_m)))
	8891	svfloat32_t svrint64x_f32_m(svfloat32_t, svbool_t, svfloat32_t);
	8892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_x)))
	8893	svfloat64_t svrint64x_f64_x(svbool_t, svfloat64_t);
	8894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_x)))
	8895	svfloat32_t svrint64x_f32_x(svbool_t, svfloat32_t);
	8896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_z)))
	8897	svfloat64_t svrint64x_f64_z(svbool_t, svfloat64_t);
	8898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_z)))
	8899	svfloat32_t svrint64x_f32_z(svbool_t, svfloat32_t);
	8900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_m)))
	8901	svfloat64_t svrint64z_f64_m(svfloat64_t, svbool_t, svfloat64_t);
	8902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_m)))
	8903	svfloat32_t svrint64z_f32_m(svfloat32_t, svbool_t, svfloat32_t);
	8904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_x)))
	8905	svfloat64_t svrint64z_f64_x(svbool_t, svfloat64_t);
	8906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_x)))
	8907	svfloat32_t svrint64z_f32_x(svbool_t, svfloat32_t);
	8908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_z)))
	8909	svfloat64_t svrint64z_f64_z(svbool_t, svfloat64_t);
	8910	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_z)))
	8911	svfloat32_t svrint64z_f32_z(svbool_t, svfloat32_t);
	8912	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt_f32_f16_z)))
	8913	svfloat32_t svcvtlt_f32_z(svbool_t, svfloat16_t);
	8914	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt_f64_f32_z)))
	8915	svfloat64_t svcvtlt_f64_z(svbool_t, svfloat32_t);
	8916	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_z)))
	8917	svbfloat16_t svcvtnt_bf16_z(svbfloat16_t, svbool_t, svfloat32_t);
	8918	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_f16_f32_z)))
	8919	svfloat16_t svcvtnt_f16_z(svfloat16_t, svbool_t, svfloat32_t);
	8920	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_f32_f64_z)))
	8921	svfloat32_t svcvtnt_f32_z(svfloat32_t, svbool_t, svfloat64_t);
	8922	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtxnt_f32_f64_z)))
	8923	svfloat32_t svcvtxnt_f32_z(svfloat32_t, svbool_t, svfloat64_t);
	8924	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_m)))
	8925	svfloat64_t svrint32x_m(svfloat64_t, svbool_t, svfloat64_t);
	8926	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_m)))
	8927	svfloat32_t svrint32x_m(svfloat32_t, svbool_t, svfloat32_t);
	8928	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_x)))
	8929	svfloat64_t svrint32x_x(svbool_t, svfloat64_t);
	8930	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_x)))
	8931	svfloat32_t svrint32x_x(svbool_t, svfloat32_t);
	8932	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f64_z)))
	8933	svfloat64_t svrint32x_z(svbool_t, svfloat64_t);
	8934	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32x_f32_z)))
	8935	svfloat32_t svrint32x_z(svbool_t, svfloat32_t);
	8936	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_m)))
	8937	svfloat64_t svrint32z_m(svfloat64_t, svbool_t, svfloat64_t);
	8938	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_m)))
	8939	svfloat32_t svrint32z_m(svfloat32_t, svbool_t, svfloat32_t);
	8940	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_x)))
	8941	svfloat64_t svrint32z_x(svbool_t, svfloat64_t);
	8942	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_x)))
	8943	svfloat32_t svrint32z_x(svbool_t, svfloat32_t);
	8944	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f64_z)))
	8945	svfloat64_t svrint32z_z(svbool_t, svfloat64_t);
	8946	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint32z_f32_z)))
	8947	svfloat32_t svrint32z_z(svbool_t, svfloat32_t);
	8948	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_m)))
	8949	svfloat64_t svrint64x_m(svfloat64_t, svbool_t, svfloat64_t);
	8950	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_m)))
	8951	svfloat32_t svrint64x_m(svfloat32_t, svbool_t, svfloat32_t);
	8952	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_x)))
	8953	svfloat64_t svrint64x_x(svbool_t, svfloat64_t);
	8954	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_x)))
	8955	svfloat32_t svrint64x_x(svbool_t, svfloat32_t);
	8956	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f64_z)))
	8957	svfloat64_t svrint64x_z(svbool_t, svfloat64_t);
	8958	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64x_f32_z)))
	8959	svfloat32_t svrint64x_z(svbool_t, svfloat32_t);
	8960	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_m)))
	8961	svfloat64_t svrint64z_m(svfloat64_t, svbool_t, svfloat64_t);
	8962	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_m)))
	8963	svfloat32_t svrint64z_m(svfloat32_t, svbool_t, svfloat32_t);
	8964	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_x)))
	8965	svfloat64_t svrint64z_x(svbool_t, svfloat64_t);
	8966	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_x)))
	8967	svfloat32_t svrint64z_x(svbool_t, svfloat32_t);
	8968	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f64_z)))
	8969	svfloat64_t svrint64z_z(svbool_t, svfloat64_t);
	8970	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrint64z_f32_z)))
	8971	svfloat32_t svrint64z_z(svbool_t, svfloat32_t);
	8972	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u8)))
	8973	svuint8_t svcompact_u8(svbool_t, svuint8_t);
	8974	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u16)))
	8975	svuint16_t svcompact_u16(svbool_t, svuint16_t);
	8976	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_bf16)))
	8977	svbfloat16_t svcompact_bf16(svbool_t, svbfloat16_t);
	8978	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s8)))
	8979	svint8_t svcompact_s8(svbool_t, svint8_t);
	8980	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f16)))
	8981	svfloat16_t svcompact_f16(svbool_t, svfloat16_t);
	8982	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_mf8)))
	8983	svmfloat8_t svcompact_mf8(svbool_t, svmfloat8_t);
	8984	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s16)))
	8985	svint16_t svcompact_s16(svbool_t, svint16_t);
	8986	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u8)))
	8987	svuint8_t svexpand_u8(svbool_t, svuint8_t);
	8988	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u32)))
	8989	svuint32_t svexpand_u32(svbool_t, svuint32_t);
	8990	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u64)))
	8991	svuint64_t svexpand_u64(svbool_t, svuint64_t);
	8992	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u16)))
	8993	svuint16_t svexpand_u16(svbool_t, svuint16_t);
	8994	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_bf16)))
	8995	svbfloat16_t svexpand_bf16(svbool_t, svbfloat16_t);
	8996	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s8)))
	8997	svint8_t svexpand_s8(svbool_t, svint8_t);
	8998	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f64)))
	8999	svfloat64_t svexpand_f64(svbool_t, svfloat64_t);
	9000	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f32)))
	9001	svfloat32_t svexpand_f32(svbool_t, svfloat32_t);
	9002	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f16)))
	9003	svfloat16_t svexpand_f16(svbool_t, svfloat16_t);
	9004	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s32)))
	9005	svint32_t svexpand_s32(svbool_t, svint32_t);
	9006	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s64)))
	9007	svint64_t svexpand_s64(svbool_t, svint64_t);
	9008	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_mf8)))
	9009	svmfloat8_t svexpand_mf8(svbool_t, svmfloat8_t);
	9010	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s16)))
	9011	svint16_t svexpand_s16(svbool_t, svint16_t);
	9012	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u8)))
	9013	svuint8_t svcompact(svbool_t, svuint8_t);
	9014	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u16)))
	9015	svuint16_t svcompact(svbool_t, svuint16_t);
	9016	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_bf16)))
	9017	svbfloat16_t svcompact(svbool_t, svbfloat16_t);
	9018	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s8)))
	9019	svint8_t svcompact(svbool_t, svint8_t);
	9020	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f16)))
	9021	svfloat16_t svcompact(svbool_t, svfloat16_t);
	9022	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_mf8)))
	9023	svmfloat8_t svcompact(svbool_t, svmfloat8_t);
	9024	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s16)))
	9025	svint16_t svcompact(svbool_t, svint16_t);
	9026	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u8)))
	9027	svuint8_t svexpand(svbool_t, svuint8_t);
	9028	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u32)))
	9029	svuint32_t svexpand(svbool_t, svuint32_t);
	9030	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u64)))
	9031	svuint64_t svexpand(svbool_t, svuint64_t);
	9032	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_u16)))
	9033	svuint16_t svexpand(svbool_t, svuint16_t);
	9034	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_bf16)))
	9035	svbfloat16_t svexpand(svbool_t, svbfloat16_t);
	9036	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s8)))
	9037	svint8_t svexpand(svbool_t, svint8_t);
	9038	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f64)))
	9039	svfloat64_t svexpand(svbool_t, svfloat64_t);
	9040	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f32)))
	9041	svfloat32_t svexpand(svbool_t, svfloat32_t);
	9042	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_f16)))
	9043	svfloat16_t svexpand(svbool_t, svfloat16_t);
	9044	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s32)))
	9045	svint32_t svexpand(svbool_t, svint32_t);
	9046	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s64)))
	9047	svint64_t svexpand(svbool_t, svint64_t);
	9048	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_mf8)))
	9049	svmfloat8_t svexpand(svbool_t, svmfloat8_t);
	9050	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpand_s16)))
	9051	svint16_t svexpand(svbool_t, svint16_t);
12538	9052	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s8)))
12539	9053	svint8_t svaba_n_s8(svint8_t, svint8_t, int8_t);
12540	9054	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaba_n_s32)))
...	...	@@ -19059,6 +15573,3848 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svxar_n_s64)))
19059	15573	svint64_t svxar(svint64_t, svint64_t, uint64_t);
19060	15574	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svxar_n_s16)))
19061	15575	svint16_t svxar(svint16_t, svint16_t, uint64_t);
	15576	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
	15577	svfloat64_t svamax_n_f64_m(svbool_t, svfloat64_t, float64_t);
	15578	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
	15579	svfloat32_t svamax_n_f32_m(svbool_t, svfloat32_t, float32_t);
	15580	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
	15581	svfloat16_t svamax_n_f16_m(svbool_t, svfloat16_t, float16_t);
	15582	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
	15583	svfloat64_t svamax_n_f64_x(svbool_t, svfloat64_t, float64_t);
	15584	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
	15585	svfloat32_t svamax_n_f32_x(svbool_t, svfloat32_t, float32_t);
	15586	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
	15587	svfloat16_t svamax_n_f16_x(svbool_t, svfloat16_t, float16_t);
	15588	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
	15589	svfloat64_t svamax_n_f64_z(svbool_t, svfloat64_t, float64_t);
	15590	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
	15591	svfloat32_t svamax_n_f32_z(svbool_t, svfloat32_t, float32_t);
	15592	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
	15593	svfloat16_t svamax_n_f16_z(svbool_t, svfloat16_t, float16_t);
	15594	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
	15595	svfloat64_t svamax_f64_m(svbool_t, svfloat64_t, svfloat64_t);
	15596	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
	15597	svfloat32_t svamax_f32_m(svbool_t, svfloat32_t, svfloat32_t);
	15598	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
	15599	svfloat16_t svamax_f16_m(svbool_t, svfloat16_t, svfloat16_t);
	15600	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
	15601	svfloat64_t svamax_f64_x(svbool_t, svfloat64_t, svfloat64_t);
	15602	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
	15603	svfloat32_t svamax_f32_x(svbool_t, svfloat32_t, svfloat32_t);
	15604	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
	15605	svfloat16_t svamax_f16_x(svbool_t, svfloat16_t, svfloat16_t);
	15606	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
	15607	svfloat64_t svamax_f64_z(svbool_t, svfloat64_t, svfloat64_t);
	15608	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
	15609	svfloat32_t svamax_f32_z(svbool_t, svfloat32_t, svfloat32_t);
	15610	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
	15611	svfloat16_t svamax_f16_z(svbool_t, svfloat16_t, svfloat16_t);
	15612	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
	15613	svfloat64_t svamin_n_f64_m(svbool_t, svfloat64_t, float64_t);
	15614	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
	15615	svfloat32_t svamin_n_f32_m(svbool_t, svfloat32_t, float32_t);
	15616	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
	15617	svfloat16_t svamin_n_f16_m(svbool_t, svfloat16_t, float16_t);
	15618	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
	15619	svfloat64_t svamin_n_f64_x(svbool_t, svfloat64_t, float64_t);
	15620	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
	15621	svfloat32_t svamin_n_f32_x(svbool_t, svfloat32_t, float32_t);
	15622	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
	15623	svfloat16_t svamin_n_f16_x(svbool_t, svfloat16_t, float16_t);
	15624	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
	15625	svfloat64_t svamin_n_f64_z(svbool_t, svfloat64_t, float64_t);
	15626	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
	15627	svfloat32_t svamin_n_f32_z(svbool_t, svfloat32_t, float32_t);
	15628	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
	15629	svfloat16_t svamin_n_f16_z(svbool_t, svfloat16_t, float16_t);
	15630	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
	15631	svfloat64_t svamin_f64_m(svbool_t, svfloat64_t, svfloat64_t);
	15632	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
	15633	svfloat32_t svamin_f32_m(svbool_t, svfloat32_t, svfloat32_t);
	15634	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
	15635	svfloat16_t svamin_f16_m(svbool_t, svfloat16_t, svfloat16_t);
	15636	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
	15637	svfloat64_t svamin_f64_x(svbool_t, svfloat64_t, svfloat64_t);
	15638	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
	15639	svfloat32_t svamin_f32_x(svbool_t, svfloat32_t, svfloat32_t);
	15640	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
	15641	svfloat16_t svamin_f16_x(svbool_t, svfloat16_t, svfloat16_t);
	15642	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
	15643	svfloat64_t svamin_f64_z(svbool_t, svfloat64_t, svfloat64_t);
	15644	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
	15645	svfloat32_t svamin_f32_z(svbool_t, svfloat32_t, svfloat32_t);
	15646	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
	15647	svfloat16_t svamin_f16_z(svbool_t, svfloat16_t, svfloat16_t);
	15648	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_m)))
	15649	svfloat64_t svamax_m(svbool_t, svfloat64_t, float64_t);
	15650	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_m)))
	15651	svfloat32_t svamax_m(svbool_t, svfloat32_t, float32_t);
	15652	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_m)))
	15653	svfloat16_t svamax_m(svbool_t, svfloat16_t, float16_t);
	15654	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_x)))
	15655	svfloat64_t svamax_x(svbool_t, svfloat64_t, float64_t);
	15656	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_x)))
	15657	svfloat32_t svamax_x(svbool_t, svfloat32_t, float32_t);
	15658	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_x)))
	15659	svfloat16_t svamax_x(svbool_t, svfloat16_t, float16_t);
	15660	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f64_z)))
	15661	svfloat64_t svamax_z(svbool_t, svfloat64_t, float64_t);
	15662	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f32_z)))
	15663	svfloat32_t svamax_z(svbool_t, svfloat32_t, float32_t);
	15664	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_n_f16_z)))
	15665	svfloat16_t svamax_z(svbool_t, svfloat16_t, float16_t);
	15666	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_m)))
	15667	svfloat64_t svamax_m(svbool_t, svfloat64_t, svfloat64_t);
	15668	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_m)))
	15669	svfloat32_t svamax_m(svbool_t, svfloat32_t, svfloat32_t);
	15670	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_m)))
	15671	svfloat16_t svamax_m(svbool_t, svfloat16_t, svfloat16_t);
	15672	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_x)))
	15673	svfloat64_t svamax_x(svbool_t, svfloat64_t, svfloat64_t);
	15674	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_x)))
	15675	svfloat32_t svamax_x(svbool_t, svfloat32_t, svfloat32_t);
	15676	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_x)))
	15677	svfloat16_t svamax_x(svbool_t, svfloat16_t, svfloat16_t);
	15678	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f64_z)))
	15679	svfloat64_t svamax_z(svbool_t, svfloat64_t, svfloat64_t);
	15680	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f32_z)))
	15681	svfloat32_t svamax_z(svbool_t, svfloat32_t, svfloat32_t);
	15682	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamax_f16_z)))
	15683	svfloat16_t svamax_z(svbool_t, svfloat16_t, svfloat16_t);
	15684	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_m)))
	15685	svfloat64_t svamin_m(svbool_t, svfloat64_t, float64_t);
	15686	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_m)))
	15687	svfloat32_t svamin_m(svbool_t, svfloat32_t, float32_t);
	15688	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_m)))
	15689	svfloat16_t svamin_m(svbool_t, svfloat16_t, float16_t);
	15690	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_x)))
	15691	svfloat64_t svamin_x(svbool_t, svfloat64_t, float64_t);
	15692	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_x)))
	15693	svfloat32_t svamin_x(svbool_t, svfloat32_t, float32_t);
	15694	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_x)))
	15695	svfloat16_t svamin_x(svbool_t, svfloat16_t, float16_t);
	15696	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f64_z)))
	15697	svfloat64_t svamin_z(svbool_t, svfloat64_t, float64_t);
	15698	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f32_z)))
	15699	svfloat32_t svamin_z(svbool_t, svfloat32_t, float32_t);
	15700	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_n_f16_z)))
	15701	svfloat16_t svamin_z(svbool_t, svfloat16_t, float16_t);
	15702	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_m)))
	15703	svfloat64_t svamin_m(svbool_t, svfloat64_t, svfloat64_t);
	15704	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_m)))
	15705	svfloat32_t svamin_m(svbool_t, svfloat32_t, svfloat32_t);
	15706	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_m)))
	15707	svfloat16_t svamin_m(svbool_t, svfloat16_t, svfloat16_t);
	15708	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_x)))
	15709	svfloat64_t svamin_x(svbool_t, svfloat64_t, svfloat64_t);
	15710	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_x)))
	15711	svfloat32_t svamin_x(svbool_t, svfloat32_t, svfloat32_t);
	15712	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_x)))
	15713	svfloat16_t svamin_x(svbool_t, svfloat16_t, svfloat16_t);
	15714	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f64_z)))
	15715	svfloat64_t svamin_z(svbool_t, svfloat64_t, svfloat64_t);
	15716	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f32_z)))
	15717	svfloat32_t svamin_z(svbool_t, svfloat32_t, svfloat32_t);
	15718	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svamin_f16_z)))
	15719	svfloat16_t svamin_z(svbool_t, svfloat16_t, svfloat16_t);
	15720	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
	15721	svbfloat16_t svcvt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
	15722	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
	15723	svfloat16_t svcvt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
	15724	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
	15725	svbfloat16_t svcvt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
	15726	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
	15727	svfloat16_t svcvt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
	15728	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
	15729	svbfloat16_t svcvtlt1_bf16_mf8_fpm(svmfloat8_t, fpm_t);
	15730	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
	15731	svfloat16_t svcvtlt1_f16_mf8_fpm(svmfloat8_t, fpm_t);
	15732	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
	15733	svbfloat16_t svcvtlt2_bf16_mf8_fpm(svmfloat8_t, fpm_t);
	15734	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
	15735	svfloat16_t svcvtlt2_f16_mf8_fpm(svmfloat8_t, fpm_t);
	15736	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
	15737	svmfloat8_t svcvtn_mf8_bf16_x2_fpm(svbfloat16x2_t, fpm_t);
	15738	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
	15739	svmfloat8_t svcvtn_mf8_f16_x2_fpm(svfloat16x2_t, fpm_t);
	15740	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
	15741	svmfloat8_t svcvtnb_mf8_f32_x2_fpm(svfloat32x2_t, fpm_t);
	15742	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
	15743	svmfloat8_t svcvtnt_mf8_f32_x2_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
	15744	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_bf16_mf8_fpm)))
	15745	svbfloat16_t svcvt1_bf16_fpm(svmfloat8_t, fpm_t);
	15746	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt1_f16_mf8_fpm)))
	15747	svfloat16_t svcvt1_f16_fpm(svmfloat8_t, fpm_t);
	15748	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_bf16_mf8_fpm)))
	15749	svbfloat16_t svcvt2_bf16_fpm(svmfloat8_t, fpm_t);
	15750	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt2_f16_mf8_fpm)))
	15751	svfloat16_t svcvt2_f16_fpm(svmfloat8_t, fpm_t);
	15752	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_bf16_mf8_fpm)))
	15753	svbfloat16_t svcvtlt1_bf16_fpm(svmfloat8_t, fpm_t);
	15754	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt1_f16_mf8_fpm)))
	15755	svfloat16_t svcvtlt1_f16_fpm(svmfloat8_t, fpm_t);
	15756	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_bf16_mf8_fpm)))
	15757	svbfloat16_t svcvtlt2_bf16_fpm(svmfloat8_t, fpm_t);
	15758	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtlt2_f16_mf8_fpm)))
	15759	svfloat16_t svcvtlt2_f16_fpm(svmfloat8_t, fpm_t);
	15760	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_bf16_x2_fpm)))
	15761	svmfloat8_t svcvtn_mf8_fpm(svbfloat16x2_t, fpm_t);
	15762	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtn_mf8_f16_x2_fpm)))
	15763	svmfloat8_t svcvtn_mf8_fpm(svfloat16x2_t, fpm_t);
	15764	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnb_mf8_f32_x2_fpm)))
	15765	svmfloat8_t svcvtnb_mf8_fpm(svfloat32x2_t, fpm_t);
	15766	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_mf8_f32_x2_fpm)))
	15767	svmfloat8_t svcvtnt_mf8_fpm(svmfloat8_t, svfloat32x2_t, fpm_t);
	15768	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
	15769	svuint8_t svluti2_lane_u8(svuint8_t, svuint8_t, uint64_t);
	15770	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
	15771	svint8_t svluti2_lane_s8(svint8_t, svuint8_t, uint64_t);
	15772	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
	15773	svuint16_t svluti2_lane_u16(svuint16_t, svuint8_t, uint64_t);
	15774	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
	15775	svbfloat16_t svluti2_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
	15776	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
	15777	svfloat16_t svluti2_lane_f16(svfloat16_t, svuint8_t, uint64_t);
	15778	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
	15779	svint16_t svluti2_lane_s16(svint16_t, svuint8_t, uint64_t);
	15780	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
	15781	svuint8_t svluti4_lane_u8(svuint8_t, svuint8_t, uint64_t);
	15782	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
	15783	svint8_t svluti4_lane_s8(svint8_t, svuint8_t, uint64_t);
	15784	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
	15785	svuint16_t svluti4_lane_u16(svuint16_t, svuint8_t, uint64_t);
	15786	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
	15787	svbfloat16_t svluti4_lane_bf16(svbfloat16_t, svuint8_t, uint64_t);
	15788	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
	15789	svfloat16_t svluti4_lane_f16(svfloat16_t, svuint8_t, uint64_t);
	15790	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
	15791	svint16_t svluti4_lane_s16(svint16_t, svuint8_t, uint64_t);
	15792	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
	15793	svuint16_t svluti4_lane_u16_x2(svuint16x2_t, svuint8_t, uint64_t);
	15794	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
	15795	svbfloat16_t svluti4_lane_bf16_x2(svbfloat16x2_t, svuint8_t, uint64_t);
	15796	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
	15797	svfloat16_t svluti4_lane_f16_x2(svfloat16x2_t, svuint8_t, uint64_t);
	15798	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
	15799	svint16_t svluti4_lane_s16_x2(svint16x2_t, svuint8_t, uint64_t);
	15800	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u8)))
	15801	svuint8_t svluti2_lane(svuint8_t, svuint8_t, uint64_t);
	15802	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s8)))
	15803	svint8_t svluti2_lane(svint8_t, svuint8_t, uint64_t);
	15804	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_u16)))
	15805	svuint16_t svluti2_lane(svuint16_t, svuint8_t, uint64_t);
	15806	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_bf16)))
	15807	svbfloat16_t svluti2_lane(svbfloat16_t, svuint8_t, uint64_t);
	15808	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_f16)))
	15809	svfloat16_t svluti2_lane(svfloat16_t, svuint8_t, uint64_t);
	15810	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti2_lane_s16)))
	15811	svint16_t svluti2_lane(svint16_t, svuint8_t, uint64_t);
	15812	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u8)))
	15813	svuint8_t svluti4_lane(svuint8_t, svuint8_t, uint64_t);
	15814	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s8)))
	15815	svint8_t svluti4_lane(svint8_t, svuint8_t, uint64_t);
	15816	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16)))
	15817	svuint16_t svluti4_lane(svuint16_t, svuint8_t, uint64_t);
	15818	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16)))
	15819	svbfloat16_t svluti4_lane(svbfloat16_t, svuint8_t, uint64_t);
	15820	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16)))
	15821	svfloat16_t svluti4_lane(svfloat16_t, svuint8_t, uint64_t);
	15822	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16)))
	15823	svint16_t svluti4_lane(svint16_t, svuint8_t, uint64_t);
	15824	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_u16_x2)))
	15825	svuint16_t svluti4_lane(svuint16x2_t, svuint8_t, uint64_t);
	15826	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_bf16_x2)))
	15827	svbfloat16_t svluti4_lane(svbfloat16x2_t, svuint8_t, uint64_t);
	15828	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_f16_x2)))
	15829	svfloat16_t svluti4_lane(svfloat16x2_t, svuint8_t, uint64_t);
	15830	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svluti4_lane_s16_x2)))
	15831	svint16_t svluti4_lane(svint16x2_t, svuint8_t, uint64_t);
	15832	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmmla_f32)))
	15833	svfloat32_t svbfmmla_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15834	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmmla_f32)))
	15835	svfloat32_t svbfmmla(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15836	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_n_f32)))
	15837	svfloat32_t svbfdot_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
	15838	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_f32)))
	15839	svfloat32_t svbfdot_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15840	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_lane_f32)))
	15841	svfloat32_t svbfdot_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15842	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_n_f32)))
	15843	svfloat32_t svbfmlalb_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
	15844	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_f32)))
	15845	svfloat32_t svbfmlalb_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15846	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_lane_f32)))
	15847	svfloat32_t svbfmlalb_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15848	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_n_f32)))
	15849	svfloat32_t svbfmlalt_n_f32(svfloat32_t, svbfloat16_t, bfloat16_t);
	15850	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_f32)))
	15851	svfloat32_t svbfmlalt_f32(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15852	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_lane_f32)))
	15853	svfloat32_t svbfmlalt_lane_f32(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15854	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_m)))
	15855	svbfloat16_t svcvt_bf16_f32_m(svbfloat16_t, svbool_t, svfloat32_t);
	15856	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_x)))
	15857	svbfloat16_t svcvt_bf16_f32_x(svbool_t, svfloat32_t);
	15858	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_z)))
	15859	svbfloat16_t svcvt_bf16_f32_z(svbool_t, svfloat32_t);
	15860	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_m)))
	15861	svbfloat16_t svcvtnt_bf16_f32_m(svbfloat16_t, svbool_t, svfloat32_t);
	15862	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_n_f32)))
	15863	svfloat32_t svbfdot(svfloat32_t, svbfloat16_t, bfloat16_t);
	15864	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_f32)))
	15865	svfloat32_t svbfdot(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15866	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfdot_lane_f32)))
	15867	svfloat32_t svbfdot_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15868	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_n_f32)))
	15869	svfloat32_t svbfmlalb(svfloat32_t, svbfloat16_t, bfloat16_t);
	15870	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_f32)))
	15871	svfloat32_t svbfmlalb(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15872	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalb_lane_f32)))
	15873	svfloat32_t svbfmlalb_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15874	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_n_f32)))
	15875	svfloat32_t svbfmlalt(svfloat32_t, svbfloat16_t, bfloat16_t);
	15876	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_f32)))
	15877	svfloat32_t svbfmlalt(svfloat32_t, svbfloat16_t, svbfloat16_t);
	15878	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbfmlalt_lane_f32)))
	15879	svfloat32_t svbfmlalt_lane(svfloat32_t, svbfloat16_t, svbfloat16_t, uint64_t);
	15880	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_m)))
	15881	svbfloat16_t svcvt_bf16_m(svbfloat16_t, svbool_t, svfloat32_t);
	15882	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_x)))
	15883	svbfloat16_t svcvt_bf16_x(svbool_t, svfloat32_t);
	15884	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvt_bf16_f32_z)))
	15885	svbfloat16_t svcvt_bf16_z(svbool_t, svfloat32_t);
	15886	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcvtnt_bf16_f32_m)))
	15887	svbfloat16_t svcvtnt_bf16_m(svbfloat16_t, svbool_t, svfloat32_t);
	15888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32)))
	15889	svfloat32_t svmmla_f32(svfloat32_t, svfloat32_t, svfloat32_t);
	15890	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32)))
	15891	svfloat32_t svmmla(svfloat32_t, svfloat32_t, svfloat32_t);
	15892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u8)))
	15893	svuint8_t svld1ro_u8(svbool_t, uint8_t const *);
	15894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u32)))
	15895	svuint32_t svld1ro_u32(svbool_t, uint32_t const *);
	15896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u64)))
	15897	svuint64_t svld1ro_u64(svbool_t, uint64_t const *);
	15898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u16)))
	15899	svuint16_t svld1ro_u16(svbool_t, uint16_t const *);
	15900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_bf16)))
	15901	svbfloat16_t svld1ro_bf16(svbool_t, bfloat16_t const *);
	15902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s8)))
	15903	svint8_t svld1ro_s8(svbool_t, int8_t const *);
	15904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f64)))
	15905	svfloat64_t svld1ro_f64(svbool_t, float64_t const *);
	15906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f32)))
	15907	svfloat32_t svld1ro_f32(svbool_t, float32_t const *);
	15908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f16)))
	15909	svfloat16_t svld1ro_f16(svbool_t, float16_t const *);
	15910	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s32)))
	15911	svint32_t svld1ro_s32(svbool_t, int32_t const *);
	15912	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s64)))
	15913	svint64_t svld1ro_s64(svbool_t, int64_t const *);
	15914	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_mf8)))
	15915	svmfloat8_t svld1ro_mf8(svbool_t, mfloat8_t const *);
	15916	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s16)))
	15917	svint16_t svld1ro_s16(svbool_t, int16_t const *);
	15918	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f64)))
	15919	svfloat64_t svmmla_f64(svfloat64_t, svfloat64_t, svfloat64_t);
	15920	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u8)))
	15921	svuint8_t svtrn1q_u8(svuint8_t, svuint8_t);
	15922	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u32)))
	15923	svuint32_t svtrn1q_u32(svuint32_t, svuint32_t);
	15924	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u64)))
	15925	svuint64_t svtrn1q_u64(svuint64_t, svuint64_t);
	15926	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u16)))
	15927	svuint16_t svtrn1q_u16(svuint16_t, svuint16_t);
	15928	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_bf16)))
	15929	svbfloat16_t svtrn1q_bf16(svbfloat16_t, svbfloat16_t);
	15930	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s8)))
	15931	svint8_t svtrn1q_s8(svint8_t, svint8_t);
	15932	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f64)))
	15933	svfloat64_t svtrn1q_f64(svfloat64_t, svfloat64_t);
	15934	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f32)))
	15935	svfloat32_t svtrn1q_f32(svfloat32_t, svfloat32_t);
	15936	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f16)))
	15937	svfloat16_t svtrn1q_f16(svfloat16_t, svfloat16_t);
	15938	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s32)))
	15939	svint32_t svtrn1q_s32(svint32_t, svint32_t);
	15940	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s64)))
	15941	svint64_t svtrn1q_s64(svint64_t, svint64_t);
	15942	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s16)))
	15943	svint16_t svtrn1q_s16(svint16_t, svint16_t);
	15944	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u8)))
	15945	svuint8_t svtrn2q_u8(svuint8_t, svuint8_t);
	15946	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u32)))
	15947	svuint32_t svtrn2q_u32(svuint32_t, svuint32_t);
	15948	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u64)))
	15949	svuint64_t svtrn2q_u64(svuint64_t, svuint64_t);
	15950	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u16)))
	15951	svuint16_t svtrn2q_u16(svuint16_t, svuint16_t);
	15952	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_bf16)))
	15953	svbfloat16_t svtrn2q_bf16(svbfloat16_t, svbfloat16_t);
	15954	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s8)))
	15955	svint8_t svtrn2q_s8(svint8_t, svint8_t);
	15956	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f64)))
	15957	svfloat64_t svtrn2q_f64(svfloat64_t, svfloat64_t);
	15958	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f32)))
	15959	svfloat32_t svtrn2q_f32(svfloat32_t, svfloat32_t);
	15960	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f16)))
	15961	svfloat16_t svtrn2q_f16(svfloat16_t, svfloat16_t);
	15962	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s32)))
	15963	svint32_t svtrn2q_s32(svint32_t, svint32_t);
	15964	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s64)))
	15965	svint64_t svtrn2q_s64(svint64_t, svint64_t);
	15966	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s16)))
	15967	svint16_t svtrn2q_s16(svint16_t, svint16_t);
	15968	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u8)))
	15969	svuint8_t svuzp1q_u8(svuint8_t, svuint8_t);
	15970	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u32)))
	15971	svuint32_t svuzp1q_u32(svuint32_t, svuint32_t);
	15972	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u64)))
	15973	svuint64_t svuzp1q_u64(svuint64_t, svuint64_t);
	15974	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u16)))
	15975	svuint16_t svuzp1q_u16(svuint16_t, svuint16_t);
	15976	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_bf16)))
	15977	svbfloat16_t svuzp1q_bf16(svbfloat16_t, svbfloat16_t);
	15978	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s8)))
	15979	svint8_t svuzp1q_s8(svint8_t, svint8_t);
	15980	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f64)))
	15981	svfloat64_t svuzp1q_f64(svfloat64_t, svfloat64_t);
	15982	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f32)))
	15983	svfloat32_t svuzp1q_f32(svfloat32_t, svfloat32_t);
	15984	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f16)))
	15985	svfloat16_t svuzp1q_f16(svfloat16_t, svfloat16_t);
	15986	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s32)))
	15987	svint32_t svuzp1q_s32(svint32_t, svint32_t);
	15988	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s64)))
	15989	svint64_t svuzp1q_s64(svint64_t, svint64_t);
	15990	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s16)))
	15991	svint16_t svuzp1q_s16(svint16_t, svint16_t);
	15992	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u8)))
	15993	svuint8_t svuzp2q_u8(svuint8_t, svuint8_t);
	15994	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u32)))
	15995	svuint32_t svuzp2q_u32(svuint32_t, svuint32_t);
	15996	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u64)))
	15997	svuint64_t svuzp2q_u64(svuint64_t, svuint64_t);
	15998	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u16)))
	15999	svuint16_t svuzp2q_u16(svuint16_t, svuint16_t);
	16000	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_bf16)))
	16001	svbfloat16_t svuzp2q_bf16(svbfloat16_t, svbfloat16_t);
	16002	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s8)))
	16003	svint8_t svuzp2q_s8(svint8_t, svint8_t);
	16004	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f64)))
	16005	svfloat64_t svuzp2q_f64(svfloat64_t, svfloat64_t);
	16006	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f32)))
	16007	svfloat32_t svuzp2q_f32(svfloat32_t, svfloat32_t);
	16008	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f16)))
	16009	svfloat16_t svuzp2q_f16(svfloat16_t, svfloat16_t);
	16010	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s32)))
	16011	svint32_t svuzp2q_s32(svint32_t, svint32_t);
	16012	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s64)))
	16013	svint64_t svuzp2q_s64(svint64_t, svint64_t);
	16014	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s16)))
	16015	svint16_t svuzp2q_s16(svint16_t, svint16_t);
	16016	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u8)))
	16017	svuint8_t svzip1q_u8(svuint8_t, svuint8_t);
	16018	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u32)))
	16019	svuint32_t svzip1q_u32(svuint32_t, svuint32_t);
	16020	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u64)))
	16021	svuint64_t svzip1q_u64(svuint64_t, svuint64_t);
	16022	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u16)))
	16023	svuint16_t svzip1q_u16(svuint16_t, svuint16_t);
	16024	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_bf16)))
	16025	svbfloat16_t svzip1q_bf16(svbfloat16_t, svbfloat16_t);
	16026	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s8)))
	16027	svint8_t svzip1q_s8(svint8_t, svint8_t);
	16028	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f64)))
	16029	svfloat64_t svzip1q_f64(svfloat64_t, svfloat64_t);
	16030	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f32)))
	16031	svfloat32_t svzip1q_f32(svfloat32_t, svfloat32_t);
	16032	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f16)))
	16033	svfloat16_t svzip1q_f16(svfloat16_t, svfloat16_t);
	16034	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s32)))
	16035	svint32_t svzip1q_s32(svint32_t, svint32_t);
	16036	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s64)))
	16037	svint64_t svzip1q_s64(svint64_t, svint64_t);
	16038	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s16)))
	16039	svint16_t svzip1q_s16(svint16_t, svint16_t);
	16040	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u8)))
	16041	svuint8_t svzip2q_u8(svuint8_t, svuint8_t);
	16042	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u32)))
	16043	svuint32_t svzip2q_u32(svuint32_t, svuint32_t);
	16044	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u64)))
	16045	svuint64_t svzip2q_u64(svuint64_t, svuint64_t);
	16046	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u16)))
	16047	svuint16_t svzip2q_u16(svuint16_t, svuint16_t);
	16048	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_bf16)))
	16049	svbfloat16_t svzip2q_bf16(svbfloat16_t, svbfloat16_t);
	16050	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s8)))
	16051	svint8_t svzip2q_s8(svint8_t, svint8_t);
	16052	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f64)))
	16053	svfloat64_t svzip2q_f64(svfloat64_t, svfloat64_t);
	16054	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f32)))
	16055	svfloat32_t svzip2q_f32(svfloat32_t, svfloat32_t);
	16056	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f16)))
	16057	svfloat16_t svzip2q_f16(svfloat16_t, svfloat16_t);
	16058	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s32)))
	16059	svint32_t svzip2q_s32(svint32_t, svint32_t);
	16060	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s64)))
	16061	svint64_t svzip2q_s64(svint64_t, svint64_t);
	16062	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s16)))
	16063	svint16_t svzip2q_s16(svint16_t, svint16_t);
	16064	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u8)))
	16065	svuint8_t svld1ro(svbool_t, uint8_t const *);
	16066	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u32)))
	16067	svuint32_t svld1ro(svbool_t, uint32_t const *);
	16068	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u64)))
	16069	svuint64_t svld1ro(svbool_t, uint64_t const *);
	16070	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_u16)))
	16071	svuint16_t svld1ro(svbool_t, uint16_t const *);
	16072	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_bf16)))
	16073	svbfloat16_t svld1ro(svbool_t, bfloat16_t const *);
	16074	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s8)))
	16075	svint8_t svld1ro(svbool_t, int8_t const *);
	16076	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f64)))
	16077	svfloat64_t svld1ro(svbool_t, float64_t const *);
	16078	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f32)))
	16079	svfloat32_t svld1ro(svbool_t, float32_t const *);
	16080	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_f16)))
	16081	svfloat16_t svld1ro(svbool_t, float16_t const *);
	16082	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s32)))
	16083	svint32_t svld1ro(svbool_t, int32_t const *);
	16084	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s64)))
	16085	svint64_t svld1ro(svbool_t, int64_t const *);
	16086	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_mf8)))
	16087	svmfloat8_t svld1ro(svbool_t, mfloat8_t const *);
	16088	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1ro_s16)))
	16089	svint16_t svld1ro(svbool_t, int16_t const *);
	16090	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f64)))
	16091	svfloat64_t svmmla(svfloat64_t, svfloat64_t, svfloat64_t);
	16092	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u8)))
	16093	svuint8_t svtrn1q(svuint8_t, svuint8_t);
	16094	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u32)))
	16095	svuint32_t svtrn1q(svuint32_t, svuint32_t);
	16096	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u64)))
	16097	svuint64_t svtrn1q(svuint64_t, svuint64_t);
	16098	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_u16)))
	16099	svuint16_t svtrn1q(svuint16_t, svuint16_t);
	16100	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_bf16)))
	16101	svbfloat16_t svtrn1q(svbfloat16_t, svbfloat16_t);
	16102	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s8)))
	16103	svint8_t svtrn1q(svint8_t, svint8_t);
	16104	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f64)))
	16105	svfloat64_t svtrn1q(svfloat64_t, svfloat64_t);
	16106	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f32)))
	16107	svfloat32_t svtrn1q(svfloat32_t, svfloat32_t);
	16108	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_f16)))
	16109	svfloat16_t svtrn1q(svfloat16_t, svfloat16_t);
	16110	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s32)))
	16111	svint32_t svtrn1q(svint32_t, svint32_t);
	16112	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s64)))
	16113	svint64_t svtrn1q(svint64_t, svint64_t);
	16114	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn1q_s16)))
	16115	svint16_t svtrn1q(svint16_t, svint16_t);
	16116	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u8)))
	16117	svuint8_t svtrn2q(svuint8_t, svuint8_t);
	16118	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u32)))
	16119	svuint32_t svtrn2q(svuint32_t, svuint32_t);
	16120	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u64)))
	16121	svuint64_t svtrn2q(svuint64_t, svuint64_t);
	16122	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_u16)))
	16123	svuint16_t svtrn2q(svuint16_t, svuint16_t);
	16124	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_bf16)))
	16125	svbfloat16_t svtrn2q(svbfloat16_t, svbfloat16_t);
	16126	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s8)))
	16127	svint8_t svtrn2q(svint8_t, svint8_t);
	16128	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f64)))
	16129	svfloat64_t svtrn2q(svfloat64_t, svfloat64_t);
	16130	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f32)))
	16131	svfloat32_t svtrn2q(svfloat32_t, svfloat32_t);
	16132	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_f16)))
	16133	svfloat16_t svtrn2q(svfloat16_t, svfloat16_t);
	16134	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s32)))
	16135	svint32_t svtrn2q(svint32_t, svint32_t);
	16136	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s64)))
	16137	svint64_t svtrn2q(svint64_t, svint64_t);
	16138	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svtrn2q_s16)))
	16139	svint16_t svtrn2q(svint16_t, svint16_t);
	16140	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u8)))
	16141	svuint8_t svuzp1q(svuint8_t, svuint8_t);
	16142	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u32)))
	16143	svuint32_t svuzp1q(svuint32_t, svuint32_t);
	16144	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u64)))
	16145	svuint64_t svuzp1q(svuint64_t, svuint64_t);
	16146	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_u16)))
	16147	svuint16_t svuzp1q(svuint16_t, svuint16_t);
	16148	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_bf16)))
	16149	svbfloat16_t svuzp1q(svbfloat16_t, svbfloat16_t);
	16150	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s8)))
	16151	svint8_t svuzp1q(svint8_t, svint8_t);
	16152	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f64)))
	16153	svfloat64_t svuzp1q(svfloat64_t, svfloat64_t);
	16154	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f32)))
	16155	svfloat32_t svuzp1q(svfloat32_t, svfloat32_t);
	16156	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_f16)))
	16157	svfloat16_t svuzp1q(svfloat16_t, svfloat16_t);
	16158	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s32)))
	16159	svint32_t svuzp1q(svint32_t, svint32_t);
	16160	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s64)))
	16161	svint64_t svuzp1q(svint64_t, svint64_t);
	16162	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp1q_s16)))
	16163	svint16_t svuzp1q(svint16_t, svint16_t);
	16164	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u8)))
	16165	svuint8_t svuzp2q(svuint8_t, svuint8_t);
	16166	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u32)))
	16167	svuint32_t svuzp2q(svuint32_t, svuint32_t);
	16168	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u64)))
	16169	svuint64_t svuzp2q(svuint64_t, svuint64_t);
	16170	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_u16)))
	16171	svuint16_t svuzp2q(svuint16_t, svuint16_t);
	16172	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_bf16)))
	16173	svbfloat16_t svuzp2q(svbfloat16_t, svbfloat16_t);
	16174	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s8)))
	16175	svint8_t svuzp2q(svint8_t, svint8_t);
	16176	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f64)))
	16177	svfloat64_t svuzp2q(svfloat64_t, svfloat64_t);
	16178	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f32)))
	16179	svfloat32_t svuzp2q(svfloat32_t, svfloat32_t);
	16180	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_f16)))
	16181	svfloat16_t svuzp2q(svfloat16_t, svfloat16_t);
	16182	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s32)))
	16183	svint32_t svuzp2q(svint32_t, svint32_t);
	16184	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s64)))
	16185	svint64_t svuzp2q(svint64_t, svint64_t);
	16186	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svuzp2q_s16)))
	16187	svint16_t svuzp2q(svint16_t, svint16_t);
	16188	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u8)))
	16189	svuint8_t svzip1q(svuint8_t, svuint8_t);
	16190	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u32)))
	16191	svuint32_t svzip1q(svuint32_t, svuint32_t);
	16192	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u64)))
	16193	svuint64_t svzip1q(svuint64_t, svuint64_t);
	16194	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_u16)))
	16195	svuint16_t svzip1q(svuint16_t, svuint16_t);
	16196	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_bf16)))
	16197	svbfloat16_t svzip1q(svbfloat16_t, svbfloat16_t);
	16198	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s8)))
	16199	svint8_t svzip1q(svint8_t, svint8_t);
	16200	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f64)))
	16201	svfloat64_t svzip1q(svfloat64_t, svfloat64_t);
	16202	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f32)))
	16203	svfloat32_t svzip1q(svfloat32_t, svfloat32_t);
	16204	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_f16)))
	16205	svfloat16_t svzip1q(svfloat16_t, svfloat16_t);
	16206	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s32)))
	16207	svint32_t svzip1q(svint32_t, svint32_t);
	16208	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s64)))
	16209	svint64_t svzip1q(svint64_t, svint64_t);
	16210	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip1q_s16)))
	16211	svint16_t svzip1q(svint16_t, svint16_t);
	16212	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u8)))
	16213	svuint8_t svzip2q(svuint8_t, svuint8_t);
	16214	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u32)))
	16215	svuint32_t svzip2q(svuint32_t, svuint32_t);
	16216	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u64)))
	16217	svuint64_t svzip2q(svuint64_t, svuint64_t);
	16218	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_u16)))
	16219	svuint16_t svzip2q(svuint16_t, svuint16_t);
	16220	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_bf16)))
	16221	svbfloat16_t svzip2q(svbfloat16_t, svbfloat16_t);
	16222	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s8)))
	16223	svint8_t svzip2q(svint8_t, svint8_t);
	16224	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f64)))
	16225	svfloat64_t svzip2q(svfloat64_t, svfloat64_t);
	16226	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f32)))
	16227	svfloat32_t svzip2q(svfloat32_t, svfloat32_t);
	16228	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_f16)))
	16229	svfloat16_t svzip2q(svfloat16_t, svfloat16_t);
	16230	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s32)))
	16231	svint32_t svzip2q(svint32_t, svint32_t);
	16232	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s64)))
	16233	svint64_t svzip2q(svint64_t, svint64_t);
	16234	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svzip2q_s16)))
	16235	svint16_t svzip2q(svint16_t, svint16_t);
	16236	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_s32)))
	16237	svint32_t svmmla_s32(svint32_t, svint8_t, svint8_t);
	16238	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_u32)))
	16239	svuint32_t svmmla_u32(svuint32_t, svuint8_t, svuint8_t);
	16240	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusmmla_s32)))
	16241	svint32_t svusmmla_s32(svint32_t, svuint8_t, svint8_t);
	16242	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_s32)))
	16243	svint32_t svmmla(svint32_t, svint8_t, svint8_t);
	16244	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_u32)))
	16245	svuint32_t svmmla(svuint32_t, svuint8_t, svuint8_t);
	16246	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusmmla_s32)))
	16247	svint32_t svusmmla(svint32_t, svuint8_t, svint8_t);
	16248	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_n_s32)))
	16249	svint32_t svsudot_n_s32(svint32_t, svint8_t, uint8_t);
	16250	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_s32)))
	16251	svint32_t svsudot_s32(svint32_t, svint8_t, svuint8_t);
	16252	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_lane_s32)))
	16253	svint32_t svsudot_lane_s32(svint32_t, svint8_t, svuint8_t, uint64_t);
	16254	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_n_s32)))
	16255	svint32_t svusdot_n_s32(svint32_t, svuint8_t, int8_t);
	16256	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_s32)))
	16257	svint32_t svusdot_s32(svint32_t, svuint8_t, svint8_t);
	16258	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_lane_s32)))
	16259	svint32_t svusdot_lane_s32(svint32_t, svuint8_t, svint8_t, uint64_t);
	16260	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_n_s32)))
	16261	svint32_t svsudot(svint32_t, svint8_t, uint8_t);
	16262	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_s32)))
	16263	svint32_t svsudot(svint32_t, svint8_t, svuint8_t);
	16264	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsudot_lane_s32)))
	16265	svint32_t svsudot_lane(svint32_t, svint8_t, svuint8_t, uint64_t);
	16266	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_n_s32)))
	16267	svint32_t svusdot(svint32_t, svuint8_t, int8_t);
	16268	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_s32)))
	16269	svint32_t svusdot(svint32_t, svuint8_t, svint8_t);
	16270	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svusdot_lane_s32)))
	16271	svint32_t svusdot_lane(svint32_t, svuint8_t, svint8_t, uint64_t);
	16272	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_lane_u8_x2)))
	16273	svuint8x2_t svaesd_lane_u8_x2(svuint8x2_t, svuint8_t, uint64_t);
	16274	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_lane_u8_x4)))
	16275	svuint8x4_t svaesd_lane_u8_x4(svuint8x4_t, svuint8_t, uint64_t);
	16276	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesdimc_lane_u8_x2)))
	16277	svuint8x2_t svaesdimc_lane_u8_x2(svuint8x2_t, svuint8_t, uint64_t);
	16278	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesdimc_lane_u8_x4)))
	16279	svuint8x4_t svaesdimc_lane_u8_x4(svuint8x4_t, svuint8_t, uint64_t);
	16280	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_lane_u8_x2)))
	16281	svuint8x2_t svaese_lane_u8_x2(svuint8x2_t, svuint8_t, uint64_t);
	16282	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_lane_u8_x4)))
	16283	svuint8x4_t svaese_lane_u8_x4(svuint8x4_t, svuint8_t, uint64_t);
	16284	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesemc_lane_u8_x2)))
	16285	svuint8x2_t svaesemc_lane_u8_x2(svuint8x2_t, svuint8_t, uint64_t);
	16286	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesemc_lane_u8_x4)))
	16287	svuint8x4_t svaesemc_lane_u8_x4(svuint8x4_t, svuint8_t, uint64_t);
	16288	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmlal_pair_n_u64_x2)))
	16289	svuint64x2_t svpmlal_pair_n_u64_x2(svuint64x2_t, svuint64_t, uint64_t);
	16290	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmlal_pair_u64_x2)))
	16291	svuint64x2_t svpmlal_pair_u64_x2(svuint64x2_t, svuint64_t, svuint64_t);
	16292	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmull_pair_n_u64_x2)))
	16293	svuint64x2_t svpmull_pair_n_u64_x2(svuint64_t, uint64_t);
	16294	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmull_pair_u64_x2)))
	16295	svuint64x2_t svpmull_pair_u64_x2(svuint64_t, svuint64_t);
	16296	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_lane_u8_x2)))
	16297	svuint8x2_t svaesd_lane(svuint8x2_t, svuint8_t, uint64_t);
	16298	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_lane_u8_x4)))
	16299	svuint8x4_t svaesd_lane(svuint8x4_t, svuint8_t, uint64_t);
	16300	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesdimc_lane_u8_x2)))
	16301	svuint8x2_t svaesdimc_lane(svuint8x2_t, svuint8_t, uint64_t);
	16302	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesdimc_lane_u8_x4)))
	16303	svuint8x4_t svaesdimc_lane(svuint8x4_t, svuint8_t, uint64_t);
	16304	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_lane_u8_x2)))
	16305	svuint8x2_t svaese_lane(svuint8x2_t, svuint8_t, uint64_t);
	16306	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_lane_u8_x4)))
	16307	svuint8x4_t svaese_lane(svuint8x4_t, svuint8_t, uint64_t);
	16308	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesemc_lane_u8_x2)))
	16309	svuint8x2_t svaesemc_lane(svuint8x2_t, svuint8_t, uint64_t);
	16310	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesemc_lane_u8_x4)))
	16311	svuint8x4_t svaesemc_lane(svuint8x4_t, svuint8_t, uint64_t);
	16312	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmlal_pair_n_u64_x2)))
	16313	svuint64x2_t svpmlal_pair(svuint64x2_t, svuint64_t, uint64_t);
	16314	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmlal_pair_u64_x2)))
	16315	svuint64x2_t svpmlal_pair(svuint64x2_t, svuint64_t, svuint64_t);
	16316	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmull_pair_n_u64_x2)))
	16317	svuint64x2_t svpmull_pair(svuint64_t, uint64_t);
	16318	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmull_pair_u64_x2)))
	16319	svuint64x2_t svpmull_pair(svuint64_t, svuint64_t);
	16320	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
	16321	svuint8_t svaesd_u8(svuint8_t, svuint8_t);
	16322	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
	16323	svuint8_t svaese_u8(svuint8_t, svuint8_t);
	16324	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
	16325	svuint8_t svaesimc_u8(svuint8_t);
	16326	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
	16327	svuint8_t svaesmc_u8(svuint8_t);
	16328	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
	16329	svuint64_t svpmullb_pair_n_u64(svuint64_t, uint64_t);
	16330	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
	16331	svuint64_t svpmullb_pair_u64(svuint64_t, svuint64_t);
	16332	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
	16333	svuint64_t svpmullt_pair_n_u64(svuint64_t, uint64_t);
	16334	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
	16335	svuint64_t svpmullt_pair_u64(svuint64_t, svuint64_t);
	16336	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesd_u8)))
	16337	svuint8_t svaesd(svuint8_t, svuint8_t);
	16338	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaese_u8)))
	16339	svuint8_t svaese(svuint8_t, svuint8_t);
	16340	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesimc_u8)))
	16341	svuint8_t svaesimc(svuint8_t);
	16342	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svaesmc_u8)))
	16343	svuint8_t svaesmc(svuint8_t);
	16344	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_n_u64)))
	16345	svuint64_t svpmullb_pair(svuint64_t, uint64_t);
	16346	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullb_pair_u64)))
	16347	svuint64_t svpmullb_pair(svuint64_t, svuint64_t);
	16348	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_n_u64)))
	16349	svuint64_t svpmullt_pair(svuint64_t, uint64_t);
	16350	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpmullt_pair_u64)))
	16351	svuint64_t svpmullt_pair(svuint64_t, svuint64_t);
	16352	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
	16353	svbfloat16_t svadd_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16354	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
	16355	svbfloat16_t svadd_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16356	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_z)))
	16357	svbfloat16_t svadd_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16358	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_m)))
	16359	svbfloat16_t svadd_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16360	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_x)))
	16361	svbfloat16_t svadd_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16362	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_z)))
	16363	svbfloat16_t svadd_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16364	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_bf16)))
	16365	svbfloat16_t svclamp_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16366	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_m)))
	16367	svbfloat16_t svmax_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16368	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_x)))
	16369	svbfloat16_t svmax_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16370	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_z)))
	16371	svbfloat16_t svmax_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16372	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_m)))
	16373	svbfloat16_t svmax_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16374	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_x)))
	16375	svbfloat16_t svmax_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16376	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_z)))
	16377	svbfloat16_t svmax_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16378	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_m)))
	16379	svbfloat16_t svmaxnm_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16380	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_x)))
	16381	svbfloat16_t svmaxnm_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16382	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_z)))
	16383	svbfloat16_t svmaxnm_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16384	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_m)))
	16385	svbfloat16_t svmaxnm_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16386	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_x)))
	16387	svbfloat16_t svmaxnm_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16388	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_z)))
	16389	svbfloat16_t svmaxnm_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16390	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_m)))
	16391	svbfloat16_t svmin_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16392	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_x)))
	16393	svbfloat16_t svmin_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16394	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_z)))
	16395	svbfloat16_t svmin_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16396	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_m)))
	16397	svbfloat16_t svmin_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16398	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_x)))
	16399	svbfloat16_t svmin_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16400	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_z)))
	16401	svbfloat16_t svmin_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16402	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_m)))
	16403	svbfloat16_t svminnm_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16404	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_x)))
	16405	svbfloat16_t svminnm_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16406	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_z)))
	16407	svbfloat16_t svminnm_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16408	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_m)))
	16409	svbfloat16_t svminnm_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16410	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x)))
	16411	svbfloat16_t svminnm_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16412	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_z)))
	16413	svbfloat16_t svminnm_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16414	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_m)))
	16415	svbfloat16_t svmla_n_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16416	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_x)))
	16417	svbfloat16_t svmla_n_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16418	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_z)))
	16419	svbfloat16_t svmla_n_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16420	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_m)))
	16421	svbfloat16_t svmla_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16422	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_x)))
	16423	svbfloat16_t svmla_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16424	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_z)))
	16425	svbfloat16_t svmla_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16426	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_lane_bf16)))
	16427	svbfloat16_t svmla_lane_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
	16428	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_m)))
	16429	svbfloat16_t svmls_n_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16430	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_x)))
	16431	svbfloat16_t svmls_n_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16432	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_z)))
	16433	svbfloat16_t svmls_n_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16434	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_m)))
	16435	svbfloat16_t svmls_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16436	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_x)))
	16437	svbfloat16_t svmls_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16438	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_z)))
	16439	svbfloat16_t svmls_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16440	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_lane_bf16)))
	16441	svbfloat16_t svmls_lane_bf16(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
	16442	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_m)))
	16443	svbfloat16_t svmul_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16444	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_x)))
	16445	svbfloat16_t svmul_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16446	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_z)))
	16447	svbfloat16_t svmul_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16448	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_m)))
	16449	svbfloat16_t svmul_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16450	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x)))
	16451	svbfloat16_t svmul_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16452	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_z)))
	16453	svbfloat16_t svmul_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16454	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_lane_bf16)))
	16455	svbfloat16_t svmul_lane_bf16(svbfloat16_t, svbfloat16_t, uint64_t);
	16456	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_m)))
	16457	svbfloat16_t svsub_n_bf16_m(svbool_t, svbfloat16_t, bfloat16_t);
	16458	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_x)))
	16459	svbfloat16_t svsub_n_bf16_x(svbool_t, svbfloat16_t, bfloat16_t);
	16460	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_z)))
	16461	svbfloat16_t svsub_n_bf16_z(svbool_t, svbfloat16_t, bfloat16_t);
	16462	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_m)))
	16463	svbfloat16_t svsub_bf16_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16464	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
	16465	svbfloat16_t svsub_bf16_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16466	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
	16467	svbfloat16_t svsub_bf16_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16468	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_m)))
	16469	svbfloat16_t svadd_m(svbool_t, svbfloat16_t, bfloat16_t);
	16470	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_x)))
	16471	svbfloat16_t svadd_x(svbool_t, svbfloat16_t, bfloat16_t);
	16472	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_n_bf16_z)))
	16473	svbfloat16_t svadd_z(svbool_t, svbfloat16_t, bfloat16_t);
	16474	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_m)))
	16475	svbfloat16_t svadd_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16476	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_x)))
	16477	svbfloat16_t svadd_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16478	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svadd_bf16_z)))
	16479	svbfloat16_t svadd_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16480	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_bf16)))
	16481	svbfloat16_t svclamp(svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16482	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_m)))
	16483	svbfloat16_t svmax_m(svbool_t, svbfloat16_t, bfloat16_t);
	16484	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_x)))
	16485	svbfloat16_t svmax_x(svbool_t, svbfloat16_t, bfloat16_t);
	16486	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_n_bf16_z)))
	16487	svbfloat16_t svmax_z(svbool_t, svbfloat16_t, bfloat16_t);
	16488	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_m)))
	16489	svbfloat16_t svmax_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16490	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_x)))
	16491	svbfloat16_t svmax_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16492	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmax_bf16_z)))
	16493	svbfloat16_t svmax_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16494	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_m)))
	16495	svbfloat16_t svmaxnm_m(svbool_t, svbfloat16_t, bfloat16_t);
	16496	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_x)))
	16497	svbfloat16_t svmaxnm_x(svbool_t, svbfloat16_t, bfloat16_t);
	16498	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_n_bf16_z)))
	16499	svbfloat16_t svmaxnm_z(svbool_t, svbfloat16_t, bfloat16_t);
	16500	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_m)))
	16501	svbfloat16_t svmaxnm_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16502	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_x)))
	16503	svbfloat16_t svmaxnm_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16504	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmaxnm_bf16_z)))
	16505	svbfloat16_t svmaxnm_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16506	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_m)))
	16507	svbfloat16_t svmin_m(svbool_t, svbfloat16_t, bfloat16_t);
	16508	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_x)))
	16509	svbfloat16_t svmin_x(svbool_t, svbfloat16_t, bfloat16_t);
	16510	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_n_bf16_z)))
	16511	svbfloat16_t svmin_z(svbool_t, svbfloat16_t, bfloat16_t);
	16512	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_m)))
	16513	svbfloat16_t svmin_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16514	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_x)))
	16515	svbfloat16_t svmin_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16516	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmin_bf16_z)))
	16517	svbfloat16_t svmin_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16518	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_m)))
	16519	svbfloat16_t svminnm_m(svbool_t, svbfloat16_t, bfloat16_t);
	16520	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_x)))
	16521	svbfloat16_t svminnm_x(svbool_t, svbfloat16_t, bfloat16_t);
	16522	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_n_bf16_z)))
	16523	svbfloat16_t svminnm_z(svbool_t, svbfloat16_t, bfloat16_t);
	16524	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_m)))
	16525	svbfloat16_t svminnm_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16526	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_x)))
	16527	svbfloat16_t svminnm_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16528	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svminnm_bf16_z)))
	16529	svbfloat16_t svminnm_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16530	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_m)))
	16531	svbfloat16_t svmla_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16532	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_x)))
	16533	svbfloat16_t svmla_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16534	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_n_bf16_z)))
	16535	svbfloat16_t svmla_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16536	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_m)))
	16537	svbfloat16_t svmla_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16538	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_x)))
	16539	svbfloat16_t svmla_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16540	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_bf16_z)))
	16541	svbfloat16_t svmla_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16542	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmla_lane_bf16)))
	16543	svbfloat16_t svmla_lane(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
	16544	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_m)))
	16545	svbfloat16_t svmls_m(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16546	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_x)))
	16547	svbfloat16_t svmls_x(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16548	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_n_bf16_z)))
	16549	svbfloat16_t svmls_z(svbool_t, svbfloat16_t, svbfloat16_t, bfloat16_t);
	16550	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_m)))
	16551	svbfloat16_t svmls_m(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16552	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_x)))
	16553	svbfloat16_t svmls_x(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16554	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_bf16_z)))
	16555	svbfloat16_t svmls_z(svbool_t, svbfloat16_t, svbfloat16_t, svbfloat16_t);
	16556	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmls_lane_bf16)))
	16557	svbfloat16_t svmls_lane(svbfloat16_t, svbfloat16_t, svbfloat16_t, uint64_t);
	16558	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_m)))
	16559	svbfloat16_t svmul_m(svbool_t, svbfloat16_t, bfloat16_t);
	16560	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_x)))
	16561	svbfloat16_t svmul_x(svbool_t, svbfloat16_t, bfloat16_t);
	16562	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_n_bf16_z)))
	16563	svbfloat16_t svmul_z(svbool_t, svbfloat16_t, bfloat16_t);
	16564	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_m)))
	16565	svbfloat16_t svmul_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16566	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_x)))
	16567	svbfloat16_t svmul_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16568	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_bf16_z)))
	16569	svbfloat16_t svmul_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16570	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmul_lane_bf16)))
	16571	svbfloat16_t svmul_lane(svbfloat16_t, svbfloat16_t, uint64_t);
	16572	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_m)))
	16573	svbfloat16_t svsub_m(svbool_t, svbfloat16_t, bfloat16_t);
	16574	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_x)))
	16575	svbfloat16_t svsub_x(svbool_t, svbfloat16_t, bfloat16_t);
	16576	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_n_bf16_z)))
	16577	svbfloat16_t svsub_z(svbool_t, svbfloat16_t, bfloat16_t);
	16578	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_m)))
	16579	svbfloat16_t svsub_m(svbool_t, svbfloat16_t, svbfloat16_t);
	16580	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_x)))
	16581	svbfloat16_t svsub_x(svbool_t, svbfloat16_t, svbfloat16_t);
	16582	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsub_bf16_z)))
	16583	svbfloat16_t svsub_z(svbool_t, svbfloat16_t, svbfloat16_t);
	16584	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_m)))
	16585	svbfloat16_t svscale_n_bf16_m(svbool_t, svbfloat16_t, int16_t);
	16586	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_x)))
	16587	svbfloat16_t svscale_n_bf16_x(svbool_t, svbfloat16_t, int16_t);
	16588	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_z)))
	16589	svbfloat16_t svscale_n_bf16_z(svbool_t, svbfloat16_t, int16_t);
	16590	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_m)))
	16591	svbfloat16_t svscale_bf16_m(svbool_t, svbfloat16_t, svint16_t);
	16592	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x)))
	16593	svbfloat16_t svscale_bf16_x(svbool_t, svbfloat16_t, svint16_t);
	16594	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_z)))
	16595	svbfloat16_t svscale_bf16_z(svbool_t, svbfloat16_t, svint16_t);
	16596	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_m)))
	16597	svbfloat16_t svscale_m(svbool_t, svbfloat16_t, int16_t);
	16598	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_x)))
	16599	svbfloat16_t svscale_x(svbool_t, svbfloat16_t, int16_t);
	16600	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_n_bf16_z)))
	16601	svbfloat16_t svscale_z(svbool_t, svbfloat16_t, int16_t);
	16602	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_m)))
	16603	svbfloat16_t svscale_m(svbool_t, svbfloat16_t, svint16_t);
	16604	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_x)))
	16605	svbfloat16_t svscale_x(svbool_t, svbfloat16_t, svint16_t);
	16606	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svscale_bf16_z)))
	16607	svbfloat16_t svscale_z(svbool_t, svbfloat16_t, svint16_t);
	16608	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
	16609	svuint8_t svbdep_n_u8(svuint8_t, uint8_t);
	16610	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
	16611	svuint32_t svbdep_n_u32(svuint32_t, uint32_t);
	16612	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u64)))
	16613	svuint64_t svbdep_n_u64(svuint64_t, uint64_t);
	16614	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u16)))
	16615	svuint16_t svbdep_n_u16(svuint16_t, uint16_t);
	16616	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u8)))
	16617	svuint8_t svbdep_u8(svuint8_t, svuint8_t);
	16618	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u32)))
	16619	svuint32_t svbdep_u32(svuint32_t, svuint32_t);
	16620	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u64)))
	16621	svuint64_t svbdep_u64(svuint64_t, svuint64_t);
	16622	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u16)))
	16623	svuint16_t svbdep_u16(svuint16_t, svuint16_t);
	16624	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u8)))
	16625	svuint8_t svbext_n_u8(svuint8_t, uint8_t);
	16626	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u32)))
	16627	svuint32_t svbext_n_u32(svuint32_t, uint32_t);
	16628	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u64)))
	16629	svuint64_t svbext_n_u64(svuint64_t, uint64_t);
	16630	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u16)))
	16631	svuint16_t svbext_n_u16(svuint16_t, uint16_t);
	16632	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u8)))
	16633	svuint8_t svbext_u8(svuint8_t, svuint8_t);
	16634	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u32)))
	16635	svuint32_t svbext_u32(svuint32_t, svuint32_t);
	16636	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u64)))
	16637	svuint64_t svbext_u64(svuint64_t, svuint64_t);
	16638	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u16)))
	16639	svuint16_t svbext_u16(svuint16_t, svuint16_t);
	16640	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u8)))
	16641	svuint8_t svbgrp_n_u8(svuint8_t, uint8_t);
	16642	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u32)))
	16643	svuint32_t svbgrp_n_u32(svuint32_t, uint32_t);
	16644	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u64)))
	16645	svuint64_t svbgrp_n_u64(svuint64_t, uint64_t);
	16646	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u16)))
	16647	svuint16_t svbgrp_n_u16(svuint16_t, uint16_t);
	16648	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u8)))
	16649	svuint8_t svbgrp_u8(svuint8_t, svuint8_t);
	16650	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u32)))
	16651	svuint32_t svbgrp_u32(svuint32_t, svuint32_t);
	16652	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u64)))
	16653	svuint64_t svbgrp_u64(svuint64_t, svuint64_t);
	16654	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u16)))
	16655	svuint16_t svbgrp_u16(svuint16_t, svuint16_t);
	16656	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u8)))
	16657	svuint8_t svbdep(svuint8_t, uint8_t);
	16658	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u32)))
	16659	svuint32_t svbdep(svuint32_t, uint32_t);
	16660	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u64)))
	16661	svuint64_t svbdep(svuint64_t, uint64_t);
	16662	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_n_u16)))
	16663	svuint16_t svbdep(svuint16_t, uint16_t);
	16664	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u8)))
	16665	svuint8_t svbdep(svuint8_t, svuint8_t);
	16666	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u32)))
	16667	svuint32_t svbdep(svuint32_t, svuint32_t);
	16668	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u64)))
	16669	svuint64_t svbdep(svuint64_t, svuint64_t);
	16670	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbdep_u16)))
	16671	svuint16_t svbdep(svuint16_t, svuint16_t);
	16672	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u8)))
	16673	svuint8_t svbext(svuint8_t, uint8_t);
	16674	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u32)))
	16675	svuint32_t svbext(svuint32_t, uint32_t);
	16676	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u64)))
	16677	svuint64_t svbext(svuint64_t, uint64_t);
	16678	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_n_u16)))
	16679	svuint16_t svbext(svuint16_t, uint16_t);
	16680	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u8)))
	16681	svuint8_t svbext(svuint8_t, svuint8_t);
	16682	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u32)))
	16683	svuint32_t svbext(svuint32_t, svuint32_t);
	16684	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u64)))
	16685	svuint64_t svbext(svuint64_t, svuint64_t);
	16686	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbext_u16)))
	16687	svuint16_t svbext(svuint16_t, svuint16_t);
	16688	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u8)))
	16689	svuint8_t svbgrp(svuint8_t, uint8_t);
	16690	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u32)))
	16691	svuint32_t svbgrp(svuint32_t, uint32_t);
	16692	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u64)))
	16693	svuint64_t svbgrp(svuint64_t, uint64_t);
	16694	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_n_u16)))
	16695	svuint16_t svbgrp(svuint16_t, uint16_t);
	16696	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u8)))
	16697	svuint8_t svbgrp(svuint8_t, svuint8_t);
	16698	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u32)))
	16699	svuint32_t svbgrp(svuint32_t, svuint32_t);
	16700	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u64)))
	16701	svuint64_t svbgrp(svuint64_t, svuint64_t);
	16702	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svbgrp_u16)))
	16703	svuint16_t svbgrp(svuint16_t, svuint16_t);
	16704	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32_f16)))
	16705	svfloat32_t svmmla_f32_f16(svfloat32_t, svfloat16_t, svfloat16_t);
	16706	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32_f16)))
	16707	svfloat32_t svmmla(svfloat32_t, svfloat16_t, svfloat16_t);
	16708	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_u64)))
	16709	svuint64_t svrax1_u64(svuint64_t, svuint64_t);
	16710	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_s64)))
	16711	svint64_t svrax1_s64(svint64_t, svint64_t);
	16712	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_u64)))
	16713	svuint64_t svrax1(svuint64_t, svuint64_t);
	16714	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svrax1_s64)))
	16715	svint64_t svrax1(svint64_t, svint64_t);
	16716	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
	16717	svuint32_t svsm4e_u32(svuint32_t, svuint32_t);
	16718	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
	16719	svuint32_t svsm4ekey_u32(svuint32_t, svuint32_t);
	16720	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
	16721	svuint32_t svsm4e(svuint32_t, svuint32_t);
	16722	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
	16723	svuint32_t svsm4ekey(svuint32_t, svuint32_t);
	16724	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u32_z)))
	16725	svuint32_t svhistcnt_u32_z(svbool_t, svuint32_t, svuint32_t);
	16726	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u64_z)))
	16727	svuint64_t svhistcnt_u64_z(svbool_t, svuint64_t, svuint64_t);
	16728	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s32_z)))
	16729	svuint32_t svhistcnt_s32_z(svbool_t, svint32_t, svint32_t);
	16730	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s64_z)))
	16731	svuint64_t svhistcnt_s64_z(svbool_t, svint64_t, svint64_t);
	16732	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_u8)))
	16733	svuint8_t svhistseg_u8(svuint8_t, svuint8_t);
	16734	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_s8)))
	16735	svuint8_t svhistseg_s8(svint8_t, svint8_t);
	16736	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_u32)))
	16737	svuint32_t svldnt1_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
	16738	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_u64)))
	16739	svuint64_t svldnt1_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	16740	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_f64)))
	16741	svfloat64_t svldnt1_gather_u64base_index_f64(svbool_t, svuint64_t, int64_t);
	16742	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_f32)))
	16743	svfloat32_t svldnt1_gather_u32base_index_f32(svbool_t, svuint32_t, int64_t);
	16744	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_s32)))
	16745	svint32_t svldnt1_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
	16746	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_s64)))
	16747	svint64_t svldnt1_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	16748	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_u32)))
	16749	svuint32_t svldnt1_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
	16750	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_u64)))
	16751	svuint64_t svldnt1_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16752	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_f64)))
	16753	svfloat64_t svldnt1_gather_u64base_offset_f64(svbool_t, svuint64_t, int64_t);
	16754	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_f32)))
	16755	svfloat32_t svldnt1_gather_u32base_offset_f32(svbool_t, svuint32_t, int64_t);
	16756	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_s32)))
	16757	svint32_t svldnt1_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
	16758	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_s64)))
	16759	svint64_t svldnt1_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16760	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_u32)))
	16761	svuint32_t svldnt1_gather_u32base_u32(svbool_t, svuint32_t);
	16762	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_u64)))
	16763	svuint64_t svldnt1_gather_u64base_u64(svbool_t, svuint64_t);
	16764	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_f64)))
	16765	svfloat64_t svldnt1_gather_u64base_f64(svbool_t, svuint64_t);
	16766	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_f32)))
	16767	svfloat32_t svldnt1_gather_u32base_f32(svbool_t, svuint32_t);
	16768	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_s32)))
	16769	svint32_t svldnt1_gather_u32base_s32(svbool_t, svuint32_t);
	16770	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_s64)))
	16771	svint64_t svldnt1_gather_u64base_s64(svbool_t, svuint64_t);
	16772	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_u64)))
	16773	svuint64_t svldnt1_gather_s64index_u64(svbool_t, uint64_t const *, svint64_t);
	16774	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_f64)))
	16775	svfloat64_t svldnt1_gather_s64index_f64(svbool_t, float64_t const *, svint64_t);
	16776	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_s64)))
	16777	svint64_t svldnt1_gather_s64index_s64(svbool_t, int64_t const *, svint64_t);
	16778	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_u64)))
	16779	svuint64_t svldnt1_gather_u64index_u64(svbool_t, uint64_t const *, svuint64_t);
	16780	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_f64)))
	16781	svfloat64_t svldnt1_gather_u64index_f64(svbool_t, float64_t const *, svuint64_t);
	16782	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_s64)))
	16783	svint64_t svldnt1_gather_u64index_s64(svbool_t, int64_t const *, svuint64_t);
	16784	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_u32)))
	16785	svuint32_t svldnt1_gather_u32offset_u32(svbool_t, uint32_t const *, svuint32_t);
	16786	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_f32)))
	16787	svfloat32_t svldnt1_gather_u32offset_f32(svbool_t, float32_t const *, svuint32_t);
	16788	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_s32)))
	16789	svint32_t svldnt1_gather_u32offset_s32(svbool_t, int32_t const *, svuint32_t);
	16790	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_u64)))
	16791	svuint64_t svldnt1_gather_s64offset_u64(svbool_t, uint64_t const *, svint64_t);
	16792	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_f64)))
	16793	svfloat64_t svldnt1_gather_s64offset_f64(svbool_t, float64_t const *, svint64_t);
	16794	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_s64)))
	16795	svint64_t svldnt1_gather_s64offset_s64(svbool_t, int64_t const *, svint64_t);
	16796	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_u64)))
	16797	svuint64_t svldnt1_gather_u64offset_u64(svbool_t, uint64_t const *, svuint64_t);
	16798	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_f64)))
	16799	svfloat64_t svldnt1_gather_u64offset_f64(svbool_t, float64_t const *, svuint64_t);
	16800	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_s64)))
	16801	svint64_t svldnt1_gather_u64offset_s64(svbool_t, int64_t const *, svuint64_t);
	16802	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_u32)))
	16803	svuint32_t svldnt1sb_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
	16804	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_u64)))
	16805	svuint64_t svldnt1sb_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16806	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_s32)))
	16807	svint32_t svldnt1sb_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
	16808	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_s64)))
	16809	svint64_t svldnt1sb_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16810	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_u32)))
	16811	svuint32_t svldnt1sb_gather_u32base_u32(svbool_t, svuint32_t);
	16812	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_u64)))
	16813	svuint64_t svldnt1sb_gather_u64base_u64(svbool_t, svuint64_t);
	16814	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_s32)))
	16815	svint32_t svldnt1sb_gather_u32base_s32(svbool_t, svuint32_t);
	16816	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_s64)))
	16817	svint64_t svldnt1sb_gather_u64base_s64(svbool_t, svuint64_t);
	16818	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_u32)))
	16819	svuint32_t svldnt1sb_gather_u32offset_u32(svbool_t, int8_t const *, svuint32_t);
	16820	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_s32)))
	16821	svint32_t svldnt1sb_gather_u32offset_s32(svbool_t, int8_t const *, svuint32_t);
	16822	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_u64)))
	16823	svuint64_t svldnt1sb_gather_s64offset_u64(svbool_t, int8_t const *, svint64_t);
	16824	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_s64)))
	16825	svint64_t svldnt1sb_gather_s64offset_s64(svbool_t, int8_t const *, svint64_t);
	16826	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_u64)))
	16827	svuint64_t svldnt1sb_gather_u64offset_u64(svbool_t, int8_t const *, svuint64_t);
	16828	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_s64)))
	16829	svint64_t svldnt1sb_gather_u64offset_s64(svbool_t, int8_t const *, svuint64_t);
	16830	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_u32)))
	16831	svuint32_t svldnt1sh_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
	16832	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_u64)))
	16833	svuint64_t svldnt1sh_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	16834	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_s32)))
	16835	svint32_t svldnt1sh_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
	16836	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_s64)))
	16837	svint64_t svldnt1sh_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	16838	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_u32)))
	16839	svuint32_t svldnt1sh_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
	16840	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_u64)))
	16841	svuint64_t svldnt1sh_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16842	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_s32)))
	16843	svint32_t svldnt1sh_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
	16844	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_s64)))
	16845	svint64_t svldnt1sh_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16846	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_u32)))
	16847	svuint32_t svldnt1sh_gather_u32base_u32(svbool_t, svuint32_t);
	16848	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_u64)))
	16849	svuint64_t svldnt1sh_gather_u64base_u64(svbool_t, svuint64_t);
	16850	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_s32)))
	16851	svint32_t svldnt1sh_gather_u32base_s32(svbool_t, svuint32_t);
	16852	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_s64)))
	16853	svint64_t svldnt1sh_gather_u64base_s64(svbool_t, svuint64_t);
	16854	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_u64)))
	16855	svuint64_t svldnt1sh_gather_s64index_u64(svbool_t, int16_t const *, svint64_t);
	16856	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_s64)))
	16857	svint64_t svldnt1sh_gather_s64index_s64(svbool_t, int16_t const *, svint64_t);
	16858	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_u64)))
	16859	svuint64_t svldnt1sh_gather_u64index_u64(svbool_t, int16_t const *, svuint64_t);
	16860	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_s64)))
	16861	svint64_t svldnt1sh_gather_u64index_s64(svbool_t, int16_t const *, svuint64_t);
	16862	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_u32)))
	16863	svuint32_t svldnt1sh_gather_u32offset_u32(svbool_t, int16_t const *, svuint32_t);
	16864	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_s32)))
	16865	svint32_t svldnt1sh_gather_u32offset_s32(svbool_t, int16_t const *, svuint32_t);
	16866	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_u64)))
	16867	svuint64_t svldnt1sh_gather_s64offset_u64(svbool_t, int16_t const *, svint64_t);
	16868	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_s64)))
	16869	svint64_t svldnt1sh_gather_s64offset_s64(svbool_t, int16_t const *, svint64_t);
	16870	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_u64)))
	16871	svuint64_t svldnt1sh_gather_u64offset_u64(svbool_t, int16_t const *, svuint64_t);
	16872	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_s64)))
	16873	svint64_t svldnt1sh_gather_u64offset_s64(svbool_t, int16_t const *, svuint64_t);
	16874	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_u64)))
	16875	svuint64_t svldnt1sw_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	16876	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_s64)))
	16877	svint64_t svldnt1sw_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	16878	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_u64)))
	16879	svuint64_t svldnt1sw_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16880	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_s64)))
	16881	svint64_t svldnt1sw_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16882	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_u64)))
	16883	svuint64_t svldnt1sw_gather_u64base_u64(svbool_t, svuint64_t);
	16884	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_s64)))
	16885	svint64_t svldnt1sw_gather_u64base_s64(svbool_t, svuint64_t);
	16886	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_u64)))
	16887	svuint64_t svldnt1sw_gather_s64index_u64(svbool_t, int32_t const *, svint64_t);
	16888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_s64)))
	16889	svint64_t svldnt1sw_gather_s64index_s64(svbool_t, int32_t const *, svint64_t);
	16890	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_u64)))
	16891	svuint64_t svldnt1sw_gather_u64index_u64(svbool_t, int32_t const *, svuint64_t);
	16892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_s64)))
	16893	svint64_t svldnt1sw_gather_u64index_s64(svbool_t, int32_t const *, svuint64_t);
	16894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_u64)))
	16895	svuint64_t svldnt1sw_gather_s64offset_u64(svbool_t, int32_t const *, svint64_t);
	16896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_s64)))
	16897	svint64_t svldnt1sw_gather_s64offset_s64(svbool_t, int32_t const *, svint64_t);
	16898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_u64)))
	16899	svuint64_t svldnt1sw_gather_u64offset_u64(svbool_t, int32_t const *, svuint64_t);
	16900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_s64)))
	16901	svint64_t svldnt1sw_gather_u64offset_s64(svbool_t, int32_t const *, svuint64_t);
	16902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_u32)))
	16903	svuint32_t svldnt1ub_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
	16904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_u64)))
	16905	svuint64_t svldnt1ub_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_s32)))
	16907	svint32_t svldnt1ub_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
	16908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_s64)))
	16909	svint64_t svldnt1ub_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16910	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_u32)))
	16911	svuint32_t svldnt1ub_gather_u32base_u32(svbool_t, svuint32_t);
	16912	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_u64)))
	16913	svuint64_t svldnt1ub_gather_u64base_u64(svbool_t, svuint64_t);
	16914	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_s32)))
	16915	svint32_t svldnt1ub_gather_u32base_s32(svbool_t, svuint32_t);
	16916	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_s64)))
	16917	svint64_t svldnt1ub_gather_u64base_s64(svbool_t, svuint64_t);
	16918	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_u32)))
	16919	svuint32_t svldnt1ub_gather_u32offset_u32(svbool_t, uint8_t const *, svuint32_t);
	16920	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_s32)))
	16921	svint32_t svldnt1ub_gather_u32offset_s32(svbool_t, uint8_t const *, svuint32_t);
	16922	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_u64)))
	16923	svuint64_t svldnt1ub_gather_s64offset_u64(svbool_t, uint8_t const *, svint64_t);
	16924	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_s64)))
	16925	svint64_t svldnt1ub_gather_s64offset_s64(svbool_t, uint8_t const *, svint64_t);
	16926	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_u64)))
	16927	svuint64_t svldnt1ub_gather_u64offset_u64(svbool_t, uint8_t const *, svuint64_t);
	16928	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_s64)))
	16929	svint64_t svldnt1ub_gather_u64offset_s64(svbool_t, uint8_t const *, svuint64_t);
	16930	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_u32)))
	16931	svuint32_t svldnt1uh_gather_u32base_index_u32(svbool_t, svuint32_t, int64_t);
	16932	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_u64)))
	16933	svuint64_t svldnt1uh_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	16934	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_s32)))
	16935	svint32_t svldnt1uh_gather_u32base_index_s32(svbool_t, svuint32_t, int64_t);
	16936	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_s64)))
	16937	svint64_t svldnt1uh_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	16938	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_u32)))
	16939	svuint32_t svldnt1uh_gather_u32base_offset_u32(svbool_t, svuint32_t, int64_t);
	16940	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_u64)))
	16941	svuint64_t svldnt1uh_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16942	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_s32)))
	16943	svint32_t svldnt1uh_gather_u32base_offset_s32(svbool_t, svuint32_t, int64_t);
	16944	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_s64)))
	16945	svint64_t svldnt1uh_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16946	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_u32)))
	16947	svuint32_t svldnt1uh_gather_u32base_u32(svbool_t, svuint32_t);
	16948	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_u64)))
	16949	svuint64_t svldnt1uh_gather_u64base_u64(svbool_t, svuint64_t);
	16950	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_s32)))
	16951	svint32_t svldnt1uh_gather_u32base_s32(svbool_t, svuint32_t);
	16952	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_s64)))
	16953	svint64_t svldnt1uh_gather_u64base_s64(svbool_t, svuint64_t);
	16954	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_u64)))
	16955	svuint64_t svldnt1uh_gather_s64index_u64(svbool_t, uint16_t const *, svint64_t);
	16956	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_s64)))
	16957	svint64_t svldnt1uh_gather_s64index_s64(svbool_t, uint16_t const *, svint64_t);
	16958	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_u64)))
	16959	svuint64_t svldnt1uh_gather_u64index_u64(svbool_t, uint16_t const *, svuint64_t);
	16960	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_s64)))
	16961	svint64_t svldnt1uh_gather_u64index_s64(svbool_t, uint16_t const *, svuint64_t);
	16962	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_u32)))
	16963	svuint32_t svldnt1uh_gather_u32offset_u32(svbool_t, uint16_t const *, svuint32_t);
	16964	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_s32)))
	16965	svint32_t svldnt1uh_gather_u32offset_s32(svbool_t, uint16_t const *, svuint32_t);
	16966	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_u64)))
	16967	svuint64_t svldnt1uh_gather_s64offset_u64(svbool_t, uint16_t const *, svint64_t);
	16968	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_s64)))
	16969	svint64_t svldnt1uh_gather_s64offset_s64(svbool_t, uint16_t const *, svint64_t);
	16970	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_u64)))
	16971	svuint64_t svldnt1uh_gather_u64offset_u64(svbool_t, uint16_t const *, svuint64_t);
	16972	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_s64)))
	16973	svint64_t svldnt1uh_gather_u64offset_s64(svbool_t, uint16_t const *, svuint64_t);
	16974	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_u64)))
	16975	svuint64_t svldnt1uw_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	16976	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_s64)))
	16977	svint64_t svldnt1uw_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	16978	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_u64)))
	16979	svuint64_t svldnt1uw_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	16980	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_s64)))
	16981	svint64_t svldnt1uw_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	16982	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_u64)))
	16983	svuint64_t svldnt1uw_gather_u64base_u64(svbool_t, svuint64_t);
	16984	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_s64)))
	16985	svint64_t svldnt1uw_gather_u64base_s64(svbool_t, svuint64_t);
	16986	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_u64)))
	16987	svuint64_t svldnt1uw_gather_s64index_u64(svbool_t, uint32_t const *, svint64_t);
	16988	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_s64)))
	16989	svint64_t svldnt1uw_gather_s64index_s64(svbool_t, uint32_t const *, svint64_t);
	16990	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_u64)))
	16991	svuint64_t svldnt1uw_gather_u64index_u64(svbool_t, uint32_t const *, svuint64_t);
	16992	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_s64)))
	16993	svint64_t svldnt1uw_gather_u64index_s64(svbool_t, uint32_t const *, svuint64_t);
	16994	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_u64)))
	16995	svuint64_t svldnt1uw_gather_s64offset_u64(svbool_t, uint32_t const *, svint64_t);
	16996	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_s64)))
	16997	svint64_t svldnt1uw_gather_s64offset_s64(svbool_t, uint32_t const *, svint64_t);
	16998	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_u64)))
	16999	svuint64_t svldnt1uw_gather_u64offset_u64(svbool_t, uint32_t const *, svuint64_t);
	17000	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_s64)))
	17001	svint64_t svldnt1uw_gather_u64offset_s64(svbool_t, uint32_t const *, svuint64_t);
	17002	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u8)))
	17003	svbool_t svmatch_u8(svbool_t, svuint8_t, svuint8_t);
	17004	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u16)))
	17005	svbool_t svmatch_u16(svbool_t, svuint16_t, svuint16_t);
	17006	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s8)))
	17007	svbool_t svmatch_s8(svbool_t, svint8_t, svint8_t);
	17008	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s16)))
	17009	svbool_t svmatch_s16(svbool_t, svint16_t, svint16_t);
	17010	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u8)))
	17011	svbool_t svnmatch_u8(svbool_t, svuint8_t, svuint8_t);
	17012	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u16)))
	17013	svbool_t svnmatch_u16(svbool_t, svuint16_t, svuint16_t);
	17014	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s8)))
	17015	svbool_t svnmatch_s8(svbool_t, svint8_t, svint8_t);
	17016	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s16)))
	17017	svbool_t svnmatch_s16(svbool_t, svint16_t, svint16_t);
	17018	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_u32)))
	17019	void svstnt1_scatter_u32base_index_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
	17020	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_u64)))
	17021	void svstnt1_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17022	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_f64)))
	17023	void svstnt1_scatter_u64base_index_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17024	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_f32)))
	17025	void svstnt1_scatter_u32base_index_f32(svbool_t, svuint32_t, int64_t, svfloat32_t);
	17026	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_s32)))
	17027	void svstnt1_scatter_u32base_index_s32(svbool_t, svuint32_t, int64_t, svint32_t);
	17028	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_s64)))
	17029	void svstnt1_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17030	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_u32)))
	17031	void svstnt1_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
	17032	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_u64)))
	17033	void svstnt1_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17034	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_f64)))
	17035	void svstnt1_scatter_u64base_offset_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17036	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_f32)))
	17037	void svstnt1_scatter_u32base_offset_f32(svbool_t, svuint32_t, int64_t, svfloat32_t);
	17038	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_s32)))
	17039	void svstnt1_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
	17040	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_s64)))
	17041	void svstnt1_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17042	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_u32)))
	17043	void svstnt1_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
	17044	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_u64)))
	17045	void svstnt1_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
	17046	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_f64)))
	17047	void svstnt1_scatter_u64base_f64(svbool_t, svuint64_t, svfloat64_t);
	17048	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_f32)))
	17049	void svstnt1_scatter_u32base_f32(svbool_t, svuint32_t, svfloat32_t);
	17050	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_s32)))
	17051	void svstnt1_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
	17052	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_s64)))
	17053	void svstnt1_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
	17054	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_u64)))
	17055	void svstnt1_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
	17056	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_f64)))
	17057	void svstnt1_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
	17058	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_s64)))
	17059	void svstnt1_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
	17060	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_u64)))
	17061	void svstnt1_scatter_u64index_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	17062	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_f64)))
	17063	void svstnt1_scatter_u64index_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	17064	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_s64)))
	17065	void svstnt1_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
	17066	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_u32)))
	17067	void svstnt1_scatter_u32offset_u32(svbool_t, uint32_t *, svuint32_t, svuint32_t);
	17068	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_f32)))
	17069	void svstnt1_scatter_u32offset_f32(svbool_t, float32_t *, svuint32_t, svfloat32_t);
	17070	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_s32)))
	17071	void svstnt1_scatter_u32offset_s32(svbool_t, int32_t *, svuint32_t, svint32_t);
	17072	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_u64)))
	17073	void svstnt1_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
	17074	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_f64)))
	17075	void svstnt1_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
	17076	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_s64)))
	17077	void svstnt1_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
	17078	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_u64)))
	17079	void svstnt1_scatter_u64offset_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	17080	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_f64)))
	17081	void svstnt1_scatter_u64offset_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	17082	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_s64)))
	17083	void svstnt1_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
	17084	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_u32)))
	17085	void svstnt1b_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
	17086	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_u64)))
	17087	void svstnt1b_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17088	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_s32)))
	17089	void svstnt1b_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
	17090	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_s64)))
	17091	void svstnt1b_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17092	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_u32)))
	17093	void svstnt1b_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
	17094	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_u64)))
	17095	void svstnt1b_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
	17096	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_s32)))
	17097	void svstnt1b_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
	17098	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_s64)))
	17099	void svstnt1b_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
	17100	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_s32)))
	17101	void svstnt1b_scatter_u32offset_s32(svbool_t, int8_t *, svuint32_t, svint32_t);
	17102	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_u32)))
	17103	void svstnt1b_scatter_u32offset_u32(svbool_t, uint8_t *, svuint32_t, svuint32_t);
	17104	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_s64)))
	17105	void svstnt1b_scatter_s64offset_s64(svbool_t, int8_t *, svint64_t, svint64_t);
	17106	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_u64)))
	17107	void svstnt1b_scatter_s64offset_u64(svbool_t, uint8_t *, svint64_t, svuint64_t);
	17108	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_s64)))
	17109	void svstnt1b_scatter_u64offset_s64(svbool_t, int8_t *, svuint64_t, svint64_t);
	17110	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_u64)))
	17111	void svstnt1b_scatter_u64offset_u64(svbool_t, uint8_t *, svuint64_t, svuint64_t);
	17112	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_u32)))
	17113	void svstnt1h_scatter_u32base_index_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
	17114	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_u64)))
	17115	void svstnt1h_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17116	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_s32)))
	17117	void svstnt1h_scatter_u32base_index_s32(svbool_t, svuint32_t, int64_t, svint32_t);
	17118	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_s64)))
	17119	void svstnt1h_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17120	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_u32)))
	17121	void svstnt1h_scatter_u32base_offset_u32(svbool_t, svuint32_t, int64_t, svuint32_t);
	17122	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_u64)))
	17123	void svstnt1h_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17124	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_s32)))
	17125	void svstnt1h_scatter_u32base_offset_s32(svbool_t, svuint32_t, int64_t, svint32_t);
	17126	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_s64)))
	17127	void svstnt1h_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17128	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_u32)))
	17129	void svstnt1h_scatter_u32base_u32(svbool_t, svuint32_t, svuint32_t);
	17130	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_u64)))
	17131	void svstnt1h_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
	17132	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_s32)))
	17133	void svstnt1h_scatter_u32base_s32(svbool_t, svuint32_t, svint32_t);
	17134	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_s64)))
	17135	void svstnt1h_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
	17136	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_s64)))
	17137	void svstnt1h_scatter_s64index_s64(svbool_t, int16_t *, svint64_t, svint64_t);
	17138	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_u64)))
	17139	void svstnt1h_scatter_s64index_u64(svbool_t, uint16_t *, svint64_t, svuint64_t);
	17140	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_s64)))
	17141	void svstnt1h_scatter_u64index_s64(svbool_t, int16_t *, svuint64_t, svint64_t);
	17142	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_u64)))
	17143	void svstnt1h_scatter_u64index_u64(svbool_t, uint16_t *, svuint64_t, svuint64_t);
	17144	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_s32)))
	17145	void svstnt1h_scatter_u32offset_s32(svbool_t, int16_t *, svuint32_t, svint32_t);
	17146	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_u32)))
	17147	void svstnt1h_scatter_u32offset_u32(svbool_t, uint16_t *, svuint32_t, svuint32_t);
	17148	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_s64)))
	17149	void svstnt1h_scatter_s64offset_s64(svbool_t, int16_t *, svint64_t, svint64_t);
	17150	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_u64)))
	17151	void svstnt1h_scatter_s64offset_u64(svbool_t, uint16_t *, svint64_t, svuint64_t);
	17152	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_s64)))
	17153	void svstnt1h_scatter_u64offset_s64(svbool_t, int16_t *, svuint64_t, svint64_t);
	17154	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_u64)))
	17155	void svstnt1h_scatter_u64offset_u64(svbool_t, uint16_t *, svuint64_t, svuint64_t);
	17156	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_u64)))
	17157	void svstnt1w_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17158	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_s64)))
	17159	void svstnt1w_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17160	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_u64)))
	17161	void svstnt1w_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17162	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_s64)))
	17163	void svstnt1w_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17164	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_u64)))
	17165	void svstnt1w_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
	17166	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_s64)))
	17167	void svstnt1w_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
	17168	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_s64)))
	17169	void svstnt1w_scatter_s64index_s64(svbool_t, int32_t *, svint64_t, svint64_t);
	17170	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_u64)))
	17171	void svstnt1w_scatter_s64index_u64(svbool_t, uint32_t *, svint64_t, svuint64_t);
	17172	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_s64)))
	17173	void svstnt1w_scatter_u64index_s64(svbool_t, int32_t *, svuint64_t, svint64_t);
	17174	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_u64)))
	17175	void svstnt1w_scatter_u64index_u64(svbool_t, uint32_t *, svuint64_t, svuint64_t);
	17176	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_s64)))
	17177	void svstnt1w_scatter_s64offset_s64(svbool_t, int32_t *, svint64_t, svint64_t);
	17178	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_u64)))
	17179	void svstnt1w_scatter_s64offset_u64(svbool_t, uint32_t *, svint64_t, svuint64_t);
	17180	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_s64)))
	17181	void svstnt1w_scatter_u64offset_s64(svbool_t, int32_t *, svuint64_t, svint64_t);
	17182	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
	17183	void svstnt1w_scatter_u64offset_u64(svbool_t, uint32_t *, svuint64_t, svuint64_t);
	17184	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u32_z)))
	17185	svuint32_t svhistcnt_z(svbool_t, svuint32_t, svuint32_t);
	17186	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_u64_z)))
	17187	svuint64_t svhistcnt_z(svbool_t, svuint64_t, svuint64_t);
	17188	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s32_z)))
	17189	svuint32_t svhistcnt_z(svbool_t, svint32_t, svint32_t);
	17190	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistcnt_s64_z)))
	17191	svuint64_t svhistcnt_z(svbool_t, svint64_t, svint64_t);
	17192	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_u8)))
	17193	svuint8_t svhistseg(svuint8_t, svuint8_t);
	17194	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svhistseg_s8)))
	17195	svuint8_t svhistseg(svint8_t, svint8_t);
	17196	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_u32)))
	17197	svuint32_t svldnt1_gather_index_u32(svbool_t, svuint32_t, int64_t);
	17198	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_u64)))
	17199	svuint64_t svldnt1_gather_index_u64(svbool_t, svuint64_t, int64_t);
	17200	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_f64)))
	17201	svfloat64_t svldnt1_gather_index_f64(svbool_t, svuint64_t, int64_t);
	17202	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_f32)))
	17203	svfloat32_t svldnt1_gather_index_f32(svbool_t, svuint32_t, int64_t);
	17204	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_index_s32)))
	17205	svint32_t svldnt1_gather_index_s32(svbool_t, svuint32_t, int64_t);
	17206	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_index_s64)))
	17207	svint64_t svldnt1_gather_index_s64(svbool_t, svuint64_t, int64_t);
	17208	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_u32)))
	17209	svuint32_t svldnt1_gather_offset_u32(svbool_t, svuint32_t, int64_t);
	17210	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_u64)))
	17211	svuint64_t svldnt1_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17212	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_f64)))
	17213	svfloat64_t svldnt1_gather_offset_f64(svbool_t, svuint64_t, int64_t);
	17214	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_f32)))
	17215	svfloat32_t svldnt1_gather_offset_f32(svbool_t, svuint32_t, int64_t);
	17216	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_offset_s32)))
	17217	svint32_t svldnt1_gather_offset_s32(svbool_t, svuint32_t, int64_t);
	17218	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_offset_s64)))
	17219	svint64_t svldnt1_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17220	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_u32)))
	17221	svuint32_t svldnt1_gather_u32(svbool_t, svuint32_t);
	17222	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_u64)))
	17223	svuint64_t svldnt1_gather_u64(svbool_t, svuint64_t);
	17224	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_f64)))
	17225	svfloat64_t svldnt1_gather_f64(svbool_t, svuint64_t);
	17226	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_f32)))
	17227	svfloat32_t svldnt1_gather_f32(svbool_t, svuint32_t);
	17228	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32base_s32)))
	17229	svint32_t svldnt1_gather_s32(svbool_t, svuint32_t);
	17230	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64base_s64)))
	17231	svint64_t svldnt1_gather_s64(svbool_t, svuint64_t);
	17232	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_u64)))
	17233	svuint64_t svldnt1_gather_index(svbool_t, uint64_t const *, svint64_t);
	17234	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_f64)))
	17235	svfloat64_t svldnt1_gather_index(svbool_t, float64_t const *, svint64_t);
	17236	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64index_s64)))
	17237	svint64_t svldnt1_gather_index(svbool_t, int64_t const *, svint64_t);
	17238	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_u64)))
	17239	svuint64_t svldnt1_gather_index(svbool_t, uint64_t const *, svuint64_t);
	17240	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_f64)))
	17241	svfloat64_t svldnt1_gather_index(svbool_t, float64_t const *, svuint64_t);
	17242	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64index_s64)))
	17243	svint64_t svldnt1_gather_index(svbool_t, int64_t const *, svuint64_t);
	17244	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_u32)))
	17245	svuint32_t svldnt1_gather_offset(svbool_t, uint32_t const *, svuint32_t);
	17246	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_f32)))
	17247	svfloat32_t svldnt1_gather_offset(svbool_t, float32_t const *, svuint32_t);
	17248	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u32offset_s32)))
	17249	svint32_t svldnt1_gather_offset(svbool_t, int32_t const *, svuint32_t);
	17250	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_u64)))
	17251	svuint64_t svldnt1_gather_offset(svbool_t, uint64_t const *, svint64_t);
	17252	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_f64)))
	17253	svfloat64_t svldnt1_gather_offset(svbool_t, float64_t const *, svint64_t);
	17254	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_s64offset_s64)))
	17255	svint64_t svldnt1_gather_offset(svbool_t, int64_t const *, svint64_t);
	17256	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_u64)))
	17257	svuint64_t svldnt1_gather_offset(svbool_t, uint64_t const *, svuint64_t);
	17258	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_f64)))
	17259	svfloat64_t svldnt1_gather_offset(svbool_t, float64_t const *, svuint64_t);
	17260	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_gather_u64offset_s64)))
	17261	svint64_t svldnt1_gather_offset(svbool_t, int64_t const *, svuint64_t);
	17262	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_u32)))
	17263	svuint32_t svldnt1sb_gather_offset_u32(svbool_t, svuint32_t, int64_t);
	17264	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_u64)))
	17265	svuint64_t svldnt1sb_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17266	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_offset_s32)))
	17267	svint32_t svldnt1sb_gather_offset_s32(svbool_t, svuint32_t, int64_t);
	17268	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_offset_s64)))
	17269	svint64_t svldnt1sb_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17270	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_u32)))
	17271	svuint32_t svldnt1sb_gather_u32(svbool_t, svuint32_t);
	17272	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_u64)))
	17273	svuint64_t svldnt1sb_gather_u64(svbool_t, svuint64_t);
	17274	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32base_s32)))
	17275	svint32_t svldnt1sb_gather_s32(svbool_t, svuint32_t);
	17276	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64base_s64)))
	17277	svint64_t svldnt1sb_gather_s64(svbool_t, svuint64_t);
	17278	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_u32)))
	17279	svuint32_t svldnt1sb_gather_offset_u32(svbool_t, int8_t const *, svuint32_t);
	17280	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u32offset_s32)))
	17281	svint32_t svldnt1sb_gather_offset_s32(svbool_t, int8_t const *, svuint32_t);
	17282	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_u64)))
	17283	svuint64_t svldnt1sb_gather_offset_u64(svbool_t, int8_t const *, svint64_t);
	17284	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_s64offset_s64)))
	17285	svint64_t svldnt1sb_gather_offset_s64(svbool_t, int8_t const *, svint64_t);
	17286	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_u64)))
	17287	svuint64_t svldnt1sb_gather_offset_u64(svbool_t, int8_t const *, svuint64_t);
	17288	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sb_gather_u64offset_s64)))
	17289	svint64_t svldnt1sb_gather_offset_s64(svbool_t, int8_t const *, svuint64_t);
	17290	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_u32)))
	17291	svuint32_t svldnt1sh_gather_index_u32(svbool_t, svuint32_t, int64_t);
	17292	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_u64)))
	17293	svuint64_t svldnt1sh_gather_index_u64(svbool_t, svuint64_t, int64_t);
	17294	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_index_s32)))
	17295	svint32_t svldnt1sh_gather_index_s32(svbool_t, svuint32_t, int64_t);
	17296	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_index_s64)))
	17297	svint64_t svldnt1sh_gather_index_s64(svbool_t, svuint64_t, int64_t);
	17298	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_u32)))
	17299	svuint32_t svldnt1sh_gather_offset_u32(svbool_t, svuint32_t, int64_t);
	17300	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_u64)))
	17301	svuint64_t svldnt1sh_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17302	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_offset_s32)))
	17303	svint32_t svldnt1sh_gather_offset_s32(svbool_t, svuint32_t, int64_t);
	17304	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_offset_s64)))
	17305	svint64_t svldnt1sh_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17306	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_u32)))
	17307	svuint32_t svldnt1sh_gather_u32(svbool_t, svuint32_t);
	17308	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_u64)))
	17309	svuint64_t svldnt1sh_gather_u64(svbool_t, svuint64_t);
	17310	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32base_s32)))
	17311	svint32_t svldnt1sh_gather_s32(svbool_t, svuint32_t);
	17312	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64base_s64)))
	17313	svint64_t svldnt1sh_gather_s64(svbool_t, svuint64_t);
	17314	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_u64)))
	17315	svuint64_t svldnt1sh_gather_index_u64(svbool_t, int16_t const *, svint64_t);
	17316	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64index_s64)))
	17317	svint64_t svldnt1sh_gather_index_s64(svbool_t, int16_t const *, svint64_t);
	17318	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_u64)))
	17319	svuint64_t svldnt1sh_gather_index_u64(svbool_t, int16_t const *, svuint64_t);
	17320	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64index_s64)))
	17321	svint64_t svldnt1sh_gather_index_s64(svbool_t, int16_t const *, svuint64_t);
	17322	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_u32)))
	17323	svuint32_t svldnt1sh_gather_offset_u32(svbool_t, int16_t const *, svuint32_t);
	17324	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u32offset_s32)))
	17325	svint32_t svldnt1sh_gather_offset_s32(svbool_t, int16_t const *, svuint32_t);
	17326	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_u64)))
	17327	svuint64_t svldnt1sh_gather_offset_u64(svbool_t, int16_t const *, svint64_t);
	17328	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_s64offset_s64)))
	17329	svint64_t svldnt1sh_gather_offset_s64(svbool_t, int16_t const *, svint64_t);
	17330	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_u64)))
	17331	svuint64_t svldnt1sh_gather_offset_u64(svbool_t, int16_t const *, svuint64_t);
	17332	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sh_gather_u64offset_s64)))
	17333	svint64_t svldnt1sh_gather_offset_s64(svbool_t, int16_t const *, svuint64_t);
	17334	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_u64)))
	17335	svuint64_t svldnt1sw_gather_index_u64(svbool_t, svuint64_t, int64_t);
	17336	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_index_s64)))
	17337	svint64_t svldnt1sw_gather_index_s64(svbool_t, svuint64_t, int64_t);
	17338	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_u64)))
	17339	svuint64_t svldnt1sw_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17340	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_offset_s64)))
	17341	svint64_t svldnt1sw_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17342	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_u64)))
	17343	svuint64_t svldnt1sw_gather_u64(svbool_t, svuint64_t);
	17344	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64base_s64)))
	17345	svint64_t svldnt1sw_gather_s64(svbool_t, svuint64_t);
	17346	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_u64)))
	17347	svuint64_t svldnt1sw_gather_index_u64(svbool_t, int32_t const *, svint64_t);
	17348	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64index_s64)))
	17349	svint64_t svldnt1sw_gather_index_s64(svbool_t, int32_t const *, svint64_t);
	17350	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_u64)))
	17351	svuint64_t svldnt1sw_gather_index_u64(svbool_t, int32_t const *, svuint64_t);
	17352	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64index_s64)))
	17353	svint64_t svldnt1sw_gather_index_s64(svbool_t, int32_t const *, svuint64_t);
	17354	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_u64)))
	17355	svuint64_t svldnt1sw_gather_offset_u64(svbool_t, int32_t const *, svint64_t);
	17356	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_s64offset_s64)))
	17357	svint64_t svldnt1sw_gather_offset_s64(svbool_t, int32_t const *, svint64_t);
	17358	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_u64)))
	17359	svuint64_t svldnt1sw_gather_offset_u64(svbool_t, int32_t const *, svuint64_t);
	17360	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1sw_gather_u64offset_s64)))
	17361	svint64_t svldnt1sw_gather_offset_s64(svbool_t, int32_t const *, svuint64_t);
	17362	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_u32)))
	17363	svuint32_t svldnt1ub_gather_offset_u32(svbool_t, svuint32_t, int64_t);
	17364	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_u64)))
	17365	svuint64_t svldnt1ub_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17366	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_offset_s32)))
	17367	svint32_t svldnt1ub_gather_offset_s32(svbool_t, svuint32_t, int64_t);
	17368	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_offset_s64)))
	17369	svint64_t svldnt1ub_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17370	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_u32)))
	17371	svuint32_t svldnt1ub_gather_u32(svbool_t, svuint32_t);
	17372	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_u64)))
	17373	svuint64_t svldnt1ub_gather_u64(svbool_t, svuint64_t);
	17374	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32base_s32)))
	17375	svint32_t svldnt1ub_gather_s32(svbool_t, svuint32_t);
	17376	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64base_s64)))
	17377	svint64_t svldnt1ub_gather_s64(svbool_t, svuint64_t);
	17378	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_u32)))
	17379	svuint32_t svldnt1ub_gather_offset_u32(svbool_t, uint8_t const *, svuint32_t);
	17380	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u32offset_s32)))
	17381	svint32_t svldnt1ub_gather_offset_s32(svbool_t, uint8_t const *, svuint32_t);
	17382	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_u64)))
	17383	svuint64_t svldnt1ub_gather_offset_u64(svbool_t, uint8_t const *, svint64_t);
	17384	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_s64offset_s64)))
	17385	svint64_t svldnt1ub_gather_offset_s64(svbool_t, uint8_t const *, svint64_t);
	17386	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_u64)))
	17387	svuint64_t svldnt1ub_gather_offset_u64(svbool_t, uint8_t const *, svuint64_t);
	17388	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1ub_gather_u64offset_s64)))
	17389	svint64_t svldnt1ub_gather_offset_s64(svbool_t, uint8_t const *, svuint64_t);
	17390	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_u32)))
	17391	svuint32_t svldnt1uh_gather_index_u32(svbool_t, svuint32_t, int64_t);
	17392	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_u64)))
	17393	svuint64_t svldnt1uh_gather_index_u64(svbool_t, svuint64_t, int64_t);
	17394	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_index_s32)))
	17395	svint32_t svldnt1uh_gather_index_s32(svbool_t, svuint32_t, int64_t);
	17396	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_index_s64)))
	17397	svint64_t svldnt1uh_gather_index_s64(svbool_t, svuint64_t, int64_t);
	17398	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_u32)))
	17399	svuint32_t svldnt1uh_gather_offset_u32(svbool_t, svuint32_t, int64_t);
	17400	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_u64)))
	17401	svuint64_t svldnt1uh_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17402	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_offset_s32)))
	17403	svint32_t svldnt1uh_gather_offset_s32(svbool_t, svuint32_t, int64_t);
	17404	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_offset_s64)))
	17405	svint64_t svldnt1uh_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17406	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_u32)))
	17407	svuint32_t svldnt1uh_gather_u32(svbool_t, svuint32_t);
	17408	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_u64)))
	17409	svuint64_t svldnt1uh_gather_u64(svbool_t, svuint64_t);
	17410	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32base_s32)))
	17411	svint32_t svldnt1uh_gather_s32(svbool_t, svuint32_t);
	17412	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64base_s64)))
	17413	svint64_t svldnt1uh_gather_s64(svbool_t, svuint64_t);
	17414	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_u64)))
	17415	svuint64_t svldnt1uh_gather_index_u64(svbool_t, uint16_t const *, svint64_t);
	17416	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64index_s64)))
	17417	svint64_t svldnt1uh_gather_index_s64(svbool_t, uint16_t const *, svint64_t);
	17418	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_u64)))
	17419	svuint64_t svldnt1uh_gather_index_u64(svbool_t, uint16_t const *, svuint64_t);
	17420	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64index_s64)))
	17421	svint64_t svldnt1uh_gather_index_s64(svbool_t, uint16_t const *, svuint64_t);
	17422	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_u32)))
	17423	svuint32_t svldnt1uh_gather_offset_u32(svbool_t, uint16_t const *, svuint32_t);
	17424	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u32offset_s32)))
	17425	svint32_t svldnt1uh_gather_offset_s32(svbool_t, uint16_t const *, svuint32_t);
	17426	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_u64)))
	17427	svuint64_t svldnt1uh_gather_offset_u64(svbool_t, uint16_t const *, svint64_t);
	17428	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_s64offset_s64)))
	17429	svint64_t svldnt1uh_gather_offset_s64(svbool_t, uint16_t const *, svint64_t);
	17430	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_u64)))
	17431	svuint64_t svldnt1uh_gather_offset_u64(svbool_t, uint16_t const *, svuint64_t);
	17432	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uh_gather_u64offset_s64)))
	17433	svint64_t svldnt1uh_gather_offset_s64(svbool_t, uint16_t const *, svuint64_t);
	17434	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_u64)))
	17435	svuint64_t svldnt1uw_gather_index_u64(svbool_t, svuint64_t, int64_t);
	17436	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_index_s64)))
	17437	svint64_t svldnt1uw_gather_index_s64(svbool_t, svuint64_t, int64_t);
	17438	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_u64)))
	17439	svuint64_t svldnt1uw_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	17440	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_offset_s64)))
	17441	svint64_t svldnt1uw_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	17442	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_u64)))
	17443	svuint64_t svldnt1uw_gather_u64(svbool_t, svuint64_t);
	17444	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64base_s64)))
	17445	svint64_t svldnt1uw_gather_s64(svbool_t, svuint64_t);
	17446	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_u64)))
	17447	svuint64_t svldnt1uw_gather_index_u64(svbool_t, uint32_t const *, svint64_t);
	17448	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64index_s64)))
	17449	svint64_t svldnt1uw_gather_index_s64(svbool_t, uint32_t const *, svint64_t);
	17450	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_u64)))
	17451	svuint64_t svldnt1uw_gather_index_u64(svbool_t, uint32_t const *, svuint64_t);
	17452	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64index_s64)))
	17453	svint64_t svldnt1uw_gather_index_s64(svbool_t, uint32_t const *, svuint64_t);
	17454	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_u64)))
	17455	svuint64_t svldnt1uw_gather_offset_u64(svbool_t, uint32_t const *, svint64_t);
	17456	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_s64offset_s64)))
	17457	svint64_t svldnt1uw_gather_offset_s64(svbool_t, uint32_t const *, svint64_t);
	17458	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_u64)))
	17459	svuint64_t svldnt1uw_gather_offset_u64(svbool_t, uint32_t const *, svuint64_t);
	17460	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1uw_gather_u64offset_s64)))
	17461	svint64_t svldnt1uw_gather_offset_s64(svbool_t, uint32_t const *, svuint64_t);
	17462	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u8)))
	17463	svbool_t svmatch(svbool_t, svuint8_t, svuint8_t);
	17464	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_u16)))
	17465	svbool_t svmatch(svbool_t, svuint16_t, svuint16_t);
	17466	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s8)))
	17467	svbool_t svmatch(svbool_t, svint8_t, svint8_t);
	17468	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmatch_s16)))
	17469	svbool_t svmatch(svbool_t, svint16_t, svint16_t);
	17470	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u8)))
	17471	svbool_t svnmatch(svbool_t, svuint8_t, svuint8_t);
	17472	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_u16)))
	17473	svbool_t svnmatch(svbool_t, svuint16_t, svuint16_t);
	17474	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s8)))
	17475	svbool_t svnmatch(svbool_t, svint8_t, svint8_t);
	17476	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svnmatch_s16)))
	17477	svbool_t svnmatch(svbool_t, svint16_t, svint16_t);
	17478	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_u32)))
	17479	void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svuint32_t);
	17480	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_u64)))
	17481	void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
	17482	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_f64)))
	17483	void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17484	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_f32)))
	17485	void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svfloat32_t);
	17486	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_index_s32)))
	17487	void svstnt1_scatter_index(svbool_t, svuint32_t, int64_t, svint32_t);
	17488	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_index_s64)))
	17489	void svstnt1_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
	17490	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_u32)))
	17491	void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
	17492	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_u64)))
	17493	void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
	17494	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_f64)))
	17495	void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17496	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_f32)))
	17497	void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svfloat32_t);
	17498	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_offset_s32)))
	17499	void svstnt1_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
	17500	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_offset_s64)))
	17501	void svstnt1_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
	17502	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_u32)))
	17503	void svstnt1_scatter(svbool_t, svuint32_t, svuint32_t);
	17504	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_u64)))
	17505	void svstnt1_scatter(svbool_t, svuint64_t, svuint64_t);
	17506	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_f64)))
	17507	void svstnt1_scatter(svbool_t, svuint64_t, svfloat64_t);
	17508	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_f32)))
	17509	void svstnt1_scatter(svbool_t, svuint32_t, svfloat32_t);
	17510	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32base_s32)))
	17511	void svstnt1_scatter(svbool_t, svuint32_t, svint32_t);
	17512	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64base_s64)))
	17513	void svstnt1_scatter(svbool_t, svuint64_t, svint64_t);
	17514	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_u64)))
	17515	void svstnt1_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
	17516	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_f64)))
	17517	void svstnt1_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
	17518	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64index_s64)))
	17519	void svstnt1_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
	17520	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_u64)))
	17521	void svstnt1_scatter_index(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	17522	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_f64)))
	17523	void svstnt1_scatter_index(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	17524	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64index_s64)))
	17525	void svstnt1_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
	17526	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_u32)))
	17527	void svstnt1_scatter_offset(svbool_t, uint32_t *, svuint32_t, svuint32_t);
	17528	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_f32)))
	17529	void svstnt1_scatter_offset(svbool_t, float32_t *, svuint32_t, svfloat32_t);
	17530	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u32offset_s32)))
	17531	void svstnt1_scatter_offset(svbool_t, int32_t *, svuint32_t, svint32_t);
	17532	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_u64)))
	17533	void svstnt1_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
	17534	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_f64)))
	17535	void svstnt1_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
	17536	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_s64offset_s64)))
	17537	void svstnt1_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
	17538	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_u64)))
	17539	void svstnt1_scatter_offset(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	17540	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_f64)))
	17541	void svstnt1_scatter_offset(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	17542	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_scatter_u64offset_s64)))
	17543	void svstnt1_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
	17544	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_u32)))
	17545	void svstnt1b_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
	17546	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_u64)))
	17547	void svstnt1b_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
	17548	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_offset_s32)))
	17549	void svstnt1b_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
	17550	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_offset_s64)))
	17551	void svstnt1b_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
	17552	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_u32)))
	17553	void svstnt1b_scatter(svbool_t, svuint32_t, svuint32_t);
	17554	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_u64)))
	17555	void svstnt1b_scatter(svbool_t, svuint64_t, svuint64_t);
	17556	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32base_s32)))
	17557	void svstnt1b_scatter(svbool_t, svuint32_t, svint32_t);
	17558	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64base_s64)))
	17559	void svstnt1b_scatter(svbool_t, svuint64_t, svint64_t);
	17560	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_s32)))
	17561	void svstnt1b_scatter_offset(svbool_t, int8_t *, svuint32_t, svint32_t);
	17562	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u32offset_u32)))
	17563	void svstnt1b_scatter_offset(svbool_t, uint8_t *, svuint32_t, svuint32_t);
	17564	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_s64)))
	17565	void svstnt1b_scatter_offset(svbool_t, int8_t *, svint64_t, svint64_t);
	17566	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_s64offset_u64)))
	17567	void svstnt1b_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint64_t);
	17568	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_s64)))
	17569	void svstnt1b_scatter_offset(svbool_t, int8_t *, svuint64_t, svint64_t);
	17570	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1b_scatter_u64offset_u64)))
	17571	void svstnt1b_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint64_t);
	17572	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_u32)))
	17573	void svstnt1h_scatter_index(svbool_t, svuint32_t, int64_t, svuint32_t);
	17574	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_u64)))
	17575	void svstnt1h_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
	17576	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_index_s32)))
	17577	void svstnt1h_scatter_index(svbool_t, svuint32_t, int64_t, svint32_t);
	17578	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_index_s64)))
	17579	void svstnt1h_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
	17580	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_u32)))
	17581	void svstnt1h_scatter_offset(svbool_t, svuint32_t, int64_t, svuint32_t);
	17582	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_u64)))
	17583	void svstnt1h_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
	17584	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_offset_s32)))
	17585	void svstnt1h_scatter_offset(svbool_t, svuint32_t, int64_t, svint32_t);
	17586	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_offset_s64)))
	17587	void svstnt1h_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
	17588	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_u32)))
	17589	void svstnt1h_scatter(svbool_t, svuint32_t, svuint32_t);
	17590	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_u64)))
	17591	void svstnt1h_scatter(svbool_t, svuint64_t, svuint64_t);
	17592	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32base_s32)))
	17593	void svstnt1h_scatter(svbool_t, svuint32_t, svint32_t);
	17594	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64base_s64)))
	17595	void svstnt1h_scatter(svbool_t, svuint64_t, svint64_t);
	17596	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_s64)))
	17597	void svstnt1h_scatter_index(svbool_t, int16_t *, svint64_t, svint64_t);
	17598	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64index_u64)))
	17599	void svstnt1h_scatter_index(svbool_t, uint16_t *, svint64_t, svuint64_t);
	17600	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_s64)))
	17601	void svstnt1h_scatter_index(svbool_t, int16_t *, svuint64_t, svint64_t);
	17602	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64index_u64)))
	17603	void svstnt1h_scatter_index(svbool_t, uint16_t *, svuint64_t, svuint64_t);
	17604	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_s32)))
	17605	void svstnt1h_scatter_offset(svbool_t, int16_t *, svuint32_t, svint32_t);
	17606	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u32offset_u32)))
	17607	void svstnt1h_scatter_offset(svbool_t, uint16_t *, svuint32_t, svuint32_t);
	17608	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_s64)))
	17609	void svstnt1h_scatter_offset(svbool_t, int16_t *, svint64_t, svint64_t);
	17610	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_s64offset_u64)))
	17611	void svstnt1h_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint64_t);
	17612	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_s64)))
	17613	void svstnt1h_scatter_offset(svbool_t, int16_t *, svuint64_t, svint64_t);
	17614	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1h_scatter_u64offset_u64)))
	17615	void svstnt1h_scatter_offset(svbool_t, uint16_t *, svuint64_t, svuint64_t);
	17616	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_u64)))
	17617	void svstnt1w_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
	17618	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_index_s64)))
	17619	void svstnt1w_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
	17620	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_u64)))
	17621	void svstnt1w_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
	17622	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_offset_s64)))
	17623	void svstnt1w_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
	17624	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_u64)))
	17625	void svstnt1w_scatter(svbool_t, svuint64_t, svuint64_t);
	17626	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64base_s64)))
	17627	void svstnt1w_scatter(svbool_t, svuint64_t, svint64_t);
	17628	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_s64)))
	17629	void svstnt1w_scatter_index(svbool_t, int32_t *, svint64_t, svint64_t);
	17630	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64index_u64)))
	17631	void svstnt1w_scatter_index(svbool_t, uint32_t *, svint64_t, svuint64_t);
	17632	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_s64)))
	17633	void svstnt1w_scatter_index(svbool_t, int32_t *, svuint64_t, svint64_t);
	17634	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64index_u64)))
	17635	void svstnt1w_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint64_t);
	17636	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_s64)))
	17637	void svstnt1w_scatter_offset(svbool_t, int32_t *, svint64_t, svint64_t);
	17638	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_s64offset_u64)))
	17639	void svstnt1w_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint64_t);
	17640	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_s64)))
	17641	void svstnt1w_scatter_offset(svbool_t, int32_t *, svuint64_t, svint64_t);
	17642	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1w_scatter_u64offset_u64)))
	17643	void svstnt1w_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint64_t);
	17644	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16_mf8_fpm)))
	17645	svfloat16_t svmmla_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17646	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f16_mf8_fpm)))
	17647	svfloat16_t svmmla_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17648	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32_mf8_fpm)))
	17649	svfloat32_t svmmla_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17650	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmmla_f32_mf8_fpm)))
	17651	svfloat32_t svmmla_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17652	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
	17653	svfloat16_t svdot_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17654	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
	17655	svfloat16_t svdot_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17656	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
	17657	svfloat16_t svdot_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17658	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f16_mf8_fpm)))
	17659	svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17660	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f16_mf8_fpm)))
	17661	svfloat16_t svdot_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17662	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f16_mf8_fpm)))
	17663	svfloat16_t svdot_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17664	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
	17665	svfloat32_t svdot_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17666	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
	17667	svfloat32_t svdot_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17668	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
	17669	svfloat32_t svdot_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17670	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_f32_mf8_fpm)))
	17671	svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17672	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_n_f32_mf8_fpm)))
	17673	svfloat32_t svdot_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17674	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdot_lane_f32_mf8_fpm)))
	17675	svfloat32_t svdot_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17676	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
	17677	svfloat16_t svmlalb_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17678	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
	17679	svfloat16_t svmlalb_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17680	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
	17681	svfloat16_t svmlalb_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17682	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
	17683	svfloat32_t svmlallbb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17684	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
	17685	svfloat32_t svmlallbb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17686	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
	17687	svfloat32_t svmlallbb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17688	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
	17689	svfloat32_t svmlallbt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17690	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
	17691	svfloat32_t svmlallbt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17692	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
	17693	svfloat32_t svmlallbt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17694	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
	17695	svfloat32_t svmlalltb_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17696	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
	17697	svfloat32_t svmlalltb_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17698	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
	17699	svfloat32_t svmlalltb_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17700	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
	17701	svfloat32_t svmlalltt_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17702	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
	17703	svfloat32_t svmlalltt_n_f32_mf8_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17704	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
	17705	svfloat32_t svmlalltt_lane_f32_mf8_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17706	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
	17707	svfloat16_t svmlalt_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17708	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
	17709	svfloat16_t svmlalt_n_f16_mf8_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17710	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
	17711	svfloat16_t svmlalt_lane_f16_mf8_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17712	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_f16_mf8_fpm)))
	17713	svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17714	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_n_f16_mf8_fpm)))
	17715	svfloat16_t svmlalb_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17716	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalb_lane_f16_mf8_fpm)))
	17717	svfloat16_t svmlalb_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17718	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_f32_mf8_fpm)))
	17719	svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17720	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_n_f32_mf8_fpm)))
	17721	svfloat32_t svmlallbb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17722	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbb_lane_f32_mf8_fpm)))
	17723	svfloat32_t svmlallbb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17724	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_f32_mf8_fpm)))
	17725	svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17726	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_n_f32_mf8_fpm)))
	17727	svfloat32_t svmlallbt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17728	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlallbt_lane_f32_mf8_fpm)))
	17729	svfloat32_t svmlallbt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17730	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_f32_mf8_fpm)))
	17731	svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17732	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_n_f32_mf8_fpm)))
	17733	svfloat32_t svmlalltb_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17734	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltb_lane_f32_mf8_fpm)))
	17735	svfloat32_t svmlalltb_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17736	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_f32_mf8_fpm)))
	17737	svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17738	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_n_f32_mf8_fpm)))
	17739	svfloat32_t svmlalltt_fpm(svfloat32_t, svmfloat8_t, mfloat8_t, fpm_t);
	17740	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalltt_lane_f32_mf8_fpm)))
	17741	svfloat32_t svmlalltt_lane_fpm(svfloat32_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17742	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_f16_mf8_fpm)))
	17743	svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, fpm_t);
	17744	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_n_f16_mf8_fpm)))
	17745	svfloat16_t svmlalt_fpm(svfloat16_t, svmfloat8_t, mfloat8_t, fpm_t);
	17746	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svmlalt_lane_f16_mf8_fpm)))
	17747	svfloat16_t svmlalt_lane_fpm(svfloat16_t, svmfloat8_t, svmfloat8_t, uint64_t, fpm_t);
	17748	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
	17749	svuint32_t svld1q_gather_u64base_index_u32(svbool_t, svuint64_t, int64_t);
	17750	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u64)))
	17751	svuint64_t svld1q_gather_u64base_index_u64(svbool_t, svuint64_t, int64_t);
	17752	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u16)))
	17753	svuint16_t svld1q_gather_u64base_index_u16(svbool_t, svuint64_t, int64_t);
	17754	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_bf16)))
	17755	svbfloat16_t svld1q_gather_u64base_index_bf16(svbool_t, svuint64_t, int64_t);
	17756	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f64)))
	17757	svfloat64_t svld1q_gather_u64base_index_f64(svbool_t, svuint64_t, int64_t);
	17758	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f32)))
	17759	svfloat32_t svld1q_gather_u64base_index_f32(svbool_t, svuint64_t, int64_t);
	17760	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f16)))
	17761	svfloat16_t svld1q_gather_u64base_index_f16(svbool_t, svuint64_t, int64_t);
	17762	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s32)))
	17763	svint32_t svld1q_gather_u64base_index_s32(svbool_t, svuint64_t, int64_t);
	17764	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s64)))
	17765	svint64_t svld1q_gather_u64base_index_s64(svbool_t, svuint64_t, int64_t);
	17766	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s16)))
	17767	svint16_t svld1q_gather_u64base_index_s16(svbool_t, svuint64_t, int64_t);
	17768	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u8)))
	17769	svuint8_t svld1q_gather_u64base_offset_u8(svbool_t, svuint64_t, int64_t);
	17770	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u32)))
	17771	svuint32_t svld1q_gather_u64base_offset_u32(svbool_t, svuint64_t, int64_t);
	17772	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u64)))
	17773	svuint64_t svld1q_gather_u64base_offset_u64(svbool_t, svuint64_t, int64_t);
	17774	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u16)))
	17775	svuint16_t svld1q_gather_u64base_offset_u16(svbool_t, svuint64_t, int64_t);
	17776	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_bf16)))
	17777	svbfloat16_t svld1q_gather_u64base_offset_bf16(svbool_t, svuint64_t, int64_t);
	17778	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s8)))
	17779	svint8_t svld1q_gather_u64base_offset_s8(svbool_t, svuint64_t, int64_t);
	17780	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f64)))
	17781	svfloat64_t svld1q_gather_u64base_offset_f64(svbool_t, svuint64_t, int64_t);
	17782	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f32)))
	17783	svfloat32_t svld1q_gather_u64base_offset_f32(svbool_t, svuint64_t, int64_t);
	17784	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f16)))
	17785	svfloat16_t svld1q_gather_u64base_offset_f16(svbool_t, svuint64_t, int64_t);
	17786	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s32)))
	17787	svint32_t svld1q_gather_u64base_offset_s32(svbool_t, svuint64_t, int64_t);
	17788	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s64)))
	17789	svint64_t svld1q_gather_u64base_offset_s64(svbool_t, svuint64_t, int64_t);
	17790	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_mf8)))
	17791	svmfloat8_t svld1q_gather_u64base_offset_mf8(svbool_t, svuint64_t, int64_t);
	17792	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s16)))
	17793	svint16_t svld1q_gather_u64base_offset_s16(svbool_t, svuint64_t, int64_t);
	17794	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u8)))
	17795	svuint8_t svld1q_gather_u64base_u8(svbool_t, svuint64_t);
	17796	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u32)))
	17797	svuint32_t svld1q_gather_u64base_u32(svbool_t, svuint64_t);
	17798	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u64)))
	17799	svuint64_t svld1q_gather_u64base_u64(svbool_t, svuint64_t);
	17800	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u16)))
	17801	svuint16_t svld1q_gather_u64base_u16(svbool_t, svuint64_t);
	17802	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_bf16)))
	17803	svbfloat16_t svld1q_gather_u64base_bf16(svbool_t, svuint64_t);
	17804	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s8)))
	17805	svint8_t svld1q_gather_u64base_s8(svbool_t, svuint64_t);
	17806	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f64)))
	17807	svfloat64_t svld1q_gather_u64base_f64(svbool_t, svuint64_t);
	17808	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f32)))
	17809	svfloat32_t svld1q_gather_u64base_f32(svbool_t, svuint64_t);
	17810	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f16)))
	17811	svfloat16_t svld1q_gather_u64base_f16(svbool_t, svuint64_t);
	17812	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s32)))
	17813	svint32_t svld1q_gather_u64base_s32(svbool_t, svuint64_t);
	17814	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s64)))
	17815	svint64_t svld1q_gather_u64base_s64(svbool_t, svuint64_t);
	17816	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_mf8)))
	17817	svmfloat8_t svld1q_gather_u64base_mf8(svbool_t, svuint64_t);
	17818	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s16)))
	17819	svint16_t svld1q_gather_u64base_s16(svbool_t, svuint64_t);
	17820	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u32)))
	17821	svuint32_t svld1q_gather_u64index_u32(svbool_t, uint32_t const *, svuint64_t);
	17822	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u64)))
	17823	svuint64_t svld1q_gather_u64index_u64(svbool_t, uint64_t const *, svuint64_t);
	17824	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u16)))
	17825	svuint16_t svld1q_gather_u64index_u16(svbool_t, uint16_t const *, svuint64_t);
	17826	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_bf16)))
	17827	svbfloat16_t svld1q_gather_u64index_bf16(svbool_t, bfloat16_t const *, svuint64_t);
	17828	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f64)))
	17829	svfloat64_t svld1q_gather_u64index_f64(svbool_t, float64_t const *, svuint64_t);
	17830	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f32)))
	17831	svfloat32_t svld1q_gather_u64index_f32(svbool_t, float32_t const *, svuint64_t);
	17832	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f16)))
	17833	svfloat16_t svld1q_gather_u64index_f16(svbool_t, float16_t const *, svuint64_t);
	17834	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s32)))
	17835	svint32_t svld1q_gather_u64index_s32(svbool_t, int32_t const *, svuint64_t);
	17836	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s64)))
	17837	svint64_t svld1q_gather_u64index_s64(svbool_t, int64_t const *, svuint64_t);
	17838	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s16)))
	17839	svint16_t svld1q_gather_u64index_s16(svbool_t, int16_t const *, svuint64_t);
	17840	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u8)))
	17841	svuint8_t svld1q_gather_u64offset_u8(svbool_t, uint8_t const *, svuint64_t);
	17842	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u32)))
	17843	svuint32_t svld1q_gather_u64offset_u32(svbool_t, uint32_t const *, svuint64_t);
	17844	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u64)))
	17845	svuint64_t svld1q_gather_u64offset_u64(svbool_t, uint64_t const *, svuint64_t);
	17846	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u16)))
	17847	svuint16_t svld1q_gather_u64offset_u16(svbool_t, uint16_t const *, svuint64_t);
	17848	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_bf16)))
	17849	svbfloat16_t svld1q_gather_u64offset_bf16(svbool_t, bfloat16_t const *, svuint64_t);
	17850	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s8)))
	17851	svint8_t svld1q_gather_u64offset_s8(svbool_t, int8_t const *, svuint64_t);
	17852	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f64)))
	17853	svfloat64_t svld1q_gather_u64offset_f64(svbool_t, float64_t const *, svuint64_t);
	17854	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f32)))
	17855	svfloat32_t svld1q_gather_u64offset_f32(svbool_t, float32_t const *, svuint64_t);
	17856	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f16)))
	17857	svfloat16_t svld1q_gather_u64offset_f16(svbool_t, float16_t const *, svuint64_t);
	17858	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s32)))
	17859	svint32_t svld1q_gather_u64offset_s32(svbool_t, int32_t const *, svuint64_t);
	17860	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s64)))
	17861	svint64_t svld1q_gather_u64offset_s64(svbool_t, int64_t const *, svuint64_t);
	17862	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_mf8)))
	17863	svmfloat8_t svld1q_gather_u64offset_mf8(svbool_t, mfloat8_t const *, svuint64_t);
	17864	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s16)))
	17865	svint16_t svld1q_gather_u64offset_s16(svbool_t, int16_t const *, svuint64_t);
	17866	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_u64)))
	17867	svuint64_t svld1udq_u64(svbool_t, uint64_t const *);
	17868	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_f64)))
	17869	svfloat64_t svld1udq_f64(svbool_t, float64_t const *);
	17870	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_s64)))
	17871	svint64_t svld1udq_s64(svbool_t, int64_t const *);
	17872	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_u64)))
	17873	svuint64_t svld1udq_vnum_u64(svbool_t, uint64_t const *, int64_t);
	17874	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_f64)))
	17875	svfloat64_t svld1udq_vnum_f64(svbool_t, float64_t const *, int64_t);
	17876	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_s64)))
	17877	svint64_t svld1udq_vnum_s64(svbool_t, int64_t const *, int64_t);
	17878	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_u32)))
	17879	svuint32_t svld1uwq_u32(svbool_t, uint32_t const *);
	17880	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_f32)))
	17881	svfloat32_t svld1uwq_f32(svbool_t, float32_t const *);
	17882	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_s32)))
	17883	svint32_t svld1uwq_s32(svbool_t, int32_t const *);
	17884	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_u32)))
	17885	svuint32_t svld1uwq_vnum_u32(svbool_t, uint32_t const *, int64_t);
	17886	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_f32)))
	17887	svfloat32_t svld1uwq_vnum_f32(svbool_t, float32_t const *, int64_t);
	17888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_s32)))
	17889	svint32_t svld1uwq_vnum_s32(svbool_t, int32_t const *, int64_t);
	17890	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
	17891	void svst1dq_u64(svbool_t, uint64_t *, svuint64_t);
	17892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
	17893	void svst1dq_f64(svbool_t, float64_t *, svfloat64_t);
	17894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
	17895	void svst1dq_s64(svbool_t, int64_t *, svint64_t);
	17896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
	17897	void svst1dq_vnum_u64(svbool_t, uint64_t *, int64_t, svuint64_t);
	17898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
	17899	void svst1dq_vnum_f64(svbool_t, float64_t *, int64_t, svfloat64_t);
	17900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
	17901	void svst1dq_vnum_s64(svbool_t, int64_t *, int64_t, svint64_t);
	17902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
	17903	void svst1q_scatter_u64base_u8(svbool_t, svuint64_t, svuint8_t);
	17904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
	17905	void svst1q_scatter_u64base_u32(svbool_t, svuint64_t, svuint32_t);
	17906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u64)))
	17907	void svst1q_scatter_u64base_u64(svbool_t, svuint64_t, svuint64_t);
	17908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u16)))
	17909	void svst1q_scatter_u64base_u16(svbool_t, svuint64_t, svuint16_t);
	17910	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_bf16)))
	17911	void svst1q_scatter_u64base_bf16(svbool_t, svuint64_t, svbfloat16_t);
	17912	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s8)))
	17913	void svst1q_scatter_u64base_s8(svbool_t, svuint64_t, svint8_t);
	17914	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f64)))
	17915	void svst1q_scatter_u64base_f64(svbool_t, svuint64_t, svfloat64_t);
	17916	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f32)))
	17917	void svst1q_scatter_u64base_f32(svbool_t, svuint64_t, svfloat32_t);
	17918	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f16)))
	17919	void svst1q_scatter_u64base_f16(svbool_t, svuint64_t, svfloat16_t);
	17920	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s32)))
	17921	void svst1q_scatter_u64base_s32(svbool_t, svuint64_t, svint32_t);
	17922	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s64)))
	17923	void svst1q_scatter_u64base_s64(svbool_t, svuint64_t, svint64_t);
	17924	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_mf8)))
	17925	void svst1q_scatter_u64base_mf8(svbool_t, svuint64_t, svmfloat8_t);
	17926	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s16)))
	17927	void svst1q_scatter_u64base_s16(svbool_t, svuint64_t, svint16_t);
	17928	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u32)))
	17929	void svst1q_scatter_u64base_index_u32(svbool_t, svuint64_t, int64_t, svuint32_t);
	17930	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u64)))
	17931	void svst1q_scatter_u64base_index_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17932	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u16)))
	17933	void svst1q_scatter_u64base_index_u16(svbool_t, svuint64_t, int64_t, svuint16_t);
	17934	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_bf16)))
	17935	void svst1q_scatter_u64base_index_bf16(svbool_t, svuint64_t, int64_t, svbfloat16_t);
	17936	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f64)))
	17937	void svst1q_scatter_u64base_index_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17938	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f32)))
	17939	void svst1q_scatter_u64base_index_f32(svbool_t, svuint64_t, int64_t, svfloat32_t);
	17940	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f16)))
	17941	void svst1q_scatter_u64base_index_f16(svbool_t, svuint64_t, int64_t, svfloat16_t);
	17942	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s32)))
	17943	void svst1q_scatter_u64base_index_s32(svbool_t, svuint64_t, int64_t, svint32_t);
	17944	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s64)))
	17945	void svst1q_scatter_u64base_index_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17946	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s16)))
	17947	void svst1q_scatter_u64base_index_s16(svbool_t, svuint64_t, int64_t, svint16_t);
	17948	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u8)))
	17949	void svst1q_scatter_u64base_offset_u8(svbool_t, svuint64_t, int64_t, svuint8_t);
	17950	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u32)))
	17951	void svst1q_scatter_u64base_offset_u32(svbool_t, svuint64_t, int64_t, svuint32_t);
	17952	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u64)))
	17953	void svst1q_scatter_u64base_offset_u64(svbool_t, svuint64_t, int64_t, svuint64_t);
	17954	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u16)))
	17955	void svst1q_scatter_u64base_offset_u16(svbool_t, svuint64_t, int64_t, svuint16_t);
	17956	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_bf16)))
	17957	void svst1q_scatter_u64base_offset_bf16(svbool_t, svuint64_t, int64_t, svbfloat16_t);
	17958	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s8)))
	17959	void svst1q_scatter_u64base_offset_s8(svbool_t, svuint64_t, int64_t, svint8_t);
	17960	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f64)))
	17961	void svst1q_scatter_u64base_offset_f64(svbool_t, svuint64_t, int64_t, svfloat64_t);
	17962	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f32)))
	17963	void svst1q_scatter_u64base_offset_f32(svbool_t, svuint64_t, int64_t, svfloat32_t);
	17964	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f16)))
	17965	void svst1q_scatter_u64base_offset_f16(svbool_t, svuint64_t, int64_t, svfloat16_t);
	17966	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s32)))
	17967	void svst1q_scatter_u64base_offset_s32(svbool_t, svuint64_t, int64_t, svint32_t);
	17968	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s64)))
	17969	void svst1q_scatter_u64base_offset_s64(svbool_t, svuint64_t, int64_t, svint64_t);
	17970	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_mf8)))
	17971	void svst1q_scatter_u64base_offset_mf8(svbool_t, svuint64_t, int64_t, svmfloat8_t);
	17972	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
	17973	void svst1q_scatter_u64base_offset_s16(svbool_t, svuint64_t, int64_t, svint16_t);
	17974	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
	17975	void svst1q_scatter_s64index_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
	17976	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
	17977	void svst1q_scatter_s64index_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
	17978	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
	17979	void svst1q_scatter_s64index_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
	17980	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
	17981	void svst1q_scatter_s64index_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
	17982	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
	17983	void svst1q_scatter_s64index_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
	17984	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
	17985	void svst1q_scatter_s64index_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
	17986	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
	17987	void svst1q_scatter_s64index_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
	17988	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
	17989	void svst1q_scatter_s64index_s32(svbool_t, int32_t *, svint64_t, svint32_t);
	17990	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
	17991	void svst1q_scatter_s64index_s64(svbool_t, int64_t *, svint64_t, svint64_t);
	17992	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
	17993	void svst1q_scatter_s64index_s16(svbool_t, int16_t *, svint64_t, svint16_t);
	17994	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
	17995	void svst1q_scatter_u64index_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
	17996	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
	17997	void svst1q_scatter_u64index_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	17998	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u16)))
	17999	void svst1q_scatter_u64index_u16(svbool_t, uint16_t *, svuint64_t, svuint16_t);
	18000	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_bf16)))
	18001	void svst1q_scatter_u64index_bf16(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
	18002	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f64)))
	18003	void svst1q_scatter_u64index_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	18004	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f32)))
	18005	void svst1q_scatter_u64index_f32(svbool_t, float32_t *, svuint64_t, svfloat32_t);
	18006	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f16)))
	18007	void svst1q_scatter_u64index_f16(svbool_t, float16_t *, svuint64_t, svfloat16_t);
	18008	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s32)))
	18009	void svst1q_scatter_u64index_s32(svbool_t, int32_t *, svuint64_t, svint32_t);
	18010	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s64)))
	18011	void svst1q_scatter_u64index_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
	18012	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
	18013	void svst1q_scatter_u64index_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
	18014	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
	18015	void svst1q_scatter_s64offset_u8(svbool_t, uint8_t *, svint64_t, svuint8_t);
	18016	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
	18017	void svst1q_scatter_s64offset_u32(svbool_t, uint32_t *, svint64_t, svuint32_t);
	18018	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
	18019	void svst1q_scatter_s64offset_u64(svbool_t, uint64_t *, svint64_t, svuint64_t);
	18020	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
	18021	void svst1q_scatter_s64offset_u16(svbool_t, uint16_t *, svint64_t, svuint16_t);
	18022	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
	18023	void svst1q_scatter_s64offset_bf16(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
	18024	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
	18025	void svst1q_scatter_s64offset_s8(svbool_t, int8_t *, svint64_t, svint8_t);
	18026	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
	18027	void svst1q_scatter_s64offset_f64(svbool_t, float64_t *, svint64_t, svfloat64_t);
	18028	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
	18029	void svst1q_scatter_s64offset_f32(svbool_t, float32_t *, svint64_t, svfloat32_t);
	18030	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
	18031	void svst1q_scatter_s64offset_f16(svbool_t, float16_t *, svint64_t, svfloat16_t);
	18032	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
	18033	void svst1q_scatter_s64offset_s32(svbool_t, int32_t *, svint64_t, svint32_t);
	18034	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
	18035	void svst1q_scatter_s64offset_s64(svbool_t, int64_t *, svint64_t, svint64_t);
	18036	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_mf8)))
	18037	void svst1q_scatter_s64offset_mf8(svbool_t, mfloat8_t *, svint64_t, svmfloat8_t);
	18038	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
	18039	void svst1q_scatter_s64offset_s16(svbool_t, int16_t *, svint64_t, svint16_t);
	18040	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
	18041	void svst1q_scatter_u64offset_u8(svbool_t, uint8_t *, svuint64_t, svuint8_t);
	18042	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
	18043	void svst1q_scatter_u64offset_u32(svbool_t, uint32_t *, svuint64_t, svuint32_t);
	18044	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u64)))
	18045	void svst1q_scatter_u64offset_u64(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	18046	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u16)))
	18047	void svst1q_scatter_u64offset_u16(svbool_t, uint16_t *, svuint64_t, svuint16_t);
	18048	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_bf16)))
	18049	void svst1q_scatter_u64offset_bf16(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
	18050	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s8)))
	18051	void svst1q_scatter_u64offset_s8(svbool_t, int8_t *, svuint64_t, svint8_t);
	18052	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f64)))
	18053	void svst1q_scatter_u64offset_f64(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	18054	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f32)))
	18055	void svst1q_scatter_u64offset_f32(svbool_t, float32_t *, svuint64_t, svfloat32_t);
	18056	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f16)))
	18057	void svst1q_scatter_u64offset_f16(svbool_t, float16_t *, svuint64_t, svfloat16_t);
	18058	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s32)))
	18059	void svst1q_scatter_u64offset_s32(svbool_t, int32_t *, svuint64_t, svint32_t);
	18060	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s64)))
	18061	void svst1q_scatter_u64offset_s64(svbool_t, int64_t *, svuint64_t, svint64_t);
	18062	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_mf8)))
	18063	void svst1q_scatter_u64offset_mf8(svbool_t, mfloat8_t *, svuint64_t, svmfloat8_t);
	18064	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
	18065	void svst1q_scatter_u64offset_s16(svbool_t, int16_t *, svuint64_t, svint16_t);
	18066	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
	18067	void svst1wq_u32(svbool_t, uint32_t *, svuint32_t);
	18068	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
	18069	void svst1wq_f32(svbool_t, float32_t *, svfloat32_t);
	18070	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
	18071	void svst1wq_s32(svbool_t, int32_t *, svint32_t);
	18072	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
	18073	void svst1wq_vnum_u32(svbool_t, uint32_t *, int64_t, svuint32_t);
	18074	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
	18075	void svst1wq_vnum_f32(svbool_t, float32_t *, int64_t, svfloat32_t);
	18076	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
	18077	void svst1wq_vnum_s32(svbool_t, int32_t *, int64_t, svint32_t);
	18078	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u32)))
	18079	svuint32_t svld1q_gather_index_u32(svbool_t, svuint64_t, int64_t);
	18080	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u64)))
	18081	svuint64_t svld1q_gather_index_u64(svbool_t, svuint64_t, int64_t);
	18082	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_u16)))
	18083	svuint16_t svld1q_gather_index_u16(svbool_t, svuint64_t, int64_t);
	18084	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_bf16)))
	18085	svbfloat16_t svld1q_gather_index_bf16(svbool_t, svuint64_t, int64_t);
	18086	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f64)))
	18087	svfloat64_t svld1q_gather_index_f64(svbool_t, svuint64_t, int64_t);
	18088	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f32)))
	18089	svfloat32_t svld1q_gather_index_f32(svbool_t, svuint64_t, int64_t);
	18090	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_f16)))
	18091	svfloat16_t svld1q_gather_index_f16(svbool_t, svuint64_t, int64_t);
	18092	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s32)))
	18093	svint32_t svld1q_gather_index_s32(svbool_t, svuint64_t, int64_t);
	18094	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s64)))
	18095	svint64_t svld1q_gather_index_s64(svbool_t, svuint64_t, int64_t);
	18096	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_index_s16)))
	18097	svint16_t svld1q_gather_index_s16(svbool_t, svuint64_t, int64_t);
	18098	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u8)))
	18099	svuint8_t svld1q_gather_offset_u8(svbool_t, svuint64_t, int64_t);
	18100	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u32)))
	18101	svuint32_t svld1q_gather_offset_u32(svbool_t, svuint64_t, int64_t);
	18102	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u64)))
	18103	svuint64_t svld1q_gather_offset_u64(svbool_t, svuint64_t, int64_t);
	18104	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_u16)))
	18105	svuint16_t svld1q_gather_offset_u16(svbool_t, svuint64_t, int64_t);
	18106	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_bf16)))
	18107	svbfloat16_t svld1q_gather_offset_bf16(svbool_t, svuint64_t, int64_t);
	18108	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s8)))
	18109	svint8_t svld1q_gather_offset_s8(svbool_t, svuint64_t, int64_t);
	18110	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f64)))
	18111	svfloat64_t svld1q_gather_offset_f64(svbool_t, svuint64_t, int64_t);
	18112	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f32)))
	18113	svfloat32_t svld1q_gather_offset_f32(svbool_t, svuint64_t, int64_t);
	18114	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_f16)))
	18115	svfloat16_t svld1q_gather_offset_f16(svbool_t, svuint64_t, int64_t);
	18116	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s32)))
	18117	svint32_t svld1q_gather_offset_s32(svbool_t, svuint64_t, int64_t);
	18118	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s64)))
	18119	svint64_t svld1q_gather_offset_s64(svbool_t, svuint64_t, int64_t);
	18120	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_mf8)))
	18121	svmfloat8_t svld1q_gather_offset_mf8(svbool_t, svuint64_t, int64_t);
	18122	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_offset_s16)))
	18123	svint16_t svld1q_gather_offset_s16(svbool_t, svuint64_t, int64_t);
	18124	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u8)))
	18125	svuint8_t svld1q_gather_u8(svbool_t, svuint64_t);
	18126	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u32)))
	18127	svuint32_t svld1q_gather_u32(svbool_t, svuint64_t);
	18128	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u64)))
	18129	svuint64_t svld1q_gather_u64(svbool_t, svuint64_t);
	18130	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_u16)))
	18131	svuint16_t svld1q_gather_u16(svbool_t, svuint64_t);
	18132	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_bf16)))
	18133	svbfloat16_t svld1q_gather_bf16(svbool_t, svuint64_t);
	18134	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s8)))
	18135	svint8_t svld1q_gather_s8(svbool_t, svuint64_t);
	18136	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f64)))
	18137	svfloat64_t svld1q_gather_f64(svbool_t, svuint64_t);
	18138	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f32)))
	18139	svfloat32_t svld1q_gather_f32(svbool_t, svuint64_t);
	18140	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_f16)))
	18141	svfloat16_t svld1q_gather_f16(svbool_t, svuint64_t);
	18142	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s32)))
	18143	svint32_t svld1q_gather_s32(svbool_t, svuint64_t);
	18144	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s64)))
	18145	svint64_t svld1q_gather_s64(svbool_t, svuint64_t);
	18146	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_mf8)))
	18147	svmfloat8_t svld1q_gather_mf8(svbool_t, svuint64_t);
	18148	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64base_s16)))
	18149	svint16_t svld1q_gather_s16(svbool_t, svuint64_t);
	18150	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u32)))
	18151	svuint32_t svld1q_gather_index(svbool_t, uint32_t const *, svuint64_t);
	18152	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u64)))
	18153	svuint64_t svld1q_gather_index(svbool_t, uint64_t const *, svuint64_t);
	18154	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_u16)))
	18155	svuint16_t svld1q_gather_index(svbool_t, uint16_t const *, svuint64_t);
	18156	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_bf16)))
	18157	svbfloat16_t svld1q_gather_index(svbool_t, bfloat16_t const *, svuint64_t);
	18158	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f64)))
	18159	svfloat64_t svld1q_gather_index(svbool_t, float64_t const *, svuint64_t);
	18160	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f32)))
	18161	svfloat32_t svld1q_gather_index(svbool_t, float32_t const *, svuint64_t);
	18162	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_f16)))
	18163	svfloat16_t svld1q_gather_index(svbool_t, float16_t const *, svuint64_t);
	18164	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s32)))
	18165	svint32_t svld1q_gather_index(svbool_t, int32_t const *, svuint64_t);
	18166	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s64)))
	18167	svint64_t svld1q_gather_index(svbool_t, int64_t const *, svuint64_t);
	18168	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64index_s16)))
	18169	svint16_t svld1q_gather_index(svbool_t, int16_t const *, svuint64_t);
	18170	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u8)))
	18171	svuint8_t svld1q_gather_offset(svbool_t, uint8_t const *, svuint64_t);
	18172	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u32)))
	18173	svuint32_t svld1q_gather_offset(svbool_t, uint32_t const *, svuint64_t);
	18174	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u64)))
	18175	svuint64_t svld1q_gather_offset(svbool_t, uint64_t const *, svuint64_t);
	18176	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_u16)))
	18177	svuint16_t svld1q_gather_offset(svbool_t, uint16_t const *, svuint64_t);
	18178	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_bf16)))
	18179	svbfloat16_t svld1q_gather_offset(svbool_t, bfloat16_t const *, svuint64_t);
	18180	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s8)))
	18181	svint8_t svld1q_gather_offset(svbool_t, int8_t const *, svuint64_t);
	18182	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f64)))
	18183	svfloat64_t svld1q_gather_offset(svbool_t, float64_t const *, svuint64_t);
	18184	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f32)))
	18185	svfloat32_t svld1q_gather_offset(svbool_t, float32_t const *, svuint64_t);
	18186	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_f16)))
	18187	svfloat16_t svld1q_gather_offset(svbool_t, float16_t const *, svuint64_t);
	18188	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s32)))
	18189	svint32_t svld1q_gather_offset(svbool_t, int32_t const *, svuint64_t);
	18190	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s64)))
	18191	svint64_t svld1q_gather_offset(svbool_t, int64_t const *, svuint64_t);
	18192	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_mf8)))
	18193	svmfloat8_t svld1q_gather_offset(svbool_t, mfloat8_t const *, svuint64_t);
	18194	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1q_gather_u64offset_s16)))
	18195	svint16_t svld1q_gather_offset(svbool_t, int16_t const *, svuint64_t);
	18196	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_u64)))
	18197	svuint64_t svld1udq(svbool_t, uint64_t const *);
	18198	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_f64)))
	18199	svfloat64_t svld1udq(svbool_t, float64_t const *);
	18200	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_s64)))
	18201	svint64_t svld1udq(svbool_t, int64_t const *);
	18202	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_u64)))
	18203	svuint64_t svld1udq_vnum(svbool_t, uint64_t const *, int64_t);
	18204	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_f64)))
	18205	svfloat64_t svld1udq_vnum(svbool_t, float64_t const *, int64_t);
	18206	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1udq_vnum_s64)))
	18207	svint64_t svld1udq_vnum(svbool_t, int64_t const *, int64_t);
	18208	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_u32)))
	18209	svuint32_t svld1uwq(svbool_t, uint32_t const *);
	18210	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_f32)))
	18211	svfloat32_t svld1uwq(svbool_t, float32_t const *);
	18212	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_s32)))
	18213	svint32_t svld1uwq(svbool_t, int32_t const *);
	18214	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_u32)))
	18215	svuint32_t svld1uwq_vnum(svbool_t, uint32_t const *, int64_t);
	18216	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_f32)))
	18217	svfloat32_t svld1uwq_vnum(svbool_t, float32_t const *, int64_t);
	18218	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1uwq_vnum_s32)))
	18219	svint32_t svld1uwq_vnum(svbool_t, int32_t const *, int64_t);
	18220	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_u64)))
	18221	void svst1dq(svbool_t, uint64_t *, svuint64_t);
	18222	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_f64)))
	18223	void svst1dq(svbool_t, float64_t *, svfloat64_t);
	18224	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_s64)))
	18225	void svst1dq(svbool_t, int64_t *, svint64_t);
	18226	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_u64)))
	18227	void svst1dq_vnum(svbool_t, uint64_t *, int64_t, svuint64_t);
	18228	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_f64)))
	18229	void svst1dq_vnum(svbool_t, float64_t *, int64_t, svfloat64_t);
	18230	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1dq_vnum_s64)))
	18231	void svst1dq_vnum(svbool_t, int64_t *, int64_t, svint64_t);
	18232	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u8)))
	18233	void svst1q_scatter(svbool_t, svuint64_t, svuint8_t);
	18234	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u32)))
	18235	void svst1q_scatter(svbool_t, svuint64_t, svuint32_t);
	18236	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u64)))
	18237	void svst1q_scatter(svbool_t, svuint64_t, svuint64_t);
	18238	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_u16)))
	18239	void svst1q_scatter(svbool_t, svuint64_t, svuint16_t);
	18240	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_bf16)))
	18241	void svst1q_scatter(svbool_t, svuint64_t, svbfloat16_t);
	18242	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s8)))
	18243	void svst1q_scatter(svbool_t, svuint64_t, svint8_t);
	18244	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f64)))
	18245	void svst1q_scatter(svbool_t, svuint64_t, svfloat64_t);
	18246	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f32)))
	18247	void svst1q_scatter(svbool_t, svuint64_t, svfloat32_t);
	18248	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_f16)))
	18249	void svst1q_scatter(svbool_t, svuint64_t, svfloat16_t);
	18250	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s32)))
	18251	void svst1q_scatter(svbool_t, svuint64_t, svint32_t);
	18252	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s64)))
	18253	void svst1q_scatter(svbool_t, svuint64_t, svint64_t);
	18254	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_mf8)))
	18255	void svst1q_scatter(svbool_t, svuint64_t, svmfloat8_t);
	18256	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_s16)))
	18257	void svst1q_scatter(svbool_t, svuint64_t, svint16_t);
	18258	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u32)))
	18259	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint32_t);
	18260	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u64)))
	18261	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint64_t);
	18262	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_u16)))
	18263	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svuint16_t);
	18264	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_bf16)))
	18265	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svbfloat16_t);
	18266	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f64)))
	18267	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat64_t);
	18268	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f32)))
	18269	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat32_t);
	18270	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_f16)))
	18271	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svfloat16_t);
	18272	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s32)))
	18273	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint32_t);
	18274	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s64)))
	18275	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint64_t);
	18276	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_index_s16)))
	18277	void svst1q_scatter_index(svbool_t, svuint64_t, int64_t, svint16_t);
	18278	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u8)))
	18279	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint8_t);
	18280	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u32)))
	18281	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint32_t);
	18282	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u64)))
	18283	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint64_t);
	18284	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_u16)))
	18285	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svuint16_t);
	18286	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_bf16)))
	18287	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svbfloat16_t);
	18288	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s8)))
	18289	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint8_t);
	18290	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f64)))
	18291	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat64_t);
	18292	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f32)))
	18293	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat32_t);
	18294	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_f16)))
	18295	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svfloat16_t);
	18296	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s32)))
	18297	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint32_t);
	18298	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s64)))
	18299	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint64_t);
	18300	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_mf8)))
	18301	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svmfloat8_t);
	18302	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64base_offset_s16)))
	18303	void svst1q_scatter_offset(svbool_t, svuint64_t, int64_t, svint16_t);
	18304	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u32)))
	18305	void svst1q_scatter_index(svbool_t, uint32_t *, svint64_t, svuint32_t);
	18306	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u64)))
	18307	void svst1q_scatter_index(svbool_t, uint64_t *, svint64_t, svuint64_t);
	18308	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_u16)))
	18309	void svst1q_scatter_index(svbool_t, uint16_t *, svint64_t, svuint16_t);
	18310	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_bf16)))
	18311	void svst1q_scatter_index(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
	18312	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f64)))
	18313	void svst1q_scatter_index(svbool_t, float64_t *, svint64_t, svfloat64_t);
	18314	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f32)))
	18315	void svst1q_scatter_index(svbool_t, float32_t *, svint64_t, svfloat32_t);
	18316	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_f16)))
	18317	void svst1q_scatter_index(svbool_t, float16_t *, svint64_t, svfloat16_t);
	18318	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s32)))
	18319	void svst1q_scatter_index(svbool_t, int32_t *, svint64_t, svint32_t);
	18320	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s64)))
	18321	void svst1q_scatter_index(svbool_t, int64_t *, svint64_t, svint64_t);
	18322	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64index_s16)))
	18323	void svst1q_scatter_index(svbool_t, int16_t *, svint64_t, svint16_t);
	18324	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u32)))
	18325	void svst1q_scatter_index(svbool_t, uint32_t *, svuint64_t, svuint32_t);
	18326	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u64)))
	18327	void svst1q_scatter_index(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	18328	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_u16)))
	18329	void svst1q_scatter_index(svbool_t, uint16_t *, svuint64_t, svuint16_t);
	18330	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_bf16)))
	18331	void svst1q_scatter_index(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
	18332	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f64)))
	18333	void svst1q_scatter_index(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	18334	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f32)))
	18335	void svst1q_scatter_index(svbool_t, float32_t *, svuint64_t, svfloat32_t);
	18336	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_f16)))
	18337	void svst1q_scatter_index(svbool_t, float16_t *, svuint64_t, svfloat16_t);
	18338	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s32)))
	18339	void svst1q_scatter_index(svbool_t, int32_t *, svuint64_t, svint32_t);
	18340	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s64)))
	18341	void svst1q_scatter_index(svbool_t, int64_t *, svuint64_t, svint64_t);
	18342	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64index_s16)))
	18343	void svst1q_scatter_index(svbool_t, int16_t *, svuint64_t, svint16_t);
	18344	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u8)))
	18345	void svst1q_scatter_offset(svbool_t, uint8_t *, svint64_t, svuint8_t);
	18346	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u32)))
	18347	void svst1q_scatter_offset(svbool_t, uint32_t *, svint64_t, svuint32_t);
	18348	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u64)))
	18349	void svst1q_scatter_offset(svbool_t, uint64_t *, svint64_t, svuint64_t);
	18350	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_u16)))
	18351	void svst1q_scatter_offset(svbool_t, uint16_t *, svint64_t, svuint16_t);
	18352	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_bf16)))
	18353	void svst1q_scatter_offset(svbool_t, bfloat16_t *, svint64_t, svbfloat16_t);
	18354	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s8)))
	18355	void svst1q_scatter_offset(svbool_t, int8_t *, svint64_t, svint8_t);
	18356	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f64)))
	18357	void svst1q_scatter_offset(svbool_t, float64_t *, svint64_t, svfloat64_t);
	18358	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f32)))
	18359	void svst1q_scatter_offset(svbool_t, float32_t *, svint64_t, svfloat32_t);
	18360	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_f16)))
	18361	void svst1q_scatter_offset(svbool_t, float16_t *, svint64_t, svfloat16_t);
	18362	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s32)))
	18363	void svst1q_scatter_offset(svbool_t, int32_t *, svint64_t, svint32_t);
	18364	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s64)))
	18365	void svst1q_scatter_offset(svbool_t, int64_t *, svint64_t, svint64_t);
	18366	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_mf8)))
	18367	void svst1q_scatter_offset(svbool_t, mfloat8_t *, svint64_t, svmfloat8_t);
	18368	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_s64offset_s16)))
	18369	void svst1q_scatter_offset(svbool_t, int16_t *, svint64_t, svint16_t);
	18370	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u8)))
	18371	void svst1q_scatter_offset(svbool_t, uint8_t *, svuint64_t, svuint8_t);
	18372	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u32)))
	18373	void svst1q_scatter_offset(svbool_t, uint32_t *, svuint64_t, svuint32_t);
	18374	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u64)))
	18375	void svst1q_scatter_offset(svbool_t, uint64_t *, svuint64_t, svuint64_t);
	18376	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_u16)))
	18377	void svst1q_scatter_offset(svbool_t, uint16_t *, svuint64_t, svuint16_t);
	18378	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_bf16)))
	18379	void svst1q_scatter_offset(svbool_t, bfloat16_t *, svuint64_t, svbfloat16_t);
	18380	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s8)))
	18381	void svst1q_scatter_offset(svbool_t, int8_t *, svuint64_t, svint8_t);
	18382	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f64)))
	18383	void svst1q_scatter_offset(svbool_t, float64_t *, svuint64_t, svfloat64_t);
	18384	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f32)))
	18385	void svst1q_scatter_offset(svbool_t, float32_t *, svuint64_t, svfloat32_t);
	18386	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_f16)))
	18387	void svst1q_scatter_offset(svbool_t, float16_t *, svuint64_t, svfloat16_t);
	18388	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s32)))
	18389	void svst1q_scatter_offset(svbool_t, int32_t *, svuint64_t, svint32_t);
	18390	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s64)))
	18391	void svst1q_scatter_offset(svbool_t, int64_t *, svuint64_t, svint64_t);
	18392	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_mf8)))
	18393	void svst1q_scatter_offset(svbool_t, mfloat8_t *, svuint64_t, svmfloat8_t);
	18394	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1q_scatter_u64offset_s16)))
	18395	void svst1q_scatter_offset(svbool_t, int16_t *, svuint64_t, svint16_t);
	18396	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_u32)))
	18397	void svst1wq(svbool_t, uint32_t *, svuint32_t);
	18398	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_f32)))
	18399	void svst1wq(svbool_t, float32_t *, svfloat32_t);
	18400	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_s32)))
	18401	void svst1wq(svbool_t, int32_t *, svint32_t);
	18402	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_u32)))
	18403	void svst1wq_vnum(svbool_t, uint32_t *, int64_t, svuint32_t);
	18404	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_f32)))
	18405	void svst1wq_vnum(svbool_t, float32_t *, int64_t, svfloat32_t);
	18406	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1wq_vnum_s32)))
	18407	void svst1wq_vnum(svbool_t, int32_t *, int64_t, svint32_t);
	18408	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c8)))
	18409	uint64_t svcntp_c8(svcount_t, uint64_t);
	18410	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c32)))
	18411	uint64_t svcntp_c32(svcount_t, uint64_t);
	18412	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c64)))
	18413	uint64_t svcntp_c64(svcount_t, uint64_t);
	18414	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcntp_c16)))
	18415	uint64_t svcntp_c16(svcount_t, uint64_t);
	18416	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x2)))
	18417	svuint8x2_t svld1_u8_x2(svcount_t, uint8_t const *);
	18418	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x2)))
	18419	svint8x2_t svld1_s8_x2(svcount_t, int8_t const *);
	18420	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x2)))
	18421	svmfloat8x2_t svld1_mf8_x2(svcount_t, mfloat8_t const *);
	18422	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x2)))
	18423	svuint64x2_t svld1_u64_x2(svcount_t, uint64_t const *);
	18424	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x2)))
	18425	svfloat64x2_t svld1_f64_x2(svcount_t, float64_t const *);
	18426	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x2)))
	18427	svint64x2_t svld1_s64_x2(svcount_t, int64_t const *);
	18428	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x2)))
	18429	svuint16x2_t svld1_u16_x2(svcount_t, uint16_t const *);
	18430	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x2)))
	18431	svbfloat16x2_t svld1_bf16_x2(svcount_t, bfloat16_t const *);
	18432	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x2)))
	18433	svfloat16x2_t svld1_f16_x2(svcount_t, float16_t const *);
	18434	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x2)))
	18435	svint16x2_t svld1_s16_x2(svcount_t, int16_t const *);
	18436	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x2)))
	18437	svuint32x2_t svld1_u32_x2(svcount_t, uint32_t const *);
	18438	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x2)))
	18439	svfloat32x2_t svld1_f32_x2(svcount_t, float32_t const *);
	18440	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x2)))
	18441	svint32x2_t svld1_s32_x2(svcount_t, int32_t const *);
	18442	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x4)))
	18443	svuint8x4_t svld1_u8_x4(svcount_t, uint8_t const *);
	18444	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x4)))
	18445	svint8x4_t svld1_s8_x4(svcount_t, int8_t const *);
	18446	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x4)))
	18447	svmfloat8x4_t svld1_mf8_x4(svcount_t, mfloat8_t const *);
	18448	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x4)))
	18449	svuint64x4_t svld1_u64_x4(svcount_t, uint64_t const *);
	18450	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x4)))
	18451	svfloat64x4_t svld1_f64_x4(svcount_t, float64_t const *);
	18452	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x4)))
	18453	svint64x4_t svld1_s64_x4(svcount_t, int64_t const *);
	18454	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x4)))
	18455	svuint16x4_t svld1_u16_x4(svcount_t, uint16_t const *);
	18456	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x4)))
	18457	svbfloat16x4_t svld1_bf16_x4(svcount_t, bfloat16_t const *);
	18458	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x4)))
	18459	svfloat16x4_t svld1_f16_x4(svcount_t, float16_t const *);
	18460	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x4)))
	18461	svint16x4_t svld1_s16_x4(svcount_t, int16_t const *);
	18462	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x4)))
	18463	svuint32x4_t svld1_u32_x4(svcount_t, uint32_t const *);
	18464	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x4)))
	18465	svfloat32x4_t svld1_f32_x4(svcount_t, float32_t const *);
	18466	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x4)))
	18467	svint32x4_t svld1_s32_x4(svcount_t, int32_t const *);
	18468	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x2)))
	18469	svuint8x2_t svld1_vnum_u8_x2(svcount_t, uint8_t const *, int64_t);
	18470	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x2)))
	18471	svint8x2_t svld1_vnum_s8_x2(svcount_t, int8_t const *, int64_t);
	18472	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x2)))
	18473	svmfloat8x2_t svld1_vnum_mf8_x2(svcount_t, mfloat8_t const *, int64_t);
	18474	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x2)))
	18475	svuint64x2_t svld1_vnum_u64_x2(svcount_t, uint64_t const *, int64_t);
	18476	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x2)))
	18477	svfloat64x2_t svld1_vnum_f64_x2(svcount_t, float64_t const *, int64_t);
	18478	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x2)))
	18479	svint64x2_t svld1_vnum_s64_x2(svcount_t, int64_t const *, int64_t);
	18480	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x2)))
	18481	svuint16x2_t svld1_vnum_u16_x2(svcount_t, uint16_t const *, int64_t);
	18482	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x2)))
	18483	svbfloat16x2_t svld1_vnum_bf16_x2(svcount_t, bfloat16_t const *, int64_t);
	18484	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x2)))
	18485	svfloat16x2_t svld1_vnum_f16_x2(svcount_t, float16_t const *, int64_t);
	18486	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x2)))
	18487	svint16x2_t svld1_vnum_s16_x2(svcount_t, int16_t const *, int64_t);
	18488	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x2)))
	18489	svuint32x2_t svld1_vnum_u32_x2(svcount_t, uint32_t const *, int64_t);
	18490	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x2)))
	18491	svfloat32x2_t svld1_vnum_f32_x2(svcount_t, float32_t const *, int64_t);
	18492	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x2)))
	18493	svint32x2_t svld1_vnum_s32_x2(svcount_t, int32_t const *, int64_t);
	18494	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x4)))
	18495	svuint8x4_t svld1_vnum_u8_x4(svcount_t, uint8_t const *, int64_t);
	18496	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x4)))
	18497	svint8x4_t svld1_vnum_s8_x4(svcount_t, int8_t const *, int64_t);
	18498	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x4)))
	18499	svmfloat8x4_t svld1_vnum_mf8_x4(svcount_t, mfloat8_t const *, int64_t);
	18500	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x4)))
	18501	svuint64x4_t svld1_vnum_u64_x4(svcount_t, uint64_t const *, int64_t);
	18502	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x4)))
	18503	svfloat64x4_t svld1_vnum_f64_x4(svcount_t, float64_t const *, int64_t);
	18504	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x4)))
	18505	svint64x4_t svld1_vnum_s64_x4(svcount_t, int64_t const *, int64_t);
	18506	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x4)))
	18507	svuint16x4_t svld1_vnum_u16_x4(svcount_t, uint16_t const *, int64_t);
	18508	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x4)))
	18509	svbfloat16x4_t svld1_vnum_bf16_x4(svcount_t, bfloat16_t const *, int64_t);
	18510	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x4)))
	18511	svfloat16x4_t svld1_vnum_f16_x4(svcount_t, float16_t const *, int64_t);
	18512	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x4)))
	18513	svint16x4_t svld1_vnum_s16_x4(svcount_t, int16_t const *, int64_t);
	18514	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x4)))
	18515	svuint32x4_t svld1_vnum_u32_x4(svcount_t, uint32_t const *, int64_t);
	18516	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x4)))
	18517	svfloat32x4_t svld1_vnum_f32_x4(svcount_t, float32_t const *, int64_t);
	18518	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x4)))
	18519	svint32x4_t svld1_vnum_s32_x4(svcount_t, int32_t const *, int64_t);
	18520	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x2)))
	18521	svuint8x2_t svldnt1_u8_x2(svcount_t, uint8_t const *);
	18522	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x2)))
	18523	svint8x2_t svldnt1_s8_x2(svcount_t, int8_t const *);
	18524	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x2)))
	18525	svmfloat8x2_t svldnt1_mf8_x2(svcount_t, mfloat8_t const *);
	18526	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x2)))
	18527	svuint64x2_t svldnt1_u64_x2(svcount_t, uint64_t const *);
	18528	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x2)))
	18529	svfloat64x2_t svldnt1_f64_x2(svcount_t, float64_t const *);
	18530	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x2)))
	18531	svint64x2_t svldnt1_s64_x2(svcount_t, int64_t const *);
	18532	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x2)))
	18533	svuint16x2_t svldnt1_u16_x2(svcount_t, uint16_t const *);
	18534	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x2)))
	18535	svbfloat16x2_t svldnt1_bf16_x2(svcount_t, bfloat16_t const *);
	18536	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x2)))
	18537	svfloat16x2_t svldnt1_f16_x2(svcount_t, float16_t const *);
	18538	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x2)))
	18539	svint16x2_t svldnt1_s16_x2(svcount_t, int16_t const *);
	18540	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x2)))
	18541	svuint32x2_t svldnt1_u32_x2(svcount_t, uint32_t const *);
	18542	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x2)))
	18543	svfloat32x2_t svldnt1_f32_x2(svcount_t, float32_t const *);
	18544	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x2)))
	18545	svint32x2_t svldnt1_s32_x2(svcount_t, int32_t const *);
	18546	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x4)))
	18547	svuint8x4_t svldnt1_u8_x4(svcount_t, uint8_t const *);
	18548	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x4)))
	18549	svint8x4_t svldnt1_s8_x4(svcount_t, int8_t const *);
	18550	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x4)))
	18551	svmfloat8x4_t svldnt1_mf8_x4(svcount_t, mfloat8_t const *);
	18552	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x4)))
	18553	svuint64x4_t svldnt1_u64_x4(svcount_t, uint64_t const *);
	18554	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x4)))
	18555	svfloat64x4_t svldnt1_f64_x4(svcount_t, float64_t const *);
	18556	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x4)))
	18557	svint64x4_t svldnt1_s64_x4(svcount_t, int64_t const *);
	18558	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x4)))
	18559	svuint16x4_t svldnt1_u16_x4(svcount_t, uint16_t const *);
	18560	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x4)))
	18561	svbfloat16x4_t svldnt1_bf16_x4(svcount_t, bfloat16_t const *);
	18562	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x4)))
	18563	svfloat16x4_t svldnt1_f16_x4(svcount_t, float16_t const *);
	18564	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x4)))
	18565	svint16x4_t svldnt1_s16_x4(svcount_t, int16_t const *);
	18566	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x4)))
	18567	svuint32x4_t svldnt1_u32_x4(svcount_t, uint32_t const *);
	18568	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x4)))
	18569	svfloat32x4_t svldnt1_f32_x4(svcount_t, float32_t const *);
	18570	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x4)))
	18571	svint32x4_t svldnt1_s32_x4(svcount_t, int32_t const *);
	18572	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x2)))
	18573	svuint8x2_t svldnt1_vnum_u8_x2(svcount_t, uint8_t const *, int64_t);
	18574	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x2)))
	18575	svint8x2_t svldnt1_vnum_s8_x2(svcount_t, int8_t const *, int64_t);
	18576	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x2)))
	18577	svmfloat8x2_t svldnt1_vnum_mf8_x2(svcount_t, mfloat8_t const *, int64_t);
	18578	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x2)))
	18579	svuint64x2_t svldnt1_vnum_u64_x2(svcount_t, uint64_t const *, int64_t);
	18580	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x2)))
	18581	svfloat64x2_t svldnt1_vnum_f64_x2(svcount_t, float64_t const *, int64_t);
	18582	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x2)))
	18583	svint64x2_t svldnt1_vnum_s64_x2(svcount_t, int64_t const *, int64_t);
	18584	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x2)))
	18585	svuint16x2_t svldnt1_vnum_u16_x2(svcount_t, uint16_t const *, int64_t);
	18586	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x2)))
	18587	svbfloat16x2_t svldnt1_vnum_bf16_x2(svcount_t, bfloat16_t const *, int64_t);
	18588	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x2)))
	18589	svfloat16x2_t svldnt1_vnum_f16_x2(svcount_t, float16_t const *, int64_t);
	18590	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x2)))
	18591	svint16x2_t svldnt1_vnum_s16_x2(svcount_t, int16_t const *, int64_t);
	18592	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x2)))
	18593	svuint32x2_t svldnt1_vnum_u32_x2(svcount_t, uint32_t const *, int64_t);
	18594	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x2)))
	18595	svfloat32x2_t svldnt1_vnum_f32_x2(svcount_t, float32_t const *, int64_t);
	18596	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x2)))
	18597	svint32x2_t svldnt1_vnum_s32_x2(svcount_t, int32_t const *, int64_t);
	18598	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x4)))
	18599	svuint8x4_t svldnt1_vnum_u8_x4(svcount_t, uint8_t const *, int64_t);
	18600	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x4)))
	18601	svint8x4_t svldnt1_vnum_s8_x4(svcount_t, int8_t const *, int64_t);
	18602	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x4)))
	18603	svmfloat8x4_t svldnt1_vnum_mf8_x4(svcount_t, mfloat8_t const *, int64_t);
	18604	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x4)))
	18605	svuint64x4_t svldnt1_vnum_u64_x4(svcount_t, uint64_t const *, int64_t);
	18606	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x4)))
	18607	svfloat64x4_t svldnt1_vnum_f64_x4(svcount_t, float64_t const *, int64_t);
	18608	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x4)))
	18609	svint64x4_t svldnt1_vnum_s64_x4(svcount_t, int64_t const *, int64_t);
	18610	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x4)))
	18611	svuint16x4_t svldnt1_vnum_u16_x4(svcount_t, uint16_t const *, int64_t);
	18612	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x4)))
	18613	svbfloat16x4_t svldnt1_vnum_bf16_x4(svcount_t, bfloat16_t const *, int64_t);
	18614	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x4)))
	18615	svfloat16x4_t svldnt1_vnum_f16_x4(svcount_t, float16_t const *, int64_t);
	18616	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x4)))
	18617	svint16x4_t svldnt1_vnum_s16_x4(svcount_t, int16_t const *, int64_t);
	18618	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x4)))
	18619	svuint32x4_t svldnt1_vnum_u32_x4(svcount_t, uint32_t const *, int64_t);
	18620	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x4)))
	18621	svfloat32x4_t svldnt1_vnum_f32_x4(svcount_t, float32_t const *, int64_t);
	18622	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x4)))
	18623	svint32x4_t svldnt1_vnum_s32_x4(svcount_t, int32_t const *, int64_t);
	18624	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c8)))
	18625	svbool_t svpext_lane_c8(svcount_t, uint64_t);
	18626	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c32)))
	18627	svbool_t svpext_lane_c32(svcount_t, uint64_t);
	18628	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c64)))
	18629	svbool_t svpext_lane_c64(svcount_t, uint64_t);
	18630	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c16)))
	18631	svbool_t svpext_lane_c16(svcount_t, uint64_t);
	18632	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c8_x2)))
	18633	svboolx2_t svpext_lane_c8_x2(svcount_t, uint64_t);
	18634	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c32_x2)))
	18635	svboolx2_t svpext_lane_c32_x2(svcount_t, uint64_t);
	18636	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c64_x2)))
	18637	svboolx2_t svpext_lane_c64_x2(svcount_t, uint64_t);
	18638	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpext_lane_c16_x2)))
	18639	svboolx2_t svpext_lane_c16_x2(svcount_t, uint64_t);
	18640	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpfalse_c)))
	18641	svcount_t svpfalse_c(void);
	18642	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c16)))
	18643	svcount_t svpsel_lane_c16(svcount_t, svbool_t, uint32_t);
	18644	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c32)))
	18645	svcount_t svpsel_lane_c32(svcount_t, svbool_t, uint32_t);
	18646	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c64)))
	18647	svcount_t svpsel_lane_c64(svcount_t, svbool_t, uint32_t);
	18648	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svpsel_lane_c8)))
	18649	svcount_t svpsel_lane_c8(svcount_t, svbool_t, uint32_t);
	18650	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c8)))
	18651	svcount_t svptrue_c8(void);
	18652	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c32)))
	18653	svcount_t svptrue_c32(void);
	18654	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c64)))
	18655	svcount_t svptrue_c64(void);
	18656	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c16)))
	18657	svcount_t svptrue_c16(void);
	18658	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_b)))
	18659	svbool_t svreinterpret_b(svcount_t);
	18660	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_c)))
	18661	svcount_t svreinterpret_c(svbool_t);
	18662	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x2)))
	18663	void svst1_u8_x2(svcount_t, uint8_t *, svuint8x2_t);
	18664	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x2)))
	18665	void svst1_s8_x2(svcount_t, int8_t *, svint8x2_t);
	18666	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x2)))
	18667	void svst1_mf8_x2(svcount_t, mfloat8_t *, svmfloat8x2_t);
	18668	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x2)))
	18669	void svst1_u64_x2(svcount_t, uint64_t *, svuint64x2_t);
	18670	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x2)))
	18671	void svst1_f64_x2(svcount_t, float64_t *, svfloat64x2_t);
	18672	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x2)))
	18673	void svst1_s64_x2(svcount_t, int64_t *, svint64x2_t);
	18674	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x2)))
	18675	void svst1_u16_x2(svcount_t, uint16_t *, svuint16x2_t);
	18676	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x2)))
	18677	void svst1_bf16_x2(svcount_t, bfloat16_t *, svbfloat16x2_t);
	18678	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x2)))
	18679	void svst1_f16_x2(svcount_t, float16_t *, svfloat16x2_t);
	18680	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x2)))
	18681	void svst1_s16_x2(svcount_t, int16_t *, svint16x2_t);
	18682	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x2)))
	18683	void svst1_u32_x2(svcount_t, uint32_t *, svuint32x2_t);
	18684	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x2)))
	18685	void svst1_f32_x2(svcount_t, float32_t *, svfloat32x2_t);
	18686	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x2)))
	18687	void svst1_s32_x2(svcount_t, int32_t *, svint32x2_t);
	18688	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x4)))
	18689	void svst1_u8_x4(svcount_t, uint8_t *, svuint8x4_t);
	18690	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x4)))
	18691	void svst1_s8_x4(svcount_t, int8_t *, svint8x4_t);
	18692	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x4)))
	18693	void svst1_mf8_x4(svcount_t, mfloat8_t *, svmfloat8x4_t);
	18694	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x4)))
	18695	void svst1_u64_x4(svcount_t, uint64_t *, svuint64x4_t);
	18696	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x4)))
	18697	void svst1_f64_x4(svcount_t, float64_t *, svfloat64x4_t);
	18698	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x4)))
	18699	void svst1_s64_x4(svcount_t, int64_t *, svint64x4_t);
	18700	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x4)))
	18701	void svst1_u16_x4(svcount_t, uint16_t *, svuint16x4_t);
	18702	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x4)))
	18703	void svst1_bf16_x4(svcount_t, bfloat16_t *, svbfloat16x4_t);
	18704	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x4)))
	18705	void svst1_f16_x4(svcount_t, float16_t *, svfloat16x4_t);
	18706	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x4)))
	18707	void svst1_s16_x4(svcount_t, int16_t *, svint16x4_t);
	18708	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x4)))
	18709	void svst1_u32_x4(svcount_t, uint32_t *, svuint32x4_t);
	18710	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x4)))
	18711	void svst1_f32_x4(svcount_t, float32_t *, svfloat32x4_t);
	18712	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x4)))
	18713	void svst1_s32_x4(svcount_t, int32_t *, svint32x4_t);
	18714	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x2)))
	18715	void svst1_vnum_u8_x2(svcount_t, uint8_t *, int64_t, svuint8x2_t);
	18716	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x2)))
	18717	void svst1_vnum_s8_x2(svcount_t, int8_t *, int64_t, svint8x2_t);
	18718	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x2)))
	18719	void svst1_vnum_mf8_x2(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
	18720	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x2)))
	18721	void svst1_vnum_u64_x2(svcount_t, uint64_t *, int64_t, svuint64x2_t);
	18722	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x2)))
	18723	void svst1_vnum_f64_x2(svcount_t, float64_t *, int64_t, svfloat64x2_t);
	18724	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x2)))
	18725	void svst1_vnum_s64_x2(svcount_t, int64_t *, int64_t, svint64x2_t);
	18726	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x2)))
	18727	void svst1_vnum_u16_x2(svcount_t, uint16_t *, int64_t, svuint16x2_t);
	18728	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x2)))
	18729	void svst1_vnum_bf16_x2(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
	18730	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x2)))
	18731	void svst1_vnum_f16_x2(svcount_t, float16_t *, int64_t, svfloat16x2_t);
	18732	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x2)))
	18733	void svst1_vnum_s16_x2(svcount_t, int16_t *, int64_t, svint16x2_t);
	18734	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x2)))
	18735	void svst1_vnum_u32_x2(svcount_t, uint32_t *, int64_t, svuint32x2_t);
	18736	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x2)))
	18737	void svst1_vnum_f32_x2(svcount_t, float32_t *, int64_t, svfloat32x2_t);
	18738	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x2)))
	18739	void svst1_vnum_s32_x2(svcount_t, int32_t *, int64_t, svint32x2_t);
	18740	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x4)))
	18741	void svst1_vnum_u8_x4(svcount_t, uint8_t *, int64_t, svuint8x4_t);
	18742	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x4)))
	18743	void svst1_vnum_s8_x4(svcount_t, int8_t *, int64_t, svint8x4_t);
	18744	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x4)))
	18745	void svst1_vnum_mf8_x4(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
	18746	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x4)))
	18747	void svst1_vnum_u64_x4(svcount_t, uint64_t *, int64_t, svuint64x4_t);
	18748	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x4)))
	18749	void svst1_vnum_f64_x4(svcount_t, float64_t *, int64_t, svfloat64x4_t);
	18750	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x4)))
	18751	void svst1_vnum_s64_x4(svcount_t, int64_t *, int64_t, svint64x4_t);
	18752	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x4)))
	18753	void svst1_vnum_u16_x4(svcount_t, uint16_t *, int64_t, svuint16x4_t);
	18754	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x4)))
	18755	void svst1_vnum_bf16_x4(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
	18756	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x4)))
	18757	void svst1_vnum_f16_x4(svcount_t, float16_t *, int64_t, svfloat16x4_t);
	18758	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x4)))
	18759	void svst1_vnum_s16_x4(svcount_t, int16_t *, int64_t, svint16x4_t);
	18760	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x4)))
	18761	void svst1_vnum_u32_x4(svcount_t, uint32_t *, int64_t, svuint32x4_t);
	18762	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x4)))
	18763	void svst1_vnum_f32_x4(svcount_t, float32_t *, int64_t, svfloat32x4_t);
	18764	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x4)))
	18765	void svst1_vnum_s32_x4(svcount_t, int32_t *, int64_t, svint32x4_t);
	18766	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x2)))
	18767	void svstnt1_u8_x2(svcount_t, uint8_t *, svuint8x2_t);
	18768	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x2)))
	18769	void svstnt1_s8_x2(svcount_t, int8_t *, svint8x2_t);
	18770	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x2)))
	18771	void svstnt1_mf8_x2(svcount_t, mfloat8_t *, svmfloat8x2_t);
	18772	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x2)))
	18773	void svstnt1_u64_x2(svcount_t, uint64_t *, svuint64x2_t);
	18774	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x2)))
	18775	void svstnt1_f64_x2(svcount_t, float64_t *, svfloat64x2_t);
	18776	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x2)))
	18777	void svstnt1_s64_x2(svcount_t, int64_t *, svint64x2_t);
	18778	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x2)))
	18779	void svstnt1_u16_x2(svcount_t, uint16_t *, svuint16x2_t);
	18780	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x2)))
	18781	void svstnt1_bf16_x2(svcount_t, bfloat16_t *, svbfloat16x2_t);
	18782	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x2)))
	18783	void svstnt1_f16_x2(svcount_t, float16_t *, svfloat16x2_t);
	18784	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x2)))
	18785	void svstnt1_s16_x2(svcount_t, int16_t *, svint16x2_t);
	18786	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x2)))
	18787	void svstnt1_u32_x2(svcount_t, uint32_t *, svuint32x2_t);
	18788	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x2)))
	18789	void svstnt1_f32_x2(svcount_t, float32_t *, svfloat32x2_t);
	18790	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x2)))
	18791	void svstnt1_s32_x2(svcount_t, int32_t *, svint32x2_t);
	18792	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x4)))
	18793	void svstnt1_u8_x4(svcount_t, uint8_t *, svuint8x4_t);
	18794	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x4)))
	18795	void svstnt1_s8_x4(svcount_t, int8_t *, svint8x4_t);
	18796	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x4)))
	18797	void svstnt1_mf8_x4(svcount_t, mfloat8_t *, svmfloat8x4_t);
	18798	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x4)))
	18799	void svstnt1_u64_x4(svcount_t, uint64_t *, svuint64x4_t);
	18800	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x4)))
	18801	void svstnt1_f64_x4(svcount_t, float64_t *, svfloat64x4_t);
	18802	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x4)))
	18803	void svstnt1_s64_x4(svcount_t, int64_t *, svint64x4_t);
	18804	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x4)))
	18805	void svstnt1_u16_x4(svcount_t, uint16_t *, svuint16x4_t);
	18806	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x4)))
	18807	void svstnt1_bf16_x4(svcount_t, bfloat16_t *, svbfloat16x4_t);
	18808	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x4)))
	18809	void svstnt1_f16_x4(svcount_t, float16_t *, svfloat16x4_t);
	18810	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x4)))
	18811	void svstnt1_s16_x4(svcount_t, int16_t *, svint16x4_t);
	18812	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x4)))
	18813	void svstnt1_u32_x4(svcount_t, uint32_t *, svuint32x4_t);
	18814	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x4)))
	18815	void svstnt1_f32_x4(svcount_t, float32_t *, svfloat32x4_t);
	18816	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x4)))
	18817	void svstnt1_s32_x4(svcount_t, int32_t *, svint32x4_t);
	18818	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x2)))
	18819	void svstnt1_vnum_u8_x2(svcount_t, uint8_t *, int64_t, svuint8x2_t);
	18820	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x2)))
	18821	void svstnt1_vnum_s8_x2(svcount_t, int8_t *, int64_t, svint8x2_t);
	18822	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x2)))
	18823	void svstnt1_vnum_mf8_x2(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
	18824	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x2)))
	18825	void svstnt1_vnum_u64_x2(svcount_t, uint64_t *, int64_t, svuint64x2_t);
	18826	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x2)))
	18827	void svstnt1_vnum_f64_x2(svcount_t, float64_t *, int64_t, svfloat64x2_t);
	18828	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x2)))
	18829	void svstnt1_vnum_s64_x2(svcount_t, int64_t *, int64_t, svint64x2_t);
	18830	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x2)))
	18831	void svstnt1_vnum_u16_x2(svcount_t, uint16_t *, int64_t, svuint16x2_t);
	18832	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x2)))
	18833	void svstnt1_vnum_bf16_x2(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
	18834	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x2)))
	18835	void svstnt1_vnum_f16_x2(svcount_t, float16_t *, int64_t, svfloat16x2_t);
	18836	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x2)))
	18837	void svstnt1_vnum_s16_x2(svcount_t, int16_t *, int64_t, svint16x2_t);
	18838	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x2)))
	18839	void svstnt1_vnum_u32_x2(svcount_t, uint32_t *, int64_t, svuint32x2_t);
	18840	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x2)))
	18841	void svstnt1_vnum_f32_x2(svcount_t, float32_t *, int64_t, svfloat32x2_t);
	18842	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x2)))
	18843	void svstnt1_vnum_s32_x2(svcount_t, int32_t *, int64_t, svint32x2_t);
	18844	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x4)))
	18845	void svstnt1_vnum_u8_x4(svcount_t, uint8_t *, int64_t, svuint8x4_t);
	18846	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x4)))
	18847	void svstnt1_vnum_s8_x4(svcount_t, int8_t *, int64_t, svint8x4_t);
	18848	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x4)))
	18849	void svstnt1_vnum_mf8_x4(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
	18850	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x4)))
	18851	void svstnt1_vnum_u64_x4(svcount_t, uint64_t *, int64_t, svuint64x4_t);
	18852	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x4)))
	18853	void svstnt1_vnum_f64_x4(svcount_t, float64_t *, int64_t, svfloat64x4_t);
	18854	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x4)))
	18855	void svstnt1_vnum_s64_x4(svcount_t, int64_t *, int64_t, svint64x4_t);
	18856	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x4)))
	18857	void svstnt1_vnum_u16_x4(svcount_t, uint16_t *, int64_t, svuint16x4_t);
	18858	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x4)))
	18859	void svstnt1_vnum_bf16_x4(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
	18860	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x4)))
	18861	void svstnt1_vnum_f16_x4(svcount_t, float16_t *, int64_t, svfloat16x4_t);
	18862	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x4)))
	18863	void svstnt1_vnum_s16_x4(svcount_t, int16_t *, int64_t, svint16x4_t);
	18864	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x4)))
	18865	void svstnt1_vnum_u32_x4(svcount_t, uint32_t *, int64_t, svuint32x4_t);
	18866	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x4)))
	18867	void svstnt1_vnum_f32_x4(svcount_t, float32_t *, int64_t, svfloat32x4_t);
	18868	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x4)))
	18869	void svstnt1_vnum_s32_x4(svcount_t, int32_t *, int64_t, svint32x4_t);
	18870	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_s64)))
	18871	svcount_t svwhilege_c8_s64(int64_t, int64_t, uint64_t);
	18872	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_s64)))
	18873	svcount_t svwhilege_c32_s64(int64_t, int64_t, uint64_t);
	18874	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_s64)))
	18875	svcount_t svwhilege_c64_s64(int64_t, int64_t, uint64_t);
	18876	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_s64)))
	18877	svcount_t svwhilege_c16_s64(int64_t, int64_t, uint64_t);
	18878	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_u64)))
	18879	svcount_t svwhilege_c8_u64(uint64_t, uint64_t, uint64_t);
	18880	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_u64)))
	18881	svcount_t svwhilege_c32_u64(uint64_t, uint64_t, uint64_t);
	18882	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_u64)))
	18883	svcount_t svwhilege_c64_u64(uint64_t, uint64_t, uint64_t);
	18884	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_u64)))
	18885	svcount_t svwhilege_c16_u64(uint64_t, uint64_t, uint64_t);
	18886	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_s64)))
	18887	svcount_t svwhilegt_c8_s64(int64_t, int64_t, uint64_t);
	18888	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_s64)))
	18889	svcount_t svwhilegt_c32_s64(int64_t, int64_t, uint64_t);
	18890	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_s64)))
	18891	svcount_t svwhilegt_c64_s64(int64_t, int64_t, uint64_t);
	18892	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_s64)))
	18893	svcount_t svwhilegt_c16_s64(int64_t, int64_t, uint64_t);
	18894	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_u64)))
	18895	svcount_t svwhilegt_c8_u64(uint64_t, uint64_t, uint64_t);
	18896	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_u64)))
	18897	svcount_t svwhilegt_c32_u64(uint64_t, uint64_t, uint64_t);
	18898	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_u64)))
	18899	svcount_t svwhilegt_c64_u64(uint64_t, uint64_t, uint64_t);
	18900	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_u64)))
	18901	svcount_t svwhilegt_c16_u64(uint64_t, uint64_t, uint64_t);
	18902	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_s64)))
	18903	svcount_t svwhilele_c8_s64(int64_t, int64_t, uint64_t);
	18904	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_s64)))
	18905	svcount_t svwhilele_c32_s64(int64_t, int64_t, uint64_t);
	18906	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_s64)))
	18907	svcount_t svwhilele_c64_s64(int64_t, int64_t, uint64_t);
	18908	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_s64)))
	18909	svcount_t svwhilele_c16_s64(int64_t, int64_t, uint64_t);
	18910	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_u64)))
	18911	svcount_t svwhilele_c8_u64(uint64_t, uint64_t, uint64_t);
	18912	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_u64)))
	18913	svcount_t svwhilele_c32_u64(uint64_t, uint64_t, uint64_t);
	18914	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_u64)))
	18915	svcount_t svwhilele_c64_u64(uint64_t, uint64_t, uint64_t);
	18916	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_u64)))
	18917	svcount_t svwhilele_c16_u64(uint64_t, uint64_t, uint64_t);
	18918	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_u64)))
	18919	svcount_t svwhilelt_c8_u64(uint64_t, uint64_t, uint64_t);
	18920	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_u64)))
	18921	svcount_t svwhilelt_c32_u64(uint64_t, uint64_t, uint64_t);
	18922	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_u64)))
	18923	svcount_t svwhilelt_c64_u64(uint64_t, uint64_t, uint64_t);
	18924	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_u64)))
	18925	svcount_t svwhilelt_c16_u64(uint64_t, uint64_t, uint64_t);
	18926	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_s64)))
	18927	svcount_t svwhilelt_c8_s64(int64_t, int64_t, uint64_t);
	18928	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_s64)))
	18929	svcount_t svwhilelt_c32_s64(int64_t, int64_t, uint64_t);
	18930	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64)))
	18931	svcount_t svwhilelt_c64_s64(int64_t, int64_t, uint64_t);
	18932	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64)))
	18933	svcount_t svwhilelt_c16_s64(int64_t, int64_t, uint64_t);
	18934	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x2)))
	18935	svuint8x2_t svld1_x2(svcount_t, uint8_t const *);
	18936	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x2)))
	18937	svint8x2_t svld1_x2(svcount_t, int8_t const *);
	18938	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x2)))
	18939	svmfloat8x2_t svld1_x2(svcount_t, mfloat8_t const *);
	18940	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x2)))
	18941	svuint64x2_t svld1_x2(svcount_t, uint64_t const *);
	18942	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x2)))
	18943	svfloat64x2_t svld1_x2(svcount_t, float64_t const *);
	18944	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x2)))
	18945	svint64x2_t svld1_x2(svcount_t, int64_t const *);
	18946	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x2)))
	18947	svuint16x2_t svld1_x2(svcount_t, uint16_t const *);
	18948	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x2)))
	18949	svbfloat16x2_t svld1_x2(svcount_t, bfloat16_t const *);
	18950	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x2)))
	18951	svfloat16x2_t svld1_x2(svcount_t, float16_t const *);
	18952	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x2)))
	18953	svint16x2_t svld1_x2(svcount_t, int16_t const *);
	18954	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x2)))
	18955	svuint32x2_t svld1_x2(svcount_t, uint32_t const *);
	18956	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x2)))
	18957	svfloat32x2_t svld1_x2(svcount_t, float32_t const *);
	18958	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x2)))
	18959	svint32x2_t svld1_x2(svcount_t, int32_t const *);
	18960	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u8_x4)))
	18961	svuint8x4_t svld1_x4(svcount_t, uint8_t const *);
	18962	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s8_x4)))
	18963	svint8x4_t svld1_x4(svcount_t, int8_t const *);
	18964	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_mf8_x4)))
	18965	svmfloat8x4_t svld1_x4(svcount_t, mfloat8_t const *);
	18966	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u64_x4)))
	18967	svuint64x4_t svld1_x4(svcount_t, uint64_t const *);
	18968	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f64_x4)))
	18969	svfloat64x4_t svld1_x4(svcount_t, float64_t const *);
	18970	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s64_x4)))
	18971	svint64x4_t svld1_x4(svcount_t, int64_t const *);
	18972	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u16_x4)))
	18973	svuint16x4_t svld1_x4(svcount_t, uint16_t const *);
	18974	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_bf16_x4)))
	18975	svbfloat16x4_t svld1_x4(svcount_t, bfloat16_t const *);
	18976	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f16_x4)))
	18977	svfloat16x4_t svld1_x4(svcount_t, float16_t const *);
	18978	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s16_x4)))
	18979	svint16x4_t svld1_x4(svcount_t, int16_t const *);
	18980	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_u32_x4)))
	18981	svuint32x4_t svld1_x4(svcount_t, uint32_t const *);
	18982	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_f32_x4)))
	18983	svfloat32x4_t svld1_x4(svcount_t, float32_t const *);
	18984	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_s32_x4)))
	18985	svint32x4_t svld1_x4(svcount_t, int32_t const *);
	18986	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x2)))
	18987	svuint8x2_t svld1_vnum_x2(svcount_t, uint8_t const *, int64_t);
	18988	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x2)))
	18989	svint8x2_t svld1_vnum_x2(svcount_t, int8_t const *, int64_t);
	18990	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x2)))
	18991	svmfloat8x2_t svld1_vnum_x2(svcount_t, mfloat8_t const *, int64_t);
	18992	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x2)))
	18993	svuint64x2_t svld1_vnum_x2(svcount_t, uint64_t const *, int64_t);
	18994	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x2)))
	18995	svfloat64x2_t svld1_vnum_x2(svcount_t, float64_t const *, int64_t);
	18996	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x2)))
	18997	svint64x2_t svld1_vnum_x2(svcount_t, int64_t const *, int64_t);
	18998	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x2)))
	18999	svuint16x2_t svld1_vnum_x2(svcount_t, uint16_t const *, int64_t);
	19000	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x2)))
	19001	svbfloat16x2_t svld1_vnum_x2(svcount_t, bfloat16_t const *, int64_t);
	19002	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x2)))
	19003	svfloat16x2_t svld1_vnum_x2(svcount_t, float16_t const *, int64_t);
	19004	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x2)))
	19005	svint16x2_t svld1_vnum_x2(svcount_t, int16_t const *, int64_t);
	19006	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x2)))
	19007	svuint32x2_t svld1_vnum_x2(svcount_t, uint32_t const *, int64_t);
	19008	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x2)))
	19009	svfloat32x2_t svld1_vnum_x2(svcount_t, float32_t const *, int64_t);
	19010	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x2)))
	19011	svint32x2_t svld1_vnum_x2(svcount_t, int32_t const *, int64_t);
	19012	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u8_x4)))
	19013	svuint8x4_t svld1_vnum_x4(svcount_t, uint8_t const *, int64_t);
	19014	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s8_x4)))
	19015	svint8x4_t svld1_vnum_x4(svcount_t, int8_t const *, int64_t);
	19016	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_mf8_x4)))
	19017	svmfloat8x4_t svld1_vnum_x4(svcount_t, mfloat8_t const *, int64_t);
	19018	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u64_x4)))
	19019	svuint64x4_t svld1_vnum_x4(svcount_t, uint64_t const *, int64_t);
	19020	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f64_x4)))
	19021	svfloat64x4_t svld1_vnum_x4(svcount_t, float64_t const *, int64_t);
	19022	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s64_x4)))
	19023	svint64x4_t svld1_vnum_x4(svcount_t, int64_t const *, int64_t);
	19024	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u16_x4)))
	19025	svuint16x4_t svld1_vnum_x4(svcount_t, uint16_t const *, int64_t);
	19026	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_bf16_x4)))
	19027	svbfloat16x4_t svld1_vnum_x4(svcount_t, bfloat16_t const *, int64_t);
	19028	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f16_x4)))
	19029	svfloat16x4_t svld1_vnum_x4(svcount_t, float16_t const *, int64_t);
	19030	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s16_x4)))
	19031	svint16x4_t svld1_vnum_x4(svcount_t, int16_t const *, int64_t);
	19032	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_u32_x4)))
	19033	svuint32x4_t svld1_vnum_x4(svcount_t, uint32_t const *, int64_t);
	19034	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_f32_x4)))
	19035	svfloat32x4_t svld1_vnum_x4(svcount_t, float32_t const *, int64_t);
	19036	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svld1_vnum_s32_x4)))
	19037	svint32x4_t svld1_vnum_x4(svcount_t, int32_t const *, int64_t);
	19038	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x2)))
	19039	svuint8x2_t svldnt1_x2(svcount_t, uint8_t const *);
	19040	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x2)))
	19041	svint8x2_t svldnt1_x2(svcount_t, int8_t const *);
	19042	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x2)))
	19043	svmfloat8x2_t svldnt1_x2(svcount_t, mfloat8_t const *);
	19044	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x2)))
	19045	svuint64x2_t svldnt1_x2(svcount_t, uint64_t const *);
	19046	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x2)))
	19047	svfloat64x2_t svldnt1_x2(svcount_t, float64_t const *);
	19048	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x2)))
	19049	svint64x2_t svldnt1_x2(svcount_t, int64_t const *);
	19050	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x2)))
	19051	svuint16x2_t svldnt1_x2(svcount_t, uint16_t const *);
	19052	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x2)))
	19053	svbfloat16x2_t svldnt1_x2(svcount_t, bfloat16_t const *);
	19054	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x2)))
	19055	svfloat16x2_t svldnt1_x2(svcount_t, float16_t const *);
	19056	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x2)))
	19057	svint16x2_t svldnt1_x2(svcount_t, int16_t const *);
	19058	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x2)))
	19059	svuint32x2_t svldnt1_x2(svcount_t, uint32_t const *);
	19060	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x2)))
	19061	svfloat32x2_t svldnt1_x2(svcount_t, float32_t const *);
	19062	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x2)))
	19063	svint32x2_t svldnt1_x2(svcount_t, int32_t const *);
	19064	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u8_x4)))
	19065	svuint8x4_t svldnt1_x4(svcount_t, uint8_t const *);
	19066	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s8_x4)))
	19067	svint8x4_t svldnt1_x4(svcount_t, int8_t const *);
	19068	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_mf8_x4)))
	19069	svmfloat8x4_t svldnt1_x4(svcount_t, mfloat8_t const *);
	19070	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u64_x4)))
	19071	svuint64x4_t svldnt1_x4(svcount_t, uint64_t const *);
	19072	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f64_x4)))
	19073	svfloat64x4_t svldnt1_x4(svcount_t, float64_t const *);
	19074	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s64_x4)))
	19075	svint64x4_t svldnt1_x4(svcount_t, int64_t const *);
	19076	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u16_x4)))
	19077	svuint16x4_t svldnt1_x4(svcount_t, uint16_t const *);
	19078	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_bf16_x4)))
	19079	svbfloat16x4_t svldnt1_x4(svcount_t, bfloat16_t const *);
	19080	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f16_x4)))
	19081	svfloat16x4_t svldnt1_x4(svcount_t, float16_t const *);
	19082	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s16_x4)))
	19083	svint16x4_t svldnt1_x4(svcount_t, int16_t const *);
	19084	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_u32_x4)))
	19085	svuint32x4_t svldnt1_x4(svcount_t, uint32_t const *);
	19086	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_f32_x4)))
	19087	svfloat32x4_t svldnt1_x4(svcount_t, float32_t const *);
	19088	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_s32_x4)))
	19089	svint32x4_t svldnt1_x4(svcount_t, int32_t const *);
	19090	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x2)))
	19091	svuint8x2_t svldnt1_vnum_x2(svcount_t, uint8_t const *, int64_t);
	19092	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x2)))
	19093	svint8x2_t svldnt1_vnum_x2(svcount_t, int8_t const *, int64_t);
	19094	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x2)))
	19095	svmfloat8x2_t svldnt1_vnum_x2(svcount_t, mfloat8_t const *, int64_t);
	19096	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x2)))
	19097	svuint64x2_t svldnt1_vnum_x2(svcount_t, uint64_t const *, int64_t);
	19098	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x2)))
	19099	svfloat64x2_t svldnt1_vnum_x2(svcount_t, float64_t const *, int64_t);
	19100	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x2)))
	19101	svint64x2_t svldnt1_vnum_x2(svcount_t, int64_t const *, int64_t);
	19102	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x2)))
	19103	svuint16x2_t svldnt1_vnum_x2(svcount_t, uint16_t const *, int64_t);
	19104	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x2)))
	19105	svbfloat16x2_t svldnt1_vnum_x2(svcount_t, bfloat16_t const *, int64_t);
	19106	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x2)))
	19107	svfloat16x2_t svldnt1_vnum_x2(svcount_t, float16_t const *, int64_t);
	19108	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x2)))
	19109	svint16x2_t svldnt1_vnum_x2(svcount_t, int16_t const *, int64_t);
	19110	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x2)))
	19111	svuint32x2_t svldnt1_vnum_x2(svcount_t, uint32_t const *, int64_t);
	19112	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x2)))
	19113	svfloat32x2_t svldnt1_vnum_x2(svcount_t, float32_t const *, int64_t);
	19114	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x2)))
	19115	svint32x2_t svldnt1_vnum_x2(svcount_t, int32_t const *, int64_t);
	19116	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u8_x4)))
	19117	svuint8x4_t svldnt1_vnum_x4(svcount_t, uint8_t const *, int64_t);
	19118	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s8_x4)))
	19119	svint8x4_t svldnt1_vnum_x4(svcount_t, int8_t const *, int64_t);
	19120	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_mf8_x4)))
	19121	svmfloat8x4_t svldnt1_vnum_x4(svcount_t, mfloat8_t const *, int64_t);
	19122	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u64_x4)))
	19123	svuint64x4_t svldnt1_vnum_x4(svcount_t, uint64_t const *, int64_t);
	19124	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f64_x4)))
	19125	svfloat64x4_t svldnt1_vnum_x4(svcount_t, float64_t const *, int64_t);
	19126	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s64_x4)))
	19127	svint64x4_t svldnt1_vnum_x4(svcount_t, int64_t const *, int64_t);
	19128	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u16_x4)))
	19129	svuint16x4_t svldnt1_vnum_x4(svcount_t, uint16_t const *, int64_t);
	19130	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_bf16_x4)))
	19131	svbfloat16x4_t svldnt1_vnum_x4(svcount_t, bfloat16_t const *, int64_t);
	19132	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f16_x4)))
	19133	svfloat16x4_t svldnt1_vnum_x4(svcount_t, float16_t const *, int64_t);
	19134	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s16_x4)))
	19135	svint16x4_t svldnt1_vnum_x4(svcount_t, int16_t const *, int64_t);
	19136	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_u32_x4)))
	19137	svuint32x4_t svldnt1_vnum_x4(svcount_t, uint32_t const *, int64_t);
	19138	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_f32_x4)))
	19139	svfloat32x4_t svldnt1_vnum_x4(svcount_t, float32_t const *, int64_t);
	19140	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svldnt1_vnum_s32_x4)))
	19141	svint32x4_t svldnt1_vnum_x4(svcount_t, int32_t const *, int64_t);
	19142	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_b)))
	19143	svbool_t svreinterpret(svcount_t);
	19144	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svreinterpret_c)))
	19145	svcount_t svreinterpret(svbool_t);
	19146	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x2)))
	19147	void svst1(svcount_t, uint8_t *, svuint8x2_t);
	19148	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x2)))
	19149	void svst1(svcount_t, int8_t *, svint8x2_t);
	19150	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x2)))
	19151	void svst1(svcount_t, mfloat8_t *, svmfloat8x2_t);
	19152	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x2)))
	19153	void svst1(svcount_t, uint64_t *, svuint64x2_t);
	19154	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x2)))
	19155	void svst1(svcount_t, float64_t *, svfloat64x2_t);
	19156	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x2)))
	19157	void svst1(svcount_t, int64_t *, svint64x2_t);
	19158	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x2)))
	19159	void svst1(svcount_t, uint16_t *, svuint16x2_t);
	19160	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x2)))
	19161	void svst1(svcount_t, bfloat16_t *, svbfloat16x2_t);
	19162	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x2)))
	19163	void svst1(svcount_t, float16_t *, svfloat16x2_t);
	19164	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x2)))
	19165	void svst1(svcount_t, int16_t *, svint16x2_t);
	19166	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x2)))
	19167	void svst1(svcount_t, uint32_t *, svuint32x2_t);
	19168	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x2)))
	19169	void svst1(svcount_t, float32_t *, svfloat32x2_t);
	19170	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x2)))
	19171	void svst1(svcount_t, int32_t *, svint32x2_t);
	19172	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u8_x4)))
	19173	void svst1(svcount_t, uint8_t *, svuint8x4_t);
	19174	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s8_x4)))
	19175	void svst1(svcount_t, int8_t *, svint8x4_t);
	19176	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_mf8_x4)))
	19177	void svst1(svcount_t, mfloat8_t *, svmfloat8x4_t);
	19178	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u64_x4)))
	19179	void svst1(svcount_t, uint64_t *, svuint64x4_t);
	19180	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f64_x4)))
	19181	void svst1(svcount_t, float64_t *, svfloat64x4_t);
	19182	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s64_x4)))
	19183	void svst1(svcount_t, int64_t *, svint64x4_t);
	19184	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u16_x4)))
	19185	void svst1(svcount_t, uint16_t *, svuint16x4_t);
	19186	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_bf16_x4)))
	19187	void svst1(svcount_t, bfloat16_t *, svbfloat16x4_t);
	19188	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f16_x4)))
	19189	void svst1(svcount_t, float16_t *, svfloat16x4_t);
	19190	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s16_x4)))
	19191	void svst1(svcount_t, int16_t *, svint16x4_t);
	19192	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_u32_x4)))
	19193	void svst1(svcount_t, uint32_t *, svuint32x4_t);
	19194	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_f32_x4)))
	19195	void svst1(svcount_t, float32_t *, svfloat32x4_t);
	19196	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_s32_x4)))
	19197	void svst1(svcount_t, int32_t *, svint32x4_t);
	19198	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x2)))
	19199	void svst1_vnum(svcount_t, uint8_t *, int64_t, svuint8x2_t);
	19200	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x2)))
	19201	void svst1_vnum(svcount_t, int8_t *, int64_t, svint8x2_t);
	19202	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x2)))
	19203	void svst1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
	19204	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x2)))
	19205	void svst1_vnum(svcount_t, uint64_t *, int64_t, svuint64x2_t);
	19206	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x2)))
	19207	void svst1_vnum(svcount_t, float64_t *, int64_t, svfloat64x2_t);
	19208	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x2)))
	19209	void svst1_vnum(svcount_t, int64_t *, int64_t, svint64x2_t);
	19210	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x2)))
	19211	void svst1_vnum(svcount_t, uint16_t *, int64_t, svuint16x2_t);
	19212	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x2)))
	19213	void svst1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
	19214	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x2)))
	19215	void svst1_vnum(svcount_t, float16_t *, int64_t, svfloat16x2_t);
	19216	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x2)))
	19217	void svst1_vnum(svcount_t, int16_t *, int64_t, svint16x2_t);
	19218	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x2)))
	19219	void svst1_vnum(svcount_t, uint32_t *, int64_t, svuint32x2_t);
	19220	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x2)))
	19221	void svst1_vnum(svcount_t, float32_t *, int64_t, svfloat32x2_t);
	19222	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x2)))
	19223	void svst1_vnum(svcount_t, int32_t *, int64_t, svint32x2_t);
	19224	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u8_x4)))
	19225	void svst1_vnum(svcount_t, uint8_t *, int64_t, svuint8x4_t);
	19226	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s8_x4)))
	19227	void svst1_vnum(svcount_t, int8_t *, int64_t, svint8x4_t);
	19228	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_mf8_x4)))
	19229	void svst1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
	19230	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u64_x4)))
	19231	void svst1_vnum(svcount_t, uint64_t *, int64_t, svuint64x4_t);
	19232	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f64_x4)))
	19233	void svst1_vnum(svcount_t, float64_t *, int64_t, svfloat64x4_t);
	19234	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s64_x4)))
	19235	void svst1_vnum(svcount_t, int64_t *, int64_t, svint64x4_t);
	19236	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u16_x4)))
	19237	void svst1_vnum(svcount_t, uint16_t *, int64_t, svuint16x4_t);
	19238	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_bf16_x4)))
	19239	void svst1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
	19240	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f16_x4)))
	19241	void svst1_vnum(svcount_t, float16_t *, int64_t, svfloat16x4_t);
	19242	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s16_x4)))
	19243	void svst1_vnum(svcount_t, int16_t *, int64_t, svint16x4_t);
	19244	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_u32_x4)))
	19245	void svst1_vnum(svcount_t, uint32_t *, int64_t, svuint32x4_t);
	19246	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_f32_x4)))
	19247	void svst1_vnum(svcount_t, float32_t *, int64_t, svfloat32x4_t);
	19248	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svst1_vnum_s32_x4)))
	19249	void svst1_vnum(svcount_t, int32_t *, int64_t, svint32x4_t);
	19250	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x2)))
	19251	void svstnt1(svcount_t, uint8_t *, svuint8x2_t);
	19252	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x2)))
	19253	void svstnt1(svcount_t, int8_t *, svint8x2_t);
	19254	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x2)))
	19255	void svstnt1(svcount_t, mfloat8_t *, svmfloat8x2_t);
	19256	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x2)))
	19257	void svstnt1(svcount_t, uint64_t *, svuint64x2_t);
	19258	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x2)))
	19259	void svstnt1(svcount_t, float64_t *, svfloat64x2_t);
	19260	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x2)))
	19261	void svstnt1(svcount_t, int64_t *, svint64x2_t);
	19262	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x2)))
	19263	void svstnt1(svcount_t, uint16_t *, svuint16x2_t);
	19264	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x2)))
	19265	void svstnt1(svcount_t, bfloat16_t *, svbfloat16x2_t);
	19266	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x2)))
	19267	void svstnt1(svcount_t, float16_t *, svfloat16x2_t);
	19268	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x2)))
	19269	void svstnt1(svcount_t, int16_t *, svint16x2_t);
	19270	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x2)))
	19271	void svstnt1(svcount_t, uint32_t *, svuint32x2_t);
	19272	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x2)))
	19273	void svstnt1(svcount_t, float32_t *, svfloat32x2_t);
	19274	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x2)))
	19275	void svstnt1(svcount_t, int32_t *, svint32x2_t);
	19276	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u8_x4)))
	19277	void svstnt1(svcount_t, uint8_t *, svuint8x4_t);
	19278	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s8_x4)))
	19279	void svstnt1(svcount_t, int8_t *, svint8x4_t);
	19280	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_mf8_x4)))
	19281	void svstnt1(svcount_t, mfloat8_t *, svmfloat8x4_t);
	19282	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u64_x4)))
	19283	void svstnt1(svcount_t, uint64_t *, svuint64x4_t);
	19284	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f64_x4)))
	19285	void svstnt1(svcount_t, float64_t *, svfloat64x4_t);
	19286	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s64_x4)))
	19287	void svstnt1(svcount_t, int64_t *, svint64x4_t);
	19288	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u16_x4)))
	19289	void svstnt1(svcount_t, uint16_t *, svuint16x4_t);
	19290	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_bf16_x4)))
	19291	void svstnt1(svcount_t, bfloat16_t *, svbfloat16x4_t);
	19292	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f16_x4)))
	19293	void svstnt1(svcount_t, float16_t *, svfloat16x4_t);
	19294	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s16_x4)))
	19295	void svstnt1(svcount_t, int16_t *, svint16x4_t);
	19296	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_u32_x4)))
	19297	void svstnt1(svcount_t, uint32_t *, svuint32x4_t);
	19298	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_f32_x4)))
	19299	void svstnt1(svcount_t, float32_t *, svfloat32x4_t);
	19300	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_s32_x4)))
	19301	void svstnt1(svcount_t, int32_t *, svint32x4_t);
	19302	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x2)))
	19303	void svstnt1_vnum(svcount_t, uint8_t *, int64_t, svuint8x2_t);
	19304	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x2)))
	19305	void svstnt1_vnum(svcount_t, int8_t *, int64_t, svint8x2_t);
	19306	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x2)))
	19307	void svstnt1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x2_t);
	19308	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x2)))
	19309	void svstnt1_vnum(svcount_t, uint64_t *, int64_t, svuint64x2_t);
	19310	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x2)))
	19311	void svstnt1_vnum(svcount_t, float64_t *, int64_t, svfloat64x2_t);
	19312	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x2)))
	19313	void svstnt1_vnum(svcount_t, int64_t *, int64_t, svint64x2_t);
	19314	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x2)))
	19315	void svstnt1_vnum(svcount_t, uint16_t *, int64_t, svuint16x2_t);
	19316	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x2)))
	19317	void svstnt1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x2_t);
	19318	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x2)))
	19319	void svstnt1_vnum(svcount_t, float16_t *, int64_t, svfloat16x2_t);
	19320	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x2)))
	19321	void svstnt1_vnum(svcount_t, int16_t *, int64_t, svint16x2_t);
	19322	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x2)))
	19323	void svstnt1_vnum(svcount_t, uint32_t *, int64_t, svuint32x2_t);
	19324	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x2)))
	19325	void svstnt1_vnum(svcount_t, float32_t *, int64_t, svfloat32x2_t);
	19326	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x2)))
	19327	void svstnt1_vnum(svcount_t, int32_t *, int64_t, svint32x2_t);
	19328	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u8_x4)))
	19329	void svstnt1_vnum(svcount_t, uint8_t *, int64_t, svuint8x4_t);
	19330	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s8_x4)))
	19331	void svstnt1_vnum(svcount_t, int8_t *, int64_t, svint8x4_t);
	19332	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_mf8_x4)))
	19333	void svstnt1_vnum(svcount_t, mfloat8_t *, int64_t, svmfloat8x4_t);
	19334	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u64_x4)))
	19335	void svstnt1_vnum(svcount_t, uint64_t *, int64_t, svuint64x4_t);
	19336	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f64_x4)))
	19337	void svstnt1_vnum(svcount_t, float64_t *, int64_t, svfloat64x4_t);
	19338	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s64_x4)))
	19339	void svstnt1_vnum(svcount_t, int64_t *, int64_t, svint64x4_t);
	19340	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u16_x4)))
	19341	void svstnt1_vnum(svcount_t, uint16_t *, int64_t, svuint16x4_t);
	19342	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_bf16_x4)))
	19343	void svstnt1_vnum(svcount_t, bfloat16_t *, int64_t, svbfloat16x4_t);
	19344	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f16_x4)))
	19345	void svstnt1_vnum(svcount_t, float16_t *, int64_t, svfloat16x4_t);
	19346	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s16_x4)))
	19347	void svstnt1_vnum(svcount_t, int16_t *, int64_t, svint16x4_t);
	19348	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_u32_x4)))
	19349	void svstnt1_vnum(svcount_t, uint32_t *, int64_t, svuint32x4_t);
	19350	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_f32_x4)))
	19351	void svstnt1_vnum(svcount_t, float32_t *, int64_t, svfloat32x4_t);
	19352	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svstnt1_vnum_s32_x4)))
	19353	void svstnt1_vnum(svcount_t, int32_t *, int64_t, svint32x4_t);
	19354	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_s64)))
	19355	svcount_t svwhilege_c8(int64_t, int64_t, uint64_t);
	19356	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_s64)))
	19357	svcount_t svwhilege_c32(int64_t, int64_t, uint64_t);
	19358	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_s64)))
	19359	svcount_t svwhilege_c64(int64_t, int64_t, uint64_t);
	19360	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_s64)))
	19361	svcount_t svwhilege_c16(int64_t, int64_t, uint64_t);
	19362	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c8_u64)))
	19363	svcount_t svwhilege_c8(uint64_t, uint64_t, uint64_t);
	19364	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c32_u64)))
	19365	svcount_t svwhilege_c32(uint64_t, uint64_t, uint64_t);
	19366	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c64_u64)))
	19367	svcount_t svwhilege_c64(uint64_t, uint64_t, uint64_t);
	19368	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilege_c16_u64)))
	19369	svcount_t svwhilege_c16(uint64_t, uint64_t, uint64_t);
	19370	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_s64)))
	19371	svcount_t svwhilegt_c8(int64_t, int64_t, uint64_t);
	19372	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_s64)))
	19373	svcount_t svwhilegt_c32(int64_t, int64_t, uint64_t);
	19374	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_s64)))
	19375	svcount_t svwhilegt_c64(int64_t, int64_t, uint64_t);
	19376	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_s64)))
	19377	svcount_t svwhilegt_c16(int64_t, int64_t, uint64_t);
	19378	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c8_u64)))
	19379	svcount_t svwhilegt_c8(uint64_t, uint64_t, uint64_t);
	19380	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c32_u64)))
	19381	svcount_t svwhilegt_c32(uint64_t, uint64_t, uint64_t);
	19382	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c64_u64)))
	19383	svcount_t svwhilegt_c64(uint64_t, uint64_t, uint64_t);
	19384	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilegt_c16_u64)))
	19385	svcount_t svwhilegt_c16(uint64_t, uint64_t, uint64_t);
	19386	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_s64)))
	19387	svcount_t svwhilele_c8(int64_t, int64_t, uint64_t);
	19388	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_s64)))
	19389	svcount_t svwhilele_c32(int64_t, int64_t, uint64_t);
	19390	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_s64)))
	19391	svcount_t svwhilele_c64(int64_t, int64_t, uint64_t);
	19392	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_s64)))
	19393	svcount_t svwhilele_c16(int64_t, int64_t, uint64_t);
	19394	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c8_u64)))
	19395	svcount_t svwhilele_c8(uint64_t, uint64_t, uint64_t);
	19396	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c32_u64)))
	19397	svcount_t svwhilele_c32(uint64_t, uint64_t, uint64_t);
	19398	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c64_u64)))
	19399	svcount_t svwhilele_c64(uint64_t, uint64_t, uint64_t);
	19400	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilele_c16_u64)))
	19401	svcount_t svwhilele_c16(uint64_t, uint64_t, uint64_t);
	19402	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_u64)))
	19403	svcount_t svwhilelt_c8(uint64_t, uint64_t, uint64_t);
	19404	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_u64)))
	19405	svcount_t svwhilelt_c32(uint64_t, uint64_t, uint64_t);
	19406	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_u64)))
	19407	svcount_t svwhilelt_c64(uint64_t, uint64_t, uint64_t);
	19408	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_u64)))
	19409	svcount_t svwhilelt_c16(uint64_t, uint64_t, uint64_t);
	19410	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c8_s64)))
	19411	svcount_t svwhilelt_c8(int64_t, int64_t, uint64_t);
	19412	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c32_s64)))
	19413	svcount_t svwhilelt_c32(int64_t, int64_t, uint64_t);
	19414	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c64_s64)))
	19415	svcount_t svwhilelt_c64(int64_t, int64_t, uint64_t);
	19416	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svwhilelt_c16_s64)))
	19417	svcount_t svwhilelt_c16(int64_t, int64_t, uint64_t);
19062	19418	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f64_m)))
19063	19419	svfloat64_t svabd_n_f64_m(svbool_t, svfloat64_t, float64_t);
19064	19420	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svabd_n_f32_m)))
...	...	@@ -31877,6 +32233,30 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpa_f32)))
31877	32233	svfloat32_t svexpa(svuint32_t);
31878	32234	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svexpa_f16)))
31879	32235	svfloat16_t svexpa(svuint16_t);
	32236	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u32)))
	32237	svuint32_t svcompact_u32(svbool_t, svuint32_t);
	32238	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u64)))
	32239	svuint64_t svcompact_u64(svbool_t, svuint64_t);
	32240	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f64)))
	32241	svfloat64_t svcompact_f64(svbool_t, svfloat64_t);
	32242	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f32)))
	32243	svfloat32_t svcompact_f32(svbool_t, svfloat32_t);
	32244	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s32)))
	32245	svint32_t svcompact_s32(svbool_t, svint32_t);
	32246	__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s64)))
	32247	svint64_t svcompact_s64(svbool_t, svint64_t);
	32248	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u32)))
	32249	svuint32_t svcompact(svbool_t, svuint32_t);
	32250	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_u64)))
	32251	svuint64_t svcompact(svbool_t, svuint64_t);
	32252	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f64)))
	32253	svfloat64_t svcompact(svbool_t, svfloat64_t);
	32254	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_f32)))
	32255	svfloat32_t svcompact(svbool_t, svfloat32_t);
	32256	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s32)))
	32257	svint32_t svcompact(svbool_t, svint32_t);
	32258	__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svcompact_s64)))
	32259	svint64_t svcompact(svbool_t, svint64_t);
31880	32260	#define svcvtnt_bf16_x svcvtnt_bf16_m
31881	32261	#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m
31882	32262	#define svcvtnt_f16_x svcvtnt_f16_m

lib/include/avx10_2_512bf16intrin.h+19-12

...	...	@@ -21,9 +21,15 @@ typedef __bf16 __m512bh_u __attribute__((__vector_size__(64), __aligned__(1)));
21	21
22	22	/* Define the default attributes for the functions in this file. */
23	23	#define __DEFAULT_FN_ATTRS512 \
24		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
25	25	__min_vector_width__(512)))
26	26
	27	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	28	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
	29	#else
	30	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
	31	#endif
	32
27	33	static __inline __m512bh __DEFAULT_FN_ATTRS512 _mm512_setzero_pbh(void) {
28	34	return __builtin_bit_cast(__m512bh, _mm512_setzero_ps());
29	35	}
...	...	@@ -167,13 +173,13 @@ _mm512_mask_blend_pbh(__mmask32 __U, __m512bh __A, __m512bh __W) {
167	173	(__v32bf)__A);
168	174	}
169	175
170		static __inline__ __m512bh __DEFAULT_FN_ATTRS512
	176	static __inline__ __m512bh __DEFAULT_FN_ATTRS512_CONSTEXPR
171	177	_mm512_permutex2var_pbh(__m512bh __A, __m512i __I, __m512bh __B) {
172	178	return (__m512bh)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I,
173	179	(__v32hi)__B);
174	180	}
175	181
176		static __inline__ __m512bh __DEFAULT_FN_ATTRS512
	182	static __inline__ __m512bh __DEFAULT_FN_ATTRS512_CONSTEXPR
177	183	_mm512_permutexvar_pbh(__m512i __A, __m512bh __B) {
178	184	return (__m512bh)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A);
179	185	}
...	...	@@ -423,7 +429,7 @@ _mm512_maskz_rsqrt_pbh(__mmask32 __U, __m512bh __A) {
423	429	(__v32bf)_mm512_setzero_pbh(), (__mmask32)(__U)))
424	430
425	431	static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_sqrt_pbh(__m512bh __A) {
426		return (__m512bh)__builtin_ia32_vsqrtbf16512((__v32bf)__A);
	432	return __builtin_elementwise_sqrt(__A);
427	433	}
428	434
429	435	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
...	...	@@ -441,8 +447,8 @@ _mm512_maskz_sqrt_pbh(__mmask32 __U, __m512bh __A) {
441	447
442	448	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
443	449	_mm512_fmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
444		return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
445		(__v32bf)__C);
	450	return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
	451	(__v32bf)__C);
446	452	}
447	453
448	454	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
...	...	@@ -469,8 +475,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmadd_pbh(
469	475
470	476	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
471	477	_mm512_fmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
472		return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
473		-(__v32bf)__C);
	478	return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
	479	-(__v32bf)__C);
474	480	}
475	481
476	482	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
...	...	@@ -497,8 +503,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmsub_pbh(
497	503
498	504	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
499	505	_mm512_fnmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
500		return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
501		(__v32bf)__C);
	506	return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
	507	(__v32bf)__C);
502	508	}
503	509
504	510	static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmadd_pbh(
...	...	@@ -527,8 +533,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmadd_pbh(
527	533
528	534	static __inline__ __m512bh __DEFAULT_FN_ATTRS512
529	535	_mm512_fnmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
530		return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
531		-(__v32bf)__C);
	536	return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
	537	-(__v32bf)__C);
532	538	}
533	539
534	540	static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmsub_pbh(
...	...	@@ -555,6 +561,7 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmsub_pbh(
555	561	(__v32bf)_mm512_setzero_pbh());
556	562	}
557	563
	564	#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
558	565	#undef __DEFAULT_FN_ATTRS512
559	566
560	567	#endif

lib/include/avx10_2_512convertintrin.h+1-1

...	...	@@ -18,7 +18,7 @@
18	18
19	19	/* Define the default attributes for the functions in this file. */
20	20	#define __DEFAULT_FN_ATTRS512 \
21		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
	21	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
22	22	__min_vector_width__(512)))
23	23
24	24	static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_cvtx2ps_ph(__m512 __A,

lib/include/avx10_2_512niintrin.h+31-31

...	...	@@ -17,7 +17,7 @@
17	17	#define __AVX10_2_512NIINTRIN_H
18	18
19	19	#define __DEFAULT_FN_ATTRS \
20		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
21	21	__min_vector_width__(512)))
22	22
23	23	/* VNNI FP16 */
...	...	@@ -64,8 +64,8 @@ static __inline__ __m512 __DEFAULT_FN_ATTRS _mm512_maskz_dpph_ps(__mmask16 __U,
64	64	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssd_epi32(__m512i __W,
65	65	__m512i __A,
66	66	__m512i __B) {
67		return (__m512i)__builtin_ia32_vpdpbssd512((__v16si)__W, (__v16si)__A,
68		(__v16si)__B);
	67	return (__m512i)__builtin_ia32_vpdpbssd512((__v16si)__W, (__v64qi)__A,
	68	(__v64qi)__B);
69	69	}
70	70
71	71	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -84,8 +84,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssd_epi32(
84	84	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbssds_epi32(__m512i __W,
85	85	__m512i __A,
86	86	__m512i __B) {
87		return (__m512i)__builtin_ia32_vpdpbssds512((__v16si)__W, (__v16si)__A,
88		(__v16si)__B);
	87	return (__m512i)__builtin_ia32_vpdpbssds512((__v16si)__W, (__v64qi)__A,
	88	(__v64qi)__B);
89	89	}
90	90
91	91	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbssds_epi32(
...	...	@@ -104,8 +104,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbssds_epi32(
104	104	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsud_epi32(__m512i __W,
105	105	__m512i __A,
106	106	__m512i __B) {
107		return (__m512i)__builtin_ia32_vpdpbsud512((__v16si)__W, (__v16si)__A,
108		(__v16si)__B);
	107	return (__m512i)__builtin_ia32_vpdpbsud512((__v16si)__W, (__v64qi)__A,
	108	(__v64qu)__B);
109	109	}
110	110
111	111	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -124,8 +124,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsud_epi32(
124	124	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbsuds_epi32(__m512i __W,
125	125	__m512i __A,
126	126	__m512i __B) {
127		return (__m512i)__builtin_ia32_vpdpbsuds512((__v16si)__W, (__v16si)__A,
128		(__v16si)__B);
	127	return (__m512i)__builtin_ia32_vpdpbsuds512((__v16si)__W, (__v64qi)__A,
	128	(__v64qu)__B);
129	129	}
130	130
131	131	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbsuds_epi32(
...	...	@@ -144,8 +144,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbsuds_epi32(
144	144	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuud_epi32(__m512i __W,
145	145	__m512i __A,
146	146	__m512i __B) {
147		return (__m512i)__builtin_ia32_vpdpbuud512((__v16si)__W, (__v16si)__A,
148		(__v16si)__B);
	147	return (__m512i)__builtin_ia32_vpdpbuud512((__v16si)__W, (__v64qu)__A,
	148	(__v64qu)__B);
149	149	}
150	150
151	151	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -164,8 +164,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuud_epi32(
164	164	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpbuuds_epi32(__m512i __W,
165	165	__m512i __A,
166	166	__m512i __B) {
167		return (__m512i)__builtin_ia32_vpdpbuuds512((__v16si)__W, (__v16si)__A,
168		(__v16si)__B);
	167	return (__m512i)__builtin_ia32_vpdpbuuds512((__v16si)__W, (__v64qu)__A,
	168	(__v64qu)__B);
169	169	}
170	170
171	171	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpbuuds_epi32(
...	...	@@ -185,8 +185,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpbuuds_epi32(
185	185	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsud_epi32(__m512i __A,
186	186	__m512i __B,
187	187	__m512i __C) {
188		return (__m512i)__builtin_ia32_vpdpwsud512((__v16si)__A, (__v16si)__B,
189		(__v16si)__C);
	188	return (__m512i)__builtin_ia32_vpdpwsud512((__v16si)__A, (__v32hi)__B,
	189	(__v32hu)__C);
190	190	}
191	191
192	192	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -197,7 +197,7 @@ _mm512_mask_dpwsud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
197	197	}
198	198
199	199	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsud_epi32(
200		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	200	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
201	201	return (__m512i)__builtin_ia32_selectd_512(
202	202	(__mmask16)__U, (__v16si)_mm512_dpwsud_epi32(__A, __B, __C),
203	203	(__v16si)_mm512_setzero_si512());
...	...	@@ -206,8 +206,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsud_epi32(
206	206	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwsuds_epi32(__m512i __A,
207	207	__m512i __B,
208	208	__m512i __C) {
209		return (__m512i)__builtin_ia32_vpdpwsuds512((__v16si)__A, (__v16si)__B,
210		(__v16si)__C);
	209	return (__m512i)__builtin_ia32_vpdpwsuds512((__v16si)__A, (__v32hi)__B,
	210	(__v32hu)__C);
211	211	}
212	212
213	213	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwsuds_epi32(
...	...	@@ -218,7 +218,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwsuds_epi32(
218	218	}
219	219
220	220	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsuds_epi32(
221		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	221	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
222	222	return (__m512i)__builtin_ia32_selectd_512(
223	223	(__mmask16)__U, (__v16si)_mm512_dpwsuds_epi32(__A, __B, __C),
224	224	(__v16si)_mm512_setzero_si512());
...	...	@@ -227,8 +227,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwsuds_epi32(
227	227	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusd_epi32(__m512i __A,
228	228	__m512i __B,
229	229	__m512i __C) {
230		return (__m512i)__builtin_ia32_vpdpwusd512((__v16si)__A, (__v16si)__B,
231		(__v16si)__C);
	230	return (__m512i)__builtin_ia32_vpdpwusd512((__v16si)__A, (__v32hu)__B,
	231	(__v32hi)__C);
232	232	}
233	233
234	234	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -239,7 +239,7 @@ _mm512_mask_dpwusd_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
239	239	}
240	240
241	241	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusd_epi32(
242		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	242	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
243	243	return (__m512i)__builtin_ia32_selectd_512(
244	244	(__mmask16)__U, (__v16si)_mm512_dpwusd_epi32(__A, __B, __C),
245	245	(__v16si)_mm512_setzero_si512());
...	...	@@ -248,8 +248,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusd_epi32(
248	248	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwusds_epi32(__m512i __A,
249	249	__m512i __B,
250	250	__m512i __C) {
251		return (__m512i)__builtin_ia32_vpdpwusds512((__v16si)__A, (__v16si)__B,
252		(__v16si)__C);
	251	return (__m512i)__builtin_ia32_vpdpwusds512((__v16si)__A, (__v32hu)__B,
	252	(__v32hi)__C);
253	253	}
254	254
255	255	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwusds_epi32(
...	...	@@ -260,7 +260,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwusds_epi32(
260	260	}
261	261
262	262	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusds_epi32(
263		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	263	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
264	264	return (__m512i)__builtin_ia32_selectd_512(
265	265	(__mmask16)__U, (__v16si)_mm512_dpwusds_epi32(__A, __B, __C),
266	266	(__v16si)_mm512_setzero_si512());
...	...	@@ -269,8 +269,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwusds_epi32(
269	269	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuud_epi32(__m512i __A,
270	270	__m512i __B,
271	271	__m512i __C) {
272		return (__m512i)__builtin_ia32_vpdpwuud512((__v16si)__A, (__v16si)__B,
273		(__v16si)__C);
	272	return (__m512i)__builtin_ia32_vpdpwuud512((__v16si)__A, (__v32hu)__B,
	273	(__v32hu)__C);
274	274	}
275	275
276	276	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -281,7 +281,7 @@ _mm512_mask_dpwuud_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
281	281	}
282	282
283	283	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuud_epi32(
284		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	284	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
285	285	return (__m512i)__builtin_ia32_selectd_512(
286	286	(__mmask16)__U, (__v16si)_mm512_dpwuud_epi32(__A, __B, __C),
287	287	(__v16si)_mm512_setzero_si512());
...	...	@@ -290,8 +290,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuud_epi32(
290	290	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_dpwuuds_epi32(__m512i __A,
291	291	__m512i __B,
292	292	__m512i __C) {
293		return (__m512i)__builtin_ia32_vpdpwuuds512((__v16si)__A, (__v16si)__B,
294		(__v16si)__C);
	293	return (__m512i)__builtin_ia32_vpdpwuuds512((__v16si)__A, (__v32hu)__B,
	294	(__v32hu)__C);
295	295	}
296	296
297	297	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwuuds_epi32(
...	...	@@ -302,7 +302,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_dpwuuds_epi32(
302	302	}
303	303
304	304	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_dpwuuds_epi32(
305		__m512i __A, __mmask16 __U, __m512i __B, __m512i __C) {
	305	__mmask16 __U, __m512i __A, __m512i __B, __m512i __C) {
306	306	return (__m512i)__builtin_ia32_selectd_512(
307	307	(__mmask16)__U, (__v16si)_mm512_dpwuuds_epi32(__A, __B, __C),
308	308	(__v16si)_mm512_setzero_si512());

lib/include/avx10_2_512satcvtdsintrin.h+1-1

...	...	@@ -16,7 +16,7 @@
16	16
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-512"), \
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
20	20	__min_vector_width__(512)))
21	21
22	22	// 512 bit : Double -> Int

lib/include/avx10_2bf16intrin.h+54-45

...	...	@@ -21,12 +21,20 @@ typedef __bf16 __m256bh_u __attribute__((__vector_size__(32), __aligned__(1)));
21	21
22	22	/* Define the default attributes for the functions in this file. */
23	23	#define __DEFAULT_FN_ATTRS256 \
24		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
25	25	__min_vector_width__(256)))
26	26	#define __DEFAULT_FN_ATTRS128 \
27		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	27	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
28	28	__min_vector_width__(128)))
29	29
	30	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	31	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	32	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	33	#else
	34	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	35	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	36	#endif
	37
30	38	static __inline __m256bh __DEFAULT_FN_ATTRS256 _mm256_setzero_pbh(void) {
31	39	return __builtin_bit_cast(__m256bh, _mm256_setzero_ps());
32	40	}
...	...	@@ -213,12 +221,12 @@ static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_move_sbh(__m128bh __a,
213	221	return __a;
214	222	}
215	223
216		static __inline__ __m128bh __DEFAULT_FN_ATTRS128
	224	static __inline__ __m128bh __DEFAULT_FN_ATTRS128_CONSTEXPR
217	225	_mm_mask_move_sbh(__m128bh __W, __mmask8 __U, __m128bh __A, __m128bh __B) {
218	226	return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B), __W);
219	227	}
220	228
221		static __inline__ __m128bh __DEFAULT_FN_ATTRS128
	229	static __inline__ __m128bh __DEFAULT_FN_ATTRS128_CONSTEXPR
222	230	_mm_maskz_move_sbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
223	231	return __builtin_ia32_selectsbf_128(__U, _mm_move_sbh(__A, __B),
224	232	_mm_setzero_pbh());
...	...	@@ -287,24 +295,24 @@ _mm256_mask_blend_pbh(__mmask16 __U, __m256bh __A, __m256bh __W) {
287	295	(__v16bf)__A);
288	296	}
289	297
290		static __inline__ __m128bh __DEFAULT_FN_ATTRS128
	298	static __inline__ __m128bh __DEFAULT_FN_ATTRS128_CONSTEXPR
291	299	_mm_permutex2var_pbh(__m128bh __A, __m128i __I, __m128bh __B) {
292	300	return (__m128bh)__builtin_ia32_vpermi2varhi128((__v8hi)__A, (__v8hi)__I,
293	301	(__v8hi)__B);
294	302	}
295	303
296		static __inline__ __m256bh __DEFAULT_FN_ATTRS256
	304	static __inline__ __m256bh __DEFAULT_FN_ATTRS256_CONSTEXPR
297	305	_mm256_permutex2var_pbh(__m256bh __A, __m256i __I, __m256bh __B) {
298	306	return (__m256bh)__builtin_ia32_vpermi2varhi256((__v16hi)__A, (__v16hi)__I,
299	307	(__v16hi)__B);
300	308	}
301	309
302		static __inline__ __m128bh __DEFAULT_FN_ATTRS128
	310	static __inline__ __m128bh __DEFAULT_FN_ATTRS128_CONSTEXPR
303	311	_mm_permutexvar_pbh(__m128i __A, __m128bh __B) {
304	312	return (__m128bh)__builtin_ia32_permvarhi128((__v8hi)__B, (__v8hi)__A);
305	313	}
306	314
307		static __inline__ __m256bh __DEFAULT_FN_ATTRS256
	315	static __inline__ __m256bh __DEFAULT_FN_ATTRS256_CONSTEXPR
308	316	_mm256_permutexvar_pbh(__m256i __A, __m256bh __B) {
309	317	return (__m256bh)__builtin_ia32_permvarhi256((__v16hi)__B, (__v16hi)__A);
310	318	}
...	...	@@ -519,34 +527,34 @@ _mm_maskz_min_pbh(__mmask8 __U, __m128bh __A, __m128bh __B) {
519	527	(__mmask8)__U, (__v8bf)_mm_min_pbh(__A, __B), (__v8bf)_mm_setzero_pbh());
520	528	}
521	529
522		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh A,
523		__m128bh B) {
524		return __builtin_ia32_vcomisbf16eq((__v8bf)A, (__v8bf)B);
	530	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comieq_sbh(__m128bh __A,
	531	__m128bh __B) {
	532	return __builtin_ia32_vcomisbf16eq((__v8bf)__A, (__v8bf)__B);
525	533	}
526	534
527		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh A,
528		__m128bh B) {
529		return __builtin_ia32_vcomisbf16lt((__v8bf)A, (__v8bf)B);
	535	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comilt_sbh(__m128bh __A,
	536	__m128bh __B) {
	537	return __builtin_ia32_vcomisbf16lt((__v8bf)__A, (__v8bf)__B);
530	538	}
531	539
532		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh A,
533		__m128bh B) {
534		return __builtin_ia32_vcomisbf16le((__v8bf)A, (__v8bf)B);
	540	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comile_sbh(__m128bh __A,
	541	__m128bh __B) {
	542	return __builtin_ia32_vcomisbf16le((__v8bf)__A, (__v8bf)__B);
535	543	}
536	544
537		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh A,
538		__m128bh B) {
539		return __builtin_ia32_vcomisbf16gt((__v8bf)A, (__v8bf)B);
	545	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comigt_sbh(__m128bh __A,
	546	__m128bh __B) {
	547	return __builtin_ia32_vcomisbf16gt((__v8bf)__A, (__v8bf)__B);
540	548	}
541	549
542		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh A,
543		__m128bh B) {
544		return __builtin_ia32_vcomisbf16ge((__v8bf)A, (__v8bf)B);
	550	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comige_sbh(__m128bh __A,
	551	__m128bh __B) {
	552	return __builtin_ia32_vcomisbf16ge((__v8bf)__A, (__v8bf)__B);
545	553	}
546	554
547		static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh A,
548		__m128bh B) {
549		return __builtin_ia32_vcomisbf16neq((__v8bf)A, (__v8bf)B);
	555	static __inline__ int __DEFAULT_FN_ATTRS128 _mm_comineq_sbh(__m128bh __A,
	556	__m128bh __B) {
	557	return __builtin_ia32_vcomisbf16neq((__v8bf)__A, (__v8bf)__B);
550	558	}
551	559
552	560	#define _mm256_cmp_pbh_mask(__A, __B, __P) \
...	...	@@ -818,7 +826,7 @@ _mm_maskz_rsqrt_pbh(__mmask8 __U, __m128bh __A) {
818	826	(__v8bf)_mm_setzero_pbh(), (__mmask8)(__U)))
819	827
820	828	static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_sqrt_pbh(__m256bh __A) {
821		return (__m256bh)__builtin_ia32_vsqrtbf16256((__v16bf)__A);
	829	return __builtin_elementwise_sqrt(__A);
822	830	}
823	831
824	832	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
...	...	@@ -835,7 +843,7 @@ _mm256_maskz_sqrt_pbh(__mmask16 __U, __m256bh __A) {
835	843	}
836	844
837	845	static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_sqrt_pbh(__m128bh __A) {
838		return (__m128bh)__builtin_ia32_vsqrtbf16((__v8bf)__A);
	846	return __builtin_elementwise_sqrt(__A);
839	847	}
840	848
841	849	static __inline__ __m128bh __DEFAULT_FN_ATTRS128
...	...	@@ -852,8 +860,8 @@ _mm_maskz_sqrt_pbh(__mmask8 __U, __m128bh __A) {
852	860
853	861	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
854	862	_mm256_fmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
855		return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
856		(__v16bf)__C);
	863	return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
	864	(__v16bf)__C);
857	865	}
858	866
859	867	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
...	...	@@ -880,8 +888,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmadd_pbh(
880	888
881	889	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
882	890	_mm256_fmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
883		return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
884		-(__v16bf)__C);
	891	return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
	892	-(__v16bf)__C);
885	893	}
886	894
887	895	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
...	...	@@ -908,8 +916,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmsub_pbh(
908	916
909	917	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
910	918	_mm256_fnmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
911		return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
912		(__v16bf)__C);
	919	return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
	920	(__v16bf)__C);
913	921	}
914	922
915	923	static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmadd_pbh(
...	...	@@ -938,8 +946,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmadd_pbh(
938	946
939	947	static __inline__ __m256bh __DEFAULT_FN_ATTRS256
940	948	_mm256_fnmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
941		return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
942		-(__v16bf)__C);
	949	return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
	950	-(__v16bf)__C);
943	951	}
944	952
945	953	static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmsub_pbh(
...	...	@@ -969,8 +977,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmsub_pbh(
969	977	static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmadd_pbh(__m128bh __A,
970	978	__m128bh __B,
971	979	__m128bh __C) {
972		return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
973		(__v8bf)__C);
	980	return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
	981	(__v8bf)__C);
974	982	}
975	983
976	984	static __inline__ __m128bh __DEFAULT_FN_ATTRS128
...	...	@@ -997,8 +1005,8 @@ _mm_maskz_fmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
997	1005	static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmsub_pbh(__m128bh __A,
998	1006	__m128bh __B,
999	1007	__m128bh __C) {
1000		return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
1001		-(__v8bf)__C);
	1008	return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
	1009	-(__v8bf)__C);
1002	1010	}
1003	1011
1004	1012	static __inline__ __m128bh __DEFAULT_FN_ATTRS128
...	...	@@ -1025,8 +1033,8 @@ _mm_maskz_fmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1025	1033	static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmadd_pbh(__m128bh __A,
1026	1034	__m128bh __B,
1027	1035	__m128bh __C) {
1028		return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
1029		(__v8bf)__C);
	1036	return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
	1037	(__v8bf)__C);
1030	1038	}
1031	1039
1032	1040	static __inline__ __m128bh __DEFAULT_FN_ATTRS128
...	...	@@ -1053,8 +1061,8 @@ _mm_maskz_fnmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1053	1061	static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmsub_pbh(__m128bh __A,
1054	1062	__m128bh __B,
1055	1063	__m128bh __C) {
1056		return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
1057		-(__v8bf)__C);
	1064	return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
	1065	-(__v8bf)__C);
1058	1066	}
1059	1067
1060	1068	static __inline__ __m128bh __DEFAULT_FN_ATTRS128
...	...	@@ -1080,6 +1088,7 @@ _mm_maskz_fnmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
1080	1088
1081	1089	#undef __DEFAULT_FN_ATTRS128
1082	1090	#undef __DEFAULT_FN_ATTRS256
1083
	1091	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	1092	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
1084	1093	#endif
1085	1094	#endif

lib/include/avx10_2convertintrin.h+2-2

...	...	@@ -18,10 +18,10 @@
18	18
19	19	/* Define the default attributes for the functions in this file. */
20	20	#define __DEFAULT_FN_ATTRS128 \
21		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	21	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
22	22	__min_vector_width__(128)))
23	23	#define __DEFAULT_FN_ATTRS256 \
24		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
25	25	__min_vector_width__(256)))
26	26
27	27	// clang-format off

lib/include/avx10_2copyintrin.h+1-1

...	...	@@ -16,7 +16,7 @@
16	16
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS128 \
19		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
20	20	__min_vector_width__(128)))
21	21
22	22	/// Constructs a 128-bit integer vector, setting the lower 32 bits to the

lib/include/avx10_2niintrin.h+14-14

...	...	@@ -16,10 +16,10 @@
16	16	#define __AVX10_2NIINTRIN_H
17	17
18	18	#define __DEFAULT_FN_ATTRS128 \
19		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
20	20	__min_vector_width__(128)))
21	21	#define __DEFAULT_FN_ATTRS256 \
22		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	22	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
23	23	__min_vector_width__(256)))
24	24
25	25	/* VNNI FP16 */
...	...	@@ -253,7 +253,7 @@ _mm_mask_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
253	253	}
254	254
255	255	static __inline__ __m128i __DEFAULT_FN_ATTRS128
256		_mm_maskz_dpwsud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	256	_mm_maskz_dpwsud_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
257	257	return (__m128i)__builtin_ia32_selectd_128(
258	258	(__mmask8)__U, (__v4si)_mm_dpwsud_epi32(__A, __B, __C),
259	259	(__v4si)_mm_setzero_si128());
...	...	@@ -266,7 +266,7 @@ _mm256_mask_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
266	266	}
267	267
268	268	static __inline__ __m256i __DEFAULT_FN_ATTRS256
269		_mm256_maskz_dpwsud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	269	_mm256_maskz_dpwsud_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
270	270	return (__m256i)__builtin_ia32_selectd_256(
271	271	(__mmask8)__U, (__v8si)_mm256_dpwsud_epi32(__A, __B, __C),
272	272	(__v8si)_mm256_setzero_si256());
...	...	@@ -279,7 +279,7 @@ _mm_mask_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
279	279	}
280	280
281	281	static __inline__ __m128i __DEFAULT_FN_ATTRS128
282		_mm_maskz_dpwsuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	282	_mm_maskz_dpwsuds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
283	283	return (__m128i)__builtin_ia32_selectd_128(
284	284	(__mmask8)__U, (__v4si)_mm_dpwsuds_epi32(__A, __B, __C),
285	285	(__v4si)_mm_setzero_si128());
...	...	@@ -292,7 +292,7 @@ _mm256_mask_dpwsuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
292	292	}
293	293
294	294	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwsuds_epi32(
295		__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	295	__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
296	296	return (__m256i)__builtin_ia32_selectd_256(
297	297	(__mmask8)__U, (__v8si)_mm256_dpwsuds_epi32(__A, __B, __C),
298	298	(__v8si)_mm256_setzero_si256());
...	...	@@ -305,7 +305,7 @@ _mm_mask_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
305	305	}
306	306
307	307	static __inline__ __m128i __DEFAULT_FN_ATTRS128
308		_mm_maskz_dpwusd_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	308	_mm_maskz_dpwusd_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
309	309	return (__m128i)__builtin_ia32_selectd_128(
310	310	(__mmask8)__U, (__v4si)_mm_dpwusd_epi32(__A, __B, __C),
311	311	(__v4si)_mm_setzero_si128());
...	...	@@ -318,7 +318,7 @@ _mm256_mask_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
318	318	}
319	319
320	320	static __inline__ __m256i __DEFAULT_FN_ATTRS256
321		_mm256_maskz_dpwusd_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	321	_mm256_maskz_dpwusd_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
322	322	return (__m256i)__builtin_ia32_selectd_256(
323	323	(__mmask8)__U, (__v8si)_mm256_dpwusd_epi32(__A, __B, __C),
324	324	(__v8si)_mm256_setzero_si256());
...	...	@@ -331,7 +331,7 @@ _mm_mask_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
331	331	}
332	332
333	333	static __inline__ __m128i __DEFAULT_FN_ATTRS128
334		_mm_maskz_dpwusds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	334	_mm_maskz_dpwusds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
335	335	return (__m128i)__builtin_ia32_selectd_128(
336	336	(__mmask8)__U, (__v4si)_mm_dpwusds_epi32(__A, __B, __C),
337	337	(__v4si)_mm_setzero_si128());
...	...	@@ -344,7 +344,7 @@ _mm256_mask_dpwusds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
344	344	}
345	345
346	346	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwusds_epi32(
347		__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	347	__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
348	348	return (__m256i)__builtin_ia32_selectd_256(
349	349	(__mmask8)__U, (__v8si)_mm256_dpwusds_epi32(__A, __B, __C),
350	350	(__v8si)_mm256_setzero_si256());
...	...	@@ -357,7 +357,7 @@ _mm_mask_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
357	357	}
358	358
359	359	static __inline__ __m128i __DEFAULT_FN_ATTRS128
360		_mm_maskz_dpwuud_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	360	_mm_maskz_dpwuud_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
361	361	return (__m128i)__builtin_ia32_selectd_128(
362	362	(__mmask8)__U, (__v4si)_mm_dpwuud_epi32(__A, __B, __C),
363	363	(__v4si)_mm_setzero_si128());
...	...	@@ -370,7 +370,7 @@ _mm256_mask_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
370	370	}
371	371
372	372	static __inline__ __m256i __DEFAULT_FN_ATTRS256
373		_mm256_maskz_dpwuud_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	373	_mm256_maskz_dpwuud_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
374	374	return (__m256i)__builtin_ia32_selectd_256(
375	375	(__mmask8)__U, (__v8si)_mm256_dpwuud_epi32(__A, __B, __C),
376	376	(__v8si)_mm256_setzero_si256());
...	...	@@ -383,7 +383,7 @@ _mm_mask_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
383	383	}
384	384
385	385	static __inline__ __m128i __DEFAULT_FN_ATTRS128
386		_mm_maskz_dpwuuds_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
	386	_mm_maskz_dpwuuds_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
387	387	return (__m128i)__builtin_ia32_selectd_128(
388	388	(__mmask8)__U, (__v4si)_mm_dpwuuds_epi32(__A, __B, __C),
389	389	(__v4si)_mm_setzero_si128());
...	...	@@ -396,7 +396,7 @@ _mm256_mask_dpwuuds_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
396	396	}
397	397
398	398	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_dpwuuds_epi32(
399		__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
	399	__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
400	400	return (__m256i)__builtin_ia32_selectd_256(
401	401	(__mmask8)__U, (__v8si)_mm256_dpwuuds_epi32(__A, __B, __C),
402	402	(__v8si)_mm256_setzero_si256());

lib/include/avx10_2satcvtdsintrin.h+2-2

...	...	@@ -17,11 +17,11 @@
17	17
18	18	/* Define the default attributes for the functions in this file. */
19	19	#define __DEFAULT_FN_ATTRS256 \
20		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
21	21	__min_vector_width__(256)))
22	22
23	23	#define __DEFAULT_FN_ATTRS128 \
24		__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"), \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx10.2"), \
25	25	__min_vector_width__(128)))
26	26
27	27	#define _mm_cvtts_roundsd_i32(__A, __R) \

lib/include/avx2intrin.h+275-377

...	...	@@ -15,20 +15,19 @@
15	15	#define __AVX2INTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
18		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
19		#define __DEFAULT_FN_ATTRS256 \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("avx2,no-evex512"), __min_vector_width__(256)))
22		#define __DEFAULT_FN_ATTRS128 \
23		__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx2,no-evex512"), __min_vector_width__(128)))
25		#else
26	18	#define __DEFAULT_FN_ATTRS256 \
27	19	__attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
28	20	__min_vector_width__(256)))
29	21	#define __DEFAULT_FN_ATTRS128 \
30	22	__attribute__((__always_inline__, __nodebug__, __target__("avx2"), \
31	23	__min_vector_width__(128)))
	24
	25	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	26	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	27	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	28	#else
	29	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	30	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
32	31	#endif
33	32
34	33	/* SSE4 Multiple Packed Sums of Absolute Difference. */
...	...	@@ -104,10 +103,9 @@
104	103	/// \param __a
105	104	/// A 256-bit integer vector.
106	105	/// \returns A 256-bit integer vector containing the result.
107		static __inline__ __m256i __DEFAULT_FN_ATTRS256
108		_mm256_abs_epi8(__m256i __a)
109		{
110		return (__m256i)__builtin_elementwise_abs((__v32qs)__a);
	106	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	107	_mm256_abs_epi8(__m256i __a) {
	108	return (__m256i)__builtin_elementwise_abs((__v32qs)__a);
111	109	}
112	110
113	111	/// Computes the absolute value of each signed 16-bit element in the 256-bit
...	...	@@ -121,10 +119,9 @@ _mm256_abs_epi8(__m256i __a)
121	119	/// \param __a
122	120	/// A 256-bit vector of [16 x i16].
123	121	/// \returns A 256-bit vector of [16 x i16] containing the result.
124		static __inline__ __m256i __DEFAULT_FN_ATTRS256
125		_mm256_abs_epi16(__m256i __a)
126		{
127		return (__m256i)__builtin_elementwise_abs((__v16hi)__a);
	122	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	123	_mm256_abs_epi16(__m256i __a) {
	124	return (__m256i)__builtin_elementwise_abs((__v16hi)__a);
128	125	}
129	126
130	127	/// Computes the absolute value of each signed 32-bit element in the 256-bit
...	...	@@ -138,10 +135,9 @@ _mm256_abs_epi16(__m256i __a)
138	135	/// \param __a
139	136	/// A 256-bit vector of [8 x i32].
140	137	/// \returns A 256-bit vector of [8 x i32] containing the result.
141		static __inline__ __m256i __DEFAULT_FN_ATTRS256
142		_mm256_abs_epi32(__m256i __a)
143		{
144		return (__m256i)__builtin_elementwise_abs((__v8si)__a);
	138	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	139	_mm256_abs_epi32(__m256i __a) {
	140	return (__m256i)__builtin_elementwise_abs((__v8si)__a);
145	141	}
146	142
147	143	/// Converts the elements of two 256-bit vectors of [16 x i16] to 8-bit
...	...	@@ -169,9 +165,8 @@ _mm256_abs_epi32(__m256i __a)
169	165	/// A 256-bit vector of [16 x i16] used to generate result[127:64] and
170	166	/// result[255:192].
171	167	/// \returns A 256-bit integer vector containing the result.
172		static __inline__ __m256i __DEFAULT_FN_ATTRS256
173		_mm256_packs_epi16(__m256i __a, __m256i __b)
174		{
	168	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	169	_mm256_packs_epi16(__m256i __a, __m256i __b) {
175	170	return (__m256i)__builtin_ia32_packsswb256((__v16hi)__a, (__v16hi)__b);
176	171	}
177	172
...	...	@@ -201,9 +196,8 @@ _mm256_packs_epi16(__m256i __a, __m256i __b)
201	196	/// A 256-bit vector of [8 x i32] used to generate result[127:64] and
202	197	/// result[255:192].
203	198	/// \returns A 256-bit vector of [16 x i16] containing the result.
204		static __inline__ __m256i __DEFAULT_FN_ATTRS256
205		_mm256_packs_epi32(__m256i __a, __m256i __b)
206		{
	199	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	200	_mm256_packs_epi32(__m256i __a, __m256i __b) {
207	201	return (__m256i)__builtin_ia32_packssdw256((__v8si)__a, (__v8si)__b);
208	202	}
209	203
...	...	@@ -232,9 +226,8 @@ _mm256_packs_epi32(__m256i __a, __m256i __b)
232	226	/// A 256-bit vector of [16 x i16] used to generate result[127:64] and
233	227	/// result[255:192].
234	228	/// \returns A 256-bit integer vector containing the result.
235		static __inline__ __m256i __DEFAULT_FN_ATTRS256
236		_mm256_packus_epi16(__m256i __a, __m256i __b)
237		{
	229	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	230	_mm256_packus_epi16(__m256i __a, __m256i __b) {
238	231	return (__m256i)__builtin_ia32_packuswb256((__v16hi)__a, (__v16hi)__b);
239	232	}
240	233
...	...	@@ -264,9 +257,8 @@ _mm256_packus_epi16(__m256i __a, __m256i __b)
264	257	/// A 256-bit vector of [8 x i32] used to generate result[127:64] and
265	258	/// result[255:192].
266	259	/// \returns A 256-bit vector of [16 x i16] containing the result.
267		static __inline__ __m256i __DEFAULT_FN_ATTRS256
268		_mm256_packus_epi32(__m256i __V1, __m256i __V2)
269		{
	260	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	261	_mm256_packus_epi32(__m256i __V1, __m256i __V2) {
270	262	return (__m256i) __builtin_ia32_packusdw256((__v8si)__V1, (__v8si)__V2);
271	263	}
272	264
...	...	@@ -283,9 +275,8 @@ _mm256_packus_epi32(__m256i __V1, __m256i __V2)
283	275	/// \param __b
284	276	/// A 256-bit integer vector containing one of the source operands.
285	277	/// \returns A 256-bit integer vector containing the sums.
286		static __inline__ __m256i __DEFAULT_FN_ATTRS256
287		_mm256_add_epi8(__m256i __a, __m256i __b)
288		{
	278	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	279	_mm256_add_epi8(__m256i __a, __m256i __b) {
289	280	return (__m256i)((__v32qu)__a + (__v32qu)__b);
290	281	}
291	282
...	...	@@ -302,9 +293,8 @@ _mm256_add_epi8(__m256i __a, __m256i __b)
302	293	/// \param __b
303	294	/// A 256-bit vector of [16 x i16] containing one of the source operands.
304	295	/// \returns A 256-bit vector of [16 x i16] containing the sums.
305		static __inline__ __m256i __DEFAULT_FN_ATTRS256
306		_mm256_add_epi16(__m256i __a, __m256i __b)
307		{
	296	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	297	_mm256_add_epi16(__m256i __a, __m256i __b) {
308	298	return (__m256i)((__v16hu)__a + (__v16hu)__b);
309	299	}
310	300
...	...	@@ -321,9 +311,8 @@ _mm256_add_epi16(__m256i __a, __m256i __b)
321	311	/// \param __b
322	312	/// A 256-bit vector of [8 x i32] containing one of the source operands.
323	313	/// \returns A 256-bit vector of [8 x i32] containing the sums.
324		static __inline__ __m256i __DEFAULT_FN_ATTRS256
325		_mm256_add_epi32(__m256i __a, __m256i __b)
326		{
	314	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	315	_mm256_add_epi32(__m256i __a, __m256i __b) {
327	316	return (__m256i)((__v8su)__a + (__v8su)__b);
328	317	}
329	318
...	...	@@ -340,9 +329,8 @@ _mm256_add_epi32(__m256i __a, __m256i __b)
340	329	/// \param __b
341	330	/// A 256-bit vector of [4 x i64] containing one of the source operands.
342	331	/// \returns A 256-bit vector of [4 x i64] containing the sums.
343		static __inline__ __m256i __DEFAULT_FN_ATTRS256
344		_mm256_add_epi64(__m256i __a, __m256i __b)
345		{
	332	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	333	_mm256_add_epi64(__m256i __a, __m256i __b) {
346	334	return (__m256i)((__v4du)__a + (__v4du)__b);
347	335	}
348	336
...	...	@@ -359,9 +347,8 @@ _mm256_add_epi64(__m256i __a, __m256i __b)
359	347	/// \param __b
360	348	/// A 256-bit integer vector containing one of the source operands.
361	349	/// \returns A 256-bit integer vector containing the sums.
362		static __inline__ __m256i __DEFAULT_FN_ATTRS256
363		_mm256_adds_epi8(__m256i __a, __m256i __b)
364		{
	350	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	351	_mm256_adds_epi8(__m256i __a, __m256i __b) {
365	352	return (__m256i)__builtin_elementwise_add_sat((__v32qs)__a, (__v32qs)__b);
366	353	}
367	354
...	...	@@ -377,9 +364,8 @@ _mm256_adds_epi8(__m256i __a, __m256i __b)
377	364	/// \param __b
378	365	/// A 256-bit vector of [16 x i16] containing one of the source operands.
379	366	/// \returns A 256-bit vector of [16 x i16] containing the sums.
380		static __inline__ __m256i __DEFAULT_FN_ATTRS256
381		_mm256_adds_epi16(__m256i __a, __m256i __b)
382		{
	367	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	368	_mm256_adds_epi16(__m256i __a, __m256i __b) {
383	369	return (__m256i)__builtin_elementwise_add_sat((__v16hi)__a, (__v16hi)__b);
384	370	}
385	371
...	...	@@ -396,9 +382,8 @@ _mm256_adds_epi16(__m256i __a, __m256i __b)
396	382	/// \param __b
397	383	/// A 256-bit integer vector containing one of the source operands.
398	384	/// \returns A 256-bit integer vector containing the sums.
399		static __inline__ __m256i __DEFAULT_FN_ATTRS256
400		_mm256_adds_epu8(__m256i __a, __m256i __b)
401		{
	385	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	386	_mm256_adds_epu8(__m256i __a, __m256i __b) {
402	387	return (__m256i)__builtin_elementwise_add_sat((__v32qu)__a, (__v32qu)__b);
403	388	}
404	389
...	...	@@ -414,9 +399,8 @@ _mm256_adds_epu8(__m256i __a, __m256i __b)
414	399	/// \param __b
415	400	/// A 256-bit vector of [16 x i16] containing one of the source operands.
416	401	/// \returns A 256-bit vector of [16 x i16] containing the sums.
417		static __inline__ __m256i __DEFAULT_FN_ATTRS256
418		_mm256_adds_epu16(__m256i __a, __m256i __b)
419		{
	402	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	403	_mm256_adds_epu16(__m256i __a, __m256i __b) {
420	404	return (__m256i)__builtin_elementwise_add_sat((__v16hu)__a, (__v16hu)__b);
421	405	}
422	406
...	...	@@ -460,7 +444,7 @@ _mm256_adds_epu16(__m256i __a, __m256i __b)
460	444	/// \param __b
461	445	/// A 256-bit integer vector.
462	446	/// \returns A 256-bit integer vector containing the result.
463		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	447	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
464	448	_mm256_and_si256(__m256i __a, __m256i __b)
465	449	{
466	450	return (__m256i)((__v4du)__a & (__v4du)__b);
...	...	@@ -478,7 +462,7 @@ _mm256_and_si256(__m256i __a, __m256i __b)
478	462	/// \param __b
479	463	/// A 256-bit integer vector.
480	464	/// \returns A 256-bit integer vector containing the result.
481		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	465	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
482	466	_mm256_andnot_si256(__m256i __a, __m256i __b)
483	467	{
484	468	return (__m256i)(~(__v4du)__a & (__v4du)__b);
...	...	@@ -504,10 +488,9 @@ _mm256_andnot_si256(__m256i __a, __m256i __b)
504	488	/// \param __b
505	489	/// A 256-bit integer vector.
506	490	/// \returns A 256-bit integer vector containing the result.
507		static __inline__ __m256i __DEFAULT_FN_ATTRS256
508		_mm256_avg_epu8(__m256i __a, __m256i __b)
509		{
510		return (__m256i)__builtin_ia32_pavgb256((__v32qi)__a, (__v32qi)__b);
	491	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	492	_mm256_avg_epu8(__m256i __a, __m256i __b) {
	493	return (__m256i)__builtin_ia32_pavgb256((__v32qu)__a, (__v32qu)__b);
511	494	}
512	495
513	496	/// Computes the averages of the corresponding unsigned 16-bit integers in
...	...	@@ -530,10 +513,9 @@ _mm256_avg_epu8(__m256i __a, __m256i __b)
530	513	/// \param __b
531	514	/// A 256-bit vector of [16 x i16].
532	515	/// \returns A 256-bit vector of [16 x i16] containing the result.
533		static __inline__ __m256i __DEFAULT_FN_ATTRS256
534		_mm256_avg_epu16(__m256i __a, __m256i __b)
535		{
536		return (__m256i)__builtin_ia32_pavgw256((__v16hi)__a, (__v16hi)__b);
	516	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	517	_mm256_avg_epu16(__m256i __a, __m256i __b) {
	518	return (__m256i)__builtin_ia32_pavgw256((__v16hu)__a, (__v16hu)__b);
537	519	}
538	520
539	521	/// Merges 8-bit integer values from either of the two 256-bit vectors
...	...	@@ -565,9 +547,8 @@ _mm256_avg_epu16(__m256i __a, __m256i __b)
565	547	/// is 0, the byte is copied from \a __V1; otherwise, it is copied from
566	548	/// \a __V2.
567	549	/// \returns A 256-bit integer vector containing the result.
568		static __inline__ __m256i __DEFAULT_FN_ATTRS256
569		_mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M)
570		{
	550	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	551	_mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M) {
571	552	return (__m256i)__builtin_ia32_pblendvb256((__v32qi)__V1, (__v32qi)__V2,
572	553	(__v32qi)__M);
573	554	}
...	...	@@ -633,7 +614,7 @@ _mm256_blendv_epi8(__m256i __V1, __m256i __V2, __m256i __M)
633	614	/// \param __b
634	615	/// A 256-bit integer vector containing one of the inputs.
635	616	/// \returns A 256-bit integer vector containing the result.
636		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	617	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
637	618	_mm256_cmpeq_epi8(__m256i __a, __m256i __b)
638	619	{
639	620	return (__m256i)((__v32qi)__a == (__v32qi)__b);
...	...	@@ -659,7 +640,7 @@ _mm256_cmpeq_epi8(__m256i __a, __m256i __b)
659	640	/// \param __b
660	641	/// A 256-bit vector of [16 x i16] containing one of the inputs.
661	642	/// \returns A 256-bit vector of [16 x i16] containing the result.
662		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	643	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
663	644	_mm256_cmpeq_epi16(__m256i __a, __m256i __b)
664	645	{
665	646	return (__m256i)((__v16hi)__a == (__v16hi)__b);
...	...	@@ -685,7 +666,7 @@ _mm256_cmpeq_epi16(__m256i __a, __m256i __b)
685	666	/// \param __b
686	667	/// A 256-bit vector of [8 x i32] containing one of the inputs.
687	668	/// \returns A 256-bit vector of [8 x i32] containing the result.
688		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	669	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
689	670	_mm256_cmpeq_epi32(__m256i __a, __m256i __b)
690	671	{
691	672	return (__m256i)((__v8si)__a == (__v8si)__b);
...	...	@@ -711,7 +692,7 @@ _mm256_cmpeq_epi32(__m256i __a, __m256i __b)
711	692	/// \param __b
712	693	/// A 256-bit vector of [4 x i64] containing one of the inputs.
713	694	/// \returns A 256-bit vector of [4 x i64] containing the result.
714		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	695	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
715	696	_mm256_cmpeq_epi64(__m256i __a, __m256i __b)
716	697	{
717	698	return (__m256i)((__v4di)__a == (__v4di)__b);
...	...	@@ -737,7 +718,7 @@ _mm256_cmpeq_epi64(__m256i __a, __m256i __b)
737	718	/// \param __b
738	719	/// A 256-bit integer vector containing one of the inputs.
739	720	/// \returns A 256-bit integer vector containing the result.
740		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	721	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
741	722	_mm256_cmpgt_epi8(__m256i __a, __m256i __b)
742	723	{
743	724	/* This function always performs a signed comparison, but __v32qi is a char
...	...	@@ -765,7 +746,7 @@ _mm256_cmpgt_epi8(__m256i __a, __m256i __b)
765	746	/// \param __b
766	747	/// A 256-bit vector of [16 x i16] containing one of the inputs.
767	748	/// \returns A 256-bit vector of [16 x i16] containing the result.
768		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	749	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
769	750	_mm256_cmpgt_epi16(__m256i __a, __m256i __b)
770	751	{
771	752	return (__m256i)((__v16hi)__a > (__v16hi)__b);
...	...	@@ -791,7 +772,7 @@ _mm256_cmpgt_epi16(__m256i __a, __m256i __b)
791	772	/// \param __b
792	773	/// A 256-bit vector of [8 x i32] containing one of the inputs.
793	774	/// \returns A 256-bit vector of [8 x i32] containing the result.
794		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	775	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
795	776	_mm256_cmpgt_epi32(__m256i __a, __m256i __b)
796	777	{
797	778	return (__m256i)((__v8si)__a > (__v8si)__b);
...	...	@@ -817,7 +798,7 @@ _mm256_cmpgt_epi32(__m256i __a, __m256i __b)
817	798	/// \param __b
818	799	/// A 256-bit vector of [4 x i64] containing one of the inputs.
819	800	/// \returns A 256-bit vector of [4 x i64] containing the result.
820		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	801	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
821	802	_mm256_cmpgt_epi64(__m256i __a, __m256i __b)
822	803	{
823	804	return (__m256i)((__v4di)__a > (__v4di)__b);
...	...	@@ -853,10 +834,9 @@ _mm256_cmpgt_epi64(__m256i __a, __m256i __b)
853	834	/// \param __b
854	835	/// A 256-bit vector of [16 x i16] containing one of the source operands.
855	836	/// \returns A 256-bit vector of [16 x i16] containing the sums.
856		static __inline__ __m256i __DEFAULT_FN_ATTRS256
857		_mm256_hadd_epi16(__m256i __a, __m256i __b)
858		{
859		return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b);
	837	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	838	_mm256_hadd_epi16(__m256i __a, __m256i __b) {
	839	return (__m256i)__builtin_ia32_phaddw256((__v16hi)__a, (__v16hi)__b);
860	840	}
861	841
862	842	/// Horizontally adds the adjacent pairs of 32-bit integers from two 256-bit
...	...	@@ -885,10 +865,9 @@ _mm256_hadd_epi16(__m256i __a, __m256i __b)
885	865	/// \param __b
886	866	/// A 256-bit vector of [8 x i32] containing one of the source operands.
887	867	/// \returns A 256-bit vector of [8 x i32] containing the sums.
888		static __inline__ __m256i __DEFAULT_FN_ATTRS256
889		_mm256_hadd_epi32(__m256i __a, __m256i __b)
890		{
891		return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b);
	868	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	869	_mm256_hadd_epi32(__m256i __a, __m256i __b) {
	870	return (__m256i)__builtin_ia32_phaddd256((__v8si)__a, (__v8si)__b);
892	871	}
893	872
894	873	/// Horizontally adds the adjacent pairs of 16-bit integers from two 256-bit
...	...	@@ -920,10 +899,9 @@ _mm256_hadd_epi32(__m256i __a, __m256i __b)
920	899	/// \param __b
921	900	/// A 256-bit vector of [16 x i16] containing one of the source operands.
922	901	/// \returns A 256-bit vector of [16 x i16] containing the sums.
923		static __inline__ __m256i __DEFAULT_FN_ATTRS256
924		_mm256_hadds_epi16(__m256i __a, __m256i __b)
925		{
926		return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b);
	902	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	903	_mm256_hadds_epi16(__m256i __a, __m256i __b) {
	904	return (__m256i)__builtin_ia32_phaddsw256((__v16hi)__a, (__v16hi)__b);
927	905	}
928	906
929	907	/// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit
...	...	@@ -956,10 +934,9 @@ _mm256_hadds_epi16(__m256i __a, __m256i __b)
956	934	/// \param __b
957	935	/// A 256-bit vector of [16 x i16] containing one of the source operands.
958	936	/// \returns A 256-bit vector of [16 x i16] containing the differences.
959		static __inline__ __m256i __DEFAULT_FN_ATTRS256
960		_mm256_hsub_epi16(__m256i __a, __m256i __b)
961		{
962		return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b);
	937	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	938	_mm256_hsub_epi16(__m256i __a, __m256i __b) {
	939	return (__m256i)__builtin_ia32_phsubw256((__v16hi)__a, (__v16hi)__b);
963	940	}
964	941
965	942	/// Horizontally subtracts adjacent pairs of 32-bit integers from two 256-bit
...	...	@@ -988,10 +965,9 @@ _mm256_hsub_epi16(__m256i __a, __m256i __b)
988	965	/// \param __b
989	966	/// A 256-bit vector of [8 x i32] containing one of the source operands.
990	967	/// \returns A 256-bit vector of [8 x i32] containing the differences.
991		static __inline__ __m256i __DEFAULT_FN_ATTRS256
992		_mm256_hsub_epi32(__m256i __a, __m256i __b)
993		{
994		return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b);
	968	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	969	_mm256_hsub_epi32(__m256i __a, __m256i __b) {
	970	return (__m256i)__builtin_ia32_phsubd256((__v8si)__a, (__v8si)__b);
995	971	}
996	972
997	973	/// Horizontally subtracts adjacent pairs of 16-bit integers from two 256-bit
...	...	@@ -1024,10 +1000,9 @@ _mm256_hsub_epi32(__m256i __a, __m256i __b)
1024	1000	/// \param __b
1025	1001	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1026	1002	/// \returns A 256-bit vector of [16 x i16] containing the differences.
1027		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1028		_mm256_hsubs_epi16(__m256i __a, __m256i __b)
1029		{
1030		return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b);
	1003	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1004	_mm256_hsubs_epi16(__m256i __a, __m256i __b) {
	1005	return (__m256i)__builtin_ia32_phsubsw256((__v16hi)__a, (__v16hi)__b);
1031	1006	}
1032	1007
1033	1008	/// Multiplies each unsigned byte from the 256-bit integer vector in \a __a
...	...	@@ -1054,10 +1029,9 @@ _mm256_hsubs_epi16(__m256i __a, __m256i __b)
1054	1029	/// \param __b
1055	1030	/// A 256-bit vector containing one of the source operands.
1056	1031	/// \returns A 256-bit vector of [16 x i16] containing the result.
1057		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1058		_mm256_maddubs_epi16(__m256i __a, __m256i __b)
1059		{
1060		return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b);
	1032	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1033	_mm256_maddubs_epi16(__m256i __a, __m256i __b) {
	1034	return (__m256i)__builtin_ia32_pmaddubsw256((__v32qi)__a, (__v32qi)__b);
1061	1035	}
1062	1036
1063	1037	/// Multiplies corresponding 16-bit elements of two 256-bit vectors of
...	...	@@ -1086,9 +1060,8 @@ _mm256_maddubs_epi16(__m256i __a, __m256i __b)
1086	1060	/// \param __b
1087	1061	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1088	1062	/// \returns A 256-bit vector of [8 x i32] containing the result.
1089		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1090		_mm256_madd_epi16(__m256i __a, __m256i __b)
1091		{
	1063	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1064	_mm256_madd_epi16(__m256i __a, __m256i __b) {
1092	1065	return (__m256i)__builtin_ia32_pmaddwd256((__v16hi)__a, (__v16hi)__b);
1093	1066	}
1094	1067
...	...	@@ -1105,9 +1078,8 @@ _mm256_madd_epi16(__m256i __a, __m256i __b)
1105	1078	/// \param __b
1106	1079	/// A 256-bit integer vector.
1107	1080	/// \returns A 256-bit integer vector containing the result.
1108		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1109		_mm256_max_epi8(__m256i __a, __m256i __b)
1110		{
	1081	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1082	_mm256_max_epi8(__m256i __a, __m256i __b) {
1111	1083	return (__m256i)__builtin_elementwise_max((__v32qs)__a, (__v32qs)__b);
1112	1084	}
1113	1085
...	...	@@ -1124,9 +1096,8 @@ _mm256_max_epi8(__m256i __a, __m256i __b)
1124	1096	/// \param __b
1125	1097	/// A 256-bit vector of [16 x i16].
1126	1098	/// \returns A 256-bit vector of [16 x i16] containing the result.
1127		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1128		_mm256_max_epi16(__m256i __a, __m256i __b)
1129		{
	1099	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1100	_mm256_max_epi16(__m256i __a, __m256i __b) {
1130	1101	return (__m256i)__builtin_elementwise_max((__v16hi)__a, (__v16hi)__b);
1131	1102	}
1132	1103
...	...	@@ -1143,9 +1114,8 @@ _mm256_max_epi16(__m256i __a, __m256i __b)
1143	1114	/// \param __b
1144	1115	/// A 256-bit vector of [8 x i32].
1145	1116	/// \returns A 256-bit vector of [8 x i32] containing the result.
1146		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1147		_mm256_max_epi32(__m256i __a, __m256i __b)
1148		{
	1117	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1118	_mm256_max_epi32(__m256i __a, __m256i __b) {
1149	1119	return (__m256i)__builtin_elementwise_max((__v8si)__a, (__v8si)__b);
1150	1120	}
1151	1121
...	...	@@ -1162,9 +1132,8 @@ _mm256_max_epi32(__m256i __a, __m256i __b)
1162	1132	/// \param __b
1163	1133	/// A 256-bit integer vector.
1164	1134	/// \returns A 256-bit integer vector containing the result.
1165		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1166		_mm256_max_epu8(__m256i __a, __m256i __b)
1167		{
	1135	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1136	_mm256_max_epu8(__m256i __a, __m256i __b) {
1168	1137	return (__m256i)__builtin_elementwise_max((__v32qu)__a, (__v32qu)__b);
1169	1138	}
1170	1139
...	...	@@ -1181,9 +1150,8 @@ _mm256_max_epu8(__m256i __a, __m256i __b)
1181	1150	/// \param __b
1182	1151	/// A 256-bit vector of [16 x i16].
1183	1152	/// \returns A 256-bit vector of [16 x i16] containing the result.
1184		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1185		_mm256_max_epu16(__m256i __a, __m256i __b)
1186		{
	1153	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1154	_mm256_max_epu16(__m256i __a, __m256i __b) {
1187	1155	return (__m256i)__builtin_elementwise_max((__v16hu)__a, (__v16hu)__b);
1188	1156	}
1189	1157
...	...	@@ -1200,9 +1168,8 @@ _mm256_max_epu16(__m256i __a, __m256i __b)
1200	1168	/// \param __b
1201	1169	/// A 256-bit vector of [8 x i32].
1202	1170	/// \returns A 256-bit vector of [8 x i32] containing the result.
1203		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1204		_mm256_max_epu32(__m256i __a, __m256i __b)
1205		{
	1171	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1172	_mm256_max_epu32(__m256i __a, __m256i __b) {
1206	1173	return (__m256i)__builtin_elementwise_max((__v8su)__a, (__v8su)__b);
1207	1174	}
1208	1175
...	...	@@ -1219,9 +1186,8 @@ _mm256_max_epu32(__m256i __a, __m256i __b)
1219	1186	/// \param __b
1220	1187	/// A 256-bit integer vector.
1221	1188	/// \returns A 256-bit integer vector containing the result.
1222		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1223		_mm256_min_epi8(__m256i __a, __m256i __b)
1224		{
	1189	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1190	_mm256_min_epi8(__m256i __a, __m256i __b) {
1225	1191	return (__m256i)__builtin_elementwise_min((__v32qs)__a, (__v32qs)__b);
1226	1192	}
1227	1193
...	...	@@ -1238,9 +1204,8 @@ _mm256_min_epi8(__m256i __a, __m256i __b)
1238	1204	/// \param __b
1239	1205	/// A 256-bit vector of [16 x i16].
1240	1206	/// \returns A 256-bit vector of [16 x i16] containing the result.
1241		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1242		_mm256_min_epi16(__m256i __a, __m256i __b)
1243		{
	1207	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1208	_mm256_min_epi16(__m256i __a, __m256i __b) {
1244	1209	return (__m256i)__builtin_elementwise_min((__v16hi)__a, (__v16hi)__b);
1245	1210	}
1246	1211
...	...	@@ -1257,9 +1222,8 @@ _mm256_min_epi16(__m256i __a, __m256i __b)
1257	1222	/// \param __b
1258	1223	/// A 256-bit vector of [8 x i32].
1259	1224	/// \returns A 256-bit vector of [8 x i32] containing the result.
1260		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1261		_mm256_min_epi32(__m256i __a, __m256i __b)
1262		{
	1225	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1226	_mm256_min_epi32(__m256i __a, __m256i __b) {
1263	1227	return (__m256i)__builtin_elementwise_min((__v8si)__a, (__v8si)__b);
1264	1228	}
1265	1229
...	...	@@ -1276,9 +1240,8 @@ _mm256_min_epi32(__m256i __a, __m256i __b)
1276	1240	/// \param __b
1277	1241	/// A 256-bit integer vector.
1278	1242	/// \returns A 256-bit integer vector containing the result.
1279		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1280		_mm256_min_epu8(__m256i __a, __m256i __b)
1281		{
	1243	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1244	_mm256_min_epu8(__m256i __a, __m256i __b) {
1282	1245	return (__m256i)__builtin_elementwise_min((__v32qu)__a, (__v32qu)__b);
1283	1246	}
1284	1247
...	...	@@ -1295,9 +1258,8 @@ _mm256_min_epu8(__m256i __a, __m256i __b)
1295	1258	/// \param __b
1296	1259	/// A 256-bit vector of [16 x i16].
1297	1260	/// \returns A 256-bit vector of [16 x i16] containing the result.
1298		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1299		_mm256_min_epu16(__m256i __a, __m256i __b)
1300		{
	1261	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1262	_mm256_min_epu16(__m256i __a, __m256i __b) {
1301	1263	return (__m256i)__builtin_elementwise_min((__v16hu)__a, (__v16hu)__b);
1302	1264	}
1303	1265
...	...	@@ -1314,9 +1276,8 @@ _mm256_min_epu16(__m256i __a, __m256i __b)
1314	1276	/// \param __b
1315	1277	/// A 256-bit vector of [8 x i32].
1316	1278	/// \returns A 256-bit vector of [8 x i32] containing the result.
1317		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1318		_mm256_min_epu32(__m256i __a, __m256i __b)
1319		{
	1279	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1280	_mm256_min_epu32(__m256i __a, __m256i __b) {
1320	1281	return (__m256i)__builtin_elementwise_min((__v8su)__a, (__v8su)__b);
1321	1282	}
1322	1283
...	...	@@ -1337,9 +1298,8 @@ _mm256_min_epu32(__m256i __a, __m256i __b)
1337	1298	/// \param __a
1338	1299	/// A 256-bit integer vector containing the source bytes.
1339	1300	/// \returns The 32-bit integer mask.
1340		static __inline__ int __DEFAULT_FN_ATTRS256
1341		_mm256_movemask_epi8(__m256i __a)
1342		{
	1301	static __inline__ int __DEFAULT_FN_ATTRS256_CONSTEXPR
	1302	_mm256_movemask_epi8(__m256i __a) {
1343	1303	return __builtin_ia32_pmovmskb256((__v32qi)__a);
1344	1304	}
1345	1305
...	...	@@ -1363,9 +1323,8 @@ _mm256_movemask_epi8(__m256i __a)
1363	1323	/// A 128-bit integer vector containing the source bytes.
1364	1324	/// \returns A 256-bit vector of [16 x i16] containing the sign-extended
1365	1325	/// values.
1366		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1367		_mm256_cvtepi8_epi16(__m128i __V)
1368		{
	1326	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1327	_mm256_cvtepi8_epi16(__m128i __V) {
1369	1328	/* This function always performs a signed extension, but __v16qi is a char
1370	1329	which may be signed or unsigned, so use __v16qs. */
1371	1330	return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi);
...	...	@@ -1391,9 +1350,8 @@ _mm256_cvtepi8_epi16(__m128i __V)
1391	1350	/// A 128-bit integer vector containing the source bytes.
1392	1351	/// \returns A 256-bit vector of [8 x i32] containing the sign-extended
1393	1352	/// values.
1394		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1395		_mm256_cvtepi8_epi32(__m128i __V)
1396		{
	1353	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1354	_mm256_cvtepi8_epi32(__m128i __V) {
1397	1355	/* This function always performs a signed extension, but __v16qi is a char
1398	1356	which may be signed or unsigned, so use __v16qs. */
1399	1357	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
...	...	@@ -1418,9 +1376,8 @@ _mm256_cvtepi8_epi32(__m128i __V)
1418	1376	/// A 128-bit integer vector containing the source bytes.
1419	1377	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
1420	1378	/// values.
1421		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1422		_mm256_cvtepi8_epi64(__m128i __V)
1423		{
	1379	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1380	_mm256_cvtepi8_epi64(__m128i __V) {
1424	1381	/* This function always performs a signed extension, but __v16qi is a char
1425	1382	which may be signed or unsigned, so use __v16qs. */
1426	1383	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di);
...	...	@@ -1446,9 +1403,8 @@ _mm256_cvtepi8_epi64(__m128i __V)
1446	1403	/// A 128-bit vector of [8 x i16] containing the source values.
1447	1404	/// \returns A 256-bit vector of [8 x i32] containing the sign-extended
1448	1405	/// values.
1449		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1450		_mm256_cvtepi16_epi32(__m128i __V)
1451		{
	1406	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1407	_mm256_cvtepi16_epi32(__m128i __V) {
1452	1408	return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si);
1453	1409	}
1454	1410
...	...	@@ -1471,9 +1427,8 @@ _mm256_cvtepi16_epi32(__m128i __V)
1471	1427	/// A 128-bit vector of [8 x i16] containing the source values.
1472	1428	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
1473	1429	/// values.
1474		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1475		_mm256_cvtepi16_epi64(__m128i __V)
1476		{
	1430	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1431	_mm256_cvtepi16_epi64(__m128i __V) {
1477	1432	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di);
1478	1433	}
1479	1434
...	...	@@ -1496,9 +1451,8 @@ _mm256_cvtepi16_epi64(__m128i __V)
1496	1451	/// A 128-bit vector of [4 x i32] containing the source values.
1497	1452	/// \returns A 256-bit vector of [4 x i64] containing the sign-extended
1498	1453	/// values.
1499		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1500		_mm256_cvtepi32_epi64(__m128i __V)
1501		{
	1454	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1455	_mm256_cvtepi32_epi64(__m128i __V) {
1502	1456	return (__m256i)__builtin_convertvector((__v4si)__V, __v4di);
1503	1457	}
1504	1458
...	...	@@ -1522,9 +1476,8 @@ _mm256_cvtepi32_epi64(__m128i __V)
1522	1476	/// A 128-bit integer vector containing the source bytes.
1523	1477	/// \returns A 256-bit vector of [16 x i16] containing the zero-extended
1524	1478	/// values.
1525		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1526		_mm256_cvtepu8_epi16(__m128i __V)
1527		{
	1479	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1480	_mm256_cvtepu8_epi16(__m128i __V) {
1528	1481	return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi);
1529	1482	}
1530	1483
...	...	@@ -1548,9 +1501,8 @@ _mm256_cvtepu8_epi16(__m128i __V)
1548	1501	/// A 128-bit integer vector containing the source bytes.
1549	1502	/// \returns A 256-bit vector of [8 x i32] containing the zero-extended
1550	1503	/// values.
1551		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1552		_mm256_cvtepu8_epi32(__m128i __V)
1553		{
	1504	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1505	_mm256_cvtepu8_epi32(__m128i __V) {
1554	1506	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
1555	1507	}
1556	1508
...	...	@@ -1573,9 +1525,8 @@ _mm256_cvtepu8_epi32(__m128i __V)
1573	1525	/// A 128-bit integer vector containing the source bytes.
1574	1526	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
1575	1527	/// values.
1576		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1577		_mm256_cvtepu8_epi64(__m128i __V)
1578		{
	1528	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1529	_mm256_cvtepu8_epi64(__m128i __V) {
1579	1530	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di);
1580	1531	}
1581	1532
...	...	@@ -1599,9 +1550,8 @@ _mm256_cvtepu8_epi64(__m128i __V)
1599	1550	/// A 128-bit vector of [8 x i16] containing the source values.
1600	1551	/// \returns A 256-bit vector of [8 x i32] containing the zero-extended
1601	1552	/// values.
1602		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1603		_mm256_cvtepu16_epi32(__m128i __V)
1604		{
	1553	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1554	_mm256_cvtepu16_epi32(__m128i __V) {
1605	1555	return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si);
1606	1556	}
1607	1557
...	...	@@ -1624,9 +1574,8 @@ _mm256_cvtepu16_epi32(__m128i __V)
1624	1574	/// A 128-bit vector of [8 x i16] containing the source values.
1625	1575	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
1626	1576	/// values.
1627		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1628		_mm256_cvtepu16_epi64(__m128i __V)
1629		{
	1577	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1578	_mm256_cvtepu16_epi64(__m128i __V) {
1630	1579	return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di);
1631	1580	}
1632	1581
...	...	@@ -1649,9 +1598,8 @@ _mm256_cvtepu16_epi64(__m128i __V)
1649	1598	/// A 128-bit vector of [4 x i32] containing the source values.
1650	1599	/// \returns A 256-bit vector of [4 x i64] containing the zero-extended
1651	1600	/// values.
1652		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1653		_mm256_cvtepu32_epi64(__m128i __V)
1654		{
	1601	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1602	_mm256_cvtepu32_epi64(__m128i __V) {
1655	1603	return (__m256i)__builtin_convertvector((__v4su)__V, __v4di);
1656	1604	}
1657	1605
...	...	@@ -1675,9 +1623,8 @@ _mm256_cvtepu32_epi64(__m128i __V)
1675	1623	/// \param __b
1676	1624	/// A 256-bit vector of [8 x i32] containing one of the source operands.
1677	1625	/// \returns A 256-bit vector of [4 x i64] containing the products.
1678		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1679		_mm256_mul_epi32(__m256i __a, __m256i __b)
1680		{
	1626	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1627	_mm256_mul_epi32(__m256i __a, __m256i __b) {
1681	1628	return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b);
1682	1629	}
1683	1630
...	...	@@ -1702,9 +1649,8 @@ _mm256_mul_epi32(__m256i __a, __m256i __b)
1702	1649	/// \param __b
1703	1650	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1704	1651	/// \returns A 256-bit vector of [16 x i16] containing the rounded products.
1705		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1706		_mm256_mulhrs_epi16(__m256i __a, __m256i __b)
1707		{
	1652	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1653	_mm256_mulhrs_epi16(__m256i __a, __m256i __b) {
1708	1654	return (__m256i)__builtin_ia32_pmulhrsw256((__v16hi)__a, (__v16hi)__b);
1709	1655	}
1710	1656
...	...	@@ -1721,10 +1667,9 @@ _mm256_mulhrs_epi16(__m256i __a, __m256i __b)
1721	1667	/// \param __b
1722	1668	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1723	1669	/// \returns A 256-bit vector of [16 x i16] containing the products.
1724		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1725		_mm256_mulhi_epu16(__m256i __a, __m256i __b)
1726		{
1727		return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b);
	1670	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1671	_mm256_mulhi_epu16(__m256i __a, __m256i __b) {
	1672	return (__m256i)__builtin_ia32_pmulhuw256((__v16hu)__a, (__v16hu)__b);
1728	1673	}
1729	1674
1730	1675	/// Multiplies signed 16-bit integer elements of two 256-bit vectors of
...	...	@@ -1740,7 +1685,7 @@ _mm256_mulhi_epu16(__m256i __a, __m256i __b)
1740	1685	/// \param __b
1741	1686	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1742	1687	/// \returns A 256-bit vector of [16 x i16] containing the products.
1743		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1688	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1744	1689	_mm256_mulhi_epi16(__m256i __a, __m256i __b)
1745	1690	{
1746	1691	return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b);
...	...	@@ -1759,7 +1704,7 @@ _mm256_mulhi_epi16(__m256i __a, __m256i __b)
1759	1704	/// \param __b
1760	1705	/// A 256-bit vector of [16 x i16] containing one of the source operands.
1761	1706	/// \returns A 256-bit vector of [16 x i16] containing the products.
1762		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1707	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1763	1708	_mm256_mullo_epi16(__m256i __a, __m256i __b)
1764	1709	{
1765	1710	return (__m256i)((__v16hu)__a * (__v16hu)__b);
...	...	@@ -1778,9 +1723,8 @@ _mm256_mullo_epi16(__m256i __a, __m256i __b)
1778	1723	/// \param __b
1779	1724	/// A 256-bit vector of [8 x i32] containing one of the source operands.
1780	1725	/// \returns A 256-bit vector of [8 x i32] containing the products.
1781		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1782		_mm256_mullo_epi32 (__m256i __a, __m256i __b)
1783		{
	1726	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1727	_mm256_mullo_epi32(__m256i __a, __m256i __b) {
1784	1728	return (__m256i)((__v8su)__a * (__v8su)__b);
1785	1729	}
1786	1730
...	...	@@ -1804,9 +1748,8 @@ _mm256_mullo_epi32 (__m256i __a, __m256i __b)
1804	1748	/// \param __b
1805	1749	/// A 256-bit vector of [8 x i32] containing one of the source operands.
1806	1750	/// \returns A 256-bit vector of [4 x i64] containing the products.
1807		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1808		_mm256_mul_epu32(__m256i __a, __m256i __b)
1809		{
	1751	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1752	_mm256_mul_epu32(__m256i __a, __m256i __b) {
1810	1753	return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b);
1811	1754	}
1812	1755
...	...	@@ -1822,7 +1765,7 @@ _mm256_mul_epu32(__m256i __a, __m256i __b)
1822	1765	/// \param __b
1823	1766	/// A 256-bit integer vector.
1824	1767	/// \returns A 256-bit integer vector containing the result.
1825		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1768	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1826	1769	_mm256_or_si256(__m256i __a, __m256i __b)
1827	1770	{
1828	1771	return (__m256i)((__v4du)__a \| (__v4du)__b);
...	...	@@ -1906,9 +1849,8 @@ _mm256_sad_epu8(__m256i __a, __m256i __b)
1906	1849	/// control byte specify the index (within the same 128-bit half) of \a __a
1907	1850	/// to copy to the result byte.
1908	1851	/// \returns A 256-bit integer vector containing the result.
1909		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1910		_mm256_shuffle_epi8(__m256i __a, __m256i __b)
1911		{
	1852	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1853	_mm256_shuffle_epi8(__m256i __a, __m256i __b) {
1912	1854	return (__m256i)__builtin_ia32_pshufb256((__v32qi)__a, (__v32qi)__b);
1913	1855	}
1914	1856
...	...	@@ -2033,10 +1975,9 @@ _mm256_shuffle_epi8(__m256i __a, __m256i __b)
2033	1975	/// \param __b
2034	1976	/// A 256-bit integer vector].
2035	1977	/// \returns A 256-bit integer vector containing the result.
2036		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2037		_mm256_sign_epi8(__m256i __a, __m256i __b)
2038		{
2039		return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b);
	1978	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1979	_mm256_sign_epi8(__m256i __a, __m256i __b) {
	1980	return (__m256i)__builtin_ia32_psignb256((__v32qi)__a, (__v32qi)__b);
2040	1981	}
2041	1982
2042	1983	/// Sets each element of the result to the corresponding element of the
...	...	@@ -2054,10 +1995,9 @@ _mm256_sign_epi8(__m256i __a, __m256i __b)
2054	1995	/// \param __b
2055	1996	/// A 256-bit vector of [16 x i16].
2056	1997	/// \returns A 256-bit vector of [16 x i16] containing the result.
2057		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2058		_mm256_sign_epi16(__m256i __a, __m256i __b)
2059		{
2060		return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b);
	1998	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1999	_mm256_sign_epi16(__m256i __a, __m256i __b) {
	2000	return (__m256i)__builtin_ia32_psignw256((__v16hi)__a, (__v16hi)__b);
2061	2001	}
2062	2002
2063	2003	/// Sets each element of the result to the corresponding element of the
...	...	@@ -2075,10 +2015,9 @@ _mm256_sign_epi16(__m256i __a, __m256i __b)
2075	2015	/// \param __b
2076	2016	/// A 256-bit vector of [8 x i32].
2077	2017	/// \returns A 256-bit vector of [8 x i32] containing the result.
2078		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2079		_mm256_sign_epi32(__m256i __a, __m256i __b)
2080		{
2081		return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b);
	2018	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2019	_mm256_sign_epi32(__m256i __a, __m256i __b) {
	2020	return (__m256i)__builtin_ia32_psignd256((__v8si)__a, (__v8si)__b);
2082	2021	}
2083	2022
2084	2023	/// Shifts each 128-bit half of the 256-bit integer vector \a a left by
...	...	@@ -2098,8 +2037,9 @@ _mm256_sign_epi32(__m256i __a, __m256i __b)
2098	2037	/// \param imm
2099	2038	/// An unsigned immediate value specifying the shift count (in bytes).
2100	2039	/// \returns A 256-bit integer vector containing the result.
2101		#define _mm256_slli_si256(a, imm) \
2102		((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
	2040	#define _mm256_slli_si256(a, imm) \
	2041	((__m256i)__builtin_ia32_pslldqi256_byteshift((__v32qi)(__m256i)(a), \
	2042	(int)(imm)))
2103	2043
2104	2044	/// Shifts each 128-bit half of the 256-bit integer vector \a a left by
2105	2045	/// \a imm bytes, shifting in zero bytes, and returns the result. If \a imm
...	...	@@ -2118,8 +2058,9 @@ _mm256_sign_epi32(__m256i __a, __m256i __b)
2118	2058	/// \param imm
2119	2059	/// An unsigned immediate value specifying the shift count (in bytes).
2120	2060	/// \returns A 256-bit integer vector containing the result.
2121		#define _mm256_bslli_epi128(a, imm) \
2122		((__m256i)__builtin_ia32_pslldqi256_byteshift((__v4di)(__m256i)(a), (int)(imm)))
	2061	#define _mm256_bslli_epi128(a, imm) \
	2062	((__m256i)__builtin_ia32_pslldqi256_byteshift((__v32qi)(__m256i)(a), \
	2063	(int)(imm)))
2123	2064
2124	2065	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
2125	2066	/// left by \a __count bits, shifting in zero bits, and returns the result.
...	...	@@ -2134,9 +2075,8 @@ _mm256_sign_epi32(__m256i __a, __m256i __b)
2134	2075	/// \param __count
2135	2076	/// An unsigned integer value specifying the shift count (in bits).
2136	2077	/// \returns A 256-bit vector of [16 x i16] containing the result.
2137		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2138		_mm256_slli_epi16(__m256i __a, int __count)
2139		{
	2078	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2079	_mm256_slli_epi16(__m256i __a, int __count) {
2140	2080	return (__m256i)__builtin_ia32_psllwi256((__v16hi)__a, __count);
2141	2081	}
2142	2082
...	...	@@ -2155,9 +2095,8 @@ _mm256_slli_epi16(__m256i __a, int __count)
2155	2095	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2156	2096	/// shift count (in bits). The upper element is ignored.
2157	2097	/// \returns A 256-bit vector of [16 x i16] containing the result.
2158		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2159		_mm256_sll_epi16(__m256i __a, __m128i __count)
2160		{
	2098	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2099	_mm256_sll_epi16(__m256i __a, __m128i __count) {
2161	2100	return (__m256i)__builtin_ia32_psllw256((__v16hi)__a, (__v8hi)__count);
2162	2101	}
2163	2102
...	...	@@ -2174,9 +2113,8 @@ _mm256_sll_epi16(__m256i __a, __m128i __count)
2174	2113	/// \param __count
2175	2114	/// An unsigned integer value specifying the shift count (in bits).
2176	2115	/// \returns A 256-bit vector of [8 x i32] containing the result.
2177		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2178		_mm256_slli_epi32(__m256i __a, int __count)
2179		{
	2116	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2117	_mm256_slli_epi32(__m256i __a, int __count) {
2180	2118	return (__m256i)__builtin_ia32_pslldi256((__v8si)__a, __count);
2181	2119	}
2182	2120
...	...	@@ -2195,9 +2133,8 @@ _mm256_slli_epi32(__m256i __a, int __count)
2195	2133	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2196	2134	/// shift count (in bits). The upper element is ignored.
2197	2135	/// \returns A 256-bit vector of [8 x i32] containing the result.
2198		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2199		_mm256_sll_epi32(__m256i __a, __m128i __count)
2200		{
	2136	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2137	_mm256_sll_epi32(__m256i __a, __m128i __count) {
2201	2138	return (__m256i)__builtin_ia32_pslld256((__v8si)__a, (__v4si)__count);
2202	2139	}
2203	2140
...	...	@@ -2214,9 +2151,8 @@ _mm256_sll_epi32(__m256i __a, __m128i __count)
2214	2151	/// \param __count
2215	2152	/// An unsigned integer value specifying the shift count (in bits).
2216	2153	/// \returns A 256-bit vector of [4 x i64] containing the result.
2217		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2218		_mm256_slli_epi64(__m256i __a, int __count)
2219		{
	2154	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2155	_mm256_slli_epi64(__m256i __a, int __count) {
2220	2156	return __builtin_ia32_psllqi256((__v4di)__a, __count);
2221	2157	}
2222	2158
...	...	@@ -2235,9 +2171,8 @@ _mm256_slli_epi64(__m256i __a, int __count)
2235	2171	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2236	2172	/// shift count (in bits). The upper element is ignored.
2237	2173	/// \returns A 256-bit vector of [4 x i64] containing the result.
2238		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2239		_mm256_sll_epi64(__m256i __a, __m128i __count)
2240		{
	2174	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2175	_mm256_sll_epi64(__m256i __a, __m128i __count) {
2241	2176	return __builtin_ia32_psllq256((__v4di)__a, __count);
2242	2177	}
2243	2178
...	...	@@ -2255,9 +2190,8 @@ _mm256_sll_epi64(__m256i __a, __m128i __count)
2255	2190	/// \param __count
2256	2191	/// An unsigned integer value specifying the shift count (in bits).
2257	2192	/// \returns A 256-bit vector of [16 x i16] containing the result.
2258		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2259		_mm256_srai_epi16(__m256i __a, int __count)
2260		{
	2193	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2194	_mm256_srai_epi16(__m256i __a, int __count) {
2261	2195	return (__m256i)__builtin_ia32_psrawi256((__v16hi)__a, __count);
2262	2196	}
2263	2197
...	...	@@ -2277,9 +2211,8 @@ _mm256_srai_epi16(__m256i __a, int __count)
2277	2211	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2278	2212	/// shift count (in bits). The upper element is ignored.
2279	2213	/// \returns A 256-bit vector of [16 x i16] containing the result.
2280		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2281		_mm256_sra_epi16(__m256i __a, __m128i __count)
2282		{
	2214	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2215	_mm256_sra_epi16(__m256i __a, __m128i __count) {
2283	2216	return (__m256i)__builtin_ia32_psraw256((__v16hi)__a, (__v8hi)__count);
2284	2217	}
2285	2218
...	...	@@ -2297,9 +2230,8 @@ _mm256_sra_epi16(__m256i __a, __m128i __count)
2297	2230	/// \param __count
2298	2231	/// An unsigned integer value specifying the shift count (in bits).
2299	2232	/// \returns A 256-bit vector of [8 x i32] containing the result.
2300		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2301		_mm256_srai_epi32(__m256i __a, int __count)
2302		{
	2233	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2234	_mm256_srai_epi32(__m256i __a, int __count) {
2303	2235	return (__m256i)__builtin_ia32_psradi256((__v8si)__a, __count);
2304	2236	}
2305	2237
...	...	@@ -2319,9 +2251,8 @@ _mm256_srai_epi32(__m256i __a, int __count)
2319	2251	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2320	2252	/// shift count (in bits). The upper element is ignored.
2321	2253	/// \returns A 256-bit vector of [8 x i32] containing the result.
2322		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2323		_mm256_sra_epi32(__m256i __a, __m128i __count)
2324		{
	2254	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2255	_mm256_sra_epi32(__m256i __a, __m128i __count) {
2325	2256	return (__m256i)__builtin_ia32_psrad256((__v8si)__a, (__v4si)__count);
2326	2257	}
2327	2258
...	...	@@ -2342,8 +2273,9 @@ _mm256_sra_epi32(__m256i __a, __m128i __count)
2342	2273	/// \param imm
2343	2274	/// An unsigned immediate value specifying the shift count (in bytes).
2344	2275	/// \returns A 256-bit integer vector containing the result.
2345		#define _mm256_srli_si256(a, imm) \
2346		((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
	2276	#define _mm256_srli_si256(a, imm) \
	2277	((__m256i)__builtin_ia32_psrldqi256_byteshift((__v32qi)(__m256i)(a), \
	2278	(int)(imm)))
2347	2279
2348	2280	/// Shifts each 128-bit half of the 256-bit integer vector in \a a right by
2349	2281	/// \a imm bytes, shifting in zero bytes, and returns the result. If
...	...	@@ -2362,8 +2294,9 @@ _mm256_sra_epi32(__m256i __a, __m128i __count)
2362	2294	/// \param imm
2363	2295	/// An unsigned immediate value specifying the shift count (in bytes).
2364	2296	/// \returns A 256-bit integer vector containing the result.
2365		#define _mm256_bsrli_epi128(a, imm) \
2366		((__m256i)__builtin_ia32_psrldqi256_byteshift((__m256i)(a), (int)(imm)))
	2297	#define _mm256_bsrli_epi128(a, imm) \
	2298	((__m256i)__builtin_ia32_psrldqi256_byteshift((__v32qi)(__m256i)(a), \
	2299	(int)(imm)))
2367	2300
2368	2301	/// Shifts each 16-bit element of the 256-bit vector of [16 x i16] in \a __a
2369	2302	/// right by \a __count bits, shifting in zero bits, and returns the result.
...	...	@@ -2378,9 +2311,8 @@ _mm256_sra_epi32(__m256i __a, __m128i __count)
2378	2311	/// \param __count
2379	2312	/// An unsigned integer value specifying the shift count (in bits).
2380	2313	/// \returns A 256-bit vector of [16 x i16] containing the result.
2381		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2382		_mm256_srli_epi16(__m256i __a, int __count)
2383		{
	2314	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2315	_mm256_srli_epi16(__m256i __a, int __count) {
2384	2316	return (__m256i)__builtin_ia32_psrlwi256((__v16hi)__a, __count);
2385	2317	}
2386	2318
...	...	@@ -2399,9 +2331,8 @@ _mm256_srli_epi16(__m256i __a, int __count)
2399	2331	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2400	2332	/// shift count (in bits). The upper element is ignored.
2401	2333	/// \returns A 256-bit vector of [16 x i16] containing the result.
2402		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2403		_mm256_srl_epi16(__m256i __a, __m128i __count)
2404		{
	2334	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2335	_mm256_srl_epi16(__m256i __a, __m128i __count) {
2405	2336	return (__m256i)__builtin_ia32_psrlw256((__v16hi)__a, (__v8hi)__count);
2406	2337	}
2407	2338
...	...	@@ -2418,9 +2349,8 @@ _mm256_srl_epi16(__m256i __a, __m128i __count)
2418	2349	/// \param __count
2419	2350	/// An unsigned integer value specifying the shift count (in bits).
2420	2351	/// \returns A 256-bit vector of [8 x i32] containing the result.
2421		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2422		_mm256_srli_epi32(__m256i __a, int __count)
2423		{
	2352	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2353	_mm256_srli_epi32(__m256i __a, int __count) {
2424	2354	return (__m256i)__builtin_ia32_psrldi256((__v8si)__a, __count);
2425	2355	}
2426	2356
...	...	@@ -2439,9 +2369,8 @@ _mm256_srli_epi32(__m256i __a, int __count)
2439	2369	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2440	2370	/// shift count (in bits). The upper element is ignored.
2441	2371	/// \returns A 256-bit vector of [8 x i32] containing the result.
2442		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2443		_mm256_srl_epi32(__m256i __a, __m128i __count)
2444		{
	2372	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2373	_mm256_srl_epi32(__m256i __a, __m128i __count) {
2445	2374	return (__m256i)__builtin_ia32_psrld256((__v8si)__a, (__v4si)__count);
2446	2375	}
2447	2376
...	...	@@ -2458,9 +2387,8 @@ _mm256_srl_epi32(__m256i __a, __m128i __count)
2458	2387	/// \param __count
2459	2388	/// An unsigned integer value specifying the shift count (in bits).
2460	2389	/// \returns A 256-bit vector of [4 x i64] containing the result.
2461		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2462		_mm256_srli_epi64(__m256i __a, int __count)
2463		{
	2390	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2391	_mm256_srli_epi64(__m256i __a, int __count) {
2464	2392	return __builtin_ia32_psrlqi256((__v4di)__a, __count);
2465	2393	}
2466	2394
...	...	@@ -2479,9 +2407,8 @@ _mm256_srli_epi64(__m256i __a, int __count)
2479	2407	/// A 128-bit vector of [2 x i64] whose lower element gives the unsigned
2480	2408	/// shift count (in bits). The upper element is ignored.
2481	2409	/// \returns A 256-bit vector of [4 x i64] containing the result.
2482		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2483		_mm256_srl_epi64(__m256i __a, __m128i __count)
2484		{
	2410	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2411	_mm256_srl_epi64(__m256i __a, __m128i __count) {
2485	2412	return __builtin_ia32_psrlq256((__v4di)__a, __count);
2486	2413	}
2487	2414
...	...	@@ -2506,9 +2433,8 @@ _mm256_srl_epi64(__m256i __a, __m128i __count)
2506	2433	/// \param __b
2507	2434	/// A 256-bit integer vector containing the subtrahends.
2508	2435	/// \returns A 256-bit integer vector containing the differences.
2509		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2510		_mm256_sub_epi8(__m256i __a, __m256i __b)
2511		{
	2436	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2437	_mm256_sub_epi8(__m256i __a, __m256i __b) {
2512	2438	return (__m256i)((__v32qu)__a - (__v32qu)__b);
2513	2439	}
2514	2440
...	...	@@ -2533,9 +2459,8 @@ _mm256_sub_epi8(__m256i __a, __m256i __b)
2533	2459	/// \param __b
2534	2460	/// A 256-bit vector of [16 x i16] containing the subtrahends.
2535	2461	/// \returns A 256-bit vector of [16 x i16] containing the differences.
2536		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2537		_mm256_sub_epi16(__m256i __a, __m256i __b)
2538		{
	2462	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2463	_mm256_sub_epi16(__m256i __a, __m256i __b) {
2539	2464	return (__m256i)((__v16hu)__a - (__v16hu)__b);
2540	2465	}
2541	2466
...	...	@@ -2559,9 +2484,8 @@ _mm256_sub_epi16(__m256i __a, __m256i __b)
2559	2484	/// \param __b
2560	2485	/// A 256-bit vector of [8 x i32] containing the subtrahends.
2561	2486	/// \returns A 256-bit vector of [8 x i32] containing the differences.
2562		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2563		_mm256_sub_epi32(__m256i __a, __m256i __b)
2564		{
	2487	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2488	_mm256_sub_epi32(__m256i __a, __m256i __b) {
2565	2489	return (__m256i)((__v8su)__a - (__v8su)__b);
2566	2490	}
2567	2491
...	...	@@ -2585,9 +2509,8 @@ _mm256_sub_epi32(__m256i __a, __m256i __b)
2585	2509	/// \param __b
2586	2510	/// A 256-bit vector of [4 x i64] containing the subtrahends.
2587	2511	/// \returns A 256-bit vector of [4 x i64] containing the differences.
2588		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2589		_mm256_sub_epi64(__m256i __a, __m256i __b)
2590		{
	2512	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2513	_mm256_sub_epi64(__m256i __a, __m256i __b) {
2591	2514	return (__m256i)((__v4du)__a - (__v4du)__b);
2592	2515	}
2593	2516
...	...	@@ -2611,9 +2534,8 @@ _mm256_sub_epi64(__m256i __a, __m256i __b)
2611	2534	/// \param __b
2612	2535	/// A 256-bit integer vector containing the subtrahends.
2613	2536	/// \returns A 256-bit integer vector containing the differences.
2614		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2615		_mm256_subs_epi8(__m256i __a, __m256i __b)
2616		{
	2537	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2538	_mm256_subs_epi8(__m256i __a, __m256i __b) {
2617	2539	return (__m256i)__builtin_elementwise_sub_sat((__v32qs)__a, (__v32qs)__b);
2618	2540	}
2619	2541
...	...	@@ -2637,9 +2559,8 @@ _mm256_subs_epi8(__m256i __a, __m256i __b)
2637	2559	/// \param __b
2638	2560	/// A 256-bit vector of [16 x i16] containing the subtrahends.
2639	2561	/// \returns A 256-bit vector of [16 x i16] containing the differences.
2640		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2641		_mm256_subs_epi16(__m256i __a, __m256i __b)
2642		{
	2562	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2563	_mm256_subs_epi16(__m256i __a, __m256i __b) {
2643	2564	return (__m256i)__builtin_elementwise_sub_sat((__v16hi)__a, (__v16hi)__b);
2644	2565	}
2645	2566
...	...	@@ -2664,9 +2585,8 @@ _mm256_subs_epi16(__m256i __a, __m256i __b)
2664	2585	/// \param __b
2665	2586	/// A 256-bit integer vector containing the subtrahends.
2666	2587	/// \returns A 256-bit integer vector containing the differences.
2667		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2668		_mm256_subs_epu8(__m256i __a, __m256i __b)
2669		{
	2588	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2589	_mm256_subs_epu8(__m256i __a, __m256i __b) {
2670	2590	return (__m256i)__builtin_elementwise_sub_sat((__v32qu)__a, (__v32qu)__b);
2671	2591	}
2672	2592
...	...	@@ -2690,9 +2610,8 @@ _mm256_subs_epu8(__m256i __a, __m256i __b)
2690	2610	/// \param __b
2691	2611	/// A 256-bit vector of [16 x i16] containing the subtrahends.
2692	2612	/// \returns A 256-bit vector of [16 x i16] containing the differences.
2693		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2694		_mm256_subs_epu16(__m256i __a, __m256i __b)
2695		{
	2613	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2614	_mm256_subs_epu16(__m256i __a, __m256i __b) {
2696	2615	return (__m256i)__builtin_elementwise_sub_sat((__v16hu)__a, (__v16hu)__b);
2697	2616	}
2698	2617
...	...	@@ -2724,9 +2643,8 @@ _mm256_subs_epu16(__m256i __a, __m256i __b)
2724	2643	/// A 256-bit integer vector used as the source for the odd-numbered bytes
2725	2644	/// of the result.
2726	2645	/// \returns A 256-bit integer vector containing the result.
2727		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2728		_mm256_unpackhi_epi8(__m256i __a, __m256i __b)
2729		{
	2646	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2647	_mm256_unpackhi_epi8(__m256i __a, __m256i __b) {
2730	2648	return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31);
2731	2649	}
2732	2650
...	...	@@ -2759,9 +2677,8 @@ _mm256_unpackhi_epi8(__m256i __a, __m256i __b)
2759	2677	/// A 256-bit vector of [16 x i16] used as the source for the odd-numbered
2760	2678	/// elements of the result.
2761	2679	/// \returns A 256-bit vector of [16 x i16] containing the result.
2762		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2763		_mm256_unpackhi_epi16(__m256i __a, __m256i __b)
2764		{
	2680	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2681	_mm256_unpackhi_epi16(__m256i __a, __m256i __b) {
2765	2682	return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
2766	2683	}
2767	2684
...	...	@@ -2793,9 +2710,8 @@ _mm256_unpackhi_epi16(__m256i __a, __m256i __b)
2793	2710	/// A 256-bit vector of [8 x i32] used as the source for the odd-numbered
2794	2711	/// elements of the result.
2795	2712	/// \returns A 256-bit vector of [8 x i32] containing the result.
2796		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2797		_mm256_unpackhi_epi32(__m256i __a, __m256i __b)
2798		{
	2713	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2714	_mm256_unpackhi_epi32(__m256i __a, __m256i __b) {
2799	2715	return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7);
2800	2716	}
2801	2717
...	...	@@ -2823,9 +2739,8 @@ _mm256_unpackhi_epi32(__m256i __a, __m256i __b)
2823	2739	/// A 256-bit vector of [4 x i64] used as the source for the odd-numbered
2824	2740	/// elements of the result.
2825	2741	/// \returns A 256-bit vector of [4 x i64] containing the result.
2826		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2827		_mm256_unpackhi_epi64(__m256i __a, __m256i __b)
2828		{
	2742	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2743	_mm256_unpackhi_epi64(__m256i __a, __m256i __b) {
2829	2744	return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3);
2830	2745	}
2831	2746
...	...	@@ -2857,9 +2772,8 @@ _mm256_unpackhi_epi64(__m256i __a, __m256i __b)
2857	2772	/// A 256-bit integer vector used as the source for the odd-numbered bytes
2858	2773	/// of the result.
2859	2774	/// \returns A 256-bit integer vector containing the result.
2860		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2861		_mm256_unpacklo_epi8(__m256i __a, __m256i __b)
2862		{
	2775	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2776	_mm256_unpacklo_epi8(__m256i __a, __m256i __b) {
2863	2777	return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23);
2864	2778	}
2865	2779
...	...	@@ -2892,9 +2806,8 @@ _mm256_unpacklo_epi8(__m256i __a, __m256i __b)
2892	2806	/// A 256-bit vector of [16 x i16] used as the source for the odd-numbered
2893	2807	/// elements of the result.
2894	2808	/// \returns A 256-bit vector of [16 x i16] containing the result.
2895		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2896		_mm256_unpacklo_epi16(__m256i __a, __m256i __b)
2897		{
	2809	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2810	_mm256_unpacklo_epi16(__m256i __a, __m256i __b) {
2898	2811	return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11);
2899	2812	}
2900	2813
...	...	@@ -2926,9 +2839,8 @@ _mm256_unpacklo_epi16(__m256i __a, __m256i __b)
2926	2839	/// A 256-bit vector of [8 x i32] used as the source for the odd-numbered
2927	2840	/// elements of the result.
2928	2841	/// \returns A 256-bit vector of [8 x i32] containing the result.
2929		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2930		_mm256_unpacklo_epi32(__m256i __a, __m256i __b)
2931		{
	2842	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2843	_mm256_unpacklo_epi32(__m256i __a, __m256i __b) {
2932	2844	return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5);
2933	2845	}
2934	2846
...	...	@@ -2956,9 +2868,8 @@ _mm256_unpacklo_epi32(__m256i __a, __m256i __b)
2956	2868	/// A 256-bit vector of [4 x i64] used as the source for the odd-numbered
2957	2869	/// elements of the result.
2958	2870	/// \returns A 256-bit vector of [4 x i64] containing the result.
2959		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2960		_mm256_unpacklo_epi64(__m256i __a, __m256i __b)
2961		{
	2871	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2872	_mm256_unpacklo_epi64(__m256i __a, __m256i __b) {
2962	2873	return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2);
2963	2874	}
2964	2875
...	...	@@ -2974,7 +2885,7 @@ _mm256_unpacklo_epi64(__m256i __a, __m256i __b)
2974	2885	/// \param __b
2975	2886	/// A 256-bit integer vector.
2976	2887	/// \returns A 256-bit integer vector containing the result.
2977		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2888	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2978	2889	_mm256_xor_si256(__m256i __a, __m256i __b)
2979	2890	{
2980	2891	return (__m256i)((__v4du)__a ^ (__v4du)__b);
...	...	@@ -3009,9 +2920,8 @@ _mm256_stream_load_si256(const void *__V)
3009	2920	/// \param __X
3010	2921	/// A 128-bit vector of [4 x float] whose low element will be broadcast.
3011	2922	/// \returns A 128-bit vector of [4 x float] containing the result.
3012		static __inline__ __m128 __DEFAULT_FN_ATTRS128
3013		_mm_broadcastss_ps(__m128 __X)
3014		{
	2923	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2924	_mm_broadcastss_ps(__m128 __X) {
3015	2925	return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0);
3016	2926	}
3017	2927
...	...	@@ -3026,9 +2936,8 @@ _mm_broadcastss_ps(__m128 __X)
3026	2936	/// \param __a
3027	2937	/// A 128-bit vector of [2 x double] whose low element will be broadcast.
3028	2938	/// \returns A 128-bit vector of [2 x double] containing the result.
3029		static __inline__ __m128d __DEFAULT_FN_ATTRS128
3030		_mm_broadcastsd_pd(__m128d __a)
3031		{
	2939	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2940	_mm_broadcastsd_pd(__m128d __a) {
3032	2941	return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
3033	2942	}
3034	2943
...	...	@@ -3043,9 +2952,8 @@ _mm_broadcastsd_pd(__m128d __a)
3043	2952	/// \param __X
3044	2953	/// A 128-bit vector of [4 x float] whose low element will be broadcast.
3045	2954	/// \returns A 256-bit vector of [8 x float] containing the result.
3046		static __inline__ __m256 __DEFAULT_FN_ATTRS256
3047		_mm256_broadcastss_ps(__m128 __X)
3048		{
	2955	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2956	_mm256_broadcastss_ps(__m128 __X) {
3049	2957	return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0);
3050	2958	}
3051	2959
...	...	@@ -3060,9 +2968,8 @@ _mm256_broadcastss_ps(__m128 __X)
3060	2968	/// \param __X
3061	2969	/// A 128-bit vector of [2 x double] whose low element will be broadcast.
3062	2970	/// \returns A 256-bit vector of [4 x double] containing the result.
3063		static __inline__ __m256d __DEFAULT_FN_ATTRS256
3064		_mm256_broadcastsd_pd(__m128d __X)
3065		{
	2971	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	2972	_mm256_broadcastsd_pd(__m128d __X) {
3066	2973	return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0);
3067	2974	}
3068	2975
...	...	@@ -3076,9 +2983,8 @@ _mm256_broadcastsd_pd(__m128d __X)
3076	2983	/// \param __X
3077	2984	/// A 128-bit integer vector to be broadcast.
3078	2985	/// \returns A 256-bit integer vector containing the result.
3079		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3080		_mm256_broadcastsi128_si256(__m128i __X)
3081		{
	2986	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2987	_mm256_broadcastsi128_si256(__m128i __X) {
3082	2988	return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 1, 0, 1);
3083	2989	}
3084	2990
...	...	@@ -3168,9 +3074,8 @@ _mm256_broadcastsi128_si256(__m128i __X)
3168	3074	/// \param __X
3169	3075	/// A 128-bit integer vector whose low byte will be broadcast.
3170	3076	/// \returns A 256-bit integer vector containing the result.
3171		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3172		_mm256_broadcastb_epi8(__m128i __X)
3173		{
	3077	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3078	_mm256_broadcastb_epi8(__m128i __X) {
3174	3079	return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
3175	3080	}
3176	3081
...	...	@@ -3184,9 +3089,8 @@ _mm256_broadcastb_epi8(__m128i __X)
3184	3089	/// \param __X
3185	3090	/// A 128-bit vector of [8 x i16] whose low element will be broadcast.
3186	3091	/// \returns A 256-bit vector of [16 x i16] containing the result.
3187		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3188		_mm256_broadcastw_epi16(__m128i __X)
3189		{
	3092	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3093	_mm256_broadcastw_epi16(__m128i __X) {
3190	3094	return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
3191	3095	}
3192	3096
...	...	@@ -3200,9 +3104,8 @@ _mm256_broadcastw_epi16(__m128i __X)
3200	3104	/// \param __X
3201	3105	/// A 128-bit vector of [4 x i32] whose low element will be broadcast.
3202	3106	/// \returns A 256-bit vector of [8 x i32] containing the result.
3203		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3204		_mm256_broadcastd_epi32(__m128i __X)
3205		{
	3107	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3108	_mm256_broadcastd_epi32(__m128i __X) {
3206	3109	return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0);
3207	3110	}
3208	3111
...	...	@@ -3216,9 +3119,8 @@ _mm256_broadcastd_epi32(__m128i __X)
3216	3119	/// \param __X
3217	3120	/// A 128-bit vector of [2 x i64] whose low element will be broadcast.
3218	3121	/// \returns A 256-bit vector of [4 x i64] containing the result.
3219		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3220		_mm256_broadcastq_epi64(__m128i __X)
3221		{
	3122	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3123	_mm256_broadcastq_epi64(__m128i __X) {
3222	3124	return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0);
3223	3125	}
3224	3126
...	...	@@ -3232,9 +3134,8 @@ _mm256_broadcastq_epi64(__m128i __X)
3232	3134	/// \param __X
3233	3135	/// A 128-bit integer vector whose low byte will be broadcast.
3234	3136	/// \returns A 128-bit integer vector containing the result.
3235		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3236		_mm_broadcastb_epi8(__m128i __X)
3237		{
	3137	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3138	_mm_broadcastb_epi8(__m128i __X) {
3238	3139	return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
3239	3140	}
3240	3141
...	...	@@ -3248,9 +3149,8 @@ _mm_broadcastb_epi8(__m128i __X)
3248	3149	/// \param __X
3249	3150	/// A 128-bit vector of [8 x i16] whose low element will be broadcast.
3250	3151	/// \returns A 128-bit vector of [8 x i16] containing the result.
3251		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3252		_mm_broadcastw_epi16(__m128i __X)
3253		{
	3152	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3153	_mm_broadcastw_epi16(__m128i __X) {
3254	3154	return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0);
3255	3155	}
3256	3156
...	...	@@ -3264,9 +3164,8 @@ _mm_broadcastw_epi16(__m128i __X)
3264	3164	/// \param __X
3265	3165	/// A 128-bit vector of [4 x i32] whose low element will be broadcast.
3266	3166	/// \returns A 128-bit vector of [4 x i32] containing the result.
3267		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3268		_mm_broadcastd_epi32(__m128i __X)
3269		{
	3167	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3168	_mm_broadcastd_epi32(__m128i __X) {
3270	3169	return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0);
3271	3170	}
3272	3171
...	...	@@ -3280,9 +3179,8 @@ _mm_broadcastd_epi32(__m128i __X)
3280	3179	/// \param __X
3281	3180	/// A 128-bit vector of [2 x i64] whose low element will be broadcast.
3282	3181	/// \returns A 128-bit vector of [2 x i64] containing the result.
3283		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3284		_mm_broadcastq_epi64(__m128i __X)
3285		{
	3182	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3183	_mm_broadcastq_epi64(__m128i __X) {
3286	3184	return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0);
3287	3185	}
3288	3186
...	...	@@ -3308,9 +3206,8 @@ _mm_broadcastq_epi64(__m128i __X)
3308	3206	/// A 256-bit vector of [8 x i32] containing indexes of values to use from
3309	3207	/// \a __a.
3310	3208	/// \returns A 256-bit vector of [8 x i32] containing the result.
3311		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3312		_mm256_permutevar8x32_epi32(__m256i __a, __m256i __b)
3313		{
	3209	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3210	_mm256_permutevar8x32_epi32(__m256i __a, __m256i __b) {
3314	3211	return (__m256i)__builtin_ia32_permvarsi256((__v8si)__a, (__v8si)__b);
3315	3212	}
3316	3213
...	...	@@ -3366,9 +3263,8 @@ _mm256_permutevar8x32_epi32(__m256i __a, __m256i __b)
3366	3263	/// A 256-bit vector of [8 x i32] containing indexes of values to use from
3367	3264	/// \a __a.
3368	3265	/// \returns A 256-bit vector of [8 x float] containing the result.
3369		static __inline__ __m256 __DEFAULT_FN_ATTRS256
3370		_mm256_permutevar8x32_ps(__m256 __a, __m256i __b)
3371		{
	3266	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	3267	_mm256_permutevar8x32_ps(__m256 __a, __m256i __b) {
3372	3268	return (__m256)__builtin_ia32_permvarsf256((__v8sf)__a, (__v8si)__b);
3373	3269	}
3374	3270
...	...	@@ -3756,7 +3652,7 @@ _mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y)
3756	3652	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
3757	3653	/// bits).
3758	3654	/// \returns A 256-bit vector of [8 x i32] containing the result.
3759		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3655	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3760	3656	_mm256_sllv_epi32(__m256i __X, __m256i __Y)
3761	3657	{
3762	3658	return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y);
...	...	@@ -3778,7 +3674,7 @@ _mm256_sllv_epi32(__m256i __X, __m256i __Y)
3778	3674	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
3779	3675	/// bits).
3780	3676	/// \returns A 128-bit vector of [4 x i32] containing the result.
3781		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3677	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3782	3678	_mm_sllv_epi32(__m128i __X, __m128i __Y)
3783	3679	{
3784	3680	return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y);
...	...	@@ -3800,7 +3696,7 @@ _mm_sllv_epi32(__m128i __X, __m128i __Y)
3800	3696	/// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
3801	3697	/// bits).
3802	3698	/// \returns A 256-bit vector of [4 x i64] containing the result.
3803		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3699	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3804	3700	_mm256_sllv_epi64(__m256i __X, __m256i __Y)
3805	3701	{
3806	3702	return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y);
...	...	@@ -3822,7 +3718,7 @@ _mm256_sllv_epi64(__m256i __X, __m256i __Y)
3822	3718	/// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
3823	3719	/// bits).
3824	3720	/// \returns A 128-bit vector of [2 x i64] containing the result.
3825		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3721	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3826	3722	_mm_sllv_epi64(__m128i __X, __m128i __Y)
3827	3723	{
3828	3724	return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y);
...	...	@@ -3845,7 +3741,7 @@ _mm_sllv_epi64(__m128i __X, __m128i __Y)
3845	3741	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
3846	3742	/// bits).
3847	3743	/// \returns A 256-bit vector of [8 x i32] containing the result.
3848		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3744	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3849	3745	_mm256_srav_epi32(__m256i __X, __m256i __Y)
3850	3746	{
3851	3747	return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y);
...	...	@@ -3868,7 +3764,7 @@ _mm256_srav_epi32(__m256i __X, __m256i __Y)
3868	3764	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
3869	3765	/// bits).
3870	3766	/// \returns A 128-bit vector of [4 x i32] containing the result.
3871		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3767	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3872	3768	_mm_srav_epi32(__m128i __X, __m128i __Y)
3873	3769	{
3874	3770	return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y);
...	...	@@ -3890,7 +3786,7 @@ _mm_srav_epi32(__m128i __X, __m128i __Y)
3890	3786	/// A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
3891	3787	/// bits).
3892	3788	/// \returns A 256-bit vector of [8 x i32] containing the result.
3893		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3789	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3894	3790	_mm256_srlv_epi32(__m256i __X, __m256i __Y)
3895	3791	{
3896	3792	return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y);
...	...	@@ -3912,7 +3808,7 @@ _mm256_srlv_epi32(__m256i __X, __m256i __Y)
3912	3808	/// A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
3913	3809	/// bits).
3914	3810	/// \returns A 128-bit vector of [4 x i32] containing the result.
3915		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3811	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3916	3812	_mm_srlv_epi32(__m128i __X, __m128i __Y)
3917	3813	{
3918	3814	return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y);
...	...	@@ -3934,7 +3830,7 @@ _mm_srlv_epi32(__m128i __X, __m128i __Y)
3934	3830	/// A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
3935	3831	/// bits).
3936	3832	/// \returns A 256-bit vector of [4 x i64] containing the result.
3937		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3833	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3938	3834	_mm256_srlv_epi64(__m256i __X, __m256i __Y)
3939	3835	{
3940	3836	return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y);
...	...	@@ -3956,7 +3852,7 @@ _mm256_srlv_epi64(__m256i __X, __m256i __Y)
3956	3852	/// A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
3957	3853	/// bits).
3958	3854	/// \returns A 128-bit vector of [2 x i64] containing the result.
3959		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3855	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3960	3856	_mm_srlv_epi64(__m128i __X, __m128i __Y)
3961	3857	{
3962	3858	return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y);
...	...	@@ -5289,5 +5185,7 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
5289	5185
5290	5186	#undef __DEFAULT_FN_ATTRS256
5291	5187	#undef __DEFAULT_FN_ATTRS128
	5188	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
	5189	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
5292	5190
5293	5191	#endif /* __AVX2INTRIN_H */

lib/include/avx512bf16intrin.h+25-16

...	...	@@ -19,12 +19,19 @@ typedef __bf16 __v32bf __attribute__((__vector_size__(64), __aligned__(64)));
19	19	typedef __bf16 __m512bh __attribute__((__vector_size__(64), __aligned__(64)));
20	20	typedef __bf16 __bfloat16 __attribute__((deprecated("use __bf16 instead")));
21	21
22		#define __DEFAULT_FN_ATTRS512 \
23		__attribute__((__always_inline__, __nodebug__, __target__("avx512bf16,evex512"), \
	22	#define __DEFAULT_FN_ATTRS512 \
	23	__attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \
24	24	__min_vector_width__(512)))
25	25	#define __DEFAULT_FN_ATTRS \
26		__attribute__((__always_inline__, __nodebug__, \
27		__target__("avx512bf16,no-evex512")))
	26	__attribute__((__always_inline__, __nodebug__, __target__("avx512bf16")))
	27
	28	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	29	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
	30	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	31	#else
	32	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
	33	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	34	#endif
28	35
29	36	/// Convert One BF16 Data to One Single Float Data.
30	37	///
...	...	@@ -36,8 +43,8 @@ typedef __bf16 __bfloat16 __attribute__((deprecated("use __bf16 instead")));
36	43	/// A bfloat data.
37	44	/// \returns A float data whose sign field and exponent field keep unchanged,
38	45	/// and fraction field is extended to 23 bits.
39		static __inline__ float __DEFAULT_FN_ATTRS _mm_cvtsbh_ss(__bf16 __A) {
40		return __builtin_ia32_cvtsbf162ss_32(__A);
	46	static __inline__ float __DEFAULT_FN_ATTRS_CONSTEXPR _mm_cvtsbh_ss(__bf16 __A) {
	47	return (float)(__A);
41	48	}
42	49
43	50	/// Convert Two Packed Single Data to One Packed BF16 Data.
...	...	@@ -236,9 +243,9 @@ _mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) {
236	243	/// \param __A
237	244	/// A 256-bit vector of [16 x bfloat].
238	245	/// \returns A 512-bit vector of [16 x float] come from conversion of __A
239		static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
240		return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
241		(__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
	246	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	247	_mm512_cvtpbh_ps(__m256bh __A) {
	248	return (__m512) __builtin_convertvector(__A, __v16sf);
242	249	}
243	250
244	251	/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
...	...	@@ -251,10 +258,11 @@ static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_cvtpbh_ps(__m256bh __A) {
251	258	/// \param __A
252	259	/// A 256-bit vector of [16 x bfloat].
253	260	/// \returns A 512-bit vector of [16 x float] come from conversion of __A
254		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	261	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
255	262	_mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
256		return _mm512_castsi512_ps((__m512i)_mm512_slli_epi32(
257		(__m512i)_mm512_maskz_cvtepi16_epi32((__mmask16)__U, (__m256i)__A), 16));
	263	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
	264	(__v16sf)_mm512_cvtpbh_ps(__A),
	265	(__v16sf)_mm512_setzero_ps());
258	266	}
259	267
260	268	/// Convert Packed BF16 Data to Packed float Data using merging mask.
...	...	@@ -269,15 +277,16 @@ _mm512_maskz_cvtpbh_ps(__mmask16 __U, __m256bh __A) {
269	277	/// \param __A
270	278	/// A 256-bit vector of [16 x bfloat].
271	279	/// \returns A 512-bit vector of [16 x float] come from conversion of __A
272		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	280	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
273	281	_mm512_mask_cvtpbh_ps(__m512 __S, __mmask16 __U, __m256bh __A) {
274		return _mm512_castsi512_ps((__m512i)_mm512_mask_slli_epi32(
275		(__m512i)__S, (__mmask16)__U,
276		(__m512i)_mm512_cvtepi16_epi32((__m256i)__A), 16));
	282	return (__m512)__builtin_ia32_selectps_512(
	283	(__mmask16)__U, (__v16sf)_mm512_cvtpbh_ps(__A), (__v16sf)__S);
277	284	}
278	285
279	286	#undef __DEFAULT_FN_ATTRS
	287	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
280	288	#undef __DEFAULT_FN_ATTRS512
	289	#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
281	290
282	291	#endif
283	292	#endif

lib/include/avx512bitalgintrin.h+19-30

...	...	@@ -15,53 +15,44 @@
15	15	#define __AVX512BITALGINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512bitalg,evex512"), \
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx512bitalg"), \
	21	__min_vector_width__(512))) constexpr
	22	#else
	23	#define __DEFAULT_FN_ATTRS \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx512bitalg"), \
21	25	__min_vector_width__(512)))
	26	#endif
22	27
23		static __inline__ __m512i __DEFAULT_FN_ATTRS
24		_mm512_popcnt_epi16(__m512i __A)
25		{
	28	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi16(__m512i __A) {
26	29	return (__m512i)__builtin_elementwise_popcount((__v32hu)__A);
27	30	}
28	31
29	32	static __inline__ __m512i __DEFAULT_FN_ATTRS
30		_mm512_mask_popcnt_epi16(__m512i __A, __mmask32 __U, __m512i __B)
31		{
32		return (__m512i) __builtin_ia32_selectw_512((__mmask32) __U,
33		(__v32hi) _mm512_popcnt_epi16(__B),
34		(__v32hi) __A);
	33	_mm512_mask_popcnt_epi16(__m512i __A, __mmask32 __U, __m512i __B) {
	34	return (__m512i)__builtin_ia32_selectw_512(
	35	(__mmask32)__U, (__v32hi)_mm512_popcnt_epi16(__B), (__v32hi)__A);
35	36	}
36	37
37	38	static __inline__ __m512i __DEFAULT_FN_ATTRS
38		_mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B)
39		{
40		return _mm512_mask_popcnt_epi16((__m512i) _mm512_setzero_si512(),
41		__U,
42		__B);
	39	_mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B) {
	40	return _mm512_mask_popcnt_epi16((__m512i)_mm512_setzero_si512(), __U, __B);
43	41	}
44	42
45		static __inline__ __m512i __DEFAULT_FN_ATTRS
46		_mm512_popcnt_epi8(__m512i __A)
47		{
	43	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi8(__m512i __A) {
48	44	return (__m512i)__builtin_elementwise_popcount((__v64qu)__A);
49	45	}
50	46
51	47	static __inline__ __m512i __DEFAULT_FN_ATTRS
52		_mm512_mask_popcnt_epi8(__m512i __A, __mmask64 __U, __m512i __B)
53		{
54		return (__m512i) __builtin_ia32_selectb_512((__mmask64) __U,
55		(__v64qi) _mm512_popcnt_epi8(__B),
56		(__v64qi) __A);
	48	_mm512_mask_popcnt_epi8(__m512i __A, __mmask64 __U, __m512i __B) {
	49	return (__m512i)__builtin_ia32_selectb_512(
	50	(__mmask64)__U, (__v64qi)_mm512_popcnt_epi8(__B), (__v64qi)__A);
57	51	}
58	52
59	53	static __inline__ __m512i __DEFAULT_FN_ATTRS
60		_mm512_maskz_popcnt_epi8(__mmask64 __U, __m512i __B)
61		{
62		return _mm512_mask_popcnt_epi8((__m512i) _mm512_setzero_si512(),
63		__U,
64		__B);
	54	_mm512_maskz_popcnt_epi8(__mmask64 __U, __m512i __B) {
	55	return _mm512_mask_popcnt_epi8((__m512i)_mm512_setzero_si512(), __U, __B);
65	56	}
66	57
67	58	static __inline__ __mmask64 __DEFAULT_FN_ATTRS
...	...	@@ -80,7 +71,5 @@ _mm512_bitshuffle_epi64_mask(__m512i __A, __m512i __B)
80	71	__B);
81	72	}
82	73
83
84	74	#undef __DEFAULT_FN_ATTRS
85
86	75	#endif

lib/include/avx512bwintrin.h+386-505

...	...	@@ -19,153 +19,150 @@ typedef unsigned long long __mmask64;
19	19
20	20	/* Define the default attributes for the functions in this file. */
21	21	#define __DEFAULT_FN_ATTRS512 \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("avx512bw,evex512"), __min_vector_width__(512)))
	22	__attribute__((__always_inline__, __nodebug__, __target__("avx512bw"), \
	23	__min_vector_width__(512)))
24	24	#define __DEFAULT_FN_ATTRS \
25		__attribute__((__always_inline__, __nodebug__, \
26		__target__("avx512bw,no-evex512")))
	25	__attribute__((__always_inline__, __nodebug__, __target__("avx512bw")))
	26
	27	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	28	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
	29	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	30	#else
	31	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
	32	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	33	#endif
27	34
28		static __inline __mmask32 __DEFAULT_FN_ATTRS
29		_knot_mask32(__mmask32 __M)
30		{
	35	static __inline __mmask32
	36	__DEFAULT_FN_ATTRS_CONSTEXPR _knot_mask32(__mmask32 __M) {
31	37	return __builtin_ia32_knotsi(__M);
32	38	}
33	39
34		static __inline __mmask64 __DEFAULT_FN_ATTRS _knot_mask64(__mmask64 __M) {
	40	static __inline __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	41	_knot_mask64(__mmask64 __M) {
35	42	return __builtin_ia32_knotdi(__M);
36	43	}
37	44
38		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
39		_kand_mask32(__mmask32 __A, __mmask32 __B)
40		{
	45	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	46	_kand_mask32(__mmask32 __A, __mmask32 __B) {
41	47	return (__mmask32)__builtin_ia32_kandsi((__mmask32)__A, (__mmask32)__B);
42	48	}
43	49
44		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kand_mask64(__mmask64 __A,
45		__mmask64 __B) {
	50	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	51	_kand_mask64(__mmask64 __A, __mmask64 __B) {
46	52	return (__mmask64)__builtin_ia32_kanddi((__mmask64)__A, (__mmask64)__B);
47	53	}
48	54
49		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
50		_kandn_mask32(__mmask32 __A, __mmask32 __B)
51		{
	55	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	56	_kandn_mask32(__mmask32 __A, __mmask32 __B) {
52	57	return (__mmask32)__builtin_ia32_kandnsi((__mmask32)__A, (__mmask32)__B);
53	58	}
54	59
55		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kandn_mask64(__mmask64 __A,
56		__mmask64 __B) {
	60	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	61	_kandn_mask64(__mmask64 __A, __mmask64 __B) {
57	62	return (__mmask64)__builtin_ia32_kandndi((__mmask64)__A, (__mmask64)__B);
58	63	}
59	64
60		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
61		_kor_mask32(__mmask32 __A, __mmask32 __B)
62		{
	65	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	66	_kor_mask32(__mmask32 __A, __mmask32 __B) {
63	67	return (__mmask32)__builtin_ia32_korsi((__mmask32)__A, (__mmask32)__B);
64	68	}
65	69
66		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kor_mask64(__mmask64 __A,
67		__mmask64 __B) {
	70	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	71	_kor_mask64(__mmask64 __A, __mmask64 __B) {
68	72	return (__mmask64)__builtin_ia32_kordi((__mmask64)__A, (__mmask64)__B);
69	73	}
70	74
71		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
72		_kxnor_mask32(__mmask32 __A, __mmask32 __B)
73		{
	75	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	76	_kxnor_mask32(__mmask32 __A, __mmask32 __B) {
74	77	return (__mmask32)__builtin_ia32_kxnorsi((__mmask32)__A, (__mmask32)__B);
75	78	}
76	79
77		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kxnor_mask64(__mmask64 __A,
78		__mmask64 __B) {
	80	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	81	_kxnor_mask64(__mmask64 __A, __mmask64 __B) {
79	82	return (__mmask64)__builtin_ia32_kxnordi((__mmask64)__A, (__mmask64)__B);
80	83	}
81	84
82		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
83		_kxor_mask32(__mmask32 __A, __mmask32 __B)
84		{
	85	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	86	_kxor_mask32(__mmask32 __A, __mmask32 __B) {
85	87	return (__mmask32)__builtin_ia32_kxorsi((__mmask32)__A, (__mmask32)__B);
86	88	}
87	89
88		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kxor_mask64(__mmask64 __A,
89		__mmask64 __B) {
	90	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	91	_kxor_mask64(__mmask64 __A, __mmask64 __B) {
90	92	return (__mmask64)__builtin_ia32_kxordi((__mmask64)__A, (__mmask64)__B);
91	93	}
92	94
93		static __inline__ unsigned char __DEFAULT_FN_ATTRS
94		_kortestc_mask32_u8(__mmask32 __A, __mmask32 __B)
95		{
	95	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	96	_kortestc_mask32_u8(__mmask32 __A, __mmask32 __B) {
96	97	return (unsigned char)__builtin_ia32_kortestcsi(__A, __B);
97	98	}
98	99
99		static __inline__ unsigned char __DEFAULT_FN_ATTRS
100		_kortestz_mask32_u8(__mmask32 __A, __mmask32 __B)
101		{
	100	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	101	_kortestz_mask32_u8(__mmask32 __A, __mmask32 __B) {
102	102	return (unsigned char)__builtin_ia32_kortestzsi(__A, __B);
103	103	}
104	104
105		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	105	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
106	106	_kortest_mask32_u8(__mmask32 __A, __mmask32 __B, unsigned char *__C) {
107	107	*__C = (unsigned char)__builtin_ia32_kortestcsi(__A, __B);
108	108	return (unsigned char)__builtin_ia32_kortestzsi(__A, __B);
109	109	}
110	110
111		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	111	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
112	112	_kortestc_mask64_u8(__mmask64 __A, __mmask64 __B) {
113	113	return (unsigned char)__builtin_ia32_kortestcdi(__A, __B);
114	114	}
115	115
116		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	116	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
117	117	_kortestz_mask64_u8(__mmask64 __A, __mmask64 __B) {
118	118	return (unsigned char)__builtin_ia32_kortestzdi(__A, __B);
119	119	}
120	120
121		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	121	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
122	122	_kortest_mask64_u8(__mmask64 __A, __mmask64 __B, unsigned char *__C) {
123	123	*__C = (unsigned char)__builtin_ia32_kortestcdi(__A, __B);
124	124	return (unsigned char)__builtin_ia32_kortestzdi(__A, __B);
125	125	}
126	126
127		static __inline__ unsigned char __DEFAULT_FN_ATTRS
128		_ktestc_mask32_u8(__mmask32 __A, __mmask32 __B)
129		{
	127	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	128	_ktestc_mask32_u8(__mmask32 __A, __mmask32 __B) {
130	129	return (unsigned char)__builtin_ia32_ktestcsi(__A, __B);
131	130	}
132	131
133		static __inline__ unsigned char __DEFAULT_FN_ATTRS
134		_ktestz_mask32_u8(__mmask32 __A, __mmask32 __B)
135		{
	132	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	133	_ktestz_mask32_u8(__mmask32 __A, __mmask32 __B) {
136	134	return (unsigned char)__builtin_ia32_ktestzsi(__A, __B);
137	135	}
138	136
139		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	137	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
140	138	_ktest_mask32_u8(__mmask32 __A, __mmask32 __B, unsigned char *__C) {
141	139	*__C = (unsigned char)__builtin_ia32_ktestcsi(__A, __B);
142	140	return (unsigned char)__builtin_ia32_ktestzsi(__A, __B);
143	141	}
144	142
145		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	143	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
146	144	_ktestc_mask64_u8(__mmask64 __A, __mmask64 __B) {
147	145	return (unsigned char)__builtin_ia32_ktestcdi(__A, __B);
148	146	}
149	147
150		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	148	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
151	149	_ktestz_mask64_u8(__mmask64 __A, __mmask64 __B) {
152	150	return (unsigned char)__builtin_ia32_ktestzdi(__A, __B);
153	151	}
154	152
155		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	153	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
156	154	_ktest_mask64_u8(__mmask64 __A, __mmask64 __B, unsigned char *__C) {
157	155	*__C = (unsigned char)__builtin_ia32_ktestcdi(__A, __B);
158	156	return (unsigned char)__builtin_ia32_ktestzdi(__A, __B);
159	157	}
160	158
161		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
162		_kadd_mask32(__mmask32 __A, __mmask32 __B)
163		{
	159	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	160	_kadd_mask32(__mmask32 __A, __mmask32 __B) {
164	161	return (__mmask32)__builtin_ia32_kaddsi((__mmask32)__A, (__mmask32)__B);
165	162	}
166	163
167		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kadd_mask64(__mmask64 __A,
168		__mmask64 __B) {
	164	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	165	_kadd_mask64(__mmask64 __A, __mmask64 __B) {
169	166	return (__mmask64)__builtin_ia32_kadddi((__mmask64)__A, (__mmask64)__B);
170	167	}
171	168
...	...	@@ -181,22 +178,22 @@ static __inline__ __mmask64 __DEFAULT_FN_ATTRS _kadd_mask64(__mmask64 __A,
181	178	#define _kshiftri_mask64(A, I) \
182	179	((__mmask64)__builtin_ia32_kshiftridi((__mmask64)(A), (unsigned int)(I)))
183	180
184		static __inline__ unsigned int __DEFAULT_FN_ATTRS
185		_cvtmask32_u32(__mmask32 __A) {
	181	static __inline__ unsigned int
	182	__DEFAULT_FN_ATTRS_CONSTEXPR _cvtmask32_u32(__mmask32 __A) {
186	183	return (unsigned int)__builtin_ia32_kmovd((__mmask32)__A);
187	184	}
188	185
189		static __inline__ unsigned long long __DEFAULT_FN_ATTRS
	186	static __inline__ unsigned long long __DEFAULT_FN_ATTRS_CONSTEXPR
190	187	_cvtmask64_u64(__mmask64 __A) {
191	188	return (unsigned long long)__builtin_ia32_kmovq((__mmask64)__A);
192	189	}
193	190
194		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
	191	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
195	192	_cvtu32_mask32(unsigned int __A) {
196	193	return (__mmask32)__builtin_ia32_kmovd((__mmask32)__A);
197	194	}
198	195
199		static __inline__ __mmask64 __DEFAULT_FN_ATTRS
	196	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
200	197	_cvtu64_mask64(unsigned long long __A) {
201	198	return (__mmask64)__builtin_ia32_kmovq((__mmask64)__A);
202	199	}
...	...	@@ -362,168 +359,159 @@ static __inline__ void __DEFAULT_FN_ATTRS _store_mask64(__mmask64 *__A,
362	359	#define _mm512_mask_cmpneq_epu16_mask(k, A, B) \
363	360	_mm512_mask_cmp_epu16_mask((k), (A), (B), _MM_CMPINT_NE)
364	361
365		static __inline__ __m512i __DEFAULT_FN_ATTRS512
366		_mm512_add_epi8 (__m512i __A, __m512i __B) {
	362	static __inline__ __m512i
	363	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_add_epi8(__m512i __A, __m512i __B) {
367	364	return (__m512i) ((__v64qu) __A + (__v64qu) __B);
368	365	}
369	366
370		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	367	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
371	368	_mm512_mask_add_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
372	369	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
373	370	(__v64qi)_mm512_add_epi8(__A, __B),
374	371	(__v64qi)__W);
375	372	}
376	373
377		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	374	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
378	375	_mm512_maskz_add_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
379	376	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
380	377	(__v64qi)_mm512_add_epi8(__A, __B),
381	378	(__v64qi)_mm512_setzero_si512());
382	379	}
383	380
384		static __inline__ __m512i __DEFAULT_FN_ATTRS512
385		_mm512_sub_epi8 (__m512i __A, __m512i __B) {
	381	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	382	_mm512_sub_epi8(__m512i __A, __m512i __B) {
386	383	return (__m512i) ((__v64qu) __A - (__v64qu) __B);
387	384	}
388	385
389		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	386	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
390	387	_mm512_mask_sub_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
391	388	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
392	389	(__v64qi)_mm512_sub_epi8(__A, __B),
393	390	(__v64qi)__W);
394	391	}
395	392
396		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	393	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
397	394	_mm512_maskz_sub_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
398	395	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
399	396	(__v64qi)_mm512_sub_epi8(__A, __B),
400	397	(__v64qi)_mm512_setzero_si512());
401	398	}
402	399
403		static __inline__ __m512i __DEFAULT_FN_ATTRS512
404		_mm512_add_epi16 (__m512i __A, __m512i __B) {
	400	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	401	_mm512_add_epi16(__m512i __A, __m512i __B) {
405	402	return (__m512i) ((__v32hu) __A + (__v32hu) __B);
406	403	}
407	404
408		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	405	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
409	406	_mm512_mask_add_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
410	407	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
411	408	(__v32hi)_mm512_add_epi16(__A, __B),
412	409	(__v32hi)__W);
413	410	}
414	411
415		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	412	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
416	413	_mm512_maskz_add_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
417	414	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
418	415	(__v32hi)_mm512_add_epi16(__A, __B),
419	416	(__v32hi)_mm512_setzero_si512());
420	417	}
421	418
422		static __inline__ __m512i __DEFAULT_FN_ATTRS512
423		_mm512_sub_epi16 (__m512i __A, __m512i __B) {
	419	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	420	_mm512_sub_epi16(__m512i __A, __m512i __B) {
424	421	return (__m512i) ((__v32hu) __A - (__v32hu) __B);
425	422	}
426	423
427		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	424	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
428	425	_mm512_mask_sub_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
429	426	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
430	427	(__v32hi)_mm512_sub_epi16(__A, __B),
431	428	(__v32hi)__W);
432	429	}
433	430
434		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	431	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
435	432	_mm512_maskz_sub_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
436	433	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
437	434	(__v32hi)_mm512_sub_epi16(__A, __B),
438	435	(__v32hi)_mm512_setzero_si512());
439	436	}
440	437
441		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	438	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
442	439	_mm512_mullo_epi16 (__m512i __A, __m512i __B) {
443	440	return (__m512i) ((__v32hu) __A * (__v32hu) __B);
444	441	}
445	442
446		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	443	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
447	444	_mm512_mask_mullo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
448	445	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
449	446	(__v32hi)_mm512_mullo_epi16(__A, __B),
450	447	(__v32hi)__W);
451	448	}
452	449
453		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	450	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
454	451	_mm512_maskz_mullo_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
455	452	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
456	453	(__v32hi)_mm512_mullo_epi16(__A, __B),
457	454	(__v32hi)_mm512_setzero_si512());
458	455	}
459	456
460		static __inline__ __m512i __DEFAULT_FN_ATTRS512
461		_mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W)
462		{
	457	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	458	_mm512_mask_blend_epi8(__mmask64 __U, __m512i __A, __m512i __W) {
463	459	return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U,
464	460	(__v64qi) __W,
465	461	(__v64qi) __A);
466	462	}
467	463
468		static __inline__ __m512i __DEFAULT_FN_ATTRS512
469		_mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W)
470		{
	464	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	465	_mm512_mask_blend_epi16(__mmask32 __U, __m512i __A, __m512i __W) {
471	466	return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U,
472	467	(__v32hi) __W,
473	468	(__v32hi) __A);
474	469	}
475	470
476		static __inline__ __m512i __DEFAULT_FN_ATTRS512
477		_mm512_abs_epi8 (__m512i __A)
478		{
	471	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	472	_mm512_abs_epi8(__m512i __A) {
479	473	return (__m512i)__builtin_elementwise_abs((__v64qs)__A);
480	474	}
481	475
482		static __inline__ __m512i __DEFAULT_FN_ATTRS512
483		_mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A)
484		{
	476	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	477	_mm512_mask_abs_epi8(__m512i __W, __mmask64 __U, __m512i __A) {
485	478	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
486	479	(__v64qi)_mm512_abs_epi8(__A),
487	480	(__v64qi)__W);
488	481	}
489	482
490		static __inline__ __m512i __DEFAULT_FN_ATTRS512
491		_mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A)
492		{
	483	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	484	_mm512_maskz_abs_epi8(__mmask64 __U, __m512i __A) {
493	485	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
494	486	(__v64qi)_mm512_abs_epi8(__A),
495	487	(__v64qi)_mm512_setzero_si512());
496	488	}
497	489
498		static __inline__ __m512i __DEFAULT_FN_ATTRS512
499		_mm512_abs_epi16 (__m512i __A)
500		{
	490	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	491	_mm512_abs_epi16(__m512i __A) {
501	492	return (__m512i)__builtin_elementwise_abs((__v32hi)__A);
502	493	}
503	494
504		static __inline__ __m512i __DEFAULT_FN_ATTRS512
505		_mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A)
506		{
	495	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	496	_mm512_mask_abs_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
507	497	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
508	498	(__v32hi)_mm512_abs_epi16(__A),
509	499	(__v32hi)__W);
510	500	}
511	501
512		static __inline__ __m512i __DEFAULT_FN_ATTRS512
513		_mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A)
514		{
	502	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	503	_mm512_maskz_abs_epi16(__mmask32 __U, __m512i __A) {
515	504	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
516	505	(__v32hi)_mm512_abs_epi16(__A),
517	506	(__v32hi)_mm512_setzero_si512());
518	507	}
519	508
520		static __inline__ __m512i __DEFAULT_FN_ATTRS512
521		_mm512_packs_epi32(__m512i __A, __m512i __B)
522		{
	509	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	510	_mm512_packs_epi32(__m512i __A, __m512i __B) {
523	511	return (__m512i)__builtin_ia32_packssdw512((__v16si)__A, (__v16si)__B);
524	512	}
525	513
526		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	514	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
527	515	_mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B)
528	516	{
529	517	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
...	...	@@ -531,21 +519,19 @@ _mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B)
531	519	(__v32hi)_mm512_setzero_si512());
532	520	}
533	521
534		static __inline__ __m512i __DEFAULT_FN_ATTRS512
535		_mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
536		{
	522	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	523	_mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
537	524	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
538	525	(__v32hi)_mm512_packs_epi32(__A, __B),
539	526	(__v32hi)__W);
540	527	}
541	528
542		static __inline__ __m512i __DEFAULT_FN_ATTRS512
543		_mm512_packs_epi16(__m512i __A, __m512i __B)
544		{
	529	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	530	_mm512_packs_epi16(__m512i __A, __m512i __B) {
545	531	return (__m512i)__builtin_ia32_packsswb512((__v32hi)__A, (__v32hi) __B);
546	532	}
547	533
548		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	534	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
549	535	_mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
550	536	{
551	537	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
...	...	@@ -553,7 +539,7 @@ _mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
553	539	(__v64qi)__W);
554	540	}
555	541
556		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	542	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
557	543	_mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B)
558	544	{
559	545	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
...	...	@@ -561,13 +547,12 @@ _mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B)
561	547	(__v64qi)_mm512_setzero_si512());
562	548	}
563	549
564		static __inline__ __m512i __DEFAULT_FN_ATTRS512
565		_mm512_packus_epi32(__m512i __A, __m512i __B)
566		{
	550	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	551	_mm512_packus_epi32(__m512i __A, __m512i __B) {
567	552	return (__m512i)__builtin_ia32_packusdw512((__v16si) __A, (__v16si) __B);
568	553	}
569	554
570		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	555	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
571	556	_mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B)
572	557	{
573	558	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
...	...	@@ -575,7 +560,7 @@ _mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B)
575	560	(__v32hi)_mm512_setzero_si512());
576	561	}
577	562
578		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	563	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
579	564	_mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
580	565	{
581	566	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
...	...	@@ -583,13 +568,12 @@ _mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
583	568	(__v32hi)__W);
584	569	}
585	570
586		static __inline__ __m512i __DEFAULT_FN_ATTRS512
587		_mm512_packus_epi16(__m512i __A, __m512i __B)
588		{
	571	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	572	_mm512_packus_epi16(__m512i __A, __m512i __B) {
589	573	return (__m512i)__builtin_ia32_packuswb512((__v32hi) __A, (__v32hi) __B);
590	574	}
591	575
592		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	576	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
593	577	_mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
594	578	{
595	579	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
...	...	@@ -597,7 +581,7 @@ _mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
597	581	(__v64qi)__W);
598	582	}
599	583
600		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	584	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
601	585	_mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B)
602	586	{
603	587	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
...	...	@@ -605,35 +589,31 @@ _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B)
605	589	(__v64qi)_mm512_setzero_si512());
606	590	}
607	591
608		static __inline__ __m512i __DEFAULT_FN_ATTRS512
609		_mm512_adds_epi8 (__m512i __A, __m512i __B)
610		{
	592	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	593	_mm512_adds_epi8(__m512i __A, __m512i __B) {
611	594	return (__m512i)__builtin_elementwise_add_sat((__v64qs)__A, (__v64qs)__B);
612	595	}
613	596
614		static __inline__ __m512i __DEFAULT_FN_ATTRS512
615		_mm512_mask_adds_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
616		{
	597	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	598	_mm512_mask_adds_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
617	599	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
618	600	(__v64qi)_mm512_adds_epi8(__A, __B),
619	601	(__v64qi)__W);
620	602	}
621	603
622		static __inline__ __m512i __DEFAULT_FN_ATTRS512
623		_mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B)
624		{
	604	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	605	_mm512_maskz_adds_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
625	606	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
626	607	(__v64qi)_mm512_adds_epi8(__A, __B),
627	608	(__v64qi)_mm512_setzero_si512());
628	609	}
629	610
630		static __inline__ __m512i __DEFAULT_FN_ATTRS512
631		_mm512_adds_epi16 (__m512i __A, __m512i __B)
632		{
	611	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	612	_mm512_adds_epi16(__m512i __A, __m512i __B) {
633	613	return (__m512i)__builtin_elementwise_add_sat((__v32hi)__A, (__v32hi)__B);
634	614	}
635	615
636		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	616	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
637	617	_mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
638	618	{
639	619	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -641,7 +621,7 @@ _mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
641	621	(__v32hi)__W);
642	622	}
643	623
644		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	624	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
645	625	_mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
646	626	{
647	627	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -649,13 +629,12 @@ _mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
649	629	(__v32hi)_mm512_setzero_si512());
650	630	}
651	631
652		static __inline__ __m512i __DEFAULT_FN_ATTRS512
653		_mm512_adds_epu8 (__m512i __A, __m512i __B)
654		{
	632	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	633	_mm512_adds_epu8(__m512i __A, __m512i __B) {
655	634	return (__m512i)__builtin_elementwise_add_sat((__v64qu) __A, (__v64qu) __B);
656	635	}
657	636
658		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	637	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
659	638	_mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
660	639	{
661	640	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -663,7 +642,7 @@ _mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
663	642	(__v64qi)__W);
664	643	}
665	644
666		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	645	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
667	646	_mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
668	647	{
669	648	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -671,280 +650,238 @@ _mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
671	650	(__v64qi)_mm512_setzero_si512());
672	651	}
673	652
674		static __inline__ __m512i __DEFAULT_FN_ATTRS512
675		_mm512_adds_epu16 (__m512i __A, __m512i __B)
676		{
	653	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	654	_mm512_adds_epu16(__m512i __A, __m512i __B) {
677	655	return (__m512i)__builtin_elementwise_add_sat((__v32hu) __A, (__v32hu) __B);
678	656	}
679	657
680		static __inline__ __m512i __DEFAULT_FN_ATTRS512
681		_mm512_mask_adds_epu16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
682		{
	658	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	659	_mm512_mask_adds_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
683	660	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
684	661	(__v32hi)_mm512_adds_epu16(__A, __B),
685	662	(__v32hi)__W);
686	663	}
687	664
688		static __inline__ __m512i __DEFAULT_FN_ATTRS512
689		_mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
690		{
	665	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	666	_mm512_maskz_adds_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
691	667	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
692	668	(__v32hi)_mm512_adds_epu16(__A, __B),
693	669	(__v32hi)_mm512_setzero_si512());
694	670	}
695	671
696		static __inline__ __m512i __DEFAULT_FN_ATTRS512
697		_mm512_avg_epu8 (__m512i __A, __m512i __B)
698		{
699		return (__m512i)__builtin_ia32_pavgb512((__v64qi)__A, (__v64qi)__B);
	672	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	673	_mm512_avg_epu8(__m512i __A, __m512i __B) {
	674	return (__m512i)__builtin_ia32_pavgb512((__v64qu)__A, (__v64qu)__B);
700	675	}
701	676
702		static __inline__ __m512i __DEFAULT_FN_ATTRS512
703		_mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A,
704		__m512i __B)
705		{
706		return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
707		(__v64qi)_mm512_avg_epu8(__A, __B),
708		(__v64qi)__W);
	677	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	678	_mm512_mask_avg_epu8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
	679	return (__m512i)__builtin_ia32_selectb_512(
	680	(__mmask64)__U, (__v64qi)_mm512_avg_epu8(__A, __B), (__v64qi)__W);
709	681	}
710	682
711		static __inline__ __m512i __DEFAULT_FN_ATTRS512
712		_mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
713		{
	683	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	684	_mm512_maskz_avg_epu8(__mmask64 __U, __m512i __A, __m512i __B) {
714	685	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
715		(__v64qi)_mm512_avg_epu8(__A, __B),
716		(__v64qi)_mm512_setzero_si512());
	686	(__v64qi)_mm512_avg_epu8(__A, __B),
	687	(__v64qi)_mm512_setzero_si512());
717	688	}
718	689
719		static __inline__ __m512i __DEFAULT_FN_ATTRS512
720		_mm512_avg_epu16 (__m512i __A, __m512i __B)
721		{
722		return (__m512i)__builtin_ia32_pavgw512((__v32hi)__A, (__v32hi)__B);
	690	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	691	_mm512_avg_epu16(__m512i __A, __m512i __B) {
	692	return (__m512i)__builtin_ia32_pavgw512((__v32hu)__A, (__v32hu)__B);
723	693	}
724	694
725		static __inline__ __m512i __DEFAULT_FN_ATTRS512
726		_mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A,
727		__m512i __B)
728		{
729		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
730		(__v32hi)_mm512_avg_epu16(__A, __B),
731		(__v32hi)__W);
	695	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	696	_mm512_mask_avg_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
	697	return (__m512i)__builtin_ia32_selectw_512(
	698	(__mmask32)__U, (__v32hi)_mm512_avg_epu16(__A, __B), (__v32hi)__W);
732	699	}
733	700
734		static __inline__ __m512i __DEFAULT_FN_ATTRS512
735		_mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
736		{
737		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
738		(__v32hi)_mm512_avg_epu16(__A, __B),
739		(__v32hi) _mm512_setzero_si512());
	701	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	702	_mm512_maskz_avg_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
	703	return (__m512i)__builtin_ia32_selectw_512(
	704	(__mmask32)__U, (__v32hi)_mm512_avg_epu16(__A, __B),
	705	(__v32hi)_mm512_setzero_si512());
740	706	}
741	707
742		static __inline__ __m512i __DEFAULT_FN_ATTRS512
743		_mm512_max_epi8 (__m512i __A, __m512i __B)
744		{
	708	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	709	_mm512_max_epi8(__m512i __A, __m512i __B) {
745	710	return (__m512i)__builtin_elementwise_max((__v64qs) __A, (__v64qs) __B);
746	711	}
747	712
748		static __inline__ __m512i __DEFAULT_FN_ATTRS512
749		_mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B)
750		{
	713	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	714	_mm512_maskz_max_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
751	715	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
752	716	(__v64qi)_mm512_max_epi8(__A, __B),
753	717	(__v64qi)_mm512_setzero_si512());
754	718	}
755	719
756		static __inline__ __m512i __DEFAULT_FN_ATTRS512
757		_mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
758		{
	720	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	721	_mm512_mask_max_epi8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
759	722	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
760	723	(__v64qi)_mm512_max_epi8(__A, __B),
761	724	(__v64qi)__W);
762	725	}
763	726
764		static __inline__ __m512i __DEFAULT_FN_ATTRS512
765		_mm512_max_epi16 (__m512i __A, __m512i __B)
766		{
	727	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	728	_mm512_max_epi16(__m512i __A, __m512i __B) {
767	729	return (__m512i)__builtin_elementwise_max((__v32hi) __A, (__v32hi) __B);
768	730	}
769	731
770		static __inline__ __m512i __DEFAULT_FN_ATTRS512
771		_mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B)
772		{
	732	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	733	_mm512_maskz_max_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
773	734	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
774	735	(__v32hi)_mm512_max_epi16(__A, __B),
775	736	(__v32hi)_mm512_setzero_si512());
776	737	}
777	738
778		static __inline__ __m512i __DEFAULT_FN_ATTRS512
779		_mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A,
780		__m512i __B)
781		{
	739	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	740	_mm512_mask_max_epi16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
782	741	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
783	742	(__v32hi)_mm512_max_epi16(__A, __B),
784	743	(__v32hi)__W);
785	744	}
786	745
787		static __inline__ __m512i __DEFAULT_FN_ATTRS512
788		_mm512_max_epu8 (__m512i __A, __m512i __B)
789		{
	746	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	747	_mm512_max_epu8(__m512i __A, __m512i __B) {
790	748	return (__m512i)__builtin_elementwise_max((__v64qu)__A, (__v64qu)__B);
791	749	}
792	750
793		static __inline__ __m512i __DEFAULT_FN_ATTRS512
794		_mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B)
795		{
	751	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	752	_mm512_maskz_max_epu8(__mmask64 __M, __m512i __A, __m512i __B) {
796	753	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
797	754	(__v64qi)_mm512_max_epu8(__A, __B),
798	755	(__v64qi)_mm512_setzero_si512());
799	756	}
800	757
801		static __inline__ __m512i __DEFAULT_FN_ATTRS512
802		_mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
803		{
	758	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	759	_mm512_mask_max_epu8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
804	760	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
805	761	(__v64qi)_mm512_max_epu8(__A, __B),
806	762	(__v64qi)__W);
807	763	}
808	764
809		static __inline__ __m512i __DEFAULT_FN_ATTRS512
810		_mm512_max_epu16 (__m512i __A, __m512i __B)
811		{
	765	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	766	_mm512_max_epu16(__m512i __A, __m512i __B) {
812	767	return (__m512i)__builtin_elementwise_max((__v32hu)__A, (__v32hu)__B);
813	768	}
814	769
815		static __inline__ __m512i __DEFAULT_FN_ATTRS512
816		_mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B)
817		{
	770	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	771	_mm512_maskz_max_epu16(__mmask32 __M, __m512i __A, __m512i __B) {
818	772	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
819	773	(__v32hi)_mm512_max_epu16(__A, __B),
820	774	(__v32hi)_mm512_setzero_si512());
821	775	}
822	776
823		static __inline__ __m512i __DEFAULT_FN_ATTRS512
824		_mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
825		{
	777	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	778	_mm512_mask_max_epu16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
826	779	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
827	780	(__v32hi)_mm512_max_epu16(__A, __B),
828	781	(__v32hi)__W);
829	782	}
830	783
831		static __inline__ __m512i __DEFAULT_FN_ATTRS512
832		_mm512_min_epi8 (__m512i __A, __m512i __B)
833		{
	784	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	785	_mm512_min_epi8(__m512i __A, __m512i __B) {
834	786	return (__m512i)__builtin_elementwise_min((__v64qs) __A, (__v64qs) __B);
835	787	}
836	788
837		static __inline__ __m512i __DEFAULT_FN_ATTRS512
838		_mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B)
839		{
	789	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	790	_mm512_maskz_min_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
840	791	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
841	792	(__v64qi)_mm512_min_epi8(__A, __B),
842	793	(__v64qi)_mm512_setzero_si512());
843	794	}
844	795
845		static __inline__ __m512i __DEFAULT_FN_ATTRS512
846		_mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
847		{
	796	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	797	_mm512_mask_min_epi8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
848	798	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
849	799	(__v64qi)_mm512_min_epi8(__A, __B),
850	800	(__v64qi)__W);
851	801	}
852	802
853		static __inline__ __m512i __DEFAULT_FN_ATTRS512
854		_mm512_min_epi16 (__m512i __A, __m512i __B)
855		{
	803	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	804	_mm512_min_epi16(__m512i __A, __m512i __B) {
856	805	return (__m512i)__builtin_elementwise_min((__v32hi) __A, (__v32hi) __B);
857	806	}
858	807
859		static __inline__ __m512i __DEFAULT_FN_ATTRS512
860		_mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B)
861		{
	808	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	809	_mm512_maskz_min_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
862	810	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
863	811	(__v32hi)_mm512_min_epi16(__A, __B),
864	812	(__v32hi)_mm512_setzero_si512());
865	813	}
866	814
867		static __inline__ __m512i __DEFAULT_FN_ATTRS512
868		_mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
869		{
	815	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	816	_mm512_mask_min_epi16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
870	817	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
871	818	(__v32hi)_mm512_min_epi16(__A, __B),
872	819	(__v32hi)__W);
873	820	}
874	821
875		static __inline__ __m512i __DEFAULT_FN_ATTRS512
876		_mm512_min_epu8 (__m512i __A, __m512i __B)
877		{
	822	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	823	_mm512_min_epu8(__m512i __A, __m512i __B) {
878	824	return (__m512i)__builtin_elementwise_min((__v64qu)__A, (__v64qu)__B);
879	825	}
880	826
881		static __inline__ __m512i __DEFAULT_FN_ATTRS512
882		_mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B)
883		{
	827	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	828	_mm512_maskz_min_epu8(__mmask64 __M, __m512i __A, __m512i __B) {
884	829	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
885	830	(__v64qi)_mm512_min_epu8(__A, __B),
886	831	(__v64qi)_mm512_setzero_si512());
887	832	}
888	833
889		static __inline__ __m512i __DEFAULT_FN_ATTRS512
890		_mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, __m512i __B)
891		{
	834	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	835	_mm512_mask_min_epu8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
892	836	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
893	837	(__v64qi)_mm512_min_epu8(__A, __B),
894	838	(__v64qi)__W);
895	839	}
896	840
897		static __inline__ __m512i __DEFAULT_FN_ATTRS512
898		_mm512_min_epu16 (__m512i __A, __m512i __B)
899		{
	841	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	842	_mm512_min_epu16(__m512i __A, __m512i __B) {
900	843	return (__m512i)__builtin_elementwise_min((__v32hu)__A, (__v32hu)__B);
901	844	}
902	845
903		static __inline__ __m512i __DEFAULT_FN_ATTRS512
904		_mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B)
905		{
	846	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	847	_mm512_maskz_min_epu16(__mmask32 __M, __m512i __A, __m512i __B) {
906	848	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
907	849	(__v32hi)_mm512_min_epu16(__A, __B),
908	850	(__v32hi)_mm512_setzero_si512());
909	851	}
910	852
911		static __inline__ __m512i __DEFAULT_FN_ATTRS512
912		_mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, __m512i __B)
913		{
	853	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	854	_mm512_mask_min_epu16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
914	855	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
915	856	(__v32hi)_mm512_min_epu16(__A, __B),
916	857	(__v32hi)__W);
917	858	}
918	859
919		static __inline__ __m512i __DEFAULT_FN_ATTRS512
920		_mm512_shuffle_epi8(__m512i __A, __m512i __B)
921		{
	860	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	861	_mm512_shuffle_epi8(__m512i __A, __m512i __B) {
922	862	return (__m512i)__builtin_ia32_pshufb512((__v64qi)__A,(__v64qi)__B);
923	863	}
924	864
925		static __inline__ __m512i __DEFAULT_FN_ATTRS512
926		_mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
927		{
	865	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	866	_mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
928	867	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
929	868	(__v64qi)_mm512_shuffle_epi8(__A, __B),
930	869	(__v64qi)__W);
931	870	}
932	871
933		static __inline__ __m512i __DEFAULT_FN_ATTRS512
934		_mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B)
935		{
	872	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	873	_mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
936	874	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
937	875	(__v64qi)_mm512_shuffle_epi8(__A, __B),
938	876	(__v64qi)_mm512_setzero_si512());
939	877	}
940	878
941		static __inline__ __m512i __DEFAULT_FN_ATTRS512
942		_mm512_subs_epi8 (__m512i __A, __m512i __B)
943		{
	879	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	880	_mm512_subs_epi8(__m512i __A, __m512i __B) {
944	881	return (__m512i)__builtin_elementwise_sub_sat((__v64qs)__A, (__v64qs)__B);
945	882	}
946	883
947		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	884	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
948	885	_mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
949	886	{
950	887	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -952,7 +889,7 @@ _mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
952	889	(__v64qi)__W);
953	890	}
954	891
955		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	892	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
956	893	_mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B)
957	894	{
958	895	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -960,13 +897,12 @@ _mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B)
960	897	(__v64qi)_mm512_setzero_si512());
961	898	}
962	899
963		static __inline__ __m512i __DEFAULT_FN_ATTRS512
964		_mm512_subs_epi16 (__m512i __A, __m512i __B)
965		{
	900	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	901	_mm512_subs_epi16(__m512i __A, __m512i __B) {
966	902	return (__m512i)__builtin_elementwise_sub_sat((__v32hi)__A, (__v32hi)__B);
967	903	}
968	904
969		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	905	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
970	906	_mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
971	907	{
972	908	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -974,7 +910,7 @@ _mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
974	910	(__v32hi)__W);
975	911	}
976	912
977		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	913	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
978	914	_mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
979	915	{
980	916	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -982,13 +918,12 @@ _mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
982	918	(__v32hi)_mm512_setzero_si512());
983	919	}
984	920
985		static __inline__ __m512i __DEFAULT_FN_ATTRS512
986		_mm512_subs_epu8 (__m512i __A, __m512i __B)
987		{
	921	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	922	_mm512_subs_epu8(__m512i __A, __m512i __B) {
988	923	return (__m512i)__builtin_elementwise_sub_sat((__v64qu) __A, (__v64qu) __B);
989	924	}
990	925
991		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	926	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
992	927	_mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
993	928	{
994	929	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -996,7 +931,7 @@ _mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B)
996	931	(__v64qi)__W);
997	932	}
998	933
999		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	934	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1000	935	_mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
1001	936	{
1002	937	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
...	...	@@ -1004,13 +939,12 @@ _mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
1004	939	(__v64qi)_mm512_setzero_si512());
1005	940	}
1006	941
1007		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1008		_mm512_subs_epu16 (__m512i __A, __m512i __B)
1009		{
	942	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	943	_mm512_subs_epu16(__m512i __A, __m512i __B) {
1010	944	return (__m512i)__builtin_elementwise_sub_sat((__v32hu) __A, (__v32hu) __B);
1011	945	}
1012	946
1013		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	947	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1014	948	_mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1015	949	{
1016	950	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1018,7 +952,7 @@ _mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1018	952	(__v32hi)__W);
1019	953	}
1020	954
1021		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	955	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1022	956	_mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
1023	957	{
1024	958	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1026,113 +960,97 @@ _mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
1026	960	(__v32hi)_mm512_setzero_si512());
1027	961	}
1028	962
1029		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1030		_mm512_permutex2var_epi16(__m512i __A, __m512i __I, __m512i __B)
1031		{
	963	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	964	_mm512_permutex2var_epi16(__m512i __A, __m512i __I, __m512i __B) {
1032	965	return (__m512i)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I,
1033	966	(__v32hi)__B);
1034	967	}
1035	968
1036		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	969	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1037	970	_mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I,
1038		__m512i __B)
1039		{
	971	__m512i __B) {
1040	972	return (__m512i)__builtin_ia32_selectw_512(__U,
1041	973	(__v32hi)_mm512_permutex2var_epi16(__A, __I, __B),
1042	974	(__v32hi)__A);
1043	975	}
1044	976
1045		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	977	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1046	978	_mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U,
1047		__m512i __B)
1048		{
	979	__m512i __B) {
1049	980	return (__m512i)__builtin_ia32_selectw_512(__U,
1050	981	(__v32hi)_mm512_permutex2var_epi16(__A, __I, __B),
1051	982	(__v32hi)__I);
1052	983	}
1053	984
1054		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	985	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1055	986	_mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I,
1056		__m512i __B)
1057		{
	987	__m512i __B) {
1058	988	return (__m512i)__builtin_ia32_selectw_512(__U,
1059	989	(__v32hi)_mm512_permutex2var_epi16(__A, __I, __B),
1060	990	(__v32hi)_mm512_setzero_si512());
1061	991	}
1062	992
1063		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1064		_mm512_mulhrs_epi16(__m512i __A, __m512i __B)
1065		{
	993	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	994	_mm512_mulhrs_epi16(__m512i __A, __m512i __B) {
1066	995	return (__m512i)__builtin_ia32_pmulhrsw512((__v32hi)__A, (__v32hi)__B);
1067	996	}
1068	997
1069		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1070		_mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1071		{
	998	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	999	_mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
1072	1000	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1073	1001	(__v32hi)_mm512_mulhrs_epi16(__A, __B),
1074	1002	(__v32hi)__W);
1075	1003	}
1076	1004
1077		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1078		_mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1079		{
	1005	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1006	_mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
1080	1007	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1081	1008	(__v32hi)_mm512_mulhrs_epi16(__A, __B),
1082	1009	(__v32hi)_mm512_setzero_si512());
1083	1010	}
1084	1011
1085		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1086		_mm512_mulhi_epi16(__m512i __A, __m512i __B)
1087		{
1088		return (__m512i)__builtin_ia32_pmulhw512((__v32hi) __A, (__v32hi) __B);
	1012	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1013	_mm512_mulhi_epi16(__m512i __A, __m512i __B) {
	1014	return (__m512i)__builtin_ia32_pmulhw512((__v32hi)__A, (__v32hi)__B);
1089	1015	}
1090	1016
1091		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1092		_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A,
1093		__m512i __B)
1094		{
1095		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1096		(__v32hi)_mm512_mulhi_epi16(__A, __B),
1097		(__v32hi)__W);
	1017	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1018	_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
	1019	return (__m512i)__builtin_ia32_selectw_512(
	1020	(__mmask32)__U, (__v32hi)_mm512_mulhi_epi16(__A, __B), (__v32hi)__W);
1098	1021	}
1099	1022
1100		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1101		_mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1102		{
1103		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1104		(__v32hi)_mm512_mulhi_epi16(__A, __B),
1105		(__v32hi)_mm512_setzero_si512());
	1023	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1024	_mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
	1025	return (__m512i)__builtin_ia32_selectw_512(
	1026	(__mmask32)__U, (__v32hi)_mm512_mulhi_epi16(__A, __B),
	1027	(__v32hi)_mm512_setzero_si512());
1106	1028	}
1107	1029
1108		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1109		_mm512_mulhi_epu16(__m512i __A, __m512i __B)
1110		{
1111		return (__m512i)__builtin_ia32_pmulhuw512((__v32hi) __A, (__v32hi) __B);
	1030	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1031	_mm512_mulhi_epu16(__m512i __A, __m512i __B) {
	1032	return (__m512i)__builtin_ia32_pmulhuw512((__v32hu)__A, (__v32hu)__B);
1112	1033	}
1113	1034
1114		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1115		_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1116		{
1117		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1118		(__v32hi)_mm512_mulhi_epu16(__A, __B),
1119		(__v32hi)__W);
	1035	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1036	_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
	1037	return (__m512i)__builtin_ia32_selectw_512(
	1038	(__mmask32)__U, (__v32hi)_mm512_mulhi_epu16(__A, __B), (__v32hi)__W);
1120	1039	}
1121	1040
1122		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1123		_mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
1124		{
1125		return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1126		(__v32hi)_mm512_mulhi_epu16(__A, __B),
1127		(__v32hi)_mm512_setzero_si512());
	1041	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1042	_mm512_maskz_mulhi_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
	1043	return (__m512i)__builtin_ia32_selectw_512(
	1044	(__mmask32)__U, (__v32hi)_mm512_mulhi_epu16(__A, __B),
	1045	(__v32hi)_mm512_setzero_si512());
1128	1046	}
1129	1047
1130		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1048	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1131	1049	_mm512_maddubs_epi16(__m512i __X, __m512i __Y) {
1132	1050	return (__m512i)__builtin_ia32_pmaddubsw512((__v64qi)__X, (__v64qi)__Y);
1133	1051	}
1134	1052
1135		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1053	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1136	1054	_mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X,
1137	1055	__m512i __Y) {
1138	1056	return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U,
...	...	@@ -1140,26 +1058,26 @@ _mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X,
1140	1058	(__v32hi)__W);
1141	1059	}
1142	1060
1143		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1061	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1144	1062	_mm512_maskz_maddubs_epi16(__mmask32 __U, __m512i __X, __m512i __Y) {
1145	1063	return (__m512i)__builtin_ia32_selectw_512((__mmask32) __U,
1146	1064	(__v32hi)_mm512_maddubs_epi16(__X, __Y),
1147	1065	(__v32hi)_mm512_setzero_si512());
1148	1066	}
1149	1067
1150		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1068	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1151	1069	_mm512_madd_epi16(__m512i __A, __m512i __B) {
1152	1070	return (__m512i)__builtin_ia32_pmaddwd512((__v32hi)__A, (__v32hi)__B);
1153	1071	}
1154	1072
1155		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1073	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1156	1074	_mm512_mask_madd_epi16(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
1157	1075	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
1158	1076	(__v16si)_mm512_madd_epi16(__A, __B),
1159	1077	(__v16si)__W);
1160	1078	}
1161	1079
1162		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1080	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1163	1081	_mm512_maskz_madd_epi16(__mmask16 __U, __m512i __A, __m512i __B) {
1164	1082	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
1165	1083	(__v16si)_mm512_madd_epi16(__A, __B),
...	...	@@ -1247,7 +1165,7 @@ _mm512_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A)
1247	1165	__builtin_ia32_pmovuswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M);
1248	1166	}
1249	1167
1250		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1168	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1251	1169	_mm512_unpackhi_epi8(__m512i __A, __m512i __B) {
1252	1170	return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B,
1253	1171	8, 64+8, 9, 64+9,
...	...	@@ -1268,21 +1186,21 @@ _mm512_unpackhi_epi8(__m512i __A, __m512i __B) {
1268	1186	62, 64+62, 63, 64+63);
1269	1187	}
1270	1188
1271		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1189	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1272	1190	_mm512_mask_unpackhi_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
1273	1191	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
1274	1192	(__v64qi)_mm512_unpackhi_epi8(__A, __B),
1275	1193	(__v64qi)__W);
1276	1194	}
1277	1195
1278		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1196	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1279	1197	_mm512_maskz_unpackhi_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
1280	1198	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
1281	1199	(__v64qi)_mm512_unpackhi_epi8(__A, __B),
1282	1200	(__v64qi)_mm512_setzero_si512());
1283	1201	}
1284	1202
1285		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1203	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1286	1204	_mm512_unpackhi_epi16(__m512i __A, __m512i __B) {
1287	1205	return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B,
1288	1206	4, 32+4, 5, 32+5,
...	...	@@ -1295,21 +1213,21 @@ _mm512_unpackhi_epi16(__m512i __A, __m512i __B) {
1295	1213	30, 32+30, 31, 32+31);
1296	1214	}
1297	1215
1298		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1216	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1299	1217	_mm512_mask_unpackhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
1300	1218	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1301	1219	(__v32hi)_mm512_unpackhi_epi16(__A, __B),
1302	1220	(__v32hi)__W);
1303	1221	}
1304	1222
1305		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1223	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1306	1224	_mm512_maskz_unpackhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
1307	1225	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1308	1226	(__v32hi)_mm512_unpackhi_epi16(__A, __B),
1309	1227	(__v32hi)_mm512_setzero_si512());
1310	1228	}
1311	1229
1312		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1230	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1313	1231	_mm512_unpacklo_epi8(__m512i __A, __m512i __B) {
1314	1232	return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B,
1315	1233	0, 64+0, 1, 64+1,
...	...	@@ -1330,21 +1248,21 @@ _mm512_unpacklo_epi8(__m512i __A, __m512i __B) {
1330	1248	54, 64+54, 55, 64+55);
1331	1249	}
1332	1250
1333		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1251	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1334	1252	_mm512_mask_unpacklo_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
1335	1253	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
1336	1254	(__v64qi)_mm512_unpacklo_epi8(__A, __B),
1337	1255	(__v64qi)__W);
1338	1256	}
1339	1257
1340		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1258	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1341	1259	_mm512_maskz_unpacklo_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
1342	1260	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
1343	1261	(__v64qi)_mm512_unpacklo_epi8(__A, __B),
1344	1262	(__v64qi)_mm512_setzero_si512());
1345	1263	}
1346	1264
1347		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1265	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1348	1266	_mm512_unpacklo_epi16(__m512i __A, __m512i __B) {
1349	1267	return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B,
1350	1268	0, 32+0, 1, 32+1,
...	...	@@ -1357,67 +1275,60 @@ _mm512_unpacklo_epi16(__m512i __A, __m512i __B) {
1357	1275	26, 32+26, 27, 32+27);
1358	1276	}
1359	1277
1360		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1278	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1361	1279	_mm512_mask_unpacklo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
1362	1280	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1363	1281	(__v32hi)_mm512_unpacklo_epi16(__A, __B),
1364	1282	(__v32hi)__W);
1365	1283	}
1366	1284
1367		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1285	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1368	1286	_mm512_maskz_unpacklo_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
1369	1287	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1370	1288	(__v32hi)_mm512_unpacklo_epi16(__A, __B),
1371	1289	(__v32hi)_mm512_setzero_si512());
1372	1290	}
1373	1291
1374		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1375		_mm512_cvtepi8_epi16(__m256i __A)
1376		{
	1292	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1293	_mm512_cvtepi8_epi16(__m256i __A) {
1377	1294	/* This function always performs a signed extension, but __v32qi is a char
1378	1295	which may be signed or unsigned, so use __v32qs. */
1379	1296	return (__m512i)__builtin_convertvector((__v32qs)__A, __v32hi);
1380	1297	}
1381	1298
1382		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1383		_mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A)
1384		{
	1299	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1300	_mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) {
1385	1301	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1386	1302	(__v32hi)_mm512_cvtepi8_epi16(__A),
1387	1303	(__v32hi)__W);
1388	1304	}
1389	1305
1390		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1391		_mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A)
1392		{
	1306	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1307	_mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) {
1393	1308	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1394	1309	(__v32hi)_mm512_cvtepi8_epi16(__A),
1395	1310	(__v32hi)_mm512_setzero_si512());
1396	1311	}
1397	1312
1398		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1399		_mm512_cvtepu8_epi16(__m256i __A)
1400		{
	1313	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1314	_mm512_cvtepu8_epi16(__m256i __A) {
1401	1315	return (__m512i)__builtin_convertvector((__v32qu)__A, __v32hi);
1402	1316	}
1403	1317
1404		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1405		_mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A)
1406		{
	1318	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1319	_mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) {
1407	1320	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1408	1321	(__v32hi)_mm512_cvtepu8_epi16(__A),
1409	1322	(__v32hi)__W);
1410	1323	}
1411	1324
1412		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1413		_mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A)
1414		{
	1325	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1326	_mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) {
1415	1327	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1416	1328	(__v32hi)_mm512_cvtepu8_epi16(__A),
1417	1329	(__v32hi)_mm512_setzero_si512());
1418	1330	}
1419	1331
1420
1421	1332	#define _mm512_shufflehi_epi16(A, imm) \
1422	1333	((__m512i)__builtin_ia32_pshufhw512((__v32hi)(__m512i)(A), (int)(imm)))
1423	1334
...	...	@@ -1450,13 +1361,13 @@ _mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A)
1450	1361	(imm)), \
1451	1362	(__v32hi)_mm512_setzero_si512()))
1452	1363
1453		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1364	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1454	1365	_mm512_sllv_epi16(__m512i __A, __m512i __B)
1455	1366	{
1456	1367	return (__m512i)__builtin_ia32_psllv32hi((__v32hi) __A, (__v32hi) __B);
1457	1368	}
1458	1369
1459		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1370	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1460	1371	_mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1461	1372	{
1462	1373	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1464,7 +1375,7 @@ _mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1464	1375	(__v32hi)__W);
1465	1376	}
1466	1377
1467		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1378	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1468	1379	_mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1469	1380	{
1470	1381	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1472,61 +1383,56 @@ _mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1472	1383	(__v32hi)_mm512_setzero_si512());
1473	1384	}
1474	1385
1475		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1476		_mm512_sll_epi16(__m512i __A, __m128i __B)
1477		{
	1386	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1387	_mm512_sll_epi16(__m512i __A, __m128i __B) {
1478	1388	return (__m512i)__builtin_ia32_psllw512((__v32hi) __A, (__v8hi) __B);
1479	1389	}
1480	1390
1481		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1482		_mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B)
1483		{
	1391	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1392	_mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
1484	1393	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1485	1394	(__v32hi)_mm512_sll_epi16(__A, __B),
1486	1395	(__v32hi)__W);
1487	1396	}
1488	1397
1489		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1490		_mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B)
1491		{
	1398	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1399	_mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
1492	1400	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1493	1401	(__v32hi)_mm512_sll_epi16(__A, __B),
1494	1402	(__v32hi)_mm512_setzero_si512());
1495	1403	}
1496	1404
1497		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1498		_mm512_slli_epi16(__m512i __A, unsigned int __B)
1499		{
	1405	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1406	_mm512_slli_epi16(__m512i __A, unsigned int __B) {
1500	1407	return (__m512i)__builtin_ia32_psllwi512((__v32hi)__A, (int)__B);
1501	1408	}
1502	1409
1503		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1410	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1504	1411	_mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A,
1505		unsigned int __B)
1506		{
	1412	unsigned int __B) {
1507	1413	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1508	1414	(__v32hi)_mm512_slli_epi16(__A, __B),
1509	1415	(__v32hi)__W);
1510	1416	}
1511	1417
1512		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1513		_mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, unsigned int __B)
1514		{
	1418	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1419	_mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A, unsigned int __B) {
1515	1420	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1516	1421	(__v32hi)_mm512_slli_epi16(__A, __B),
1517	1422	(__v32hi)_mm512_setzero_si512());
1518	1423	}
1519	1424
1520		#define _mm512_bslli_epi128(a, imm) \
1521		((__m512i)__builtin_ia32_pslldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)))
	1425	#define _mm512_bslli_epi128(a, imm) \
	1426	((__m512i)__builtin_ia32_pslldqi512_byteshift((__v64qi)(__m512i)(a), \
	1427	(int)(imm)))
1522	1428
1523		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1429	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1524	1430	_mm512_srlv_epi16(__m512i __A, __m512i __B)
1525	1431	{
1526	1432	return (__m512i)__builtin_ia32_psrlv32hi((__v32hi)__A, (__v32hi)__B);
1527	1433	}
1528	1434
1529		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1435	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1530	1436	_mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1531	1437	{
1532	1438	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1534,7 +1440,7 @@ _mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1534	1440	(__v32hi)__W);
1535	1441	}
1536	1442
1537		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1443	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1538	1444	_mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1539	1445	{
1540	1446	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1542,13 +1448,13 @@ _mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1542	1448	(__v32hi)_mm512_setzero_si512());
1543	1449	}
1544	1450
1545		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1451	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1546	1452	_mm512_srav_epi16(__m512i __A, __m512i __B)
1547	1453	{
1548	1454	return (__m512i)__builtin_ia32_psrav32hi((__v32hi)__A, (__v32hi)__B);
1549	1455	}
1550	1456
1551		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1457	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1552	1458	_mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1553	1459	{
1554	1460	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1556,7 +1462,7 @@ _mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
1556	1462	(__v32hi)__W);
1557	1463	}
1558	1464
1559		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1465	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1560	1466	_mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1561	1467	{
1562	1468	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
...	...	@@ -1564,100 +1470,89 @@ _mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B)
1564	1470	(__v32hi)_mm512_setzero_si512());
1565	1471	}
1566	1472
1567		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1568		_mm512_sra_epi16(__m512i __A, __m128i __B)
1569		{
	1473	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1474	_mm512_sra_epi16(__m512i __A, __m128i __B) {
1570	1475	return (__m512i)__builtin_ia32_psraw512((__v32hi) __A, (__v8hi) __B);
1571	1476	}
1572	1477
1573		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1574		_mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B)
1575		{
	1478	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1479	_mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
1576	1480	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1577	1481	(__v32hi)_mm512_sra_epi16(__A, __B),
1578	1482	(__v32hi)__W);
1579	1483	}
1580	1484
1581		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1582		_mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B)
1583		{
	1485	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1486	_mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
1584	1487	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1585	1488	(__v32hi)_mm512_sra_epi16(__A, __B),
1586	1489	(__v32hi)_mm512_setzero_si512());
1587	1490	}
1588	1491
1589		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1590		_mm512_srai_epi16(__m512i __A, unsigned int __B)
1591		{
	1492	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1493	_mm512_srai_epi16(__m512i __A, unsigned int __B) {
1592	1494	return (__m512i)__builtin_ia32_psrawi512((__v32hi)__A, (int)__B);
1593	1495	}
1594	1496
1595		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1497	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1596	1498	_mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A,
1597		unsigned int __B)
1598		{
	1499	unsigned int __B) {
1599	1500	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1600	1501	(__v32hi)_mm512_srai_epi16(__A, __B),
1601	1502	(__v32hi)__W);
1602	1503	}
1603	1504
1604		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1605		_mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, unsigned int __B)
1606		{
	1505	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1506	_mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A, unsigned int __B) {
1607	1507	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1608	1508	(__v32hi)_mm512_srai_epi16(__A, __B),
1609	1509	(__v32hi)_mm512_setzero_si512());
1610	1510	}
1611	1511
1612		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1613		_mm512_srl_epi16(__m512i __A, __m128i __B)
1614		{
	1512	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1513	_mm512_srl_epi16(__m512i __A, __m128i __B) {
1615	1514	return (__m512i)__builtin_ia32_psrlw512((__v32hi) __A, (__v8hi) __B);
1616	1515	}
1617	1516
1618		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1619		_mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B)
1620		{
	1517	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1518	_mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
1621	1519	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1622	1520	(__v32hi)_mm512_srl_epi16(__A, __B),
1623	1521	(__v32hi)__W);
1624	1522	}
1625	1523
1626		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1627		_mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B)
1628		{
	1524	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1525	_mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
1629	1526	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1630	1527	(__v32hi)_mm512_srl_epi16(__A, __B),
1631	1528	(__v32hi)_mm512_setzero_si512());
1632	1529	}
1633	1530
1634		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1635		_mm512_srli_epi16(__m512i __A, unsigned int __B)
1636		{
	1531	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1532	_mm512_srli_epi16(__m512i __A, unsigned int __B) {
1637	1533	return (__m512i)__builtin_ia32_psrlwi512((__v32hi)__A, (int)__B);
1638	1534	}
1639	1535
1640		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1536	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1641	1537	_mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A,
1642		unsigned int __B)
1643		{
	1538	unsigned int __B) {
1644	1539	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1645	1540	(__v32hi)_mm512_srli_epi16(__A, __B),
1646	1541	(__v32hi)__W);
1647	1542	}
1648	1543
1649		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1650		_mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B)
1651		{
	1544	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1545	_mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A, int __B) {
1652	1546	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
1653	1547	(__v32hi)_mm512_srli_epi16(__A, (unsigned int)__B),
1654	1548	(__v32hi)_mm512_setzero_si512());
1655	1549	}
1656	1550
1657		#define _mm512_bsrli_epi128(a, imm) \
1658		((__m512i)__builtin_ia32_psrldqi512_byteshift((__v8di)(__m512i)(a), (int)(imm)))
	1551	#define _mm512_bsrli_epi128(a, imm) \
	1552	((__m512i)__builtin_ia32_psrldqi512_byteshift((__v64qi)(__m512i)(a), \
	1553	(int)(imm)))
1659	1554
1660		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1555	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1661	1556	_mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A)
1662	1557	{
1663	1558	return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U,
...	...	@@ -1665,23 +1560,21 @@ _mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A)
1665	1560	(__v32hi) __W);
1666	1561	}
1667	1562
1668		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1669		_mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A)
1670		{
	1563	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1564	_mm512_maskz_mov_epi16(__mmask32 __U, __m512i __A) {
1671	1565	return (__m512i) __builtin_ia32_selectw_512 ((__mmask32) __U,
1672	1566	(__v32hi) __A,
1673	1567	(__v32hi) _mm512_setzero_si512 ());
1674	1568	}
1675	1569
1676		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1677		_mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A)
1678		{
	1570	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1571	_mm512_mask_mov_epi8(__m512i __W, __mmask64 __U, __m512i __A) {
1679	1572	return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U,
1680	1573	(__v64qi) __A,
1681	1574	(__v64qi) __W);
1682	1575	}
1683	1576
1684		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1577	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1685	1578	_mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A)
1686	1579	{
1687	1580	return (__m512i) __builtin_ia32_selectb_512 ((__mmask64) __U,
...	...	@@ -1689,7 +1582,7 @@ _mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A)
1689	1582	(__v64qi) _mm512_setzero_si512 ());
1690	1583	}
1691	1584
1692		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1585	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1693	1586	_mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A)
1694	1587	{
1695	1588	return (__m512i) __builtin_ia32_selectb_512(__M,
...	...	@@ -1697,23 +1590,21 @@ _mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A)
1697	1590	(__v64qi) __O);
1698	1591	}
1699	1592
1700		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1701		_mm512_maskz_set1_epi8 (__mmask64 __M, char __A)
1702		{
	1593	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1594	_mm512_maskz_set1_epi8(__mmask64 __M, char __A) {
1703	1595	return (__m512i) __builtin_ia32_selectb_512(__M,
1704	1596	(__v64qi) _mm512_set1_epi8(__A),
1705	1597	(__v64qi) _mm512_setzero_si512());
1706	1598	}
1707	1599
1708		static __inline__ __mmask64 __DEFAULT_FN_ATTRS _mm512_kunpackd(__mmask64 __A,
1709		__mmask64 __B) {
	1600	static __inline__ __mmask64 __DEFAULT_FN_ATTRS_CONSTEXPR
	1601	_mm512_kunpackd(__mmask64 __A, __mmask64 __B) {
1710	1602	return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A,
1711	1603	(__mmask64) __B);
1712	1604	}
1713	1605
1714		static __inline__ __mmask32 __DEFAULT_FN_ATTRS
1715		_mm512_kunpackw (__mmask32 __A, __mmask32 __B)
1716		{
	1606	static __inline__ __mmask32 __DEFAULT_FN_ATTRS_CONSTEXPR
	1607	_mm512_kunpackw(__mmask32 __A, __mmask32 __B) {
1717	1608	return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A,
1718	1609	(__mmask32) __B);
1719	1610	}
...	...	@@ -1859,33 +1750,28 @@ _mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B)
1859	1750	_mm512_setzero_si512());
1860	1751	}
1861	1752
1862		static __inline__ __mmask64 __DEFAULT_FN_ATTRS512
1863		_mm512_movepi8_mask (__m512i __A)
1864		{
	1753	static __inline__ __mmask64 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1754	_mm512_movepi8_mask(__m512i __A) {
1865	1755	return (__mmask64) __builtin_ia32_cvtb2mask512 ((__v64qi) __A);
1866	1756	}
1867	1757
1868		static __inline__ __mmask32 __DEFAULT_FN_ATTRS512
1869		_mm512_movepi16_mask (__m512i __A)
1870		{
	1758	static __inline__ __mmask32 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1759	_mm512_movepi16_mask(__m512i __A) {
1871	1760	return (__mmask32) __builtin_ia32_cvtw2mask512 ((__v32hi) __A);
1872	1761	}
1873	1762
1874		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1875		_mm512_movm_epi8 (__mmask64 __A)
1876		{
	1763	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1764	_mm512_movm_epi8(__mmask64 __A) {
1877	1765	return (__m512i) __builtin_ia32_cvtmask2b512 (__A);
1878	1766	}
1879	1767
1880		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1881		_mm512_movm_epi16 (__mmask32 __A)
1882		{
	1768	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1769	_mm512_movm_epi16(__mmask32 __A) {
1883	1770	return (__m512i) __builtin_ia32_cvtmask2w512 (__A);
1884	1771	}
1885	1772
1886		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1887		_mm512_broadcastb_epi8 (__m128i __A)
1888		{
	1773	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1774	_mm512_broadcastb_epi8(__m128i __A) {
1889	1775	return (__m512i)__builtin_shufflevector((__v16qi) __A, (__v16qi) __A,
1890	1776	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1891	1777	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
...	...	@@ -1893,7 +1779,7 @@ _mm512_broadcastb_epi8 (__m128i __A)
1893	1779	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
1894	1780	}
1895	1781
1896		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1782	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1897	1783	_mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A)
1898	1784	{
1899	1785	return (__m512i)__builtin_ia32_selectb_512(__M,
...	...	@@ -1901,15 +1787,14 @@ _mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A)
1901	1787	(__v64qi) __O);
1902	1788	}
1903	1789
1904		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1905		_mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A)
1906		{
	1790	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1791	_mm512_maskz_broadcastb_epi8(__mmask64 __M, __m128i __A) {
1907	1792	return (__m512i)__builtin_ia32_selectb_512(__M,
1908	1793	(__v64qi) _mm512_broadcastb_epi8(__A),
1909	1794	(__v64qi) _mm512_setzero_si512());
1910	1795	}
1911	1796
1912		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1797	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1913	1798	_mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A)
1914	1799	{
1915	1800	return (__m512i) __builtin_ia32_selectw_512(__M,
...	...	@@ -1917,23 +1802,21 @@ _mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A)
1917	1802	(__v32hi) __O);
1918	1803	}
1919	1804
1920		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1921		_mm512_maskz_set1_epi16 (__mmask32 __M, short __A)
1922		{
	1805	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1806	_mm512_maskz_set1_epi16(__mmask32 __M, short __A) {
1923	1807	return (__m512i) __builtin_ia32_selectw_512(__M,
1924	1808	(__v32hi) _mm512_set1_epi16(__A),
1925	1809	(__v32hi) _mm512_setzero_si512());
1926	1810	}
1927	1811
1928		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1929		_mm512_broadcastw_epi16 (__m128i __A)
1930		{
	1812	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1813	_mm512_broadcastw_epi16(__m128i __A) {
1931	1814	return (__m512i)__builtin_shufflevector((__v8hi) __A, (__v8hi) __A,
1932	1815	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1933	1816	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
1934	1817	}
1935	1818
1936		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1819	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1937	1820	_mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A)
1938	1821	{
1939	1822	return (__m512i)__builtin_ia32_selectw_512(__M,
...	...	@@ -1941,7 +1824,7 @@ _mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A)
1941	1824	(__v32hi) __O);
1942	1825	}
1943	1826
1944		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1827	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1945	1828	_mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A)
1946	1829	{
1947	1830	return (__m512i)__builtin_ia32_selectw_512(__M,
...	...	@@ -1949,25 +1832,21 @@ _mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A)
1949	1832	(__v32hi) _mm512_setzero_si512());
1950	1833	}
1951	1834
1952		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1953		_mm512_permutexvar_epi16 (__m512i __A, __m512i __B)
1954		{
	1835	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1836	_mm512_permutexvar_epi16(__m512i __A, __m512i __B) {
1955	1837	return (__m512i)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A);
1956	1838	}
1957	1839
1958		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1959		_mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A,
1960		__m512i __B)
1961		{
	1840	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1841	_mm512_maskz_permutexvar_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
1962	1842	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
1963	1843	(__v32hi)_mm512_permutexvar_epi16(__A, __B),
1964	1844	(__v32hi)_mm512_setzero_si512());
1965	1845	}
1966	1846
1967		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1968		_mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A,
1969		__m512i __B)
1970		{
	1847	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1848	_mm512_mask_permutexvar_epi16(__m512i __W, __mmask32 __M, __m512i __A,
	1849	__m512i __B) {
1971	1850	return (__m512i)__builtin_ia32_selectw_512((__mmask32)__M,
1972	1851	(__v32hi)_mm512_permutexvar_epi16(__A, __B),
1973	1852	(__v32hi)__W);
...	...	@@ -2010,5 +1889,7 @@ _mm512_sad_epu8 (__m512i __A, __m512i __B)
2010	1889
2011	1890	#undef __DEFAULT_FN_ATTRS512
2012	1891	#undef __DEFAULT_FN_ATTRS
	1892	#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
	1893	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
2013	1894
2014	1895	#endif

lib/include/avx512cdintrin.h+42-55

...	...	@@ -15,109 +15,96 @@
15	15	#define __AVX512CDINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512cd,evex512"), __min_vector_width__(512)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx512cd"), \
	21	__min_vector_width__(512))) constexpr
	22	#else
	23	#define __DEFAULT_FN_ATTRS \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx512cd"), \
	25	__min_vector_width__(512)))
	26	#endif
21	27
22	28	static __inline__ __m512i __DEFAULT_FN_ATTRS
23		_mm512_conflict_epi64 (__m512i __A)
24		{
25		return (__m512i) __builtin_ia32_vpconflictdi_512 ((__v8di) __A);
	29	_mm512_conflict_epi64(__m512i __A) {
	30	return (__m512i)__builtin_ia32_vpconflictdi_512((__v8di)__A);
26	31	}
27	32
28	33	static __inline__ __m512i __DEFAULT_FN_ATTRS
29		_mm512_mask_conflict_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
30		{
31		return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
32		(__v8di)_mm512_conflict_epi64(__A),
33		(__v8di)__W);
	34	_mm512_mask_conflict_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
	35	return (__m512i)__builtin_ia32_selectq_512(
	36	(__mmask8)__U, (__v8di)_mm512_conflict_epi64(__A), (__v8di)__W);
34	37	}
35	38
36	39	static __inline__ __m512i __DEFAULT_FN_ATTRS
37		_mm512_maskz_conflict_epi64 (__mmask8 __U, __m512i __A)
38		{
	40	_mm512_maskz_conflict_epi64(__mmask8 __U, __m512i __A) {
39	41	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
40	42	(__v8di)_mm512_conflict_epi64(__A),
41		(__v8di)_mm512_setzero_si512 ());
	43	(__v8di)_mm512_setzero_si512());
42	44	}
43	45
44	46	static __inline__ __m512i __DEFAULT_FN_ATTRS
45		_mm512_conflict_epi32 (__m512i __A)
46		{
47		return (__m512i) __builtin_ia32_vpconflictsi_512 ((__v16si) __A);
	47	_mm512_conflict_epi32(__m512i __A) {
	48	return (__m512i)__builtin_ia32_vpconflictsi_512((__v16si)__A);
48	49	}
49	50
50	51	static __inline__ __m512i __DEFAULT_FN_ATTRS
51		_mm512_mask_conflict_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
52		{
53		return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
54		(__v16si)_mm512_conflict_epi32(__A),
55		(__v16si)__W);
	52	_mm512_mask_conflict_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
	53	return (__m512i)__builtin_ia32_selectd_512(
	54	(__mmask16)__U, (__v16si)_mm512_conflict_epi32(__A), (__v16si)__W);
56	55	}
57	56
58	57	static __inline__ __m512i __DEFAULT_FN_ATTRS
59		_mm512_maskz_conflict_epi32 (__mmask16 __U, __m512i __A)
60		{
61		return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
62		(__v16si)_mm512_conflict_epi32(__A),
63		(__v16si)_mm512_setzero_si512());
	58	_mm512_maskz_conflict_epi32(__mmask16 __U, __m512i __A) {
	59	return (__m512i)__builtin_ia32_selectd_512(
	60	(__mmask16)__U, (__v16si)_mm512_conflict_epi32(__A),
	61	(__v16si)_mm512_setzero_si512());
64	62	}
65	63
66		static __inline__ __m512i __DEFAULT_FN_ATTRS
67		_mm512_lzcnt_epi32 (__m512i __A)
68		{
69		return (__m512i) __builtin_ia32_vplzcntd_512 ((__v16si) __A);
	64	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_lzcnt_epi32(__m512i __A) {
	65	return (__m512i)__builtin_elementwise_clzg((__v16si)__A,
	66	(__v16si)_mm512_set1_epi32(32));
70	67	}
71	68
72	69	static __inline__ __m512i __DEFAULT_FN_ATTRS
73		_mm512_mask_lzcnt_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
74		{
75		return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
76		(__v16si)_mm512_lzcnt_epi32(__A),
77		(__v16si)__W);
	70	_mm512_mask_lzcnt_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
	71	return (__m512i)__builtin_ia32_selectd_512(
	72	(__mmask16)__U, (__v16si)_mm512_lzcnt_epi32(__A), (__v16si)__W);
78	73	}
79	74
80	75	static __inline__ __m512i __DEFAULT_FN_ATTRS
81		_mm512_maskz_lzcnt_epi32 (__mmask16 __U, __m512i __A)
82		{
	76	_mm512_maskz_lzcnt_epi32(__mmask16 __U, __m512i __A) {
83	77	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
84	78	(__v16si)_mm512_lzcnt_epi32(__A),
85	79	(__v16si)_mm512_setzero_si512());
86	80	}
87	81
88		static __inline__ __m512i __DEFAULT_FN_ATTRS
89		_mm512_lzcnt_epi64 (__m512i __A)
90		{
91		return (__m512i) __builtin_ia32_vplzcntq_512 ((__v8di) __A);
	82	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_lzcnt_epi64(__m512i __A) {
	83	return (__m512i)__builtin_elementwise_clzg(
	84	(__v8di)__A, (__v8di)_mm512_set1_epi64((long long)64));
92	85	}
93	86
94	87	static __inline__ __m512i __DEFAULT_FN_ATTRS
95		_mm512_mask_lzcnt_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
96		{
97		return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
98		(__v8di)_mm512_lzcnt_epi64(__A),
99		(__v8di)__W);
	88	_mm512_mask_lzcnt_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
	89	return (__m512i)__builtin_ia32_selectq_512(
	90	(__mmask8)__U, (__v8di)_mm512_lzcnt_epi64(__A), (__v8di)__W);
100	91	}
101	92
102	93	static __inline__ __m512i __DEFAULT_FN_ATTRS
103		_mm512_maskz_lzcnt_epi64 (__mmask8 __U, __m512i __A)
104		{
	94	_mm512_maskz_lzcnt_epi64(__mmask8 __U, __m512i __A) {
105	95	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
106	96	(__v8di)_mm512_lzcnt_epi64(__A),
107	97	(__v8di)_mm512_setzero_si512());
108	98	}
109	99
110	100	static __inline__ __m512i __DEFAULT_FN_ATTRS
111		_mm512_broadcastmb_epi64 (__mmask8 __A)
112		{
113		return (__m512i) _mm512_set1_epi64((long long) __A);
	101	_mm512_broadcastmb_epi64(__mmask8 __A) {
	102	return (__m512i)_mm512_set1_epi64((long long)__A);
114	103	}
115	104
116	105	static __inline__ __m512i __DEFAULT_FN_ATTRS
117		_mm512_broadcastmw_epi32 (__mmask16 __A)
118		{
119		return (__m512i) _mm512_set1_epi32((int) __A);
120
	106	_mm512_broadcastmw_epi32(__mmask16 __A) {
	107	return (__m512i)_mm512_set1_epi32((int)__A);
121	108	}
122	109
123	110	#undef __DEFAULT_FN_ATTRS

lib/include/avx512dqintrin.h+132-161

...	...	@@ -15,110 +15,105 @@
15	15	#define __AVX512DQINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
18		#define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512dq,evex512"), __min_vector_width__(512)))
	18	#define __DEFAULT_FN_ATTRS512 \
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx512dq"), \
	20	__min_vector_width__(512)))
19	21	#define __DEFAULT_FN_ATTRS \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("avx512dq,no-evex512")))
	22	__attribute__((__always_inline__, __nodebug__, __target__("avx512dq")))
	23
	24	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	25	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
	26	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	27	#else
	28	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
	29	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	30	#endif
22	31
23		static __inline __mmask8 __DEFAULT_FN_ATTRS
24		_knot_mask8(__mmask8 __M)
25		{
	32	static __inline __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	33	_knot_mask8(__mmask8 __M) {
26	34	return __builtin_ia32_knotqi(__M);
27	35	}
28	36
29		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
30		_kand_mask8(__mmask8 __A, __mmask8 __B)
31		{
	37	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	38	_kand_mask8(__mmask8 __A, __mmask8 __B) {
32	39	return (__mmask8)__builtin_ia32_kandqi((__mmask8)__A, (__mmask8)__B);
33	40	}
34	41
35		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
36		_kandn_mask8(__mmask8 __A, __mmask8 __B)
37		{
	42	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	43	_kandn_mask8(__mmask8 __A, __mmask8 __B) {
38	44	return (__mmask8)__builtin_ia32_kandnqi((__mmask8)__A, (__mmask8)__B);
39	45	}
40	46
41		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
42		_kor_mask8(__mmask8 __A, __mmask8 __B)
43		{
	47	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	48	_kor_mask8(__mmask8 __A, __mmask8 __B) {
44	49	return (__mmask8)__builtin_ia32_korqi((__mmask8)__A, (__mmask8)__B);
45	50	}
46	51
47		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
48		_kxnor_mask8(__mmask8 __A, __mmask8 __B)
49		{
	52	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	53	_kxnor_mask8(__mmask8 __A, __mmask8 __B) {
50	54	return (__mmask8)__builtin_ia32_kxnorqi((__mmask8)__A, (__mmask8)__B);
51	55	}
52	56
53		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
54		_kxor_mask8(__mmask8 __A, __mmask8 __B)
55		{
	57	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	58	_kxor_mask8(__mmask8 __A, __mmask8 __B) {
56	59	return (__mmask8)__builtin_ia32_kxorqi((__mmask8)__A, (__mmask8)__B);
57	60	}
58	61
59		static __inline__ unsigned char __DEFAULT_FN_ATTRS
60		_kortestc_mask8_u8(__mmask8 __A, __mmask8 __B)
61		{
	62	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	63	_kortestc_mask8_u8(__mmask8 __A, __mmask8 __B) {
62	64	return (unsigned char)__builtin_ia32_kortestcqi(__A, __B);
63	65	}
64	66
65		static __inline__ unsigned char __DEFAULT_FN_ATTRS
66		_kortestz_mask8_u8(__mmask8 __A, __mmask8 __B)
67		{
	67	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	68	_kortestz_mask8_u8(__mmask8 __A, __mmask8 __B) {
68	69	return (unsigned char)__builtin_ia32_kortestzqi(__A, __B);
69	70	}
70	71
71		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	72	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
72	73	_kortest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) {
73	74	*__C = (unsigned char)__builtin_ia32_kortestcqi(__A, __B);
74	75	return (unsigned char)__builtin_ia32_kortestzqi(__A, __B);
75	76	}
76	77
77		static __inline__ unsigned char __DEFAULT_FN_ATTRS
78		_ktestc_mask8_u8(__mmask8 __A, __mmask8 __B)
79		{
	78	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	79	_ktestc_mask8_u8(__mmask8 __A, __mmask8 __B) {
80	80	return (unsigned char)__builtin_ia32_ktestcqi(__A, __B);
81	81	}
82	82
83		static __inline__ unsigned char __DEFAULT_FN_ATTRS
84		_ktestz_mask8_u8(__mmask8 __A, __mmask8 __B)
85		{
	83	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	84	_ktestz_mask8_u8(__mmask8 __A, __mmask8 __B) {
86	85	return (unsigned char)__builtin_ia32_ktestzqi(__A, __B);
87	86	}
88	87
89		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	88	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
90	89	_ktest_mask8_u8(__mmask8 __A, __mmask8 __B, unsigned char *__C) {
91	90	*__C = (unsigned char)__builtin_ia32_ktestcqi(__A, __B);
92	91	return (unsigned char)__builtin_ia32_ktestzqi(__A, __B);
93	92	}
94	93
95		static __inline__ unsigned char __DEFAULT_FN_ATTRS
96		_ktestc_mask16_u8(__mmask16 __A, __mmask16 __B)
97		{
	94	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	95	_ktestc_mask16_u8(__mmask16 __A, __mmask16 __B) {
98	96	return (unsigned char)__builtin_ia32_ktestchi(__A, __B);
99	97	}
100	98
101		static __inline__ unsigned char __DEFAULT_FN_ATTRS
102		_ktestz_mask16_u8(__mmask16 __A, __mmask16 __B)
103		{
	99	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	100	_ktestz_mask16_u8(__mmask16 __A, __mmask16 __B) {
104	101	return (unsigned char)__builtin_ia32_ktestzhi(__A, __B);
105	102	}
106	103
107		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	104	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
108	105	_ktest_mask16_u8(__mmask16 __A, __mmask16 __B, unsigned char *__C) {
109	106	*__C = (unsigned char)__builtin_ia32_ktestchi(__A, __B);
110	107	return (unsigned char)__builtin_ia32_ktestzhi(__A, __B);
111	108	}
112	109
113		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
114		_kadd_mask8(__mmask8 __A, __mmask8 __B)
115		{
	110	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
	111	_kadd_mask8(__mmask8 __A, __mmask8 __B) {
116	112	return (__mmask8)__builtin_ia32_kaddqi((__mmask8)__A, (__mmask8)__B);
117	113	}
118	114
119		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
120		_kadd_mask16(__mmask16 __A, __mmask16 __B)
121		{
	115	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	116	_kadd_mask16(__mmask16 __A, __mmask16 __B) {
122	117	return (__mmask16)__builtin_ia32_kaddhi((__mmask16)__A, (__mmask16)__B);
123	118	}
124	119
...	...	@@ -128,12 +123,12 @@ _kadd_mask16(__mmask16 __A, __mmask16 __B)
128	123	#define _kshiftri_mask8(A, I) \
129	124	((__mmask8)__builtin_ia32_kshiftriqi((__mmask8)(A), (unsigned int)(I)))
130	125
131		static __inline__ unsigned int __DEFAULT_FN_ATTRS
132		_cvtmask8_u32(__mmask8 __A) {
	126	static __inline__ unsigned int
	127	__DEFAULT_FN_ATTRS_CONSTEXPR _cvtmask8_u32(__mmask8 __A) {
133	128	return (unsigned int)__builtin_ia32_kmovb((__mmask8)__A);
134	129	}
135	130
136		static __inline__ __mmask8 __DEFAULT_FN_ATTRS
	131	static __inline__ __mmask8 __DEFAULT_FN_ATTRS_CONSTEXPR
137	132	_cvtu32_mask8(unsigned int __A) {
138	133	return (__mmask8)__builtin_ia32_kmovb((__mmask8)__A);
139	134	}
...	...	@@ -148,26 +143,26 @@ _store_mask8(__mmask8 *__A, __mmask8 __B) {
148	143	(__mmask8 )__A = __builtin_ia32_kmovb((__mmask8)__B);
149	144	}
150	145
151		static __inline__ __m512i __DEFAULT_FN_ATTRS512
152		_mm512_mullo_epi64 (__m512i __A, __m512i __B) {
	146	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	147	_mm512_mullo_epi64(__m512i __A, __m512i __B) {
153	148	return (__m512i) ((__v8du) __A * (__v8du) __B);
154	149	}
155	150
156		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	151	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
157	152	_mm512_mask_mullo_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
158	153	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
159	154	(__v8di)_mm512_mullo_epi64(__A, __B),
160	155	(__v8di)__W);
161	156	}
162	157
163		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	158	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
164	159	_mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
165	160	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
166	161	(__v8di)_mm512_mullo_epi64(__A, __B),
167	162	(__v8di)_mm512_setzero_si512());
168	163	}
169	164
170		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	165	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
171	166	_mm512_xor_pd(__m512d __A, __m512d __B) {
172	167	return (__m512d)((__v8du)__A ^ (__v8du)__B);
173	168	}
...	...	@@ -186,7 +181,7 @@ _mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) {
186	181	(__v8df)_mm512_setzero_pd());
187	182	}
188	183
189		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	184	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
190	185	_mm512_xor_ps (__m512 __A, __m512 __B) {
191	186	return (__m512)((__v16su)__A ^ (__v16su)__B);
192	187	}
...	...	@@ -205,7 +200,7 @@ _mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) {
205	200	(__v16sf)_mm512_setzero_ps());
206	201	}
207	202
208		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	203	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
209	204	_mm512_or_pd(__m512d __A, __m512d __B) {
210	205	return (__m512d)((__v8du)__A \| (__v8du)__B);
211	206	}
...	...	@@ -224,7 +219,7 @@ _mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) {
224	219	(__v8df)_mm512_setzero_pd());
225	220	}
226	221
227		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	222	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
228	223	_mm512_or_ps(__m512 __A, __m512 __B) {
229	224	return (__m512)((__v16su)__A \| (__v16su)__B);
230	225	}
...	...	@@ -243,7 +238,7 @@ _mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) {
243	238	(__v16sf)_mm512_setzero_ps());
244	239	}
245	240
246		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	241	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
247	242	_mm512_and_pd(__m512d __A, __m512d __B) {
248	243	return (__m512d)((__v8du)__A & (__v8du)__B);
249	244	}
...	...	@@ -262,7 +257,7 @@ _mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) {
262	257	(__v8df)_mm512_setzero_pd());
263	258	}
264	259
265		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	260	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
266	261	_mm512_and_ps(__m512 __A, __m512 __B) {
267	262	return (__m512)((__v16su)__A & (__v16su)__B);
268	263	}
...	...	@@ -281,7 +276,7 @@ _mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) {
281	276	(__v16sf)_mm512_setzero_ps());
282	277	}
283	278
284		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	279	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
285	280	_mm512_andnot_pd(__m512d __A, __m512d __B) {
286	281	return (__m512d)(~(__v8du)__A & (__v8du)__B);
287	282	}
...	...	@@ -300,7 +295,7 @@ _mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) {
300	295	(__v8df)_mm512_setzero_pd());
301	296	}
302	297
303		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	298	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
304	299	_mm512_andnot_ps(__m512 __A, __m512 __B) {
305	300	return (__m512)(~(__v16su)__A & (__v16su)__B);
306	301	}
...	...	@@ -475,21 +470,20 @@ _mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) {
475	470	(__v8di)_mm512_setzero_si512(), \
476	471	(__mmask8)(U), (int)(R)))
477	472
478
479		static __inline__ __m512d __DEFAULT_FN_ATTRS512
480		_mm512_cvtepi64_pd (__m512i __A) {
	473	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	474	_mm512_cvtepi64_pd(__m512i __A) {
481	475	return (__m512d)__builtin_convertvector((__v8di)__A, __v8df);
482	476	}
483	477
484		static __inline__ __m512d __DEFAULT_FN_ATTRS512
485		_mm512_mask_cvtepi64_pd (__m512d __W, __mmask8 __U, __m512i __A) {
	478	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	479	_mm512_mask_cvtepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
486	480	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
487	481	(__v8df)_mm512_cvtepi64_pd(__A),
488	482	(__v8df)__W);
489	483	}
490	484
491		static __inline__ __m512d __DEFAULT_FN_ATTRS512
492		_mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) {
	485	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	486	_mm512_maskz_cvtepi64_pd(__mmask8 __U, __m512i __A) {
493	487	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
494	488	(__v8df)_mm512_cvtepi64_pd(__A),
495	489	(__v8df)_mm512_setzero_pd());
...	...	@@ -706,20 +700,20 @@ _mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) {
706	700	(__v8di)_mm512_setzero_si512(), \
707	701	(__mmask8)(U), (int)(R)))
708	702
709		static __inline__ __m512d __DEFAULT_FN_ATTRS512
710		_mm512_cvtepu64_pd (__m512i __A) {
	703	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	704	_mm512_cvtepu64_pd(__m512i __A) {
711	705	return (__m512d)__builtin_convertvector((__v8du)__A, __v8df);
712	706	}
713	707
714		static __inline__ __m512d __DEFAULT_FN_ATTRS512
715		_mm512_mask_cvtepu64_pd (__m512d __W, __mmask8 __U, __m512i __A) {
	708	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	709	_mm512_mask_cvtepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
716	710	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
717	711	(__v8df)_mm512_cvtepu64_pd(__A),
718	712	(__v8df)__W);
719	713	}
720	714
721		static __inline__ __m512d __DEFAULT_FN_ATTRS512
722		_mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) {
	715	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	716	_mm512_maskz_cvtepu64_pd(__mmask8 __U, __m512i __A) {
723	717	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
724	718	(__v8df)_mm512_cvtepu64_pd(__A),
725	719	(__v8df)_mm512_setzero_pd());
...	...	@@ -1052,177 +1046,154 @@ _mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) {
1052	1046	(__v2df)_mm_setzero_pd(), \
1053	1047	(__mmask8)(U), (int)(C), (int)(R)))
1054	1048
1055		static __inline__ __mmask16 __DEFAULT_FN_ATTRS512
1056		_mm512_movepi32_mask (__m512i __A)
1057		{
	1049	static __inline__ __mmask16
	1050	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_movepi32_mask(__m512i __A) {
1058	1051	return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A);
1059	1052	}
1060	1053
1061		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1062		_mm512_movm_epi32 (__mmask16 __A)
1063		{
	1054	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1055	_mm512_movm_epi32(__mmask16 __A) {
1064	1056	return (__m512i) __builtin_ia32_cvtmask2d512 (__A);
1065	1057	}
1066	1058
1067		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1068		_mm512_movm_epi64 (__mmask8 __A)
1069		{
	1059	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1060	_mm512_movm_epi64(__mmask8 __A) {
1070	1061	return (__m512i) __builtin_ia32_cvtmask2q512 (__A);
1071	1062	}
1072	1063
1073		static __inline__ __mmask8 __DEFAULT_FN_ATTRS512
1074		_mm512_movepi64_mask (__m512i __A)
1075		{
	1064	static __inline__ __mmask8 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1065	_mm512_movepi64_mask(__m512i __A) {
1076	1066	return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A);
1077	1067	}
1078	1068
1079
1080		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1081		_mm512_broadcast_f32x2 (__m128 __A)
1082		{
	1069	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1070	_mm512_broadcast_f32x2(__m128 __A) {
1083	1071	return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
1084	1072	0, 1, 0, 1, 0, 1, 0, 1,
1085	1073	0, 1, 0, 1, 0, 1, 0, 1);
1086	1074	}
1087	1075
1088		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1089		_mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A)
1090		{
	1076	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1077	_mm512_mask_broadcast_f32x2(__m512 __O, __mmask16 __M, __m128 __A) {
1091	1078	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
1092	1079	(__v16sf)_mm512_broadcast_f32x2(__A),
1093	1080	(__v16sf)__O);
1094	1081	}
1095	1082
1096		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1097		_mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A)
1098		{
	1083	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1084	_mm512_maskz_broadcast_f32x2(__mmask16 __M, __m128 __A) {
1099	1085	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
1100	1086	(__v16sf)_mm512_broadcast_f32x2(__A),
1101	1087	(__v16sf)_mm512_setzero_ps());
1102	1088	}
1103	1089
1104		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1105		_mm512_broadcast_f32x8(__m256 __A)
1106		{
	1090	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1091	_mm512_broadcast_f32x8(__m256 __A) {
1107	1092	return (__m512)__builtin_shufflevector((__v8sf)__A, (__v8sf)__A,
1108	1093	0, 1, 2, 3, 4, 5, 6, 7,
1109	1094	0, 1, 2, 3, 4, 5, 6, 7);
1110	1095	}
1111	1096
1112		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1113		_mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A)
1114		{
	1097	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1098	_mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, __m256 __A) {
1115	1099	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
1116	1100	(__v16sf)_mm512_broadcast_f32x8(__A),
1117	1101	(__v16sf)__O);
1118	1102	}
1119	1103
1120		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1121		_mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A)
1122		{
	1104	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1105	_mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A) {
1123	1106	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
1124	1107	(__v16sf)_mm512_broadcast_f32x8(__A),
1125	1108	(__v16sf)_mm512_setzero_ps());
1126	1109	}
1127	1110
1128		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1129		_mm512_broadcast_f64x2(__m128d __A)
1130		{
	1111	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	1112	_mm512_broadcast_f64x2(__m128d __A) {
1131	1113	return (__m512d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,
1132	1114	0, 1, 0, 1, 0, 1, 0, 1);
1133	1115	}
1134	1116
1135		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1136		_mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A)
1137		{
	1117	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	1118	_mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, __m128d __A) {
1138	1119	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,
1139	1120	(__v8df)_mm512_broadcast_f64x2(__A),
1140	1121	(__v8df)__O);
1141	1122	}
1142	1123
1143		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1144		_mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A)
1145		{
	1124	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	1125	_mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) {
1146	1126	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__M,
1147	1127	(__v8df)_mm512_broadcast_f64x2(__A),
1148	1128	(__v8df)_mm512_setzero_pd());
1149	1129	}
1150	1130
1151		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1152		_mm512_broadcast_i32x2 (__m128i __A)
1153		{
	1131	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1132	_mm512_broadcast_i32x2(__m128i __A) {
1154	1133	return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
1155	1134	0, 1, 0, 1, 0, 1, 0, 1,
1156	1135	0, 1, 0, 1, 0, 1, 0, 1);
1157	1136	}
1158	1137
1159		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1160		_mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A)
1161		{
	1138	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1139	_mm512_mask_broadcast_i32x2(__m512i __O, __mmask16 __M, __m128i __A) {
1162	1140	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1163	1141	(__v16si)_mm512_broadcast_i32x2(__A),
1164	1142	(__v16si)__O);
1165	1143	}
1166	1144
1167		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1168		_mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A)
1169		{
	1145	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1146	_mm512_maskz_broadcast_i32x2(__mmask16 __M, __m128i __A) {
1170	1147	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1171	1148	(__v16si)_mm512_broadcast_i32x2(__A),
1172	1149	(__v16si)_mm512_setzero_si512());
1173	1150	}
1174	1151
1175		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1176		_mm512_broadcast_i32x8(__m256i __A)
1177		{
	1152	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1153	_mm512_broadcast_i32x8(__m256i __A) {
1178	1154	return (__m512i)__builtin_shufflevector((__v8si)__A, (__v8si)__A,
1179	1155	0, 1, 2, 3, 4, 5, 6, 7,
1180	1156	0, 1, 2, 3, 4, 5, 6, 7);
1181	1157	}
1182	1158
1183		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1184		_mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A)
1185		{
	1159	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1160	_mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i __A) {
1186	1161	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1187	1162	(__v16si)_mm512_broadcast_i32x8(__A),
1188	1163	(__v16si)__O);
1189	1164	}
1190	1165
1191		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1192		_mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A)
1193		{
	1166	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1167	_mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A) {
1194	1168	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1195	1169	(__v16si)_mm512_broadcast_i32x8(__A),
1196	1170	(__v16si)_mm512_setzero_si512());
1197	1171	}
1198	1172
1199		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1200		_mm512_broadcast_i64x2(__m128i __A)
1201		{
	1173	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1174	_mm512_broadcast_i64x2(__m128i __A) {
1202	1175	return (__m512i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,
1203	1176	0, 1, 0, 1, 0, 1, 0, 1);
1204	1177	}
1205	1178
1206		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1207		_mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A)
1208		{
	1179	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1180	_mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i __A) {
1209	1181	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1210	1182	(__v8di)_mm512_broadcast_i64x2(__A),
1211	1183	(__v8di)__O);
1212	1184	}
1213	1185
1214		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1215		_mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
1216		{
	1186	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1187	_mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) {
1217	1188	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1218	1189	(__v8di)_mm512_broadcast_i64x2(__A),
1219	1190	(__v8di)_mm512_setzero_si512());
1220	1191	}
1221	1192
1222		#define _mm512_extractf32x8_ps(A, imm) \
1223		((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \
1224		(__v8sf)_mm256_undefined_ps(), \
1225		(__mmask8)-1))
	1193	#define _mm512_extractf32x8_ps(A, imm) \
	1194	((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \
	1195	(__v8sf)_mm256_setzero_ps(), \
	1196	(__mmask8) - 1))
1226	1197
1227	1198	#define _mm512_mask_extractf32x8_ps(W, U, A, imm) \
1228	1199	((__m256)__builtin_ia32_extractf32x8_mask((__v16sf)(__m512)(A), (int)(imm), \
...	...	@@ -1234,11 +1205,10 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
1234	1205	(__v8sf)_mm256_setzero_ps(), \
1235	1206	(__mmask8)(U)))
1236	1207
1237		#define _mm512_extractf64x2_pd(A, imm) \
1238		((__m128d)__builtin_ia32_extractf64x2_512_mask((__v8df)(__m512d)(A), \
1239		(int)(imm), \
1240		(__v2df)_mm_undefined_pd(), \
1241		(__mmask8)-1))
	1208	#define _mm512_extractf64x2_pd(A, imm) \
	1209	((__m128d)__builtin_ia32_extractf64x2_512_mask( \
	1210	(__v8df)(__m512d)(A), (int)(imm), (__v2df)_mm_setzero_pd(), \
	1211	(__mmask8) - 1))
1242	1212
1243	1213	#define _mm512_mask_extractf64x2_pd(W, U, A, imm) \
1244	1214	((__m128d)__builtin_ia32_extractf64x2_512_mask((__v8df)(__m512d)(A), \
...	...	@@ -1252,10 +1222,10 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
1252	1222	(__v2df)_mm_setzero_pd(), \
1253	1223	(__mmask8)(U)))
1254	1224
1255		#define _mm512_extracti32x8_epi32(A, imm) \
1256		((__m256i)__builtin_ia32_extracti32x8_mask((__v16si)(__m512i)(A), (int)(imm), \
1257		(__v8si)_mm256_undefined_si256(), \
1258		(__mmask8)-1))
	1225	#define _mm512_extracti32x8_epi32(A, imm) \
	1226	((__m256i)__builtin_ia32_extracti32x8_mask( \
	1227	(__v16si)(__m512i)(A), (int)(imm), (__v8si)_mm256_setzero_si256(), \
	1228	(__mmask8) - 1))
1259	1229
1260	1230	#define _mm512_mask_extracti32x8_epi32(W, U, A, imm) \
1261	1231	((__m256i)__builtin_ia32_extracti32x8_mask((__v16si)(__m512i)(A), (int)(imm), \
...	...	@@ -1267,11 +1237,10 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
1267	1237	(__v8si)_mm256_setzero_si256(), \
1268	1238	(__mmask8)(U)))
1269	1239
1270		#define _mm512_extracti64x2_epi64(A, imm) \
1271		((__m128i)__builtin_ia32_extracti64x2_512_mask((__v8di)(__m512i)(A), \
1272		(int)(imm), \
1273		(__v2di)_mm_undefined_si128(), \
1274		(__mmask8)-1))
	1240	#define _mm512_extracti64x2_epi64(A, imm) \
	1241	((__m128i)__builtin_ia32_extracti64x2_512_mask( \
	1242	(__v8di)(__m512i)(A), (int)(imm), (__v2di)_mm_setzero_si128(), \
	1243	(__mmask8) - 1))
1275	1244
1276	1245	#define _mm512_mask_extracti64x2_epi64(W, U, A, imm) \
1277	1246	((__m128i)__builtin_ia32_extracti64x2_512_mask((__v8di)(__m512i)(A), \
...	...	@@ -1375,5 +1344,7 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
1375	1344
1376	1345	#undef __DEFAULT_FN_ATTRS512
1377	1346	#undef __DEFAULT_FN_ATTRS
	1347	#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
	1348	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
1378	1349
1379	1350	#endif

lib/include/avx512fintrin.h+1015-1424

...	...	@@ -167,22 +167,23 @@ typedef enum
167	167	} _MM_MANTISSA_SIGN_ENUM;
168	168
169	169	/* Define the default attributes for the functions in this file. */
170		#define __DEFAULT_FN_ATTRS512 __attribute__((__always_inline__, __nodebug__, __target__("avx512f,evex512"), __min_vector_width__(512)))
	170	#define __DEFAULT_FN_ATTRS512 \
	171	__attribute__((__always_inline__, __nodebug__, __target__("avx512f"), \
	172	__min_vector_width__(512)))
171	173	#define __DEFAULT_FN_ATTRS128 \
172		__attribute__((__always_inline__, __nodebug__, \
173		__target__("avx512f,no-evex512"), __min_vector_width__(128)))
	174	__attribute__((__always_inline__, __nodebug__, __target__("avx512f"), \
	175	__min_vector_width__(128)))
174	176	#define __DEFAULT_FN_ATTRS \
175		__attribute__((__always_inline__, __nodebug__, \
176		__target__("avx512f,no-evex512")))
	177	__attribute__((__always_inline__, __nodebug__, __target__("avx512f")))
177	178
178	179	#if defined(__cplusplus) && (__cplusplus >= 201103L)
179	180	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
180	181	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
181	182	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
182	183	#else
183		#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
	184	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
184	185	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
185		#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
	186	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
186	187	#endif
187	188
188	189	/* Create vectors with repeated elements */
...	...	@@ -206,9 +207,7 @@ _mm512_undefined(void)
206	207	return (__m512)__builtin_ia32_undef512();
207	208	}
208	209
209		static __inline__ __m512 __DEFAULT_FN_ATTRS512
210		_mm512_undefined_ps(void)
211		{
	210	static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_undefined_ps(void) {
212	211	return (__m512)__builtin_ia32_undef512();
213	212	}
214	213
...	...	@@ -218,48 +217,40 @@ _mm512_undefined_epi32(void)
218	217	return (__m512i)__builtin_ia32_undef512();
219	218	}
220	219
221		static __inline__ __m512i __DEFAULT_FN_ATTRS512
222		_mm512_broadcastd_epi32 (__m128i __A)
223		{
	220	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	221	_mm512_broadcastd_epi32(__m128i __A) {
224	222	return (__m512i)__builtin_shufflevector((__v4si) __A, (__v4si) __A,
225	223	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
226	224	}
227	225
228		static __inline__ __m512i __DEFAULT_FN_ATTRS512
229		_mm512_mask_broadcastd_epi32 (__m512i __O, __mmask16 __M, __m128i __A)
230		{
	226	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	227	_mm512_mask_broadcastd_epi32(__m512i __O, __mmask16 __M, __m128i __A) {
231	228	return (__m512i)__builtin_ia32_selectd_512(__M,
232	229	(__v16si) _mm512_broadcastd_epi32(__A),
233	230	(__v16si) __O);
234	231	}
235	232
236		static __inline__ __m512i __DEFAULT_FN_ATTRS512
237		_mm512_maskz_broadcastd_epi32 (__mmask16 __M, __m128i __A)
238		{
	233	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	234	_mm512_maskz_broadcastd_epi32(__mmask16 __M, __m128i __A) {
239	235	return (__m512i)__builtin_ia32_selectd_512(__M,
240	236	(__v16si) _mm512_broadcastd_epi32(__A),
241	237	(__v16si) _mm512_setzero_si512());
242	238	}
243	239
244		static __inline__ __m512i __DEFAULT_FN_ATTRS512
245		_mm512_broadcastq_epi64 (__m128i __A)
246		{
	240	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	241	_mm512_broadcastq_epi64(__m128i __A) {
247	242	return (__m512i)__builtin_shufflevector((__v2di) __A, (__v2di) __A,
248	243	0, 0, 0, 0, 0, 0, 0, 0);
249	244	}
250	245
251		static __inline__ __m512i __DEFAULT_FN_ATTRS512
252		_mm512_mask_broadcastq_epi64 (__m512i __O, __mmask8 __M, __m128i __A)
253		{
254		return (__m512i)__builtin_ia32_selectq_512(__M,
255		(__v8di) _mm512_broadcastq_epi64(__A),
256		(__v8di) __O);
257
	246	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	247	_mm512_mask_broadcastq_epi64(__m512i __O, __mmask8 __M, __m128i __A) {
	248	return (__m512i)__builtin_ia32_selectq_512(
	249	__M, (__v8di)_mm512_broadcastq_epi64(__A), (__v8di)__O);
258	250	}
259	251
260		static __inline__ __m512i __DEFAULT_FN_ATTRS512
261		_mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
262		{
	252	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	253	_mm512_maskz_broadcastq_epi64(__mmask8 __M, __m128i __A) {
263	254	return (__m512i)__builtin_ia32_selectq_512(__M,
264	255	(__v8di) _mm512_broadcastq_epi64(__A),
265	256	(__v8di) _mm512_setzero_si512());
...	...	@@ -277,20 +268,20 @@ _mm512_setzero_pd(void) {
277	268	return __extension__(__m512d){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
278	269	}
279	270
280		static __inline __m512 __DEFAULT_FN_ATTRS512
	271	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
281	272	_mm512_set1_ps(float __w)
282	273	{
283	274	return __extension__ (__m512){ __w, __w, __w, __w, __w, __w, __w, __w,
284	275	__w, __w, __w, __w, __w, __w, __w, __w };
285	276	}
286	277
287		static __inline __m512d __DEFAULT_FN_ATTRS512
	278	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
288	279	_mm512_set1_pd(double __w)
289	280	{
290	281	return __extension__ (__m512d){ __w, __w, __w, __w, __w, __w, __w, __w };
291	282	}
292	283
293		static __inline __m512i __DEFAULT_FN_ATTRS512
	284	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
294	285	_mm512_set1_epi8(char __w)
295	286	{
296	287	return __extension__ (__m512i)(__v64qi){
...	...	@@ -304,7 +295,7 @@ _mm512_set1_epi8(char __w)
304	295	__w, __w, __w, __w, __w, __w, __w, __w };
305	296	}
306	297
307		static __inline __m512i __DEFAULT_FN_ATTRS512
	298	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
308	299	_mm512_set1_epi16(short __w)
309	300	{
310	301	return __extension__ (__m512i)(__v32hi){
...	...	@@ -314,7 +305,7 @@ _mm512_set1_epi16(short __w)
314	305	__w, __w, __w, __w, __w, __w, __w, __w };
315	306	}
316	307
317		static __inline __m512i __DEFAULT_FN_ATTRS512
	308	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
318	309	_mm512_set1_epi32(int __s)
319	310	{
320	311	return __extension__ (__m512i)(__v16si){
...	...	@@ -322,81 +313,80 @@ _mm512_set1_epi32(int __s)
322	313	__s, __s, __s, __s, __s, __s, __s, __s };
323	314	}
324	315
325		static __inline __m512i __DEFAULT_FN_ATTRS512
326		_mm512_maskz_set1_epi32(__mmask16 __M, int __A)
327		{
	316	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	317	_mm512_maskz_set1_epi32(__mmask16 __M, int __A) {
328	318	return (__m512i)__builtin_ia32_selectd_512(__M,
329	319	(__v16si)_mm512_set1_epi32(__A),
330	320	(__v16si)_mm512_setzero_si512());
331	321	}
332	322
333		static __inline __m512i __DEFAULT_FN_ATTRS512
	323	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
334	324	_mm512_set1_epi64(long long __d)
335	325	{
336	326	return __extension__(__m512i)(__v8di){ __d, __d, __d, __d, __d, __d, __d, __d };
337	327	}
338	328
339		static __inline __m512i __DEFAULT_FN_ATTRS512
340		_mm512_maskz_set1_epi64(__mmask8 __M, long long __A)
341		{
	329	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	330	_mm512_maskz_set1_epi64(__mmask8 __M, long long __A) {
342	331	return (__m512i)__builtin_ia32_selectq_512(__M,
343	332	(__v8di)_mm512_set1_epi64(__A),
344	333	(__v8di)_mm512_setzero_si512());
345	334	}
346	335
347		static __inline__ __m512 __DEFAULT_FN_ATTRS512
348		_mm512_broadcastss_ps(__m128 __A)
349		{
	336	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	337	_mm512_broadcastss_ps(__m128 __A) {
350	338	return (__m512)__builtin_shufflevector((__v4sf) __A, (__v4sf) __A,
351	339	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
352	340	}
353	341
354		static __inline __m512i __DEFAULT_FN_ATTRS512
355		_mm512_set4_epi32 (int __A, int __B, int __C, int __D)
356		{
	342	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	343	_mm512_set4_epi32(int __A, int __B, int __C, int __D) {
357	344	return __extension__ (__m512i)(__v16si)
358	345	{ __D, __C, __B, __A, __D, __C, __B, __A,
359	346	__D, __C, __B, __A, __D, __C, __B, __A };
360	347	}
361	348
362		static __inline __m512i __DEFAULT_FN_ATTRS512
363		_mm512_set4_epi64 (long long __A, long long __B, long long __C,
364		long long __D)
365		{
	349	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	350	_mm512_set4_epi64(long long __A, long long __B, long long __C, long long __D) {
366	351	return __extension__ (__m512i) (__v8di)
367	352	{ __D, __C, __B, __A, __D, __C, __B, __A };
368	353	}
369	354
370		static __inline __m512d __DEFAULT_FN_ATTRS512
371		_mm512_set4_pd (double __A, double __B, double __C, double __D)
372		{
	355	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	356	_mm512_set4_pd(double __A, double __B, double __C, double __D) {
373	357	return __extension__ (__m512d)
374	358	{ __D, __C, __B, __A, __D, __C, __B, __A };
375	359	}
376	360
377		static __inline __m512 __DEFAULT_FN_ATTRS512
378		_mm512_set4_ps (float __A, float __B, float __C, float __D)
379		{
	361	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	362	_mm512_set4_ps(float __A, float __B, float __C, float __D) {
380	363	return __extension__ (__m512)
381	364	{ __D, __C, __B, __A, __D, __C, __B, __A,
382	365	__D, __C, __B, __A, __D, __C, __B, __A };
383	366	}
384	367
385		#define _mm512_setr4_epi32(e0,e1,e2,e3) \
386		_mm512_set4_epi32((e3),(e2),(e1),(e0))
	368	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	369	_mm512_setr4_epi32(int e0, int e1, int e2, int e3) {
	370	return _mm512_set4_epi32(e3, e2, e1, e0);
	371	}
387	372
388		#define _mm512_setr4_epi64(e0,e1,e2,e3) \
389		_mm512_set4_epi64((e3),(e2),(e1),(e0))
	373	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	374	_mm512_setr4_epi64(long long e0, long long e1, long long e2, long long e3) {
	375	return _mm512_set4_epi64(e3, e2, e1, e0);
	376	}
390	377
391		#define _mm512_setr4_pd(e0,e1,e2,e3) \
392		_mm512_set4_pd((e3),(e2),(e1),(e0))
	378	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	379	_mm512_setr4_pd(double e0, double e1, double e2, double e3) {
	380	return _mm512_set4_pd(e3, e2, e1, e0);
	381	}
393	382
394		#define _mm512_setr4_ps(e0,e1,e2,e3) \
395		_mm512_set4_ps((e3),(e2),(e1),(e0))
	383	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	384	_mm512_setr4_ps(float e0, float e1, float e2, float e3) {
	385	return _mm512_set4_ps(e3, e2, e1, e0);
	386	}
396	387
397		static __inline__ __m512d __DEFAULT_FN_ATTRS512
398		_mm512_broadcastsd_pd(__m128d __A)
399		{
	388	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	389	_mm512_broadcastsd_pd(__m128d __A) {
400	390	return (__m512d)__builtin_shufflevector((__v2df) __A, (__v2df) __A,
401	391	0, 0, 0, 0, 0, 0, 0, 0);
402	392	}
...	...	@@ -417,37 +407,36 @@ _mm512_castps256_ps512(__m256 __a)
417	407	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
418	408	}
419	409
420		static __inline __m128d __DEFAULT_FN_ATTRS512
	410	static __inline __m128d __DEFAULT_FN_ATTRS512_CONSTEXPR
421	411	_mm512_castpd512_pd128(__m512d __a)
422	412	{
423	413	return __builtin_shufflevector(__a, __a, 0, 1);
424	414	}
425	415
426		static __inline __m256d __DEFAULT_FN_ATTRS512
	416	static __inline __m256d __DEFAULT_FN_ATTRS512_CONSTEXPR
427	417	_mm512_castpd512_pd256 (__m512d __A)
428	418	{
429	419	return __builtin_shufflevector(__A, __A, 0, 1, 2, 3);
430	420	}
431	421
432		static __inline __m128 __DEFAULT_FN_ATTRS512
	422	static __inline __m128 __DEFAULT_FN_ATTRS512_CONSTEXPR
433	423	_mm512_castps512_ps128(__m512 __a)
434	424	{
435	425	return __builtin_shufflevector(__a, __a, 0, 1, 2, 3);
436	426	}
437	427
438		static __inline __m256 __DEFAULT_FN_ATTRS512
439		_mm512_castps512_ps256 (__m512 __A)
440		{
	428	static __inline __m256 __DEFAULT_FN_ATTRS512_CONSTEXPR
	429	_mm512_castps512_ps256(__m512 __A) {
441	430	return __builtin_shufflevector(__A, __A, 0, 1, 2, 3, 4, 5, 6, 7);
442	431	}
443	432
444		static __inline __m512 __DEFAULT_FN_ATTRS512
	433	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
445	434	_mm512_castpd_ps (__m512d __A)
446	435	{
447	436	return (__m512) (__A);
448	437	}
449	438
450		static __inline __m512i __DEFAULT_FN_ATTRS512
	439	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
451	440	_mm512_castpd_si512 (__m512d __A)
452	441	{
453	442	return (__m512i) (__A);
...	...	@@ -462,13 +451,13 @@ _mm512_castpd128_pd512 (__m128d __A)
462	451	__B, 0, 1, 2, 3, 4, 5, 6, 7);
463	452	}
464	453
465		static __inline __m512d __DEFAULT_FN_ATTRS512
	454	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
466	455	_mm512_castps_pd (__m512 __A)
467	456	{
468	457	return (__m512d) (__A);
469	458	}
470	459
471		static __inline __m512i __DEFAULT_FN_ATTRS512
	460	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
472	461	_mm512_castps_si512 (__m512 __A)
473	462	{
474	463	return (__m512i) (__A);
...	...	@@ -498,39 +487,36 @@ _mm512_castsi256_si512 (__m256i __A)
498	487	return __builtin_shufflevector( __A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7);
499	488	}
500	489
501		static __inline __m512 __DEFAULT_FN_ATTRS512
	490	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
502	491	_mm512_castsi512_ps (__m512i __A)
503	492	{
504	493	return (__m512) (__A);
505	494	}
506	495
507		static __inline __m512d __DEFAULT_FN_ATTRS512
	496	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
508	497	_mm512_castsi512_pd (__m512i __A)
509	498	{
510	499	return (__m512d) (__A);
511	500	}
512	501
513		static __inline __m128i __DEFAULT_FN_ATTRS512
	502	static __inline __m128i __DEFAULT_FN_ATTRS512_CONSTEXPR
514	503	_mm512_castsi512_si128 (__m512i __A)
515	504	{
516	505	return (__m128i)__builtin_shufflevector(__A, __A , 0, 1);
517	506	}
518	507
519		static __inline __m256i __DEFAULT_FN_ATTRS512
520		_mm512_castsi512_si256 (__m512i __A)
521		{
	508	static __inline __m256i __DEFAULT_FN_ATTRS512_CONSTEXPR
	509	_mm512_castsi512_si256(__m512i __A) {
522	510	return (__m256i)__builtin_shufflevector(__A, __A , 0, 1, 2, 3);
523	511	}
524	512
525		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
526		_mm512_int2mask(int __a)
527		{
	513	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	514	_mm512_int2mask(int __a) {
528	515	return (__mmask16)__a;
529	516	}
530	517
531		static __inline__ int __DEFAULT_FN_ATTRS
532		_mm512_mask2int(__mmask16 __a)
533		{
	518	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	519	_mm512_mask2int(__mmask16 __a) {
534	520	return (int)__a;
535	521	}
536	522
...	...	@@ -547,9 +533,8 @@ _mm512_mask2int(__mmask16 __a)
547	533	/// A 128-bit vector of [2 x double].
548	534	/// \returns A 512-bit floating-point vector of [8 x double]. The lower 128 bits
549	535	/// contain the value of the parameter. The upper 384 bits are set to zero.
550		static __inline __m512d __DEFAULT_FN_ATTRS512
551		_mm512_zextpd128_pd512(__m128d __a)
552		{
	536	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	537	_mm512_zextpd128_pd512(__m128d __a) {
553	538	return __builtin_shufflevector((__v2df)__a, (__v2df)_mm_setzero_pd(), 0, 1, 2, 3, 2, 3, 2, 3);
554	539	}
555	540
...	...	@@ -566,9 +551,8 @@ _mm512_zextpd128_pd512(__m128d __a)
566	551	/// A 256-bit vector of [4 x double].
567	552	/// \returns A 512-bit floating-point vector of [8 x double]. The lower 256 bits
568	553	/// contain the value of the parameter. The upper 256 bits are set to zero.
569		static __inline __m512d __DEFAULT_FN_ATTRS512
570		_mm512_zextpd256_pd512(__m256d __a)
571		{
	554	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	555	_mm512_zextpd256_pd512(__m256d __a) {
572	556	return __builtin_shufflevector((__v4df)__a, (__v4df)_mm256_setzero_pd(), 0, 1, 2, 3, 4, 5, 6, 7);
573	557	}
574	558
...	...	@@ -584,9 +568,8 @@ _mm512_zextpd256_pd512(__m256d __a)
584	568	/// A 128-bit vector of [4 x float].
585	569	/// \returns A 512-bit floating-point vector of [16 x float]. The lower 128 bits
586	570	/// contain the value of the parameter. The upper 384 bits are set to zero.
587		static __inline __m512 __DEFAULT_FN_ATTRS512
588		_mm512_zextps128_ps512(__m128 __a)
589		{
	571	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	572	_mm512_zextps128_ps512(__m128 __a) {
590	573	return __builtin_shufflevector((__v4sf)__a, (__v4sf)_mm_setzero_ps(), 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7, 4, 5, 6, 7);
591	574	}
592	575
...	...	@@ -602,9 +585,8 @@ _mm512_zextps128_ps512(__m128 __a)
602	585	/// A 256-bit vector of [8 x float].
603	586	/// \returns A 512-bit floating-point vector of [16 x float]. The lower 256 bits
604	587	/// contain the value of the parameter. The upper 256 bits are set to zero.
605		static __inline __m512 __DEFAULT_FN_ATTRS512
606		_mm512_zextps256_ps512(__m256 __a)
607		{
	588	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	589	_mm512_zextps256_ps512(__m256 __a) {
608	590	return __builtin_shufflevector((__v8sf)__a, (__v8sf)_mm256_setzero_ps(), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
609	591	}
610	592
...	...	@@ -620,9 +602,8 @@ _mm512_zextps256_ps512(__m256 __a)
620	602	/// A 128-bit integer vector.
621	603	/// \returns A 512-bit integer vector. The lower 128 bits contain the value of
622	604	/// the parameter. The upper 384 bits are set to zero.
623		static __inline __m512i __DEFAULT_FN_ATTRS512
624		_mm512_zextsi128_si512(__m128i __a)
625		{
	605	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	606	_mm512_zextsi128_si512(__m128i __a) {
626	607	return __builtin_shufflevector((__v2di)__a, (__v2di)_mm_setzero_si128(), 0, 1, 2, 3, 2, 3, 2, 3);
627	608	}
628	609
...	...	@@ -638,22 +619,20 @@ _mm512_zextsi128_si512(__m128i __a)
638	619	/// A 256-bit integer vector.
639	620	/// \returns A 512-bit integer vector. The lower 256 bits contain the value of
640	621	/// the parameter. The upper 256 bits are set to zero.
641		static __inline __m512i __DEFAULT_FN_ATTRS512
642		_mm512_zextsi256_si512(__m256i __a)
643		{
	622	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	623	_mm512_zextsi256_si512(__m256i __a) {
644	624	return __builtin_shufflevector((__v4di)__a, (__v4di)_mm256_setzero_si256(), 0, 1, 2, 3, 4, 5, 6, 7);
645	625	}
646	626
647	627	/* Bitwise operators */
648		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	628	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
649	629	_mm512_and_epi32(__m512i __a, __m512i __b)
650	630	{
651	631	return (__m512i)((__v16su)__a & (__v16su)__b);
652	632	}
653	633
654		static __inline__ __m512i __DEFAULT_FN_ATTRS512
655		_mm512_mask_and_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b)
656		{
	634	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	635	_mm512_mask_and_epi32(__m512i __src, __mmask16 __k, __m512i __a, __m512i __b) {
657	636	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__k,
658	637	(__v16si) _mm512_and_epi32(__a, __b),
659	638	(__v16si) __src);
...	...	@@ -666,18 +645,16 @@ _mm512_maskz_and_epi32(__mmask16 __k, __m512i __a, __m512i __b)
666	645	__k, __a, __b);
667	646	}
668	647
669		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	648	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
670	649	_mm512_and_epi64(__m512i __a, __m512i __b)
671	650	{
672	651	return (__m512i)((__v8du)__a & (__v8du)__b);
673	652	}
674	653
675		static __inline__ __m512i __DEFAULT_FN_ATTRS512
676		_mm512_mask_and_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b)
677		{
678		return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __k,
679		(__v8di) _mm512_and_epi64(__a, __b),
680		(__v8di) __src);
	654	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	655	_mm512_mask_and_epi64(__m512i __src, __mmask8 __k, __m512i __a, __m512i __b) {
	656	return (__m512i)__builtin_ia32_selectq_512(
	657	(__mmask8)__k, (__v8di)_mm512_and_epi64(__a, __b), (__v8di)__src);
681	658	}
682	659
683	660	static __inline__ __m512i __DEFAULT_FN_ATTRS512
...	...	@@ -687,13 +664,13 @@ _mm512_maskz_and_epi64(__mmask8 __k, __m512i __a, __m512i __b)
687	664	__k, __a, __b);
688	665	}
689	666
690		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	667	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
691	668	_mm512_andnot_si512 (__m512i __A, __m512i __B)
692	669	{
693	670	return (__m512i)(~(__v8du)__A & (__v8du)__B);
694	671	}
695	672
696		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	673	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
697	674	_mm512_andnot_epi32 (__m512i __A, __m512i __B)
698	675	{
699	676	return (__m512i)(~(__v16su)__A & (__v16su)__B);
...	...	@@ -714,7 +691,7 @@ _mm512_maskz_andnot_epi32(__mmask16 __U, __m512i __A, __m512i __B)
714	691	__U, __A, __B);
715	692	}
716	693
717		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	694	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
718	695	_mm512_andnot_epi64(__m512i __A, __m512i __B)
719	696	{
720	697	return (__m512i)(~(__v8du)__A & (__v8du)__B);
...	...	@@ -735,7 +712,7 @@ _mm512_maskz_andnot_epi64(__mmask8 __U, __m512i __A, __m512i __B)
735	712	__U, __A, __B);
736	713	}
737	714
738		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	715	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
739	716	_mm512_or_epi32(__m512i __a, __m512i __b)
740	717	{
741	718	return (__m512i)((__v16su)__a \| (__v16su)__b);
...	...	@@ -755,7 +732,7 @@ _mm512_maskz_or_epi32(__mmask16 __k, __m512i __a, __m512i __b)
755	732	return (__m512i)_mm512_mask_or_epi32(_mm512_setzero_si512(), __k, __a, __b);
756	733	}
757	734
758		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	735	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
759	736	_mm512_or_epi64(__m512i __a, __m512i __b)
760	737	{
761	738	return (__m512i)((__v8du)__a \| (__v8du)__b);
...	...	@@ -775,7 +752,7 @@ _mm512_maskz_or_epi64(__mmask8 __k, __m512i __a, __m512i __b)
775	752	return (__m512i)_mm512_mask_or_epi64(_mm512_setzero_si512(), __k, __a, __b);
776	753	}
777	754
778		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	755	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
779	756	_mm512_xor_epi32(__m512i __a, __m512i __b)
780	757	{
781	758	return (__m512i)((__v16su)__a ^ (__v16su)__b);
...	...	@@ -795,7 +772,7 @@ _mm512_maskz_xor_epi32(__mmask16 __k, __m512i __a, __m512i __b)
795	772	return (__m512i)_mm512_mask_xor_epi32(_mm512_setzero_si512(), __k, __a, __b);
796	773	}
797	774
798		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	775	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
799	776	_mm512_xor_epi64(__m512i __a, __m512i __b)
800	777	{
801	778	return (__m512i)((__v8du)__a ^ (__v8du)__b);
...	...	@@ -815,19 +792,19 @@ _mm512_maskz_xor_epi64(__mmask8 __k, __m512i __a, __m512i __b)
815	792	return (__m512i)_mm512_mask_xor_epi64(_mm512_setzero_si512(), __k, __a, __b);
816	793	}
817	794
818		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	795	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
819	796	_mm512_and_si512(__m512i __a, __m512i __b)
820	797	{
821	798	return (__m512i)((__v8du)__a & (__v8du)__b);
822	799	}
823	800
824		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	801	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
825	802	_mm512_or_si512(__m512i __a, __m512i __b)
826	803	{
827	804	return (__m512i)((__v8du)__a \| (__v8du)__b);
828	805	}
829	806
830		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	807	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
831	808	_mm512_xor_si512(__m512i __a, __m512i __b)
832	809	{
833	810	return (__m512i)((__v8du)__a ^ (__v8du)__b);
...	...	@@ -835,125 +812,107 @@ _mm512_xor_si512(__m512i __a, __m512i __b)
835	812
836	813	/* Arithmetic */
837	814
838		static __inline __m512d __DEFAULT_FN_ATTRS512
839		_mm512_add_pd(__m512d __a, __m512d __b)
840		{
	815	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	816	_mm512_add_pd(__m512d __a, __m512d __b) {
841	817	return (__m512d)((__v8df)__a + (__v8df)__b);
842	818	}
843	819
844		static __inline __m512 __DEFAULT_FN_ATTRS512
845		_mm512_add_ps(__m512 __a, __m512 __b)
846		{
	820	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	821	_mm512_add_ps(__m512 __a, __m512 __b) {
847	822	return (__m512)((__v16sf)__a + (__v16sf)__b);
848	823	}
849	824
850		static __inline __m512d __DEFAULT_FN_ATTRS512
851		_mm512_mul_pd(__m512d __a, __m512d __b)
852		{
	825	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	826	_mm512_mul_pd(__m512d __a, __m512d __b) {
853	827	return (__m512d)((__v8df)__a * (__v8df)__b);
854	828	}
855	829
856		static __inline __m512 __DEFAULT_FN_ATTRS512
857		_mm512_mul_ps(__m512 __a, __m512 __b)
858		{
	830	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	831	_mm512_mul_ps(__m512 __a, __m512 __b) {
859	832	return (__m512)((__v16sf)__a * (__v16sf)__b);
860	833	}
861	834
862		static __inline __m512d __DEFAULT_FN_ATTRS512
863		_mm512_sub_pd(__m512d __a, __m512d __b)
864		{
	835	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	836	_mm512_sub_pd(__m512d __a, __m512d __b) {
865	837	return (__m512d)((__v8df)__a - (__v8df)__b);
866	838	}
867	839
868		static __inline __m512 __DEFAULT_FN_ATTRS512
869		_mm512_sub_ps(__m512 __a, __m512 __b)
870		{
	840	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	841	_mm512_sub_ps(__m512 __a, __m512 __b) {
871	842	return (__m512)((__v16sf)__a - (__v16sf)__b);
872	843	}
873	844
874		static __inline__ __m512i __DEFAULT_FN_ATTRS512
875		_mm512_add_epi64 (__m512i __A, __m512i __B)
876		{
	845	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	846	_mm512_add_epi64(__m512i __A, __m512i __B) {
877	847	return (__m512i) ((__v8du) __A + (__v8du) __B);
878	848	}
879	849
880		static __inline__ __m512i __DEFAULT_FN_ATTRS512
881		_mm512_mask_add_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
882		{
	850	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	851	_mm512_mask_add_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
883	852	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
884	853	(__v8di)_mm512_add_epi64(__A, __B),
885	854	(__v8di)__W);
886	855	}
887	856
888		static __inline__ __m512i __DEFAULT_FN_ATTRS512
889		_mm512_maskz_add_epi64(__mmask8 __U, __m512i __A, __m512i __B)
890		{
	857	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	858	_mm512_maskz_add_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
891	859	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
892	860	(__v8di)_mm512_add_epi64(__A, __B),
893	861	(__v8di)_mm512_setzero_si512());
894	862	}
895	863
896		static __inline__ __m512i __DEFAULT_FN_ATTRS512
897		_mm512_sub_epi64 (__m512i __A, __m512i __B)
898		{
	864	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	865	_mm512_sub_epi64(__m512i __A, __m512i __B) {
899	866	return (__m512i) ((__v8du) __A - (__v8du) __B);
900	867	}
901	868
902		static __inline__ __m512i __DEFAULT_FN_ATTRS512
903		_mm512_mask_sub_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
904		{
	869	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	870	_mm512_mask_sub_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
905	871	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
906	872	(__v8di)_mm512_sub_epi64(__A, __B),
907	873	(__v8di)__W);
908	874	}
909	875
910		static __inline__ __m512i __DEFAULT_FN_ATTRS512
911		_mm512_maskz_sub_epi64(__mmask8 __U, __m512i __A, __m512i __B)
912		{
	876	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	877	_mm512_maskz_sub_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
913	878	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
914	879	(__v8di)_mm512_sub_epi64(__A, __B),
915	880	(__v8di)_mm512_setzero_si512());
916	881	}
917	882
918		static __inline__ __m512i __DEFAULT_FN_ATTRS512
919		_mm512_add_epi32 (__m512i __A, __m512i __B)
920		{
	883	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	884	_mm512_add_epi32(__m512i __A, __m512i __B) {
921	885	return (__m512i) ((__v16su) __A + (__v16su) __B);
922	886	}
923	887
924		static __inline__ __m512i __DEFAULT_FN_ATTRS512
925		_mm512_mask_add_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
926		{
	888	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	889	_mm512_mask_add_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
927	890	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
928	891	(__v16si)_mm512_add_epi32(__A, __B),
929	892	(__v16si)__W);
930	893	}
931	894
932		static __inline__ __m512i __DEFAULT_FN_ATTRS512
933		_mm512_maskz_add_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
934		{
	895	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	896	_mm512_maskz_add_epi32(__mmask16 __U, __m512i __A, __m512i __B) {
935	897	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
936	898	(__v16si)_mm512_add_epi32(__A, __B),
937	899	(__v16si)_mm512_setzero_si512());
938	900	}
939	901
940		static __inline__ __m512i __DEFAULT_FN_ATTRS512
941		_mm512_sub_epi32 (__m512i __A, __m512i __B)
942		{
	902	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	903	_mm512_sub_epi32(__m512i __A, __m512i __B) {
943	904	return (__m512i) ((__v16su) __A - (__v16su) __B);
944	905	}
945	906
946		static __inline__ __m512i __DEFAULT_FN_ATTRS512
947		_mm512_mask_sub_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
948		{
	907	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	908	_mm512_mask_sub_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
949	909	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
950	910	(__v16si)_mm512_sub_epi32(__A, __B),
951	911	(__v16si)__W);
952	912	}
953	913
954		static __inline__ __m512i __DEFAULT_FN_ATTRS512
955		_mm512_maskz_sub_epi32(__mmask16 __U, __m512i __A, __m512i __B)
956		{
	914	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	915	_mm512_maskz_sub_epi32(__mmask16 __U, __m512i __A, __m512i __B) {
957	916	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
958	917	(__v16si)_mm512_sub_epi32(__A, __B),
959	918	(__v16si)_mm512_setzero_si512());
...	...	@@ -973,24 +932,21 @@ _mm512_maskz_sub_epi32(__mmask16 __U, __m512i __A, __m512i __B)
973	932	(__v8df)_mm512_max_round_pd((A), (B), (R)), \
974	933	(__v8df)_mm512_setzero_pd()))
975	934
976		static __inline__ __m512d __DEFAULT_FN_ATTRS512
977		_mm512_max_pd(__m512d __A, __m512d __B)
978		{
	935	static __inline__ __m512d
	936	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_max_pd(__m512d __A, __m512d __B) {
979	937	return (__m512d) __builtin_ia32_maxpd512((__v8df) __A, (__v8df) __B,
980	938	_MM_FROUND_CUR_DIRECTION);
981	939	}
982	940
983		static __inline__ __m512d __DEFAULT_FN_ATTRS512
984		_mm512_mask_max_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
985		{
	941	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	942	_mm512_mask_max_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
986	943	return (__m512d)__builtin_ia32_selectpd_512(__U,
987	944	(__v8df)_mm512_max_pd(__A, __B),
988	945	(__v8df)__W);
989	946	}
990	947
991		static __inline__ __m512d __DEFAULT_FN_ATTRS512
992		_mm512_maskz_max_pd (__mmask8 __U, __m512d __A, __m512d __B)
993		{
	948	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	949	_mm512_maskz_max_pd(__mmask8 __U, __m512d __A, __m512d __B) {
994	950	return (__m512d)__builtin_ia32_selectpd_512(__U,
995	951	(__v8df)_mm512_max_pd(__A, __B),
996	952	(__v8df)_mm512_setzero_pd());
...	...	@@ -1010,31 +966,30 @@ _mm512_maskz_max_pd (__mmask8 __U, __m512d __A, __m512d __B)
1010	966	(__v16sf)_mm512_max_round_ps((A), (B), (R)), \
1011	967	(__v16sf)_mm512_setzero_ps()))
1012	968
1013		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1014		_mm512_max_ps(__m512 __A, __m512 __B)
1015		{
	969	static __inline__ __m512
	970	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_max_ps(__m512 __A, __m512 __B) {
1016	971	return (__m512) __builtin_ia32_maxps512((__v16sf) __A, (__v16sf) __B,
1017	972	_MM_FROUND_CUR_DIRECTION);
1018	973	}
1019	974
1020		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1021		_mm512_mask_max_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
1022		{
	975	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	976	_mm512_mask_max_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
1023	977	return (__m512)__builtin_ia32_selectps_512(__U,
1024	978	(__v16sf)_mm512_max_ps(__A, __B),
1025	979	(__v16sf)__W);
1026	980	}
1027	981
1028		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1029		_mm512_maskz_max_ps (__mmask16 __U, __m512 __A, __m512 __B)
1030		{
	982	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	983	_mm512_maskz_max_ps(__mmask16 __U, __m512 __A, __m512 __B) {
1031	984	return (__m512)__builtin_ia32_selectps_512(__U,
1032	985	(__v16sf)_mm512_max_ps(__A, __B),
1033	986	(__v16sf)_mm512_setzero_ps());
1034	987	}
1035	988
1036		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1037		_mm_mask_max_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	989	static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_max_ss(__m128 __W,
	990	__mmask8 __U,
	991	__m128 __A,
	992	__m128 __B) {
1038	993	return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
1039	994	(__v4sf) __B,
1040	995	(__v4sf) __W,
...	...	@@ -1042,8 +997,9 @@ _mm_mask_max_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
1042	997	_MM_FROUND_CUR_DIRECTION);
1043	998	}
1044	999
1045		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1046		_mm_maskz_max_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	1000	static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_max_ss(__mmask8 __U,
	1001	__m128 __A,
	1002	__m128 __B) {
1047	1003	return (__m128) __builtin_ia32_maxss_round_mask ((__v4sf) __A,
1048	1004	(__v4sf) __B,
1049	1005	(__v4sf) _mm_setzero_ps (),
...	...	@@ -1069,8 +1025,10 @@ _mm_maskz_max_ss(__mmask8 __U,__m128 __A, __m128 __B) {
1069	1025	(__v4sf)_mm_setzero_ps(), \
1070	1026	(__mmask8)(U), (int)(R)))
1071	1027
1072		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1073		_mm_mask_max_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	1028	static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_max_sd(__m128d __W,
	1029	__mmask8 __U,
	1030	__m128d __A,
	1031	__m128d __B) {
1074	1032	return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
1075	1033	(__v2df) __B,
1076	1034	(__v2df) __W,
...	...	@@ -1078,8 +1036,9 @@ _mm_mask_max_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
1078	1036	_MM_FROUND_CUR_DIRECTION);
1079	1037	}
1080	1038
1081		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1082		_mm_maskz_max_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	1039	static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_max_sd(__mmask8 __U,
	1040	__m128d __A,
	1041	__m128d __B) {
1083	1042	return (__m128d) __builtin_ia32_maxsd_round_mask ((__v2df) __A,
1084	1043	(__v2df) __B,
1085	1044	(__v2df) _mm_setzero_pd (),
...	...	@@ -1106,89 +1065,76 @@ _mm_maskz_max_sd(__mmask8 __U,__m128d __A, __m128d __B) {
1106	1065	(__mmask8)(U), (int)(R)))
1107	1066
1108	1067	static __inline __m512i
1109		__DEFAULT_FN_ATTRS512
1110		_mm512_max_epi32(__m512i __A, __m512i __B)
1111		{
	1068	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_max_epi32(__m512i __A, __m512i __B) {
1112	1069	return (__m512i)__builtin_elementwise_max((__v16si)__A, (__v16si)__B);
1113	1070	}
1114	1071
1115		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1116		_mm512_mask_max_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1117		{
	1072	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1073	_mm512_mask_max_epi32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) {
1118	1074	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1119	1075	(__v16si)_mm512_max_epi32(__A, __B),
1120	1076	(__v16si)__W);
1121	1077	}
1122	1078
1123		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1124		_mm512_maskz_max_epi32 (__mmask16 __M, __m512i __A, __m512i __B)
1125		{
	1079	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1080	_mm512_maskz_max_epi32(__mmask16 __M, __m512i __A, __m512i __B) {
1126	1081	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1127	1082	(__v16si)_mm512_max_epi32(__A, __B),
1128	1083	(__v16si)_mm512_setzero_si512());
1129	1084	}
1130	1085
1131		static __inline __m512i __DEFAULT_FN_ATTRS512
1132		_mm512_max_epu32(__m512i __A, __m512i __B)
1133		{
	1086	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1087	_mm512_max_epu32(__m512i __A, __m512i __B) {
1134	1088	return (__m512i)__builtin_elementwise_max((__v16su)__A, (__v16su)__B);
1135	1089	}
1136	1090
1137		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1138		_mm512_mask_max_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1139		{
	1091	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1092	_mm512_mask_max_epu32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) {
1140	1093	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1141	1094	(__v16si)_mm512_max_epu32(__A, __B),
1142	1095	(__v16si)__W);
1143	1096	}
1144	1097
1145		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1146		_mm512_maskz_max_epu32 (__mmask16 __M, __m512i __A, __m512i __B)
1147		{
	1098	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1099	_mm512_maskz_max_epu32(__mmask16 __M, __m512i __A, __m512i __B) {
1148	1100	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1149	1101	(__v16si)_mm512_max_epu32(__A, __B),
1150	1102	(__v16si)_mm512_setzero_si512());
1151	1103	}
1152	1104
1153		static __inline __m512i __DEFAULT_FN_ATTRS512
1154		_mm512_max_epi64(__m512i __A, __m512i __B)
1155		{
	1105	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1106	_mm512_max_epi64(__m512i __A, __m512i __B) {
1156	1107	return (__m512i)__builtin_elementwise_max((__v8di)__A, (__v8di)__B);
1157	1108	}
1158	1109
1159		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1160		_mm512_mask_max_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1161		{
	1110	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1111	_mm512_mask_max_epi64(__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) {
1162	1112	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1163	1113	(__v8di)_mm512_max_epi64(__A, __B),
1164	1114	(__v8di)__W);
1165	1115	}
1166	1116
1167		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1168		_mm512_maskz_max_epi64 (__mmask8 __M, __m512i __A, __m512i __B)
1169		{
	1117	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1118	_mm512_maskz_max_epi64(__mmask8 __M, __m512i __A, __m512i __B) {
1170	1119	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1171	1120	(__v8di)_mm512_max_epi64(__A, __B),
1172	1121	(__v8di)_mm512_setzero_si512());
1173	1122	}
1174	1123
1175		static __inline __m512i __DEFAULT_FN_ATTRS512
1176		_mm512_max_epu64(__m512i __A, __m512i __B)
1177		{
	1124	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1125	_mm512_max_epu64(__m512i __A, __m512i __B) {
1178	1126	return (__m512i)__builtin_elementwise_max((__v8du)__A, (__v8du)__B);
1179	1127	}
1180	1128
1181		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1182		_mm512_mask_max_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1183		{
	1129	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1130	_mm512_mask_max_epu64(__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) {
1184	1131	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1185	1132	(__v8di)_mm512_max_epu64(__A, __B),
1186	1133	(__v8di)__W);
1187	1134	}
1188	1135
1189		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1190		_mm512_maskz_max_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
1191		{
	1136	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1137	_mm512_maskz_max_epu64(__mmask8 __M, __m512i __A, __m512i __B) {
1192	1138	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1193	1139	(__v8di)_mm512_max_epu64(__A, __B),
1194	1140	(__v8di)_mm512_setzero_si512());
...	...	@@ -1208,24 +1154,21 @@ _mm512_maskz_max_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
1208	1154	(__v8df)_mm512_min_round_pd((A), (B), (R)), \
1209	1155	(__v8df)_mm512_setzero_pd()))
1210	1156
1211		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1212		_mm512_min_pd(__m512d __A, __m512d __B)
1213		{
	1157	static __inline__ __m512d
	1158	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_min_pd(__m512d __A, __m512d __B) {
1214	1159	return (__m512d) __builtin_ia32_minpd512((__v8df) __A, (__v8df) __B,
1215	1160	_MM_FROUND_CUR_DIRECTION);
1216	1161	}
1217	1162
1218		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1219		_mm512_mask_min_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B)
1220		{
	1163	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	1164	_mm512_mask_min_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
1221	1165	return (__m512d)__builtin_ia32_selectpd_512(__U,
1222	1166	(__v8df)_mm512_min_pd(__A, __B),
1223	1167	(__v8df)__W);
1224	1168	}
1225	1169
1226		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1227		_mm512_maskz_min_pd (__mmask8 __U, __m512d __A, __m512d __B)
1228		{
	1170	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	1171	_mm512_maskz_min_pd(__mmask8 __U, __m512d __A, __m512d __B) {
1229	1172	return (__m512d)__builtin_ia32_selectpd_512(__U,
1230	1173	(__v8df)_mm512_min_pd(__A, __B),
1231	1174	(__v8df)_mm512_setzero_pd());
...	...	@@ -1245,31 +1188,30 @@ _mm512_maskz_min_pd (__mmask8 __U, __m512d __A, __m512d __B)
1245	1188	(__v16sf)_mm512_min_round_ps((A), (B), (R)), \
1246	1189	(__v16sf)_mm512_setzero_ps()))
1247	1190
1248		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1249		_mm512_min_ps(__m512 __A, __m512 __B)
1250		{
	1191	static __inline__ __m512
	1192	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_min_ps(__m512 __A, __m512 __B) {
1251	1193	return (__m512) __builtin_ia32_minps512((__v16sf) __A, (__v16sf) __B,
1252	1194	_MM_FROUND_CUR_DIRECTION);
1253	1195	}
1254	1196
1255		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1256		_mm512_mask_min_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
1257		{
	1197	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1198	_mm512_mask_min_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
1258	1199	return (__m512)__builtin_ia32_selectps_512(__U,
1259	1200	(__v16sf)_mm512_min_ps(__A, __B),
1260	1201	(__v16sf)__W);
1261	1202	}
1262	1203
1263		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1264		_mm512_maskz_min_ps (__mmask16 __U, __m512 __A, __m512 __B)
1265		{
	1204	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	1205	_mm512_maskz_min_ps(__mmask16 __U, __m512 __A, __m512 __B) {
1266	1206	return (__m512)__builtin_ia32_selectps_512(__U,
1267	1207	(__v16sf)_mm512_min_ps(__A, __B),
1268	1208	(__v16sf)_mm512_setzero_ps());
1269	1209	}
1270	1210
1271		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1272		_mm_mask_min_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	1211	static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_mask_min_ss(__m128 __W,
	1212	__mmask8 __U,
	1213	__m128 __A,
	1214	__m128 __B) {
1273	1215	return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
1274	1216	(__v4sf) __B,
1275	1217	(__v4sf) __W,
...	...	@@ -1277,8 +1219,9 @@ _mm_mask_min_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
1277	1219	_MM_FROUND_CUR_DIRECTION);
1278	1220	}
1279	1221
1280		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1281		_mm_maskz_min_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	1222	static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_maskz_min_ss(__mmask8 __U,
	1223	__m128 __A,
	1224	__m128 __B) {
1282	1225	return (__m128) __builtin_ia32_minss_round_mask ((__v4sf) __A,
1283	1226	(__v4sf) __B,
1284	1227	(__v4sf) _mm_setzero_ps (),
...	...	@@ -1304,8 +1247,10 @@ _mm_maskz_min_ss(__mmask8 __U,__m128 __A, __m128 __B) {
1304	1247	(__v4sf)_mm_setzero_ps(), \
1305	1248	(__mmask8)(U), (int)(R)))
1306	1249
1307		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1308		_mm_mask_min_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	1250	static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_mask_min_sd(__m128d __W,
	1251	__mmask8 __U,
	1252	__m128d __A,
	1253	__m128d __B) {
1309	1254	return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
1310	1255	(__v2df) __B,
1311	1256	(__v2df) __W,
...	...	@@ -1313,8 +1258,9 @@ _mm_mask_min_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
1313	1258	_MM_FROUND_CUR_DIRECTION);
1314	1259	}
1315	1260
1316		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1317		_mm_maskz_min_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	1261	static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_maskz_min_sd(__mmask8 __U,
	1262	__m128d __A,
	1263	__m128d __B) {
1318	1264	return (__m128d) __builtin_ia32_minsd_round_mask ((__v2df) __A,
1319	1265	(__v2df) __B,
1320	1266	(__v2df) _mm_setzero_pd (),
...	...	@@ -1341,166 +1287,144 @@ _mm_maskz_min_sd(__mmask8 __U,__m128d __A, __m128d __B) {
1341	1287	(__mmask8)(U), (int)(R)))
1342	1288
1343	1289	static __inline __m512i
1344		__DEFAULT_FN_ATTRS512
1345		_mm512_min_epi32(__m512i __A, __m512i __B)
1346		{
	1290	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_min_epi32(__m512i __A, __m512i __B) {
1347	1291	return (__m512i)__builtin_elementwise_min((__v16si)__A, (__v16si)__B);
1348	1292	}
1349	1293
1350		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1351		_mm512_mask_min_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1352		{
	1294	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1295	_mm512_mask_min_epi32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) {
1353	1296	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1354	1297	(__v16si)_mm512_min_epi32(__A, __B),
1355	1298	(__v16si)__W);
1356	1299	}
1357	1300
1358		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1359		_mm512_maskz_min_epi32 (__mmask16 __M, __m512i __A, __m512i __B)
1360		{
	1301	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1302	_mm512_maskz_min_epi32(__mmask16 __M, __m512i __A, __m512i __B) {
1361	1303	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1362	1304	(__v16si)_mm512_min_epi32(__A, __B),
1363	1305	(__v16si)_mm512_setzero_si512());
1364	1306	}
1365	1307
1366		static __inline __m512i __DEFAULT_FN_ATTRS512
1367		_mm512_min_epu32(__m512i __A, __m512i __B)
1368		{
	1308	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1309	_mm512_min_epu32(__m512i __A, __m512i __B) {
1369	1310	return (__m512i)__builtin_elementwise_min((__v16su)__A, (__v16su)__B);
1370	1311	}
1371	1312
1372		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1373		_mm512_mask_min_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1374		{
	1313	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1314	_mm512_mask_min_epu32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) {
1375	1315	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1376	1316	(__v16si)_mm512_min_epu32(__A, __B),
1377	1317	(__v16si)__W);
1378	1318	}
1379	1319
1380		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1381		_mm512_maskz_min_epu32 (__mmask16 __M, __m512i __A, __m512i __B)
1382		{
	1320	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1321	_mm512_maskz_min_epu32(__mmask16 __M, __m512i __A, __m512i __B) {
1383	1322	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1384	1323	(__v16si)_mm512_min_epu32(__A, __B),
1385	1324	(__v16si)_mm512_setzero_si512());
1386	1325	}
1387	1326
1388		static __inline __m512i __DEFAULT_FN_ATTRS512
1389		_mm512_min_epi64(__m512i __A, __m512i __B)
1390		{
	1327	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1328	_mm512_min_epi64(__m512i __A, __m512i __B) {
1391	1329	return (__m512i)__builtin_elementwise_min((__v8di)__A, (__v8di)__B);
1392	1330	}
1393	1331
1394		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1395		_mm512_mask_min_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1396		{
	1332	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1333	_mm512_mask_min_epi64(__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) {
1397	1334	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1398	1335	(__v8di)_mm512_min_epi64(__A, __B),
1399	1336	(__v8di)__W);
1400	1337	}
1401	1338
1402		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1403		_mm512_maskz_min_epi64 (__mmask8 __M, __m512i __A, __m512i __B)
1404		{
	1339	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1340	_mm512_maskz_min_epi64(__mmask8 __M, __m512i __A, __m512i __B) {
1405	1341	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1406	1342	(__v8di)_mm512_min_epi64(__A, __B),
1407	1343	(__v8di)_mm512_setzero_si512());
1408	1344	}
1409	1345
1410		static __inline __m512i __DEFAULT_FN_ATTRS512
1411		_mm512_min_epu64(__m512i __A, __m512i __B)
1412		{
	1346	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1347	_mm512_min_epu64(__m512i __A, __m512i __B) {
1413	1348	return (__m512i)__builtin_elementwise_min((__v8du)__A, (__v8du)__B);
1414	1349	}
1415	1350
1416		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1417		_mm512_mask_min_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B)
1418		{
	1351	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1352	_mm512_mask_min_epu64(__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) {
1419	1353	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1420	1354	(__v8di)_mm512_min_epu64(__A, __B),
1421	1355	(__v8di)__W);
1422	1356	}
1423	1357
1424		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1425		_mm512_maskz_min_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
1426		{
	1358	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1359	_mm512_maskz_min_epu64(__mmask8 __M, __m512i __A, __m512i __B) {
1427	1360	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1428	1361	(__v8di)_mm512_min_epu64(__A, __B),
1429	1362	(__v8di)_mm512_setzero_si512());
1430	1363	}
1431	1364
1432		static __inline __m512i __DEFAULT_FN_ATTRS512
1433		_mm512_mul_epi32(__m512i __X, __m512i __Y)
1434		{
	1365	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1366	_mm512_mul_epi32(__m512i __X, __m512i __Y) {
1435	1367	return (__m512i)__builtin_ia32_pmuldq512((__v16si)__X, (__v16si) __Y);
1436	1368	}
1437	1369
1438		static __inline __m512i __DEFAULT_FN_ATTRS512
1439		_mm512_mask_mul_epi32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
1440		{
	1370	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1371	_mm512_mask_mul_epi32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
1441	1372	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1442	1373	(__v8di)_mm512_mul_epi32(__X, __Y),
1443	1374	(__v8di)__W);
1444	1375	}
1445	1376
1446		static __inline __m512i __DEFAULT_FN_ATTRS512
1447		_mm512_maskz_mul_epi32(__mmask8 __M, __m512i __X, __m512i __Y)
1448		{
	1377	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1378	_mm512_maskz_mul_epi32(__mmask8 __M, __m512i __X, __m512i __Y) {
1449	1379	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1450	1380	(__v8di)_mm512_mul_epi32(__X, __Y),
1451	1381	(__v8di)_mm512_setzero_si512 ());
1452	1382	}
1453	1383
1454		static __inline __m512i __DEFAULT_FN_ATTRS512
1455		_mm512_mul_epu32(__m512i __X, __m512i __Y)
1456		{
	1384	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1385	_mm512_mul_epu32(__m512i __X, __m512i __Y) {
1457	1386	return (__m512i)__builtin_ia32_pmuludq512((__v16si)__X, (__v16si)__Y);
1458	1387	}
1459	1388
1460		static __inline __m512i __DEFAULT_FN_ATTRS512
1461		_mm512_mask_mul_epu32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
1462		{
	1389	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1390	_mm512_mask_mul_epu32(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
1463	1391	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1464	1392	(__v8di)_mm512_mul_epu32(__X, __Y),
1465	1393	(__v8di)__W);
1466	1394	}
1467	1395
1468		static __inline __m512i __DEFAULT_FN_ATTRS512
1469		_mm512_maskz_mul_epu32(__mmask8 __M, __m512i __X, __m512i __Y)
1470		{
	1396	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1397	_mm512_maskz_mul_epu32(__mmask8 __M, __m512i __X, __m512i __Y) {
1471	1398	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
1472	1399	(__v8di)_mm512_mul_epu32(__X, __Y),
1473	1400	(__v8di)_mm512_setzero_si512 ());
1474	1401	}
1475	1402
1476		static __inline __m512i __DEFAULT_FN_ATTRS512
1477		_mm512_mullo_epi32 (__m512i __A, __m512i __B)
1478		{
	1403	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1404	_mm512_mullo_epi32(__m512i __A, __m512i __B) {
1479	1405	return (__m512i) ((__v16su) __A * (__v16su) __B);
1480	1406	}
1481	1407
1482		static __inline __m512i __DEFAULT_FN_ATTRS512
1483		_mm512_maskz_mullo_epi32(__mmask16 __M, __m512i __A, __m512i __B)
1484		{
	1408	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1409	_mm512_maskz_mullo_epi32(__mmask16 __M, __m512i __A, __m512i __B) {
1485	1410	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1486	1411	(__v16si)_mm512_mullo_epi32(__A, __B),
1487	1412	(__v16si)_mm512_setzero_si512());
1488	1413	}
1489	1414
1490		static __inline __m512i __DEFAULT_FN_ATTRS512
1491		_mm512_mask_mullo_epi32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B)
1492		{
	1415	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1416	_mm512_mask_mullo_epi32(__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) {
1493	1417	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
1494	1418	(__v16si)_mm512_mullo_epi32(__A, __B),
1495	1419	(__v16si)__W);
1496	1420	}
1497	1421
1498		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1499		_mm512_mullox_epi64 (__m512i __A, __m512i __B) {
	1422	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1423	_mm512_mullox_epi64(__m512i __A, __m512i __B) {
1500	1424	return (__m512i) ((__v8du) __A * (__v8du) __B);
1501	1425	}
1502	1426
1503		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	1427	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
1504	1428	_mm512_mask_mullox_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
1505	1429	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1506	1430	(__v8di)_mm512_mullox_epi64(__A, __B),
...	...	@@ -1520,26 +1444,19 @@ _mm512_mask_mullox_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
1520	1444	(__v8df)_mm512_sqrt_round_pd((A), (R)), \
1521	1445	(__v8df)_mm512_setzero_pd()))
1522	1446
1523		static __inline__ __m512d __DEFAULT_FN_ATTRS512
1524		_mm512_sqrt_pd(__m512d __A)
1525		{
1526		return (__m512d)__builtin_ia32_sqrtpd512((__v8df)__A,
1527		_MM_FROUND_CUR_DIRECTION);
	1447	static __inline__ __m512d __DEFAULT_FN_ATTRS512 _mm512_sqrt_pd(__m512d __A) {
	1448	return (__m512d)__builtin_elementwise_sqrt((__v8df)__A);
1528	1449	}
1529	1450
1530	1451	static __inline__ __m512d __DEFAULT_FN_ATTRS512
1531		_mm512_mask_sqrt_pd (__m512d __W, __mmask8 __U, __m512d __A)
1532		{
1533		return (__m512d)__builtin_ia32_selectpd_512(__U,
1534		(__v8df)_mm512_sqrt_pd(__A),
	1452	_mm512_mask_sqrt_pd(__m512d __W, __mmask8 __U, __m512d __A) {
	1453	return (__m512d)__builtin_ia32_selectpd_512(__U, (__v8df)_mm512_sqrt_pd(__A),
1535	1454	(__v8df)__W);
1536	1455	}
1537	1456
1538	1457	static __inline__ __m512d __DEFAULT_FN_ATTRS512
1539		_mm512_maskz_sqrt_pd (__mmask8 __U, __m512d __A)
1540		{
1541		return (__m512d)__builtin_ia32_selectpd_512(__U,
1542		(__v8df)_mm512_sqrt_pd(__A),
	1458	_mm512_maskz_sqrt_pd(__mmask8 __U, __m512d __A) {
	1459	return (__m512d)__builtin_ia32_selectpd_512(__U, (__v8df)_mm512_sqrt_pd(__A),
1543	1460	(__v8df)_mm512_setzero_pd());
1544	1461	}
1545	1462
...	...	@@ -1556,26 +1473,19 @@ _mm512_maskz_sqrt_pd (__mmask8 __U, __m512d __A)
1556	1473	(__v16sf)_mm512_sqrt_round_ps((A), (R)), \
1557	1474	(__v16sf)_mm512_setzero_ps()))
1558	1475
1559		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1560		_mm512_sqrt_ps(__m512 __A)
1561		{
1562		return (__m512)__builtin_ia32_sqrtps512((__v16sf)__A,
1563		_MM_FROUND_CUR_DIRECTION);
	1476	static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_sqrt_ps(__m512 __A) {
	1477	return (__m512)__builtin_elementwise_sqrt((__v16sf)__A);
1564	1478	}
1565	1479
1566		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1567		_mm512_mask_sqrt_ps(__m512 __W, __mmask16 __U, __m512 __A)
1568		{
1569		return (__m512)__builtin_ia32_selectps_512(__U,
1570		(__v16sf)_mm512_sqrt_ps(__A),
	1480	static __inline__ __m512 __DEFAULT_FN_ATTRS512
	1481	_mm512_mask_sqrt_ps(__m512 __W, __mmask16 __U, __m512 __A) {
	1482	return (__m512)__builtin_ia32_selectps_512(__U, (__v16sf)_mm512_sqrt_ps(__A),
1571	1483	(__v16sf)__W);
1572	1484	}
1573	1485
1574		static __inline__ __m512 __DEFAULT_FN_ATTRS512
1575		_mm512_maskz_sqrt_ps( __mmask16 __U, __m512 __A)
1576		{
1577		return (__m512)__builtin_ia32_selectps_512(__U,
1578		(__v16sf)_mm512_sqrt_ps(__A),
	1486	static __inline__ __m512 __DEFAULT_FN_ATTRS512
	1487	_mm512_maskz_sqrt_ps(__mmask16 __U, __m512 __A) {
	1488	return (__m512)__builtin_ia32_selectps_512(__U, (__v16sf)_mm512_sqrt_ps(__A),
1579	1489	(__v16sf)_mm512_setzero_ps());
1580	1490	}
1581	1491
...	...	@@ -1866,58 +1776,52 @@ _mm512_mask_ceil_pd (__m512d __W, __mmask8 __U, __m512d __A)
1866	1776	_MM_FROUND_CUR_DIRECTION);
1867	1777	}
1868	1778
1869		static __inline __m512i __DEFAULT_FN_ATTRS512
1870		_mm512_abs_epi64(__m512i __A)
1871		{
	1779	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1780	_mm512_abs_epi64(__m512i __A) {
1872	1781	return (__m512i)__builtin_elementwise_abs((__v8di)__A);
1873	1782	}
1874	1783
1875		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1876		_mm512_mask_abs_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
1877		{
	1784	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1785	_mm512_mask_abs_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
1878	1786	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1879	1787	(__v8di)_mm512_abs_epi64(__A),
1880	1788	(__v8di)__W);
1881	1789	}
1882	1790
1883		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1884		_mm512_maskz_abs_epi64 (__mmask8 __U, __m512i __A)
1885		{
	1791	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1792	_mm512_maskz_abs_epi64(__mmask8 __U, __m512i __A) {
1886	1793	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
1887	1794	(__v8di)_mm512_abs_epi64(__A),
1888	1795	(__v8di)_mm512_setzero_si512());
1889	1796	}
1890	1797
1891		static __inline __m512i __DEFAULT_FN_ATTRS512
1892		_mm512_abs_epi32(__m512i __A)
1893		{
	1798	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1799	_mm512_abs_epi32(__m512i __A) {
1894	1800	return (__m512i)__builtin_elementwise_abs((__v16si) __A);
1895	1801	}
1896	1802
1897		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1898		_mm512_mask_abs_epi32 (__m512i __W, __mmask16 __U, __m512i __A)
1899		{
	1803	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1804	_mm512_mask_abs_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
1900	1805	return (__m512i)__builtin_ia32_selectd_512(__U,
1901	1806	(__v16si)_mm512_abs_epi32(__A),
1902	1807	(__v16si)__W);
1903	1808	}
1904	1809
1905		static __inline__ __m512i __DEFAULT_FN_ATTRS512
1906		_mm512_maskz_abs_epi32 (__mmask16 __U, __m512i __A)
1907		{
	1810	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	1811	_mm512_maskz_abs_epi32(__mmask16 __U, __m512i __A) {
1908	1812	return (__m512i)__builtin_ia32_selectd_512(__U,
1909	1813	(__v16si)_mm512_abs_epi32(__A),
1910	1814	(__v16si)_mm512_setzero_si512());
1911	1815	}
1912	1816
1913		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1914		_mm_mask_add_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	1817	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1818	_mm_mask_add_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
1915	1819	__A = _mm_add_ss(__A, __B);
1916	1820	return __builtin_ia32_selectss_128(__U, __A, __W);
1917	1821	}
1918	1822
1919		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1920		_mm_maskz_add_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	1823	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1824	_mm_maskz_add_ss(__mmask8 __U, __m128 __A, __m128 __B) {
1921	1825	__A = _mm_add_ss(__A, __B);
1922	1826	return __builtin_ia32_selectss_128(__U, __A, _mm_setzero_ps());
1923	1827	}
...	...	@@ -1940,14 +1844,14 @@ _mm_maskz_add_ss(__mmask8 __U,__m128 __A, __m128 __B) {
1940	1844	(__v4sf)_mm_setzero_ps(), \
1941	1845	(__mmask8)(U), (int)(R)))
1942	1846
1943		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1944		_mm_mask_add_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	1847	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1848	_mm_mask_add_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
1945	1849	__A = _mm_add_sd(__A, __B);
1946	1850	return __builtin_ia32_selectsd_128(__U, __A, __W);
1947	1851	}
1948	1852
1949		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1950		_mm_maskz_add_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	1853	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1854	_mm_maskz_add_sd(__mmask8 __U, __m128d __A, __m128d __B) {
1951	1855	__A = _mm_add_sd(__A, __B);
1952	1856	return __builtin_ia32_selectsd_128(__U, __A, _mm_setzero_pd());
1953	1857	}
...	...	@@ -1969,28 +1873,28 @@ _mm_maskz_add_sd(__mmask8 __U,__m128d __A, __m128d __B) {
1969	1873	(__v2df)_mm_setzero_pd(), \
1970	1874	(__mmask8)(U), (int)(R)))
1971	1875
1972		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	1876	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
1973	1877	_mm512_mask_add_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
1974	1878	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
1975	1879	(__v8df)_mm512_add_pd(__A, __B),
1976	1880	(__v8df)__W);
1977	1881	}
1978	1882
1979		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	1883	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
1980	1884	_mm512_maskz_add_pd(__mmask8 __U, __m512d __A, __m512d __B) {
1981	1885	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
1982	1886	(__v8df)_mm512_add_pd(__A, __B),
1983	1887	(__v8df)_mm512_setzero_pd());
1984	1888	}
1985	1889
1986		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	1890	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
1987	1891	_mm512_mask_add_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
1988	1892	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
1989	1893	(__v16sf)_mm512_add_ps(__A, __B),
1990	1894	(__v16sf)__W);
1991	1895	}
1992	1896
1993		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	1897	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
1994	1898	_mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {
1995	1899	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
1996	1900	(__v16sf)_mm512_add_ps(__A, __B),
...	...	@@ -2025,14 +1929,14 @@ _mm512_maskz_add_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2025	1929	(__v16sf)_mm512_add_round_ps((A), (B), (R)), \
2026	1930	(__v16sf)_mm512_setzero_ps()))
2027	1931
2028		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2029		_mm_mask_sub_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	1932	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1933	_mm_mask_sub_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2030	1934	__A = _mm_sub_ss(__A, __B);
2031	1935	return __builtin_ia32_selectss_128(__U, __A, __W);
2032	1936	}
2033	1937
2034		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2035		_mm_maskz_sub_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	1938	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1939	_mm_maskz_sub_ss(__mmask8 __U, __m128 __A, __m128 __B) {
2036	1940	__A = _mm_sub_ss(__A, __B);
2037	1941	return __builtin_ia32_selectss_128(__U, __A, _mm_setzero_ps());
2038	1942	}
...	...	@@ -2054,14 +1958,14 @@ _mm_maskz_sub_ss(__mmask8 __U,__m128 __A, __m128 __B) {
2054	1958	(__v4sf)_mm_setzero_ps(), \
2055	1959	(__mmask8)(U), (int)(R)))
2056	1960
2057		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2058		_mm_mask_sub_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	1961	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1962	_mm_mask_sub_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2059	1963	__A = _mm_sub_sd(__A, __B);
2060	1964	return __builtin_ia32_selectsd_128(__U, __A, __W);
2061	1965	}
2062	1966
2063		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2064		_mm_maskz_sub_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	1967	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1968	_mm_maskz_sub_sd(__mmask8 __U, __m128d __A, __m128d __B) {
2065	1969	__A = _mm_sub_sd(__A, __B);
2066	1970	return __builtin_ia32_selectsd_128(__U, __A, _mm_setzero_pd());
2067	1971	}
...	...	@@ -2084,28 +1988,28 @@ _mm_maskz_sub_sd(__mmask8 __U,__m128d __A, __m128d __B) {
2084	1988	(__v2df)_mm_setzero_pd(), \
2085	1989	(__mmask8)(U), (int)(R)))
2086	1990
2087		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	1991	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2088	1992	_mm512_mask_sub_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
2089	1993	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2090	1994	(__v8df)_mm512_sub_pd(__A, __B),
2091	1995	(__v8df)__W);
2092	1996	}
2093	1997
2094		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	1998	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2095	1999	_mm512_maskz_sub_pd(__mmask8 __U, __m512d __A, __m512d __B) {
2096	2000	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2097	2001	(__v8df)_mm512_sub_pd(__A, __B),
2098	2002	(__v8df)_mm512_setzero_pd());
2099	2003	}
2100	2004
2101		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2005	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2102	2006	_mm512_mask_sub_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
2103	2007	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2104	2008	(__v16sf)_mm512_sub_ps(__A, __B),
2105	2009	(__v16sf)__W);
2106	2010	}
2107	2011
2108		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2012	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2109	2013	_mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2110	2014	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2111	2015	(__v16sf)_mm512_sub_ps(__A, __B),
...	...	@@ -2140,14 +2044,14 @@ _mm512_maskz_sub_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2140	2044	(__v16sf)_mm512_sub_round_ps((A), (B), (R)), \
2141	2045	(__v16sf)_mm512_setzero_ps()))
2142	2046
2143		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2144		_mm_mask_mul_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	2047	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2048	_mm_mask_mul_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2145	2049	__A = _mm_mul_ss(__A, __B);
2146	2050	return __builtin_ia32_selectss_128(__U, __A, __W);
2147	2051	}
2148	2052
2149		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2150		_mm_maskz_mul_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	2053	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2054	_mm_maskz_mul_ss(__mmask8 __U, __m128 __A, __m128 __B) {
2151	2055	__A = _mm_mul_ss(__A, __B);
2152	2056	return __builtin_ia32_selectss_128(__U, __A, _mm_setzero_ps());
2153	2057	}
...	...	@@ -2169,14 +2073,14 @@ _mm_maskz_mul_ss(__mmask8 __U,__m128 __A, __m128 __B) {
2169	2073	(__v4sf)_mm_setzero_ps(), \
2170	2074	(__mmask8)(U), (int)(R)))
2171	2075
2172		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2173		_mm_mask_mul_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	2076	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2077	_mm_mask_mul_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2174	2078	__A = _mm_mul_sd(__A, __B);
2175	2079	return __builtin_ia32_selectsd_128(__U, __A, __W);
2176	2080	}
2177	2081
2178		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2179		_mm_maskz_mul_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	2082	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2083	_mm_maskz_mul_sd(__mmask8 __U, __m128d __A, __m128d __B) {
2180	2084	__A = _mm_mul_sd(__A, __B);
2181	2085	return __builtin_ia32_selectsd_128(__U, __A, _mm_setzero_pd());
2182	2086	}
...	...	@@ -2199,28 +2103,28 @@ _mm_maskz_mul_sd(__mmask8 __U,__m128d __A, __m128d __B) {
2199	2103	(__v2df)_mm_setzero_pd(), \
2200	2104	(__mmask8)(U), (int)(R)))
2201	2105
2202		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	2106	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2203	2107	_mm512_mask_mul_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
2204	2108	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2205	2109	(__v8df)_mm512_mul_pd(__A, __B),
2206	2110	(__v8df)__W);
2207	2111	}
2208	2112
2209		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	2113	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2210	2114	_mm512_maskz_mul_pd(__mmask8 __U, __m512d __A, __m512d __B) {
2211	2115	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2212	2116	(__v8df)_mm512_mul_pd(__A, __B),
2213	2117	(__v8df)_mm512_setzero_pd());
2214	2118	}
2215	2119
2216		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2120	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2217	2121	_mm512_mask_mul_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
2218	2122	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2219	2123	(__v16sf)_mm512_mul_ps(__A, __B),
2220	2124	(__v16sf)__W);
2221	2125	}
2222	2126
2223		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2127	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2224	2128	_mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2225	2129	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2226	2130	(__v16sf)_mm512_mul_ps(__A, __B),
...	...	@@ -2255,14 +2159,14 @@ _mm512_maskz_mul_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2255	2159	(__v16sf)_mm512_mul_round_ps((A), (B), (R)), \
2256	2160	(__v16sf)_mm512_setzero_ps()))
2257	2161
2258		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2259		_mm_mask_div_ss(__m128 __W, __mmask8 __U,__m128 __A, __m128 __B) {
	2162	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2163	_mm_mask_div_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2260	2164	__A = _mm_div_ss(__A, __B);
2261	2165	return __builtin_ia32_selectss_128(__U, __A, __W);
2262	2166	}
2263	2167
2264		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2265		_mm_maskz_div_ss(__mmask8 __U,__m128 __A, __m128 __B) {
	2168	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2169	_mm_maskz_div_ss(__mmask8 __U, __m128 __A, __m128 __B) {
2266	2170	__A = _mm_div_ss(__A, __B);
2267	2171	return __builtin_ia32_selectss_128(__U, __A, _mm_setzero_ps());
2268	2172	}
...	...	@@ -2285,14 +2189,14 @@ _mm_maskz_div_ss(__mmask8 __U,__m128 __A, __m128 __B) {
2285	2189	(__v4sf)_mm_setzero_ps(), \
2286	2190	(__mmask8)(U), (int)(R)))
2287	2191
2288		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2289		_mm_mask_div_sd(__m128d __W, __mmask8 __U,__m128d __A, __m128d __B) {
	2192	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2193	_mm_mask_div_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2290	2194	__A = _mm_div_sd(__A, __B);
2291	2195	return __builtin_ia32_selectsd_128(__U, __A, __W);
2292	2196	}
2293	2197
2294		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2295		_mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) {
	2198	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2199	_mm_maskz_div_sd(__mmask8 __U, __m128d __A, __m128d __B) {
2296	2200	__A = _mm_div_sd(__A, __B);
2297	2201	return __builtin_ia32_selectsd_128(__U, __A, _mm_setzero_pd());
2298	2202	}
...	...	@@ -2315,40 +2219,38 @@ _mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) {
2315	2219	(__v2df)_mm_setzero_pd(), \
2316	2220	(__mmask8)(U), (int)(R)))
2317	2221
2318		static __inline __m512d __DEFAULT_FN_ATTRS512
2319		_mm512_div_pd(__m512d __a, __m512d __b)
2320		{
	2222	static __inline __m512d
	2223	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_div_pd(__m512d __a, __m512d __b) {
2321	2224	return (__m512d)((__v8df)__a/(__v8df)__b);
2322	2225	}
2323	2226
2324		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	2227	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2325	2228	_mm512_mask_div_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
2326	2229	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2327	2230	(__v8df)_mm512_div_pd(__A, __B),
2328	2231	(__v8df)__W);
2329	2232	}
2330	2233
2331		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	2234	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
2332	2235	_mm512_maskz_div_pd(__mmask8 __U, __m512d __A, __m512d __B) {
2333	2236	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
2334	2237	(__v8df)_mm512_div_pd(__A, __B),
2335	2238	(__v8df)_mm512_setzero_pd());
2336	2239	}
2337	2240
2338		static __inline __m512 __DEFAULT_FN_ATTRS512
2339		_mm512_div_ps(__m512 __a, __m512 __b)
2340		{
	2241	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2242	_mm512_div_ps(__m512 __a, __m512 __b) {
2341	2243	return (__m512)((__v16sf)__a/(__v16sf)__b);
2342	2244	}
2343	2245
2344		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2246	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2345	2247	_mm512_mask_div_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
2346	2248	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2347	2249	(__v16sf)_mm512_div_ps(__A, __B),
2348	2250	(__v16sf)__W);
2349	2251	}
2350	2252
2351		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	2253	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
2352	2254	_mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2353	2255	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
2354	2256	(__v16sf)_mm512_div_ps(__A, __B),
...	...	@@ -2530,125 +2432,104 @@ _mm512_maskz_div_ps(__mmask16 __U, __m512 __A, __m512 __B) {
2530	2432	-(__v8df)(__m512d)(C), \
2531	2433	(__mmask8)(U), (int)(R)))
2532	2434
	2435	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2436	_mm512_fmadd_pd(__m512d __A, __m512d __B, __m512d __C) {
	2437	return (__m512d)__builtin_elementwise_fma((__v8df)__A, (__v8df)__B,
	2438	(__v8df)__C);
	2439	}
2533	2440
2534		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2535		_mm512_fmadd_pd(__m512d __A, __m512d __B, __m512d __C)
2536		{
2537		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2538		(__v8df) __B,
2539		(__v8df) __C,
2540		(__mmask8) -1,
2541		_MM_FROUND_CUR_DIRECTION);
	2441	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2442	_mm512_mask_fmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) {
	2443	return (__m512d)__builtin_ia32_selectpd_512(
	2444	(__mmask8)__U, (__v8df)_mm512_fmadd_pd(__A, __B, __C), (__v8df)__A);
2542	2445	}
2543	2446
2544		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2545		_mm512_mask_fmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2546		{
2547		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2548		(__v8df) __B,
2549		(__v8df) __C,
2550		(__mmask8) __U,
2551		_MM_FROUND_CUR_DIRECTION);
	2447	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2448	_mm512_mask3_fmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) {
	2449	return (__m512d)__builtin_ia32_selectpd_512(
	2450	(__mmask8)__U, (__v8df)_mm512_fmadd_pd(__A, __B, __C), (__v8df)__C);
2552	2451	}
2553	2452
2554		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2555		_mm512_mask3_fmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
2556		{
2557		return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A,
2558		(__v8df) __B,
2559		(__v8df) __C,
2560		(__mmask8) __U,
2561		_MM_FROUND_CUR_DIRECTION);
	2453	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2454	_mm512_maskz_fmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) {
	2455	return (__m512d)__builtin_ia32_selectpd_512(
	2456	(__mmask8)__U, (__v8df)_mm512_fmadd_pd(__A, __B, __C),
	2457	(__v8df)_mm512_setzero_pd());
2562	2458	}
2563	2459
2564		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2565		_mm512_maskz_fmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2566		{
2567		return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
2568		(__v8df) __B,
2569		(__v8df) __C,
2570		(__mmask8) __U,
2571		_MM_FROUND_CUR_DIRECTION);
	2460	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2461	_mm512_fmsub_pd(__m512d __A, __m512d __B, __m512d __C) {
	2462	return (__m512d)__builtin_elementwise_fma((__v8df)__A, (__v8df)__B,
	2463	-(__v8df)__C);
2572	2464	}
2573	2465
2574		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2575		_mm512_fmsub_pd(__m512d __A, __m512d __B, __m512d __C)
2576		{
2577		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2578		(__v8df) __B,
2579		-(__v8df) __C,
2580		(__mmask8) -1,
2581		_MM_FROUND_CUR_DIRECTION);
	2466	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2467	_mm512_mask_fmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) {
	2468	return (__m512d)__builtin_ia32_selectpd_512(
	2469	(__mmask8)__U, (__v8df)_mm512_fmsub_pd(__A, __B, __C), (__v8df)__A);
2582	2470	}
2583	2471
2584		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2585		_mm512_mask_fmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
2586		{
2587		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2588		(__v8df) __B,
2589		-(__v8df) __C,
2590		(__mmask8) __U,
2591		_MM_FROUND_CUR_DIRECTION);
	2472	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2473	_mm512_mask3_fmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) {
	2474	return (__m512d)__builtin_ia32_selectpd_512(
	2475	(__mmask8)__U, (__v8df)_mm512_fmsub_pd(__A, __B, __C), (__v8df)__C);
2592	2476	}
2593	2477
2594		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2595		_mm512_maskz_fmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2596		{
2597		return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A,
2598		(__v8df) __B,
2599		-(__v8df) __C,
2600		(__mmask8) __U,
2601		_MM_FROUND_CUR_DIRECTION);
	2478	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2479	_mm512_maskz_fmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) {
	2480	return (__m512d)__builtin_ia32_selectpd_512(
	2481	(__mmask8)__U, (__v8df)_mm512_fmsub_pd(__A, __B, __C),
	2482	(__v8df)_mm512_setzero_pd());
2602	2483	}
2603	2484
2604		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2605		_mm512_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C)
2606		{
2607		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2608		-(__v8df) __B,
2609		(__v8df) __C,
2610		(__mmask8) -1,
2611		_MM_FROUND_CUR_DIRECTION);
	2485	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2486	_mm512_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C) {
	2487	return (__m512d)__builtin_elementwise_fma(-(__v8df)__A, (__v8df)__B,
	2488	(__v8df)__C);
2612	2489	}
2613	2490
2614		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2615		_mm512_mask3_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
2616		{
2617		return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A,
2618		(__v8df) __B,
2619		(__v8df) __C,
2620		(__mmask8) __U,
2621		_MM_FROUND_CUR_DIRECTION);
	2491	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2492	_mm512_mask_fnmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) {
	2493	return (__m512d)__builtin_ia32_selectpd_512(
	2494	(__mmask8)__U, (__v8df)_mm512_fnmadd_pd(__A, __B, __C), (__v8df)__A);
2622	2495	}
2623	2496
2624		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2625		_mm512_maskz_fnmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2626		{
2627		return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
2628		(__v8df) __B,
2629		(__v8df) __C,
2630		(__mmask8) __U,
2631		_MM_FROUND_CUR_DIRECTION);
	2497	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2498	_mm512_mask3_fnmadd_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) {
	2499	return (__m512d)__builtin_ia32_selectpd_512(
	2500	(__mmask8)__U, (__v8df)_mm512_fnmadd_pd(__A, __B, __C), (__v8df)__C);
2632	2501	}
2633	2502
2634		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2635		_mm512_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C)
2636		{
2637		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
2638		-(__v8df) __B,
2639		-(__v8df) __C,
2640		(__mmask8) -1,
2641		_MM_FROUND_CUR_DIRECTION);
	2503	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2504	_mm512_maskz_fnmadd_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) {
	2505	return (__m512d)__builtin_ia32_selectpd_512(
	2506	(__mmask8)__U, (__v8df)_mm512_fnmadd_pd(__A, __B, __C),
	2507	(__v8df)_mm512_setzero_pd());
2642	2508	}
2643	2509
2644		static __inline__ __m512d __DEFAULT_FN_ATTRS512
2645		_mm512_maskz_fnmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2646		{
2647		return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A,
2648		(__v8df) __B,
2649		-(__v8df) __C,
2650		(__mmask8) __U,
2651		_MM_FROUND_CUR_DIRECTION);
	2510	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2511	_mm512_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C) {
	2512	return (__m512d)__builtin_elementwise_fma(-(__v8df)__A, (__v8df)__B,
	2513	-(__v8df)__C);
	2514	}
	2515
	2516	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2517	_mm512_mask_fnmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) {
	2518	return (__m512d)__builtin_ia32_selectpd_512(
	2519	(__mmask8)__U, (__v8df)_mm512_fnmsub_pd(__A, __B, __C), (__v8df)__A);
	2520	}
	2521
	2522	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2523	_mm512_mask3_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) {
	2524	return (__m512d)__builtin_ia32_selectpd_512(
	2525	(__mmask8)__U, (__v8df)_mm512_fnmsub_pd(__A, __B, __C), (__v8df)__C);
	2526	}
	2527
	2528	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	2529	_mm512_maskz_fnmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) {
	2530	return (__m512d)__builtin_ia32_selectpd_512(
	2531	(__mmask8)__U, (__v8df)_mm512_fnmsub_pd(__A, __B, __C),
	2532	(__v8df)_mm512_setzero_pd());
2652	2533	}
2653	2534
2654	2535	#define _mm512_fmadd_round_ps(A, B, C, R) \
...	...	@@ -2734,125 +2615,104 @@ _mm512_maskz_fnmsub_pd(__mmask8 __U, __m512d __A, __m512d __B, __m512d __C)
2734	2615	-(__v16sf)(__m512)(C), \
2735	2616	(__mmask16)(U), (int)(R)))
2736	2617
	2618	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2619	_mm512_fmadd_ps(__m512 __A, __m512 __B, __m512 __C) {
	2620	return (__m512)__builtin_elementwise_fma((__v16sf)__A, (__v16sf)__B,
	2621	(__v16sf)__C);
	2622	}
2737	2623
2738		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2739		_mm512_fmadd_ps(__m512 __A, __m512 __B, __m512 __C)
2740		{
2741		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2742		(__v16sf) __B,
2743		(__v16sf) __C,
2744		(__mmask16) -1,
2745		_MM_FROUND_CUR_DIRECTION);
	2624	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2625	_mm512_mask_fmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) {
	2626	return (__m512)__builtin_ia32_selectps_512(
	2627	(__mmask16)__U, (__v16sf)_mm512_fmadd_ps(__A, __B, __C), (__v16sf)__A);
2746	2628	}
2747	2629
2748		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2749		_mm512_mask_fmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
2750		{
2751		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2752		(__v16sf) __B,
2753		(__v16sf) __C,
2754		(__mmask16) __U,
2755		_MM_FROUND_CUR_DIRECTION);
	2630	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2631	_mm512_mask3_fmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) {
	2632	return (__m512)__builtin_ia32_selectps_512(
	2633	(__mmask16)__U, (__v16sf)_mm512_fmadd_ps(__A, __B, __C), (__v16sf)__C);
2756	2634	}
2757	2635
2758		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2759		_mm512_mask3_fmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
2760		{
2761		return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A,
2762		(__v16sf) __B,
2763		(__v16sf) __C,
2764		(__mmask16) __U,
2765		_MM_FROUND_CUR_DIRECTION);
	2636	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2637	_mm512_maskz_fmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) {
	2638	return (__m512)__builtin_ia32_selectps_512(
	2639	(__mmask16)__U, (__v16sf)_mm512_fmadd_ps(__A, __B, __C),
	2640	(__v16sf)_mm512_setzero_ps());
2766	2641	}
2767	2642
2768		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2769		_mm512_maskz_fmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2770		{
2771		return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
2772		(__v16sf) __B,
2773		(__v16sf) __C,
2774		(__mmask16) __U,
2775		_MM_FROUND_CUR_DIRECTION);
	2643	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2644	_mm512_fmsub_ps(__m512 __A, __m512 __B, __m512 __C) {
	2645	return (__m512)__builtin_elementwise_fma((__v16sf)__A, (__v16sf)__B,
	2646	-(__v16sf)__C);
2776	2647	}
2777	2648
2778		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2779		_mm512_fmsub_ps(__m512 __A, __m512 __B, __m512 __C)
2780		{
2781		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2782		(__v16sf) __B,
2783		-(__v16sf) __C,
2784		(__mmask16) -1,
2785		_MM_FROUND_CUR_DIRECTION);
	2649	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2650	_mm512_mask_fmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) {
	2651	return (__m512)__builtin_ia32_selectps_512(
	2652	(__mmask16)__U, (__v16sf)_mm512_fmsub_ps(__A, __B, __C), (__v16sf)__A);
2786	2653	}
2787	2654
2788		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2789		_mm512_mask_fmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
2790		{
2791		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2792		(__v16sf) __B,
2793		-(__v16sf) __C,
2794		(__mmask16) __U,
2795		_MM_FROUND_CUR_DIRECTION);
	2655	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2656	_mm512_mask3_fmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) {
	2657	return (__m512)__builtin_ia32_selectps_512(
	2658	(__mmask16)__U, (__v16sf)_mm512_fmsub_ps(__A, __B, __C), (__v16sf)__C);
	2659	}
	2660
	2661	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2662	_mm512_maskz_fmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) {
	2663	return (__m512)__builtin_ia32_selectps_512(
	2664	(__mmask16)__U, (__v16sf)_mm512_fmsub_ps(__A, __B, __C),
	2665	(__v16sf)_mm512_setzero_ps());
	2666	}
	2667
	2668	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2669	_mm512_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C) {
	2670	return (__m512)__builtin_elementwise_fma(-(__v16sf)__A, (__v16sf)__B,
	2671	(__v16sf)__C);
2796	2672	}
2797	2673
2798		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2799		_mm512_maskz_fmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2800		{
2801		return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A,
2802		(__v16sf) __B,
2803		-(__v16sf) __C,
2804		(__mmask16) __U,
2805		_MM_FROUND_CUR_DIRECTION);
	2674	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2675	_mm512_mask_fnmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) {
	2676	return (__m512)__builtin_ia32_selectps_512(
	2677	(__mmask16)__U, (__v16sf)_mm512_fnmadd_ps(__A, __B, __C), (__v16sf)__A);
2806	2678	}
2807	2679
2808		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2809		_mm512_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C)
2810		{
2811		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2812		-(__v16sf) __B,
2813		(__v16sf) __C,
2814		(__mmask16) -1,
2815		_MM_FROUND_CUR_DIRECTION);
	2680	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2681	_mm512_mask3_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) {
	2682	return (__m512)__builtin_ia32_selectps_512(
	2683	(__mmask16)__U, (__v16sf)_mm512_fnmadd_ps(__A, __B, __C), (__v16sf)__C);
2816	2684	}
2817	2685
2818		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2819		_mm512_mask3_fnmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
2820		{
2821		return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A,
2822		(__v16sf) __B,
2823		(__v16sf) __C,
2824		(__mmask16) __U,
2825		_MM_FROUND_CUR_DIRECTION);
	2686	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2687	_mm512_maskz_fnmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) {
	2688	return (__m512)__builtin_ia32_selectps_512(
	2689	(__mmask16)__U, (__v16sf)_mm512_fnmadd_ps(__A, __B, __C),
	2690	(__v16sf)_mm512_setzero_ps());
2826	2691	}
2827	2692
2828		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2829		_mm512_maskz_fnmadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2830		{
2831		return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
2832		(__v16sf) __B,
2833		(__v16sf) __C,
2834		(__mmask16) __U,
2835		_MM_FROUND_CUR_DIRECTION);
	2693	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2694	_mm512_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C) {
	2695	return (__m512)__builtin_elementwise_fma(-(__v16sf)__A, (__v16sf)__B,
	2696	-(__v16sf)__C);
2836	2697	}
2837	2698
2838		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2839		_mm512_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C)
2840		{
2841		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
2842		-(__v16sf) __B,
2843		-(__v16sf) __C,
2844		(__mmask16) -1,
2845		_MM_FROUND_CUR_DIRECTION);
	2699	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2700	_mm512_mask_fnmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) {
	2701	return (__m512)__builtin_ia32_selectps_512(
	2702	(__mmask16)__U, (__v16sf)_mm512_fnmsub_ps(__A, __B, __C), (__v16sf)__A);
2846	2703	}
2847	2704
2848		static __inline__ __m512 __DEFAULT_FN_ATTRS512
2849		_mm512_maskz_fnmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
2850		{
2851		return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A,
2852		(__v16sf) __B,
2853		-(__v16sf) __C,
2854		(__mmask16) __U,
2855		_MM_FROUND_CUR_DIRECTION);
	2705	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2706	_mm512_mask3_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) {
	2707	return (__m512)__builtin_ia32_selectps_512(
	2708	(__mmask16)__U, (__v16sf)_mm512_fnmsub_ps(__A, __B, __C), (__v16sf)__C);
	2709	}
	2710
	2711	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	2712	_mm512_maskz_fnmsub_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) {
	2713	return (__m512)__builtin_ia32_selectps_512(
	2714	(__mmask16)__U, (__v16sf)_mm512_fnmsub_ps(__A, __B, __C),
	2715	(__v16sf)_mm512_setzero_ps());
2856	2716	}
2857	2717
2858	2718	#define _mm512_fmaddsub_round_pd(A, B, C, R) \
...	...	@@ -3099,33 +2959,12 @@ _mm512_maskz_fmsubadd_ps(__mmask16 __U, __m512 __A, __m512 __B, __m512 __C)
3099	2959	(__v8df)(__m512d)(C), \
3100	2960	(__mmask8)(U), (int)(R)))
3101	2961
3102
3103		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3104		_mm512_mask3_fmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3105		{
3106		return (__m512d)__builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A,
3107		(__v8df) __B,
3108		(__v8df) __C,
3109		(__mmask8) __U,
3110		_MM_FROUND_CUR_DIRECTION);
3111		}
3112
3113	2962	#define _mm512_mask3_fmsub_round_ps(A, B, C, U, R) \
3114	2963	((__m512)__builtin_ia32_vfmsubps512_mask3((__v16sf)(__m512)(A), \
3115	2964	(__v16sf)(__m512)(B), \
3116	2965	(__v16sf)(__m512)(C), \
3117	2966	(__mmask16)(U), (int)(R)))
3118	2967
3119		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3120		_mm512_mask3_fmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3121		{
3122		return (__m512)__builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A,
3123		(__v16sf) __B,
3124		(__v16sf) __C,
3125		(__mmask16) __U,
3126		_MM_FROUND_CUR_DIRECTION);
3127		}
3128
3129	2968	#define _mm512_mask3_fmsubadd_round_pd(A, B, C, U, R) \
3130	2969	((__m512d)__builtin_ia32_vfmsubaddpd512_mask3((__v8df)(__m512d)(A), \
3131	2970	(__v8df)(__m512d)(B), \
...	...	@@ -3166,34 +3005,12 @@ _mm512_mask3_fmsubadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3166	3005	(__v8df)(__m512d)(C), \
3167	3006	(__mmask8)(U), (int)(R)))
3168	3007
3169
3170		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3171		_mm512_mask_fnmadd_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
3172		{
3173		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
3174		-(__v8df) __B,
3175		(__v8df) __C,
3176		(__mmask8) __U,
3177		_MM_FROUND_CUR_DIRECTION);
3178		}
3179
3180	3008	#define _mm512_mask_fnmadd_round_ps(A, U, B, C, R) \
3181	3009	((__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
3182	3010	-(__v16sf)(__m512)(B), \
3183	3011	(__v16sf)(__m512)(C), \
3184	3012	(__mmask16)(U), (int)(R)))
3185	3013
3186
3187		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3188		_mm512_mask_fnmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3189		{
3190		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
3191		-(__v16sf) __B,
3192		(__v16sf) __C,
3193		(__mmask16) __U,
3194		_MM_FROUND_CUR_DIRECTION);
3195		}
3196
3197	3014	#define _mm512_mask_fnmsub_round_pd(A, U, B, C, R) \
3198	3015	((__m512d)__builtin_ia32_vfmaddpd512_mask((__v8df)(__m512d)(A), \
3199	3016	-(__v8df)(__m512d)(B), \
...	...	@@ -3207,27 +3024,6 @@ _mm512_mask_fnmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3207	3024	(__v8df)(__m512d)(C), \
3208	3025	(__mmask8)(U), (int)(R)))
3209	3026
3210
3211		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3212		_mm512_mask_fnmsub_pd(__m512d __A, __mmask8 __U, __m512d __B, __m512d __C)
3213		{
3214		return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A,
3215		-(__v8df) __B,
3216		-(__v8df) __C,
3217		(__mmask8) __U,
3218		_MM_FROUND_CUR_DIRECTION);
3219		}
3220
3221		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3222		_mm512_mask3_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3223		{
3224		return (__m512d) __builtin_ia32_vfmsubpd512_mask3 (-(__v8df) __A,
3225		(__v8df) __B,
3226		(__v8df) __C,
3227		(__mmask8) __U,
3228		_MM_FROUND_CUR_DIRECTION);
3229		}
3230
3231	3027	#define _mm512_mask_fnmsub_round_ps(A, U, B, C, R) \
3232	3028	((__m512)__builtin_ia32_vfmaddps512_mask((__v16sf)(__m512)(A), \
3233	3029	-(__v16sf)(__m512)(B), \
...	...	@@ -3241,94 +3037,63 @@ _mm512_mask3_fnmsub_pd(__m512d __A, __m512d __B, __m512d __C, __mmask8 __U)
3241	3037	(__v16sf)(__m512)(C), \
3242	3038	(__mmask16)(U), (int)(R)))
3243	3039
3244
3245		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3246		_mm512_mask_fnmsub_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C)
3247		{
3248		return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A,
3249		-(__v16sf) __B,
3250		-(__v16sf) __C,
3251		(__mmask16) __U,
3252		_MM_FROUND_CUR_DIRECTION);
3253		}
3254
3255		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3256		_mm512_mask3_fnmsub_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U)
3257		{
3258		return (__m512) __builtin_ia32_vfmsubps512_mask3 (-(__v16sf) __A,
3259		(__v16sf) __B,
3260		(__v16sf) __C,
3261		(__mmask16) __U,
3262		_MM_FROUND_CUR_DIRECTION);
3263		}
3264
3265
3266
3267	3040	/* Vector permutations */
3268	3041
3269		static __inline __m512i __DEFAULT_FN_ATTRS512
3270		_mm512_permutex2var_epi32(__m512i __A, __m512i __I, __m512i __B)
3271		{
	3042	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3043	_mm512_permutex2var_epi32(__m512i __A, __m512i __I, __m512i __B) {
3272	3044	return (__m512i)__builtin_ia32_vpermi2vard512((__v16si)__A, (__v16si) __I,
3273	3045	(__v16si) __B);
3274	3046	}
3275	3047
3276		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3048	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3277	3049	_mm512_mask_permutex2var_epi32(__m512i __A, __mmask16 __U, __m512i __I,
3278		__m512i __B)
3279		{
	3050	__m512i __B) {
3280	3051	return (__m512i)__builtin_ia32_selectd_512(__U,
3281	3052	(__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3282	3053	(__v16si)__A);
3283	3054	}
3284	3055
3285		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3056	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3286	3057	_mm512_mask2_permutex2var_epi32(__m512i __A, __m512i __I, __mmask16 __U,
3287		__m512i __B)
3288		{
	3058	__m512i __B) {
3289	3059	return (__m512i)__builtin_ia32_selectd_512(__U,
3290	3060	(__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3291	3061	(__v16si)__I);
3292	3062	}
3293	3063
3294		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3064	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3295	3065	_mm512_maskz_permutex2var_epi32(__mmask16 __U, __m512i __A, __m512i __I,
3296		__m512i __B)
3297		{
	3066	__m512i __B) {
3298	3067	return (__m512i)__builtin_ia32_selectd_512(__U,
3299	3068	(__v16si)_mm512_permutex2var_epi32(__A, __I, __B),
3300	3069	(__v16si)_mm512_setzero_si512());
3301	3070	}
3302	3071
3303		static __inline __m512i __DEFAULT_FN_ATTRS512
3304		_mm512_permutex2var_epi64(__m512i __A, __m512i __I, __m512i __B)
3305		{
	3072	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3073	_mm512_permutex2var_epi64(__m512i __A, __m512i __I, __m512i __B) {
3306	3074	return (__m512i)__builtin_ia32_vpermi2varq512((__v8di)__A, (__v8di) __I,
3307	3075	(__v8di) __B);
3308	3076	}
3309	3077
3310		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3078	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3311	3079	_mm512_mask_permutex2var_epi64(__m512i __A, __mmask8 __U, __m512i __I,
3312		__m512i __B)
3313		{
	3080	__m512i __B) {
3314	3081	return (__m512i)__builtin_ia32_selectq_512(__U,
3315	3082	(__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3316	3083	(__v8di)__A);
3317	3084	}
3318	3085
3319		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3086	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3320	3087	_mm512_mask2_permutex2var_epi64(__m512i __A, __m512i __I, __mmask8 __U,
3321		__m512i __B)
3322		{
	3088	__m512i __B) {
3323	3089	return (__m512i)__builtin_ia32_selectq_512(__U,
3324	3090	(__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3325	3091	(__v8di)__I);
3326	3092	}
3327	3093
3328		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	3094	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
3329	3095	_mm512_maskz_permutex2var_epi64(__mmask8 __U, __m512i __A, __m512i __I,
3330		__m512i __B)
3331		{
	3096	__m512i __B) {
3332	3097	return (__m512i)__builtin_ia32_selectq_512(__U,
3333	3098	(__v8di)_mm512_permutex2var_epi64(__A, __I, __B),
3334	3099	(__v8di)_mm512_setzero_si512());
...	...	@@ -3363,10 +3128,10 @@ _mm512_maskz_permutex2var_epi64(__mmask8 __U, __m512i __A, __m512i __I,
3363	3128	(__v16si)_mm512_setzero_si512()))
3364	3129	/* Vector Extract */
3365	3130
3366		#define _mm512_extractf64x4_pd(A, I) \
3367		((__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(I), \
3368		(__v4df)_mm256_undefined_pd(), \
3369		(__mmask8)-1))
	3131	#define _mm512_extractf64x4_pd(A, I) \
	3132	((__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(I), \
	3133	(__v4df)_mm256_setzero_pd(), \
	3134	(__mmask8) - 1))
3370	3135
3371	3136	#define _mm512_mask_extractf64x4_pd(W, U, A, imm) \
3372	3137	((__m256d)__builtin_ia32_extractf64x4_mask((__v8df)(__m512d)(A), (int)(imm), \
...	...	@@ -3378,10 +3143,10 @@ _mm512_maskz_permutex2var_epi64(__mmask8 __U, __m512i __A, __m512i __I,
3378	3143	(__v4df)_mm256_setzero_pd(), \
3379	3144	(__mmask8)(U)))
3380	3145
3381		#define _mm512_extractf32x4_ps(A, I) \
3382		((__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(I), \
3383		(__v4sf)_mm_undefined_ps(), \
3384		(__mmask8)-1))
	3146	#define _mm512_extractf32x4_ps(A, I) \
	3147	((__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(I), \
	3148	(__v4sf)_mm_setzero_ps(), \
	3149	(__mmask8) - 1))
3385	3150
3386	3151	#define _mm512_mask_extractf32x4_ps(W, U, A, imm) \
3387	3152	((__m128)__builtin_ia32_extractf32x4_mask((__v16sf)(__m512)(A), (int)(imm), \
...	...	@@ -3395,33 +3160,29 @@ _mm512_maskz_permutex2var_epi64(__mmask8 __U, __m512i __A, __m512i __I,
3395	3160
3396	3161	/* Vector Blend */
3397	3162
3398		static __inline __m512d __DEFAULT_FN_ATTRS512
3399		_mm512_mask_blend_pd(__mmask8 __U, __m512d __A, __m512d __W)
3400		{
	3163	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3164	_mm512_mask_blend_pd(__mmask8 __U, __m512d __A, __m512d __W) {
3401	3165	return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
3402	3166	(__v8df) __W,
3403	3167	(__v8df) __A);
3404	3168	}
3405	3169
3406		static __inline __m512 __DEFAULT_FN_ATTRS512
3407		_mm512_mask_blend_ps(__mmask16 __U, __m512 __A, __m512 __W)
3408		{
	3170	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3171	_mm512_mask_blend_ps(__mmask16 __U, __m512 __A, __m512 __W) {
3409	3172	return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
3410	3173	(__v16sf) __W,
3411	3174	(__v16sf) __A);
3412	3175	}
3413	3176
3414		static __inline __m512i __DEFAULT_FN_ATTRS512
3415		_mm512_mask_blend_epi64(__mmask8 __U, __m512i __A, __m512i __W)
3416		{
	3177	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3178	_mm512_mask_blend_epi64(__mmask8 __U, __m512i __A, __m512i __W) {
3417	3179	return (__m512i) __builtin_ia32_selectq_512 ((__mmask8) __U,
3418	3180	(__v8di) __W,
3419	3181	(__v8di) __A);
3420	3182	}
3421	3183
3422		static __inline __m512i __DEFAULT_FN_ATTRS512
3423		_mm512_mask_blend_epi32(__mmask16 __U, __m512i __A, __m512i __W)
3424		{
	3184	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3185	_mm512_mask_blend_epi32(__mmask16 __U, __m512i __A, __m512i __W) {
3425	3186	return (__m512i) __builtin_ia32_selectd_512 ((__mmask16) __U,
3426	3187	(__v16si) __W,
3427	3188	(__v16si) __A);
...	...	@@ -3615,115 +3376,99 @@ _mm512_maskz_cvttps_epu32 (__mmask16 __U, __m512 __A)
3615	3376	(__v16sf)_mm512_setzero_ps(), \
3616	3377	(__mmask16)(U), (int)(R)))
3617	3378
3618		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3619		_mm512_cvtepu32_ps (__m512i __A)
3620		{
	3379	static __inline__ __m512
	3380	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_cvtepu32_ps(__m512i __A) {
3621	3381	return (__m512)__builtin_convertvector((__v16su)__A, __v16sf);
3622	3382	}
3623	3383
3624		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3625		_mm512_mask_cvtepu32_ps (__m512 __W, __mmask16 __U, __m512i __A)
3626		{
	3384	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3385	_mm512_mask_cvtepu32_ps(__m512 __W, __mmask16 __U, __m512i __A) {
3627	3386	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3628	3387	(__v16sf)_mm512_cvtepu32_ps(__A),
3629	3388	(__v16sf)__W);
3630	3389	}
3631	3390
3632		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3633		_mm512_maskz_cvtepu32_ps (__mmask16 __U, __m512i __A)
3634		{
	3391	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3392	_mm512_maskz_cvtepu32_ps(__mmask16 __U, __m512i __A) {
3635	3393	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3636	3394	(__v16sf)_mm512_cvtepu32_ps(__A),
3637	3395	(__v16sf)_mm512_setzero_ps());
3638	3396	}
3639	3397
3640		static __inline __m512d __DEFAULT_FN_ATTRS512
3641		_mm512_cvtepi32_pd(__m256i __A)
3642		{
	3398	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3399	_mm512_cvtepi32_pd(__m256i __A) {
3643	3400	return (__m512d)__builtin_convertvector((__v8si)__A, __v8df);
3644	3401	}
3645	3402
3646		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3647		_mm512_mask_cvtepi32_pd (__m512d __W, __mmask8 __U, __m256i __A)
3648		{
	3403	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3404	_mm512_mask_cvtepi32_pd(__m512d __W, __mmask8 __U, __m256i __A) {
3649	3405	return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3650	3406	(__v8df)_mm512_cvtepi32_pd(__A),
3651	3407	(__v8df)__W);
3652	3408	}
3653	3409
3654		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3655		_mm512_maskz_cvtepi32_pd (__mmask8 __U, __m256i __A)
3656		{
	3410	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3411	_mm512_maskz_cvtepi32_pd(__mmask8 __U, __m256i __A) {
3657	3412	return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3658	3413	(__v8df)_mm512_cvtepi32_pd(__A),
3659	3414	(__v8df)_mm512_setzero_pd());
3660	3415	}
3661	3416
3662		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3663		_mm512_cvtepi32lo_pd(__m512i __A)
3664		{
	3417	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3418	_mm512_cvtepi32lo_pd(__m512i __A) {
3665	3419	return (__m512d) _mm512_cvtepi32_pd(_mm512_castsi512_si256(__A));
3666	3420	}
3667	3421
3668		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3669		_mm512_mask_cvtepi32lo_pd(__m512d __W, __mmask8 __U,__m512i __A)
3670		{
	3422	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3423	_mm512_mask_cvtepi32lo_pd(__m512d __W, __mmask8 __U, __m512i __A) {
3671	3424	return (__m512d) _mm512_mask_cvtepi32_pd(__W, __U, _mm512_castsi512_si256(__A));
3672	3425	}
3673	3426
3674		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3675		_mm512_cvtepi32_ps (__m512i __A)
3676		{
	3427	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3428	_mm512_cvtepi32_ps(__m512i __A) {
3677	3429	return (__m512)__builtin_convertvector((__v16si)__A, __v16sf);
3678	3430	}
3679	3431
3680		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3681		_mm512_mask_cvtepi32_ps (__m512 __W, __mmask16 __U, __m512i __A)
3682		{
	3432	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3433	_mm512_mask_cvtepi32_ps(__m512 __W, __mmask16 __U, __m512i __A) {
3683	3434	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3684	3435	(__v16sf)_mm512_cvtepi32_ps(__A),
3685	3436	(__v16sf)__W);
3686	3437	}
3687	3438
3688		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3689		_mm512_maskz_cvtepi32_ps (__mmask16 __U, __m512i __A)
3690		{
	3439	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3440	_mm512_maskz_cvtepi32_ps(__mmask16 __U, __m512i __A) {
3691	3441	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
3692	3442	(__v16sf)_mm512_cvtepi32_ps(__A),
3693	3443	(__v16sf)_mm512_setzero_ps());
3694	3444	}
3695	3445
3696		static __inline __m512d __DEFAULT_FN_ATTRS512
3697		_mm512_cvtepu32_pd(__m256i __A)
3698		{
	3446	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3447	_mm512_cvtepu32_pd(__m256i __A) {
3699	3448	return (__m512d)__builtin_convertvector((__v8su)__A, __v8df);
3700	3449	}
3701	3450
3702		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3703		_mm512_mask_cvtepu32_pd (__m512d __W, __mmask8 __U, __m256i __A)
3704		{
	3451	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3452	_mm512_mask_cvtepu32_pd(__m512d __W, __mmask8 __U, __m256i __A) {
3705	3453	return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3706	3454	(__v8df)_mm512_cvtepu32_pd(__A),
3707	3455	(__v8df)__W);
3708	3456	}
3709	3457
3710		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3711		_mm512_maskz_cvtepu32_pd (__mmask8 __U, __m256i __A)
3712		{
	3458	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3459	_mm512_maskz_cvtepu32_pd(__mmask8 __U, __m256i __A) {
3713	3460	return (__m512d)__builtin_ia32_selectpd_512((__mmask8) __U,
3714	3461	(__v8df)_mm512_cvtepu32_pd(__A),
3715	3462	(__v8df)_mm512_setzero_pd());
3716	3463	}
3717	3464
3718		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3719		_mm512_cvtepu32lo_pd(__m512i __A)
3720		{
	3465	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3466	_mm512_cvtepu32lo_pd(__m512i __A) {
3721	3467	return (__m512d) _mm512_cvtepu32_pd(_mm512_castsi512_si256(__A));
3722	3468	}
3723	3469
3724		static __inline__ __m512d __DEFAULT_FN_ATTRS512
3725		_mm512_mask_cvtepu32lo_pd(__m512d __W, __mmask8 __U,__m512i __A)
3726		{
	3470	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3471	_mm512_mask_cvtepu32lo_pd(__m512d __W, __mmask8 __U, __m512i __A) {
3727	3472	return (__m512d) _mm512_mask_cvtepu32_pd(__W, __U, _mm512_castsi512_si256(__A));
3728	3473	}
3729	3474
...	...	@@ -3742,44 +3487,38 @@ _mm512_mask_cvtepu32lo_pd(__m512d __W, __mmask8 __U,__m512i __A)
3742	3487	(__v8sf)_mm256_setzero_ps(), \
3743	3488	(__mmask8)(U), (int)(R)))
3744	3489
3745		static __inline__ __m256 __DEFAULT_FN_ATTRS512
3746		_mm512_cvtpd_ps (__m512d __A)
3747		{
3748		return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3749		(__v8sf) _mm256_undefined_ps (),
3750		(__mmask8) -1,
3751		_MM_FROUND_CUR_DIRECTION);
	3490	static __inline__ __m256
	3491	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_cvtpd_ps(__m512d __A) {
	3492	return (__m256)__builtin_ia32_cvtpd2ps512_mask(
	3493	(__v8df)__A, (__v8sf)_mm256_setzero_ps(), (__mmask8)-1,
	3494	_MM_FROUND_CUR_DIRECTION);
3752	3495	}
3753	3496
3754		static __inline__ __m256 __DEFAULT_FN_ATTRS512
3755		_mm512_mask_cvtpd_ps (__m256 __W, __mmask8 __U, __m512d __A)
3756		{
	3497	static __inline__ __m256 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3498	_mm512_mask_cvtpd_ps(__m256 __W, __mmask8 __U, __m512d __A) {
3757	3499	return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3758	3500	(__v8sf) __W,
3759	3501	(__mmask8) __U,
3760	3502	_MM_FROUND_CUR_DIRECTION);
3761	3503	}
3762	3504
3763		static __inline__ __m256 __DEFAULT_FN_ATTRS512
3764		_mm512_maskz_cvtpd_ps (__mmask8 __U, __m512d __A)
3765		{
	3505	static __inline__ __m256 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3506	_mm512_maskz_cvtpd_ps(__mmask8 __U, __m512d __A) {
3766	3507	return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A,
3767	3508	(__v8sf) _mm256_setzero_ps (),
3768	3509	(__mmask8) __U,
3769	3510	_MM_FROUND_CUR_DIRECTION);
3770	3511	}
3771	3512
3772		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3773		_mm512_cvtpd_pslo (__m512d __A)
3774		{
	3513	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3514	_mm512_cvtpd_pslo(__m512d __A) {
3775	3515	return (__m512) __builtin_shufflevector((__v8sf) _mm512_cvtpd_ps(__A),
3776	3516	(__v8sf) _mm256_setzero_ps (),
3777	3517	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
3778	3518	}
3779	3519
3780		static __inline__ __m512 __DEFAULT_FN_ATTRS512
3781		_mm512_mask_cvtpd_pslo (__m512 __W, __mmask8 __U,__m512d __A)
3782		{
	3520	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3521	_mm512_mask_cvtpd_pslo(__m512 __W, __mmask8 __U, __m512d __A) {
3783	3522	return (__m512) __builtin_shufflevector (
3784	3523	(__v8sf) _mm512_mask_cvtpd_ps (_mm512_castps512_ps256(__W),
3785	3524	__U, __A),
...	...	@@ -4123,9 +3862,8 @@ _mm512_cvtss_f32(__m512 __a)
4123	3862
4124	3863	/* Unpack and Interleave */
4125	3864
4126		static __inline __m512d __DEFAULT_FN_ATTRS512
4127		_mm512_unpackhi_pd(__m512d __a, __m512d __b)
4128		{
	3865	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3866	_mm512_unpackhi_pd(__m512d __a, __m512d __b) {
4129	3867	return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
4130	3868	1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
4131	3869	}
...	...	@@ -4146,9 +3884,8 @@ _mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B)
4146	3884	(__v8df)_mm512_setzero_pd());
4147	3885	}
4148	3886
4149		static __inline __m512d __DEFAULT_FN_ATTRS512
4150		_mm512_unpacklo_pd(__m512d __a, __m512d __b)
4151		{
	3887	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	3888	_mm512_unpacklo_pd(__m512d __a, __m512d __b) {
4152	3889	return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
4153	3890	0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
4154	3891	}
...	...	@@ -4169,9 +3906,8 @@ _mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B)
4169	3906	(__v8df)_mm512_setzero_pd());
4170	3907	}
4171	3908
4172		static __inline __m512 __DEFAULT_FN_ATTRS512
4173		_mm512_unpackhi_ps(__m512 __a, __m512 __b)
4174		{
	3909	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3910	_mm512_unpackhi_ps(__m512 __a, __m512 __b) {
4175	3911	return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
4176	3912	2, 18, 3, 19,
4177	3913	2+4, 18+4, 3+4, 19+4,
...	...	@@ -4195,9 +3931,8 @@ _mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B)
4195	3931	(__v16sf)_mm512_setzero_ps());
4196	3932	}
4197	3933
4198		static __inline __m512 __DEFAULT_FN_ATTRS512
4199		_mm512_unpacklo_ps(__m512 __a, __m512 __b)
4200		{
	3934	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	3935	_mm512_unpacklo_ps(__m512 __a, __m512 __b) {
4201	3936	return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
4202	3937	0, 16, 1, 17,
4203	3938	0+4, 16+4, 1+4, 17+4,
...	...	@@ -4221,9 +3956,8 @@ _mm512_maskz_unpacklo_ps (__mmask16 __U, __m512 __A, __m512 __B)
4221	3956	(__v16sf)_mm512_setzero_ps());
4222	3957	}
4223	3958
4224		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4225		_mm512_unpackhi_epi32(__m512i __A, __m512i __B)
4226		{
	3959	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3960	_mm512_unpackhi_epi32(__m512i __A, __m512i __B) {
4227	3961	return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
4228	3962	2, 18, 3, 19,
4229	3963	2+4, 18+4, 3+4, 19+4,
...	...	@@ -4247,9 +3981,8 @@ _mm512_maskz_unpackhi_epi32(__mmask16 __U, __m512i __A, __m512i __B)
4247	3981	(__v16si)_mm512_setzero_si512());
4248	3982	}
4249	3983
4250		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4251		_mm512_unpacklo_epi32(__m512i __A, __m512i __B)
4252		{
	3984	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	3985	_mm512_unpacklo_epi32(__m512i __A, __m512i __B) {
4253	3986	return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
4254	3987	0, 16, 1, 17,
4255	3988	0+4, 16+4, 1+4, 17+4,
...	...	@@ -4273,9 +4006,8 @@ _mm512_maskz_unpacklo_epi32(__mmask16 __U, __m512i __A, __m512i __B)
4273	4006	(__v16si)_mm512_setzero_si512());
4274	4007	}
4275	4008
4276		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4277		_mm512_unpackhi_epi64(__m512i __A, __m512i __B)
4278		{
	4009	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4010	_mm512_unpackhi_epi64(__m512i __A, __m512i __B) {
4279	4011	return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
4280	4012	1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
4281	4013	}
...	...	@@ -4296,9 +4028,8 @@ _mm512_maskz_unpackhi_epi64(__mmask8 __U, __m512i __A, __m512i __B)
4296	4028	(__v8di)_mm512_setzero_si512());
4297	4029	}
4298	4030
4299		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4300		_mm512_unpacklo_epi64 (__m512i __A, __m512i __B)
4301		{
	4031	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4032	_mm512_unpacklo_epi64(__m512i __A, __m512i __B) {
4302	4033	return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
4303	4034	0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
4304	4035	}
...	...	@@ -4619,9 +4350,8 @@ _mm512_store_epi64 (void *__P, __m512i __A)
4619	4350
4620	4351	/* Mask ops */
4621	4352
4622		static __inline __mmask16 __DEFAULT_FN_ATTRS
4623		_mm512_knot(__mmask16 __M)
4624		{
	4353	static __inline __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	4354	_mm512_knot(__mmask16 __M) {
4625	4355	return __builtin_ia32_knothi(__M);
4626	4356	}
4627	4357
...	...	@@ -4727,237 +4457,207 @@ _mm512_knot(__mmask16 __M)
4727	4457	#define _mm512_mask_cmpneq_epu64_mask(k, A, B) \
4728	4458	_mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_NE)
4729	4459
4730		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4731		_mm512_cvtepi8_epi32(__m128i __A)
4732		{
	4460	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4461	_mm512_cvtepi8_epi32(__m128i __A) {
4733	4462	/* This function always performs a signed extension, but __v16qi is a char
4734	4463	which may be signed or unsigned, so use __v16qs. */
4735	4464	return (__m512i)__builtin_convertvector((__v16qs)__A, __v16si);
4736	4465	}
4737	4466
4738		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4739		_mm512_mask_cvtepi8_epi32(__m512i __W, __mmask16 __U, __m128i __A)
4740		{
	4467	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4468	_mm512_mask_cvtepi8_epi32(__m512i __W, __mmask16 __U, __m128i __A) {
4741	4469	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4742	4470	(__v16si)_mm512_cvtepi8_epi32(__A),
4743	4471	(__v16si)__W);
4744	4472	}
4745	4473
4746		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4747		_mm512_maskz_cvtepi8_epi32(__mmask16 __U, __m128i __A)
4748		{
	4474	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4475	_mm512_maskz_cvtepi8_epi32(__mmask16 __U, __m128i __A) {
4749	4476	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4750	4477	(__v16si)_mm512_cvtepi8_epi32(__A),
4751	4478	(__v16si)_mm512_setzero_si512());
4752	4479	}
4753	4480
4754		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4755		_mm512_cvtepi8_epi64(__m128i __A)
4756		{
	4481	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4482	_mm512_cvtepi8_epi64(__m128i __A) {
4757	4483	/* This function always performs a signed extension, but __v16qi is a char
4758	4484	which may be signed or unsigned, so use __v16qs. */
4759	4485	return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__A, (__v16qs)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
4760	4486	}
4761	4487
4762		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4763		_mm512_mask_cvtepi8_epi64(__m512i __W, __mmask8 __U, __m128i __A)
4764		{
	4488	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4489	_mm512_mask_cvtepi8_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
4765	4490	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4766	4491	(__v8di)_mm512_cvtepi8_epi64(__A),
4767	4492	(__v8di)__W);
4768	4493	}
4769	4494
4770		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4771		_mm512_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A)
4772		{
	4495	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4496	_mm512_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A) {
4773	4497	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4774	4498	(__v8di)_mm512_cvtepi8_epi64(__A),
4775	4499	(__v8di)_mm512_setzero_si512 ());
4776	4500	}
4777	4501
4778		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4779		_mm512_cvtepi32_epi64(__m256i __X)
4780		{
	4502	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4503	_mm512_cvtepi32_epi64(__m256i __X) {
4781	4504	return (__m512i)__builtin_convertvector((__v8si)__X, __v8di);
4782	4505	}
4783	4506
4784		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4785		_mm512_mask_cvtepi32_epi64(__m512i __W, __mmask8 __U, __m256i __X)
4786		{
	4507	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4508	_mm512_mask_cvtepi32_epi64(__m512i __W, __mmask8 __U, __m256i __X) {
4787	4509	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4788	4510	(__v8di)_mm512_cvtepi32_epi64(__X),
4789	4511	(__v8di)__W);
4790	4512	}
4791	4513
4792		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4793		_mm512_maskz_cvtepi32_epi64(__mmask8 __U, __m256i __X)
4794		{
	4514	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4515	_mm512_maskz_cvtepi32_epi64(__mmask8 __U, __m256i __X) {
4795	4516	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4796	4517	(__v8di)_mm512_cvtepi32_epi64(__X),
4797	4518	(__v8di)_mm512_setzero_si512());
4798	4519	}
4799	4520
4800		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4801		_mm512_cvtepi16_epi32(__m256i __A)
4802		{
	4521	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4522	_mm512_cvtepi16_epi32(__m256i __A) {
4803	4523	return (__m512i)__builtin_convertvector((__v16hi)__A, __v16si);
4804	4524	}
4805	4525
4806		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4807		_mm512_mask_cvtepi16_epi32(__m512i __W, __mmask16 __U, __m256i __A)
4808		{
	4526	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4527	_mm512_mask_cvtepi16_epi32(__m512i __W, __mmask16 __U, __m256i __A) {
4809	4528	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4810	4529	(__v16si)_mm512_cvtepi16_epi32(__A),
4811	4530	(__v16si)__W);
4812	4531	}
4813	4532
4814		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4815		_mm512_maskz_cvtepi16_epi32(__mmask16 __U, __m256i __A)
4816		{
	4533	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4534	_mm512_maskz_cvtepi16_epi32(__mmask16 __U, __m256i __A) {
4817	4535	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4818	4536	(__v16si)_mm512_cvtepi16_epi32(__A),
4819	4537	(__v16si)_mm512_setzero_si512 ());
4820	4538	}
4821	4539
4822		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4823		_mm512_cvtepi16_epi64(__m128i __A)
4824		{
	4540	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4541	_mm512_cvtepi16_epi64(__m128i __A) {
4825	4542	return (__m512i)__builtin_convertvector((__v8hi)__A, __v8di);
4826	4543	}
4827	4544
4828		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4829		_mm512_mask_cvtepi16_epi64(__m512i __W, __mmask8 __U, __m128i __A)
4830		{
	4545	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4546	_mm512_mask_cvtepi16_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
4831	4547	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4832	4548	(__v8di)_mm512_cvtepi16_epi64(__A),
4833	4549	(__v8di)__W);
4834	4550	}
4835	4551
4836		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4837		_mm512_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A)
4838		{
	4552	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4553	_mm512_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A) {
4839	4554	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4840	4555	(__v8di)_mm512_cvtepi16_epi64(__A),
4841	4556	(__v8di)_mm512_setzero_si512());
4842	4557	}
4843	4558
4844		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4845		_mm512_cvtepu8_epi32(__m128i __A)
4846		{
	4559	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4560	_mm512_cvtepu8_epi32(__m128i __A) {
4847	4561	return (__m512i)__builtin_convertvector((__v16qu)__A, __v16si);
4848	4562	}
4849	4563
4850		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4851		_mm512_mask_cvtepu8_epi32(__m512i __W, __mmask16 __U, __m128i __A)
4852		{
	4564	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4565	_mm512_mask_cvtepu8_epi32(__m512i __W, __mmask16 __U, __m128i __A) {
4853	4566	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4854	4567	(__v16si)_mm512_cvtepu8_epi32(__A),
4855	4568	(__v16si)__W);
4856	4569	}
4857	4570
4858		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4859		_mm512_maskz_cvtepu8_epi32(__mmask16 __U, __m128i __A)
4860		{
	4571	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4572	_mm512_maskz_cvtepu8_epi32(__mmask16 __U, __m128i __A) {
4861	4573	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4862	4574	(__v16si)_mm512_cvtepu8_epi32(__A),
4863	4575	(__v16si)_mm512_setzero_si512());
4864	4576	}
4865	4577
4866		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4867		_mm512_cvtepu8_epi64(__m128i __A)
4868		{
	4578	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4579	_mm512_cvtepu8_epi64(__m128i __A) {
4869	4580	return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__A, (__v16qu)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
4870	4581	}
4871	4582
4872		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4873		_mm512_mask_cvtepu8_epi64(__m512i __W, __mmask8 __U, __m128i __A)
4874		{
	4583	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4584	_mm512_mask_cvtepu8_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
4875	4585	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4876	4586	(__v8di)_mm512_cvtepu8_epi64(__A),
4877	4587	(__v8di)__W);
4878	4588	}
4879	4589
4880		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4881		_mm512_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A)
4882		{
	4590	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4591	_mm512_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A) {
4883	4592	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4884	4593	(__v8di)_mm512_cvtepu8_epi64(__A),
4885	4594	(__v8di)_mm512_setzero_si512());
4886	4595	}
4887	4596
4888		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4889		_mm512_cvtepu32_epi64(__m256i __X)
4890		{
	4597	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4598	_mm512_cvtepu32_epi64(__m256i __X) {
4891	4599	return (__m512i)__builtin_convertvector((__v8su)__X, __v8di);
4892	4600	}
4893	4601
4894		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4895		_mm512_mask_cvtepu32_epi64(__m512i __W, __mmask8 __U, __m256i __X)
4896		{
	4602	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4603	_mm512_mask_cvtepu32_epi64(__m512i __W, __mmask8 __U, __m256i __X) {
4897	4604	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4898	4605	(__v8di)_mm512_cvtepu32_epi64(__X),
4899	4606	(__v8di)__W);
4900	4607	}
4901	4608
4902		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4903		_mm512_maskz_cvtepu32_epi64(__mmask8 __U, __m256i __X)
4904		{
	4609	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4610	_mm512_maskz_cvtepu32_epi64(__mmask8 __U, __m256i __X) {
4905	4611	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4906	4612	(__v8di)_mm512_cvtepu32_epi64(__X),
4907	4613	(__v8di)_mm512_setzero_si512());
4908	4614	}
4909	4615
4910		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4911		_mm512_cvtepu16_epi32(__m256i __A)
4912		{
	4616	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4617	_mm512_cvtepu16_epi32(__m256i __A) {
4913	4618	return (__m512i)__builtin_convertvector((__v16hu)__A, __v16si);
4914	4619	}
4915	4620
4916		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4917		_mm512_mask_cvtepu16_epi32(__m512i __W, __mmask16 __U, __m256i __A)
4918		{
	4621	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4622	_mm512_mask_cvtepu16_epi32(__m512i __W, __mmask16 __U, __m256i __A) {
4919	4623	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4920	4624	(__v16si)_mm512_cvtepu16_epi32(__A),
4921	4625	(__v16si)__W);
4922	4626	}
4923	4627
4924		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4925		_mm512_maskz_cvtepu16_epi32(__mmask16 __U, __m256i __A)
4926		{
	4628	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4629	_mm512_maskz_cvtepu16_epi32(__mmask16 __U, __m256i __A) {
4927	4630	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
4928	4631	(__v16si)_mm512_cvtepu16_epi32(__A),
4929	4632	(__v16si)_mm512_setzero_si512());
4930	4633	}
4931	4634
4932		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4933		_mm512_cvtepu16_epi64(__m128i __A)
4934		{
	4635	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4636	_mm512_cvtepu16_epi64(__m128i __A) {
4935	4637	return (__m512i)__builtin_convertvector((__v8hu)__A, __v8di);
4936	4638	}
4937	4639
4938		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4939		_mm512_mask_cvtepu16_epi64(__m512i __W, __mmask8 __U, __m128i __A)
4940		{
	4640	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4641	_mm512_mask_cvtepu16_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
4941	4642	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4942	4643	(__v8di)_mm512_cvtepu16_epi64(__A),
4943	4644	(__v8di)__W);
4944	4645	}
4945	4646
4946		static __inline__ __m512i __DEFAULT_FN_ATTRS512
4947		_mm512_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A)
4948		{
	4647	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4648	_mm512_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A) {
4949	4649	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
4950	4650	(__v8di)_mm512_cvtepu16_epi64(__A),
4951	4651	(__v8di)_mm512_setzero_si512());
4952	4652	}
4953	4653
4954		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4654	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4955	4655	_mm512_rorv_epi32 (__m512i __A, __m512i __B)
4956	4656	{
4957		return (__m512i)__builtin_ia32_prorvd512((__v16si)__A, (__v16si)__B);
	4657	return (__m512i)__builtin_elementwise_fshr((__v16su)__A,(__v16su)__A, (__v16su)__B);
4958	4658	}
4959	4659
4960		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4660	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4961	4661	_mm512_mask_rorv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
4962	4662	{
4963	4663	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -4965,7 +4665,7 @@ _mm512_mask_rorv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
4965	4665	(__v16si)__W);
4966	4666	}
4967	4667
4968		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4668	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4969	4669	_mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
4970	4670	{
4971	4671	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -4973,13 +4673,13 @@ _mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
4973	4673	(__v16si)_mm512_setzero_si512());
4974	4674	}
4975	4675
4976		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4676	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4977	4677	_mm512_rorv_epi64 (__m512i __A, __m512i __B)
4978	4678	{
4979		return (__m512i)__builtin_ia32_prorvq512((__v8di)__A, (__v8di)__B);
	4679	return (__m512i)__builtin_elementwise_fshr((__v8du)__A, (__v8du)__A, (__v8du)__B);
4980	4680	}
4981	4681
4982		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4682	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4983	4683	_mm512_mask_rorv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
4984	4684	{
4985	4685	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -4987,7 +4687,7 @@ _mm512_mask_rorv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
4987	4687	(__v8di)__W);
4988	4688	}
4989	4689
4990		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4690	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
4991	4691	_mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
4992	4692	{
4993	4693	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -5063,13 +4763,13 @@ _mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
5063	4763	(__v8di)_mm512_rol_epi64((a), (b)), \
5064	4764	(__v8di)_mm512_setzero_si512()))
5065	4765
5066		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4766	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5067	4767	_mm512_rolv_epi32 (__m512i __A, __m512i __B)
5068	4768	{
5069		return (__m512i)__builtin_ia32_prolvd512((__v16si)__A, (__v16si)__B);
	4769	return (__m512i)__builtin_elementwise_fshl((__v16su)__A, (__v16su)__A, (__v16su)__B);
5070	4770	}
5071	4771
5072		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4772	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5073	4773	_mm512_mask_rolv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
5074	4774	{
5075	4775	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -5077,7 +4777,7 @@ _mm512_mask_rolv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B)
5077	4777	(__v16si)__W);
5078	4778	}
5079	4779
5080		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4780	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5081	4781	_mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
5082	4782	{
5083	4783	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -5085,13 +4785,13 @@ _mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
5085	4785	(__v16si)_mm512_setzero_si512());
5086	4786	}
5087	4787
5088		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4788	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5089	4789	_mm512_rolv_epi64 (__m512i __A, __m512i __B)
5090	4790	{
5091		return (__m512i)__builtin_ia32_prolvq512((__v8di)__A, (__v8di)__B);
	4791	return (__m512i)__builtin_elementwise_fshl((__v8du)__A, (__v8du)__A, (__v8du)__B);
5092	4792	}
5093	4793
5094		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4794	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5095	4795	_mm512_mask_rolv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
5096	4796	{
5097	4797	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -5099,7 +4799,7 @@ _mm512_mask_rolv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B)
5099	4799	(__v8di)__W);
5100	4800	}
5101	4801
5102		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4802	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5103	4803	_mm512_maskz_rolv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
5104	4804	{
5105	4805	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -5133,91 +4833,81 @@ _mm512_maskz_rolv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
5133	4833	(__v8di)_mm512_ror_epi64((A), (B)), \
5134	4834	(__v8di)_mm512_setzero_si512()))
5135	4835
5136		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5137		_mm512_slli_epi32(__m512i __A, unsigned int __B)
5138		{
	4836	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4837	_mm512_slli_epi32(__m512i __A, unsigned int __B) {
5139	4838	return (__m512i)__builtin_ia32_pslldi512((__v16si)__A, (int)__B);
5140	4839	}
5141	4840
5142		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4841	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5143	4842	_mm512_mask_slli_epi32(__m512i __W, __mmask16 __U, __m512i __A,
5144		unsigned int __B)
5145		{
	4843	unsigned int __B) {
5146	4844	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5147	4845	(__v16si)_mm512_slli_epi32(__A, __B),
5148	4846	(__v16si)__W);
5149	4847	}
5150	4848
5151		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4849	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5152	4850	_mm512_maskz_slli_epi32(__mmask16 __U, __m512i __A, unsigned int __B) {
5153	4851	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5154	4852	(__v16si)_mm512_slli_epi32(__A, __B),
5155	4853	(__v16si)_mm512_setzero_si512());
5156	4854	}
5157	4855
5158		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5159		_mm512_slli_epi64(__m512i __A, unsigned int __B)
5160		{
	4856	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4857	_mm512_slli_epi64(__m512i __A, unsigned int __B) {
5161	4858	return (__m512i)__builtin_ia32_psllqi512((__v8di)__A, (int)__B);
5162	4859	}
5163	4860
5164		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5165		_mm512_mask_slli_epi64(__m512i __W, __mmask8 __U, __m512i __A, unsigned int __B)
5166		{
	4861	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4862	_mm512_mask_slli_epi64(__m512i __W, __mmask8 __U, __m512i __A,
	4863	unsigned int __B) {
5167	4864	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5168	4865	(__v8di)_mm512_slli_epi64(__A, __B),
5169	4866	(__v8di)__W);
5170	4867	}
5171	4868
5172		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5173		_mm512_maskz_slli_epi64(__mmask8 __U, __m512i __A, unsigned int __B)
5174		{
	4869	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4870	_mm512_maskz_slli_epi64(__mmask8 __U, __m512i __A, unsigned int __B) {
5175	4871	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5176	4872	(__v8di)_mm512_slli_epi64(__A, __B),
5177	4873	(__v8di)_mm512_setzero_si512());
5178	4874	}
5179	4875
5180		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5181		_mm512_srli_epi32(__m512i __A, unsigned int __B)
5182		{
	4876	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4877	_mm512_srli_epi32(__m512i __A, unsigned int __B) {
5183	4878	return (__m512i)__builtin_ia32_psrldi512((__v16si)__A, (int)__B);
5184	4879	}
5185	4880
5186		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4881	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5187	4882	_mm512_mask_srli_epi32(__m512i __W, __mmask16 __U, __m512i __A,
5188		unsigned int __B)
5189		{
	4883	unsigned int __B) {
5190	4884	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5191	4885	(__v16si)_mm512_srli_epi32(__A, __B),
5192	4886	(__v16si)__W);
5193	4887	}
5194	4888
5195		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4889	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5196	4890	_mm512_maskz_srli_epi32(__mmask16 __U, __m512i __A, unsigned int __B) {
5197	4891	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5198	4892	(__v16si)_mm512_srli_epi32(__A, __B),
5199	4893	(__v16si)_mm512_setzero_si512());
5200	4894	}
5201	4895
5202		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5203		_mm512_srli_epi64(__m512i __A, unsigned int __B)
5204		{
	4896	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4897	_mm512_srli_epi64(__m512i __A, unsigned int __B) {
5205	4898	return (__m512i)__builtin_ia32_psrlqi512((__v8di)__A, (int)__B);
5206	4899	}
5207	4900
5208		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	4901	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5209	4902	_mm512_mask_srli_epi64(__m512i __W, __mmask8 __U, __m512i __A,
5210		unsigned int __B)
5211		{
	4903	unsigned int __B) {
5212	4904	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5213	4905	(__v8di)_mm512_srli_epi64(__A, __B),
5214	4906	(__v8di)__W);
5215	4907	}
5216	4908
5217		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5218		_mm512_maskz_srli_epi64(__mmask8 __U, __m512i __A,
5219		unsigned int __B)
5220		{
	4909	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	4910	_mm512_maskz_srli_epi64(__mmask8 __U, __m512i __A, unsigned int __B) {
5221	4911	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5222	4912	(__v8di)_mm512_srli_epi64(__A, __B),
5223	4913	(__v8di)_mm512_setzero_si512());
...	...	@@ -5303,7 +4993,7 @@ _mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A)
5303	4993	(__mmask8) __U);
5304	4994	}
5305	4995
5306		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	4996	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
5307	4997	_mm512_movedup_pd (__m512d __A)
5308	4998	{
5309	4999	return (__m512d)__builtin_shufflevector((__v8df)__A, (__v8df)__A,
...	...	@@ -5664,10 +5354,9 @@ _mm_maskz_getexp_ss (__mmask8 __U, __m128 __A, __m128 __B)
5664	5354	(int)(((D)<<2) \| (C)), \
5665	5355	(__v4sf)_mm_setzero_ps(), \
5666	5356	(__mmask8)(U), (int)(R)))
5667
5668		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
5669		_mm512_kmov (__mmask16 __A)
5670		{
	5357
	5358	static __inline__ __mmask16
	5359	__DEFAULT_FN_ATTRS_CONSTEXPR _mm512_kmov(__mmask16 __A) {
5671	5360	return __A;
5672	5361	}
5673	5362
...	...	@@ -5684,79 +5373,70 @@ _mm512_kmov (__mmask16 __A)
5684	5373	((long long)__builtin_ia32_vcvtsd2si64((__v2df)(__m128d)(A), (int)(R)))
5685	5374	#endif
5686	5375
5687		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5688		_mm512_sll_epi32(__m512i __A, __m128i __B)
5689		{
	5376	static __inline__ __m512i
	5377	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_sll_epi32(__m512i __A, __m128i __B) {
5690	5378	return (__m512i)__builtin_ia32_pslld512((__v16si) __A, (__v4si)__B);
5691	5379	}
5692	5380
5693		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5694		_mm512_mask_sll_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
5695		{
	5381	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5382	_mm512_mask_sll_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) {
5696	5383	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5697	5384	(__v16si)_mm512_sll_epi32(__A, __B),
5698	5385	(__v16si)__W);
5699	5386	}
5700	5387
5701		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5702		_mm512_maskz_sll_epi32(__mmask16 __U, __m512i __A, __m128i __B)
5703		{
	5388	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5389	_mm512_maskz_sll_epi32(__mmask16 __U, __m512i __A, __m128i __B) {
5704	5390	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5705	5391	(__v16si)_mm512_sll_epi32(__A, __B),
5706	5392	(__v16si)_mm512_setzero_si512());
5707	5393	}
5708	5394
5709		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5710		_mm512_sll_epi64(__m512i __A, __m128i __B)
5711		{
	5395	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5396	_mm512_sll_epi64(__m512i __A, __m128i __B) {
5712	5397	return (__m512i)__builtin_ia32_psllq512((__v8di)__A, (__v2di)__B);
5713	5398	}
5714	5399
5715		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5716		_mm512_mask_sll_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
5717		{
	5400	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5401	_mm512_mask_sll_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) {
5718	5402	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5719	5403	(__v8di)_mm512_sll_epi64(__A, __B),
5720	5404	(__v8di)__W);
5721	5405	}
5722	5406
5723		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5724		_mm512_maskz_sll_epi64(__mmask8 __U, __m512i __A, __m128i __B)
5725		{
	5407	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5408	_mm512_maskz_sll_epi64(__mmask8 __U, __m512i __A, __m128i __B) {
5726	5409	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5727	5410	(__v8di)_mm512_sll_epi64(__A, __B),
5728	5411	(__v8di)_mm512_setzero_si512());
5729	5412	}
5730	5413
5731		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5732		_mm512_sllv_epi32(__m512i __X, __m512i __Y)
5733		{
	5414	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5415	_mm512_sllv_epi32(__m512i __X, __m512i __Y) {
5734	5416	return (__m512i)__builtin_ia32_psllv16si((__v16si)__X, (__v16si)__Y);
5735	5417	}
5736	5418
5737		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5738		_mm512_mask_sllv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
5739		{
	5419	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5420	_mm512_mask_sllv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) {
5740	5421	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5741	5422	(__v16si)_mm512_sllv_epi32(__X, __Y),
5742	5423	(__v16si)__W);
5743	5424	}
5744	5425
5745		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5746		_mm512_maskz_sllv_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
5747		{
	5426	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5427	_mm512_maskz_sllv_epi32(__mmask16 __U, __m512i __X, __m512i __Y) {
5748	5428	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5749	5429	(__v16si)_mm512_sllv_epi32(__X, __Y),
5750	5430	(__v16si)_mm512_setzero_si512());
5751	5431	}
5752	5432
5753		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5433	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5754	5434	_mm512_sllv_epi64(__m512i __X, __m512i __Y)
5755	5435	{
5756	5436	return (__m512i)__builtin_ia32_psllv8di((__v8di)__X, (__v8di)__Y);
5757	5437	}
5758	5438
5759		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5439	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5760	5440	_mm512_mask_sllv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5761	5441	{
5762	5442	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -5764,7 +5444,7 @@ _mm512_mask_sllv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5764	5444	(__v8di)__W);
5765	5445	}
5766	5446
5767		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5447	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5768	5448	_mm512_maskz_sllv_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
5769	5449	{
5770	5450	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -5772,79 +5452,70 @@ _mm512_maskz_sllv_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
5772	5452	(__v8di)_mm512_setzero_si512());
5773	5453	}
5774	5454
5775		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5776		_mm512_sra_epi32(__m512i __A, __m128i __B)
5777		{
	5455	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5456	_mm512_sra_epi32(__m512i __A, __m128i __B) {
5778	5457	return (__m512i)__builtin_ia32_psrad512((__v16si) __A, (__v4si)__B);
5779	5458	}
5780	5459
5781		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5782		_mm512_mask_sra_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
5783		{
	5460	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5461	_mm512_mask_sra_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) {
5784	5462	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5785	5463	(__v16si)_mm512_sra_epi32(__A, __B),
5786	5464	(__v16si)__W);
5787	5465	}
5788	5466
5789		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5790		_mm512_maskz_sra_epi32(__mmask16 __U, __m512i __A, __m128i __B)
5791		{
	5467	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5468	_mm512_maskz_sra_epi32(__mmask16 __U, __m512i __A, __m128i __B) {
5792	5469	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5793	5470	(__v16si)_mm512_sra_epi32(__A, __B),
5794	5471	(__v16si)_mm512_setzero_si512());
5795	5472	}
5796	5473
5797		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5798		_mm512_sra_epi64(__m512i __A, __m128i __B)
5799		{
	5474	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5475	_mm512_sra_epi64(__m512i __A, __m128i __B) {
5800	5476	return (__m512i)__builtin_ia32_psraq512((__v8di)__A, (__v2di)__B);
5801	5477	}
5802	5478
5803		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5804		_mm512_mask_sra_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
5805		{
	5479	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5480	_mm512_mask_sra_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) {
5806	5481	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5807	5482	(__v8di)_mm512_sra_epi64(__A, __B),
5808	5483	(__v8di)__W);
5809	5484	}
5810	5485
5811		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5812		_mm512_maskz_sra_epi64(__mmask8 __U, __m512i __A, __m128i __B)
5813		{
	5486	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5487	_mm512_maskz_sra_epi64(__mmask8 __U, __m512i __A, __m128i __B) {
5814	5488	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5815	5489	(__v8di)_mm512_sra_epi64(__A, __B),
5816	5490	(__v8di)_mm512_setzero_si512());
5817	5491	}
5818	5492
5819		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5820		_mm512_srav_epi32(__m512i __X, __m512i __Y)
5821		{
	5493	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5494	_mm512_srav_epi32(__m512i __X, __m512i __Y) {
5822	5495	return (__m512i)__builtin_ia32_psrav16si((__v16si)__X, (__v16si)__Y);
5823	5496	}
5824	5497
5825		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5826		_mm512_mask_srav_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
5827		{
	5498	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5499	_mm512_mask_srav_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) {
5828	5500	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5829	5501	(__v16si)_mm512_srav_epi32(__X, __Y),
5830	5502	(__v16si)__W);
5831	5503	}
5832	5504
5833		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5834		_mm512_maskz_srav_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
5835		{
	5505	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5506	_mm512_maskz_srav_epi32(__mmask16 __U, __m512i __X, __m512i __Y) {
5836	5507	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5837	5508	(__v16si)_mm512_srav_epi32(__X, __Y),
5838	5509	(__v16si)_mm512_setzero_si512());
5839	5510	}
5840	5511
5841		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5512	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5842	5513	_mm512_srav_epi64(__m512i __X, __m512i __Y)
5843	5514	{
5844	5515	return (__m512i)__builtin_ia32_psrav8di((__v8di)__X, (__v8di)__Y);
5845	5516	}
5846	5517
5847		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5518	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5848	5519	_mm512_mask_srav_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5849	5520	{
5850	5521	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -5852,7 +5523,7 @@ _mm512_mask_srav_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5852	5523	(__v8di)__W);
5853	5524	}
5854	5525
5855		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5526	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5856	5527	_mm512_maskz_srav_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
5857	5528	{
5858	5529	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -5860,79 +5531,70 @@ _mm512_maskz_srav_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
5860	5531	(__v8di)_mm512_setzero_si512());
5861	5532	}
5862	5533
5863		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5864		_mm512_srl_epi32(__m512i __A, __m128i __B)
5865		{
	5534	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5535	_mm512_srl_epi32(__m512i __A, __m128i __B) {
5866	5536	return (__m512i)__builtin_ia32_psrld512((__v16si) __A, (__v4si)__B);
5867	5537	}
5868	5538
5869		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5870		_mm512_mask_srl_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B)
5871		{
	5539	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5540	_mm512_mask_srl_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) {
5872	5541	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5873	5542	(__v16si)_mm512_srl_epi32(__A, __B),
5874	5543	(__v16si)__W);
5875	5544	}
5876	5545
5877		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5878		_mm512_maskz_srl_epi32(__mmask16 __U, __m512i __A, __m128i __B)
5879		{
	5546	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5547	_mm512_maskz_srl_epi32(__mmask16 __U, __m512i __A, __m128i __B) {
5880	5548	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5881	5549	(__v16si)_mm512_srl_epi32(__A, __B),
5882	5550	(__v16si)_mm512_setzero_si512());
5883	5551	}
5884	5552
5885		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5886		_mm512_srl_epi64(__m512i __A, __m128i __B)
5887		{
	5553	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5554	_mm512_srl_epi64(__m512i __A, __m128i __B) {
5888	5555	return (__m512i)__builtin_ia32_psrlq512((__v8di)__A, (__v2di)__B);
5889	5556	}
5890	5557
5891		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5892		_mm512_mask_srl_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B)
5893		{
	5558	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5559	_mm512_mask_srl_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) {
5894	5560	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5895	5561	(__v8di)_mm512_srl_epi64(__A, __B),
5896	5562	(__v8di)__W);
5897	5563	}
5898	5564
5899		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5900		_mm512_maskz_srl_epi64(__mmask8 __U, __m512i __A, __m128i __B)
5901		{
	5565	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5566	_mm512_maskz_srl_epi64(__mmask8 __U, __m512i __A, __m128i __B) {
5902	5567	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
5903	5568	(__v8di)_mm512_srl_epi64(__A, __B),
5904	5569	(__v8di)_mm512_setzero_si512());
5905	5570	}
5906	5571
5907		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5908		_mm512_srlv_epi32(__m512i __X, __m512i __Y)
5909		{
	5572	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5573	_mm512_srlv_epi32(__m512i __X, __m512i __Y) {
5910	5574	return (__m512i)__builtin_ia32_psrlv16si((__v16si)__X, (__v16si)__Y);
5911	5575	}
5912	5576
5913		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5914		_mm512_mask_srlv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y)
5915		{
	5577	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5578	_mm512_mask_srlv_epi32(__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) {
5916	5579	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5917	5580	(__v16si)_mm512_srlv_epi32(__X, __Y),
5918	5581	(__v16si)__W);
5919	5582	}
5920	5583
5921		static __inline__ __m512i __DEFAULT_FN_ATTRS512
5922		_mm512_maskz_srlv_epi32(__mmask16 __U, __m512i __X, __m512i __Y)
5923		{
	5584	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	5585	_mm512_maskz_srlv_epi32(__mmask16 __U, __m512i __X, __m512i __Y) {
5924	5586	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
5925	5587	(__v16si)_mm512_srlv_epi32(__X, __Y),
5926	5588	(__v16si)_mm512_setzero_si512());
5927	5589	}
5928	5590
5929		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5591	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5930	5592	_mm512_srlv_epi64 (__m512i __X, __m512i __Y)
5931	5593	{
5932	5594	return (__m512i)__builtin_ia32_psrlv8di((__v8di)__X, (__v8di)__Y);
5933	5595	}
5934	5596
5935		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5597	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5936	5598	_mm512_mask_srlv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5937	5599	{
5938	5600	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -5940,7 +5602,7 @@ _mm512_mask_srlv_epi64(__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y)
5940	5602	(__v8di)__W);
5941	5603	}
5942	5604
5943		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	5605	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
5944	5606	_mm512_maskz_srlv_epi64(__mmask8 __U, __m512i __X, __m512i __Y)
5945	5607	{
5946	5608	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
...	...	@@ -6190,115 +5852,104 @@ _mm_cvttss_u64 (__m128 __A)
6190	5852	(__v16sf)_mm512_permute_ps((X), (C)), \
6191	5853	(__v16sf)_mm512_setzero_ps()))
6192	5854
6193		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6194		_mm512_permutevar_pd(__m512d __A, __m512i __C)
6195		{
	5855	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	5856	_mm512_permutevar_pd(__m512d __A, __m512i __C) {
6196	5857	return (__m512d)__builtin_ia32_vpermilvarpd512((__v8df)__A, (__v8di)__C);
6197	5858	}
6198	5859
6199		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6200		_mm512_mask_permutevar_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512i __C)
6201		{
	5860	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	5861	_mm512_mask_permutevar_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512i __C) {
6202	5862	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
6203	5863	(__v8df)_mm512_permutevar_pd(__A, __C),
6204	5864	(__v8df)__W);
6205	5865	}
6206	5866
6207		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6208		_mm512_maskz_permutevar_pd(__mmask8 __U, __m512d __A, __m512i __C)
6209		{
	5867	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	5868	_mm512_maskz_permutevar_pd(__mmask8 __U, __m512d __A, __m512i __C) {
6210	5869	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
6211	5870	(__v8df)_mm512_permutevar_pd(__A, __C),
6212	5871	(__v8df)_mm512_setzero_pd());
6213	5872	}
6214	5873
6215		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6216		_mm512_permutevar_ps(__m512 __A, __m512i __C)
6217		{
	5874	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5875	_mm512_permutevar_ps(__m512 __A, __m512i __C) {
6218	5876	return (__m512)__builtin_ia32_vpermilvarps512((__v16sf)__A, (__v16si)__C);
6219	5877	}
6220	5878
6221		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6222		_mm512_mask_permutevar_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512i __C)
6223		{
	5879	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5880	_mm512_mask_permutevar_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512i __C) {
6224	5881	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
6225	5882	(__v16sf)_mm512_permutevar_ps(__A, __C),
6226	5883	(__v16sf)__W);
6227	5884	}
6228	5885
6229		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6230		_mm512_maskz_permutevar_ps(__mmask16 __U, __m512 __A, __m512i __C)
6231		{
	5886	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5887	_mm512_maskz_permutevar_ps(__mmask16 __U, __m512 __A, __m512i __C) {
6232	5888	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
6233	5889	(__v16sf)_mm512_permutevar_ps(__A, __C),
6234	5890	(__v16sf)_mm512_setzero_ps());
6235	5891	}
6236	5892
6237		static __inline __m512d __DEFAULT_FN_ATTRS512
6238		_mm512_permutex2var_pd(__m512d __A, __m512i __I, __m512d __B)
6239		{
	5893	static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	5894	_mm512_permutex2var_pd(__m512d __A, __m512i __I, __m512d __B) {
6240	5895	return (__m512d)__builtin_ia32_vpermi2varpd512((__v8df)__A, (__v8di)__I,
6241	5896	(__v8df)__B);
6242	5897	}
6243	5898
6244		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6245		_mm512_mask_permutex2var_pd(__m512d __A, __mmask8 __U, __m512i __I, __m512d __B)
6246		{
	5899	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	5900	_mm512_mask_permutex2var_pd(__m512d __A, __mmask8 __U, __m512i __I,
	5901	__m512d __B) {
6247	5902	return (__m512d)__builtin_ia32_selectpd_512(__U,
6248	5903	(__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6249	5904	(__v8df)__A);
6250	5905	}
6251	5906
6252		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	5907	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
6253	5908	_mm512_mask2_permutex2var_pd(__m512d __A, __m512i __I, __mmask8 __U,
6254		__m512d __B)
6255		{
	5909	__m512d __B) {
6256	5910	return (__m512d)__builtin_ia32_selectpd_512(__U,
6257	5911	(__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6258	5912	(__v8df)(__m512d)__I);
6259	5913	}
6260	5914
6261		static __inline__ __m512d __DEFAULT_FN_ATTRS512
	5915	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
6262	5916	_mm512_maskz_permutex2var_pd(__mmask8 __U, __m512d __A, __m512i __I,
6263		__m512d __B)
6264		{
	5917	__m512d __B) {
6265	5918	return (__m512d)__builtin_ia32_selectpd_512(__U,
6266	5919	(__v8df)_mm512_permutex2var_pd(__A, __I, __B),
6267	5920	(__v8df)_mm512_setzero_pd());
6268	5921	}
6269	5922
6270		static __inline __m512 __DEFAULT_FN_ATTRS512
6271		_mm512_permutex2var_ps(__m512 __A, __m512i __I, __m512 __B)
6272		{
	5923	static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5924	_mm512_permutex2var_ps(__m512 __A, __m512i __I, __m512 __B) {
6273	5925	return (__m512)__builtin_ia32_vpermi2varps512((__v16sf)__A, (__v16si)__I,
6274	5926	(__v16sf) __B);
6275	5927	}
6276	5928
6277		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6278		_mm512_mask_permutex2var_ps(__m512 __A, __mmask16 __U, __m512i __I, __m512 __B)
6279		{
	5929	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5930	_mm512_mask_permutex2var_ps(__m512 __A, __mmask16 __U, __m512i __I,
	5931	__m512 __B) {
6280	5932	return (__m512)__builtin_ia32_selectps_512(__U,
6281	5933	(__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6282	5934	(__v16sf)__A);
6283	5935	}
6284	5936
6285		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6286		_mm512_mask2_permutex2var_ps(__m512 __A, __m512i __I, __mmask16 __U, __m512 __B)
6287		{
	5937	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5938	_mm512_mask2_permutex2var_ps(__m512 __A, __m512i __I, __mmask16 __U,
	5939	__m512 __B) {
6288	5940	return (__m512)__builtin_ia32_selectps_512(__U,
6289	5941	(__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6290	5942	(__v16sf)(__m512)__I);
6291	5943	}
6292	5944
6293		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6294		_mm512_maskz_permutex2var_ps(__mmask16 __U, __m512 __A, __m512i __I, __m512 __B)
6295		{
	5945	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	5946	_mm512_maskz_permutex2var_ps(__mmask16 __U, __m512 __A, __m512i __I,
	5947	__m512 __B) {
6296	5948	return (__m512)__builtin_ia32_selectps_512(__U,
6297	5949	(__v16sf)_mm512_permutex2var_ps(__A, __I, __B),
6298	5950	(__v16sf)_mm512_setzero_ps());
6299	5951	}
6300	5952
6301
6302	5953	#define _mm512_cvtt_roundpd_epu32(A, R) \
6303	5954	((__m256i)__builtin_ia32_cvttpd2udq512_mask((__v8df)(__m512d)(A), \
6304	5955	(__v8si)_mm256_undefined_si256(), \
...	...	@@ -6622,46 +6273,41 @@ _mm_maskz_scalef_ss (__mmask8 __U, __m128 __A, __m128 __B)
6622	6273	(__mmask8)(U), \
6623	6274	(int)(R)))
6624	6275
6625		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6626		_mm512_srai_epi32(__m512i __A, unsigned int __B)
6627		{
	6276	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6277	_mm512_srai_epi32(__m512i __A, unsigned int __B) {
6628	6278	return (__m512i)__builtin_ia32_psradi512((__v16si)__A, (int)__B);
6629	6279	}
6630	6280
6631		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	6281	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
6632	6282	_mm512_mask_srai_epi32(__m512i __W, __mmask16 __U, __m512i __A,
6633		unsigned int __B)
6634		{
	6283	unsigned int __B) {
6635	6284	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
6636	6285	(__v16si)_mm512_srai_epi32(__A, __B),
6637	6286	(__v16si)__W);
6638	6287	}
6639	6288
6640		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6641		_mm512_maskz_srai_epi32(__mmask16 __U, __m512i __A,
6642		unsigned int __B) {
	6289	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6290	_mm512_maskz_srai_epi32(__mmask16 __U, __m512i __A, unsigned int __B) {
6643	6291	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__U,
6644	6292	(__v16si)_mm512_srai_epi32(__A, __B),
6645	6293	(__v16si)_mm512_setzero_si512());
6646	6294	}
6647	6295
6648		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6649		_mm512_srai_epi64(__m512i __A, unsigned int __B)
6650		{
	6296	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6297	_mm512_srai_epi64(__m512i __A, unsigned int __B) {
6651	6298	return (__m512i)__builtin_ia32_psraqi512((__v8di)__A, (int)__B);
6652	6299	}
6653	6300
6654		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6655		_mm512_mask_srai_epi64(__m512i __W, __mmask8 __U, __m512i __A, unsigned int __B)
6656		{
	6301	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6302	_mm512_mask_srai_epi64(__m512i __W, __mmask8 __U, __m512i __A,
	6303	unsigned int __B) {
6657	6304	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
6658	6305	(__v8di)_mm512_srai_epi64(__A, __B),
6659	6306	(__v8di)__W);
6660	6307	}
6661	6308
6662		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6663		_mm512_maskz_srai_epi64(__mmask8 __U, __m512i __A, unsigned int __B)
6664		{
	6309	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6310	_mm512_maskz_srai_epi64(__mmask8 __U, __m512i __A, unsigned int __B) {
6665	6311	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__U,
6666	6312	(__v8di)_mm512_srai_epi64(__A, __B),
6667	6313	(__v8di)_mm512_setzero_si512());
...	...	@@ -6827,33 +6473,29 @@ _mm_maskz_sqrt_ss (__mmask8 __U, __m128 __A, __m128 __B)
6827	6473	(__v4sf)_mm_setzero_ps(), \
6828	6474	(__mmask8)(U), (int)(R)))
6829	6475
6830		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6831		_mm512_broadcast_f32x4(__m128 __A)
6832		{
	6476	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	6477	_mm512_broadcast_f32x4(__m128 __A) {
6833	6478	return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
6834	6479	0, 1, 2, 3, 0, 1, 2, 3,
6835	6480	0, 1, 2, 3, 0, 1, 2, 3);
6836	6481	}
6837	6482
6838		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6839		_mm512_mask_broadcast_f32x4(__m512 __O, __mmask16 __M, __m128 __A)
6840		{
	6483	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	6484	_mm512_mask_broadcast_f32x4(__m512 __O, __mmask16 __M, __m128 __A) {
6841	6485	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
6842	6486	(__v16sf)_mm512_broadcast_f32x4(__A),
6843	6487	(__v16sf)__O);
6844	6488	}
6845	6489
6846		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6847		_mm512_maskz_broadcast_f32x4(__mmask16 __M, __m128 __A)
6848		{
	6490	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	6491	_mm512_maskz_broadcast_f32x4(__mmask16 __M, __m128 __A) {
6849	6492	return (__m512)__builtin_ia32_selectps_512((__mmask16)__M,
6850	6493	(__v16sf)_mm512_broadcast_f32x4(__A),
6851	6494	(__v16sf)_mm512_setzero_ps());
6852	6495	}
6853	6496
6854		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6855		_mm512_broadcast_f64x4(__m256d __A)
6856		{
	6497	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	6498	_mm512_broadcast_f64x4(__m256d __A) {
6857	6499	return (__m512d)__builtin_shufflevector((__v4df)__A, (__v4df)__A,
6858	6500	0, 1, 2, 3, 0, 1, 2, 3);
6859	6501	}
...	...	@@ -6874,33 +6516,29 @@ _mm512_maskz_broadcast_f64x4(__mmask8 __M, __m256d __A)
6874	6516	(__v8df)_mm512_setzero_pd());
6875	6517	}
6876	6518
6877		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6878		_mm512_broadcast_i32x4(__m128i __A)
6879		{
	6519	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6520	_mm512_broadcast_i32x4(__m128i __A) {
6880	6521	return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
6881	6522	0, 1, 2, 3, 0, 1, 2, 3,
6882	6523	0, 1, 2, 3, 0, 1, 2, 3);
6883	6524	}
6884	6525
6885		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6886		_mm512_mask_broadcast_i32x4(__m512i __O, __mmask16 __M, __m128i __A)
6887		{
	6526	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6527	_mm512_mask_broadcast_i32x4(__m512i __O, __mmask16 __M, __m128i __A) {
6888	6528	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
6889	6529	(__v16si)_mm512_broadcast_i32x4(__A),
6890	6530	(__v16si)__O);
6891	6531	}
6892	6532
6893		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6894		_mm512_maskz_broadcast_i32x4(__mmask16 __M, __m128i __A)
6895		{
	6533	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6534	_mm512_maskz_broadcast_i32x4(__mmask16 __M, __m128i __A) {
6896	6535	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
6897	6536	(__v16si)_mm512_broadcast_i32x4(__A),
6898	6537	(__v16si)_mm512_setzero_si512());
6899	6538	}
6900	6539
6901		static __inline__ __m512i __DEFAULT_FN_ATTRS512
6902		_mm512_broadcast_i64x4(__m256i __A)
6903		{
	6540	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	6541	_mm512_broadcast_i64x4(__m256i __A) {
6904	6542	return (__m512i)__builtin_shufflevector((__v4di)__A, (__v4di)__A,
6905	6543	0, 1, 2, 3, 0, 1, 2, 3);
6906	6544	}
...	...	@@ -6921,33 +6559,29 @@ _mm512_maskz_broadcast_i64x4(__mmask8 __M, __m256i __A)
6921	6559	(__v8di)_mm512_setzero_si512());
6922	6560	}
6923	6561
6924		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6925		_mm512_mask_broadcastsd_pd (__m512d __O, __mmask8 __M, __m128d __A)
6926		{
	6562	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	6563	_mm512_mask_broadcastsd_pd(__m512d __O, __mmask8 __M, __m128d __A) {
6927	6564	return (__m512d)__builtin_ia32_selectpd_512(__M,
6928	6565	(__v8df) _mm512_broadcastsd_pd(__A),
6929	6566	(__v8df) __O);
6930	6567	}
6931	6568
6932		static __inline__ __m512d __DEFAULT_FN_ATTRS512
6933		_mm512_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A)
6934		{
	6569	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	6570	_mm512_maskz_broadcastsd_pd(__mmask8 __M, __m128d __A) {
6935	6571	return (__m512d)__builtin_ia32_selectpd_512(__M,
6936	6572	(__v8df) _mm512_broadcastsd_pd(__A),
6937	6573	(__v8df) _mm512_setzero_pd());
6938	6574	}
6939	6575
6940		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6941		_mm512_mask_broadcastss_ps (__m512 __O, __mmask16 __M, __m128 __A)
6942		{
	6576	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	6577	_mm512_mask_broadcastss_ps(__m512 __O, __mmask16 __M, __m128 __A) {
6943	6578	return (__m512)__builtin_ia32_selectps_512(__M,
6944	6579	(__v16sf) _mm512_broadcastss_ps(__A),
6945	6580	(__v16sf) __O);
6946	6581	}
6947	6582
6948		static __inline__ __m512 __DEFAULT_FN_ATTRS512
6949		_mm512_maskz_broadcastss_ps (__mmask16 __M, __m128 __A)
6950		{
	6583	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	6584	_mm512_maskz_broadcastss_ps(__mmask16 __M, __m128 __A) {
6951	6585	return (__m512)__builtin_ia32_selectps_512(__M,
6952	6586	(__v16sf) _mm512_broadcastss_ps(__A),
6953	6587	(__v16sf) _mm512_setzero_ps());
...	...	@@ -7391,10 +7025,10 @@ _mm512_mask_cvtepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A)
7391	7025	__builtin_ia32_pmovqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M);
7392	7026	}
7393	7027
7394		#define _mm512_extracti32x4_epi32(A, imm) \
7395		((__m128i)__builtin_ia32_extracti32x4_mask((__v16si)(__m512i)(A), (int)(imm), \
7396		(__v4si)_mm_undefined_si128(), \
7397		(__mmask8)-1))
	7028	#define _mm512_extracti32x4_epi32(A, imm) \
	7029	((__m128i)__builtin_ia32_extracti32x4_mask( \
	7030	(__v16si)(__m512i)(A), (int)(imm), (__v4si)_mm_setzero_si128(), \
	7031	(__mmask8) - 1))
7398	7032
7399	7033	#define _mm512_mask_extracti32x4_epi32(W, U, A, imm) \
7400	7034	((__m128i)__builtin_ia32_extracti32x4_mask((__v16si)(__m512i)(A), (int)(imm), \
...	...	@@ -7406,10 +7040,10 @@ _mm512_mask_cvtepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A)
7406	7040	(__v4si)_mm_setzero_si128(), \
7407	7041	(__mmask8)(U)))
7408	7042
7409		#define _mm512_extracti64x4_epi64(A, imm) \
	7043	#define _mm512_extracti64x4_epi64(A, imm) \
7410	7044	((__m256i)__builtin_ia32_extracti64x4_mask((__v8di)(__m512i)(A), (int)(imm), \
7411		(__v4di)_mm256_undefined_si256(), \
7412		(__mmask8)-1))
	7045	(__v4di)_mm256_setzero_si256(), \
	7046	(__mmask8) - 1))
7413	7047
7414	7048	#define _mm512_mask_extracti64x4_epi64(W, U, A, imm) \
7415	7049	((__m256i)__builtin_ia32_extracti64x4_mask((__v8di)(__m512i)(A), (int)(imm), \
...	...	@@ -8274,93 +7908,82 @@ _mm_mask3_fnmsub_sd (__m128d __W, __m128d __X, __m128d __Y, __mmask8 __U)
8274	7908	(__v8di)_mm512_permutex_epi64((X), (C)), \
8275	7909	(__v8di)_mm512_setzero_si512()))
8276	7910
8277		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8278		_mm512_permutexvar_pd (__m512i __X, __m512d __Y)
8279		{
	7911	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	7912	_mm512_permutexvar_pd(__m512i __X, __m512d __Y) {
8280	7913	return (__m512d)__builtin_ia32_permvardf512((__v8df) __Y, (__v8di) __X);
8281	7914	}
8282	7915
8283		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8284		_mm512_mask_permutexvar_pd (__m512d __W, __mmask8 __U, __m512i __X, __m512d __Y)
8285		{
	7916	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	7917	_mm512_mask_permutexvar_pd(__m512d __W, __mmask8 __U, __m512i __X,
	7918	__m512d __Y) {
8286	7919	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8287	7920	(__v8df)_mm512_permutexvar_pd(__X, __Y),
8288	7921	(__v8df)__W);
8289	7922	}
8290	7923
8291		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8292		_mm512_maskz_permutexvar_pd (__mmask8 __U, __m512i __X, __m512d __Y)
8293		{
	7924	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	7925	_mm512_maskz_permutexvar_pd(__mmask8 __U, __m512i __X, __m512d __Y) {
8294	7926	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8295	7927	(__v8df)_mm512_permutexvar_pd(__X, __Y),
8296	7928	(__v8df)_mm512_setzero_pd());
8297	7929	}
8298	7930
8299		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8300		_mm512_permutexvar_epi64 (__m512i __X, __m512i __Y)
8301		{
	7931	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7932	_mm512_permutexvar_epi64(__m512i __X, __m512i __Y) {
8302	7933	return (__m512i)__builtin_ia32_permvardi512((__v8di)__Y, (__v8di)__X);
8303	7934	}
8304	7935
8305		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8306		_mm512_maskz_permutexvar_epi64 (__mmask8 __M, __m512i __X, __m512i __Y)
8307		{
	7936	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7937	_mm512_maskz_permutexvar_epi64(__mmask8 __M, __m512i __X, __m512i __Y) {
8308	7938	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
8309	7939	(__v8di)_mm512_permutexvar_epi64(__X, __Y),
8310	7940	(__v8di)_mm512_setzero_si512());
8311	7941	}
8312	7942
8313		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8314		_mm512_mask_permutexvar_epi64 (__m512i __W, __mmask8 __M, __m512i __X,
8315		__m512i __Y)
8316		{
	7943	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7944	_mm512_mask_permutexvar_epi64(__m512i __W, __mmask8 __M, __m512i __X,
	7945	__m512i __Y) {
8317	7946	return (__m512i)__builtin_ia32_selectq_512((__mmask8)__M,
8318	7947	(__v8di)_mm512_permutexvar_epi64(__X, __Y),
8319	7948	(__v8di)__W);
8320	7949	}
8321	7950
8322		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8323		_mm512_permutexvar_ps (__m512i __X, __m512 __Y)
8324		{
	7951	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	7952	_mm512_permutexvar_ps(__m512i __X, __m512 __Y) {
8325	7953	return (__m512)__builtin_ia32_permvarsf512((__v16sf)__Y, (__v16si)__X);
8326	7954	}
8327	7955
8328		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8329		_mm512_mask_permutexvar_ps (__m512 __W, __mmask16 __U, __m512i __X, __m512 __Y)
8330		{
	7956	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	7957	_mm512_mask_permutexvar_ps(__m512 __W, __mmask16 __U, __m512i __X, __m512 __Y) {
8331	7958	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8332	7959	(__v16sf)_mm512_permutexvar_ps(__X, __Y),
8333	7960	(__v16sf)__W);
8334	7961	}
8335	7962
8336		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8337		_mm512_maskz_permutexvar_ps (__mmask16 __U, __m512i __X, __m512 __Y)
8338		{
	7963	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	7964	_mm512_maskz_permutexvar_ps(__mmask16 __U, __m512i __X, __m512 __Y) {
8339	7965	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8340	7966	(__v16sf)_mm512_permutexvar_ps(__X, __Y),
8341	7967	(__v16sf)_mm512_setzero_ps());
8342	7968	}
8343	7969
8344		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8345		_mm512_permutexvar_epi32 (__m512i __X, __m512i __Y)
8346		{
	7970	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7971	_mm512_permutexvar_epi32(__m512i __X, __m512i __Y) {
8347	7972	return (__m512i)__builtin_ia32_permvarsi512((__v16si)__Y, (__v16si)__X);
8348	7973	}
8349	7974
8350	7975	#define _mm512_permutevar_epi32 _mm512_permutexvar_epi32
8351	7976
8352		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8353		_mm512_maskz_permutexvar_epi32 (__mmask16 __M, __m512i __X, __m512i __Y)
8354		{
	7977	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7978	_mm512_maskz_permutexvar_epi32(__mmask16 __M, __m512i __X, __m512i __Y) {
8355	7979	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
8356	7980	(__v16si)_mm512_permutexvar_epi32(__X, __Y),
8357	7981	(__v16si)_mm512_setzero_si512());
8358	7982	}
8359	7983
8360		static __inline__ __m512i __DEFAULT_FN_ATTRS512
8361		_mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X,
8362		__m512i __Y)
8363		{
	7984	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	7985	_mm512_mask_permutexvar_epi32(__m512i __W, __mmask16 __M, __m512i __X,
	7986	__m512i __Y) {
8364	7987	return (__m512i)__builtin_ia32_selectd_512((__mmask16)__M,
8365	7988	(__v16si)_mm512_permutexvar_epi32(__X, __Y),
8366	7989	(__v16si)__W);
...	...	@@ -8368,69 +7991,59 @@ _mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X,
8368	7991
8369	7992	#define _mm512_mask_permutevar_epi32 _mm512_mask_permutexvar_epi32
8370	7993
8371		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8372		_mm512_kand (__mmask16 __A, __mmask16 __B)
8373		{
	7994	static __inline__ __mmask16
	7995	__DEFAULT_FN_ATTRS_CONSTEXPR _mm512_kand(__mmask16 __A, __mmask16 __B) {
8374	7996	return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B);
8375	7997	}
8376	7998
8377		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8378		_mm512_kandn (__mmask16 __A, __mmask16 __B)
8379		{
	7999	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	8000	_mm512_kandn(__mmask16 __A, __mmask16 __B) {
8380	8001	return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, (__mmask16) __B);
8381	8002	}
8382	8003
8383		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8384		_mm512_kor (__mmask16 __A, __mmask16 __B)
8385		{
	8004	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	8005	_mm512_kor(__mmask16 __A, __mmask16 __B) {
8386	8006	return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B);
8387	8007	}
8388	8008
8389		static __inline__ int __DEFAULT_FN_ATTRS
8390		_mm512_kortestc (__mmask16 __A, __mmask16 __B)
8391		{
	8009	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	8010	_mm512_kortestc(__mmask16 __A, __mmask16 __B) {
8392	8011	return __builtin_ia32_kortestchi ((__mmask16) __A, (__mmask16) __B);
8393	8012	}
8394	8013
8395		static __inline__ int __DEFAULT_FN_ATTRS
8396		_mm512_kortestz (__mmask16 __A, __mmask16 __B)
8397		{
	8014	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	8015	_mm512_kortestz(__mmask16 __A, __mmask16 __B) {
8398	8016	return __builtin_ia32_kortestzhi ((__mmask16) __A, (__mmask16) __B);
8399	8017	}
8400	8018
8401		static __inline__ unsigned char __DEFAULT_FN_ATTRS
8402		_kortestc_mask16_u8(__mmask16 __A, __mmask16 __B)
8403		{
	8019	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	8020	_kortestc_mask16_u8(__mmask16 __A, __mmask16 __B) {
8404	8021	return (unsigned char)__builtin_ia32_kortestchi(__A, __B);
8405	8022	}
8406	8023
8407		static __inline__ unsigned char __DEFAULT_FN_ATTRS
8408		_kortestz_mask16_u8(__mmask16 __A, __mmask16 __B)
8409		{
	8024	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
	8025	_kortestz_mask16_u8(__mmask16 __A, __mmask16 __B) {
8410	8026	return (unsigned char)__builtin_ia32_kortestzhi(__A, __B);
8411	8027	}
8412	8028
8413		static __inline__ unsigned char __DEFAULT_FN_ATTRS
	8029	static __inline__ unsigned char __DEFAULT_FN_ATTRS_CONSTEXPR
8414	8030	_kortest_mask16_u8(__mmask16 __A, __mmask16 __B, unsigned char *__C) {
8415	8031	*__C = (unsigned char)__builtin_ia32_kortestchi(__A, __B);
8416	8032	return (unsigned char)__builtin_ia32_kortestzhi(__A, __B);
8417	8033	}
8418	8034
8419		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8420		_mm512_kunpackb (__mmask16 __A, __mmask16 __B)
8421		{
	8035	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	8036	_mm512_kunpackb(__mmask16 __A, __mmask16 __B) {
8422	8037	return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B);
8423	8038	}
8424	8039
8425		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8426		_mm512_kxnor (__mmask16 __A, __mmask16 __B)
8427		{
	8040	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	8041	_mm512_kxnor(__mmask16 __A, __mmask16 __B) {
8428	8042	return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B);
8429	8043	}
8430	8044
8431		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
8432		_mm512_kxor (__mmask16 __A, __mmask16 __B)
8433		{
	8045	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
	8046	_mm512_kxor(__mmask16 __A, __mmask16 __B) {
8434	8047	return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B);
8435	8048	}
8436	8049
...	...	@@ -8447,12 +8060,12 @@ _mm512_kxor (__mmask16 __A, __mmask16 __B)
8447	8060	#define _kshiftri_mask16(A, I) \
8448	8061	((__mmask16)__builtin_ia32_kshiftrihi((__mmask16)(A), (unsigned int)(I)))
8449	8062
8450		static __inline__ unsigned int __DEFAULT_FN_ATTRS
8451		_cvtmask16_u32(__mmask16 __A) {
	8063	static __inline__ unsigned int
	8064	__DEFAULT_FN_ATTRS_CONSTEXPR _cvtmask16_u32(__mmask16 __A) {
8452	8065	return (unsigned int)__builtin_ia32_kmovw((__mmask16)__A);
8453	8066	}
8454	8067
8455		static __inline__ __mmask16 __DEFAULT_FN_ATTRS
	8068	static __inline__ __mmask16 __DEFAULT_FN_ATTRS_CONSTEXPR
8456	8069	_cvtu32_mask16(unsigned int __A) {
8457	8070	return (__mmask16)__builtin_ia32_kmovw((__mmask16)__A);
8458	8071	}
...	...	@@ -8665,74 +8278,66 @@ _mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
8665	8278	_mm512_setzero_si512());
8666	8279	}
8667	8280
8668		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	8281	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
8669	8282	_mm512_movehdup_ps (__m512 __A)
8670	8283	{
8671	8284	return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
8672	8285	1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15);
8673	8286	}
8674	8287
8675		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8676		_mm512_mask_movehdup_ps (__m512 __W, __mmask16 __U, __m512 __A)
8677		{
	8288	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8289	_mm512_mask_movehdup_ps(__m512 __W, __mmask16 __U, __m512 __A) {
8678	8290	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8679	8291	(__v16sf)_mm512_movehdup_ps(__A),
8680	8292	(__v16sf)__W);
8681	8293	}
8682	8294
8683		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8684		_mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A)
8685		{
	8295	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8296	_mm512_maskz_movehdup_ps(__mmask16 __U, __m512 __A) {
8686	8297	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8687	8298	(__v16sf)_mm512_movehdup_ps(__A),
8688	8299	(__v16sf)_mm512_setzero_ps());
8689	8300	}
8690	8301
8691		static __inline__ __m512 __DEFAULT_FN_ATTRS512
	8302	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
8692	8303	_mm512_moveldup_ps (__m512 __A)
8693	8304	{
8694	8305	return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
8695	8306	0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14);
8696	8307	}
8697	8308
8698		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8699		_mm512_mask_moveldup_ps (__m512 __W, __mmask16 __U, __m512 __A)
8700		{
	8309	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8310	_mm512_mask_moveldup_ps(__m512 __W, __mmask16 __U, __m512 __A) {
8701	8311	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8702	8312	(__v16sf)_mm512_moveldup_ps(__A),
8703	8313	(__v16sf)__W);
8704	8314	}
8705	8315
8706		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8707		_mm512_maskz_moveldup_ps (__mmask16 __U, __m512 __A)
8708		{
	8316	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8317	_mm512_maskz_moveldup_ps(__mmask16 __U, __m512 __A) {
8709	8318	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U,
8710	8319	(__v16sf)_mm512_moveldup_ps(__A),
8711	8320	(__v16sf)_mm512_setzero_ps());
8712	8321	}
8713	8322
8714		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8715		_mm_mask_move_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
8716		{
	8323	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	8324	_mm_mask_move_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
8717	8325	return __builtin_ia32_selectss_128(__U, _mm_move_ss(__A, __B), __W);
8718	8326	}
8719	8327
8720		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8721		_mm_maskz_move_ss (__mmask8 __U, __m128 __A, __m128 __B)
8722		{
	8328	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	8329	_mm_maskz_move_ss(__mmask8 __U, __m128 __A, __m128 __B) {
8723	8330	return __builtin_ia32_selectss_128(__U, _mm_move_ss(__A, __B),
8724	8331	_mm_setzero_ps());
8725	8332	}
8726	8333
8727		static __inline__ __m128d __DEFAULT_FN_ATTRS128
8728		_mm_mask_move_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
8729		{
	8334	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	8335	_mm_mask_move_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
8730	8336	return __builtin_ia32_selectsd_128(__U, _mm_move_sd(__A, __B), __W);
8731	8337	}
8732	8338
8733		static __inline__ __m128d __DEFAULT_FN_ATTRS128
8734		_mm_maskz_move_sd (__mmask8 __U, __m128d __A, __m128d __B)
8735		{
	8339	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	8340	_mm_maskz_move_sd(__mmask8 __U, __m128d __A, __m128d __B) {
8736	8341	return __builtin_ia32_selectsd_128(__U, _mm_move_sd(__A, __B),
8737	8342	_mm_setzero_pd());
8738	8343	}
...	...	@@ -8941,70 +8546,57 @@ _mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A)
8941	8546	(__v8df)_mm512_setzero_pd(), \
8942	8547	(__mmask8)(U), (int)(R)))
8943	8548
8944		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8945		_mm512_cvtps_pd (__m256 __A)
8946		{
	8549	static __inline__ __m512d
	8550	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_cvtps_pd(__m256 __A) {
8947	8551	return (__m512d) __builtin_convertvector((__v8sf)__A, __v8df);
8948	8552	}
8949	8553
8950		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8951		_mm512_mask_cvtps_pd (__m512d __W, __mmask8 __U, __m256 __A)
8952		{
	8554	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8555	_mm512_mask_cvtps_pd(__m512d __W, __mmask8 __U, __m256 __A) {
8953	8556	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8954	8557	(__v8df)_mm512_cvtps_pd(__A),
8955	8558	(__v8df)__W);
8956	8559	}
8957	8560
8958		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8959		_mm512_maskz_cvtps_pd (__mmask8 __U, __m256 __A)
8960		{
	8561	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8562	_mm512_maskz_cvtps_pd(__mmask8 __U, __m256 __A) {
8961	8563	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U,
8962	8564	(__v8df)_mm512_cvtps_pd(__A),
8963	8565	(__v8df)_mm512_setzero_pd());
8964	8566	}
8965	8567
8966		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8967		_mm512_cvtpslo_pd (__m512 __A)
8968		{
	8568	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8569	_mm512_cvtpslo_pd(__m512 __A) {
8969	8570	return (__m512d) _mm512_cvtps_pd(_mm512_castps512_ps256(__A));
8970	8571	}
8971	8572
8972		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8973		_mm512_mask_cvtpslo_pd (__m512d __W, __mmask8 __U, __m512 __A)
8974		{
	8573	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8574	_mm512_mask_cvtpslo_pd(__m512d __W, __mmask8 __U, __m512 __A) {
8975	8575	return (__m512d) _mm512_mask_cvtps_pd(__W, __U, _mm512_castps512_ps256(__A));
8976	8576	}
8977	8577
8978		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8979		_mm512_mask_mov_pd (__m512d __W, __mmask8 __U, __m512d __A)
8980		{
8981		return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
8982		(__v8df) __A,
8983		(__v8df) __W);
	8578	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8579	_mm512_mask_mov_pd(__m512d __W, __mmask8 __U, __m512d __A) {
	8580	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, (__v8df)__A,
	8581	(__v8df)__W);
8984	8582	}
8985	8583
8986		static __inline__ __m512d __DEFAULT_FN_ATTRS512
8987		_mm512_maskz_mov_pd (__mmask8 __U, __m512d __A)
8988		{
8989		return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
8990		(__v8df) __A,
8991		(__v8df) _mm512_setzero_pd ());
	8584	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8585	_mm512_maskz_mov_pd(__mmask8 __U, __m512d __A) {
	8586	return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, (__v8df)__A,
	8587	(__v8df)_mm512_setzero_pd());
8992	8588	}
8993	8589
8994		static __inline__ __m512 __DEFAULT_FN_ATTRS512
8995		_mm512_mask_mov_ps (__m512 __W, __mmask16 __U, __m512 __A)
8996		{
8997		return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
8998		(__v16sf) __A,
8999		(__v16sf) __W);
	8590	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8591	_mm512_mask_mov_ps(__m512 __W, __mmask16 __U, __m512 __A) {
	8592	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, (__v16sf)__A,
	8593	(__v16sf)__W);
9000	8594	}
9001	8595
9002		static __inline__ __m512 __DEFAULT_FN_ATTRS512
9003		_mm512_maskz_mov_ps (__mmask16 __U, __m512 __A)
9004		{
9005		return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
9006		(__v16sf) __A,
9007		(__v16sf) _mm512_setzero_ps ());
	8596	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8597	_mm512_maskz_mov_ps(__mmask16 __U, __m512 __A) {
	8598	return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, (__v16sf)__A,
	8599	(__v16sf)_mm512_setzero_ps());
9008	8600	}
9009	8601
9010	8602	static __inline__ void __DEFAULT_FN_ATTRS512
...	...	@@ -9053,18 +8645,16 @@ _mm512_mask_compressstoreu_epi32 (void *__P, __mmask16 __U, __m512i __A)
9053	8645	(__v4sf)_mm_setzero_ps(), \
9054	8646	(__mmask8)(U), (int)(R)))
9055	8647
9056		static __inline__ __m128 __DEFAULT_FN_ATTRS128
9057		_mm_mask_cvtsd_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128d __B)
9058		{
	8648	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	8649	_mm_mask_cvtsd_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128d __B) {
9059	8650	return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)__A,
9060	8651	(__v2df)__B,
9061	8652	(__v4sf)__W,
9062	8653	(__mmask8)__U, _MM_FROUND_CUR_DIRECTION);
9063	8654	}
9064	8655
9065		static __inline__ __m128 __DEFAULT_FN_ATTRS128
9066		_mm_maskz_cvtsd_ss (__mmask8 __U, __m128 __A, __m128d __B)
9067		{
	8656	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	8657	_mm_maskz_cvtsd_ss(__mmask8 __U, __m128 __A, __m128d __B) {
9068	8658	return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)__A,
9069	8659	(__v2df)__B,
9070	8660	(__v4sf)_mm_setzero_ps(),
...	...	@@ -9188,34 +8778,32 @@ _mm_cvtu64_ss (__m128 __A, unsigned long long __B)
9188	8778	}
9189	8779	#endif
9190	8780
9191		static __inline__ __m512i __DEFAULT_FN_ATTRS512
9192		_mm512_mask_set1_epi32 (__m512i __O, __mmask16 __M, int __A)
9193		{
	8781	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	8782	_mm512_mask_set1_epi32(__m512i __O, __mmask16 __M, int __A) {
9194	8783	return (__m512i) __builtin_ia32_selectd_512(__M,
9195	8784	(__v16si) _mm512_set1_epi32(__A),
9196	8785	(__v16si) __O);
9197	8786	}
9198	8787
9199		static __inline__ __m512i __DEFAULT_FN_ATTRS512
9200		_mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A)
9201		{
	8788	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	8789	_mm512_mask_set1_epi64(__m512i __O, __mmask8 __M, long long __A) {
9202	8790	return (__m512i) __builtin_ia32_selectq_512(__M,
9203	8791	(__v8di) _mm512_set1_epi64(__A),
9204	8792	(__v8di) __O);
9205	8793	}
9206	8794
9207		static __inline __m512i __DEFAULT_FN_ATTRS512
9208		_mm512_set_epi8 (char __e63, char __e62, char __e61, char __e60, char __e59,
9209		char __e58, char __e57, char __e56, char __e55, char __e54, char __e53,
9210		char __e52, char __e51, char __e50, char __e49, char __e48, char __e47,
9211		char __e46, char __e45, char __e44, char __e43, char __e42, char __e41,
9212		char __e40, char __e39, char __e38, char __e37, char __e36, char __e35,
9213		char __e34, char __e33, char __e32, char __e31, char __e30, char __e29,
9214		char __e28, char __e27, char __e26, char __e25, char __e24, char __e23,
9215		char __e22, char __e21, char __e20, char __e19, char __e18, char __e17,
9216		char __e16, char __e15, char __e14, char __e13, char __e12, char __e11,
9217		char __e10, char __e9, char __e8, char __e7, char __e6, char __e5,
9218		char __e4, char __e3, char __e2, char __e1, char __e0) {
	8795	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi8(
	8796	char __e63, char __e62, char __e61, char __e60, char __e59, char __e58,
	8797	char __e57, char __e56, char __e55, char __e54, char __e53, char __e52,
	8798	char __e51, char __e50, char __e49, char __e48, char __e47, char __e46,
	8799	char __e45, char __e44, char __e43, char __e42, char __e41, char __e40,
	8800	char __e39, char __e38, char __e37, char __e36, char __e35, char __e34,
	8801	char __e33, char __e32, char __e31, char __e30, char __e29, char __e28,
	8802	char __e27, char __e26, char __e25, char __e24, char __e23, char __e22,
	8803	char __e21, char __e20, char __e19, char __e18, char __e17, char __e16,
	8804	char __e15, char __e14, char __e13, char __e12, char __e11, char __e10,
	8805	char __e9, char __e8, char __e7, char __e6, char __e5, char __e4, char __e3,
	8806	char __e2, char __e1, char __e0) {
9219	8807
9220	8808	return __extension__ (__m512i)(__v64qi)
9221	8809	{__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
...	...	@@ -9228,14 +8816,13 @@ _mm512_set_epi8 (char __e63, char __e62, char __e61, char __e60, char __e59,
9228	8816	__e56, __e57, __e58, __e59, __e60, __e61, __e62, __e63};
9229	8817	}
9230	8818
9231		static __inline __m512i __DEFAULT_FN_ATTRS512
9232		_mm512_set_epi16(short __e31, short __e30, short __e29, short __e28,
9233		short __e27, short __e26, short __e25, short __e24, short __e23,
9234		short __e22, short __e21, short __e20, short __e19, short __e18,
9235		short __e17, short __e16, short __e15, short __e14, short __e13,
9236		short __e12, short __e11, short __e10, short __e9, short __e8,
9237		short __e7, short __e6, short __e5, short __e4, short __e3,
9238		short __e2, short __e1, short __e0) {
	8819	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi16(
	8820	short __e31, short __e30, short __e29, short __e28, short __e27,
	8821	short __e26, short __e25, short __e24, short __e23, short __e22,
	8822	short __e21, short __e20, short __e19, short __e18, short __e17,
	8823	short __e16, short __e15, short __e14, short __e13, short __e12,
	8824	short __e11, short __e10, short __e9, short __e8, short __e7, short __e6,
	8825	short __e5, short __e4, short __e3, short __e2, short __e1, short __e0) {
9239	8826	return __extension__ (__m512i)(__v32hi)
9240	8827	{__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
9241	8828	__e8, __e9, __e10, __e11, __e12, __e13, __e14, __e15,
...	...	@@ -9243,81 +8830,81 @@ _mm512_set_epi16(short __e31, short __e30, short __e29, short __e28,
9243	8830	__e24, __e25, __e26, __e27, __e28, __e29, __e30, __e31 };
9244	8831	}
9245	8832
9246		static __inline __m512i __DEFAULT_FN_ATTRS512
9247		_mm512_set_epi32 (int __A, int __B, int __C, int __D,
9248		int __E, int __F, int __G, int __H,
9249		int __I, int __J, int __K, int __L,
9250		int __M, int __N, int __O, int __P)
9251		{
	8833	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi32(
	8834	int __A, int __B, int __C, int __D, int __E, int __F, int __G, int __H,
	8835	int __I, int __J, int __K, int __L, int __M, int __N, int __O, int __P) {
9252	8836	return __extension__ (__m512i)(__v16si)
9253	8837	{ __P, __O, __N, __M, __L, __K, __J, __I,
9254	8838	__H, __G, __F, __E, __D, __C, __B, __A };
9255	8839	}
9256	8840
9257		#define _mm512_setr_epi32(e0,e1,e2,e3,e4,e5,e6,e7, \
9258		e8,e9,e10,e11,e12,e13,e14,e15) \
9259		_mm512_set_epi32((e15),(e14),(e13),(e12),(e11),(e10),(e9),(e8),(e7),(e6), \
9260		(e5),(e4),(e3),(e2),(e1),(e0))
	8841	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_setr_epi32(
	8842	int e0, int e1, int e2, int e3, int e4, int e5, int e6, int e7, int e8,
	8843	int e9, int e10, int e11, int e12, int e13, int e14, int e15) {
	8844	return _mm512_set_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4,
	8845	e3, e2, e1, e0);
	8846	}
9261	8847
9262		static __inline__ __m512i __DEFAULT_FN_ATTRS512
9263		_mm512_set_epi64 (long long __A, long long __B, long long __C,
9264		long long __D, long long __E, long long __F,
9265		long long __G, long long __H)
9266		{
	8848	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	8849	_mm512_set_epi64(long long __A, long long __B, long long __C, long long __D,
	8850	long long __E, long long __F, long long __G, long long __H) {
9267	8851	return __extension__ (__m512i) (__v8di)
9268	8852	{ __H, __G, __F, __E, __D, __C, __B, __A };
9269	8853	}
9270	8854
9271		#define _mm512_setr_epi64(e0,e1,e2,e3,e4,e5,e6,e7) \
9272		_mm512_set_epi64((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
	8855	static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
	8856	_mm512_setr_epi64(long long e0, long long e1, long long e2, long long e3,
	8857	long long e4, long long e5, long long e6, long long e7) {
	8858	return _mm512_set_epi64(e7, e6, e5, e4, e3, e2, e1, e0);
	8859	}
9273	8860
9274		static __inline__ __m512d __DEFAULT_FN_ATTRS512
9275		_mm512_set_pd (double __A, double __B, double __C, double __D,
9276		double __E, double __F, double __G, double __H)
9277		{
	8861	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8862	_mm512_set_pd(double __A, double __B, double __C, double __D, double __E,
	8863	double __F, double __G, double __H) {
9278	8864	return __extension__ (__m512d)
9279	8865	{ __H, __G, __F, __E, __D, __C, __B, __A };
9280	8866	}
9281	8867
9282		#define _mm512_setr_pd(e0,e1,e2,e3,e4,e5,e6,e7) \
9283		_mm512_set_pd((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
	8868	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8869	_mm512_setr_pd(double e0, double e1, double e2, double e3, double e4, double e5,
	8870	double e6, double e7) {
	8871	return _mm512_set_pd(e7, e6, e5, e4, e3, e2, e1, e0);
	8872	}
9284	8873
9285		static __inline__ __m512 __DEFAULT_FN_ATTRS512
9286		_mm512_set_ps (float __A, float __B, float __C, float __D,
9287		float __E, float __F, float __G, float __H,
9288		float __I, float __J, float __K, float __L,
9289		float __M, float __N, float __O, float __P)
9290		{
	8874	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8875	_mm512_set_ps(float __A, float __B, float __C, float __D, float __E, float __F,
	8876	float __G, float __H, float __I, float __J, float __K, float __L,
	8877	float __M, float __N, float __O, float __P) {
9291	8878	return __extension__ (__m512)
9292	8879	{ __P, __O, __N, __M, __L, __K, __J, __I,
9293	8880	__H, __G, __F, __E, __D, __C, __B, __A };
9294	8881	}
9295	8882
9296		#define _mm512_setr_ps(e0,e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15) \
9297		_mm512_set_ps((e15),(e14),(e13),(e12),(e11),(e10),(e9),(e8),(e7),(e6),(e5), \
9298		(e4),(e3),(e2),(e1),(e0))
	8883	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8884	_mm512_setr_ps(float e0, float e1, float e2, float e3, float e4, float e5,
	8885	float e6, float e7, float e8, float e9, float e10, float e11,
	8886	float e12, float e13, float e14, float e15) {
	8887	return _mm512_set_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3,
	8888	e2, e1, e0);
	8889	}
9299	8890
9300		static __inline__ __m512 __DEFAULT_FN_ATTRS512
9301		_mm512_abs_ps(__m512 __A)
9302		{
	8891	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8892	_mm512_abs_ps(__m512 __A) {
9303	8893	return (__m512)_mm512_and_epi32(_mm512_set1_epi32(0x7FFFFFFF),(__m512i)__A) ;
9304	8894	}
9305	8895
9306		static __inline__ __m512 __DEFAULT_FN_ATTRS512
9307		_mm512_mask_abs_ps(__m512 __W, __mmask16 __K, __m512 __A)
9308		{
	8896	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
	8897	_mm512_mask_abs_ps(__m512 __W, __mmask16 __K, __m512 __A) {
9309	8898	return (__m512)_mm512_mask_and_epi32((__m512i)__W, __K, _mm512_set1_epi32(0x7FFFFFFF),(__m512i)__A) ;
9310	8899	}
9311	8900
9312		static __inline__ __m512d __DEFAULT_FN_ATTRS512
9313		_mm512_abs_pd(__m512d __A)
9314		{
	8901	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8902	_mm512_abs_pd(__m512d __A) {
9315	8903	return (__m512d)_mm512_and_epi64(_mm512_set1_epi64(0x7FFFFFFFFFFFFFFF),(__v8di)__A) ;
9316	8904	}
9317	8905
9318		static __inline__ __m512d __DEFAULT_FN_ATTRS512
9319		_mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
9320		{
	8906	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
	8907	_mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A) {
9321	8908	return (__m512d)_mm512_mask_and_epi64((__v8di)__W, __K, _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF),(__v8di)__A);
9322	8909	}
9323	8910
...	...	@@ -9337,19 +8924,23 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
9337	8924	* This takes log2(n) steps where n is the number of elements in the vector.
9338	8925	*/
9339	8926
9340		static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {
	8927	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
	8928	_mm512_reduce_add_epi64(__m512i __W) {
9341	8929	return __builtin_reduce_add((__v8di)__W);
9342	8930	}
9343	8931
9344		static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_epi64(__m512i __W) {
	8932	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
	8933	_mm512_reduce_mul_epi64(__m512i __W) {
9345	8934	return __builtin_reduce_mul((__v8di)__W);
9346	8935	}
9347	8936
9348		static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_and_epi64(__m512i __W) {
	8937	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
	8938	_mm512_reduce_and_epi64(__m512i __W) {
9349	8939	return __builtin_reduce_and((__v8di)__W);
9350	8940	}
9351	8941
9352		static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_or_epi64(__m512i __W) {
	8942	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
	8943	_mm512_reduce_or_epi64(__m512i __W) {
9353	8944	return __builtin_reduce_or((__v8di)__W);
9354	8945	}
9355	8946
...	...	@@ -9400,22 +8991,22 @@ _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
9400	8991	return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
9401	8992	}
9402	8993
9403		static __inline__ int __DEFAULT_FN_ATTRS512
	8994	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9404	8995	_mm512_reduce_add_epi32(__m512i __W) {
9405	8996	return __builtin_reduce_add((__v16si)__W);
9406	8997	}
9407	8998
9408		static __inline__ int __DEFAULT_FN_ATTRS512
	8999	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9409	9000	_mm512_reduce_mul_epi32(__m512i __W) {
9410	9001	return __builtin_reduce_mul((__v16si)__W);
9411	9002	}
9412	9003
9413		static __inline__ int __DEFAULT_FN_ATTRS512
	9004	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9414	9005	_mm512_reduce_and_epi32(__m512i __W) {
9415	9006	return __builtin_reduce_and((__v16si)__W);
9416	9007	}
9417	9008
9418		static __inline__ int __DEFAULT_FN_ATTRS512
	9009	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9419	9010	_mm512_reduce_or_epi32(__m512i __W) {
9420	9011	return __builtin_reduce_or((__v16si)__W);
9421	9012	}
...	...	@@ -9466,22 +9057,22 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
9466	9057	return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
9467	9058	}
9468	9059
9469		static __inline__ long long __DEFAULT_FN_ATTRS512
	9060	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
9470	9061	_mm512_reduce_max_epi64(__m512i __V) {
9471	9062	return __builtin_reduce_max((__v8di)__V);
9472	9063	}
9473	9064
9474		static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
	9065	static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR
9475	9066	_mm512_reduce_max_epu64(__m512i __V) {
9476	9067	return __builtin_reduce_max((__v8du)__V);
9477	9068	}
9478	9069
9479		static __inline__ long long __DEFAULT_FN_ATTRS512
	9070	static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
9480	9071	_mm512_reduce_min_epi64(__m512i __V) {
9481	9072	return __builtin_reduce_min((__v8di)__V);
9482	9073	}
9483	9074
9484		static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
	9075	static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR
9485	9076	_mm512_reduce_min_epu64(__m512i __V) {
9486	9077	return __builtin_reduce_min((__v8du)__V);
9487	9078	}
...	...	@@ -9509,22 +9100,22 @@ _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {
9509	9100	__V = _mm512_mask_mov_epi64(_mm512_set1_epi64(-1LL), __M, __V);
9510	9101	return __builtin_reduce_min((__v8du)__V);
9511	9102	}
9512		static __inline__ int __DEFAULT_FN_ATTRS512
	9103	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9513	9104	_mm512_reduce_max_epi32(__m512i __V) {
9514	9105	return __builtin_reduce_max((__v16si)__V);
9515	9106	}
9516	9107
9517		static __inline__ unsigned int __DEFAULT_FN_ATTRS512
	9108	static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR
9518	9109	_mm512_reduce_max_epu32(__m512i __V) {
9519	9110	return __builtin_reduce_max((__v16su)__V);
9520	9111	}
9521	9112
9522		static __inline__ int __DEFAULT_FN_ATTRS512
	9113	static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
9523	9114	_mm512_reduce_min_epi32(__m512i __V) {
9524	9115	return __builtin_reduce_min((__v16si)__V);
9525	9116	}
9526	9117
9527		static __inline__ unsigned int __DEFAULT_FN_ATTRS512
	9118	static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR
9528	9119	_mm512_reduce_min_epu32(__m512i __V) {
9529	9120	return __builtin_reduce_min((__v16su)__V);
9530	9121	}

lib/include/avx512fp16intrin.h+112-110

...	...	@@ -22,26 +22,36 @@ typedef _Float16 __m512h_u __attribute__((__vector_size__(64), __aligned__(1)));
22	22
23	23	/* Define the default attributes for the functions in this file. */
24	24	#define __DEFAULT_FN_ATTRS512 \
25		__attribute__((__always_inline__, __nodebug__, \
26		__target__("avx512fp16,evex512"), __min_vector_width__(512)))
	25	__attribute__((__always_inline__, __nodebug__, __target__("avx512fp16"), \
	26	__min_vector_width__(512)))
27	27	#define __DEFAULT_FN_ATTRS256 \
28		__attribute__((__always_inline__, __nodebug__, \
29		__target__("avx512fp16,no-evex512"), \
	28	__attribute__((__always_inline__, __nodebug__, __target__("avx512fp16"), \
30	29	__min_vector_width__(256)))
31	30	#define __DEFAULT_FN_ATTRS128 \
32		__attribute__((__always_inline__, __nodebug__, \
33		__target__("avx512fp16,no-evex512"), \
	31	__attribute__((__always_inline__, __nodebug__, __target__("avx512fp16"), \
34	32	__min_vector_width__(128)))
35	33
36		static __inline__ _Float16 __DEFAULT_FN_ATTRS512 _mm512_cvtsh_h(__m512h __a) {
	34	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	35	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
	36	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	37	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	38	#else
	39	#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
	40	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	41	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	42	#endif
	43
	44	static __inline__ _Float16 __DEFAULT_FN_ATTRS512_CONSTEXPR
	45	_mm512_cvtsh_h(__m512h __a) {
37	46	return __a[0];
38	47	}
39	48
40		static __inline __m128h __DEFAULT_FN_ATTRS128 _mm_setzero_ph(void) {
	49	static __inline __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_setzero_ph(void) {
41	50	return (__m128h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
42	51	}
43	52
44		static __inline __m256h __DEFAULT_FN_ATTRS256 _mm256_setzero_ph(void) {
	53	static __inline __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	54	_mm256_setzero_ph(void) {
45	55	return (__m256h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
46	56	0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
47	57	}
...	...	@@ -50,7 +60,8 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_undefined_ph(void) {
50	60	return (__m256h)__builtin_ia32_undef256();
51	61	}
52	62
53		static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_setzero_ph(void) {
	63	static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
	64	_mm512_setzero_ph(void) {
54	65	return (__m512h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
55	66	0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
56	67	0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
...	...	@@ -64,14 +75,15 @@ static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_undefined_ph(void) {
64	75	return (__m512h)__builtin_ia32_undef512();
65	76	}
66	77
67		static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_set1_ph(_Float16 __h) {
	78	static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
	79	_mm512_set1_ph(_Float16 __h) {
68	80	return (__m512h)(__v32hf){__h, __h, __h, __h, __h, __h, __h, __h,
69	81	__h, __h, __h, __h, __h, __h, __h, __h,
70	82	__h, __h, __h, __h, __h, __h, __h, __h,
71	83	__h, __h, __h, __h, __h, __h, __h, __h};
72	84	}
73	85
74		static __inline __m512h __DEFAULT_FN_ATTRS512
	86	static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
75	87	_mm512_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
76	88	_Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
77	89	_Float16 __h9, _Float16 __h10, _Float16 __h11, _Float16 __h12,
...	...	@@ -87,106 +99,111 @@ _mm512_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
87	99	__h4, __h3, __h2, __h1};
88	100	}
89	101
90		#define _mm512_setr_ph(h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, h12, h13, \
91		h14, h15, h16, h17, h18, h19, h20, h21, h22, h23, h24, \
92		h25, h26, h27, h28, h29, h30, h31, h32) \
93		_mm512_set_ph((h32), (h31), (h30), (h29), (h28), (h27), (h26), (h25), (h24), \
94		(h23), (h22), (h21), (h20), (h19), (h18), (h17), (h16), (h15), \
95		(h14), (h13), (h12), (h11), (h10), (h9), (h8), (h7), (h6), \
96		(h5), (h4), (h3), (h2), (h1))
	102	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_setr_ph(
	103	_Float16 e0, _Float16 e1, _Float16 e2, _Float16 e3, _Float16 e4,
	104	_Float16 e5, _Float16 e6, _Float16 e7, _Float16 e8, _Float16 e9,
	105	_Float16 e10, _Float16 e11, _Float16 e12, _Float16 e13, _Float16 e14,
	106	_Float16 e15, _Float16 e16, _Float16 e17, _Float16 e18, _Float16 e19,
	107	_Float16 e20, _Float16 e21, _Float16 e22, _Float16 e23, _Float16 e24,
	108	_Float16 e25, _Float16 e26, _Float16 e27, _Float16 e28, _Float16 e29,
	109	_Float16 e30, _Float16 e31) {
	110	return _mm512_set_ph(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21,
	111	e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10,
	112	e9, e8, e7, e6, e5, e4, e3, e2, e1, e0);
	113	}
97	114
98		static __inline __m512h __DEFAULT_FN_ATTRS512
	115	static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
99	116	_mm512_set1_pch(_Float16 _Complex __h) {
100	117	return (__m512h)_mm512_set1_ps(__builtin_bit_cast(float, __h));
101	118	}
102	119
103		static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_castph_ps(__m128h __a) {
	120	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castph_ps(__m128h __a) {
104	121	return (__m128)__a;
105	122	}
106	123
107		static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_castph_ps(__m256h __a) {
	124	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_castph_ps(__m256h __a) {
108	125	return (__m256)__a;
109	126	}
110	127
111		static __inline__ __m512 __DEFAULT_FN_ATTRS512 _mm512_castph_ps(__m512h __a) {
	128	static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_castph_ps(__m512h __a) {
112	129	return (__m512)__a;
113	130	}
114	131
115		static __inline__ __m128d __DEFAULT_FN_ATTRS128 _mm_castph_pd(__m128h __a) {
	132	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castph_pd(__m128h __a) {
116	133	return (__m128d)__a;
117	134	}
118	135
119		static __inline__ __m256d __DEFAULT_FN_ATTRS256 _mm256_castph_pd(__m256h __a) {
	136	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_castph_pd(__m256h __a) {
120	137	return (__m256d)__a;
121	138	}
122	139
123		static __inline__ __m512d __DEFAULT_FN_ATTRS512 _mm512_castph_pd(__m512h __a) {
	140	static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_castph_pd(__m512h __a) {
124	141	return (__m512d)__a;
125	142	}
126	143
127		static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_castph_si128(__m128h __a) {
	144	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castph_si128(__m128h __a) {
128	145	return (__m128i)__a;
129	146	}
130	147
131		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	148	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
132	149	_mm256_castph_si256(__m256h __a) {
133	150	return (__m256i)__a;
134	151	}
135	152
136		static __inline__ __m512i __DEFAULT_FN_ATTRS512
	153	static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
137	154	_mm512_castph_si512(__m512h __a) {
138	155	return (__m512i)__a;
139	156	}
140	157
141		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_castps_ph(__m128 __a) {
	158	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castps_ph(__m128 __a) {
142	159	return (__m128h)__a;
143	160	}
144	161
145		static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_castps_ph(__m256 __a) {
	162	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_castps_ph(__m256 __a) {
146	163	return (__m256h)__a;
147	164	}
148	165
149		static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_castps_ph(__m512 __a) {
	166	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_castps_ph(__m512 __a) {
150	167	return (__m512h)__a;
151	168	}
152	169
153		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_castpd_ph(__m128d __a) {
	170	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castpd_ph(__m128d __a) {
154	171	return (__m128h)__a;
155	172	}
156	173
157		static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_castpd_ph(__m256d __a) {
	174	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR _mm256_castpd_ph(__m256d __a) {
158	175	return (__m256h)__a;
159	176	}
160	177
161		static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_castpd_ph(__m512d __a) {
	178	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_castpd_ph(__m512d __a) {
162	179	return (__m512h)__a;
163	180	}
164	181
165		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_castsi128_ph(__m128i __a) {
	182	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_castsi128_ph(__m128i __a) {
166	183	return (__m128h)__a;
167	184	}
168	185
169		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	186	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
170	187	_mm256_castsi256_ph(__m256i __a) {
171	188	return (__m256h)__a;
172	189	}
173	190
174		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	191	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
175	192	_mm512_castsi512_ph(__m512i __a) {
176	193	return (__m512h)__a;
177	194	}
178	195
179		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	196	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
180	197	_mm256_castph256_ph128(__m256h __a) {
181	198	return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
182	199	}
183	200
184		static __inline__ __m128h __DEFAULT_FN_ATTRS512
	201	static __inline__ __m128h __DEFAULT_FN_ATTRS512_CONSTEXPR
185	202	_mm512_castph512_ph128(__m512h __a) {
186	203	return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7);
187	204	}
188	205
189		static __inline__ __m256h __DEFAULT_FN_ATTRS512
	206	static __inline__ __m256h __DEFAULT_FN_ATTRS512_CONSTEXPR
190	207	_mm512_castph512_ph256(__m512h __a) {
191	208	return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
192	209	12, 13, 14, 15);
...	...	@@ -229,7 +246,7 @@ _mm512_castph256_ph512(__m256h __a) {
229	246	/// A 128-bit vector of [8 x half].
230	247	/// \returns A 512-bit floating-point vector of [16 x half]. The lower 128 bits
231	248	/// contain the value of the parameter. The upper 384 bits are set to zero.
232		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	249	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
233	250	_mm256_zextph128_ph256(__m128h __a) {
234	251	return __builtin_shufflevector(__a, (__v8hf)_mm_setzero_ph(), 0, 1, 2, 3, 4,
235	252	5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
...	...	@@ -248,7 +265,7 @@ _mm256_zextph128_ph256(__m128h __a) {
248	265	/// A 128-bit vector of [8 x half].
249	266	/// \returns A 512-bit floating-point vector of [32 x half]. The lower 128 bits
250	267	/// contain the value of the parameter. The upper 384 bits are set to zero.
251		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	268	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
252	269	_mm512_zextph128_ph512(__m128h __a) {
253	270	return __builtin_shufflevector(
254	271	__a, (__v8hf)_mm_setzero_ph(), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
...	...	@@ -268,7 +285,7 @@ _mm512_zextph128_ph512(__m128h __a) {
268	285	/// A 256-bit vector of [16 x half].
269	286	/// \returns A 512-bit floating-point vector of [32 x half]. The lower 256 bits
270	287	/// contain the value of the parameter. The upper 256 bits are set to zero.
271		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	288	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
272	289	_mm512_zextph256_ph512(__m256h __a) {
273	290	return __builtin_shufflevector(__a, (__v16hf)_mm256_setzero_ph(), 0, 1, 2, 3,
274	291	4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
...	...	@@ -482,19 +499,19 @@ _mm512_maskz_div_ph(__mmask32 __U, __m512h __A, __m512h __B) {
482	499	(__mmask32)(U), (__v32hf)_mm512_div_round_ph((A), (B), (R)), \
483	500	(__v32hf)_mm512_setzero_ph()))
484	501
485		static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_min_ph(__m512h __A,
486		__m512h __B) {
	502	static __inline__ __m512h
	503	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_min_ph(__m512h __A, __m512h __B) {
487	504	return (__m512h)__builtin_ia32_minph512((__v32hf)__A, (__v32hf)__B,
488	505	_MM_FROUND_CUR_DIRECTION);
489	506	}
490	507
491		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	508	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
492	509	_mm512_mask_min_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
493	510	return (__m512h)__builtin_ia32_selectph_512(
494	511	(__mmask32)__U, (__v32hf)_mm512_min_ph(__A, __B), (__v32hf)__W);
495	512	}
496	513
497		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	514	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
498	515	_mm512_maskz_min_ph(__mmask32 __U, __m512h __A, __m512h __B) {
499	516	return (__m512h)__builtin_ia32_selectph_512((__mmask32)__U,
500	517	(__v32hf)_mm512_min_ph(__A, __B),
...	...	@@ -515,19 +532,19 @@ _mm512_maskz_min_ph(__mmask32 __U, __m512h __A, __m512h __B) {
515	532	(__mmask32)(U), (__v32hf)_mm512_min_round_ph((A), (B), (R)), \
516	533	(__v32hf)_mm512_setzero_ph()))
517	534
518		static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_max_ph(__m512h __A,
519		__m512h __B) {
	535	static __inline__ __m512h
	536	__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_max_ph(__m512h __A, __m512h __B) {
520	537	return (__m512h)__builtin_ia32_maxph512((__v32hf)__A, (__v32hf)__B,
521	538	_MM_FROUND_CUR_DIRECTION);
522	539	}
523	540
524		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	541	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
525	542	_mm512_mask_max_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
526	543	return (__m512h)__builtin_ia32_selectph_512(
527	544	(__mmask32)__U, (__v32hf)_mm512_max_ph(__A, __B), (__v32hf)__W);
528	545	}
529	546
530		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	547	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
531	548	_mm512_maskz_max_ph(__mmask32 __U, __m512h __A, __m512h __B) {
532	549	return (__m512h)__builtin_ia32_selectph_512((__mmask32)__U,
533	550	(__v32hf)_mm512_max_ph(__A, __B),
...	...	@@ -548,7 +565,8 @@ _mm512_maskz_max_ph(__mmask32 __U, __m512h __A, __m512h __B) {
548	565	(__mmask32)(U), (__v32hf)_mm512_max_round_ph((A), (B), (R)), \
549	566	(__v32hf)_mm512_setzero_ph()))
550	567
551		static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_abs_ph(__m512h __A) {
	568	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
	569	_mm512_abs_ph(__m512h __A) {
552	570	return (__m512h)_mm512_and_epi32(_mm512_set1_epi32(0x7FFF7FFF), (__m512i)__A);
553	571	}
554	572
...	...	@@ -570,23 +588,20 @@ _mm512_maskz_conj_pch(__mmask16 __U, __m512h __A) {
570	588	(__v16sf)_mm512_setzero_ps());
571	589	}
572	590
573		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_add_sh(__m128h __A,
574		__m128h __B) {
	591	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	592	_mm_add_sh(__m128h __A, __m128h __B) {
575	593	__A[0] += __B[0];
576	594	return __A;
577	595	}
578	596
579		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_add_sh(__m128h __W,
580		__mmask8 __U,
581		__m128h __A,
582		__m128h __B) {
	597	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	598	_mm_mask_add_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
583	599	__A = _mm_add_sh(__A, __B);
584	600	return __builtin_ia32_selectsh_128(__U, __A, __W);
585	601	}
586	602
587		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_add_sh(__mmask8 __U,
588		__m128h __A,
589		__m128h __B) {
	603	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	604	_mm_maskz_add_sh(__mmask8 __U, __m128h __A, __m128h __B) {
590	605	__A = _mm_add_sh(__A, __B);
591	606	return __builtin_ia32_selectsh_128(__U, __A, _mm_setzero_ph());
592	607	}
...	...	@@ -606,23 +621,20 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_add_sh(__mmask8 __U,
606	621	(__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \
607	622	(__mmask8)(U), (int)(R)))
608	623
609		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_sub_sh(__m128h __A,
610		__m128h __B) {
	624	static __inline__ __m128h
	625	__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_sub_sh(__m128h __A, __m128h __B) {
611	626	__A[0] -= __B[0];
612	627	return __A;
613	628	}
614	629
615		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_sub_sh(__m128h __W,
616		__mmask8 __U,
617		__m128h __A,
618		__m128h __B) {
	630	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	631	_mm_mask_sub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
619	632	__A = _mm_sub_sh(__A, __B);
620	633	return __builtin_ia32_selectsh_128(__U, __A, __W);
621	634	}
622	635
623		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_sub_sh(__mmask8 __U,
624		__m128h __A,
625		__m128h __B) {
	636	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	637	_mm_maskz_sub_sh(__mmask8 __U, __m128h __A, __m128h __B) {
626	638	__A = _mm_sub_sh(__A, __B);
627	639	return __builtin_ia32_selectsh_128(__U, __A, _mm_setzero_ph());
628	640	}
...	...	@@ -642,23 +654,20 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_sub_sh(__mmask8 __U,
642	654	(__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \
643	655	(__mmask8)(U), (int)(R)))
644	656
645		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mul_sh(__m128h __A,
646		__m128h __B) {
	657	static __inline__ __m128h
	658	__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mul_sh(__m128h __A, __m128h __B) {
647	659	__A[0] *= __B[0];
648	660	return __A;
649	661	}
650	662
651		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_mul_sh(__m128h __W,
652		__mmask8 __U,
653		__m128h __A,
654		__m128h __B) {
	663	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	664	_mm_mask_mul_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
655	665	__A = _mm_mul_sh(__A, __B);
656	666	return __builtin_ia32_selectsh_128(__U, __A, __W);
657	667	}
658	668
659		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_mul_sh(__mmask8 __U,
660		__m128h __A,
661		__m128h __B) {
	669	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	670	_mm_maskz_mul_sh(__mmask8 __U, __m128h __A, __m128h __B) {
662	671	__A = _mm_mul_sh(__A, __B);
663	672	return __builtin_ia32_selectsh_128(__U, __A, _mm_setzero_ph());
664	673	}
...	...	@@ -678,23 +687,20 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_mul_sh(__mmask8 __U,
678	687	(__v8hf)(__m128h)(A), (__v8hf)(__m128h)(B), (__v8hf)_mm_setzero_ph(), \
679	688	(__mmask8)(U), (int)(R)))
680	689
681		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_div_sh(__m128h __A,
682		__m128h __B) {
	690	static __inline__ __m128h
	691	__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_div_sh(__m128h __A, __m128h __B) {
683	692	__A[0] /= __B[0];
684	693	return __A;
685	694	}
686	695
687		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_div_sh(__m128h __W,
688		__mmask8 __U,
689		__m128h __A,
690		__m128h __B) {
	696	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	697	_mm_mask_div_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
691	698	__A = _mm_div_sh(__A, __B);
692	699	return __builtin_ia32_selectsh_128(__U, __A, __W);
693	700	}
694	701
695		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_div_sh(__mmask8 __U,
696		__m128h __A,
697		__m128h __B) {
	702	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	703	_mm_maskz_div_sh(__mmask8 __U, __m128h __A, __m128h __B) {
698	704	__A = _mm_div_sh(__A, __B);
699	705	return __builtin_ia32_selectsh_128(__U, __A, _mm_setzero_ph());
700	706	}
...	...	@@ -942,22 +948,19 @@ static __inline__ void __DEFAULT_FN_ATTRS128 _mm_storeu_ph(void *__P,
942	948	}
943	949
944	950	// moves with vmovsh:
945		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_move_sh(__m128h __a,
946		__m128h __b) {
	951	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	952	_mm_move_sh(__m128h __a, __m128h __b) {
947	953	__a[0] = __b[0];
948	954	return __a;
949	955	}
950	956
951		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_move_sh(__m128h __W,
952		__mmask8 __U,
953		__m128h __A,
954		__m128h __B) {
	957	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	958	_mm_mask_move_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
955	959	return __builtin_ia32_selectsh_128(__U, _mm_move_sh(__A, __B), __W);
956	960	}
957	961
958		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_move_sh(__mmask8 __U,
959		__m128h __A,
960		__m128h __B) {
	962	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	963	_mm_maskz_move_sh(__mmask8 __U, __m128h __A, __m128h __B) {
961	964	return __builtin_ia32_selectsh_128(__U, _mm_move_sh(__A, __B),
962	965	_mm_setzero_ph());
963	966	}
...	...	@@ -1383,24 +1386,20 @@ _mm_maskz_scalef_sh(__mmask8 __U, __m128h __A, __m128h __B) {
1383	1386	(__v32hf)_mm512_setzero_ph()))
1384	1387
1385	1388	static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_sqrt_ph(__m512h __A) {
1386		return (__m512h)__builtin_ia32_sqrtph512((__v32hf)__A,
1387		_MM_FROUND_CUR_DIRECTION);
	1389	return (__m512h)__builtin_elementwise_sqrt((__v32hf)__A);
1388	1390	}
1389	1391
1390	1392	static __inline__ __m512h __DEFAULT_FN_ATTRS512
1391	1393	_mm512_mask_sqrt_ph(__m512h __W, __mmask32 __U, __m512h __A) {
1392	1394	return (__m512h)__builtin_ia32_selectph_512(
1393		(__mmask32)(__U),
1394		(__v32hf)__builtin_ia32_sqrtph512((__A), (_MM_FROUND_CUR_DIRECTION)),
1395		(__v32hf)(__m512h)(__W));
	1395	(__mmask32)(__U), (__v32hf)_mm512_sqrt_ph(__A), (__v32hf)(__m512h)(__W));
1396	1396	}
1397	1397
1398	1398	static __inline__ __m512h __DEFAULT_FN_ATTRS512
1399	1399	_mm512_maskz_sqrt_ph(__mmask32 __U, __m512h __A) {
1400		return (__m512h)__builtin_ia32_selectph_512(
1401		(__mmask32)(__U),
1402		(__v32hf)__builtin_ia32_sqrtph512((__A), (_MM_FROUND_CUR_DIRECTION)),
1403		(__v32hf)_mm512_setzero_ph());
	1400	return (__m512h)__builtin_ia32_selectph_512((__mmask32)(__U),
	1401	(__v32hf)_mm512_sqrt_ph(__A),
	1402	(__v32hf)_mm512_setzero_ph());
1404	1403	}
1405	1404
1406	1405	#define _mm_sqrt_round_sh(A, B, R) \
...	...	@@ -3292,19 +3291,19 @@ _mm512_reduce_min_ph(__m512h __V) {
3292	3291	return __builtin_ia32_reduce_fmin_ph512(__V);
3293	3292	}
3294	3293
3295		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	3294	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
3296	3295	_mm512_mask_blend_ph(__mmask32 __U, __m512h __A, __m512h __W) {
3297	3296	return (__m512h)__builtin_ia32_selectph_512((__mmask32)__U, (__v32hf)__W,
3298	3297	(__v32hf)__A);
3299	3298	}
3300	3299
3301		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	3300	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
3302	3301	_mm512_permutex2var_ph(__m512h __A, __m512i __I, __m512h __B) {
3303	3302	return (__m512h)__builtin_ia32_vpermi2varhi512((__v32hi)__A, (__v32hi)__I,
3304	3303	(__v32hi)__B);
3305	3304	}
3306	3305
3307		static __inline__ __m512h __DEFAULT_FN_ATTRS512
	3306	static __inline__ __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
3308	3307	_mm512_permutexvar_ph(__m512i __A, __m512h __B) {
3309	3308	return (__m512h)__builtin_ia32_permvarhi512((__v32hi)__B, (__v32hi)__A);
3310	3309	}
...	...	@@ -3348,6 +3347,9 @@ _mm512_permutexvar_ph(__m512i __A, __m512h __B) {
3348	3347	#undef __DEFAULT_FN_ATTRS128
3349	3348	#undef __DEFAULT_FN_ATTRS256
3350	3349	#undef __DEFAULT_FN_ATTRS512
	3350	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	3351	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
	3352	#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
3351	3353
3352	3354	#endif
3353	3355	#endif

lib/include/avx512ifmaintrin.h+32-34

...	...	@@ -15,54 +15,52 @@
15	15	#define __IFMAINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512ifma,evex512"), __min_vector_width__(512)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx512ifma"), \
	21	__min_vector_width__(512))) constexpr
	22	#else
	23	#define __DEFAULT_FN_ATTRS \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx512ifma"), \
	25	__min_vector_width__(512)))
	26	#endif
21	27
22	28	static __inline__ __m512i __DEFAULT_FN_ATTRS
23		_mm512_madd52hi_epu64 (__m512i __X, __m512i __Y, __m512i __Z)
24		{
25		return (__m512i)__builtin_ia32_vpmadd52huq512((__v8di) __X, (__v8di) __Y,
26		(__v8di) __Z);
	29	_mm512_madd52hi_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
	30	return (__m512i)__builtin_ia32_vpmadd52huq512((__v8di)__X, (__v8di)__Y,
	31	(__v8di)__Z);
27	32	}
28	33
29		static __inline__ __m512i __DEFAULT_FN_ATTRS
30		_mm512_mask_madd52hi_epu64 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
31		{
32		return (__m512i)__builtin_ia32_selectq_512(__M,
33		(__v8di)_mm512_madd52hi_epu64(__W, __X, __Y),
34		(__v8di)__W);
	34	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_madd52hi_epu64(
	35	__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
	36	return (__m512i)__builtin_ia32_selectq_512(
	37	__M, (__v8di)_mm512_madd52hi_epu64(__W, __X, __Y), (__v8di)__W);
35	38	}
36	39
37		static __inline__ __m512i __DEFAULT_FN_ATTRS
38		_mm512_maskz_madd52hi_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z)
39		{
40		return (__m512i)__builtin_ia32_selectq_512(__M,
41		(__v8di)_mm512_madd52hi_epu64(__X, __Y, __Z),
42		(__v8di)_mm512_setzero_si512());
	40	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_madd52hi_epu64(
	41	__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
	42	return (__m512i)__builtin_ia32_selectq_512(
	43	__M, (__v8di)_mm512_madd52hi_epu64(__X, __Y, __Z),
	44	(__v8di)_mm512_setzero_si512());
43	45	}
44	46
45	47	static __inline__ __m512i __DEFAULT_FN_ATTRS
46		_mm512_madd52lo_epu64 (__m512i __X, __m512i __Y, __m512i __Z)
47		{
48		return (__m512i)__builtin_ia32_vpmadd52luq512((__v8di) __X, (__v8di) __Y,
49		(__v8di) __Z);
	48	_mm512_madd52lo_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
	49	return (__m512i)__builtin_ia32_vpmadd52luq512((__v8di)__X, (__v8di)__Y,
	50	(__v8di)__Z);
50	51	}
51	52
52		static __inline__ __m512i __DEFAULT_FN_ATTRS
53		_mm512_mask_madd52lo_epu64 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
54		{
55		return (__m512i)__builtin_ia32_selectq_512(__M,
56		(__v8di)_mm512_madd52lo_epu64(__W, __X, __Y),
57		(__v8di)__W);
	53	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_madd52lo_epu64(
	54	__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
	55	return (__m512i)__builtin_ia32_selectq_512(
	56	__M, (__v8di)_mm512_madd52lo_epu64(__W, __X, __Y), (__v8di)__W);
58	57	}
59	58
60		static __inline__ __m512i __DEFAULT_FN_ATTRS
61		_mm512_maskz_madd52lo_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z)
62		{
63		return (__m512i)__builtin_ia32_selectq_512(__M,
64		(__v8di)_mm512_madd52lo_epu64(__X, __Y, __Z),
65		(__v8di)_mm512_setzero_si512());
	59	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_madd52lo_epu64(
	60	__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
	61	return (__m512i)__builtin_ia32_selectq_512(
	62	__M, (__v8di)_mm512_madd52lo_epu64(__X, __Y, __Z),
	63	(__v8di)_mm512_setzero_si512());
66	64	}
67	65
68	66	#undef __DEFAULT_FN_ATTRS

lib/include/avx512ifmavlintrin.h+73-46

...	...	@@ -8,22 +8,35 @@
8	8	*===-----------------------------------------------------------------------===
9	9	*/
10	10	#ifndef __IMMINTRIN_H
11		#error "Never use <avx512ifmavlintrin.h> directly; include <immintrin.h> instead."
	11	#error \
	12	"Never use <avx512ifmavlintrin.h> directly; include <immintrin.h> instead."
12	13	#endif
13	14
14	15	#ifndef __IFMAVLINTRIN_H
15	16	#define __IFMAVLINTRIN_H
16	17
17	18	/* Define the default attributes for the functions in this file. */
	19	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	20	#define __DEFAULT_FN_ATTRS128 \
19	21	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512ifma,avx512vl,no-evex512"), \
	22	__target__("avx512ifma,avx512vl"), \
	23	__min_vector_width__(128))) constexpr
	24	#define __DEFAULT_FN_ATTRS256 \
	25	__attribute__((__always_inline__, __nodebug__, \
	26	__target__("avx512ifma,avx512vl"), \
	27	__min_vector_width__(256))) constexpr
	28	#else
	29	#define __DEFAULT_FN_ATTRS128 \
	30	__attribute__((__always_inline__, __nodebug__, \
	31	__target__("avx512ifma,avx512vl"), \
21	32	__min_vector_width__(128)))
22	33	#define __DEFAULT_FN_ATTRS256 \
23	34	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512ifma,avx512vl,no-evex512"), \
	35	__target__("avx512ifma,avx512vl"), \
25	36	__min_vector_width__(256)))
	37	#endif
26	38
	39	#if !(defined(__AVXIFMA__) \|\| defined(__AVX512IFMA__))
27	40	#define _mm_madd52hi_epu64(X, Y, Z) \
28	41	((__m128i)__builtin_ia32_vpmadd52huq128((__v2di)(X), (__v2di)(Y), \
29	42	(__v2di)(Z)))
...	...	@@ -39,71 +52,85 @@
39	52	#define _mm256_madd52lo_epu64(X, Y, Z) \
40	53	((__m256i)__builtin_ia32_vpmadd52luq256((__v4di)(X), (__v4di)(Y), \
41	54	(__v4di)(Z)))
	55	#endif
42	56
	57	#if defined(__AVX512IFMA__)
43	58	static __inline__ __m128i __DEFAULT_FN_ATTRS128
44		_mm_mask_madd52hi_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
45		{
46		return (__m128i)__builtin_ia32_selectq_128(__M,
47		(__v2di)_mm_madd52hi_epu64(__W, __X, __Y),
48		(__v2di)__W);
	59	_mm_madd52hi_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
	60	return (__m128i)__builtin_ia32_vpmadd52huq128((__v2di)__X, (__v2di)__Y,
	61	(__v2di)__Z);
	62	}
	63
	64	static __inline__ __m256i __DEFAULT_FN_ATTRS256
	65	_mm256_madd52hi_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
	66	return (__m256i)__builtin_ia32_vpmadd52huq256((__v4di)__X, (__v4di)__Y,
	67	(__v4di)__Z);
49	68	}
50	69
51	70	static __inline__ __m128i __DEFAULT_FN_ATTRS128
52		_mm_maskz_madd52hi_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z)
53		{
54		return (__m128i)__builtin_ia32_selectq_128(__M,
55		(__v2di)_mm_madd52hi_epu64(__X, __Y, __Z),
56		(__v2di)_mm_setzero_si128());
	71	_mm_madd52lo_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
	72	return (__m128i)__builtin_ia32_vpmadd52luq128((__v2di)__X, (__v2di)__Y,
	73	(__v2di)__Z);
57	74	}
58	75
59	76	static __inline__ __m256i __DEFAULT_FN_ATTRS256
60		_mm256_mask_madd52hi_epu64 (__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
61		{
62		return (__m256i)__builtin_ia32_selectq_256(__M,
63		(__v4di)_mm256_madd52hi_epu64(__W, __X, __Y),
64		(__v4di)__W);
	77	_mm256_madd52lo_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
	78	return (__m256i)__builtin_ia32_vpmadd52luq256((__v4di)__X, (__v4di)__Y,
	79	(__v4di)__Z);
65	80	}
	81	#endif
66	82
67		static __inline__ __m256i __DEFAULT_FN_ATTRS256
68		_mm256_maskz_madd52hi_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z)
69		{
70		return (__m256i)__builtin_ia32_selectq_256(__M,
71		(__v4di)_mm256_madd52hi_epu64(__X, __Y, __Z),
72		(__v4di)_mm256_setzero_si256());
	83	static __inline__ __m128i __DEFAULT_FN_ATTRS128
	84	_mm_mask_madd52hi_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
	85	return (__m128i)__builtin_ia32_selectq_128(
	86	__M, (__v2di)__builtin_ia32_vpmadd52huq128(__W, __X, __Y), (__v2di)__W);
73	87	}
74	88
75	89	static __inline__ __m128i __DEFAULT_FN_ATTRS128
76		_mm_mask_madd52lo_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
77		{
78		return (__m128i)__builtin_ia32_selectq_128(__M,
79		(__v2di)_mm_madd52lo_epu64(__W, __X, __Y),
80		(__v2di)__W);
	90	_mm_maskz_madd52hi_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
	91	return (__m128i)__builtin_ia32_selectq_128(
	92	__M, (__v2di)__builtin_ia32_vpmadd52huq128(__X, __Y, __Z),
	93	(__v2di)_mm_setzero_si128());
	94	}
	95
	96	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_madd52hi_epu64(
	97	__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
	98	return (__m256i)__builtin_ia32_selectq_256(
	99	__M, (__v4di)__builtin_ia32_vpmadd52huq256(__W, __X, __Y), (__v4di)__W);
	100	}
	101
	102	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_madd52hi_epu64(
	103	__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
	104	return (__m256i)__builtin_ia32_selectq_256(
	105	__M, (__v4di)__builtin_ia32_vpmadd52huq256(__X, __Y, __Z),
	106	(__v4di)_mm256_setzero_si256());
81	107	}
82	108
83	109	static __inline__ __m128i __DEFAULT_FN_ATTRS128
84		_mm_maskz_madd52lo_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z)
85		{
86		return (__m128i)__builtin_ia32_selectq_128(__M,
87		(__v2di)_mm_madd52lo_epu64(__X, __Y, __Z),
88		(__v2di)_mm_setzero_si128());
	110	_mm_mask_madd52lo_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
	111	return (__m128i)__builtin_ia32_selectq_128(
	112	__M, (__v2di)__builtin_ia32_vpmadd52luq128(__W, __X, __Y), (__v2di)__W);
89	113	}
90	114
91		static __inline__ __m256i __DEFAULT_FN_ATTRS256
92		_mm256_mask_madd52lo_epu64 (__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
93		{
94		return (__m256i)__builtin_ia32_selectq_256(__M,
95		(__v4di)_mm256_madd52lo_epu64(__W, __X, __Y),
96		(__v4di)__W);
	115	static __inline__ __m128i __DEFAULT_FN_ATTRS128
	116	_mm_maskz_madd52lo_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
	117	return (__m128i)__builtin_ia32_selectq_128(
	118	__M, (__v2di)__builtin_ia32_vpmadd52luq128(__X, __Y, __Z),
	119	(__v2di)_mm_setzero_si128());
97	120	}
98	121
99		static __inline__ __m256i __DEFAULT_FN_ATTRS256
100		_mm256_maskz_madd52lo_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z)
101		{
102		return (__m256i)__builtin_ia32_selectq_256(__M,
103		(__v4di)_mm256_madd52lo_epu64(__X, __Y, __Z),
104		(__v4di)_mm256_setzero_si256());
	122	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_madd52lo_epu64(
	123	__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
	124	return (__m256i)__builtin_ia32_selectq_256(
	125	__M, (__v4di)__builtin_ia32_vpmadd52luq256(__W, __X, __Y), (__v4di)__W);
105	126	}
106	127
	128	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_madd52lo_epu64(
	129	__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
	130	return (__m256i)__builtin_ia32_selectq_256(
	131	__M, (__v4di)__builtin_ia32_vpmadd52luq256(__X, __Y, __Z),
	132	(__v4di)_mm256_setzero_si256());
	133	}
107	134
108	135	#undef __DEFAULT_FN_ATTRS128
109	136	#undef __DEFAULT_FN_ATTRS256

lib/include/avx512vbmi2intrin.h+43-32

...	...	@@ -15,8 +15,15 @@
15	15	#define __AVX512VBMI2INTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
18		#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("avx512vbmi2,evex512"), __min_vector_width__(512)))
19
	18	#define __DEFAULT_FN_ATTRS \
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx512vbmi2"), \
	20	__min_vector_width__(512)))
	21
	22	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	23	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	24	#else
	25	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	26	#endif
20	27
21	28	static __inline__ __m512i __DEFAULT_FN_ATTRS
22	29	_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D)
...	...	@@ -212,14 +219,14 @@ _mm512_maskz_expandloadu_epi8(__mmask64 __U, void const *__P)
212	219	(__v32hi)_mm512_shrdi_epi16((A), (B), (I)), \
213	220	(__v32hi)_mm512_setzero_si512()))
214	221
215		static __inline__ __m512i __DEFAULT_FN_ATTRS
	222	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
216	223	_mm512_shldv_epi64(__m512i __A, __m512i __B, __m512i __C)
217	224	{
218		return (__m512i)__builtin_ia32_vpshldvq512((__v8di)__A, (__v8di)__B,
219		(__v8di)__C);
	225	return (__m512i)__builtin_elementwise_fshl((__v8du)__A, (__v8du)__B,
	226	(__v8du)__C);
220	227	}
221	228
222		static __inline__ __m512i __DEFAULT_FN_ATTRS
	229	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
223	230	_mm512_mask_shldv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
224	231	{
225	232	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -227,7 +234,7 @@ _mm512_mask_shldv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
227	234	(__v8di)__A);
228	235	}
229	236
230		static __inline__ __m512i __DEFAULT_FN_ATTRS
	237	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
231	238	_mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
232	239	{
233	240	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -235,14 +242,14 @@ _mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
235	242	(__v8di)_mm512_setzero_si512());
236	243	}
237	244
238		static __inline__ __m512i __DEFAULT_FN_ATTRS
	245	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
239	246	_mm512_shldv_epi32(__m512i __A, __m512i __B, __m512i __C)
240	247	{
241		return (__m512i)__builtin_ia32_vpshldvd512((__v16si)__A, (__v16si)__B,
242		(__v16si)__C);
	248	return (__m512i)__builtin_elementwise_fshl((__v16su)__A, (__v16su)__B,
	249	(__v16su)__C);
243	250	}
244	251
245		static __inline__ __m512i __DEFAULT_FN_ATTRS
	252	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
246	253	_mm512_mask_shldv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
247	254	{
248	255	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -250,7 +257,7 @@ _mm512_mask_shldv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
250	257	(__v16si)__A);
251	258	}
252	259
253		static __inline__ __m512i __DEFAULT_FN_ATTRS
	260	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
254	261	_mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
255	262	{
256	263	return (__m512i)__builtin_ia32_selectd_512(__U,
...	...	@@ -258,14 +265,14 @@ _mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
258	265	(__v16si)_mm512_setzero_si512());
259	266	}
260	267
261		static __inline__ __m512i __DEFAULT_FN_ATTRS
	268	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
262	269	_mm512_shldv_epi16(__m512i __A, __m512i __B, __m512i __C)
263	270	{
264		return (__m512i)__builtin_ia32_vpshldvw512((__v32hi)__A, (__v32hi)__B,
265		(__v32hi)__C);
	271	return (__m512i)__builtin_elementwise_fshl((__v32hu)__A, (__v32hu)__B,
	272	(__v32hu)__C);
266	273	}
267	274
268		static __inline__ __m512i __DEFAULT_FN_ATTRS
	275	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
269	276	_mm512_mask_shldv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
270	277	{
271	278	return (__m512i)__builtin_ia32_selectw_512(__U,
...	...	@@ -273,7 +280,7 @@ _mm512_mask_shldv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
273	280	(__v32hi)__A);
274	281	}
275	282
276		static __inline__ __m512i __DEFAULT_FN_ATTRS
	283	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
277	284	_mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
278	285	{
279	286	return (__m512i)__builtin_ia32_selectw_512(__U,
...	...	@@ -281,14 +288,15 @@ _mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
281	288	(__v32hi)_mm512_setzero_si512());
282	289	}
283	290
284		static __inline__ __m512i __DEFAULT_FN_ATTRS
	291	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
285	292	_mm512_shrdv_epi64(__m512i __A, __m512i __B, __m512i __C)
286	293	{
287		return (__m512i)__builtin_ia32_vpshrdvq512((__v8di)__A, (__v8di)__B,
288		(__v8di)__C);
	294	// Ops __A and __B are swapped.
	295	return (__m512i)__builtin_elementwise_fshr((__v8du)__B, (__v8du)__A,
	296	(__v8du)__C);
289	297	}
290	298
291		static __inline__ __m512i __DEFAULT_FN_ATTRS
	299	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
292	300	_mm512_mask_shrdv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
293	301	{
294	302	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -296,7 +304,7 @@ _mm512_mask_shrdv_epi64(__m512i __A, __mmask8 __U, __m512i __B, __m512i __C)
296	304	(__v8di)__A);
297	305	}
298	306
299		static __inline__ __m512i __DEFAULT_FN_ATTRS
	307	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
300	308	_mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
301	309	{
302	310	return (__m512i)__builtin_ia32_selectq_512(__U,
...	...	@@ -304,14 +312,15 @@ _mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
304	312	(__v8di)_mm512_setzero_si512());
305	313	}
306	314
307		static __inline__ __m512i __DEFAULT_FN_ATTRS
	315	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
308	316	_mm512_shrdv_epi32(__m512i __A, __m512i __B, __m512i __C)
309	317	{
310		return (__m512i)__builtin_ia32_vpshrdvd512((__v16si)__A, (__v16si)__B,
311		(__v16si)__C);
	318	// Ops __A and __B are swapped.
	319	return (__m512i)__builtin_elementwise_fshr((__v16su)__B, (__v16su)__A,
	320	(__v16su)__C);
312	321	}
313	322
314		static __inline__ __m512i __DEFAULT_FN_ATTRS
	323	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
315	324	_mm512_mask_shrdv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
316	325	{
317	326	return (__m512i) __builtin_ia32_selectd_512(__U,
...	...	@@ -319,7 +328,7 @@ _mm512_mask_shrdv_epi32(__m512i __A, __mmask16 __U, __m512i __B, __m512i __C)
319	328	(__v16si)__A);
320	329	}
321	330
322		static __inline__ __m512i __DEFAULT_FN_ATTRS
	331	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
323	332	_mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
324	333	{
325	334	return (__m512i) __builtin_ia32_selectd_512(__U,
...	...	@@ -327,14 +336,15 @@ _mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
327	336	(__v16si)_mm512_setzero_si512());
328	337	}
329	338
330		static __inline__ __m512i __DEFAULT_FN_ATTRS
	339	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
331	340	_mm512_shrdv_epi16(__m512i __A, __m512i __B, __m512i __C)
332	341	{
333		return (__m512i)__builtin_ia32_vpshrdvw512((__v32hi)__A, (__v32hi)__B,
334		(__v32hi)__C);
	342	// Ops __A and __B are swapped.
	343	return (__m512i)__builtin_elementwise_fshr((__v32hu)__B, (__v32hu)__A,
	344	(__v32hu)__C);
335	345	}
336	346
337		static __inline__ __m512i __DEFAULT_FN_ATTRS
	347	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
338	348	_mm512_mask_shrdv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
339	349	{
340	350	return (__m512i)__builtin_ia32_selectw_512(__U,
...	...	@@ -342,7 +352,7 @@ _mm512_mask_shrdv_epi16(__m512i __A, __mmask32 __U, __m512i __B, __m512i __C)
342	352	(__v32hi)__A);
343	353	}
344	354
345		static __inline__ __m512i __DEFAULT_FN_ATTRS
	355	static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
346	356	_mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
347	357	{
348	358	return (__m512i)__builtin_ia32_selectw_512(__U,
...	...	@@ -352,6 +362,7 @@ _mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
352	362
353	363
354	364	#undef __DEFAULT_FN_ATTRS
	365	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
355	366
356	367	#endif
357	368

lib/include/avx512vbmiintrin.h+19-27

...	...	@@ -15,63 +15,57 @@
15	15	#define __VBMIINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vbmi,evex512"), __min_vector_width__(512)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("avx512vbmi"), \
	21	__min_vector_width__(512))) constexpr
	22	#else
	23	#define __DEFAULT_FN_ATTRS \
	24	__attribute__((__always_inline__, __nodebug__, __target__("avx512vbmi"), \
	25	__min_vector_width__(512)))
	26	#endif
21	27
22	28	static __inline__ __m512i __DEFAULT_FN_ATTRS
23		_mm512_permutex2var_epi8(__m512i __A, __m512i __I, __m512i __B)
24		{
	29	_mm512_permutex2var_epi8(__m512i __A, __m512i __I, __m512i __B) {
25	30	return (__m512i)__builtin_ia32_vpermi2varqi512((__v64qi)__A, (__v64qi)__I,
26	31	(__v64qi) __B);
27	32	}
28	33
29		static __inline__ __m512i __DEFAULT_FN_ATTRS
30		_mm512_mask_permutex2var_epi8(__m512i __A, __mmask64 __U, __m512i __I,
31		__m512i __B)
32		{
	34	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_permutex2var_epi8(
	35	__m512i __A, __mmask64 __U, __m512i __I, __m512i __B) {
33	36	return (__m512i)__builtin_ia32_selectb_512(__U,
34	37	(__v64qi)_mm512_permutex2var_epi8(__A, __I, __B),
35	38	(__v64qi)__A);
36	39	}
37	40
38		static __inline__ __m512i __DEFAULT_FN_ATTRS
39		_mm512_mask2_permutex2var_epi8(__m512i __A, __m512i __I, __mmask64 __U,
40		__m512i __B)
41		{
	41	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask2_permutex2var_epi8(
	42	__m512i __A, __m512i __I, __mmask64 __U, __m512i __B) {
42	43	return (__m512i)__builtin_ia32_selectb_512(__U,
43	44	(__v64qi)_mm512_permutex2var_epi8(__A, __I, __B),
44	45	(__v64qi)__I);
45	46	}
46	47
47		static __inline__ __m512i __DEFAULT_FN_ATTRS
48		_mm512_maskz_permutex2var_epi8(__mmask64 __U, __m512i __A, __m512i __I,
49		__m512i __B)
50		{
	48	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_permutex2var_epi8(
	49	__mmask64 __U, __m512i __A, __m512i __I, __m512i __B) {
51	50	return (__m512i)__builtin_ia32_selectb_512(__U,
52	51	(__v64qi)_mm512_permutex2var_epi8(__A, __I, __B),
53	52	(__v64qi)_mm512_setzero_si512());
54	53	}
55	54
56	55	static __inline__ __m512i __DEFAULT_FN_ATTRS
57		_mm512_permutexvar_epi8 (__m512i __A, __m512i __B)
58		{
	56	_mm512_permutexvar_epi8(__m512i __A, __m512i __B) {
59	57	return (__m512i)__builtin_ia32_permvarqi512((__v64qi) __B, (__v64qi) __A);
60	58	}
61	59
62	60	static __inline__ __m512i __DEFAULT_FN_ATTRS
63		_mm512_maskz_permutexvar_epi8 (__mmask64 __M, __m512i __A,
64		__m512i __B)
65		{
	61	_mm512_maskz_permutexvar_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
66	62	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
67	63	(__v64qi)_mm512_permutexvar_epi8(__A, __B),
68	64	(__v64qi)_mm512_setzero_si512());
69	65	}
70	66
71		static __inline__ __m512i __DEFAULT_FN_ATTRS
72		_mm512_mask_permutexvar_epi8 (__m512i __W, __mmask64 __M, __m512i __A,
73		__m512i __B)
74		{
	67	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_permutexvar_epi8(
	68	__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
75	69	return (__m512i)__builtin_ia32_selectb_512((__mmask64)__M,
76	70	(__v64qi)_mm512_permutexvar_epi8(__A, __B),
77	71	(__v64qi)__W);
...	...	@@ -100,7 +94,5 @@ _mm512_maskz_multishift_epi64_epi8(__mmask64 __M, __m512i __X, __m512i __Y)
100	94	(__v64qi)_mm512_setzero_si512());
101	95	}
102	96
103
104	97	#undef __DEFAULT_FN_ATTRS
105
106	98	#endif

lib/include/avx512vbmivlintrin.h+35-49

...	...	@@ -15,126 +15,114 @@
15	15	#define __VBMIVLINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS128 \
19	20	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vbmi,avx512vl,no-evex512"), \
	21	__target__("avx512vbmi,avx512vl"), \
	22	__min_vector_width__(128))) constexpr
	23	#define __DEFAULT_FN_ATTRS256 \
	24	__attribute__((__always_inline__, __nodebug__, \
	25	__target__("avx512vbmi,avx512vl"), \
	26	__min_vector_width__(256))) constexpr
	27	#else
	28	#define __DEFAULT_FN_ATTRS128 \
	29	__attribute__((__always_inline__, __nodebug__, \
	30	__target__("avx512vbmi,avx512vl"), \
21	31	__min_vector_width__(128)))
22	32	#define __DEFAULT_FN_ATTRS256 \
23	33	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vbmi,avx512vl,no-evex512"), \
	34	__target__("avx512vbmi,avx512vl"), \
25	35	__min_vector_width__(256)))
	36	#endif
26	37
27	38	static __inline__ __m128i __DEFAULT_FN_ATTRS128
28		_mm_permutex2var_epi8(__m128i __A, __m128i __I, __m128i __B)
29		{
	39	_mm_permutex2var_epi8(__m128i __A, __m128i __I, __m128i __B) {
30	40	return (__m128i)__builtin_ia32_vpermi2varqi128((__v16qi)__A,
31	41	(__v16qi)__I,
32	42	(__v16qi)__B);
33	43	}
34	44
35		static __inline__ __m128i __DEFAULT_FN_ATTRS128
36		_mm_mask_permutex2var_epi8(__m128i __A, __mmask16 __U, __m128i __I,
37		__m128i __B)
38		{
	45	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_mask_permutex2var_epi8(
	46	__m128i __A, __mmask16 __U, __m128i __I, __m128i __B) {
39	47	return (__m128i)__builtin_ia32_selectb_128(__U,
40	48	(__v16qi)_mm_permutex2var_epi8(__A, __I, __B),
41	49	(__v16qi)__A);
42	50	}
43	51
44		static __inline__ __m128i __DEFAULT_FN_ATTRS128
45		_mm_mask2_permutex2var_epi8(__m128i __A, __m128i __I, __mmask16 __U,
46		__m128i __B)
47		{
	52	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_mask2_permutex2var_epi8(
	53	__m128i __A, __m128i __I, __mmask16 __U, __m128i __B) {
48	54	return (__m128i)__builtin_ia32_selectb_128(__U,
49	55	(__v16qi)_mm_permutex2var_epi8(__A, __I, __B),
50	56	(__v16qi)__I);
51	57	}
52	58
53		static __inline__ __m128i __DEFAULT_FN_ATTRS128
54		_mm_maskz_permutex2var_epi8(__mmask16 __U, __m128i __A, __m128i __I,
55		__m128i __B)
56		{
	59	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_maskz_permutex2var_epi8(
	60	__mmask16 __U, __m128i __A, __m128i __I, __m128i __B) {
57	61	return (__m128i)__builtin_ia32_selectb_128(__U,
58	62	(__v16qi)_mm_permutex2var_epi8(__A, __I, __B),
59	63	(__v16qi)_mm_setzero_si128());
60	64	}
61	65
62	66	static __inline__ __m256i __DEFAULT_FN_ATTRS256
63		_mm256_permutex2var_epi8(__m256i __A, __m256i __I, __m256i __B)
64		{
	67	_mm256_permutex2var_epi8(__m256i __A, __m256i __I, __m256i __B) {
65	68	return (__m256i)__builtin_ia32_vpermi2varqi256((__v32qi)__A, (__v32qi)__I,
66	69	(__v32qi)__B);
67	70	}
68	71
69		static __inline__ __m256i __DEFAULT_FN_ATTRS256
70		_mm256_mask_permutex2var_epi8(__m256i __A, __mmask32 __U, __m256i __I,
71		__m256i __B)
72		{
	72	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_permutex2var_epi8(
	73	__m256i __A, __mmask32 __U, __m256i __I, __m256i __B) {
73	74	return (__m256i)__builtin_ia32_selectb_256(__U,
74	75	(__v32qi)_mm256_permutex2var_epi8(__A, __I, __B),
75	76	(__v32qi)__A);
76	77	}
77	78
78		static __inline__ __m256i __DEFAULT_FN_ATTRS256
79		_mm256_mask2_permutex2var_epi8(__m256i __A, __m256i __I, __mmask32 __U,
80		__m256i __B)
81		{
	79	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask2_permutex2var_epi8(
	80	__m256i __A, __m256i __I, __mmask32 __U, __m256i __B) {
82	81	return (__m256i)__builtin_ia32_selectb_256(__U,
83	82	(__v32qi)_mm256_permutex2var_epi8(__A, __I, __B),
84	83	(__v32qi)__I);
85	84	}
86	85
87		static __inline__ __m256i __DEFAULT_FN_ATTRS256
88		_mm256_maskz_permutex2var_epi8(__mmask32 __U, __m256i __A, __m256i __I,
89		__m256i __B)
90		{
	86	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_permutex2var_epi8(
	87	__mmask32 __U, __m256i __A, __m256i __I, __m256i __B) {
91	88	return (__m256i)__builtin_ia32_selectb_256(__U,
92	89	(__v32qi)_mm256_permutex2var_epi8(__A, __I, __B),
93	90	(__v32qi)_mm256_setzero_si256());
94	91	}
95	92
96	93	static __inline__ __m128i __DEFAULT_FN_ATTRS128
97		_mm_permutexvar_epi8 (__m128i __A, __m128i __B)
98		{
	94	_mm_permutexvar_epi8(__m128i __A, __m128i __B) {
99	95	return (__m128i)__builtin_ia32_permvarqi128((__v16qi)__B, (__v16qi)__A);
100	96	}
101	97
102	98	static __inline__ __m128i __DEFAULT_FN_ATTRS128
103		_mm_maskz_permutexvar_epi8 (__mmask16 __M, __m128i __A, __m128i __B)
104		{
	99	_mm_maskz_permutexvar_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
105	100	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
106	101	(__v16qi)_mm_permutexvar_epi8(__A, __B),
107	102	(__v16qi)_mm_setzero_si128());
108	103	}
109	104
110		static __inline__ __m128i __DEFAULT_FN_ATTRS128
111		_mm_mask_permutexvar_epi8 (__m128i __W, __mmask16 __M, __m128i __A,
112		__m128i __B)
113		{
	105	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_mask_permutexvar_epi8(
	106	__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
114	107	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
115	108	(__v16qi)_mm_permutexvar_epi8(__A, __B),
116	109	(__v16qi)__W);
117	110	}
118	111
119	112	static __inline__ __m256i __DEFAULT_FN_ATTRS256
120		_mm256_permutexvar_epi8 (__m256i __A, __m256i __B)
121		{
	113	_mm256_permutexvar_epi8(__m256i __A, __m256i __B) {
122	114	return (__m256i)__builtin_ia32_permvarqi256((__v32qi) __B, (__v32qi) __A);
123	115	}
124	116
125	117	static __inline__ __m256i __DEFAULT_FN_ATTRS256
126		_mm256_maskz_permutexvar_epi8 (__mmask32 __M, __m256i __A,
127		__m256i __B)
128		{
	118	_mm256_maskz_permutexvar_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
129	119	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
130	120	(__v32qi)_mm256_permutexvar_epi8(__A, __B),
131	121	(__v32qi)_mm256_setzero_si256());
132	122	}
133	123
134		static __inline__ __m256i __DEFAULT_FN_ATTRS256
135		_mm256_mask_permutexvar_epi8 (__m256i __W, __mmask32 __M, __m256i __A,
136		__m256i __B)
137		{
	124	static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_permutexvar_epi8(
	125	__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
138	126	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
139	127	(__v32qi)_mm256_permutexvar_epi8(__A, __B),
140	128	(__v32qi)__W);
...	...	@@ -186,8 +174,6 @@ _mm256_maskz_multishift_epi64_epi8(__mmask32 __M, __m256i __X, __m256i __Y)
186	174	(__v32qi)_mm256_setzero_si256());
187	175	}
188	176
189
190	177	#undef __DEFAULT_FN_ATTRS128
191	178	#undef __DEFAULT_FN_ATTRS256
192
193	179	#endif

lib/include/avx512vlbf16intrin.h+32-22

...	...	@@ -17,13 +17,21 @@
17	17
18	18	#define __DEFAULT_FN_ATTRS128 \
19	19	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512bf16,no-evex512"), \
	20	__target__("avx512vl,avx512bf16"), \
21	21	__min_vector_width__(128)))
22	22	#define __DEFAULT_FN_ATTRS256 \
23	23	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512bf16,no-evex512"), \
	24	__target__("avx512vl,avx512bf16"), \
25	25	__min_vector_width__(256)))
26	26
	27	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	28	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	29	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	30	#else
	31	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	32	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	33	#endif
	34
27	35	/// Convert Two Packed Single Data to One Packed BF16 Data.
28	36	///
29	37	/// \headerfile <x86intrin.h>
...	...	@@ -421,9 +429,10 @@ static __inline__ __bf16 __DEFAULT_FN_ATTRS128 _mm_cvtness_sbh(float __A) {
421	429	/// \param __A
422	430	/// A 128-bit vector of [4 x bfloat].
423	431	/// \returns A 128-bit vector of [4 x float] come from conversion of __A
424		static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_cvtpbh_ps(__m128bh __A) {
425		return _mm_castsi128_ps(
426		(__m128i)_mm_slli_epi32((__m128i)_mm_cvtepi16_epi32((__m128i)__A), 16));
	432	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	433	_mm_cvtpbh_ps(__m128bh __A) {
	434	return (__m128)_mm256_castps256_ps128(
	435	(__m256) __builtin_convertvector(__A, __v8sf));
427	436	}
428	437
429	438	/// Convert Packed BF16 Data to Packed float Data.
...	...	@@ -433,9 +442,9 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS128 _mm_cvtpbh_ps(__m128bh __A) {
433	442	/// \param __A
434	443	/// A 128-bit vector of [8 x bfloat].
435	444	/// \returns A 256-bit vector of [8 x float] come from conversion of __A
436		static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
437		return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
438		(__m256i)_mm256_cvtepi16_epi32((__m128i)__A), 16));
	445	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	446	_mm256_cvtpbh_ps(__m128bh __A) {
	447	return (__m256) __builtin_convertvector(__A, __v8sf);
439	448	}
440	449
441	450	/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
...	...	@@ -448,10 +457,10 @@ static __inline__ __m256 __DEFAULT_FN_ATTRS256 _mm256_cvtpbh_ps(__m128bh __A) {
448	457	/// \param __A
449	458	/// A 128-bit vector of [4 x bfloat].
450	459	/// \returns A 128-bit vector of [4 x float] come from conversion of __A
451		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	460	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
452	461	_mm_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
453		return _mm_castsi128_ps((__m128i)_mm_slli_epi32(
454		(__m128i)_mm_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
	462	return (__m128)__builtin_ia32_selectps_128(
	463	(__mmask8)__U, (__v4sf)_mm_cvtpbh_ps(__A), (__v4sf)_mm_setzero_ps());
455	464	}
456	465
457	466	/// Convert Packed BF16 Data to Packed float Data using zeroing mask.
...	...	@@ -464,10 +473,11 @@ _mm_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
464	473	/// \param __A
465	474	/// A 128-bit vector of [8 x bfloat].
466	475	/// \returns A 256-bit vector of [8 x float] come from conversion of __A
467		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	476	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
468	477	_mm256_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
469		return _mm256_castsi256_ps((__m256i)_mm256_slli_epi32(
470		(__m256i)_mm256_maskz_cvtepi16_epi32((__mmask8)__U, (__m128i)__A), 16));
	478	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
	479	(__v8sf)_mm256_cvtpbh_ps(__A),
	480	(__v8sf)_mm256_setzero_ps());
471	481	}
472	482
473	483	/// Convert Packed BF16 Data to Packed float Data using merging mask.
...	...	@@ -483,11 +493,10 @@ _mm256_maskz_cvtpbh_ps(__mmask8 __U, __m128bh __A) {
483	493	/// \param __A
484	494	/// A 128-bit vector of [4 x bfloat].
485	495	/// \returns A 128-bit vector of [4 x float] come from conversion of __A
486		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	496	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
487	497	_mm_mask_cvtpbh_ps(__m128 __S, __mmask8 __U, __m128bh __A) {
488		return _mm_castsi128_ps((__m128i)_mm_mask_slli_epi32(
489		(__m128i)__S, (__mmask8)__U, (__m128i)_mm_cvtepi16_epi32((__m128i)__A),
490		16));
	498	return (__m128)__builtin_ia32_selectps_128(
	499	(__mmask8)__U, (__v4sf)_mm_cvtpbh_ps(__A), (__v4sf)__S);
491	500	}
492	501
493	502	/// Convert Packed BF16 Data to Packed float Data using merging mask.
...	...	@@ -503,15 +512,16 @@ _mm_mask_cvtpbh_ps(__m128 __S, __mmask8 __U, __m128bh __A) {
503	512	/// \param __A
504	513	/// A 128-bit vector of [8 x bfloat].
505	514	/// \returns A 256-bit vector of [8 x float] come from conversion of __A
506		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	515	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
507	516	_mm256_mask_cvtpbh_ps(__m256 __S, __mmask8 __U, __m128bh __A) {
508		return _mm256_castsi256_ps((__m256i)_mm256_mask_slli_epi32(
509		(__m256i)__S, (__mmask8)__U, (__m256i)_mm256_cvtepi16_epi32((__m128i)__A),
510		16));
	517	return (__m256)__builtin_ia32_selectps_256(
	518	(__mmask8)__U, (__v8sf)_mm256_cvtpbh_ps(__A), (__v8sf)__S);
511	519	}
512	520
513	521	#undef __DEFAULT_FN_ATTRS128
514	522	#undef __DEFAULT_FN_ATTRS256
	523	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	524	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
515	525
516	526	#endif
517	527	#endif

lib/include/avx512vlbitalgintrin.h+37-53

...	...	@@ -15,101 +15,86 @@
15	15	#define __AVX512VLBITALGINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
18	19	#define __DEFAULT_FN_ATTRS128 \
19	20	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512bitalg,no-evex512"), \
	21	__target__("avx512vl,avx512bitalg"), \
	22	__min_vector_width__(128))) constexpr
	23	#define __DEFAULT_FN_ATTRS256 \
	24	__attribute__((__always_inline__, __nodebug__, \
	25	__target__("avx512vl,avx512bitalg"), \
	26	__min_vector_width__(256))) constexpr
	27	#else
	28	#define __DEFAULT_FN_ATTRS128 \
	29	__attribute__((__always_inline__, __nodebug__, \
	30	__target__("avx512vl,avx512bitalg"), \
21	31	__min_vector_width__(128)))
22	32	#define __DEFAULT_FN_ATTRS256 \
23	33	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512bitalg,no-evex512"), \
	34	__target__("avx512vl,avx512bitalg"), \
25	35	__min_vector_width__(256)))
	36	#endif
26	37
27	38	static __inline__ __m256i __DEFAULT_FN_ATTRS256
28		_mm256_popcnt_epi16(__m256i __A)
29		{
	39	_mm256_popcnt_epi16(__m256i __A) {
30	40	return (__m256i)__builtin_elementwise_popcount((__v16hu)__A);
31	41	}
32	42
33	43	static __inline__ __m256i __DEFAULT_FN_ATTRS256
34		_mm256_mask_popcnt_epi16(__m256i __A, __mmask16 __U, __m256i __B)
35		{
36		return (__m256i) __builtin_ia32_selectw_256((__mmask16) __U,
37		(__v16hi) _mm256_popcnt_epi16(__B),
38		(__v16hi) __A);
	44	_mm256_mask_popcnt_epi16(__m256i __A, __mmask16 __U, __m256i __B) {
	45	return (__m256i)__builtin_ia32_selectw_256(
	46	(__mmask16)__U, (__v16hi)_mm256_popcnt_epi16(__B), (__v16hi)__A);
39	47	}
40	48
41	49	static __inline__ __m256i __DEFAULT_FN_ATTRS256
42		_mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B)
43		{
44		return _mm256_mask_popcnt_epi16((__m256i) _mm256_setzero_si256(),
45		__U,
46		__B);
	50	_mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B) {
	51	return _mm256_mask_popcnt_epi16((__m256i)_mm256_setzero_si256(), __U, __B);
47	52	}
48	53
49		static __inline__ __m128i __DEFAULT_FN_ATTRS128
50		_mm_popcnt_epi16(__m128i __A)
51		{
	54	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi16(__m128i __A) {
52	55	return (__m128i)__builtin_elementwise_popcount((__v8hu)__A);
53	56	}
54	57
55	58	static __inline__ __m128i __DEFAULT_FN_ATTRS128
56		_mm_mask_popcnt_epi16(__m128i __A, __mmask8 __U, __m128i __B)
57		{
58		return (__m128i) __builtin_ia32_selectw_128((__mmask8) __U,
59		(__v8hi) _mm_popcnt_epi16(__B),
60		(__v8hi) __A);
	59	_mm_mask_popcnt_epi16(__m128i __A, __mmask8 __U, __m128i __B) {
	60	return (__m128i)__builtin_ia32_selectw_128(
	61	(__mmask8)__U, (__v8hi)_mm_popcnt_epi16(__B), (__v8hi)__A);
61	62	}
62	63
63	64	static __inline__ __m128i __DEFAULT_FN_ATTRS128
64		_mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B)
65		{
66		return _mm_mask_popcnt_epi16((__m128i) _mm_setzero_si128(),
67		__U,
68		__B);
	65	_mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B) {
	66	return _mm_mask_popcnt_epi16((__m128i)_mm_setzero_si128(), __U, __B);
69	67	}
70	68
71	69	static __inline__ __m256i __DEFAULT_FN_ATTRS256
72		_mm256_popcnt_epi8(__m256i __A)
73		{
	70	_mm256_popcnt_epi8(__m256i __A) {
74	71	return (__m256i)__builtin_elementwise_popcount((__v32qu)__A);
75	72	}
76	73
77	74	static __inline__ __m256i __DEFAULT_FN_ATTRS256
78		_mm256_mask_popcnt_epi8(__m256i __A, __mmask32 __U, __m256i __B)
79		{
80		return (__m256i) __builtin_ia32_selectb_256((__mmask32) __U,
81		(__v32qi) _mm256_popcnt_epi8(__B),
82		(__v32qi) __A);
	75	_mm256_mask_popcnt_epi8(__m256i __A, __mmask32 __U, __m256i __B) {
	76	return (__m256i)__builtin_ia32_selectb_256(
	77	(__mmask32)__U, (__v32qi)_mm256_popcnt_epi8(__B), (__v32qi)__A);
83	78	}
84	79
85	80	static __inline__ __m256i __DEFAULT_FN_ATTRS256
86		_mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B)
87		{
88		return _mm256_mask_popcnt_epi8((__m256i) _mm256_setzero_si256(),
89		__U,
90		__B);
	81	_mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B) {
	82	return _mm256_mask_popcnt_epi8((__m256i)_mm256_setzero_si256(), __U, __B);
91	83	}
92	84
93		static __inline__ __m128i __DEFAULT_FN_ATTRS128
94		_mm_popcnt_epi8(__m128i __A)
95		{
	85	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi8(__m128i __A) {
96	86	return (__m128i)__builtin_elementwise_popcount((__v16qu)__A);
97	87	}
98	88
99	89	static __inline__ __m128i __DEFAULT_FN_ATTRS128
100		_mm_mask_popcnt_epi8(__m128i __A, __mmask16 __U, __m128i __B)
101		{
102		return (__m128i) __builtin_ia32_selectb_128((__mmask16) __U,
103		(__v16qi) _mm_popcnt_epi8(__B),
104		(__v16qi) __A);
	90	_mm_mask_popcnt_epi8(__m128i __A, __mmask16 __U, __m128i __B) {
	91	return (__m128i)__builtin_ia32_selectb_128(
	92	(__mmask16)__U, (__v16qi)_mm_popcnt_epi8(__B), (__v16qi)__A);
105	93	}
106	94
107	95	static __inline__ __m128i __DEFAULT_FN_ATTRS128
108		_mm_maskz_popcnt_epi8(__mmask16 __U, __m128i __B)
109		{
110		return _mm_mask_popcnt_epi8((__m128i) _mm_setzero_si128(),
111		__U,
112		__B);
	96	_mm_maskz_popcnt_epi8(__mmask16 __U, __m128i __B) {
	97	return _mm_mask_popcnt_epi8((__m128i)_mm_setzero_si128(), __U, __B);
113	98	}
114	99
115	100	static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
...	...	@@ -147,5 +132,4 @@ _mm_bitshuffle_epi64_mask(__m128i __A, __m128i __B)
147	132
148	133	#undef __DEFAULT_FN_ATTRS128
149	134	#undef __DEFAULT_FN_ATTRS256
150
151	135	#endif

lib/include/avx512vlbwintrin.h+470-573

...	...	@@ -17,12 +17,18 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS128 \
19	19	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512bw,no-evex512"), \
21		__min_vector_width__(128)))
	20	__target__("avx512vl,avx512bw"), __min_vector_width__(128)))
22	21	#define __DEFAULT_FN_ATTRS256 \
23	22	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512bw,no-evex512"), \
25		__min_vector_width__(256)))
	23	__target__("avx512vl,avx512bw"), __min_vector_width__(256)))
	24
	25	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	26	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	27	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	28	#else
	29	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	30	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	31	#endif
26	32
27	33	/* Integer compare */
28	34
...	...	@@ -306,250 +312,238 @@
306	312	#define _mm256_mask_cmpneq_epu16_mask(k, A, B) \
307	313	_mm256_mask_cmp_epu16_mask((k), (A), (B), _MM_CMPINT_NE)
308	314
309		static __inline__ __m256i __DEFAULT_FN_ATTRS256
310		_mm256_mask_add_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B){
	315	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	316	_mm256_mask_add_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
311	317	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
312	318	(__v32qi)_mm256_add_epi8(__A, __B),
313	319	(__v32qi)__W);
314	320	}
315	321
316		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	322	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
317	323	_mm256_maskz_add_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
318	324	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
319	325	(__v32qi)_mm256_add_epi8(__A, __B),
320	326	(__v32qi)_mm256_setzero_si256());
321	327	}
322	328
323		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	329	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
324	330	_mm256_mask_add_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
325	331	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
326	332	(__v16hi)_mm256_add_epi16(__A, __B),
327	333	(__v16hi)__W);
328	334	}
329	335
330		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	336	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
331	337	_mm256_maskz_add_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
332	338	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
333	339	(__v16hi)_mm256_add_epi16(__A, __B),
334	340	(__v16hi)_mm256_setzero_si256());
335	341	}
336	342
337		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	343	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
338	344	_mm256_mask_sub_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
339	345	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
340	346	(__v32qi)_mm256_sub_epi8(__A, __B),
341	347	(__v32qi)__W);
342	348	}
343	349
344		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	350	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
345	351	_mm256_maskz_sub_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
346	352	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
347	353	(__v32qi)_mm256_sub_epi8(__A, __B),
348	354	(__v32qi)_mm256_setzero_si256());
349	355	}
350	356
351		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	357	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
352	358	_mm256_mask_sub_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
353	359	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
354	360	(__v16hi)_mm256_sub_epi16(__A, __B),
355	361	(__v16hi)__W);
356	362	}
357	363
358		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	364	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
359	365	_mm256_maskz_sub_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
360	366	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
361	367	(__v16hi)_mm256_sub_epi16(__A, __B),
362	368	(__v16hi)_mm256_setzero_si256());
363	369	}
364	370
365		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	371	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
366	372	_mm_mask_add_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
367	373	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
368	374	(__v16qi)_mm_add_epi8(__A, __B),
369	375	(__v16qi)__W);
370	376	}
371	377
372		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	378	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
373	379	_mm_maskz_add_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
374	380	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
375	381	(__v16qi)_mm_add_epi8(__A, __B),
376	382	(__v16qi)_mm_setzero_si128());
377	383	}
378	384
379		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	385	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
380	386	_mm_mask_add_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
381	387	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
382	388	(__v8hi)_mm_add_epi16(__A, __B),
383	389	(__v8hi)__W);
384	390	}
385	391
386		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	392	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
387	393	_mm_maskz_add_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
388	394	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
389	395	(__v8hi)_mm_add_epi16(__A, __B),
390	396	(__v8hi)_mm_setzero_si128());
391	397	}
392	398
393		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	399	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
394	400	_mm_mask_sub_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
395	401	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
396	402	(__v16qi)_mm_sub_epi8(__A, __B),
397	403	(__v16qi)__W);
398	404	}
399	405
400		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	406	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
401	407	_mm_maskz_sub_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
402	408	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
403	409	(__v16qi)_mm_sub_epi8(__A, __B),
404	410	(__v16qi)_mm_setzero_si128());
405	411	}
406	412
407		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	413	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
408	414	_mm_mask_sub_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
409	415	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
410	416	(__v8hi)_mm_sub_epi16(__A, __B),
411	417	(__v8hi)__W);
412	418	}
413	419
414		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	420	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
415	421	_mm_maskz_sub_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
416	422	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
417	423	(__v8hi)_mm_sub_epi16(__A, __B),
418	424	(__v8hi)_mm_setzero_si128());
419	425	}
420	426
421		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	427	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
422	428	_mm256_mask_mullo_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
423	429	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
424	430	(__v16hi)_mm256_mullo_epi16(__A, __B),
425	431	(__v16hi)__W);
426	432	}
427	433
428		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	434	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
429	435	_mm256_maskz_mullo_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
430	436	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
431	437	(__v16hi)_mm256_mullo_epi16(__A, __B),
432	438	(__v16hi)_mm256_setzero_si256());
433	439	}
434	440
435		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	441	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
436	442	_mm_mask_mullo_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
437	443	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
438	444	(__v8hi)_mm_mullo_epi16(__A, __B),
439	445	(__v8hi)__W);
440	446	}
441	447
442		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	448	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
443	449	_mm_maskz_mullo_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
444	450	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
445	451	(__v8hi)_mm_mullo_epi16(__A, __B),
446	452	(__v8hi)_mm_setzero_si128());
447	453	}
448	454
449		static __inline__ __m128i __DEFAULT_FN_ATTRS128
450		_mm_mask_blend_epi8 (__mmask16 __U, __m128i __A, __m128i __W)
451		{
	455	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	456	_mm_mask_blend_epi8(__mmask16 __U, __m128i __A, __m128i __W) {
452	457	return (__m128i) __builtin_ia32_selectb_128 ((__mmask16) __U,
453	458	(__v16qi) __W,
454	459	(__v16qi) __A);
455	460	}
456	461
457		static __inline__ __m256i __DEFAULT_FN_ATTRS256
458		_mm256_mask_blend_epi8 (__mmask32 __U, __m256i __A, __m256i __W)
459		{
	462	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	463	_mm256_mask_blend_epi8(__mmask32 __U, __m256i __A, __m256i __W) {
460	464	return (__m256i) __builtin_ia32_selectb_256 ((__mmask32) __U,
461	465	(__v32qi) __W,
462	466	(__v32qi) __A);
463	467	}
464	468
465		static __inline__ __m128i __DEFAULT_FN_ATTRS128
466		_mm_mask_blend_epi16 (__mmask8 __U, __m128i __A, __m128i __W)
467		{
	469	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	470	_mm_mask_blend_epi16(__mmask8 __U, __m128i __A, __m128i __W) {
468	471	return (__m128i) __builtin_ia32_selectw_128 ((__mmask8) __U,
469	472	(__v8hi) __W,
470	473	(__v8hi) __A);
471	474	}
472	475
473		static __inline__ __m256i __DEFAULT_FN_ATTRS256
474		_mm256_mask_blend_epi16 (__mmask16 __U, __m256i __A, __m256i __W)
475		{
	476	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	477	_mm256_mask_blend_epi16(__mmask16 __U, __m256i __A, __m256i __W) {
476	478	return (__m256i) __builtin_ia32_selectw_256 ((__mmask16) __U,
477	479	(__v16hi) __W,
478	480	(__v16hi) __A);
479	481	}
480	482
481		static __inline__ __m128i __DEFAULT_FN_ATTRS128
482		_mm_mask_abs_epi8(__m128i __W, __mmask16 __U, __m128i __A)
483		{
	483	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	484	_mm_mask_abs_epi8(__m128i __W, __mmask16 __U, __m128i __A) {
484	485	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
485	486	(__v16qi)_mm_abs_epi8(__A),
486	487	(__v16qi)__W);
487	488	}
488	489
489		static __inline__ __m128i __DEFAULT_FN_ATTRS128
490		_mm_maskz_abs_epi8(__mmask16 __U, __m128i __A)
491		{
	490	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	491	_mm_maskz_abs_epi8(__mmask16 __U, __m128i __A) {
492	492	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
493	493	(__v16qi)_mm_abs_epi8(__A),
494	494	(__v16qi)_mm_setzero_si128());
495	495	}
496	496
497		static __inline__ __m256i __DEFAULT_FN_ATTRS256
498		_mm256_mask_abs_epi8(__m256i __W, __mmask32 __U, __m256i __A)
499		{
	497	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	498	_mm256_mask_abs_epi8(__m256i __W, __mmask32 __U, __m256i __A) {
500	499	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
501	500	(__v32qi)_mm256_abs_epi8(__A),
502	501	(__v32qi)__W);
503	502	}
504	503
505		static __inline__ __m256i __DEFAULT_FN_ATTRS256
506		_mm256_maskz_abs_epi8 (__mmask32 __U, __m256i __A)
507		{
	504	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	505	_mm256_maskz_abs_epi8(__mmask32 __U, __m256i __A) {
508	506	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
509	507	(__v32qi)_mm256_abs_epi8(__A),
510	508	(__v32qi)_mm256_setzero_si256());
511	509	}
512	510
513		static __inline__ __m128i __DEFAULT_FN_ATTRS128
514		_mm_mask_abs_epi16(__m128i __W, __mmask8 __U, __m128i __A)
515		{
	511	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	512	_mm_mask_abs_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
516	513	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
517	514	(__v8hi)_mm_abs_epi16(__A),
518	515	(__v8hi)__W);
519	516	}
520	517
521		static __inline__ __m128i __DEFAULT_FN_ATTRS128
522		_mm_maskz_abs_epi16(__mmask8 __U, __m128i __A)
523		{
	518	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	519	_mm_maskz_abs_epi16(__mmask8 __U, __m128i __A) {
524	520	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
525	521	(__v8hi)_mm_abs_epi16(__A),
526	522	(__v8hi)_mm_setzero_si128());
527	523	}
528	524
529		static __inline__ __m256i __DEFAULT_FN_ATTRS256
530		_mm256_mask_abs_epi16(__m256i __W, __mmask16 __U, __m256i __A)
531		{
	525	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	526	_mm256_mask_abs_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
532	527	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
533	528	(__v16hi)_mm256_abs_epi16(__A),
534	529	(__v16hi)__W);
535	530	}
536	531
537		static __inline__ __m256i __DEFAULT_FN_ATTRS256
538		_mm256_maskz_abs_epi16(__mmask16 __U, __m256i __A)
539		{
	532	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	533	_mm256_maskz_abs_epi16(__mmask16 __U, __m256i __A) {
540	534	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
541	535	(__v16hi)_mm256_abs_epi16(__A),
542	536	(__v16hi)_mm256_setzero_si256());
543	537	}
544	538
545		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	539	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
546	540	_mm_maskz_packs_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
547	541	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
548	542	(__v8hi)_mm_packs_epi32(__A, __B),
549	543	(__v8hi)_mm_setzero_si128());
550	544	}
551	545
552		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	546	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
553	547	_mm_mask_packs_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
554	548	{
555	549	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
...	...	@@ -557,7 +551,7 @@ _mm_mask_packs_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
557	551	(__v8hi)__W);
558	552	}
559	553
560		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	554	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
561	555	_mm256_maskz_packs_epi32(__mmask16 __M, __m256i __A, __m256i __B)
562	556	{
563	557	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
...	...	@@ -565,7 +559,7 @@ _mm256_maskz_packs_epi32(__mmask16 __M, __m256i __A, __m256i __B)
565	559	(__v16hi)_mm256_setzero_si256());
566	560	}
567	561
568		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	562	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
569	563	_mm256_mask_packs_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
570	564	{
571	565	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
...	...	@@ -573,7 +567,7 @@ _mm256_mask_packs_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
573	567	(__v16hi)__W);
574	568	}
575	569
576		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	570	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
577	571	_mm_maskz_packs_epi16(__mmask16 __M, __m128i __A, __m128i __B)
578	572	{
579	573	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
...	...	@@ -581,7 +575,7 @@ _mm_maskz_packs_epi16(__mmask16 __M, __m128i __A, __m128i __B)
581	575	(__v16qi)_mm_setzero_si128());
582	576	}
583	577
584		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	578	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
585	579	_mm_mask_packs_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
586	580	{
587	581	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
...	...	@@ -589,7 +583,7 @@ _mm_mask_packs_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
589	583	(__v16qi)__W);
590	584	}
591	585
592		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	586	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
593	587	_mm256_maskz_packs_epi16(__mmask32 __M, __m256i __A, __m256i __B)
594	588	{
595	589	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
...	...	@@ -597,7 +591,7 @@ _mm256_maskz_packs_epi16(__mmask32 __M, __m256i __A, __m256i __B)
597	591	(__v32qi)_mm256_setzero_si256());
598	592	}
599	593
600		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	594	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
601	595	_mm256_mask_packs_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
602	596	{
603	597	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
...	...	@@ -605,7 +599,7 @@ _mm256_mask_packs_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
605	599	(__v32qi)__W);
606	600	}
607	601
608		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	602	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
609	603	_mm_maskz_packus_epi32(__mmask8 __M, __m128i __A, __m128i __B)
610	604	{
611	605	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
...	...	@@ -613,7 +607,7 @@ _mm_maskz_packus_epi32(__mmask8 __M, __m128i __A, __m128i __B)
613	607	(__v8hi)_mm_setzero_si128());
614	608	}
615	609
616		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	610	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
617	611	_mm_mask_packus_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
618	612	{
619	613	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
...	...	@@ -621,7 +615,7 @@ _mm_mask_packus_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
621	615	(__v8hi)__W);
622	616	}
623	617
624		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	618	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
625	619	_mm256_maskz_packus_epi32(__mmask16 __M, __m256i __A, __m256i __B)
626	620	{
627	621	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
...	...	@@ -629,7 +623,7 @@ _mm256_maskz_packus_epi32(__mmask16 __M, __m256i __A, __m256i __B)
629	623	(__v16hi)_mm256_setzero_si256());
630	624	}
631	625
632		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	626	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
633	627	_mm256_mask_packus_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
634	628	{
635	629	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
...	...	@@ -637,7 +631,7 @@ _mm256_mask_packus_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
637	631	(__v16hi)__W);
638	632	}
639	633
640		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	634	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
641	635	_mm_maskz_packus_epi16(__mmask16 __M, __m128i __A, __m128i __B)
642	636	{
643	637	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
...	...	@@ -645,7 +639,7 @@ _mm_maskz_packus_epi16(__mmask16 __M, __m128i __A, __m128i __B)
645	639	(__v16qi)_mm_setzero_si128());
646	640	}
647	641
648		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	642	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
649	643	_mm_mask_packus_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
650	644	{
651	645	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
...	...	@@ -653,7 +647,7 @@ _mm_mask_packus_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
653	647	(__v16qi)__W);
654	648	}
655	649
656		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	650	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
657	651	_mm256_maskz_packus_epi16(__mmask32 __M, __m256i __A, __m256i __B)
658	652	{
659	653	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
...	...	@@ -661,7 +655,7 @@ _mm256_maskz_packus_epi16(__mmask32 __M, __m256i __A, __m256i __B)
661	655	(__v32qi)_mm256_setzero_si256());
662	656	}
663	657
664		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	658	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
665	659	_mm256_mask_packus_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
666	660	{
667	661	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
...	...	@@ -669,7 +663,7 @@ _mm256_mask_packus_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
669	663	(__v32qi)__W);
670	664	}
671	665
672		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	666	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
673	667	_mm_mask_adds_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
674	668	{
675	669	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -677,7 +671,7 @@ _mm_mask_adds_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
677	671	(__v16qi)__W);
678	672	}
679	673
680		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	674	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
681	675	_mm_maskz_adds_epi8(__mmask16 __U, __m128i __A, __m128i __B)
682	676	{
683	677	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -685,7 +679,7 @@ _mm_maskz_adds_epi8(__mmask16 __U, __m128i __A, __m128i __B)
685	679	(__v16qi)_mm_setzero_si128());
686	680	}
687	681
688		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	682	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
689	683	_mm256_mask_adds_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
690	684	{
691	685	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -693,7 +687,7 @@ _mm256_mask_adds_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
693	687	(__v32qi)__W);
694	688	}
695	689
696		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	690	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
697	691	_mm256_maskz_adds_epi8(__mmask32 __U, __m256i __A, __m256i __B)
698	692	{
699	693	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -701,7 +695,7 @@ _mm256_maskz_adds_epi8(__mmask32 __U, __m256i __A, __m256i __B)
701	695	(__v32qi)_mm256_setzero_si256());
702	696	}
703	697
704		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	698	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
705	699	_mm_mask_adds_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
706	700	{
707	701	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -709,7 +703,7 @@ _mm_mask_adds_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
709	703	(__v8hi)__W);
710	704	}
711	705
712		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	706	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
713	707	_mm_maskz_adds_epi16(__mmask8 __U, __m128i __A, __m128i __B)
714	708	{
715	709	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -717,7 +711,7 @@ _mm_maskz_adds_epi16(__mmask8 __U, __m128i __A, __m128i __B)
717	711	(__v8hi)_mm_setzero_si128());
718	712	}
719	713
720		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	714	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
721	715	_mm256_mask_adds_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
722	716	{
723	717	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -725,7 +719,7 @@ _mm256_mask_adds_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
725	719	(__v16hi)__W);
726	720	}
727	721
728		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	722	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
729	723	_mm256_maskz_adds_epi16(__mmask16 __U, __m256i __A, __m256i __B)
730	724	{
731	725	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -733,7 +727,7 @@ _mm256_maskz_adds_epi16(__mmask16 __U, __m256i __A, __m256i __B)
733	727	(__v16hi)_mm256_setzero_si256());
734	728	}
735	729
736		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	730	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
737	731	_mm_mask_adds_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
738	732	{
739	733	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -741,7 +735,7 @@ _mm_mask_adds_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
741	735	(__v16qi)__W);
742	736	}
743	737
744		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	738	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
745	739	_mm_maskz_adds_epu8(__mmask16 __U, __m128i __A, __m128i __B)
746	740	{
747	741	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -749,7 +743,7 @@ _mm_maskz_adds_epu8(__mmask16 __U, __m128i __A, __m128i __B)
749	743	(__v16qi)_mm_setzero_si128());
750	744	}
751	745
752		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	746	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
753	747	_mm256_mask_adds_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
754	748	{
755	749	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -757,7 +751,7 @@ _mm256_mask_adds_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
757	751	(__v32qi)__W);
758	752	}
759	753
760		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	754	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
761	755	_mm256_maskz_adds_epu8(__mmask32 __U, __m256i __A, __m256i __B)
762	756	{
763	757	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -765,7 +759,7 @@ _mm256_maskz_adds_epu8(__mmask32 __U, __m256i __A, __m256i __B)
765	759	(__v32qi)_mm256_setzero_si256());
766	760	}
767	761
768		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	762	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
769	763	_mm_mask_adds_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
770	764	{
771	765	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -773,7 +767,7 @@ _mm_mask_adds_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
773	767	(__v8hi)__W);
774	768	}
775	769
776		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	770	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
777	771	_mm_maskz_adds_epu16(__mmask8 __U, __m128i __A, __m128i __B)
778	772	{
779	773	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -781,7 +775,7 @@ _mm_maskz_adds_epu16(__mmask8 __U, __m128i __A, __m128i __B)
781	775	(__v8hi)_mm_setzero_si128());
782	776	}
783	777
784		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	778	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
785	779	_mm256_mask_adds_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
786	780	{
787	781	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -789,7 +783,7 @@ _mm256_mask_adds_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
789	783	(__v16hi)__W);
790	784	}
791	785
792		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	786	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
793	787	_mm256_maskz_adds_epu16(__mmask16 __U, __m256i __A, __m256i __B)
794	788	{
795	789	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -797,359 +791,311 @@ _mm256_maskz_adds_epu16(__mmask16 __U, __m256i __A, __m256i __B)
797	791	(__v16hi)_mm256_setzero_si256());
798	792	}
799	793
800		static __inline__ __m128i __DEFAULT_FN_ATTRS128
801		_mm_mask_avg_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
802		{
803		return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
804		(__v16qi)_mm_avg_epu8(__A, __B),
805		(__v16qi)__W);
	794	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	795	_mm_mask_avg_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
	796	return (__m128i)__builtin_ia32_selectb_128(
	797	(__mmask16)__U, (__v16qi)_mm_avg_epu8(__A, __B), (__v16qi)__W);
806	798	}
807	799
808		static __inline__ __m128i __DEFAULT_FN_ATTRS128
809		_mm_maskz_avg_epu8(__mmask16 __U, __m128i __A, __m128i __B)
810		{
	800	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	801	_mm_maskz_avg_epu8(__mmask16 __U, __m128i __A, __m128i __B) {
811	802	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
812	803	(__v16qi)_mm_avg_epu8(__A, __B),
813	804	(__v16qi)_mm_setzero_si128());
814	805	}
815	806
816		static __inline__ __m256i __DEFAULT_FN_ATTRS256
817		_mm256_mask_avg_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
818		{
819		return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
820		(__v32qi)_mm256_avg_epu8(__A, __B),
821		(__v32qi)__W);
	807	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	808	_mm256_mask_avg_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
	809	return (__m256i)__builtin_ia32_selectb_256(
	810	(__mmask32)__U, (__v32qi)_mm256_avg_epu8(__A, __B), (__v32qi)__W);
822	811	}
823	812
824		static __inline__ __m256i __DEFAULT_FN_ATTRS256
825		_mm256_maskz_avg_epu8(__mmask32 __U, __m256i __A, __m256i __B)
826		{
	813	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	814	_mm256_maskz_avg_epu8(__mmask32 __U, __m256i __A, __m256i __B) {
827	815	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
828	816	(__v32qi)_mm256_avg_epu8(__A, __B),
829	817	(__v32qi)_mm256_setzero_si256());
830	818	}
831	819
832		static __inline__ __m128i __DEFAULT_FN_ATTRS128
833		_mm_mask_avg_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
834		{
835		return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
836		(__v8hi)_mm_avg_epu16(__A, __B),
837		(__v8hi)__W);
	820	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	821	_mm_mask_avg_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
	822	return (__m128i)__builtin_ia32_selectw_128(
	823	(__mmask8)__U, (__v8hi)_mm_avg_epu16(__A, __B), (__v8hi)__W);
838	824	}
839	825
840		static __inline__ __m128i __DEFAULT_FN_ATTRS128
841		_mm_maskz_avg_epu16(__mmask8 __U, __m128i __A, __m128i __B)
842		{
	826	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	827	_mm_maskz_avg_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
843	828	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
844	829	(__v8hi)_mm_avg_epu16(__A, __B),
845	830	(__v8hi)_mm_setzero_si128());
846	831	}
847	832
848		static __inline__ __m256i __DEFAULT_FN_ATTRS256
849		_mm256_mask_avg_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
850		{
851		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
852		(__v16hi)_mm256_avg_epu16(__A, __B),
853		(__v16hi)__W);
	833	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	834	_mm256_mask_avg_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
	835	return (__m256i)__builtin_ia32_selectw_256(
	836	(__mmask16)__U, (__v16hi)_mm256_avg_epu16(__A, __B), (__v16hi)__W);
854	837	}
855	838
856		static __inline__ __m256i __DEFAULT_FN_ATTRS256
857		_mm256_maskz_avg_epu16(__mmask16 __U, __m256i __A, __m256i __B)
858		{
859		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
860		(__v16hi)_mm256_avg_epu16(__A, __B),
861		(__v16hi)_mm256_setzero_si256());
	839	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	840	_mm256_maskz_avg_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
	841	return (__m256i)__builtin_ia32_selectw_256(
	842	(__mmask16)__U, (__v16hi)_mm256_avg_epu16(__A, __B),
	843	(__v16hi)_mm256_setzero_si256());
862	844	}
863	845
864		static __inline__ __m128i __DEFAULT_FN_ATTRS128
865		_mm_maskz_max_epi8(__mmask16 __M, __m128i __A, __m128i __B)
866		{
	846	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	847	_mm_maskz_max_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
867	848	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
868	849	(__v16qi)_mm_max_epi8(__A, __B),
869	850	(__v16qi)_mm_setzero_si128());
870	851	}
871	852
872		static __inline__ __m128i __DEFAULT_FN_ATTRS128
873		_mm_mask_max_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
874		{
	853	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	854	_mm_mask_max_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
875	855	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
876	856	(__v16qi)_mm_max_epi8(__A, __B),
877	857	(__v16qi)__W);
878	858	}
879	859
880		static __inline__ __m256i __DEFAULT_FN_ATTRS256
881		_mm256_maskz_max_epi8(__mmask32 __M, __m256i __A, __m256i __B)
882		{
	860	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	861	_mm256_maskz_max_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
883	862	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
884	863	(__v32qi)_mm256_max_epi8(__A, __B),
885	864	(__v32qi)_mm256_setzero_si256());
886	865	}
887	866
888		static __inline__ __m256i __DEFAULT_FN_ATTRS256
889		_mm256_mask_max_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
890		{
	867	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	868	_mm256_mask_max_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
891	869	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
892	870	(__v32qi)_mm256_max_epi8(__A, __B),
893	871	(__v32qi)__W);
894	872	}
895	873
896		static __inline__ __m128i __DEFAULT_FN_ATTRS128
897		_mm_maskz_max_epi16(__mmask8 __M, __m128i __A, __m128i __B)
898		{
	874	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	875	_mm_maskz_max_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
899	876	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
900	877	(__v8hi)_mm_max_epi16(__A, __B),
901	878	(__v8hi)_mm_setzero_si128());
902	879	}
903	880
904		static __inline__ __m128i __DEFAULT_FN_ATTRS128
905		_mm_mask_max_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
906		{
	881	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	882	_mm_mask_max_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
907	883	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
908	884	(__v8hi)_mm_max_epi16(__A, __B),
909	885	(__v8hi)__W);
910	886	}
911	887
912		static __inline__ __m256i __DEFAULT_FN_ATTRS256
913		_mm256_maskz_max_epi16(__mmask16 __M, __m256i __A, __m256i __B)
914		{
	888	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	889	_mm256_maskz_max_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
915	890	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
916	891	(__v16hi)_mm256_max_epi16(__A, __B),
917	892	(__v16hi)_mm256_setzero_si256());
918	893	}
919	894
920		static __inline__ __m256i __DEFAULT_FN_ATTRS256
921		_mm256_mask_max_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
922		{
	895	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	896	_mm256_mask_max_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
923	897	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
924	898	(__v16hi)_mm256_max_epi16(__A, __B),
925	899	(__v16hi)__W);
926	900	}
927	901
928		static __inline__ __m128i __DEFAULT_FN_ATTRS128
929		_mm_maskz_max_epu8(__mmask16 __M, __m128i __A, __m128i __B)
930		{
	902	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	903	_mm_maskz_max_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
931	904	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
932	905	(__v16qi)_mm_max_epu8(__A, __B),
933	906	(__v16qi)_mm_setzero_si128());
934	907	}
935	908
936		static __inline__ __m128i __DEFAULT_FN_ATTRS128
937		_mm_mask_max_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
938		{
	909	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	910	_mm_mask_max_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
939	911	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
940	912	(__v16qi)_mm_max_epu8(__A, __B),
941	913	(__v16qi)__W);
942	914	}
943	915
944		static __inline__ __m256i __DEFAULT_FN_ATTRS256
945		_mm256_maskz_max_epu8 (__mmask32 __M, __m256i __A, __m256i __B)
946		{
	916	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	917	_mm256_maskz_max_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
947	918	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
948	919	(__v32qi)_mm256_max_epu8(__A, __B),
949	920	(__v32qi)_mm256_setzero_si256());
950	921	}
951	922
952		static __inline__ __m256i __DEFAULT_FN_ATTRS256
953		_mm256_mask_max_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
954		{
	923	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	924	_mm256_mask_max_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
955	925	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
956	926	(__v32qi)_mm256_max_epu8(__A, __B),
957	927	(__v32qi)__W);
958	928	}
959	929
960		static __inline__ __m128i __DEFAULT_FN_ATTRS128
961		_mm_maskz_max_epu16(__mmask8 __M, __m128i __A, __m128i __B)
962		{
	930	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	931	_mm_maskz_max_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
963	932	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
964	933	(__v8hi)_mm_max_epu16(__A, __B),
965	934	(__v8hi)_mm_setzero_si128());
966	935	}
967	936
968		static __inline__ __m128i __DEFAULT_FN_ATTRS128
969		_mm_mask_max_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
970		{
	937	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	938	_mm_mask_max_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
971	939	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
972	940	(__v8hi)_mm_max_epu16(__A, __B),
973	941	(__v8hi)__W);
974	942	}
975	943
976		static __inline__ __m256i __DEFAULT_FN_ATTRS256
977		_mm256_maskz_max_epu16(__mmask16 __M, __m256i __A, __m256i __B)
978		{
	944	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	945	_mm256_maskz_max_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
979	946	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
980	947	(__v16hi)_mm256_max_epu16(__A, __B),
981	948	(__v16hi)_mm256_setzero_si256());
982	949	}
983	950
984		static __inline__ __m256i __DEFAULT_FN_ATTRS256
985		_mm256_mask_max_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
986		{
	951	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	952	_mm256_mask_max_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
987	953	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
988	954	(__v16hi)_mm256_max_epu16(__A, __B),
989	955	(__v16hi)__W);
990	956	}
991	957
992		static __inline__ __m128i __DEFAULT_FN_ATTRS128
993		_mm_maskz_min_epi8(__mmask16 __M, __m128i __A, __m128i __B)
994		{
	958	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	959	_mm_maskz_min_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
995	960	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
996	961	(__v16qi)_mm_min_epi8(__A, __B),
997	962	(__v16qi)_mm_setzero_si128());
998	963	}
999	964
1000		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1001		_mm_mask_min_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
1002		{
	965	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	966	_mm_mask_min_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
1003	967	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
1004	968	(__v16qi)_mm_min_epi8(__A, __B),
1005	969	(__v16qi)__W);
1006	970	}
1007	971
1008		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1009		_mm256_maskz_min_epi8(__mmask32 __M, __m256i __A, __m256i __B)
1010		{
	972	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	973	_mm256_maskz_min_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
1011	974	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
1012	975	(__v32qi)_mm256_min_epi8(__A, __B),
1013	976	(__v32qi)_mm256_setzero_si256());
1014	977	}
1015	978
1016		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1017		_mm256_mask_min_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
1018		{
	979	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	980	_mm256_mask_min_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
1019	981	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
1020	982	(__v32qi)_mm256_min_epi8(__A, __B),
1021	983	(__v32qi)__W);
1022	984	}
1023	985
1024		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1025		_mm_maskz_min_epi16(__mmask8 __M, __m128i __A, __m128i __B)
1026		{
	986	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	987	_mm_maskz_min_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
1027	988	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
1028	989	(__v8hi)_mm_min_epi16(__A, __B),
1029	990	(__v8hi)_mm_setzero_si128());
1030	991	}
1031	992
1032		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1033		_mm_mask_min_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
1034		{
	993	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	994	_mm_mask_min_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
1035	995	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
1036	996	(__v8hi)_mm_min_epi16(__A, __B),
1037	997	(__v8hi)__W);
1038	998	}
1039	999
1040		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1041		_mm256_maskz_min_epi16(__mmask16 __M, __m256i __A, __m256i __B)
1042		{
	1000	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1001	_mm256_maskz_min_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
1043	1002	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
1044	1003	(__v16hi)_mm256_min_epi16(__A, __B),
1045	1004	(__v16hi)_mm256_setzero_si256());
1046	1005	}
1047	1006
1048		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1049		_mm256_mask_min_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
1050		{
	1007	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1008	_mm256_mask_min_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
1051	1009	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
1052	1010	(__v16hi)_mm256_min_epi16(__A, __B),
1053	1011	(__v16hi)__W);
1054	1012	}
1055	1013
1056		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1057		_mm_maskz_min_epu8(__mmask16 __M, __m128i __A, __m128i __B)
1058		{
	1014	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1015	_mm_maskz_min_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
1059	1016	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
1060	1017	(__v16qi)_mm_min_epu8(__A, __B),
1061	1018	(__v16qi)_mm_setzero_si128());
1062	1019	}
1063	1020
1064		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1065		_mm_mask_min_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B)
1066		{
	1021	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1022	_mm_mask_min_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
1067	1023	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
1068	1024	(__v16qi)_mm_min_epu8(__A, __B),
1069	1025	(__v16qi)__W);
1070	1026	}
1071	1027
1072		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1073		_mm256_maskz_min_epu8 (__mmask32 __M, __m256i __A, __m256i __B)
1074		{
	1028	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1029	_mm256_maskz_min_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
1075	1030	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
1076	1031	(__v32qi)_mm256_min_epu8(__A, __B),
1077	1032	(__v32qi)_mm256_setzero_si256());
1078	1033	}
1079	1034
1080		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1081		_mm256_mask_min_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B)
1082		{
	1035	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1036	_mm256_mask_min_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
1083	1037	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__M,
1084	1038	(__v32qi)_mm256_min_epu8(__A, __B),
1085	1039	(__v32qi)__W);
1086	1040	}
1087	1041
1088		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1089		_mm_maskz_min_epu16(__mmask8 __M, __m128i __A, __m128i __B)
1090		{
	1042	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1043	_mm_maskz_min_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
1091	1044	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
1092	1045	(__v8hi)_mm_min_epu16(__A, __B),
1093	1046	(__v8hi)_mm_setzero_si128());
1094	1047	}
1095	1048
1096		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1097		_mm_mask_min_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
1098		{
	1049	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1050	_mm_mask_min_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
1099	1051	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
1100	1052	(__v8hi)_mm_min_epu16(__A, __B),
1101	1053	(__v8hi)__W);
1102	1054	}
1103	1055
1104		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1105		_mm256_maskz_min_epu16(__mmask16 __M, __m256i __A, __m256i __B)
1106		{
	1056	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1057	_mm256_maskz_min_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
1107	1058	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
1108	1059	(__v16hi)_mm256_min_epu16(__A, __B),
1109	1060	(__v16hi)_mm256_setzero_si256());
1110	1061	}
1111	1062
1112		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1113		_mm256_mask_min_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B)
1114		{
	1063	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1064	_mm256_mask_min_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
1115	1065	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
1116	1066	(__v16hi)_mm256_min_epu16(__A, __B),
1117	1067	(__v16hi)__W);
1118	1068	}
1119	1069
1120		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1121		_mm_mask_shuffle_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
1122		{
	1070	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1071	_mm_mask_shuffle_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
1123	1072	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1124	1073	(__v16qi)_mm_shuffle_epi8(__A, __B),
1125	1074	(__v16qi)__W);
1126	1075	}
1127	1076
1128		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1129		_mm_maskz_shuffle_epi8(__mmask16 __U, __m128i __A, __m128i __B)
1130		{
	1077	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1078	_mm_maskz_shuffle_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
1131	1079	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1132	1080	(__v16qi)_mm_shuffle_epi8(__A, __B),
1133	1081	(__v16qi)_mm_setzero_si128());
1134	1082	}
1135	1083
1136		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1137		_mm256_mask_shuffle_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
1138		{
	1084	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1085	_mm256_mask_shuffle_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
1139	1086	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1140	1087	(__v32qi)_mm256_shuffle_epi8(__A, __B),
1141	1088	(__v32qi)__W);
1142	1089	}
1143	1090
1144		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1145		_mm256_maskz_shuffle_epi8(__mmask32 __U, __m256i __A, __m256i __B)
1146		{
	1091	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1092	_mm256_maskz_shuffle_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
1147	1093	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1148	1094	(__v32qi)_mm256_shuffle_epi8(__A, __B),
1149	1095	(__v32qi)_mm256_setzero_si256());
1150	1096	}
1151	1097
1152		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1098	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1153	1099	_mm_mask_subs_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
1154	1100	{
1155	1101	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -1157,7 +1103,7 @@ _mm_mask_subs_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
1157	1103	(__v16qi)__W);
1158	1104	}
1159	1105
1160		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1106	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1161	1107	_mm_maskz_subs_epi8(__mmask16 __U, __m128i __A, __m128i __B)
1162	1108	{
1163	1109	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -1165,7 +1111,7 @@ _mm_maskz_subs_epi8(__mmask16 __U, __m128i __A, __m128i __B)
1165	1111	(__v16qi)_mm_setzero_si128());
1166	1112	}
1167	1113
1168		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1114	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1169	1115	_mm256_mask_subs_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
1170	1116	{
1171	1117	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -1173,7 +1119,7 @@ _mm256_mask_subs_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
1173	1119	(__v32qi)__W);
1174	1120	}
1175	1121
1176		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1122	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1177	1123	_mm256_maskz_subs_epi8(__mmask32 __U, __m256i __A, __m256i __B)
1178	1124	{
1179	1125	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -1181,7 +1127,7 @@ _mm256_maskz_subs_epi8(__mmask32 __U, __m256i __A, __m256i __B)
1181	1127	(__v32qi)_mm256_setzero_si256());
1182	1128	}
1183	1129
1184		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1130	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1185	1131	_mm_mask_subs_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1186	1132	{
1187	1133	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1189,7 +1135,7 @@ _mm_mask_subs_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1189	1135	(__v8hi)__W);
1190	1136	}
1191	1137
1192		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1138	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1193	1139	_mm_maskz_subs_epi16(__mmask8 __U, __m128i __A, __m128i __B)
1194	1140	{
1195	1141	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1197,7 +1143,7 @@ _mm_maskz_subs_epi16(__mmask8 __U, __m128i __A, __m128i __B)
1197	1143	(__v8hi)_mm_setzero_si128());
1198	1144	}
1199	1145
1200		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1146	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1201	1147	_mm256_mask_subs_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1202	1148	{
1203	1149	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1205,7 +1151,7 @@ _mm256_mask_subs_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1205	1151	(__v16hi)__W);
1206	1152	}
1207	1153
1208		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1154	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1209	1155	_mm256_maskz_subs_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1210	1156	{
1211	1157	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1213,7 +1159,7 @@ _mm256_maskz_subs_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1213	1159	(__v16hi)_mm256_setzero_si256());
1214	1160	}
1215	1161
1216		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1162	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1217	1163	_mm_mask_subs_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
1218	1164	{
1219	1165	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -1221,7 +1167,7 @@ _mm_mask_subs_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B)
1221	1167	(__v16qi)__W);
1222	1168	}
1223	1169
1224		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1170	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1225	1171	_mm_maskz_subs_epu8(__mmask16 __U, __m128i __A, __m128i __B)
1226	1172	{
1227	1173	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
...	...	@@ -1229,7 +1175,7 @@ _mm_maskz_subs_epu8(__mmask16 __U, __m128i __A, __m128i __B)
1229	1175	(__v16qi)_mm_setzero_si128());
1230	1176	}
1231	1177
1232		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1178	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1233	1179	_mm256_mask_subs_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
1234	1180	{
1235	1181	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -1237,7 +1183,7 @@ _mm256_mask_subs_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B)
1237	1183	(__v32qi)__W);
1238	1184	}
1239	1185
1240		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1186	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1241	1187	_mm256_maskz_subs_epu8(__mmask32 __U, __m256i __A, __m256i __B)
1242	1188	{
1243	1189	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
...	...	@@ -1245,7 +1191,7 @@ _mm256_maskz_subs_epu8(__mmask32 __U, __m256i __A, __m256i __B)
1245	1191	(__v32qi)_mm256_setzero_si256());
1246	1192	}
1247	1193
1248		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1194	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1249	1195	_mm_mask_subs_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1250	1196	{
1251	1197	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1253,7 +1199,7 @@ _mm_mask_subs_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1253	1199	(__v8hi)__W);
1254	1200	}
1255	1201
1256		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1202	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1257	1203	_mm_maskz_subs_epu16(__mmask8 __U, __m128i __A, __m128i __B)
1258	1204	{
1259	1205	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1261,7 +1207,7 @@ _mm_maskz_subs_epu16(__mmask8 __U, __m128i __A, __m128i __B)
1261	1207	(__v8hi)_mm_setzero_si128());
1262	1208	}
1263	1209
1264		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1210	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1265	1211	_mm256_mask_subs_epu16(__m256i __W, __mmask16 __U, __m256i __A,
1266	1212	__m256i __B) {
1267	1213	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1269,7 +1215,7 @@ _mm256_mask_subs_epu16(__m256i __W, __mmask16 __U, __m256i __A,
1269	1215	(__v16hi)__W);
1270	1216	}
1271	1217
1272		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1218	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1273	1219	_mm256_maskz_subs_epu16(__mmask16 __U, __m256i __A, __m256i __B)
1274	1220	{
1275	1221	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1277,89 +1223,81 @@ _mm256_maskz_subs_epu16(__mmask16 __U, __m256i __A, __m256i __B)
1277	1223	(__v16hi)_mm256_setzero_si256());
1278	1224	}
1279	1225
1280		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1281		_mm_permutex2var_epi16(__m128i __A, __m128i __I, __m128i __B)
1282		{
	1226	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1227	_mm_permutex2var_epi16(__m128i __A, __m128i __I, __m128i __B) {
1283	1228	return (__m128i)__builtin_ia32_vpermi2varhi128((__v8hi)__A, (__v8hi)__I,
1284	1229	(__v8hi) __B);
1285	1230	}
1286	1231
1287		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1232	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1288	1233	_mm_mask_permutex2var_epi16(__m128i __A, __mmask8 __U, __m128i __I,
1289		__m128i __B)
1290		{
	1234	__m128i __B) {
1291	1235	return (__m128i)__builtin_ia32_selectw_128(__U,
1292	1236	(__v8hi)_mm_permutex2var_epi16(__A, __I, __B),
1293	1237	(__v8hi)__A);
1294	1238	}
1295	1239
1296		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1240	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1297	1241	_mm_mask2_permutex2var_epi16(__m128i __A, __m128i __I, __mmask8 __U,
1298		__m128i __B)
1299		{
	1242	__m128i __B) {
1300	1243	return (__m128i)__builtin_ia32_selectw_128(__U,
1301	1244	(__v8hi)_mm_permutex2var_epi16(__A, __I, __B),
1302	1245	(__v8hi)__I);
1303	1246	}
1304	1247
1305		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1306		_mm_maskz_permutex2var_epi16 (__mmask8 __U, __m128i __A, __m128i __I,
1307		__m128i __B)
1308		{
	1248	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1249	_mm_maskz_permutex2var_epi16(__mmask8 __U, __m128i __A, __m128i __I,
	1250	__m128i __B) {
1309	1251	return (__m128i)__builtin_ia32_selectw_128(__U,
1310	1252	(__v8hi)_mm_permutex2var_epi16(__A, __I, __B),
1311	1253	(__v8hi)_mm_setzero_si128());
1312	1254	}
1313	1255
1314		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1315		_mm256_permutex2var_epi16(__m256i __A, __m256i __I, __m256i __B)
1316		{
	1256	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1257	_mm256_permutex2var_epi16(__m256i __A, __m256i __I, __m256i __B) {
1317	1258	return (__m256i)__builtin_ia32_vpermi2varhi256((__v16hi)__A, (__v16hi)__I,
1318	1259	(__v16hi)__B);
1319	1260	}
1320	1261
1321		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1262	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1322	1263	_mm256_mask_permutex2var_epi16(__m256i __A, __mmask16 __U, __m256i __I,
1323		__m256i __B)
1324		{
	1264	__m256i __B) {
1325	1265	return (__m256i)__builtin_ia32_selectw_256(__U,
1326	1266	(__v16hi)_mm256_permutex2var_epi16(__A, __I, __B),
1327	1267	(__v16hi)__A);
1328	1268	}
1329	1269
1330		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1270	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1331	1271	_mm256_mask2_permutex2var_epi16(__m256i __A, __m256i __I, __mmask16 __U,
1332		__m256i __B)
1333		{
	1272	__m256i __B) {
1334	1273	return (__m256i)__builtin_ia32_selectw_256(__U,
1335	1274	(__v16hi)_mm256_permutex2var_epi16(__A, __I, __B),
1336	1275	(__v16hi)__I);
1337	1276	}
1338	1277
1339		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1340		_mm256_maskz_permutex2var_epi16 (__mmask16 __U, __m256i __A, __m256i __I,
1341		__m256i __B)
1342		{
	1278	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1279	_mm256_maskz_permutex2var_epi16(__mmask16 __U, __m256i __A, __m256i __I,
	1280	__m256i __B) {
1343	1281	return (__m256i)__builtin_ia32_selectw_256(__U,
1344	1282	(__v16hi)_mm256_permutex2var_epi16(__A, __I, __B),
1345	1283	(__v16hi)_mm256_setzero_si256());
1346	1284	}
1347	1285
1348		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1286	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1349	1287	_mm_mask_maddubs_epi16(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
1350	1288	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1351	1289	(__v8hi)_mm_maddubs_epi16(__X, __Y),
1352	1290	(__v8hi)__W);
1353	1291	}
1354	1292
1355		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1293	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1356	1294	_mm_maskz_maddubs_epi16(__mmask8 __U, __m128i __X, __m128i __Y) {
1357	1295	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1358	1296	(__v8hi)_mm_maddubs_epi16(__X, __Y),
1359	1297	(__v8hi)_mm_setzero_si128());
1360	1298	}
1361	1299
1362		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1300	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1363	1301	_mm256_mask_maddubs_epi16(__m256i __W, __mmask16 __U, __m256i __X,
1364	1302	__m256i __Y) {
1365	1303	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1367,35 +1305,35 @@ _mm256_mask_maddubs_epi16(__m256i __W, __mmask16 __U, __m256i __X,
1367	1305	(__v16hi)__W);
1368	1306	}
1369	1307
1370		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1308	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1371	1309	_mm256_maskz_maddubs_epi16(__mmask16 __U, __m256i __X, __m256i __Y) {
1372	1310	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1373	1311	(__v16hi)_mm256_maddubs_epi16(__X, __Y),
1374	1312	(__v16hi)_mm256_setzero_si256());
1375	1313	}
1376	1314
1377		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1315	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1378	1316	_mm_mask_madd_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
1379	1317	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
1380	1318	(__v4si)_mm_madd_epi16(__A, __B),
1381	1319	(__v4si)__W);
1382	1320	}
1383	1321
1384		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1322	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1385	1323	_mm_maskz_madd_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
1386	1324	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
1387	1325	(__v4si)_mm_madd_epi16(__A, __B),
1388	1326	(__v4si)_mm_setzero_si128());
1389	1327	}
1390	1328
1391		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1329	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1392	1330	_mm256_mask_madd_epi16(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
1393	1331	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
1394	1332	(__v8si)_mm256_madd_epi16(__A, __B),
1395	1333	(__v8si)__W);
1396	1334	}
1397	1335
1398		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1336	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1399	1337	_mm256_maskz_madd_epi16(__mmask8 __U, __m256i __A, __m256i __B) {
1400	1338	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
1401	1339	(__v8si)_mm256_madd_epi16(__A, __B),
...	...	@@ -1486,8 +1424,8 @@ _mm256_maskz_cvtusepi16_epi8 (__mmask16 __M, __m256i __A) {
1486	1424	__M);
1487	1425	}
1488	1426
1489		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1490		_mm_cvtepi16_epi8 (__m128i __A) {
	1427	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1428	_mm_cvtepi16_epi8(__m128i __A) {
1491	1429	return (__m128i)__builtin_shufflevector(
1492	1430	__builtin_convertvector((__v8hi)__A, __v8qi),
1493	1431	(__v8qi){0, 0, 0, 0, 0, 0, 0, 0}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
...	...	@@ -1527,20 +1465,20 @@ _mm_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
1527	1465	__builtin_ia32_pmovuswb128mem_mask ((__v16qi *) __P, (__v8hi) __A, __M);
1528	1466	}
1529	1467
1530		static __inline__ __m128i __DEFAULT_FN_ATTRS256
1531		_mm256_cvtepi16_epi8 (__m256i __A) {
	1468	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1469	_mm256_cvtepi16_epi8(__m256i __A) {
1532	1470	return (__m128i)__builtin_convertvector((__v16hi) __A, __v16qi);
1533	1471	}
1534	1472
1535		static __inline__ __m128i __DEFAULT_FN_ATTRS256
1536		_mm256_mask_cvtepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) {
	1473	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1474	_mm256_mask_cvtepi16_epi8(__m128i __O, __mmask16 __M, __m256i __A) {
1537	1475	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
1538	1476	(__v16qi)_mm256_cvtepi16_epi8(__A),
1539	1477	(__v16qi)__O);
1540	1478	}
1541	1479
1542		static __inline__ __m128i __DEFAULT_FN_ATTRS256
1543		_mm256_maskz_cvtepi16_epi8 (__mmask16 __M, __m256i __A) {
	1480	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1481	_mm256_maskz_cvtepi16_epi8(__mmask16 __M, __m256i __A) {
1544	1482	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__M,
1545	1483	(__v16qi)_mm256_cvtepi16_epi8(__A),
1546	1484	(__v16qi)_mm_setzero_si128());
...	...	@@ -1564,203 +1502,198 @@ _mm256_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask16 __M, __m256i __A)
1564	1502	__builtin_ia32_pmovuswb256mem_mask ((__v16qi*) __P, (__v16hi) __A, __M);
1565	1503	}
1566	1504
1567		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1505	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1568	1506	_mm_mask_mulhrs_epi16(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
1569	1507	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1570	1508	(__v8hi)_mm_mulhrs_epi16(__X, __Y),
1571	1509	(__v8hi)__W);
1572	1510	}
1573	1511
1574		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1512	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1575	1513	_mm_maskz_mulhrs_epi16(__mmask8 __U, __m128i __X, __m128i __Y) {
1576	1514	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1577	1515	(__v8hi)_mm_mulhrs_epi16(__X, __Y),
1578	1516	(__v8hi)_mm_setzero_si128());
1579	1517	}
1580	1518
1581		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1519	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1582	1520	_mm256_mask_mulhrs_epi16(__m256i __W, __mmask16 __U, __m256i __X, __m256i __Y) {
1583	1521	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1584	1522	(__v16hi)_mm256_mulhrs_epi16(__X, __Y),
1585	1523	(__v16hi)__W);
1586	1524	}
1587	1525
1588		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1526	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1589	1527	_mm256_maskz_mulhrs_epi16(__mmask16 __U, __m256i __X, __m256i __Y) {
1590	1528	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1591	1529	(__v16hi)_mm256_mulhrs_epi16(__X, __Y),
1592	1530	(__v16hi)_mm256_setzero_si256());
1593	1531	}
1594	1532
1595		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1533	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1596	1534	_mm_mask_mulhi_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
1597		return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1598		(__v8hi)_mm_mulhi_epu16(__A, __B),
1599		(__v8hi)__W);
	1535	return (__m128i)__builtin_ia32_selectw_128(
	1536	(__mmask8)__U, (__v8hi)_mm_mulhi_epu16(__A, __B), (__v8hi)__W);
1600	1537	}
1601	1538
1602		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1539	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1603	1540	_mm_maskz_mulhi_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
1604	1541	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1605	1542	(__v8hi)_mm_mulhi_epu16(__A, __B),
1606	1543	(__v8hi)_mm_setzero_si128());
1607	1544	}
1608	1545
1609		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1546	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1610	1547	_mm256_mask_mulhi_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
1611		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1612		(__v16hi)_mm256_mulhi_epu16(__A, __B),
1613		(__v16hi)__W);
	1548	return (__m256i)__builtin_ia32_selectw_256(
	1549	(__mmask16)__U, (__v16hi)_mm256_mulhi_epu16(__A, __B), (__v16hi)__W);
1614	1550	}
1615	1551
1616		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1552	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1617	1553	_mm256_maskz_mulhi_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
1618		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1619		(__v16hi)_mm256_mulhi_epu16(__A, __B),
1620		(__v16hi)_mm256_setzero_si256());
	1554	return (__m256i)__builtin_ia32_selectw_256(
	1555	(__mmask16)__U, (__v16hi)_mm256_mulhi_epu16(__A, __B),
	1556	(__v16hi)_mm256_setzero_si256());
1621	1557	}
1622	1558
1623		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1559	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1624	1560	_mm_mask_mulhi_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
1625		return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1626		(__v8hi)_mm_mulhi_epi16(__A, __B),
1627		(__v8hi)__W);
	1561	return (__m128i)__builtin_ia32_selectw_128(
	1562	(__mmask8)__U, (__v8hi)_mm_mulhi_epi16(__A, __B), (__v8hi)__W);
1628	1563	}
1629	1564
1630		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1565	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1631	1566	_mm_maskz_mulhi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
1632	1567	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1633	1568	(__v8hi)_mm_mulhi_epi16(__A, __B),
1634	1569	(__v8hi)_mm_setzero_si128());
1635	1570	}
1636	1571
1637		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1572	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1638	1573	_mm256_mask_mulhi_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
1639		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1640		(__v16hi)_mm256_mulhi_epi16(__A, __B),
1641		(__v16hi)__W);
	1574	return (__m256i)__builtin_ia32_selectw_256(
	1575	(__mmask16)__U, (__v16hi)_mm256_mulhi_epi16(__A, __B), (__v16hi)__W);
1642	1576	}
1643	1577
1644		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1578	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1645	1579	_mm256_maskz_mulhi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
1646		return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1647		(__v16hi)_mm256_mulhi_epi16(__A, __B),
1648		(__v16hi)_mm256_setzero_si256());
	1580	return (__m256i)__builtin_ia32_selectw_256(
	1581	(__mmask16)__U, (__v16hi)_mm256_mulhi_epi16(__A, __B),
	1582	(__v16hi)_mm256_setzero_si256());
1649	1583	}
1650	1584
1651		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1585	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1652	1586	_mm_mask_unpackhi_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
1653		return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1654		(__v16qi)_mm_unpackhi_epi8(__A, __B),
1655		(__v16qi)__W);
	1587	return (__m128i)__builtin_ia32_selectb_128(
	1588	(__mmask16)__U, (__v16qi)_mm_unpackhi_epi8(__A, __B), (__v16qi)__W);
1656	1589	}
1657	1590
1658		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1591	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1659	1592	_mm_maskz_unpackhi_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
1660	1593	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1661	1594	(__v16qi)_mm_unpackhi_epi8(__A, __B),
1662	1595	(__v16qi)_mm_setzero_si128());
1663	1596	}
1664	1597
1665		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1598	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1666	1599	_mm256_mask_unpackhi_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
1667	1600	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1668	1601	(__v32qi)_mm256_unpackhi_epi8(__A, __B),
1669	1602	(__v32qi)__W);
1670	1603	}
1671	1604
1672		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1605	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1673	1606	_mm256_maskz_unpackhi_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
1674	1607	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1675	1608	(__v32qi)_mm256_unpackhi_epi8(__A, __B),
1676	1609	(__v32qi)_mm256_setzero_si256());
1677	1610	}
1678	1611
1679		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1612	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1680	1613	_mm_mask_unpackhi_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
1681	1614	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1682	1615	(__v8hi)_mm_unpackhi_epi16(__A, __B),
1683	1616	(__v8hi)__W);
1684	1617	}
1685	1618
1686		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1619	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1687	1620	_mm_maskz_unpackhi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
1688	1621	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1689	1622	(__v8hi)_mm_unpackhi_epi16(__A, __B),
1690	1623	(__v8hi) _mm_setzero_si128());
1691	1624	}
1692	1625
1693		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1626	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1694	1627	_mm256_mask_unpackhi_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
1695	1628	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1696	1629	(__v16hi)_mm256_unpackhi_epi16(__A, __B),
1697	1630	(__v16hi)__W);
1698	1631	}
1699	1632
1700		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1633	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1701	1634	_mm256_maskz_unpackhi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
1702	1635	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1703	1636	(__v16hi)_mm256_unpackhi_epi16(__A, __B),
1704	1637	(__v16hi)_mm256_setzero_si256());
1705	1638	}
1706	1639
1707		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1640	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1708	1641	_mm_mask_unpacklo_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
1709	1642	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1710	1643	(__v16qi)_mm_unpacklo_epi8(__A, __B),
1711	1644	(__v16qi)__W);
1712	1645	}
1713	1646
1714		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1647	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1715	1648	_mm_maskz_unpacklo_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
1716	1649	return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
1717	1650	(__v16qi)_mm_unpacklo_epi8(__A, __B),
1718	1651	(__v16qi)_mm_setzero_si128());
1719	1652	}
1720	1653
1721		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1654	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1722	1655	_mm256_mask_unpacklo_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
1723	1656	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1724	1657	(__v32qi)_mm256_unpacklo_epi8(__A, __B),
1725	1658	(__v32qi)__W);
1726	1659	}
1727	1660
1728		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1661	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1729	1662	_mm256_maskz_unpacklo_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
1730	1663	return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
1731	1664	(__v32qi)_mm256_unpacklo_epi8(__A, __B),
1732	1665	(__v32qi)_mm256_setzero_si256());
1733	1666	}
1734	1667
1735		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1668	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1736	1669	_mm_mask_unpacklo_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
1737	1670	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1738	1671	(__v8hi)_mm_unpacklo_epi16(__A, __B),
1739	1672	(__v8hi)__W);
1740	1673	}
1741	1674
1742		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1675	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1743	1676	_mm_maskz_unpacklo_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
1744	1677	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
1745	1678	(__v8hi)_mm_unpacklo_epi16(__A, __B),
1746	1679	(__v8hi) _mm_setzero_si128());
1747	1680	}
1748	1681
1749		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1682	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1750	1683	_mm256_mask_unpacklo_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
1751	1684	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1752	1685	(__v16hi)_mm256_unpacklo_epi16(__A, __B),
1753	1686	(__v16hi)__W);
1754	1687	}
1755	1688
1756		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1689	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1757	1690	_mm256_maskz_unpacklo_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
1758	1691	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1759	1692	(__v16hi)_mm256_unpacklo_epi16(__A, __B),
1760	1693	(__v16hi)_mm256_setzero_si256());
1761	1694	}
1762	1695
1763		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1696	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1764	1697	_mm_mask_cvtepi8_epi16(__m128i __W, __mmask8 __U, __m128i __A)
1765	1698	{
1766	1699	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1768,7 +1701,7 @@ _mm_mask_cvtepi8_epi16(__m128i __W, __mmask8 __U, __m128i __A)
1768	1701	(__v8hi)__W);
1769	1702	}
1770	1703
1771		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1704	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1772	1705	_mm_maskz_cvtepi8_epi16(__mmask8 __U, __m128i __A)
1773	1706	{
1774	1707	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1776,7 +1709,7 @@ _mm_maskz_cvtepi8_epi16(__mmask8 __U, __m128i __A)
1776	1709	(__v8hi)_mm_setzero_si128());
1777	1710	}
1778	1711
1779		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1712	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1780	1713	_mm256_mask_cvtepi8_epi16(__m256i __W, __mmask16 __U, __m128i __A)
1781	1714	{
1782	1715	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1784,7 +1717,7 @@ _mm256_mask_cvtepi8_epi16(__m256i __W, __mmask16 __U, __m128i __A)
1784	1717	(__v16hi)__W);
1785	1718	}
1786	1719
1787		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1720	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1788	1721	_mm256_maskz_cvtepi8_epi16(__mmask16 __U, __m128i __A)
1789	1722	{
1790	1723	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1793,7 +1726,7 @@ _mm256_maskz_cvtepi8_epi16(__mmask16 __U, __m128i __A)
1793	1726	}
1794	1727
1795	1728
1796		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1729	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1797	1730	_mm_mask_cvtepu8_epi16(__m128i __W, __mmask8 __U, __m128i __A)
1798	1731	{
1799	1732	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1801,7 +1734,7 @@ _mm_mask_cvtepu8_epi16(__m128i __W, __mmask8 __U, __m128i __A)
1801	1734	(__v8hi)__W);
1802	1735	}
1803	1736
1804		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1737	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1805	1738	_mm_maskz_cvtepu8_epi16(__mmask8 __U, __m128i __A)
1806	1739	{
1807	1740	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1809,7 +1742,7 @@ _mm_maskz_cvtepu8_epi16(__mmask8 __U, __m128i __A)
1809	1742	(__v8hi)_mm_setzero_si128());
1810	1743	}
1811	1744
1812		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1745	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1813	1746	_mm256_mask_cvtepu8_epi16(__m256i __W, __mmask16 __U, __m128i __A)
1814	1747	{
1815	1748	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1817,7 +1750,7 @@ _mm256_mask_cvtepu8_epi16(__m256i __W, __mmask16 __U, __m128i __A)
1817	1750	(__v16hi)__W);
1818	1751	}
1819	1752
1820		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1753	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1821	1754	_mm256_maskz_cvtepu8_epi16 (__mmask16 __U, __m128i __A)
1822	1755	{
1823	1756	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1868,13 +1801,13 @@ _mm256_maskz_cvtepu8_epi16 (__mmask16 __U, __m128i __A)
1868	1801	(imm)), \
1869	1802	(__v16hi)_mm256_setzero_si256()))
1870	1803
1871		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1804	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1872	1805	_mm256_sllv_epi16(__m256i __A, __m256i __B)
1873	1806	{
1874	1807	return (__m256i)__builtin_ia32_psllv16hi((__v16hi)__A, (__v16hi)__B);
1875	1808	}
1876	1809
1877		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1810	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1878	1811	_mm256_mask_sllv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1879	1812	{
1880	1813	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1882,7 +1815,7 @@ _mm256_mask_sllv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1882	1815	(__v16hi)__W);
1883	1816	}
1884	1817
1885		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1818	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1886	1819	_mm256_maskz_sllv_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1887	1820	{
1888	1821	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1890,13 +1823,13 @@ _mm256_maskz_sllv_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1890	1823	(__v16hi)_mm256_setzero_si256());
1891	1824	}
1892	1825
1893		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1826	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1894	1827	_mm_sllv_epi16(__m128i __A, __m128i __B)
1895	1828	{
1896	1829	return (__m128i)__builtin_ia32_psllv8hi((__v8hi)__A, (__v8hi)__B);
1897	1830	}
1898	1831
1899		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1832	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1900	1833	_mm_mask_sllv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1901	1834	{
1902	1835	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1904,7 +1837,7 @@ _mm_mask_sllv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
1904	1837	(__v8hi)__W);
1905	1838	}
1906	1839
1907		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1840	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1908	1841	_mm_maskz_sllv_epi16(__mmask8 __U, __m128i __A, __m128i __B)
1909	1842	{
1910	1843	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1944,7 +1877,7 @@ _mm256_maskz_sll_epi16(__mmask16 __U, __m256i __A, __m128i __B)
1944	1877	(__v16hi)_mm256_setzero_si256());
1945	1878	}
1946	1879
1947		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1880	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1948	1881	_mm_mask_slli_epi16(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B)
1949	1882	{
1950	1883	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1952,7 +1885,7 @@ _mm_mask_slli_epi16(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B)
1952	1885	(__v8hi)__W);
1953	1886	}
1954	1887
1955		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1888	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
1956	1889	_mm_maskz_slli_epi16 (__mmask8 __U, __m128i __A, unsigned int __B)
1957	1890	{
1958	1891	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -1960,30 +1893,28 @@ _mm_maskz_slli_epi16 (__mmask8 __U, __m128i __A, unsigned int __B)
1960	1893	(__v8hi)_mm_setzero_si128());
1961	1894	}
1962	1895
1963		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1896	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1964	1897	_mm256_mask_slli_epi16(__m256i __W, __mmask16 __U, __m256i __A,
1965		unsigned int __B)
1966		{
	1898	unsigned int __B) {
1967	1899	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1968	1900	(__v16hi)_mm256_slli_epi16(__A, (int)__B),
1969	1901	(__v16hi)__W);
1970	1902	}
1971	1903
1972		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1973		_mm256_maskz_slli_epi16(__mmask16 __U, __m256i __A, unsigned int __B)
1974		{
	1904	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1905	_mm256_maskz_slli_epi16(__mmask16 __U, __m256i __A, unsigned int __B) {
1975	1906	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
1976	1907	(__v16hi)_mm256_slli_epi16(__A, (int)__B),
1977	1908	(__v16hi)_mm256_setzero_si256());
1978	1909	}
1979	1910
1980		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1911	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1981	1912	_mm256_srlv_epi16(__m256i __A, __m256i __B)
1982	1913	{
1983	1914	return (__m256i)__builtin_ia32_psrlv16hi((__v16hi)__A, (__v16hi)__B);
1984	1915	}
1985	1916
1986		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1917	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1987	1918	_mm256_mask_srlv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1988	1919	{
1989	1920	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1991,7 +1922,7 @@ _mm256_mask_srlv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
1991	1922	(__v16hi)__W);
1992	1923	}
1993	1924
1994		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1925	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
1995	1926	_mm256_maskz_srlv_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1996	1927	{
1997	1928	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -1999,13 +1930,13 @@ _mm256_maskz_srlv_epi16(__mmask16 __U, __m256i __A, __m256i __B)
1999	1930	(__v16hi)_mm256_setzero_si256());
2000	1931	}
2001	1932
2002		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1933	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2003	1934	_mm_srlv_epi16(__m128i __A, __m128i __B)
2004	1935	{
2005	1936	return (__m128i)__builtin_ia32_psrlv8hi((__v8hi)__A, (__v8hi)__B);
2006	1937	}
2007	1938
2008		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1939	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2009	1940	_mm_mask_srlv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
2010	1941	{
2011	1942	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -2013,7 +1944,7 @@ _mm_mask_srlv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
2013	1944	(__v8hi)__W);
2014	1945	}
2015	1946
2016		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1947	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2017	1948	_mm_maskz_srlv_epi16(__mmask8 __U, __m128i __A, __m128i __B)
2018	1949	{
2019	1950	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -2021,13 +1952,13 @@ _mm_maskz_srlv_epi16(__mmask8 __U, __m128i __A, __m128i __B)
2021	1952	(__v8hi)_mm_setzero_si128());
2022	1953	}
2023	1954
2024		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1955	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2025	1956	_mm256_srav_epi16(__m256i __A, __m256i __B)
2026	1957	{
2027	1958	return (__m256i)__builtin_ia32_psrav16hi((__v16hi)__A, (__v16hi)__B);
2028	1959	}
2029	1960
2030		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1961	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2031	1962	_mm256_mask_srav_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
2032	1963	{
2033	1964	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -2035,7 +1966,7 @@ _mm256_mask_srav_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B)
2035	1966	(__v16hi)__W);
2036	1967	}
2037	1968
2038		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	1969	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2039	1970	_mm256_maskz_srav_epi16(__mmask16 __U, __m256i __A, __m256i __B)
2040	1971	{
2041	1972	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
...	...	@@ -2043,13 +1974,13 @@ _mm256_maskz_srav_epi16(__mmask16 __U, __m256i __A, __m256i __B)
2043	1974	(__v16hi)_mm256_setzero_si256());
2044	1975	}
2045	1976
2046		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1977	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2047	1978	_mm_srav_epi16(__m128i __A, __m128i __B)
2048	1979	{
2049	1980	return (__m128i)__builtin_ia32_psrav8hi((__v8hi)__A, (__v8hi)__B);
2050	1981	}
2051	1982
2052		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1983	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2053	1984	_mm_mask_srav_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
2054	1985	{
2055	1986	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -2057,7 +1988,7 @@ _mm_mask_srav_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
2057	1988	(__v8hi)__W);
2058	1989	}
2059	1990
2060		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	1991	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2061	1992	_mm_maskz_srav_epi16(__mmask8 __U, __m128i __A, __m128i __B)
2062	1993	{
2063	1994	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
...	...	@@ -2097,34 +2028,30 @@ _mm256_maskz_sra_epi16(__mmask16 __U, __m256i __A, __m128i __B)
2097	2028	(__v16hi)_mm256_setzero_si256());
2098	2029	}
2099	2030
2100		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2101		_mm_mask_srai_epi16(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B)
2102		{
	2031	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2032	_mm_mask_srai_epi16(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
2103	2033	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
2104	2034	(__v8hi)_mm_srai_epi16(__A, (int)__B),
2105	2035	(__v8hi)__W);
2106	2036	}
2107	2037
2108		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2109		_mm_maskz_srai_epi16(__mmask8 __U, __m128i __A, unsigned int __B)
2110		{
	2038	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2039	_mm_maskz_srai_epi16(__mmask8 __U, __m128i __A, unsigned int __B) {
2111	2040	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
2112	2041	(__v8hi)_mm_srai_epi16(__A, (int)__B),
2113	2042	(__v8hi)_mm_setzero_si128());
2114	2043	}
2115	2044
2116		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2045	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2117	2046	_mm256_mask_srai_epi16(__m256i __W, __mmask16 __U, __m256i __A,
2118		unsigned int __B)
2119		{
	2047	unsigned int __B) {
2120	2048	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
2121	2049	(__v16hi)_mm256_srai_epi16(__A, (int)__B),
2122	2050	(__v16hi)__W);
2123	2051	}
2124	2052
2125		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2126		_mm256_maskz_srai_epi16(__mmask16 __U, __m256i __A, unsigned int __B)
2127		{
	2053	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2054	_mm256_maskz_srai_epi16(__mmask16 __U, __m256i __A, unsigned int __B) {
2128	2055	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
2129	2056	(__v16hi)_mm256_srai_epi16(__A, (int)__B),
2130	2057	(__v16hi)_mm256_setzero_si256());
...	...	@@ -2162,104 +2089,91 @@ _mm256_maskz_srl_epi16(__mmask16 __U, __m256i __A, __m128i __B)
2162	2089	(__v16hi)_mm256_setzero_si256());
2163	2090	}
2164	2091
2165		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2166		_mm_mask_srli_epi16(__m128i __W, __mmask8 __U, __m128i __A, int __B)
2167		{
	2092	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2093	_mm_mask_srli_epi16(__m128i __W, __mmask8 __U, __m128i __A, int __B) {
2168	2094	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
2169	2095	(__v8hi)_mm_srli_epi16(__A, __B),
2170	2096	(__v8hi)__W);
2171	2097	}
2172	2098
2173		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2174		_mm_maskz_srli_epi16 (__mmask8 __U, __m128i __A, int __B)
2175		{
	2099	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2100	_mm_maskz_srli_epi16(__mmask8 __U, __m128i __A, int __B) {
2176	2101	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
2177	2102	(__v8hi)_mm_srli_epi16(__A, __B),
2178	2103	(__v8hi)_mm_setzero_si128());
2179	2104	}
2180	2105
2181		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2182		_mm256_mask_srli_epi16(__m256i __W, __mmask16 __U, __m256i __A, int __B)
2183		{
	2106	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2107	_mm256_mask_srli_epi16(__m256i __W, __mmask16 __U, __m256i __A, int __B) {
2184	2108	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
2185	2109	(__v16hi)_mm256_srli_epi16(__A, __B),
2186	2110	(__v16hi)__W);
2187	2111	}
2188	2112
2189		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2190		_mm256_maskz_srli_epi16(__mmask16 __U, __m256i __A, int __B)
2191		{
	2113	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2114	_mm256_maskz_srli_epi16(__mmask16 __U, __m256i __A, int __B) {
2192	2115	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
2193	2116	(__v16hi)_mm256_srli_epi16(__A, __B),
2194	2117	(__v16hi)_mm256_setzero_si256());
2195	2118	}
2196	2119
2197		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2198		_mm_mask_mov_epi16 (__m128i __W, __mmask8 __U, __m128i __A)
2199		{
	2120	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2121	_mm_mask_mov_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
2200	2122	return (__m128i) __builtin_ia32_selectw_128 ((__mmask8) __U,
2201	2123	(__v8hi) __A,
2202	2124	(__v8hi) __W);
2203	2125	}
2204	2126
2205		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2206		_mm_maskz_mov_epi16 (__mmask8 __U, __m128i __A)
2207		{
	2127	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2128	_mm_maskz_mov_epi16(__mmask8 __U, __m128i __A) {
2208	2129	return (__m128i) __builtin_ia32_selectw_128 ((__mmask8) __U,
2209	2130	(__v8hi) __A,
2210	2131	(__v8hi) _mm_setzero_si128 ());
2211	2132	}
2212	2133
2213		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2214		_mm256_mask_mov_epi16 (__m256i __W, __mmask16 __U, __m256i __A)
2215		{
	2134	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2135	_mm256_mask_mov_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
2216	2136	return (__m256i) __builtin_ia32_selectw_256 ((__mmask16) __U,
2217	2137	(__v16hi) __A,
2218	2138	(__v16hi) __W);
2219	2139	}
2220	2140
2221		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2222		_mm256_maskz_mov_epi16 (__mmask16 __U, __m256i __A)
2223		{
	2141	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2142	_mm256_maskz_mov_epi16(__mmask16 __U, __m256i __A) {
2224	2143	return (__m256i) __builtin_ia32_selectw_256 ((__mmask16) __U,
2225	2144	(__v16hi) __A,
2226	2145	(__v16hi) _mm256_setzero_si256 ());
2227	2146	}
2228	2147
2229		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2230		_mm_mask_mov_epi8 (__m128i __W, __mmask16 __U, __m128i __A)
2231		{
	2148	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2149	_mm_mask_mov_epi8(__m128i __W, __mmask16 __U, __m128i __A) {
2232	2150	return (__m128i) __builtin_ia32_selectb_128 ((__mmask16) __U,
2233	2151	(__v16qi) __A,
2234	2152	(__v16qi) __W);
2235	2153	}
2236	2154
2237		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2238		_mm_maskz_mov_epi8 (__mmask16 __U, __m128i __A)
2239		{
	2155	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2156	_mm_maskz_mov_epi8(__mmask16 __U, __m128i __A) {
2240	2157	return (__m128i) __builtin_ia32_selectb_128 ((__mmask16) __U,
2241	2158	(__v16qi) __A,
2242	2159	(__v16qi) _mm_setzero_si128 ());
2243	2160	}
2244	2161
2245		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2246		_mm256_mask_mov_epi8 (__m256i __W, __mmask32 __U, __m256i __A)
2247		{
	2162	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2163	_mm256_mask_mov_epi8(__m256i __W, __mmask32 __U, __m256i __A) {
2248	2164	return (__m256i) __builtin_ia32_selectb_256 ((__mmask32) __U,
2249	2165	(__v32qi) __A,
2250	2166	(__v32qi) __W);
2251	2167	}
2252	2168
2253		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2254		_mm256_maskz_mov_epi8 (__mmask32 __U, __m256i __A)
2255		{
	2169	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2170	_mm256_maskz_mov_epi8(__mmask32 __U, __m256i __A) {
2256	2171	return (__m256i) __builtin_ia32_selectb_256 ((__mmask32) __U,
2257	2172	(__v32qi) __A,
2258	2173	(__v32qi) _mm256_setzero_si256 ());
2259	2174	}
2260	2175
2261
2262		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2176	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2263	2177	_mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A)
2264	2178	{
2265	2179	return (__m128i) __builtin_ia32_selectb_128(__M,
...	...	@@ -2267,7 +2181,7 @@ _mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A)
2267	2181	(__v16qi) __O);
2268	2182	}
2269	2183
2270		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2184	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2271	2185	_mm_maskz_set1_epi8 (__mmask16 __M, char __A)
2272	2186	{
2273	2187	return (__m128i) __builtin_ia32_selectb_128(__M,
...	...	@@ -2275,7 +2189,7 @@ _mm_maskz_set1_epi8 (__mmask16 __M, char __A)
2275	2189	(__v16qi) _mm_setzero_si128());
2276	2190	}
2277	2191
2278		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2192	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2279	2193	_mm256_mask_set1_epi8 (__m256i __O, __mmask32 __M, char __A)
2280	2194	{
2281	2195	return (__m256i) __builtin_ia32_selectb_256(__M,
...	...	@@ -2283,7 +2197,7 @@ _mm256_mask_set1_epi8 (__m256i __O, __mmask32 __M, char __A)
2283	2197	(__v32qi) __O);
2284	2198	}
2285	2199
2286		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2200	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2287	2201	_mm256_maskz_set1_epi8 (__mmask32 __M, char __A)
2288	2202	{
2289	2203	return (__m256i) __builtin_ia32_selectb_256(__M,
...	...	@@ -2463,22 +2377,19 @@ _mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A)
2463	2377	(__mmask32) __U);
2464	2378	}
2465	2379
2466		static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
2467		_mm_test_epi8_mask (__m128i __A, __m128i __B)
2468		{
	2380	static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2381	_mm_test_epi8_mask(__m128i __A, __m128i __B) {
2469	2382	return _mm_cmpneq_epi8_mask (_mm_and_si128(__A, __B), _mm_setzero_si128());
2470	2383	}
2471	2384
2472		static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
2473		_mm_mask_test_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B)
2474		{
	2385	static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2386	_mm_mask_test_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
2475	2387	return _mm_mask_cmpneq_epi8_mask (__U, _mm_and_si128 (__A, __B),
2476	2388	_mm_setzero_si128());
2477	2389	}
2478	2390
2479		static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
2480		_mm256_test_epi8_mask (__m256i __A, __m256i __B)
2481		{
	2391	static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2392	_mm256_test_epi8_mask(__m256i __A, __m256i __B) {
2482	2393	return _mm256_cmpneq_epi8_mask (_mm256_and_si256(__A, __B),
2483	2394	_mm256_setzero_si256());
2484	2395	}
...	...	@@ -2517,9 +2428,8 @@ _mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B)
2517	2428	_mm256_setzero_si256());
2518	2429	}
2519	2430
2520		static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
2521		_mm_testn_epi8_mask (__m128i __A, __m128i __B)
2522		{
	2431	static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2432	_mm_testn_epi8_mask(__m128i __A, __m128i __B) {
2523	2433	return _mm_cmpeq_epi8_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
2524	2434	}
2525	2435
...	...	@@ -2570,55 +2480,47 @@ _mm256_mask_testn_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B)
2570	2480	_mm256_setzero_si256());
2571	2481	}
2572	2482
2573		static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
2574		_mm_movepi8_mask (__m128i __A)
2575		{
	2483	static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2484	_mm_movepi8_mask(__m128i __A) {
2576	2485	return (__mmask16) __builtin_ia32_cvtb2mask128 ((__v16qi) __A);
2577	2486	}
2578	2487
2579		static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
2580		_mm256_movepi8_mask (__m256i __A)
2581		{
	2488	static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2489	_mm256_movepi8_mask(__m256i __A) {
2582	2490	return (__mmask32) __builtin_ia32_cvtb2mask256 ((__v32qi) __A);
2583	2491	}
2584	2492
2585		static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
2586		_mm_movepi16_mask (__m128i __A)
2587		{
	2493	static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2494	_mm_movepi16_mask(__m128i __A) {
2588	2495	return (__mmask8) __builtin_ia32_cvtw2mask128 ((__v8hi) __A);
2589	2496	}
2590	2497
2591		static __inline__ __mmask16 __DEFAULT_FN_ATTRS256
2592		_mm256_movepi16_mask (__m256i __A)
2593		{
	2498	static __inline__ __mmask16 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2499	_mm256_movepi16_mask(__m256i __A) {
2594	2500	return (__mmask16) __builtin_ia32_cvtw2mask256 ((__v16hi) __A);
2595	2501	}
2596	2502
2597		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2598		_mm_movm_epi8 (__mmask16 __A)
2599		{
	2503	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2504	_mm_movm_epi8(__mmask16 __A) {
2600	2505	return (__m128i) __builtin_ia32_cvtmask2b128 (__A);
2601	2506	}
2602	2507
2603		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2604		_mm256_movm_epi8 (__mmask32 __A)
2605		{
	2508	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2509	_mm256_movm_epi8(__mmask32 __A) {
2606	2510	return (__m256i) __builtin_ia32_cvtmask2b256 (__A);
2607	2511	}
2608	2512
2609		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2610		_mm_movm_epi16 (__mmask8 __A)
2611		{
	2513	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2514	_mm_movm_epi16(__mmask8 __A) {
2612	2515	return (__m128i) __builtin_ia32_cvtmask2w128 (__A);
2613	2516	}
2614	2517
2615		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2616		_mm256_movm_epi16 (__mmask16 __A)
2617		{
	2518	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2519	_mm256_movm_epi16(__mmask16 __A) {
2618	2520	return (__m256i) __builtin_ia32_cvtmask2w256 (__A);
2619	2521	}
2620	2522
2621		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2523	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2622	2524	_mm_mask_broadcastb_epi8 (__m128i __O, __mmask16 __M, __m128i __A)
2623	2525	{
2624	2526	return (__m128i)__builtin_ia32_selectb_128(__M,
...	...	@@ -2626,7 +2528,7 @@ _mm_mask_broadcastb_epi8 (__m128i __O, __mmask16 __M, __m128i __A)
2626	2528	(__v16qi) __O);
2627	2529	}
2628	2530
2629		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2531	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2630	2532	_mm_maskz_broadcastb_epi8 (__mmask16 __M, __m128i __A)
2631	2533	{
2632	2534	return (__m128i)__builtin_ia32_selectb_128(__M,
...	...	@@ -2634,7 +2536,7 @@ _mm_maskz_broadcastb_epi8 (__mmask16 __M, __m128i __A)
2634	2536	(__v16qi) _mm_setzero_si128());
2635	2537	}
2636	2538
2637		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2539	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2638	2540	_mm256_mask_broadcastb_epi8 (__m256i __O, __mmask32 __M, __m128i __A)
2639	2541	{
2640	2542	return (__m256i)__builtin_ia32_selectb_256(__M,
...	...	@@ -2642,7 +2544,7 @@ _mm256_mask_broadcastb_epi8 (__m256i __O, __mmask32 __M, __m128i __A)
2642	2544	(__v32qi) __O);
2643	2545	}
2644	2546
2645		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2547	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2646	2548	_mm256_maskz_broadcastb_epi8 (__mmask32 __M, __m128i __A)
2647	2549	{
2648	2550	return (__m256i)__builtin_ia32_selectb_256(__M,
...	...	@@ -2650,7 +2552,7 @@ _mm256_maskz_broadcastb_epi8 (__mmask32 __M, __m128i __A)
2650	2552	(__v32qi) _mm256_setzero_si256());
2651	2553	}
2652	2554
2653		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2555	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2654	2556	_mm_mask_broadcastw_epi16 (__m128i __O, __mmask8 __M, __m128i __A)
2655	2557	{
2656	2558	return (__m128i)__builtin_ia32_selectw_128(__M,
...	...	@@ -2658,7 +2560,7 @@ _mm_mask_broadcastw_epi16 (__m128i __O, __mmask8 __M, __m128i __A)
2658	2560	(__v8hi) __O);
2659	2561	}
2660	2562
2661		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2563	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2662	2564	_mm_maskz_broadcastw_epi16 (__mmask8 __M, __m128i __A)
2663	2565	{
2664	2566	return (__m128i)__builtin_ia32_selectw_128(__M,
...	...	@@ -2666,7 +2568,7 @@ _mm_maskz_broadcastw_epi16 (__mmask8 __M, __m128i __A)
2666	2568	(__v8hi) _mm_setzero_si128());
2667	2569	}
2668	2570
2669		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2571	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2670	2572	_mm256_mask_broadcastw_epi16 (__m256i __O, __mmask16 __M, __m128i __A)
2671	2573	{
2672	2574	return (__m256i)__builtin_ia32_selectw_256(__M,
...	...	@@ -2674,7 +2576,7 @@ _mm256_mask_broadcastw_epi16 (__m256i __O, __mmask16 __M, __m128i __A)
2674	2576	(__v16hi) __O);
2675	2577	}
2676	2578
2677		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2579	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2678	2580	_mm256_maskz_broadcastw_epi16 (__mmask16 __M, __m128i __A)
2679	2581	{
2680	2582	return (__m256i)__builtin_ia32_selectw_256(__M,
...	...	@@ -2682,7 +2584,7 @@ _mm256_maskz_broadcastw_epi16 (__mmask16 __M, __m128i __A)
2682	2584	(__v16hi) _mm256_setzero_si256());
2683	2585	}
2684	2586
2685		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2587	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2686	2588	_mm256_mask_set1_epi16 (__m256i __O, __mmask16 __M, short __A)
2687	2589	{
2688	2590	return (__m256i) __builtin_ia32_selectw_256 (__M,
...	...	@@ -2690,7 +2592,7 @@ _mm256_mask_set1_epi16 (__m256i __O, __mmask16 __M, short __A)
2690	2592	(__v16hi) __O);
2691	2593	}
2692	2594
2693		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2595	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2694	2596	_mm256_maskz_set1_epi16 (__mmask16 __M, short __A)
2695	2597	{
2696	2598	return (__m256i) __builtin_ia32_selectw_256(__M,
...	...	@@ -2698,7 +2600,7 @@ _mm256_maskz_set1_epi16 (__mmask16 __M, short __A)
2698	2600	(__v16hi) _mm256_setzero_si256());
2699	2601	}
2700	2602
2701		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2603	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2702	2604	_mm_mask_set1_epi16 (__m128i __O, __mmask8 __M, short __A)
2703	2605	{
2704	2606	return (__m128i) __builtin_ia32_selectw_128(__M,
...	...	@@ -2706,7 +2608,7 @@ _mm_mask_set1_epi16 (__m128i __O, __mmask8 __M, short __A)
2706	2608	(__v8hi) __O);
2707	2609	}
2708	2610
2709		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2611	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2710	2612	_mm_maskz_set1_epi16 (__mmask8 __M, short __A)
2711	2613	{
2712	2614	return (__m128i) __builtin_ia32_selectw_128(__M,
...	...	@@ -2714,48 +2616,41 @@ _mm_maskz_set1_epi16 (__mmask8 __M, short __A)
2714	2616	(__v8hi) _mm_setzero_si128());
2715	2617	}
2716	2618
2717		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2718		_mm_permutexvar_epi16 (__m128i __A, __m128i __B)
2719		{
	2619	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2620	_mm_permutexvar_epi16(__m128i __A, __m128i __B) {
2720	2621	return (__m128i)__builtin_ia32_permvarhi128((__v8hi) __B, (__v8hi) __A);
2721	2622	}
2722	2623
2723		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2724		_mm_maskz_permutexvar_epi16 (__mmask8 __M, __m128i __A, __m128i __B)
2725		{
	2624	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2625	_mm_maskz_permutexvar_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
2726	2626	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
2727	2627	(__v8hi)_mm_permutexvar_epi16(__A, __B),
2728	2628	(__v8hi) _mm_setzero_si128());
2729	2629	}
2730	2630
2731		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2732		_mm_mask_permutexvar_epi16 (__m128i __W, __mmask8 __M, __m128i __A,
2733		__m128i __B)
2734		{
	2631	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2632	_mm_mask_permutexvar_epi16(__m128i __W, __mmask8 __M, __m128i __A,
	2633	__m128i __B) {
2735	2634	return (__m128i)__builtin_ia32_selectw_128((__mmask8)__M,
2736	2635	(__v8hi)_mm_permutexvar_epi16(__A, __B),
2737	2636	(__v8hi)__W);
2738	2637	}
2739	2638
2740		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2741		_mm256_permutexvar_epi16 (__m256i __A, __m256i __B)
2742		{
	2639	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2640	_mm256_permutexvar_epi16(__m256i __A, __m256i __B) {
2743	2641	return (__m256i)__builtin_ia32_permvarhi256((__v16hi) __B, (__v16hi) __A);
2744	2642	}
2745	2643
2746		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2747		_mm256_maskz_permutexvar_epi16 (__mmask16 __M, __m256i __A,
2748		__m256i __B)
2749		{
	2644	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2645	_mm256_maskz_permutexvar_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
2750	2646	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
2751	2647	(__v16hi)_mm256_permutexvar_epi16(__A, __B),
2752	2648	(__v16hi)_mm256_setzero_si256());
2753	2649	}
2754	2650
2755		static __inline__ __m256i __DEFAULT_FN_ATTRS256
2756		_mm256_mask_permutexvar_epi16 (__m256i __W, __mmask16 __M, __m256i __A,
2757		__m256i __B)
2758		{
	2651	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2652	_mm256_mask_permutexvar_epi16(__m256i __W, __mmask16 __M, __m256i __A,
	2653	__m256i __B) {
2759	2654	return (__m256i)__builtin_ia32_selectw_256((__mmask16)__M,
2760	2655	(__v16hi)_mm256_permutexvar_epi16(__A, __B),
2761	2656	(__v16hi)__W);
...	...	@@ -2809,353 +2704,353 @@ _mm256_mask_permutexvar_epi16 (__m256i __W, __mmask16 __M, __m256i __A,
2809	2704	(__v16hi)_mm256_dbsad_epu8((A), (B), (imm)), \
2810	2705	(__v16hi)_mm256_setzero_si256()))
2811	2706
2812		static __inline__ short __DEFAULT_FN_ATTRS128
	2707	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2813	2708	_mm_reduce_add_epi16(__m128i __W) {
2814	2709	return __builtin_reduce_add((__v8hi)__W);
2815	2710	}
2816	2711
2817		static __inline__ short __DEFAULT_FN_ATTRS128
	2712	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2818	2713	_mm_reduce_mul_epi16(__m128i __W) {
2819	2714	return __builtin_reduce_mul((__v8hi)__W);
2820	2715	}
2821	2716
2822		static __inline__ short __DEFAULT_FN_ATTRS128
	2717	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2823	2718	_mm_reduce_and_epi16(__m128i __W) {
2824	2719	return __builtin_reduce_and((__v8hi)__W);
2825	2720	}
2826	2721
2827		static __inline__ short __DEFAULT_FN_ATTRS128
	2722	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2828	2723	_mm_reduce_or_epi16(__m128i __W) {
2829	2724	return __builtin_reduce_or((__v8hi)__W);
2830	2725	}
2831	2726
2832		static __inline__ short __DEFAULT_FN_ATTRS128
2833		_mm_mask_reduce_add_epi16( __mmask8 __M, __m128i __W) {
	2727	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
	2728	_mm_mask_reduce_add_epi16(__mmask8 __M, __m128i __W) {
2834	2729	__W = _mm_maskz_mov_epi16(__M, __W);
2835	2730	return __builtin_reduce_add((__v8hi)__W);
2836	2731	}
2837	2732
2838		static __inline__ short __DEFAULT_FN_ATTRS128
2839		_mm_mask_reduce_mul_epi16( __mmask8 __M, __m128i __W) {
	2733	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
	2734	_mm_mask_reduce_mul_epi16(__mmask8 __M, __m128i __W) {
2840	2735	__W = _mm_mask_mov_epi16(_mm_set1_epi16(1), __M, __W);
2841	2736	return __builtin_reduce_mul((__v8hi)__W);
2842	2737	}
2843	2738
2844		static __inline__ short __DEFAULT_FN_ATTRS128
2845		_mm_mask_reduce_and_epi16( __mmask8 __M, __m128i __W) {
	2739	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
	2740	_mm_mask_reduce_and_epi16(__mmask8 __M, __m128i __W) {
2846	2741	__W = _mm_mask_mov_epi16(_mm_set1_epi16(-1), __M, __W);
2847	2742	return __builtin_reduce_and((__v8hi)__W);
2848	2743	}
2849	2744
2850		static __inline__ short __DEFAULT_FN_ATTRS128
	2745	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2851	2746	_mm_mask_reduce_or_epi16(__mmask8 __M, __m128i __W) {
2852	2747	__W = _mm_maskz_mov_epi16(__M, __W);
2853	2748	return __builtin_reduce_or((__v8hi)__W);
2854	2749	}
2855	2750
2856		static __inline__ short __DEFAULT_FN_ATTRS128
	2751	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2857	2752	_mm_reduce_max_epi16(__m128i __V) {
2858	2753	return __builtin_reduce_max((__v8hi)__V);
2859	2754	}
2860	2755
2861		static __inline__ unsigned short __DEFAULT_FN_ATTRS128
	2756	static __inline__ unsigned short __DEFAULT_FN_ATTRS128_CONSTEXPR
2862	2757	_mm_reduce_max_epu16(__m128i __V) {
2863	2758	return __builtin_reduce_max((__v8hu)__V);
2864	2759	}
2865	2760
2866		static __inline__ short __DEFAULT_FN_ATTRS128
	2761	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2867	2762	_mm_reduce_min_epi16(__m128i __V) {
2868	2763	return __builtin_reduce_min((__v8hi)__V);
2869	2764	}
2870	2765
2871		static __inline__ unsigned short __DEFAULT_FN_ATTRS128
	2766	static __inline__ unsigned short __DEFAULT_FN_ATTRS128_CONSTEXPR
2872	2767	_mm_reduce_min_epu16(__m128i __V) {
2873	2768	return __builtin_reduce_min((__v8hu)__V);
2874	2769	}
2875	2770
2876		static __inline__ short __DEFAULT_FN_ATTRS128
	2771	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2877	2772	_mm_mask_reduce_max_epi16(__mmask16 __M, __m128i __V) {
2878	2773	__V = _mm_mask_mov_epi16(_mm_set1_epi16(-32767-1), __M, __V);
2879	2774	return __builtin_reduce_max((__v8hi)__V);
2880	2775	}
2881	2776
2882		static __inline__ unsigned short __DEFAULT_FN_ATTRS128
	2777	static __inline__ unsigned short __DEFAULT_FN_ATTRS128_CONSTEXPR
2883	2778	_mm_mask_reduce_max_epu16(__mmask16 __M, __m128i __V) {
2884	2779	__V = _mm_maskz_mov_epi16(__M, __V);
2885	2780	return __builtin_reduce_max((__v8hu)__V);
2886	2781	}
2887	2782
2888		static __inline__ short __DEFAULT_FN_ATTRS128
	2783	static __inline__ short __DEFAULT_FN_ATTRS128_CONSTEXPR
2889	2784	_mm_mask_reduce_min_epi16(__mmask16 __M, __m128i __V) {
2890	2785	__V = _mm_mask_mov_epi16(_mm_set1_epi16(32767), __M, __V);
2891	2786	return __builtin_reduce_min((__v8hi)__V);
2892	2787	}
2893	2788
2894		static __inline__ unsigned short __DEFAULT_FN_ATTRS128
	2789	static __inline__ unsigned short __DEFAULT_FN_ATTRS128_CONSTEXPR
2895	2790	_mm_mask_reduce_min_epu16(__mmask16 __M, __m128i __V) {
2896	2791	__V = _mm_mask_mov_epi16(_mm_set1_epi16(-1), __M, __V);
2897	2792	return __builtin_reduce_min((__v8hu)__V);
2898	2793	}
2899	2794
2900		static __inline__ short __DEFAULT_FN_ATTRS256
	2795	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2901	2796	_mm256_reduce_add_epi16(__m256i __W) {
2902	2797	return __builtin_reduce_add((__v16hi)__W);
2903	2798	}
2904	2799
2905		static __inline__ short __DEFAULT_FN_ATTRS256
	2800	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2906	2801	_mm256_reduce_mul_epi16(__m256i __W) {
2907	2802	return __builtin_reduce_mul((__v16hi)__W);
2908	2803	}
2909	2804
2910		static __inline__ short __DEFAULT_FN_ATTRS256
	2805	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2911	2806	_mm256_reduce_and_epi16(__m256i __W) {
2912	2807	return __builtin_reduce_and((__v16hi)__W);
2913	2808	}
2914	2809
2915		static __inline__ short __DEFAULT_FN_ATTRS256
	2810	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2916	2811	_mm256_reduce_or_epi16(__m256i __W) {
2917	2812	return __builtin_reduce_or((__v16hi)__W);
2918	2813	}
2919	2814
2920		static __inline__ short __DEFAULT_FN_ATTRS256
2921		_mm256_mask_reduce_add_epi16( __mmask16 __M, __m256i __W) {
	2815	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
	2816	_mm256_mask_reduce_add_epi16(__mmask16 __M, __m256i __W) {
2922	2817	__W = _mm256_maskz_mov_epi16(__M, __W);
2923	2818	return __builtin_reduce_add((__v16hi)__W);
2924	2819	}
2925	2820
2926		static __inline__ short __DEFAULT_FN_ATTRS256
2927		_mm256_mask_reduce_mul_epi16( __mmask16 __M, __m256i __W) {
	2821	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
	2822	_mm256_mask_reduce_mul_epi16(__mmask16 __M, __m256i __W) {
2928	2823	__W = _mm256_mask_mov_epi16(_mm256_set1_epi16(1), __M, __W);
2929	2824	return __builtin_reduce_mul((__v16hi)__W);
2930	2825	}
2931	2826
2932		static __inline__ short __DEFAULT_FN_ATTRS256
2933		_mm256_mask_reduce_and_epi16( __mmask16 __M, __m256i __W) {
	2827	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
	2828	_mm256_mask_reduce_and_epi16(__mmask16 __M, __m256i __W) {
2934	2829	__W = _mm256_mask_mov_epi16(_mm256_set1_epi16(-1), __M, __W);
2935	2830	return __builtin_reduce_and((__v16hi)__W);
2936	2831	}
2937	2832
2938		static __inline__ short __DEFAULT_FN_ATTRS256
	2833	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2939	2834	_mm256_mask_reduce_or_epi16(__mmask16 __M, __m256i __W) {
2940	2835	__W = _mm256_maskz_mov_epi16(__M, __W);
2941	2836	return __builtin_reduce_or((__v16hi)__W);
2942	2837	}
2943	2838
2944		static __inline__ short __DEFAULT_FN_ATTRS256
	2839	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2945	2840	_mm256_reduce_max_epi16(__m256i __V) {
2946	2841	return __builtin_reduce_max((__v16hi)__V);
2947	2842	}
2948	2843
2949		static __inline__ unsigned short __DEFAULT_FN_ATTRS256
	2844	static __inline__ unsigned short __DEFAULT_FN_ATTRS256_CONSTEXPR
2950	2845	_mm256_reduce_max_epu16(__m256i __V) {
2951	2846	return __builtin_reduce_max((__v16hu)__V);
2952	2847	}
2953	2848
2954		static __inline__ short __DEFAULT_FN_ATTRS256
	2849	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2955	2850	_mm256_reduce_min_epi16(__m256i __V) {
2956	2851	return __builtin_reduce_min((__v16hi)__V);
2957	2852	}
2958	2853
2959		static __inline__ unsigned short __DEFAULT_FN_ATTRS256
	2854	static __inline__ unsigned short __DEFAULT_FN_ATTRS256_CONSTEXPR
2960	2855	_mm256_reduce_min_epu16(__m256i __V) {
2961	2856	return __builtin_reduce_min((__v16hu)__V);
2962	2857	}
2963	2858
2964		static __inline__ short __DEFAULT_FN_ATTRS256
	2859	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2965	2860	_mm256_mask_reduce_max_epi16(__mmask16 __M, __m256i __V) {
2966	2861	__V = _mm256_mask_mov_epi16(_mm256_set1_epi16(-32767-1), __M, __V);
2967	2862	return __builtin_reduce_max((__v16hi)__V);
2968	2863	}
2969	2864
2970		static __inline__ unsigned short __DEFAULT_FN_ATTRS256
	2865	static __inline__ unsigned short __DEFAULT_FN_ATTRS256_CONSTEXPR
2971	2866	_mm256_mask_reduce_max_epu16(__mmask16 __M, __m256i __V) {
2972	2867	__V = _mm256_maskz_mov_epi16(__M, __V);
2973	2868	return __builtin_reduce_max((__v16hu)__V);
2974	2869	}
2975	2870
2976		static __inline__ short __DEFAULT_FN_ATTRS256
	2871	static __inline__ short __DEFAULT_FN_ATTRS256_CONSTEXPR
2977	2872	_mm256_mask_reduce_min_epi16(__mmask16 __M, __m256i __V) {
2978	2873	__V = _mm256_mask_mov_epi16(_mm256_set1_epi16(32767), __M, __V);
2979	2874	return __builtin_reduce_min((__v16hi)__V);
2980	2875	}
2981	2876
2982		static __inline__ unsigned short __DEFAULT_FN_ATTRS256
	2877	static __inline__ unsigned short __DEFAULT_FN_ATTRS256_CONSTEXPR
2983	2878	_mm256_mask_reduce_min_epu16(__mmask16 __M, __m256i __V) {
2984	2879	__V = _mm256_mask_mov_epi16(_mm256_set1_epi16(-1), __M, __V);
2985	2880	return __builtin_reduce_min((__v16hu)__V);
2986	2881	}
2987	2882
2988		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2883	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
2989	2884	_mm_reduce_add_epi8(__m128i __W) {
2990	2885	return __builtin_reduce_add((__v16qs)__W);
2991	2886	}
2992	2887
2993		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2888	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
2994	2889	_mm_reduce_mul_epi8(__m128i __W) {
2995	2890	return __builtin_reduce_mul((__v16qs)__W);
2996	2891	}
2997	2892
2998		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2893	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
2999	2894	_mm_reduce_and_epi8(__m128i __W) {
3000	2895	return __builtin_reduce_and((__v16qs)__W);
3001	2896	}
3002	2897
3003		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2898	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3004	2899	_mm_reduce_or_epi8(__m128i __W) {
3005	2900	return __builtin_reduce_or((__v16qs)__W);
3006	2901	}
3007	2902
3008		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2903	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3009	2904	_mm_mask_reduce_add_epi8(__mmask16 __M, __m128i __W) {
3010	2905	__W = _mm_maskz_mov_epi8(__M, __W);
3011	2906	return __builtin_reduce_add((__v16qs)__W);
3012	2907	}
3013	2908
3014		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2909	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3015	2910	_mm_mask_reduce_mul_epi8(__mmask16 __M, __m128i __W) {
3016	2911	__W = _mm_mask_mov_epi8(_mm_set1_epi8(1), __M, __W);
3017	2912	return __builtin_reduce_mul((__v16qs)__W);
3018	2913	}
3019	2914
3020		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2915	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3021	2916	_mm_mask_reduce_and_epi8(__mmask16 __M, __m128i __W) {
3022	2917	__W = _mm_mask_mov_epi8(_mm_set1_epi8(-1), __M, __W);
3023	2918	return __builtin_reduce_and((__v16qs)__W);
3024	2919	}
3025	2920
3026		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2921	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3027	2922	_mm_mask_reduce_or_epi8(__mmask16 __M, __m128i __W) {
3028	2923	__W = _mm_maskz_mov_epi8(__M, __W);
3029	2924	return __builtin_reduce_or((__v16qs)__W);
3030	2925	}
3031	2926
3032		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2927	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3033	2928	_mm_reduce_max_epi8(__m128i __V) {
3034	2929	return __builtin_reduce_max((__v16qs)__V);
3035	2930	}
3036	2931
3037		static __inline__ unsigned char __DEFAULT_FN_ATTRS128
	2932	static __inline__ unsigned char __DEFAULT_FN_ATTRS128_CONSTEXPR
3038	2933	_mm_reduce_max_epu8(__m128i __V) {
3039	2934	return __builtin_reduce_max((__v16qu)__V);
3040	2935	}
3041	2936
3042		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2937	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3043	2938	_mm_reduce_min_epi8(__m128i __V) {
3044	2939	return __builtin_reduce_min((__v16qs)__V);
3045	2940	}
3046	2941
3047		static __inline__ unsigned char __DEFAULT_FN_ATTRS128
	2942	static __inline__ unsigned char __DEFAULT_FN_ATTRS128_CONSTEXPR
3048	2943	_mm_reduce_min_epu8(__m128i __V) {
3049	2944	return __builtin_reduce_min((__v16qu)__V);
3050	2945	}
3051	2946
3052		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2947	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3053	2948	_mm_mask_reduce_max_epi8(__mmask16 __M, __m128i __V) {
3054	2949	__V = _mm_mask_mov_epi8(_mm_set1_epi8(-127-1), __M, __V);
3055	2950	return __builtin_reduce_max((__v16qs)__V);
3056	2951	}
3057	2952
3058		static __inline__ unsigned char __DEFAULT_FN_ATTRS128
	2953	static __inline__ unsigned char __DEFAULT_FN_ATTRS128_CONSTEXPR
3059	2954	_mm_mask_reduce_max_epu8(__mmask16 __M, __m128i __V) {
3060	2955	__V = _mm_maskz_mov_epi8(__M, __V);
3061	2956	return __builtin_reduce_max((__v16qu)__V);
3062	2957	}
3063	2958
3064		static __inline__ signed char __DEFAULT_FN_ATTRS128
	2959	static __inline__ signed char __DEFAULT_FN_ATTRS128_CONSTEXPR
3065	2960	_mm_mask_reduce_min_epi8(__mmask16 __M, __m128i __V) {
3066	2961	__V = _mm_mask_mov_epi8(_mm_set1_epi8(127), __M, __V);
3067	2962	return __builtin_reduce_min((__v16qs)__V);
3068	2963	}
3069	2964
3070		static __inline__ unsigned char __DEFAULT_FN_ATTRS128
	2965	static __inline__ unsigned char __DEFAULT_FN_ATTRS128_CONSTEXPR
3071	2966	_mm_mask_reduce_min_epu8(__mmask16 __M, __m128i __V) {
3072	2967	__V = _mm_mask_mov_epi8(_mm_set1_epi8(-1), __M, __V);
3073	2968	return __builtin_reduce_min((__v16qu)__V);
3074	2969	}
3075	2970
3076		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2971	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3077	2972	_mm256_reduce_add_epi8(__m256i __W) {
3078	2973	return __builtin_reduce_add((__v32qs)__W);
3079	2974	}
3080	2975
3081		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2976	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3082	2977	_mm256_reduce_mul_epi8(__m256i __W) {
3083	2978	return __builtin_reduce_mul((__v32qs)__W);
3084	2979	}
3085	2980
3086		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2981	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3087	2982	_mm256_reduce_and_epi8(__m256i __W) {
3088	2983	return __builtin_reduce_and((__v32qs)__W);
3089	2984	}
3090	2985
3091		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2986	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3092	2987	_mm256_reduce_or_epi8(__m256i __W) {
3093	2988	return __builtin_reduce_or((__v32qs)__W);
3094	2989	}
3095	2990
3096		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2991	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3097	2992	_mm256_mask_reduce_add_epi8(__mmask32 __M, __m256i __W) {
3098	2993	__W = _mm256_maskz_mov_epi8(__M, __W);
3099	2994	return __builtin_reduce_add((__v32qs)__W);
3100	2995	}
3101	2996
3102		static __inline__ signed char __DEFAULT_FN_ATTRS256
	2997	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3103	2998	_mm256_mask_reduce_mul_epi8(__mmask32 __M, __m256i __W) {
3104	2999	__W = _mm256_mask_mov_epi8(_mm256_set1_epi8(1), __M, __W);
3105	3000	return __builtin_reduce_mul((__v32qs)__W);
3106	3001	}
3107	3002
3108		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3003	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3109	3004	_mm256_mask_reduce_and_epi8(__mmask32 __M, __m256i __W) {
3110	3005	__W = _mm256_mask_mov_epi8(_mm256_set1_epi8(-1), __M, __W);
3111	3006	return __builtin_reduce_and((__v32qs)__W);
3112	3007	}
3113	3008
3114		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3009	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3115	3010	_mm256_mask_reduce_or_epi8(__mmask32 __M, __m256i __W) {
3116	3011	__W = _mm256_maskz_mov_epi8(__M, __W);
3117	3012	return __builtin_reduce_or((__v32qs)__W);
3118	3013	}
3119	3014
3120		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3015	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3121	3016	_mm256_reduce_max_epi8(__m256i __V) {
3122	3017	return __builtin_reduce_max((__v32qs)__V);
3123	3018	}
3124	3019
3125		static __inline__ unsigned char __DEFAULT_FN_ATTRS256
	3020	static __inline__ unsigned char __DEFAULT_FN_ATTRS256_CONSTEXPR
3126	3021	_mm256_reduce_max_epu8(__m256i __V) {
3127	3022	return __builtin_reduce_max((__v32qu)__V);
3128	3023	}
3129	3024
3130		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3025	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3131	3026	_mm256_reduce_min_epi8(__m256i __V) {
3132	3027	return __builtin_reduce_min((__v32qs)__V);
3133	3028	}
3134	3029
3135		static __inline__ unsigned char __DEFAULT_FN_ATTRS256
	3030	static __inline__ unsigned char __DEFAULT_FN_ATTRS256_CONSTEXPR
3136	3031	_mm256_reduce_min_epu8(__m256i __V) {
3137	3032	return __builtin_reduce_min((__v32qu)__V);
3138	3033	}
3139	3034
3140		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3035	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3141	3036	_mm256_mask_reduce_max_epi8(__mmask32 __M, __m256i __V) {
3142	3037	__V = _mm256_mask_mov_epi8(_mm256_set1_epi8(-127-1), __M, __V);
3143	3038	return __builtin_reduce_max((__v32qs)__V);
3144	3039	}
3145	3040
3146		static __inline__ unsigned char __DEFAULT_FN_ATTRS256
	3041	static __inline__ unsigned char __DEFAULT_FN_ATTRS256_CONSTEXPR
3147	3042	_mm256_mask_reduce_max_epu8(__mmask32 __M, __m256i __V) {
3148	3043	__V = _mm256_maskz_mov_epi8(__M, __V);
3149	3044	return __builtin_reduce_max((__v32qu)__V);
3150	3045	}
3151	3046
3152		static __inline__ signed char __DEFAULT_FN_ATTRS256
	3047	static __inline__ signed char __DEFAULT_FN_ATTRS256_CONSTEXPR
3153	3048	_mm256_mask_reduce_min_epi8(__mmask32 __M, __m256i __V) {
3154	3049	__V = _mm256_mask_mov_epi8(_mm256_set1_epi8(127), __M, __V);
3155	3050	return __builtin_reduce_min((__v32qs)__V);
3156	3051	}
3157	3052
3158		static __inline__ unsigned char __DEFAULT_FN_ATTRS256
	3053	static __inline__ unsigned char __DEFAULT_FN_ATTRS256_CONSTEXPR
3159	3054	_mm256_mask_reduce_min_epu8(__mmask32 __M, __m256i __V) {
3160	3055	__V = _mm256_mask_mov_epi8(_mm256_set1_epi8(-1), __M, __V);
3161	3056	return __builtin_reduce_min((__v32qu)__V);
...	...	@@ -3163,5 +3058,7 @@ _mm256_mask_reduce_min_epu8(__mmask32 __M, __m256i __V) {
3163	3058
3164	3059	#undef __DEFAULT_FN_ATTRS128
3165	3060	#undef __DEFAULT_FN_ATTRS256
	3061	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	3062	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
3166	3063
3167	3064	#endif /* __AVX512VLBWINTRIN_H */

lib/include/avx512vlcdintrin.h+74-102

...	...	@@ -14,211 +14,183 @@
14	14	#define __AVX512VLCDINTRIN_H
15	15
16	16	/* Define the default attributes for the functions in this file. */
	17	#if defined(__cplusplus) && (__cplusplus >= 201103L)
17	18	#define __DEFAULT_FN_ATTRS128 \
18	19	__attribute__((__always_inline__, __nodebug__, \
19		__target__("avx512vl,avx512cd,no-evex512"), \
20		__min_vector_width__(128)))
	20	__target__("avx512vl,avx512cd"), \
	21	__min_vector_width__(128))) constexpr
21	22	#define __DEFAULT_FN_ATTRS256 \
22	23	__attribute__((__always_inline__, __nodebug__, \
23		__target__("avx512vl,avx512cd,no-evex512"), \
24		__min_vector_width__(256)))
	24	__target__("avx512vl,avx512cd"), \
	25	__min_vector_width__(256))) constexpr
	26	#else
	27	#define __DEFAULT_FN_ATTRS128 \
	28	__attribute__((__always_inline__, __nodebug__, \
	29	__target__("avx512vl,avx512cd"), __min_vector_width__(128)))
	30	#define __DEFAULT_FN_ATTRS256 \
	31	__attribute__((__always_inline__, __nodebug__, \
	32	__target__("avx512vl,avx512cd"), __min_vector_width__(256)))
	33	#endif
25	34
26	35	static __inline__ __m128i __DEFAULT_FN_ATTRS128
27		_mm_broadcastmb_epi64 (__mmask8 __A)
28		{
	36	_mm_broadcastmb_epi64(__mmask8 __A) {
29	37	return (__m128i) _mm_set1_epi64x((long long) __A);
30	38	}
31	39
32	40	static __inline__ __m256i __DEFAULT_FN_ATTRS256
33		_mm256_broadcastmb_epi64 (__mmask8 __A)
34		{
35		return (__m256i) _mm256_set1_epi64x((long long)__A);
	41	_mm256_broadcastmb_epi64(__mmask8 __A) {
	42	return (__m256i)_mm256_set1_epi64x((long long)__A);
36	43	}
37	44
38	45	static __inline__ __m128i __DEFAULT_FN_ATTRS128
39		_mm_broadcastmw_epi32 (__mmask16 __A)
40		{
	46	_mm_broadcastmw_epi32(__mmask16 __A) {
41	47	return (__m128i) _mm_set1_epi32((int)__A);
42	48	}
43	49
44	50	static __inline__ __m256i __DEFAULT_FN_ATTRS256
45		_mm256_broadcastmw_epi32 (__mmask16 __A)
46		{
	51	_mm256_broadcastmw_epi32(__mmask16 __A) {
47	52	return (__m256i) _mm256_set1_epi32((int)__A);
48	53	}
49	54
50
51	55	static __inline__ __m128i __DEFAULT_FN_ATTRS128
52		_mm_conflict_epi64 (__m128i __A)
53		{
54		return (__m128i) __builtin_ia32_vpconflictdi_128 ((__v2di) __A);
	56	_mm_conflict_epi64(__m128i __A) {
	57	return (__m128i)__builtin_ia32_vpconflictdi_128((__v2di)__A);
55	58	}
56	59
57	60	static __inline__ __m128i __DEFAULT_FN_ATTRS128
58		_mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A)
59		{
60		return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
61		(__v2di)_mm_conflict_epi64(__A),
62		(__v2di)__W);
	61	_mm_mask_conflict_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
	62	return (__m128i)__builtin_ia32_selectq_128(
	63	(__mmask8)__U, (__v2di)_mm_conflict_epi64(__A), (__v2di)__W);
63	64	}
64	65
65	66	static __inline__ __m128i __DEFAULT_FN_ATTRS128
66		_mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A)
67		{
	67	_mm_maskz_conflict_epi64(__mmask8 __U, __m128i __A) {
68	68	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
69	69	(__v2di)_mm_conflict_epi64(__A),
70	70	(__v2di)_mm_setzero_si128());
71	71	}
72	72
73	73	static __inline__ __m256i __DEFAULT_FN_ATTRS256
74		_mm256_conflict_epi64 (__m256i __A)
75		{
76		return (__m256i) __builtin_ia32_vpconflictdi_256 ((__v4di) __A);
	74	_mm256_conflict_epi64(__m256i __A) {
	75	return (__m256i)__builtin_ia32_vpconflictdi_256((__v4di)__A);
77	76	}
78	77
79	78	static __inline__ __m256i __DEFAULT_FN_ATTRS256
80		_mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A)
81		{
82		return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
83		(__v4di)_mm256_conflict_epi64(__A),
84		(__v4di)__W);
	79	_mm256_mask_conflict_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
	80	return (__m256i)__builtin_ia32_selectq_256(
	81	(__mmask8)__U, (__v4di)_mm256_conflict_epi64(__A), (__v4di)__W);
85	82	}
86	83
87	84	static __inline__ __m256i __DEFAULT_FN_ATTRS256
88		_mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A)
89		{
	85	_mm256_maskz_conflict_epi64(__mmask8 __U, __m256i __A) {
90	86	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
91	87	(__v4di)_mm256_conflict_epi64(__A),
92	88	(__v4di)_mm256_setzero_si256());
93	89	}
94	90
95	91	static __inline__ __m128i __DEFAULT_FN_ATTRS128
96		_mm_conflict_epi32 (__m128i __A)
97		{
98		return (__m128i) __builtin_ia32_vpconflictsi_128 ((__v4si) __A);
	92	_mm_conflict_epi32(__m128i __A) {
	93	return (__m128i)__builtin_ia32_vpconflictsi_128((__v4si)__A);
99	94	}
100	95
101	96	static __inline__ __m128i __DEFAULT_FN_ATTRS128
102		_mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A)
103		{
104		return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
105		(__v4si)_mm_conflict_epi32(__A),
106		(__v4si)__W);
	97	_mm_mask_conflict_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
	98	return (__m128i)__builtin_ia32_selectd_128(
	99	(__mmask8)__U, (__v4si)_mm_conflict_epi32(__A), (__v4si)__W);
107	100	}
108	101
109	102	static __inline__ __m128i __DEFAULT_FN_ATTRS128
110		_mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A)
111		{
	103	_mm_maskz_conflict_epi32(__mmask8 __U, __m128i __A) {
112	104	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
113	105	(__v4si)_mm_conflict_epi32(__A),
114	106	(__v4si)_mm_setzero_si128());
115	107	}
116	108
117	109	static __inline__ __m256i __DEFAULT_FN_ATTRS256
118		_mm256_conflict_epi32 (__m256i __A)
119		{
120		return (__m256i) __builtin_ia32_vpconflictsi_256 ((__v8si) __A);
	110	_mm256_conflict_epi32(__m256i __A) {
	111	return (__m256i)__builtin_ia32_vpconflictsi_256((__v8si)__A);
121	112	}
122	113
123	114	static __inline__ __m256i __DEFAULT_FN_ATTRS256
124		_mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A)
125		{
126		return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
127		(__v8si)_mm256_conflict_epi32(__A),
128		(__v8si)__W);
	115	_mm256_mask_conflict_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
	116	return (__m256i)__builtin_ia32_selectd_256(
	117	(__mmask8)__U, (__v8si)_mm256_conflict_epi32(__A), (__v8si)__W);
129	118	}
130	119
131	120	static __inline__ __m256i __DEFAULT_FN_ATTRS256
132		_mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A)
133		{
	121	_mm256_maskz_conflict_epi32(__mmask8 __U, __m256i __A) {
134	122	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
135	123	(__v8si)_mm256_conflict_epi32(__A),
136	124	(__v8si)_mm256_setzero_si256());
137	125	}
138	126
139		static __inline__ __m128i __DEFAULT_FN_ATTRS128
140		_mm_lzcnt_epi32 (__m128i __A)
141		{
142		return (__m128i) __builtin_ia32_vplzcntd_128 ((__v4si) __A);
	127	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_lzcnt_epi32(__m128i __A) {
	128	return (__m128i)__builtin_elementwise_clzg((__v4si)__A,
	129	(__v4si)_mm_set1_epi32(32));
143	130	}
144	131
145	132	static __inline__ __m128i __DEFAULT_FN_ATTRS128
146		_mm_mask_lzcnt_epi32 (__m128i __W, __mmask8 __U, __m128i __A)
147		{
148		return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
149		(__v4si)_mm_lzcnt_epi32(__A),
150		(__v4si)__W);
	133	_mm_mask_lzcnt_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
	134	return (__m128i)__builtin_ia32_selectd_128(
	135	(__mmask8)__U, (__v4si)_mm_lzcnt_epi32(__A), (__v4si)__W);
151	136	}
152	137
153	138	static __inline__ __m128i __DEFAULT_FN_ATTRS128
154		_mm_maskz_lzcnt_epi32 (__mmask8 __U, __m128i __A)
155		{
156		return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
157		(__v4si)_mm_lzcnt_epi32(__A),
158		(__v4si)_mm_setzero_si128());
	139	_mm_maskz_lzcnt_epi32(__mmask8 __U, __m128i __A) {
	140	return (__m128i)__builtin_ia32_selectd_128(
	141	(__mmask8)__U, (__v4si)_mm_lzcnt_epi32(__A), (__v4si)_mm_setzero_si128());
159	142	}
160	143
161	144	static __inline__ __m256i __DEFAULT_FN_ATTRS256
162		_mm256_lzcnt_epi32 (__m256i __A)
163		{
164		return (__m256i) __builtin_ia32_vplzcntd_256 ((__v8si) __A);
	145	_mm256_lzcnt_epi32(__m256i __A) {
	146	return (__m256i)__builtin_elementwise_clzg((__v8si)__A,
	147	(__v8si)_mm256_set1_epi32(32));
165	148	}
166	149
167	150	static __inline__ __m256i __DEFAULT_FN_ATTRS256
168		_mm256_mask_lzcnt_epi32 (__m256i __W, __mmask8 __U, __m256i __A)
169		{
170		return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
171		(__v8si)_mm256_lzcnt_epi32(__A),
172		(__v8si)__W);
	151	_mm256_mask_lzcnt_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
	152	return (__m256i)__builtin_ia32_selectd_256(
	153	(__mmask8)__U, (__v8si)_mm256_lzcnt_epi32(__A), (__v8si)__W);
173	154	}
174	155
175	156	static __inline__ __m256i __DEFAULT_FN_ATTRS256
176		_mm256_maskz_lzcnt_epi32 (__mmask8 __U, __m256i __A)
177		{
	157	_mm256_maskz_lzcnt_epi32(__mmask8 __U, __m256i __A) {
178	158	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
179	159	(__v8si)_mm256_lzcnt_epi32(__A),
180	160	(__v8si)_mm256_setzero_si256());
181	161	}
182	162
183		static __inline__ __m128i __DEFAULT_FN_ATTRS128
184		_mm_lzcnt_epi64 (__m128i __A)
185		{
186		return (__m128i) __builtin_ia32_vplzcntq_128 ((__v2di) __A);
	163	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_lzcnt_epi64(__m128i __A) {
	164	return (__m128i)__builtin_elementwise_clzg(
	165	(__v2di)__A, (__v2di)_mm_set1_epi64x((long long)64));
187	166	}
188	167
189	168	static __inline__ __m128i __DEFAULT_FN_ATTRS128
190		_mm_mask_lzcnt_epi64 (__m128i __W, __mmask8 __U, __m128i __A)
191		{
192		return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
193		(__v2di)_mm_lzcnt_epi64(__A),
194		(__v2di)__W);
	169	_mm_mask_lzcnt_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
	170	return (__m128i)__builtin_ia32_selectq_128(
	171	(__mmask8)__U, (__v2di)_mm_lzcnt_epi64(__A), (__v2di)__W);
195	172	}
196	173
197	174	static __inline__ __m128i __DEFAULT_FN_ATTRS128
198		_mm_maskz_lzcnt_epi64 (__mmask8 __U, __m128i __A)
199		{
200		return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
201		(__v2di)_mm_lzcnt_epi64(__A),
202		(__v2di)_mm_setzero_si128());
	175	_mm_maskz_lzcnt_epi64(__mmask8 __U, __m128i __A) {
	176	return (__m128i)__builtin_ia32_selectq_128(
	177	(__mmask8)__U, (__v2di)_mm_lzcnt_epi64(__A), (__v2di)_mm_setzero_si128());
203	178	}
204	179
205	180	static __inline__ __m256i __DEFAULT_FN_ATTRS256
206		_mm256_lzcnt_epi64 (__m256i __A)
207		{
208		return (__m256i) __builtin_ia32_vplzcntq_256 ((__v4di) __A);
	181	_mm256_lzcnt_epi64(__m256i __A) {
	182	return (__m256i)__builtin_elementwise_clzg(
	183	(__v4di)__A, (__v4di)_mm256_set1_epi64x((long long)64));
209	184	}
210	185
211	186	static __inline__ __m256i __DEFAULT_FN_ATTRS256
212		_mm256_mask_lzcnt_epi64 (__m256i __W, __mmask8 __U, __m256i __A)
213		{
214		return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
215		(__v4di)_mm256_lzcnt_epi64(__A),
216		(__v4di)__W);
	187	_mm256_mask_lzcnt_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
	188	return (__m256i)__builtin_ia32_selectq_256(
	189	(__mmask8)__U, (__v4di)_mm256_lzcnt_epi64(__A), (__v4di)__W);
217	190	}
218	191
219	192	static __inline__ __m256i __DEFAULT_FN_ATTRS256
220		_mm256_maskz_lzcnt_epi64 (__mmask8 __U, __m256i __A)
221		{
	193	_mm256_maskz_lzcnt_epi64(__mmask8 __U, __m256i __A) {
222	194	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
223	195	(__v4di)_mm256_lzcnt_epi64(__A),
224	196	(__v4di)_mm256_setzero_si256());

lib/include/avx512vldqintrin.h+110-127

...	...	@@ -17,45 +17,51 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS128 \
19	19	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512dq,no-evex512"), \
21		__min_vector_width__(128)))
	20	__target__("avx512vl,avx512dq"), __min_vector_width__(128)))
22	21	#define __DEFAULT_FN_ATTRS256 \
23	22	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512dq,no-evex512"), \
25		__min_vector_width__(256)))
	23	__target__("avx512vl,avx512dq"), __min_vector_width__(256)))
	24
	25	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	26	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	27	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	28	#else
	29	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	30	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	31	#endif
26	32
27		static __inline__ __m256i __DEFAULT_FN_ATTRS256
28		_mm256_mullo_epi64 (__m256i __A, __m256i __B) {
	33	static __inline__ __m256i __DEFAULT_FN_ATTRS128_CONSTEXPR
	34	_mm256_mullo_epi64(__m256i __A, __m256i __B) {
29	35	return (__m256i) ((__v4du) __A * (__v4du) __B);
30	36	}
31	37
32		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	38	static __inline__ __m256i __DEFAULT_FN_ATTRS128_CONSTEXPR
33	39	_mm256_mask_mullo_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
34	40	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
35	41	(__v4di)_mm256_mullo_epi64(__A, __B),
36	42	(__v4di)__W);
37	43	}
38	44
39		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	45	static __inline__ __m256i __DEFAULT_FN_ATTRS128_CONSTEXPR
40	46	_mm256_maskz_mullo_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
41	47	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
42	48	(__v4di)_mm256_mullo_epi64(__A, __B),
43	49	(__v4di)_mm256_setzero_si256());
44	50	}
45	51
46		static __inline__ __m128i __DEFAULT_FN_ATTRS128
47		_mm_mullo_epi64 (__m128i __A, __m128i __B) {
	52	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	53	_mm_mullo_epi64(__m128i __A, __m128i __B) {
48	54	return (__m128i) ((__v2du) __A * (__v2du) __B);
49	55	}
50	56
51		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	57	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
52	58	_mm_mask_mullo_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
53	59	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
54	60	(__v2di)_mm_mullo_epi64(__A, __B),
55	61	(__v2di)__W);
56	62	}
57	63
58		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	64	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
59	65	_mm_maskz_mullo_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
60	66	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
61	67	(__v2di)_mm_mullo_epi64(__A, __B),
...	...	@@ -454,39 +460,39 @@ _mm256_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) {
454	460	(__mmask8) __U);
455	461	}
456	462
457		static __inline__ __m128d __DEFAULT_FN_ATTRS128
458		_mm_cvtepi64_pd (__m128i __A) {
	463	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	464	_mm_cvtepi64_pd(__m128i __A) {
459	465	return (__m128d)__builtin_convertvector((__v2di)__A, __v2df);
460	466	}
461	467
462		static __inline__ __m128d __DEFAULT_FN_ATTRS128
463		_mm_mask_cvtepi64_pd (__m128d __W, __mmask8 __U, __m128i __A) {
	468	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	469	_mm_mask_cvtepi64_pd(__m128d __W, __mmask8 __U, __m128i __A) {
464	470	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
465	471	(__v2df)_mm_cvtepi64_pd(__A),
466	472	(__v2df)__W);
467	473	}
468	474
469		static __inline__ __m128d __DEFAULT_FN_ATTRS128
470		_mm_maskz_cvtepi64_pd (__mmask8 __U, __m128i __A) {
	475	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	476	_mm_maskz_cvtepi64_pd(__mmask8 __U, __m128i __A) {
471	477	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
472	478	(__v2df)_mm_cvtepi64_pd(__A),
473	479	(__v2df)_mm_setzero_pd());
474	480	}
475	481
476		static __inline__ __m256d __DEFAULT_FN_ATTRS256
477		_mm256_cvtepi64_pd (__m256i __A) {
	482	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	483	_mm256_cvtepi64_pd(__m256i __A) {
478	484	return (__m256d)__builtin_convertvector((__v4di)__A, __v4df);
479	485	}
480	486
481		static __inline__ __m256d __DEFAULT_FN_ATTRS256
482		_mm256_mask_cvtepi64_pd (__m256d __W, __mmask8 __U, __m256i __A) {
	487	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	488	_mm256_mask_cvtepi64_pd(__m256d __W, __mmask8 __U, __m256i __A) {
483	489	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
484	490	(__v4df)_mm256_cvtepi64_pd(__A),
485	491	(__v4df)__W);
486	492	}
487	493
488		static __inline__ __m256d __DEFAULT_FN_ATTRS256
489		_mm256_maskz_cvtepi64_pd (__mmask8 __U, __m256i __A) {
	494	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	495	_mm256_maskz_cvtepi64_pd(__mmask8 __U, __m256i __A) {
490	496	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
491	497	(__v4df)_mm256_cvtepi64_pd(__A),
492	498	(__v4df)_mm256_setzero_pd());
...	...	@@ -513,20 +519,20 @@ _mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) {
513	519	(__mmask8) __U);
514	520	}
515	521
516		static __inline__ __m128 __DEFAULT_FN_ATTRS256
517		_mm256_cvtepi64_ps (__m256i __A) {
	522	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	523	_mm256_cvtepi64_ps(__m256i __A) {
518	524	return (__m128)__builtin_convertvector((__v4di)__A, __v4sf);
519	525	}
520	526
521		static __inline__ __m128 __DEFAULT_FN_ATTRS256
522		_mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
	527	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	528	_mm256_mask_cvtepi64_ps(__m128 __W, __mmask8 __U, __m256i __A) {
523	529	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
524	530	(__v4sf)_mm256_cvtepi64_ps(__A),
525	531	(__v4sf)__W);
526	532	}
527	533
528		static __inline__ __m128 __DEFAULT_FN_ATTRS256
529		_mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) {
	534	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	535	_mm256_maskz_cvtepi64_ps(__mmask8 __U, __m256i __A) {
530	536	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
531	537	(__v4sf)_mm256_cvtepi64_ps(__A),
532	538	(__v4sf)_mm_setzero_ps());
...	...	@@ -700,39 +706,39 @@ _mm256_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) {
700	706	(__mmask8) __U);
701	707	}
702	708
703		static __inline__ __m128d __DEFAULT_FN_ATTRS128
704		_mm_cvtepu64_pd (__m128i __A) {
	709	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	710	_mm_cvtepu64_pd(__m128i __A) {
705	711	return (__m128d)__builtin_convertvector((__v2du)__A, __v2df);
706	712	}
707	713
708		static __inline__ __m128d __DEFAULT_FN_ATTRS128
709		_mm_mask_cvtepu64_pd (__m128d __W, __mmask8 __U, __m128i __A) {
	714	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	715	_mm_mask_cvtepu64_pd(__m128d __W, __mmask8 __U, __m128i __A) {
710	716	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
711	717	(__v2df)_mm_cvtepu64_pd(__A),
712	718	(__v2df)__W);
713	719	}
714	720
715		static __inline__ __m128d __DEFAULT_FN_ATTRS128
716		_mm_maskz_cvtepu64_pd (__mmask8 __U, __m128i __A) {
	721	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	722	_mm_maskz_cvtepu64_pd(__mmask8 __U, __m128i __A) {
717	723	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
718	724	(__v2df)_mm_cvtepu64_pd(__A),
719	725	(__v2df)_mm_setzero_pd());
720	726	}
721	727
722		static __inline__ __m256d __DEFAULT_FN_ATTRS256
723		_mm256_cvtepu64_pd (__m256i __A) {
	728	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	729	_mm256_cvtepu64_pd(__m256i __A) {
724	730	return (__m256d)__builtin_convertvector((__v4du)__A, __v4df);
725	731	}
726	732
727		static __inline__ __m256d __DEFAULT_FN_ATTRS256
728		_mm256_mask_cvtepu64_pd (__m256d __W, __mmask8 __U, __m256i __A) {
	733	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	734	_mm256_mask_cvtepu64_pd(__m256d __W, __mmask8 __U, __m256i __A) {
729	735	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
730	736	(__v4df)_mm256_cvtepu64_pd(__A),
731	737	(__v4df)__W);
732	738	}
733	739
734		static __inline__ __m256d __DEFAULT_FN_ATTRS256
735		_mm256_maskz_cvtepu64_pd (__mmask8 __U, __m256i __A) {
	740	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	741	_mm256_maskz_cvtepu64_pd(__mmask8 __U, __m256i __A) {
736	742	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
737	743	(__v4df)_mm256_cvtepu64_pd(__A),
738	744	(__v4df)_mm256_setzero_pd());
...	...	@@ -759,20 +765,20 @@ _mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) {
759	765	(__mmask8) __U);
760	766	}
761	767
762		static __inline__ __m128 __DEFAULT_FN_ATTRS256
763		_mm256_cvtepu64_ps (__m256i __A) {
	768	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	769	_mm256_cvtepu64_ps(__m256i __A) {
764	770	return (__m128)__builtin_convertvector((__v4du)__A, __v4sf);
765	771	}
766	772
767		static __inline__ __m128 __DEFAULT_FN_ATTRS256
768		_mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) {
	773	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	774	_mm256_mask_cvtepu64_ps(__m128 __W, __mmask8 __U, __m256i __A) {
769	775	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
770	776	(__v4sf)_mm256_cvtepu64_ps(__A),
771	777	(__v4sf)__W);
772	778	}
773	779
774		static __inline__ __m128 __DEFAULT_FN_ATTRS256
775		_mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) {
	780	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	781	_mm256_maskz_cvtepu64_ps(__mmask8 __U, __m256i __A) {
776	782	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
777	783	(__v4sf)_mm256_cvtepu64_ps(__A),
778	784	(__v4sf)_mm_setzero_ps());
...	...	@@ -908,174 +914,150 @@ _mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) {
908	914	(__v8sf)_mm256_setzero_ps(), \
909	915	(__mmask8)(U)))
910	916
911		static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
912		_mm_movepi32_mask (__m128i __A)
913		{
	917	static __inline__ __mmask8
	918	__DEFAULT_FN_ATTRS128_CONSTEXPR _mm_movepi32_mask(__m128i __A) {
914	919	return (__mmask8) __builtin_ia32_cvtd2mask128 ((__v4si) __A);
915	920	}
916	921
917		static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
918		_mm256_movepi32_mask (__m256i __A)
919		{
	922	static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
	923	_mm256_movepi32_mask(__m256i __A) {
920	924	return (__mmask8) __builtin_ia32_cvtd2mask256 ((__v8si) __A);
921	925	}
922	926
923		static __inline__ __m128i __DEFAULT_FN_ATTRS128
924		_mm_movm_epi32 (__mmask8 __A)
925		{
	927	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	928	_mm_movm_epi32(__mmask8 __A) {
926	929	return (__m128i) __builtin_ia32_cvtmask2d128 (__A);
927	930	}
928	931
929		static __inline__ __m256i __DEFAULT_FN_ATTRS256
930		_mm256_movm_epi32 (__mmask8 __A)
931		{
	932	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	933	_mm256_movm_epi32(__mmask8 __A) {
932	934	return (__m256i) __builtin_ia32_cvtmask2d256 (__A);
933	935	}
934	936
935		static __inline__ __m128i __DEFAULT_FN_ATTRS128
936		_mm_movm_epi64 (__mmask8 __A)
937		{
	937	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	938	_mm_movm_epi64(__mmask8 __A) {
938	939	return (__m128i) __builtin_ia32_cvtmask2q128 (__A);
939	940	}
940	941
941		static __inline__ __m256i __DEFAULT_FN_ATTRS256
942		_mm256_movm_epi64 (__mmask8 __A)
943		{
	942	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	943	_mm256_movm_epi64(__mmask8 __A) {
944	944	return (__m256i) __builtin_ia32_cvtmask2q256 (__A);
945	945	}
946	946
947		static __inline__ __mmask8 __DEFAULT_FN_ATTRS128
948		_mm_movepi64_mask (__m128i __A)
949		{
	947	static __inline__ __mmask8 __DEFAULT_FN_ATTRS128_CONSTEXPR
	948	_mm_movepi64_mask(__m128i __A) {
950	949	return (__mmask8) __builtin_ia32_cvtq2mask128 ((__v2di) __A);
951	950	}
952	951
953		static __inline__ __mmask8 __DEFAULT_FN_ATTRS256
954		_mm256_movepi64_mask (__m256i __A)
955		{
	952	static __inline__ __mmask8 __DEFAULT_FN_ATTRS256_CONSTEXPR
	953	_mm256_movepi64_mask(__m256i __A) {
956	954	return (__mmask8) __builtin_ia32_cvtq2mask256 ((__v4di) __A);
957	955	}
958	956
959		static __inline__ __m256 __DEFAULT_FN_ATTRS256
960		_mm256_broadcast_f32x2 (__m128 __A)
961		{
	957	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	958	_mm256_broadcast_f32x2(__m128 __A) {
962	959	return (__m256)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
963	960	0, 1, 0, 1, 0, 1, 0, 1);
964	961	}
965	962
966		static __inline__ __m256 __DEFAULT_FN_ATTRS256
967		_mm256_mask_broadcast_f32x2 (__m256 __O, __mmask8 __M, __m128 __A)
968		{
	963	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	964	_mm256_mask_broadcast_f32x2(__m256 __O, __mmask8 __M, __m128 __A) {
969	965	return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
970	966	(__v8sf)_mm256_broadcast_f32x2(__A),
971	967	(__v8sf)__O);
972	968	}
973	969
974		static __inline__ __m256 __DEFAULT_FN_ATTRS256
975		_mm256_maskz_broadcast_f32x2 (__mmask8 __M, __m128 __A)
976		{
	970	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	971	_mm256_maskz_broadcast_f32x2(__mmask8 __M, __m128 __A) {
977	972	return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
978	973	(__v8sf)_mm256_broadcast_f32x2(__A),
979	974	(__v8sf)_mm256_setzero_ps());
980	975	}
981	976
982		static __inline__ __m256d __DEFAULT_FN_ATTRS256
983		_mm256_broadcast_f64x2(__m128d __A)
984		{
	977	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	978	_mm256_broadcast_f64x2(__m128d __A) {
985	979	return (__m256d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,
986	980	0, 1, 0, 1);
987	981	}
988	982
989		static __inline__ __m256d __DEFAULT_FN_ATTRS256
990		_mm256_mask_broadcast_f64x2(__m256d __O, __mmask8 __M, __m128d __A)
991		{
	983	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	984	_mm256_mask_broadcast_f64x2(__m256d __O, __mmask8 __M, __m128d __A) {
992	985	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__M,
993	986	(__v4df)_mm256_broadcast_f64x2(__A),
994	987	(__v4df)__O);
995	988	}
996	989
997		static __inline__ __m256d __DEFAULT_FN_ATTRS256
998		_mm256_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A)
999		{
	990	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	991	_mm256_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A) {
1000	992	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__M,
1001	993	(__v4df)_mm256_broadcast_f64x2(__A),
1002	994	(__v4df)_mm256_setzero_pd());
1003	995	}
1004	996
1005		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1006		_mm_broadcast_i32x2 (__m128i __A)
1007		{
	997	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	998	_mm_broadcast_i32x2(__m128i __A) {
1008	999	return (__m128i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
1009	1000	0, 1, 0, 1);
1010	1001	}
1011	1002
1012		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1013		_mm_mask_broadcast_i32x2 (__m128i __O, __mmask8 __M, __m128i __A)
1014		{
	1003	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1004	_mm_mask_broadcast_i32x2(__m128i __O, __mmask8 __M, __m128i __A) {
1015	1005	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
1016	1006	(__v4si)_mm_broadcast_i32x2(__A),
1017	1007	(__v4si)__O);
1018	1008	}
1019	1009
1020		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1021		_mm_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
1022		{
	1010	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1011	_mm_maskz_broadcast_i32x2(__mmask8 __M, __m128i __A) {
1023	1012	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
1024	1013	(__v4si)_mm_broadcast_i32x2(__A),
1025	1014	(__v4si)_mm_setzero_si128());
1026	1015	}
1027	1016
1028		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1029		_mm256_broadcast_i32x2 (__m128i __A)
1030		{
	1017	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1018	_mm256_broadcast_i32x2(__m128i __A) {
1031	1019	return (__m256i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
1032	1020	0, 1, 0, 1, 0, 1, 0, 1);
1033	1021	}
1034	1022
1035		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1036		_mm256_mask_broadcast_i32x2 (__m256i __O, __mmask8 __M, __m128i __A)
1037		{
	1023	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1024	_mm256_mask_broadcast_i32x2(__m256i __O, __mmask8 __M, __m128i __A) {
1038	1025	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
1039	1026	(__v8si)_mm256_broadcast_i32x2(__A),
1040	1027	(__v8si)__O);
1041	1028	}
1042	1029
1043		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1044		_mm256_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
1045		{
	1030	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1031	_mm256_maskz_broadcast_i32x2(__mmask8 __M, __m128i __A) {
1046	1032	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
1047	1033	(__v8si)_mm256_broadcast_i32x2(__A),
1048	1034	(__v8si)_mm256_setzero_si256());
1049	1035	}
1050	1036
1051		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1052		_mm256_broadcast_i64x2(__m128i __A)
1053		{
	1037	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1038	_mm256_broadcast_i64x2(__m128i __A) {
1054	1039	return (__m256i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,
1055	1040	0, 1, 0, 1);
1056	1041	}
1057	1042
1058		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1059		_mm256_mask_broadcast_i64x2(__m256i __O, __mmask8 __M, __m128i __A)
1060		{
	1043	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1044	_mm256_mask_broadcast_i64x2(__m256i __O, __mmask8 __M, __m128i __A) {
1061	1045	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
1062	1046	(__v4di)_mm256_broadcast_i64x2(__A),
1063	1047	(__v4di)__O);
1064	1048	}
1065	1049
1066		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1067		_mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A)
1068		{
	1050	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1051	_mm256_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A) {
1069	1052	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
1070	1053	(__v4di)_mm256_broadcast_i64x2(__A),
1071	1054	(__v4di)_mm256_setzero_si256());
1072	1055	}
1073	1056
1074		#define _mm256_extractf64x2_pd(A, imm) \
1075		((__m128d)__builtin_ia32_extractf64x2_256_mask((__v4df)(__m256d)(A), \
1076		(int)(imm), \
1077		(__v2df)_mm_undefined_pd(), \
1078		(__mmask8)-1))
	1057	#define _mm256_extractf64x2_pd(A, imm) \
	1058	((__m128d)__builtin_ia32_extractf64x2_256_mask( \
	1059	(__v4df)(__m256d)(A), (int)(imm), (__v2df)_mm_setzero_pd(), \
	1060	(__mmask8) - 1))
1079	1061
1080	1062	#define _mm256_mask_extractf64x2_pd(W, U, A, imm) \
1081	1063	((__m128d)__builtin_ia32_extractf64x2_256_mask((__v4df)(__m256d)(A), \
...	...	@@ -1089,11 +1071,10 @@ _mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A)
1089	1071	(__v2df)_mm_setzero_pd(), \
1090	1072	(__mmask8)(U)))
1091	1073
1092		#define _mm256_extracti64x2_epi64(A, imm) \
1093		((__m128i)__builtin_ia32_extracti64x2_256_mask((__v4di)(__m256i)(A), \
1094		(int)(imm), \
1095		(__v2di)_mm_undefined_si128(), \
1096		(__mmask8)-1))
	1074	#define _mm256_extracti64x2_epi64(A, imm) \
	1075	((__m128i)__builtin_ia32_extracti64x2_256_mask( \
	1076	(__v4di)(__m256i)(A), (int)(imm), (__v2di)_mm_setzero_si128(), \
	1077	(__mmask8) - 1))
1097	1078
1098	1079	#define _mm256_mask_extracti64x2_epi64(W, U, A, imm) \
1099	1080	((__m128i)__builtin_ia32_extracti64x2_256_mask((__v4di)(__m256i)(A), \
...	...	@@ -1169,5 +1150,7 @@ _mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A)
1169	1150
1170	1151	#undef __DEFAULT_FN_ATTRS128
1171	1152	#undef __DEFAULT_FN_ATTRS256
	1153	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
	1154	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
1172	1155
1173	1156	#endif

lib/include/avx512vlfp16intrin.h+130-111

...	...	@@ -19,51 +19,64 @@
19	19	/* Define the default attributes for the functions in this file. */
20	20	#define __DEFAULT_FN_ATTRS256 \
21	21	__attribute__((__always_inline__, __nodebug__, \
22		__target__("avx512fp16,avx512vl,no-evex512"), \
	22	__target__("avx512fp16,avx512vl"), \
23	23	__min_vector_width__(256)))
24	24	#define __DEFAULT_FN_ATTRS128 \
25	25	__attribute__((__always_inline__, __nodebug__, \
26		__target__("avx512fp16,avx512vl,no-evex512"), \
	26	__target__("avx512fp16,avx512vl"), \
27	27	__min_vector_width__(128)))
28	28
29		static __inline__ _Float16 __DEFAULT_FN_ATTRS128 _mm_cvtsh_h(__m128h __a) {
	29	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	30	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	31	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	32	#else
	33	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	34	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	35	#endif
	36
	37	static __inline__ _Float16 __DEFAULT_FN_ATTRS128_CONSTEXPR
	38	_mm_cvtsh_h(__m128h __a) {
30	39	return __a[0];
31	40	}
32	41
33		static __inline__ _Float16 __DEFAULT_FN_ATTRS256 _mm256_cvtsh_h(__m256h __a) {
	42	static __inline__ _Float16 __DEFAULT_FN_ATTRS256_CONSTEXPR
	43	_mm256_cvtsh_h(__m256h __a) {
34	44	return __a[0];
35	45	}
36	46
37		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_set_sh(_Float16 __h) {
	47	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	48	_mm_set_sh(_Float16 __h) {
38	49	return __extension__(__m128h){__h, 0, 0, 0, 0, 0, 0, 0};
39	50	}
40	51
41		static __inline __m128h __DEFAULT_FN_ATTRS128 _mm_set1_ph(_Float16 __h) {
	52	static __inline __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	53	_mm_set1_ph(_Float16 __h) {
42	54	return (__m128h)(__v8hf){__h, __h, __h, __h, __h, __h, __h, __h};
43	55	}
44	56
45		static __inline __m256h __DEFAULT_FN_ATTRS256 _mm256_set1_ph(_Float16 __h) {
	57	static __inline __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	58	_mm256_set1_ph(_Float16 __h) {
46	59	return (__m256h)(__v16hf){__h, __h, __h, __h, __h, __h, __h, __h,
47	60	__h, __h, __h, __h, __h, __h, __h, __h};
48	61	}
49	62
50		static __inline __m128h __DEFAULT_FN_ATTRS128
	63	static __inline __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
51	64	_mm_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
52	65	_Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8) {
53	66	return (__m128h)(__v8hf){__h8, __h7, __h6, __h5, __h4, __h3, __h2, __h1};
54	67	}
55	68
56		static __inline __m256h __DEFAULT_FN_ATTRS256
	69	static __inline __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
57	70	_mm256_set1_pch(_Float16 _Complex h) {
58	71	return (__m256h)_mm256_set1_ps(__builtin_bit_cast(float, h));
59	72	}
60	73
61		static __inline __m128h __DEFAULT_FN_ATTRS128
	74	static __inline __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
62	75	_mm_set1_pch(_Float16 _Complex h) {
63	76	return (__m128h)_mm_set1_ps(__builtin_bit_cast(float, h));
64	77	}
65	78
66		static __inline __m256h __DEFAULT_FN_ATTRS256
	79	static __inline __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
67	80	_mm256_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
68	81	_Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
69	82	_Float16 __h9, _Float16 __h10, _Float16 __h11, _Float16 __h12,
...	...	@@ -73,13 +86,20 @@ _mm256_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
73	86	__h4, __h3, __h2, __h1};
74	87	}
75	88
76		#define _mm_setr_ph(h1, h2, h3, h4, h5, h6, h7, h8) \
77		_mm_set_ph((h8), (h7), (h6), (h5), (h4), (h3), (h2), (h1))
	89	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	90	_mm_setr_ph(_Float16 e0, _Float16 e1, _Float16 e2, _Float16 e3, _Float16 e4,
	91	_Float16 e5, _Float16 e6, _Float16 e7) {
	92	return _mm_set_ph(e7, e6, e5, e4, e3, e2, e1, e0);
	93	}
78	94
79		#define _mm256_setr_ph(h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11, h12, h13, \
80		h14, h15, h16) \
81		_mm256_set_ph((h16), (h15), (h14), (h13), (h12), (h11), (h10), (h9), (h8), \
82		(h7), (h6), (h5), (h4), (h3), (h2), (h1))
	95	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	96	_mm256_setr_ph(_Float16 e0, _Float16 e1, _Float16 e2, _Float16 e3, _Float16 e4,
	97	_Float16 e5, _Float16 e6, _Float16 e7, _Float16 e8, _Float16 e9,
	98	_Float16 e10, _Float16 e11, _Float16 e12, _Float16 e13,
	99	_Float16 e14, _Float16 e15) {
	100	return _mm256_set_ph(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3,
	101	e2, e1, e0);
	102	}
83	103
84	104	static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_add_ph(__m256h __A,
85	105	__m256h __B) {
...	...	@@ -229,12 +249,12 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_div_ph(__mmask8 __U,
229	249	(__v8hf)_mm_setzero_ph());
230	250	}
231	251
232		static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_min_ph(__m256h __A,
233		__m256h __B) {
	252	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	253	_mm256_min_ph(__m256h __A, __m256h __B) {
234	254	return (__m256h)__builtin_ia32_minph256((__v16hf)__A, (__v16hf)__B);
235	255	}
236	256
237		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	257	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
238	258	_mm256_mask_min_ph(__m256h __W, __mmask16 __U, __m256h __A, __m256h __B) {
239	259	return (__m256h)__builtin_ia32_selectph_256(
240	260	(__mmask16)__U,
...	...	@@ -242,7 +262,7 @@ _mm256_mask_min_ph(__m256h __W, __mmask16 __U, __m256h __A, __m256h __B) {
242	262	(__v16hf)__W);
243	263	}
244	264
245		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	265	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
246	266	_mm256_maskz_min_ph(__mmask16 __U, __m256h __A, __m256h __B) {
247	267	return (__m256h)__builtin_ia32_selectph_256(
248	268	(__mmask16)__U,
...	...	@@ -250,34 +270,31 @@ _mm256_maskz_min_ph(__mmask16 __U, __m256h __A, __m256h __B) {
250	270	(__v16hf)_mm256_setzero_ph());
251	271	}
252	272
253		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_min_ph(__m128h __A,
254		__m128h __B) {
	273	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	274	_mm_min_ph(__m128h __A, __m128h __B) {
255	275	return (__m128h)__builtin_ia32_minph128((__v8hf)__A, (__v8hf)__B);
256	276	}
257	277
258		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_min_ph(__m128h __W,
259		__mmask8 __U,
260		__m128h __A,
261		__m128h __B) {
	278	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	279	_mm_mask_min_ph(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
262	280	return (__m128h)__builtin_ia32_selectph_128(
263	281	(__mmask8)__U, (__v8hf)__builtin_ia32_minph128((__v8hf)__A, (__v8hf)__B),
264	282	(__v8hf)__W);
265	283	}
266	284
267		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_min_ph(__mmask8 __U,
268		__m128h __A,
269		__m128h __B) {
	285	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	286	_mm_maskz_min_ph(__mmask8 __U, __m128h __A, __m128h __B) {
270	287	return (__m128h)__builtin_ia32_selectph_128(
271	288	(__mmask8)__U, (__v8hf)__builtin_ia32_minph128((__v8hf)__A, (__v8hf)__B),
272	289	(__v8hf)_mm_setzero_ph());
273	290	}
274	291
275		static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_max_ph(__m256h __A,
276		__m256h __B) {
	292	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	293	_mm256_max_ph(__m256h __A, __m256h __B) {
277	294	return (__m256h)__builtin_ia32_maxph256((__v16hf)__A, (__v16hf)__B);
278	295	}
279	296
280		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	297	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
281	298	_mm256_mask_max_ph(__m256h __W, __mmask16 __U, __m256h __A, __m256h __B) {
282	299	return (__m256h)__builtin_ia32_selectph_256(
283	300	(__mmask16)__U,
...	...	@@ -285,7 +302,7 @@ _mm256_mask_max_ph(__m256h __W, __mmask16 __U, __m256h __A, __m256h __B) {
285	302	(__v16hf)__W);
286	303	}
287	304
288		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	305	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
289	306	_mm256_maskz_max_ph(__mmask16 __U, __m256h __A, __m256h __B) {
290	307	return (__m256h)__builtin_ia32_selectph_256(
291	308	(__mmask16)__U,
...	...	@@ -293,33 +310,32 @@ _mm256_maskz_max_ph(__mmask16 __U, __m256h __A, __m256h __B) {
293	310	(__v16hf)_mm256_setzero_ph());
294	311	}
295	312
296		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_max_ph(__m128h __A,
297		__m128h __B) {
	313	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	314	_mm_max_ph(__m128h __A, __m128h __B) {
298	315	return (__m128h)__builtin_ia32_maxph128((__v8hf)__A, (__v8hf)__B);
299	316	}
300	317
301		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_max_ph(__m128h __W,
302		__mmask8 __U,
303		__m128h __A,
304		__m128h __B) {
	318	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	319	_mm_mask_max_ph(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
305	320	return (__m128h)__builtin_ia32_selectph_128(
306	321	(__mmask8)__U, (__v8hf)__builtin_ia32_maxph128((__v8hf)__A, (__v8hf)__B),
307	322	(__v8hf)__W);
308	323	}
309	324
310		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_max_ph(__mmask8 __U,
311		__m128h __A,
312		__m128h __B) {
	325	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	326	_mm_maskz_max_ph(__mmask8 __U, __m128h __A, __m128h __B) {
313	327	return (__m128h)__builtin_ia32_selectph_128(
314	328	(__mmask8)__U, (__v8hf)__builtin_ia32_maxph128((__v8hf)__A, (__v8hf)__B),
315	329	(__v8hf)_mm_setzero_ph());
316	330	}
317	331
318		static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_abs_ph(__m256h __A) {
	332	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
	333	_mm256_abs_ph(__m256h __A) {
319	334	return (__m256h)_mm256_and_epi32(_mm256_set1_epi32(0x7FFF7FFF), (__m256i)__A);
320	335	}
321	336
322		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_abs_ph(__m128h __A) {
	337	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	338	_mm_abs_ph(__m128h __A) {
323	339	return (__m128h)_mm_and_epi32(_mm_set1_epi32(0x7FFF7FFF), (__m128i)__A);
324	340	}
325	341
...	...	@@ -601,7 +617,7 @@ _mm256_maskz_scalef_ph(__mmask16 __U, __m256h __A, __m256h __B) {
601	617	(__mmask16)(U)))
602	618
603	619	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_sqrt_ph(__m128h __a) {
604		return __builtin_ia32_sqrtph((__v8hf)__a);
	620	return __builtin_elementwise_sqrt(__a);
605	621	}
606	622
607	623	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_sqrt_ph(__m128h __W,
...	...	@@ -618,7 +634,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_maskz_sqrt_ph(__mmask8 __U,
618	634	}
619	635
620	636	static __inline __m256h __DEFAULT_FN_ATTRS256 _mm256_sqrt_ph(__m256h __a) {
621		return (__m256h)__builtin_ia32_sqrtph256((__v16hf)__a);
	637	return __builtin_elementwise_sqrt(__a);
622	638	}
623	639
624	640	static __inline__ __m256h __DEFAULT_FN_ATTRS256
...	...	@@ -790,34 +806,35 @@ _mm256_maskz_cvttph_epi16(__mmask16 __U, __m256h __A) {
790	806	(__v16hf)__A, (__v16hi)_mm256_setzero_si256(), (__mmask16)__U);
791	807	}
792	808
793		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtepi16_ph(__m128i __A) {
	809	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	810	_mm_cvtepi16_ph(__m128i __A) {
794	811	return (__m128h) __builtin_convertvector((__v8hi)__A, __v8hf);
795	812	}
796	813
797		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	814	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
798	815	_mm_mask_cvtepi16_ph(__m128h __W, __mmask8 __U, __m128i __A) {
799	816	return (__m128h)__builtin_ia32_selectph_128(
800	817	(__mmask8)__U, (__v8hf)_mm_cvtepi16_ph(__A), (__v8hf)__W);
801	818	}
802	819
803		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	820	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
804	821	_mm_maskz_cvtepi16_ph(__mmask8 __U, __m128i __A) {
805	822	return (__m128h)__builtin_ia32_selectph_128(
806	823	(__mmask8)__U, (__v8hf)_mm_cvtepi16_ph(__A), (__v8hf)_mm_setzero_ph());
807	824	}
808	825
809		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	826	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
810	827	_mm256_cvtepi16_ph(__m256i __A) {
811	828	return (__m256h) __builtin_convertvector((__v16hi)__A, __v16hf);
812	829	}
813	830
814		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	831	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
815	832	_mm256_mask_cvtepi16_ph(__m256h __W, __mmask16 __U, __m256i __A) {
816	833	return (__m256h)__builtin_ia32_selectph_256(
817	834	(__mmask16)__U, (__v16hf)_mm256_cvtepi16_ph(__A), (__v16hf)__W);
818	835	}
819	836
820		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	837	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
821	838	_mm256_maskz_cvtepi16_ph(__mmask16 __U, __m256i __A) {
822	839	return (__m256h)__builtin_ia32_selectph_256((__mmask16)__U,
823	840	(__v16hf)_mm256_cvtepi16_ph(__A),
...	...	@@ -894,34 +911,35 @@ _mm256_maskz_cvttph_epu16(__mmask16 __U, __m256h __A) {
894	911	(__v16hf)__A, (__v16hu)_mm256_setzero_si256(), (__mmask16)__U);
895	912	}
896	913
897		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtepu16_ph(__m128i __A) {
	914	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	915	_mm_cvtepu16_ph(__m128i __A) {
898	916	return (__m128h) __builtin_convertvector((__v8hu)__A, __v8hf);
899	917	}
900	918
901		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	919	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
902	920	_mm_mask_cvtepu16_ph(__m128h __W, __mmask8 __U, __m128i __A) {
903	921	return (__m128h)__builtin_ia32_selectph_128(
904	922	(__mmask8)__U, (__v8hf)_mm_cvtepu16_ph(__A), (__v8hf)__W);
905	923	}
906	924
907		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	925	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
908	926	_mm_maskz_cvtepu16_ph(__mmask8 __U, __m128i __A) {
909	927	return (__m128h)__builtin_ia32_selectph_128(
910	928	(__mmask8)__U, (__v8hf)_mm_cvtepu16_ph(__A), (__v8hf)_mm_setzero_ph());
911	929	}
912	930
913		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	931	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
914	932	_mm256_cvtepu16_ph(__m256i __A) {
915	933	return (__m256h) __builtin_convertvector((__v16hu)__A, __v16hf);
916	934	}
917	935
918		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	936	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
919	937	_mm256_mask_cvtepu16_ph(__m256h __W, __mmask16 __U, __m256i __A) {
920	938	return (__m256h)__builtin_ia32_selectph_256(
921	939	(__mmask16)__U, (__v16hf)_mm256_cvtepu16_ph(__A), (__v16hf)__W);
922	940	}
923	941
924		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	942	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
925	943	_mm256_maskz_cvtepu16_ph(__mmask16 __U, __m256i __A) {
926	944	return (__m256h)__builtin_ia32_selectph_256((__mmask16)__U,
927	945	(__v16hf)_mm256_cvtepu16_ph(__A),
...	...	@@ -1015,18 +1033,18 @@ _mm_maskz_cvtepi32_ph(__mmask8 __U, __m128i __A) {
1015	1033	(__v4si)__A, (__v8hf)_mm_setzero_ph(), (__mmask8)__U);
1016	1034	}
1017	1035
1018		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1036	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1019	1037	_mm256_cvtepi32_ph(__m256i __A) {
1020	1038	return (__m128h) __builtin_convertvector((__v8si)__A, __v8hf);
1021	1039	}
1022	1040
1023		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1041	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1024	1042	_mm256_mask_cvtepi32_ph(__m128h __W, __mmask8 __U, __m256i __A) {
1025	1043	return (__m128h)__builtin_ia32_selectph_128(
1026	1044	(__mmask8)__U, (__v8hf)_mm256_cvtepi32_ph(__A), (__v8hf)__W);
1027	1045	}
1028	1046
1029		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1047	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1030	1048	_mm256_maskz_cvtepi32_ph(__mmask8 __U, __m256i __A) {
1031	1049	return (__m128h)__builtin_ia32_selectph_128(
1032	1050	(__mmask8)__U, (__v8hf)_mm256_cvtepi32_ph(__A), (__v8hf)_mm_setzero_ph());
...	...	@@ -1049,18 +1067,18 @@ _mm_maskz_cvtepu32_ph(__mmask8 __U, __m128i __A) {
1049	1067	(__v4su)__A, (__v8hf)_mm_setzero_ph(), (__mmask8)__U);
1050	1068	}
1051	1069
1052		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1070	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1053	1071	_mm256_cvtepu32_ph(__m256i __A) {
1054	1072	return (__m128h) __builtin_convertvector((__v8su)__A, __v8hf);
1055	1073	}
1056	1074
1057		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1075	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1058	1076	_mm256_mask_cvtepu32_ph(__m128h __W, __mmask8 __U, __m256i __A) {
1059	1077	return (__m128h)__builtin_ia32_selectph_128(
1060	1078	(__mmask8)__U, (__v8hf)_mm256_cvtepu32_ph(__A), (__v8hf)__W);
1061	1079	}
1062	1080
1063		static __inline__ __m128h __DEFAULT_FN_ATTRS256
	1081	static __inline__ __m128h __DEFAULT_FN_ATTRS256_CONSTEXPR
1064	1082	_mm256_maskz_cvtepu32_ph(__mmask8 __U, __m256i __A) {
1065	1083	return (__m128h)__builtin_ia32_selectph_128(
1066	1084	(__mmask8)__U, (__v8hf)_mm256_cvtepu32_ph(__A), (__v8hf)_mm_setzero_ph());
...	...	@@ -1419,8 +1437,8 @@ _mm256_maskz_cvtxps_ph(__mmask8 __U, __m256 __A) {
1419	1437	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fmadd_ph(__m128h __A,
1420	1438	__m128h __B,
1421	1439	__m128h __C) {
1422		return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B,
1423		(__v8hf)__C);
	1440	return (__m128h)__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B,
	1441	(__v8hf)__C);
1424	1442	}
1425	1443
1426	1444	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmadd_ph(__m128h __A,
...	...	@@ -1429,7 +1447,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmadd_ph(__m128h __A,
1429	1447	__m128h __C) {
1430	1448	return (__m128h)__builtin_ia32_selectph_128(
1431	1449	(__mmask8)__U,
1432		__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
	1450	__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
1433	1451	(__v8hf)__A);
1434	1452	}
1435	1453
...	...	@@ -1437,7 +1455,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1437	1455	_mm_mask3_fmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
1438	1456	return (__m128h)__builtin_ia32_selectph_128(
1439	1457	(__mmask8)__U,
1440		__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
	1458	__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
1441	1459	(__v8hf)__C);
1442	1460	}
1443	1461
...	...	@@ -1445,15 +1463,15 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1445	1463	_mm_maskz_fmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
1446	1464	return (__m128h)__builtin_ia32_selectph_128(
1447	1465	(__mmask8)__U,
1448		__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
	1466	__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
1449	1467	(__v8hf)_mm_setzero_ph());
1450	1468	}
1451	1469
1452	1470	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fmsub_ph(__m128h __A,
1453	1471	__m128h __B,
1454	1472	__m128h __C) {
1455		return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B,
1456		-(__v8hf)__C);
	1473	return (__m128h)__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B,
	1474	-(__v8hf)__C);
1457	1475	}
1458	1476
1459	1477	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmsub_ph(__m128h __A,
...	...	@@ -1476,7 +1494,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1476	1494	_mm_mask3_fnmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
1477	1495	return (__m128h)__builtin_ia32_selectph_128(
1478	1496	(__mmask8)__U,
1479		__builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
	1497	__builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
1480	1498	(__v8hf)__C);
1481	1499	}
1482	1500
...	...	@@ -1484,7 +1502,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1484	1502	_mm_maskz_fnmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
1485	1503	return (__m128h)__builtin_ia32_selectph_128(
1486	1504	(__mmask8)__U,
1487		__builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
	1505	__builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
1488	1506	(__v8hf)_mm_setzero_ph());
1489	1507	}
1490	1508
...	...	@@ -1492,22 +1510,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1492	1510	_mm_maskz_fnmsub_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
1493	1511	return (__m128h)__builtin_ia32_selectph_128(
1494	1512	(__mmask8)__U,
1495		__builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
	1513	__builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
1496	1514	(__v8hf)_mm_setzero_ph());
1497	1515	}
1498	1516
1499	1517	static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fmadd_ph(__m256h __A,
1500	1518	__m256h __B,
1501	1519	__m256h __C) {
1502		return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B,
1503		(__v16hf)__C);
	1520	return (__m256h)__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B,
	1521	(__v16hf)__C);
1504	1522	}
1505	1523
1506	1524	static __inline__ __m256h __DEFAULT_FN_ATTRS256
1507	1525	_mm256_mask_fmadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
1508	1526	return (__m256h)__builtin_ia32_selectph_256(
1509	1527	(__mmask16)__U,
1510		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
	1528	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
1511	1529	(__v16hf)__A);
1512	1530	}
1513	1531
...	...	@@ -1515,7 +1533,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1515	1533	_mm256_mask3_fmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
1516	1534	return (__m256h)__builtin_ia32_selectph_256(
1517	1535	(__mmask16)__U,
1518		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
	1536	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
1519	1537	(__v16hf)__C);
1520	1538	}
1521	1539
...	...	@@ -1523,22 +1541,22 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1523	1541	_mm256_maskz_fmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
1524	1542	return (__m256h)__builtin_ia32_selectph_256(
1525	1543	(__mmask16)__U,
1526		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
	1544	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
1527	1545	(__v16hf)_mm256_setzero_ph());
1528	1546	}
1529	1547
1530	1548	static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fmsub_ph(__m256h __A,
1531	1549	__m256h __B,
1532	1550	__m256h __C) {
1533		return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B,
1534		-(__v16hf)__C);
	1551	return (__m256h)__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B,
	1552	-(__v16hf)__C);
1535	1553	}
1536	1554
1537	1555	static __inline__ __m256h __DEFAULT_FN_ATTRS256
1538	1556	_mm256_mask_fmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
1539	1557	return (__m256h)__builtin_ia32_selectph_256(
1540	1558	(__mmask16)__U,
1541		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
	1559	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
1542	1560	(__v16hf)__A);
1543	1561	}
1544	1562
...	...	@@ -1546,7 +1564,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1546	1564	_mm256_maskz_fmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
1547	1565	return (__m256h)__builtin_ia32_selectph_256(
1548	1566	(__mmask16)__U,
1549		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
	1567	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
1550	1568	(__v16hf)_mm256_setzero_ph());
1551	1569	}
1552	1570
...	...	@@ -1554,7 +1572,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1554	1572	_mm256_mask3_fnmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
1555	1573	return (__m256h)__builtin_ia32_selectph_256(
1556	1574	(__mmask16)__U,
1557		__builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
	1575	__builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
1558	1576	(__v16hf)__C);
1559	1577	}
1560	1578
...	...	@@ -1562,7 +1580,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1562	1580	_mm256_maskz_fnmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
1563	1581	return (__m256h)__builtin_ia32_selectph_256(
1564	1582	(__mmask16)__U,
1565		__builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
	1583	__builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
1566	1584	(__v16hf)_mm256_setzero_ph());
1567	1585	}
1568	1586
...	...	@@ -1570,7 +1588,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1570	1588	_mm256_maskz_fnmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
1571	1589	return (__m256h)__builtin_ia32_selectph_256(
1572	1590	(__mmask16)__U,
1573		__builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
	1591	__builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
1574	1592	(__v16hf)_mm256_setzero_ph());
1575	1593	}
1576	1594
...	...	@@ -1684,7 +1702,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1684	1702	_mm_mask3_fmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
1685	1703	return (__m128h)__builtin_ia32_selectph_128(
1686	1704	(__mmask8)__U,
1687		__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
	1705	__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
1688	1706	(__v8hf)__C);
1689	1707	}
1690	1708
...	...	@@ -1692,7 +1710,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1692	1710	_mm256_mask3_fmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
1693	1711	return (__m256h)__builtin_ia32_selectph_256(
1694	1712	(__mmask16)__U,
1695		__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
	1713	__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
1696	1714	(__v16hf)__C);
1697	1715	}
1698	1716
...	...	@@ -1715,45 +1733,45 @@ _mm256_mask3_fmsubadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
1715	1733	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fnmadd_ph(__m128h __A,
1716	1734	__m128h __B,
1717	1735	__m128h __C) {
1718		return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B,
1719		(__v8hf)__C);
	1736	return (__m128h)__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B,
	1737	(__v8hf)__C);
1720	1738	}
1721	1739
1722	1740	static __inline__ __m128h __DEFAULT_FN_ATTRS128
1723	1741	_mm_mask_fnmadd_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
1724	1742	return (__m128h)__builtin_ia32_selectph_128(
1725	1743	(__mmask8)__U,
1726		__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, (__v8hf)__C),
	1744	__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, (__v8hf)__C),
1727	1745	(__v8hf)__A);
1728	1746	}
1729	1747
1730	1748	static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fnmadd_ph(__m256h __A,
1731	1749	__m256h __B,
1732	1750	__m256h __C) {
1733		return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B,
1734		(__v16hf)__C);
	1751	return (__m256h)__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B,
	1752	(__v16hf)__C);
1735	1753	}
1736	1754
1737	1755	static __inline__ __m256h __DEFAULT_FN_ATTRS256
1738	1756	_mm256_mask_fnmadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
1739	1757	return (__m256h)__builtin_ia32_selectph_256(
1740	1758	(__mmask16)__U,
1741		__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, (__v16hf)__C),
	1759	__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, (__v16hf)__C),
1742	1760	(__v16hf)__A);
1743	1761	}
1744	1762
1745	1763	static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fnmsub_ph(__m128h __A,
1746	1764	__m128h __B,
1747	1765	__m128h __C) {
1748		return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B,
1749		-(__v8hf)__C);
	1766	return (__m128h)__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B,
	1767	-(__v8hf)__C);
1750	1768	}
1751	1769
1752	1770	static __inline__ __m128h __DEFAULT_FN_ATTRS128
1753	1771	_mm_mask_fnmsub_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
1754	1772	return (__m128h)__builtin_ia32_selectph_128(
1755	1773	(__mmask8)__U,
1756		__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
	1774	__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
1757	1775	(__v8hf)__A);
1758	1776	}
1759	1777
...	...	@@ -1761,22 +1779,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
1761	1779	_mm_mask3_fnmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
1762	1780	return (__m128h)__builtin_ia32_selectph_128(
1763	1781	(__mmask8)__U,
1764		__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
	1782	__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
1765	1783	(__v8hf)__C);
1766	1784	}
1767	1785
1768	1786	static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fnmsub_ph(__m256h __A,
1769	1787	__m256h __B,
1770	1788	__m256h __C) {
1771		return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B,
1772		-(__v16hf)__C);
	1789	return (__m256h)__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B,
	1790	-(__v16hf)__C);
1773	1791	}
1774	1792
1775	1793	static __inline__ __m256h __DEFAULT_FN_ATTRS256
1776	1794	_mm256_mask_fnmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
1777	1795	return (__m256h)__builtin_ia32_selectph_256(
1778	1796	(__mmask16)__U,
1779		__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
	1797	__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
1780	1798	(__v16hf)__A);
1781	1799	}
1782	1800
...	...	@@ -1784,7 +1802,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
1784	1802	_mm256_mask3_fnmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
1785	1803	return (__m256h)__builtin_ia32_selectph_256(
1786	1804	(__mmask16)__U,
1787		__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
	1805	__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
1788	1806	(__v16hf)__C);
1789	1807	}
1790	1808
...	...	@@ -1974,37 +1992,36 @@ _mm256_maskz_fmadd_pch(__mmask8 __U, __m256h __A, __m256h __B, __m256h __C) {
1974	1992	(__v8sf)__C, (__mmask8)__U);
1975	1993	}
1976	1994
1977		static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_blend_ph(__mmask8 __U,
1978		__m128h __A,
1979		__m128h __W) {
	1995	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
	1996	_mm_mask_blend_ph(__mmask8 __U, __m128h __A, __m128h __W) {
1980	1997	return (__m128h)__builtin_ia32_selectph_128((__mmask8)__U, (__v8hf)__W,
1981	1998	(__v8hf)__A);
1982	1999	}
1983	2000
1984		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	2001	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
1985	2002	_mm256_mask_blend_ph(__mmask16 __U, __m256h __A, __m256h __W) {
1986	2003	return (__m256h)__builtin_ia32_selectph_256((__mmask16)__U, (__v16hf)__W,
1987	2004	(__v16hf)__A);
1988	2005	}
1989	2006
1990		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	2007	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
1991	2008	_mm_permutex2var_ph(__m128h __A, __m128i __I, __m128h __B) {
1992	2009	return (__m128h)__builtin_ia32_vpermi2varhi128((__v8hi)__A, (__v8hi)__I,
1993	2010	(__v8hi)__B);
1994	2011	}
1995	2012
1996		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	2013	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
1997	2014	_mm256_permutex2var_ph(__m256h __A, __m256i __I, __m256h __B) {
1998	2015	return (__m256h)__builtin_ia32_vpermi2varhi256((__v16hi)__A, (__v16hi)__I,
1999	2016	(__v16hi)__B);
2000	2017	}
2001	2018
2002		static __inline__ __m128h __DEFAULT_FN_ATTRS128
	2019	static __inline__ __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR
2003	2020	_mm_permutexvar_ph(__m128i __A, __m128h __B) {
2004	2021	return (__m128h)__builtin_ia32_permvarhi128((__v8hi)__B, (__v8hi)__A);
2005	2022	}
2006	2023
2007		static __inline__ __m256h __DEFAULT_FN_ATTRS256
	2024	static __inline__ __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
2008	2025	_mm256_permutexvar_ph(__m256i __A, __m256h __B) {
2009	2026	return (__m256h)__builtin_ia32_permvarhi256((__v16hi)__B, (__v16hi)__A);
2010	2027	}
...	...	@@ -2066,6 +2083,8 @@ _mm_reduce_min_ph(__m128h __V) {
2066	2083
2067	2084	#undef __DEFAULT_FN_ATTRS128
2068	2085	#undef __DEFAULT_FN_ATTRS256
	2086	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
	2087	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
2069	2088
2070	2089	#endif
2071	2090	#endif

lib/include/avx512vlintrin.h+945-1299

...	...	@@ -15,14 +15,20 @@
15	15	#define __AVX512VLINTRIN_H
16	16
17	17	#define __DEFAULT_FN_ATTRS128 \
18		__attribute__((__always_inline__, __nodebug__, \
19		__target__("avx512vl,no-evex512"), \
	18	__attribute__((__always_inline__, __nodebug__, __target__("avx512vl"), \
20	19	__min_vector_width__(128)))
21	20	#define __DEFAULT_FN_ATTRS256 \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("avx512vl,no-evex512"), \
	21	__attribute__((__always_inline__, __nodebug__, __target__("avx512vl"), \
24	22	__min_vector_width__(256)))
25	23
	24	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	25	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	26	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	27	#else
	28	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	29	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	30	#endif
	31
26	32	typedef short __v2hi __attribute__((__vector_size__(4)));
27	33	typedef char __v4qi __attribute__((__vector_size__(4)));
28	34	typedef char __v2qi __attribute__((__vector_size__(2)));
...	...	@@ -229,209 +235,183 @@ typedef char __v2qi __attribute__((__vector_size__(2)));
229	235	#define _mm256_mask_cmpneq_epu64_mask(k, A, B) \
230	236	_mm256_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_NE)
231	237
232		static __inline__ __m256i __DEFAULT_FN_ATTRS256
233		_mm256_mask_add_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
234		{
	238	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	239	_mm256_mask_add_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
235	240	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
236	241	(__v8si)_mm256_add_epi32(__A, __B),
237	242	(__v8si)__W);
238	243	}
239	244
240		static __inline__ __m256i __DEFAULT_FN_ATTRS256
241		_mm256_maskz_add_epi32(__mmask8 __U, __m256i __A, __m256i __B)
242		{
	245	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	246	_mm256_maskz_add_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
243	247	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
244	248	(__v8si)_mm256_add_epi32(__A, __B),
245	249	(__v8si)_mm256_setzero_si256());
246	250	}
247	251
248		static __inline__ __m256i __DEFAULT_FN_ATTRS256
249		_mm256_mask_add_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
250		{
	252	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	253	_mm256_mask_add_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
251	254	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
252	255	(__v4di)_mm256_add_epi64(__A, __B),
253	256	(__v4di)__W);
254	257	}
255	258
256		static __inline__ __m256i __DEFAULT_FN_ATTRS256
257		_mm256_maskz_add_epi64(__mmask8 __U, __m256i __A, __m256i __B)
258		{
	259	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	260	_mm256_maskz_add_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
259	261	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
260	262	(__v4di)_mm256_add_epi64(__A, __B),
261	263	(__v4di)_mm256_setzero_si256());
262	264	}
263	265
264		static __inline__ __m256i __DEFAULT_FN_ATTRS256
265		_mm256_mask_sub_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
266		{
	266	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	267	_mm256_mask_sub_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
267	268	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
268	269	(__v8si)_mm256_sub_epi32(__A, __B),
269	270	(__v8si)__W);
270	271	}
271	272
272		static __inline__ __m256i __DEFAULT_FN_ATTRS256
273		_mm256_maskz_sub_epi32(__mmask8 __U, __m256i __A, __m256i __B)
274		{
	273	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	274	_mm256_maskz_sub_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
275	275	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
276	276	(__v8si)_mm256_sub_epi32(__A, __B),
277	277	(__v8si)_mm256_setzero_si256());
278	278	}
279	279
280		static __inline__ __m256i __DEFAULT_FN_ATTRS256
281		_mm256_mask_sub_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
282		{
	280	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	281	_mm256_mask_sub_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
283	282	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
284	283	(__v4di)_mm256_sub_epi64(__A, __B),
285	284	(__v4di)__W);
286	285	}
287	286
288		static __inline__ __m256i __DEFAULT_FN_ATTRS256
289		_mm256_maskz_sub_epi64(__mmask8 __U, __m256i __A, __m256i __B)
290		{
	287	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	288	_mm256_maskz_sub_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
291	289	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
292	290	(__v4di)_mm256_sub_epi64(__A, __B),
293	291	(__v4di)_mm256_setzero_si256());
294	292	}
295	293
296		static __inline__ __m128i __DEFAULT_FN_ATTRS128
297		_mm_mask_add_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
298		{
	294	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	295	_mm_mask_add_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
299	296	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
300	297	(__v4si)_mm_add_epi32(__A, __B),
301	298	(__v4si)__W);
302	299	}
303	300
304		static __inline__ __m128i __DEFAULT_FN_ATTRS128
305		_mm_maskz_add_epi32(__mmask8 __U, __m128i __A, __m128i __B)
306		{
	301	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	302	_mm_maskz_add_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
307	303	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
308	304	(__v4si)_mm_add_epi32(__A, __B),
309	305	(__v4si)_mm_setzero_si128());
310	306	}
311	307
312		static __inline__ __m128i __DEFAULT_FN_ATTRS128
313		_mm_mask_add_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
314		{
	308	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	309	_mm_mask_add_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
315	310	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
316	311	(__v2di)_mm_add_epi64(__A, __B),
317	312	(__v2di)__W);
318	313	}
319	314
320		static __inline__ __m128i __DEFAULT_FN_ATTRS128
321		_mm_maskz_add_epi64(__mmask8 __U, __m128i __A, __m128i __B)
322		{
	315	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	316	_mm_maskz_add_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
323	317	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
324	318	(__v2di)_mm_add_epi64(__A, __B),
325	319	(__v2di)_mm_setzero_si128());
326	320	}
327	321
328		static __inline__ __m128i __DEFAULT_FN_ATTRS128
329		_mm_mask_sub_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
330		{
	322	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	323	_mm_mask_sub_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
331	324	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
332	325	(__v4si)_mm_sub_epi32(__A, __B),
333	326	(__v4si)__W);
334	327	}
335	328
336		static __inline__ __m128i __DEFAULT_FN_ATTRS128
337		_mm_maskz_sub_epi32(__mmask8 __U, __m128i __A, __m128i __B)
338		{
	329	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	330	_mm_maskz_sub_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
339	331	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
340	332	(__v4si)_mm_sub_epi32(__A, __B),
341	333	(__v4si)_mm_setzero_si128());
342	334	}
343	335
344		static __inline__ __m128i __DEFAULT_FN_ATTRS128
345		_mm_mask_sub_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
346		{
	336	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	337	_mm_mask_sub_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
347	338	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
348	339	(__v2di)_mm_sub_epi64(__A, __B),
349	340	(__v2di)__W);
350	341	}
351	342
352		static __inline__ __m128i __DEFAULT_FN_ATTRS128
353		_mm_maskz_sub_epi64(__mmask8 __U, __m128i __A, __m128i __B)
354		{
	343	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	344	_mm_maskz_sub_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
355	345	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
356	346	(__v2di)_mm_sub_epi64(__A, __B),
357	347	(__v2di)_mm_setzero_si128());
358	348	}
359	349
360		static __inline__ __m256i __DEFAULT_FN_ATTRS256
361		_mm256_mask_mul_epi32(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
362		{
	350	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	351	_mm256_mask_mul_epi32(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
363	352	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
364	353	(__v4di)_mm256_mul_epi32(__X, __Y),
365	354	(__v4di)__W);
366	355	}
367	356
368		static __inline__ __m256i __DEFAULT_FN_ATTRS256
369		_mm256_maskz_mul_epi32(__mmask8 __M, __m256i __X, __m256i __Y)
370		{
	357	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	358	_mm256_maskz_mul_epi32(__mmask8 __M, __m256i __X, __m256i __Y) {
371	359	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
372	360	(__v4di)_mm256_mul_epi32(__X, __Y),
373	361	(__v4di)_mm256_setzero_si256());
374	362	}
375	363
376		static __inline__ __m128i __DEFAULT_FN_ATTRS128
377		_mm_mask_mul_epi32(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
378		{
	364	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	365	_mm_mask_mul_epi32(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
379	366	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
380	367	(__v2di)_mm_mul_epi32(__X, __Y),
381	368	(__v2di)__W);
382	369	}
383	370
384		static __inline__ __m128i __DEFAULT_FN_ATTRS128
385		_mm_maskz_mul_epi32(__mmask8 __M, __m128i __X, __m128i __Y)
386		{
	371	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	372	_mm_maskz_mul_epi32(__mmask8 __M, __m128i __X, __m128i __Y) {
387	373	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
388	374	(__v2di)_mm_mul_epi32(__X, __Y),
389	375	(__v2di)_mm_setzero_si128());
390	376	}
391	377
392		static __inline__ __m256i __DEFAULT_FN_ATTRS256
393		_mm256_mask_mul_epu32(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
394		{
	378	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	379	_mm256_mask_mul_epu32(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
395	380	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
396	381	(__v4di)_mm256_mul_epu32(__X, __Y),
397	382	(__v4di)__W);
398	383	}
399	384
400		static __inline__ __m256i __DEFAULT_FN_ATTRS256
401		_mm256_maskz_mul_epu32(__mmask8 __M, __m256i __X, __m256i __Y)
402		{
	385	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	386	_mm256_maskz_mul_epu32(__mmask8 __M, __m256i __X, __m256i __Y) {
403	387	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
404	388	(__v4di)_mm256_mul_epu32(__X, __Y),
405	389	(__v4di)_mm256_setzero_si256());
406	390	}
407	391
408		static __inline__ __m128i __DEFAULT_FN_ATTRS128
409		_mm_mask_mul_epu32(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
410		{
	392	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	393	_mm_mask_mul_epu32(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
411	394	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
412	395	(__v2di)_mm_mul_epu32(__X, __Y),
413	396	(__v2di)__W);
414	397	}
415	398
416		static __inline__ __m128i __DEFAULT_FN_ATTRS128
417		_mm_maskz_mul_epu32(__mmask8 __M, __m128i __X, __m128i __Y)
418		{
	399	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	400	_mm_maskz_mul_epu32(__mmask8 __M, __m128i __X, __m128i __Y) {
419	401	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
420	402	(__v2di)_mm_mul_epu32(__X, __Y),
421	403	(__v2di)_mm_setzero_si128());
422	404	}
423	405
424		static __inline__ __m256i __DEFAULT_FN_ATTRS256
425		_mm256_maskz_mullo_epi32(__mmask8 __M, __m256i __A, __m256i __B)
426		{
	406	static __inline__ __m256i __DEFAULT_FN_ATTRS128_CONSTEXPR
	407	_mm256_maskz_mullo_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
427	408	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
428	409	(__v8si)_mm256_mullo_epi32(__A, __B),
429	410	(__v8si)_mm256_setzero_si256());
430	411	}
431	412
432		static __inline__ __m256i __DEFAULT_FN_ATTRS256
433		_mm256_mask_mullo_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B)
434		{
	413	static __inline__ __m256i __DEFAULT_FN_ATTRS128_CONSTEXPR
	414	_mm256_mask_mullo_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
435	415	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
436	416	(__v8si)_mm256_mullo_epi32(__A, __B),
437	417	(__v8si)__W);
...	...	@@ -453,9 +433,8 @@ _mm_mask_mullo_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B)
453	433	(__v4si)__W);
454	434	}
455	435
456		static __inline__ __m256i __DEFAULT_FN_ATTRS256
457		_mm256_and_epi32(__m256i __a, __m256i __b)
458		{
	436	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	437	_mm256_and_epi32(__m256i __a, __m256i __b) {
459	438	return (__m256i)((__v8su)__a & (__v8su)__b);
460	439	}
461	440
...	...	@@ -473,9 +452,8 @@ _mm256_maskz_and_epi32(__mmask8 __U, __m256i __A, __m256i __B)
473	452	return (__m256i)_mm256_mask_and_epi32(_mm256_setzero_si256(), __U, __A, __B);
474	453	}
475	454
476		static __inline__ __m128i __DEFAULT_FN_ATTRS128
477		_mm_and_epi32(__m128i __a, __m128i __b)
478		{
	455	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	456	_mm_and_epi32(__m128i __a, __m128i __b) {
479	457	return (__m128i)((__v4su)__a & (__v4su)__b);
480	458	}
481	459
...	...	@@ -896,329 +874,312 @@ _mm_maskz_xor_epi64(__mmask8 __U, __m128i __A, __m128i __B)
896	874	(__v2df)(__m128d)(b), (int)(p), \
897	875	(__mmask8)(m)))
898	876
899		static __inline__ __m128d __DEFAULT_FN_ATTRS128
900		_mm_mask_fmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
901		{
902		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
903		__builtin_ia32_vfmaddpd ((__v2df) __A,
904		(__v2df) __B,
905		(__v2df) __C),
906		(__v2df) __A);
	877	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	878	_mm_mask_fmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
	879	return (__m128d)__builtin_ia32_selectpd_128(
	880	(__mmask8)__U, (__v2df)_mm_fmadd_pd(__A, __B, __C), (__v2df)__A);
907	881	}
908	882
909		static __inline__ __m128d __DEFAULT_FN_ATTRS128
910		_mm_mask3_fmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
911		{
912		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
913		__builtin_ia32_vfmaddpd ((__v2df) __A,
914		(__v2df) __B,
915		(__v2df) __C),
916		(__v2df) __C);
	883	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	884	_mm_mask3_fmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
	885	return (__m128d)__builtin_ia32_selectpd_128(
	886	(__mmask8)__U, (__v2df)_mm_fmadd_pd(__A, __B, __C), (__v2df)__C);
917	887	}
918	888
919		static __inline__ __m128d __DEFAULT_FN_ATTRS128
920		_mm_maskz_fmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
921		{
922		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
923		__builtin_ia32_vfmaddpd ((__v2df) __A,
924		(__v2df) __B,
925		(__v2df) __C),
926		(__v2df)_mm_setzero_pd());
	889	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	890	_mm_maskz_fmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
	891	return (__m128d)__builtin_ia32_selectpd_128(
	892	(__mmask8)__U, (__v2df)_mm_fmadd_pd(__A, __B, __C),
	893	(__v2df)_mm_setzero_pd());
927	894	}
928	895
929		static __inline__ __m128d __DEFAULT_FN_ATTRS128
930		_mm_mask_fmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
931		{
932		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
933		__builtin_ia32_vfmaddpd ((__v2df) __A,
934		(__v2df) __B,
935		-(__v2df) __C),
936		(__v2df) __A);
	896	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	897	_mm_mask_fmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
	898	return (__m128d)__builtin_ia32_selectpd_128(
	899	(__mmask8)__U, (__v2df)_mm_fmsub_pd(__A, __B, __C), (__v2df)__A);
937	900	}
938	901
939		static __inline__ __m128d __DEFAULT_FN_ATTRS128
940		_mm_maskz_fmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
941		{
942		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
943		__builtin_ia32_vfmaddpd ((__v2df) __A,
944		(__v2df) __B,
945		-(__v2df) __C),
946		(__v2df)_mm_setzero_pd());
	902	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	903	_mm_mask3_fmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
	904	return (__m128d)__builtin_ia32_selectpd_128(
	905	(__mmask8)__U, (__v2df)_mm_fmsub_pd(__A, __B, __C), (__v2df)__C);
947	906	}
948	907
949		static __inline__ __m128d __DEFAULT_FN_ATTRS128
950		_mm_mask3_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
951		{
952		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
953		__builtin_ia32_vfmaddpd (-(__v2df) __A,
954		(__v2df) __B,
955		(__v2df) __C),
956		(__v2df) __C);
	908	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	909	_mm_maskz_fmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
	910	return (__m128d)__builtin_ia32_selectpd_128(
	911	(__mmask8)__U, (__v2df)_mm_fmsub_pd(__A, __B, __C),
	912	(__v2df)_mm_setzero_pd());
957	913	}
958	914
959		static __inline__ __m128d __DEFAULT_FN_ATTRS128
960		_mm_maskz_fnmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
961		{
962		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
963		__builtin_ia32_vfmaddpd (-(__v2df) __A,
964		(__v2df) __B,
965		(__v2df) __C),
966		(__v2df)_mm_setzero_pd());
	915	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	916	_mm_mask_fnmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
	917	return (__m128d)__builtin_ia32_selectpd_128(
	918	(__mmask8)__U, (__v2df)_mm_fnmadd_pd(__A, __B, __C), (__v2df)__A);
967	919	}
968	920
969		static __inline__ __m128d __DEFAULT_FN_ATTRS128
970		_mm_maskz_fnmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
971		{
972		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
973		__builtin_ia32_vfmaddpd (-(__v2df) __A,
974		(__v2df) __B,
975		-(__v2df) __C),
976		(__v2df)_mm_setzero_pd());
	921	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	922	_mm_mask3_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
	923	return (__m128d)__builtin_ia32_selectpd_128(
	924	(__mmask8)__U, (__v2df)_mm_fnmadd_pd(__A, __B, __C), (__v2df)__C);
977	925	}
978	926
979		static __inline__ __m256d __DEFAULT_FN_ATTRS256
980		_mm256_mask_fmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
981		{
982		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
983		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
984		(__v4df) __B,
985		(__v4df) __C),
986		(__v4df) __A);
	927	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	928	_mm_maskz_fnmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
	929	return (__m128d)__builtin_ia32_selectpd_128(
	930	(__mmask8)__U, (__v2df)_mm_fnmadd_pd(__A, __B, __C),
	931	(__v2df)_mm_setzero_pd());
987	932	}
988	933
989		static __inline__ __m256d __DEFAULT_FN_ATTRS256
990		_mm256_mask3_fmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
991		{
992		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
993		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
994		(__v4df) __B,
995		(__v4df) __C),
996		(__v4df) __C);
	934	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	935	_mm_mask_fnmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
	936	return (__m128d)__builtin_ia32_selectpd_128(
	937	(__mmask8)__U, (__v2df)_mm_fnmsub_pd(__A, __B, __C), (__v2df)__A);
997	938	}
998	939
999		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1000		_mm256_maskz_fmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
1001		{
1002		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1003		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1004		(__v4df) __B,
1005		(__v4df) __C),
1006		(__v4df)_mm256_setzero_pd());
	940	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	941	_mm_mask3_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
	942	return (__m128d)__builtin_ia32_selectpd_128(
	943	(__mmask8)__U, (__v2df)_mm_fnmsub_pd(__A, __B, __C), (__v2df)__C);
1007	944	}
1008	945
1009		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1010		_mm256_mask_fmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
1011		{
1012		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1013		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1014		(__v4df) __B,
1015		-(__v4df) __C),
1016		(__v4df) __A);
	946	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	947	_mm_maskz_fnmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
	948	return (__m128d)__builtin_ia32_selectpd_128(
	949	(__mmask8)__U, (__v2df)_mm_fnmsub_pd(__A, __B, __C),
	950	(__v2df)_mm_setzero_pd());
1017	951	}
1018	952
1019		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1020		_mm256_maskz_fmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
1021		{
1022		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1023		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1024		(__v4df) __B,
1025		-(__v4df) __C),
1026		(__v4df)_mm256_setzero_pd());
	953	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	954	_mm256_mask_fmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
	955	return (__m256d)__builtin_ia32_selectpd_256(
	956	(__mmask8)__U, (__v4df)_mm256_fmadd_pd(__A, __B, __C), (__v4df)__A);
1027	957	}
1028	958
1029		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1030		_mm256_mask3_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
1031		{
1032		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1033		__builtin_ia32_vfmaddpd256 (-(__v4df) __A,
1034		(__v4df) __B,
1035		(__v4df) __C),
1036		(__v4df) __C);
	959	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	960	_mm256_mask3_fmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
	961	return (__m256d)__builtin_ia32_selectpd_256(
	962	(__mmask8)__U, (__v4df)_mm256_fmadd_pd(__A, __B, __C), (__v4df)__C);
1037	963	}
1038	964
1039		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1040		_mm256_maskz_fnmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
1041		{
1042		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1043		__builtin_ia32_vfmaddpd256 (-(__v4df) __A,
1044		(__v4df) __B,
1045		(__v4df) __C),
1046		(__v4df)_mm256_setzero_pd());
	965	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	966	_mm256_maskz_fmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
	967	return (__m256d)__builtin_ia32_selectpd_256(
	968	(__mmask8)__U, (__v4df)_mm256_fmadd_pd(__A, __B, __C),
	969	(__v4df)_mm256_setzero_pd());
1047	970	}
1048	971
1049		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1050		_mm256_maskz_fnmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
1051		{
1052		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1053		__builtin_ia32_vfmaddpd256 (-(__v4df) __A,
1054		(__v4df) __B,
1055		-(__v4df) __C),
1056		(__v4df)_mm256_setzero_pd());
	972	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	973	_mm256_mask_fmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
	974	return (__m256d)__builtin_ia32_selectpd_256(
	975	(__mmask8)__U, (__v4df)_mm256_fmsub_pd(__A, __B, __C), (__v4df)__A);
1057	976	}
1058	977
1059		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1060		_mm_mask_fmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
1061		{
1062		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1063		__builtin_ia32_vfmaddps ((__v4sf) __A,
1064		(__v4sf) __B,
1065		(__v4sf) __C),
1066		(__v4sf) __A);
	978	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	979	_mm256_mask3_fmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
	980	return (__m256d)__builtin_ia32_selectpd_256(
	981	(__mmask8)__U, (__v4df)_mm256_fmsub_pd(__A, __B, __C), (__v4df)__C);
1067	982	}
1068	983
1069		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1070		_mm_mask3_fmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
1071		{
1072		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1073		__builtin_ia32_vfmaddps ((__v4sf) __A,
1074		(__v4sf) __B,
1075		(__v4sf) __C),
1076		(__v4sf) __C);
	984	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	985	_mm256_maskz_fmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
	986	return (__m256d)__builtin_ia32_selectpd_256(
	987	(__mmask8)__U, (__v4df)_mm256_fmsub_pd(__A, __B, __C),
	988	(__v4df)_mm256_setzero_pd());
1077	989	}
1078	990
1079		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1080		_mm_maskz_fmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
1081		{
1082		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1083		__builtin_ia32_vfmaddps ((__v4sf) __A,
1084		(__v4sf) __B,
1085		(__v4sf) __C),
1086		(__v4sf)_mm_setzero_ps());
	991	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	992	_mm256_mask_fnmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
	993	return (__m256d)__builtin_ia32_selectpd_256(
	994	(__mmask8)__U, (__v4df)_mm256_fnmadd_pd(__A, __B, __C), (__v4df)__A);
1087	995	}
1088	996
1089		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1090		_mm_mask_fmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
1091		{
1092		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1093		__builtin_ia32_vfmaddps ((__v4sf) __A,
1094		(__v4sf) __B,
1095		-(__v4sf) __C),
1096		(__v4sf) __A);
	997	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	998	_mm256_mask3_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
	999	return (__m256d)__builtin_ia32_selectpd_256(
	1000	(__mmask8)__U, (__v4df)_mm256_fnmadd_pd(__A, __B, __C), (__v4df)__C);
1097	1001	}
1098	1002
1099		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1100		_mm_maskz_fmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
1101		{
1102		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1103		__builtin_ia32_vfmaddps ((__v4sf) __A,
1104		(__v4sf) __B,
1105		-(__v4sf) __C),
1106		(__v4sf)_mm_setzero_ps());
	1003	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1004	_mm256_maskz_fnmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
	1005	return (__m256d)__builtin_ia32_selectpd_256(
	1006	(__mmask8)__U, (__v4df)_mm256_fnmadd_pd(__A, __B, __C),
	1007	(__v4df)_mm256_setzero_pd());
1107	1008	}
1108	1009
1109		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1110		_mm_mask3_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
1111		{
1112		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1113		__builtin_ia32_vfmaddps (-(__v4sf) __A,
1114		(__v4sf) __B,
1115		(__v4sf) __C),
1116		(__v4sf) __C);
	1010	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1011	_mm256_mask_fnmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
	1012	return (__m256d)__builtin_ia32_selectpd_256(
	1013	(__mmask8)__U, (__v4df)_mm256_fnmsub_pd(__A, __B, __C), (__v4df)__A);
1117	1014	}
1118	1015
1119		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1120		_mm_maskz_fnmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
1121		{
1122		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1123		__builtin_ia32_vfmaddps (-(__v4sf) __A,
1124		(__v4sf) __B,
1125		(__v4sf) __C),
1126		(__v4sf)_mm_setzero_ps());
	1016	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1017	_mm256_mask3_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
	1018	return (__m256d)__builtin_ia32_selectpd_256(
	1019	(__mmask8)__U, (__v4df)_mm256_fnmsub_pd(__A, __B, __C), (__v4df)__C);
1127	1020	}
1128	1021
1129		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1130		_mm_maskz_fnmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
1131		{
1132		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1133		__builtin_ia32_vfmaddps (-(__v4sf) __A,
1134		(__v4sf) __B,
1135		-(__v4sf) __C),
1136		(__v4sf)_mm_setzero_ps());
	1022	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1023	_mm256_maskz_fnmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
	1024	return (__m256d)__builtin_ia32_selectpd_256(
	1025	(__mmask8)__U, (__v4df)_mm256_fnmsub_pd(__A, __B, __C),
	1026	(__v4df)_mm256_setzero_pd());
1137	1027	}
1138	1028
1139		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1140		_mm256_mask_fmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
1141		{
1142		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1143		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1144		(__v8sf) __B,
1145		(__v8sf) __C),
1146		(__v8sf) __A);
	1029	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1030	_mm_mask_fmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
	1031	return (__m128)__builtin_ia32_selectps_128(
	1032	(__mmask8)__U, (__v4sf)_mm_fmadd_ps(__A, __B, __C), (__v4sf)__A);
1147	1033	}
1148	1034
1149		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1150		_mm256_mask3_fmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
1151		{
1152		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1153		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1154		(__v8sf) __B,
1155		(__v8sf) __C),
1156		(__v8sf) __C);
	1035	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1036	_mm_mask3_fmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
	1037	return (__m128)__builtin_ia32_selectps_128(
	1038	(__mmask8)__U, (__v4sf)_mm_fmadd_ps(__A, __B, __C), (__v4sf)__C);
1157	1039	}
1158	1040
1159		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1160		_mm256_maskz_fmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
1161		{
1162		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1163		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1164		(__v8sf) __B,
1165		(__v8sf) __C),
1166		(__v8sf)_mm256_setzero_ps());
	1041	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1042	_mm_maskz_fmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
	1043	return (__m128)__builtin_ia32_selectps_128(
	1044	(__mmask8)__U, (__v4sf)_mm_fmadd_ps(__A, __B, __C),
	1045	(__v4sf)_mm_setzero_ps());
1167	1046	}
1168	1047
1169		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1170		_mm256_mask_fmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
1171		{
1172		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1173		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1174		(__v8sf) __B,
1175		-(__v8sf) __C),
1176		(__v8sf) __A);
	1048	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1049	_mm_mask_fmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
	1050	return (__m128)__builtin_ia32_selectps_128(
	1051	(__mmask8)__U, (__v4sf)_mm_fmsub_ps(__A, __B, __C), (__v4sf)__A);
1177	1052	}
1178	1053
1179		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1180		_mm256_maskz_fmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
1181		{
1182		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1183		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1184		(__v8sf) __B,
1185		-(__v8sf) __C),
1186		(__v8sf)_mm256_setzero_ps());
	1054	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1055	_mm_mask3_fmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
	1056	return (__m128)__builtin_ia32_selectps_128(
	1057	(__mmask8)__U, (__v4sf)_mm_fmsub_ps(__A, __B, __C), (__v4sf)__C);
1187	1058	}
1188	1059
1189		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1190		_mm256_mask3_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
1191		{
1192		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1193		__builtin_ia32_vfmaddps256 (-(__v8sf) __A,
1194		(__v8sf) __B,
1195		(__v8sf) __C),
1196		(__v8sf) __C);
	1060	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1061	_mm_maskz_fmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
	1062	return (__m128)__builtin_ia32_selectps_128(
	1063	(__mmask8)__U, (__v4sf)_mm_fmsub_ps(__A, __B, __C),
	1064	(__v4sf)_mm_setzero_ps());
1197	1065	}
1198	1066
1199		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1200		_mm256_maskz_fnmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
1201		{
1202		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1203		__builtin_ia32_vfmaddps256 (-(__v8sf) __A,
1204		(__v8sf) __B,
1205		(__v8sf) __C),
1206		(__v8sf)_mm256_setzero_ps());
	1067	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1068	_mm_mask_fnmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
	1069	return (__m128)__builtin_ia32_selectps_128(
	1070	(__mmask8)__U, (__v4sf)_mm_fnmadd_ps(__A, __B, __C), (__v4sf)__A);
1207	1071	}
1208	1072
1209		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1210		_mm256_maskz_fnmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
1211		{
1212		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1213		__builtin_ia32_vfmaddps256 (-(__v8sf) __A,
1214		(__v8sf) __B,
1215		-(__v8sf) __C),
1216		(__v8sf)_mm256_setzero_ps());
	1073	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1074	_mm_mask3_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
	1075	return (__m128)__builtin_ia32_selectps_128(
	1076	(__mmask8)__U, (__v4sf)_mm_fnmadd_ps(__A, __B, __C), (__v4sf)__C);
	1077	}
	1078
	1079	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1080	_mm_maskz_fnmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
	1081	return (__m128)__builtin_ia32_selectps_128(
	1082	(__mmask8)__U, (__v4sf)_mm_fnmadd_ps(__A, __B, __C),
	1083	(__v4sf)_mm_setzero_ps());
	1084	}
	1085
	1086	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1087	_mm_mask_fnmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
	1088	return (__m128)__builtin_ia32_selectps_128(
	1089	(__mmask8)__U, (__v4sf)_mm_fnmsub_ps(__A, __B, __C), (__v4sf)__A);
	1090	}
	1091
	1092	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1093	_mm_mask3_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
	1094	return (__m128)__builtin_ia32_selectps_128(
	1095	(__mmask8)__U, (__v4sf)_mm_fnmsub_ps(__A, __B, __C), (__v4sf)__C);
	1096	}
	1097
	1098	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1099	_mm_maskz_fnmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
	1100	return (__m128)__builtin_ia32_selectps_128(
	1101	(__mmask8)__U, (__v4sf)_mm_fnmsub_ps(__A, __B, __C),
	1102	(__v4sf)_mm_setzero_ps());
	1103	}
	1104
	1105	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1106	_mm256_mask_fmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
	1107	return (__m256)__builtin_ia32_selectps_256(
	1108	(__mmask8)__U, (__v8sf)_mm256_fmadd_ps(__A, __B, __C), (__v8sf)__A);
	1109	}
	1110
	1111	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1112	_mm256_mask3_fmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
	1113	return (__m256)__builtin_ia32_selectps_256(
	1114	(__mmask8)__U, (__v8sf)_mm256_fmadd_ps(__A, __B, __C), (__v8sf)__C);
	1115	}
	1116
	1117	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1118	_mm256_maskz_fmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
	1119	return (__m256)__builtin_ia32_selectps_256(
	1120	(__mmask8)__U, (__v8sf)_mm256_fmadd_ps(__A, __B, __C),
	1121	(__v8sf)_mm256_setzero_ps());
	1122	}
	1123
	1124	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1125	_mm256_mask_fmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
	1126	return (__m256)__builtin_ia32_selectps_256(
	1127	(__mmask8)__U, (__v8sf)_mm256_fmsub_ps(__A, __B, __C), (__v8sf)__A);
	1128	}
	1129
	1130	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1131	_mm256_mask3_fmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
	1132	return (__m256)__builtin_ia32_selectps_256(
	1133	(__mmask8)__U, (__v8sf)_mm256_fmsub_ps(__A, __B, __C), (__v8sf)__C);
	1134	}
	1135
	1136	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1137	_mm256_maskz_fmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
	1138	return (__m256)__builtin_ia32_selectps_256(
	1139	(__mmask8)__U, (__v8sf)_mm256_fmsub_ps(__A, __B, __C),
	1140	(__v8sf)_mm256_setzero_ps());
	1141	}
	1142
	1143	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1144	_mm256_mask_fnmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
	1145	return (__m256)__builtin_ia32_selectps_256(
	1146	(__mmask8)__U, (__v8sf)_mm256_fnmadd_ps(__A, __B, __C), (__v8sf)__A);
	1147	}
	1148
	1149	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1150	_mm256_mask3_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
	1151	return (__m256)__builtin_ia32_selectps_256(
	1152	(__mmask8)__U, (__v8sf)_mm256_fnmadd_ps(__A, __B, __C), (__v8sf)__C);
	1153	}
	1154
	1155	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1156	_mm256_maskz_fnmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
	1157	return (__m256)__builtin_ia32_selectps_256(
	1158	(__mmask8)__U, (__v8sf)_mm256_fnmadd_ps(__A, __B, __C),
	1159	(__v8sf)_mm256_setzero_ps());
	1160	}
	1161
	1162	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1163	_mm256_mask_fnmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
	1164	return (__m256)__builtin_ia32_selectps_256(
	1165	(__mmask8)__U, (__v8sf)_mm256_fnmsub_ps(__A, __B, __C), (__v8sf)__A);
	1166	}
	1167
	1168	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1169	_mm256_mask3_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
	1170	return (__m256)__builtin_ia32_selectps_256(
	1171	(__mmask8)__U, (__v8sf)_mm256_fnmsub_ps(__A, __B, __C), (__v8sf)__C);
	1172	}
	1173
	1174	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1175	_mm256_maskz_fnmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
	1176	return (__m256)__builtin_ia32_selectps_256(
	1177	(__mmask8)__U, (__v8sf)_mm256_fnmsub_ps(__A, __B, __C),
	1178	(__v8sf)_mm256_setzero_ps());
1217	1179	}
1218	1180
1219	1181	static __inline__ __m128d __DEFAULT_FN_ATTRS128
1220		_mm_mask_fmaddsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
1221		{
	1182	_mm_mask_fmaddsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
1222	1183	return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
1223	1184	__builtin_ia32_vfmaddsubpd ((__v2df) __A,
1224	1185	(__v2df) __B,
...	...	@@ -1417,46 +1378,6 @@ _mm256_maskz_fmsubadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
1417	1378	(__v8sf)_mm256_setzero_ps());
1418	1379	}
1419	1380
1420		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1421		_mm_mask3_fmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
1422		{
1423		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
1424		__builtin_ia32_vfmaddpd ((__v2df) __A,
1425		(__v2df) __B,
1426		-(__v2df) __C),
1427		(__v2df) __C);
1428		}
1429
1430		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1431		_mm256_mask3_fmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
1432		{
1433		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1434		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1435		(__v4df) __B,
1436		-(__v4df) __C),
1437		(__v4df) __C);
1438		}
1439
1440		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1441		_mm_mask3_fmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
1442		{
1443		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1444		__builtin_ia32_vfmaddps ((__v4sf) __A,
1445		(__v4sf) __B,
1446		-(__v4sf) __C),
1447		(__v4sf) __C);
1448		}
1449
1450		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1451		_mm256_mask3_fmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
1452		{
1453		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1454		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1455		(__v8sf) __B,
1456		-(__v8sf) __C),
1457		(__v8sf) __C);
1458		}
1459
1460	1381	static __inline__ __m128d __DEFAULT_FN_ATTRS128
1461	1382	_mm_mask3_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
1462	1383	{
...	...	@@ -1497,233 +1418,113 @@ _mm256_mask3_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
1497	1418	(__v8sf) __C);
1498	1419	}
1499	1420
1500		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1501		_mm_mask_fnmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
1502		{
1503		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
1504		__builtin_ia32_vfmaddpd ((__v2df) __A,
1505		-(__v2df) __B,
1506		(__v2df) __C),
1507		(__v2df) __A);
1508		}
1509
1510		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1511		_mm256_mask_fnmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
1512		{
1513		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1514		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1515		-(__v4df) __B,
1516		(__v4df) __C),
1517		(__v4df) __A);
1518		}
1519
1520		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1521		_mm_mask_fnmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
1522		{
1523		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1524		__builtin_ia32_vfmaddps ((__v4sf) __A,
1525		-(__v4sf) __B,
1526		(__v4sf) __C),
1527		(__v4sf) __A);
1528		}
1529
1530		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1531		_mm256_mask_fnmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
1532		{
1533		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1534		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1535		-(__v8sf) __B,
1536		(__v8sf) __C),
1537		(__v8sf) __A);
1538		}
1539
1540		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1541		_mm_mask_fnmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
1542		{
1543		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
1544		__builtin_ia32_vfmaddpd ((__v2df) __A,
1545		-(__v2df) __B,
1546		-(__v2df) __C),
1547		(__v2df) __A);
1548		}
1549
1550		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1551		_mm_mask3_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
1552		{
1553		return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
1554		__builtin_ia32_vfmaddpd ((__v2df) __A,
1555		-(__v2df) __B,
1556		-(__v2df) __C),
1557		(__v2df) __C);
1558		}
1559
1560		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1561		_mm256_mask_fnmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
1562		{
1563		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1564		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1565		-(__v4df) __B,
1566		-(__v4df) __C),
1567		(__v4df) __A);
1568		}
1569
1570		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1571		_mm256_mask3_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
1572		{
1573		return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
1574		__builtin_ia32_vfmaddpd256 ((__v4df) __A,
1575		-(__v4df) __B,
1576		-(__v4df) __C),
1577		(__v4df) __C);
1578		}
1579
1580		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1581		_mm_mask_fnmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
1582		{
1583		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1584		__builtin_ia32_vfmaddps ((__v4sf) __A,
1585		-(__v4sf) __B,
1586		-(__v4sf) __C),
1587		(__v4sf) __A);
1588		}
1589
1590		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1591		_mm_mask3_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
1592		{
1593		return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
1594		__builtin_ia32_vfmaddps ((__v4sf) __A,
1595		-(__v4sf) __B,
1596		-(__v4sf) __C),
1597		(__v4sf) __C);
1598		}
1599
1600		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1601		_mm256_mask_fnmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
1602		{
1603		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1604		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1605		-(__v8sf) __B,
1606		-(__v8sf) __C),
1607		(__v8sf) __A);
1608		}
1609
1610		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1611		_mm256_mask3_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
1612		{
1613		return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
1614		__builtin_ia32_vfmaddps256 ((__v8sf) __A,
1615		-(__v8sf) __B,
1616		-(__v8sf) __C),
1617		(__v8sf) __C);
1618		}
1619
1620		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	1421	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
1621	1422	_mm_mask_add_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
1622	1423	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
1623	1424	(__v2df)_mm_add_pd(__A, __B),
1624	1425	(__v2df)__W);
1625	1426	}
1626	1427
1627		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	1428	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
1628	1429	_mm_maskz_add_pd(__mmask8 __U, __m128d __A, __m128d __B) {
1629	1430	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
1630	1431	(__v2df)_mm_add_pd(__A, __B),
1631	1432	(__v2df)_mm_setzero_pd());
1632	1433	}
1633	1434
1634		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	1435	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
1635	1436	_mm256_mask_add_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
1636	1437	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
1637	1438	(__v4df)_mm256_add_pd(__A, __B),
1638	1439	(__v4df)__W);
1639	1440	}
1640	1441
1641		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	1442	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
1642	1443	_mm256_maskz_add_pd(__mmask8 __U, __m256d __A, __m256d __B) {
1643	1444	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
1644	1445	(__v4df)_mm256_add_pd(__A, __B),
1645	1446	(__v4df)_mm256_setzero_pd());
1646	1447	}
1647	1448
1648		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	1449	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
1649	1450	_mm_mask_add_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
1650	1451	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
1651	1452	(__v4sf)_mm_add_ps(__A, __B),
1652	1453	(__v4sf)__W);
1653	1454	}
1654	1455
1655		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	1456	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
1656	1457	_mm_maskz_add_ps(__mmask8 __U, __m128 __A, __m128 __B) {
1657	1458	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
1658	1459	(__v4sf)_mm_add_ps(__A, __B),
1659	1460	(__v4sf)_mm_setzero_ps());
1660	1461	}
1661	1462
1662		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	1463	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
1663	1464	_mm256_mask_add_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
1664	1465	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
1665	1466	(__v8sf)_mm256_add_ps(__A, __B),
1666	1467	(__v8sf)__W);
1667	1468	}
1668	1469
1669		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	1470	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
1670	1471	_mm256_maskz_add_ps(__mmask8 __U, __m256 __A, __m256 __B) {
1671	1472	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
1672	1473	(__v8sf)_mm256_add_ps(__A, __B),
1673	1474	(__v8sf)_mm256_setzero_ps());
1674	1475	}
1675	1476
1676		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1677		_mm_mask_blend_epi32 (__mmask8 __U, __m128i __A, __m128i __W) {
	1477	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1478	_mm_mask_blend_epi32(__mmask8 __U, __m128i __A, __m128i __W) {
1678	1479	return (__m128i) __builtin_ia32_selectd_128 ((__mmask8) __U,
1679	1480	(__v4si) __W,
1680	1481	(__v4si) __A);
1681	1482	}
1682	1483
1683		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1684		_mm256_mask_blend_epi32 (__mmask8 __U, __m256i __A, __m256i __W) {
	1484	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1485	_mm256_mask_blend_epi32(__mmask8 __U, __m256i __A, __m256i __W) {
1685	1486	return (__m256i) __builtin_ia32_selectd_256 ((__mmask8) __U,
1686	1487	(__v8si) __W,
1687	1488	(__v8si) __A);
1688	1489	}
1689	1490
1690		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1691		_mm_mask_blend_pd (__mmask8 __U, __m128d __A, __m128d __W) {
	1491	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1492	_mm_mask_blend_pd(__mmask8 __U, __m128d __A, __m128d __W) {
1692	1493	return (__m128d) __builtin_ia32_selectpd_128 ((__mmask8) __U,
1693	1494	(__v2df) __W,
1694	1495	(__v2df) __A);
1695	1496	}
1696	1497
1697		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1698		_mm256_mask_blend_pd (__mmask8 __U, __m256d __A, __m256d __W) {
	1498	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1499	_mm256_mask_blend_pd(__mmask8 __U, __m256d __A, __m256d __W) {
1699	1500	return (__m256d) __builtin_ia32_selectpd_256 ((__mmask8) __U,
1700	1501	(__v4df) __W,
1701	1502	(__v4df) __A);
1702	1503	}
1703	1504
1704		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1705		_mm_mask_blend_ps (__mmask8 __U, __m128 __A, __m128 __W) {
	1505	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1506	_mm_mask_blend_ps(__mmask8 __U, __m128 __A, __m128 __W) {
1706	1507	return (__m128) __builtin_ia32_selectps_128 ((__mmask8) __U,
1707	1508	(__v4sf) __W,
1708	1509	(__v4sf) __A);
1709	1510	}
1710	1511
1711		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1712		_mm256_mask_blend_ps (__mmask8 __U, __m256 __A, __m256 __W) {
	1512	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1513	_mm256_mask_blend_ps(__mmask8 __U, __m256 __A, __m256 __W) {
1713	1514	return (__m256) __builtin_ia32_selectps_256 ((__mmask8) __U,
1714	1515	(__v8sf) __W,
1715	1516	(__v8sf) __A);
1716	1517	}
1717	1518
1718		static __inline__ __m128i __DEFAULT_FN_ATTRS128
1719		_mm_mask_blend_epi64 (__mmask8 __U, __m128i __A, __m128i __W) {
	1519	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	1520	_mm_mask_blend_epi64(__mmask8 __U, __m128i __A, __m128i __W) {
1720	1521	return (__m128i) __builtin_ia32_selectq_128 ((__mmask8) __U,
1721	1522	(__v2di) __W,
1722	1523	(__v2di) __A);
1723	1524	}
1724	1525
1725		static __inline__ __m256i __DEFAULT_FN_ATTRS256
1726		_mm256_mask_blend_epi64 (__mmask8 __U, __m256i __A, __m256i __W) {
	1526	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	1527	_mm256_mask_blend_epi64(__mmask8 __U, __m256i __A, __m256i __W) {
1727	1528	return (__m256i) __builtin_ia32_selectq_256 ((__mmask8) __U,
1728	1529	(__v4di) __W,
1729	1530	(__v4di) __A);
...	...	@@ -1905,57 +1706,57 @@ _mm256_mask_compressstoreu_epi32 (void *__P, __mmask8 __U, __m256i __A) {
1905	1706	(__mmask8) __U);
1906	1707	}
1907	1708
1908		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1909		_mm_mask_cvtepi32_pd (__m128d __W, __mmask8 __U, __m128i __A) {
	1709	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1710	_mm_mask_cvtepi32_pd(__m128d __W, __mmask8 __U, __m128i __A) {
1910	1711	return (__m128d)__builtin_ia32_selectpd_128((__mmask8) __U,
1911	1712	(__v2df)_mm_cvtepi32_pd(__A),
1912	1713	(__v2df)__W);
1913	1714	}
1914	1715
1915		static __inline__ __m128d __DEFAULT_FN_ATTRS128
1916		_mm_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) {
	1716	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	1717	_mm_maskz_cvtepi32_pd(__mmask8 __U, __m128i __A) {
1917	1718	return (__m128d)__builtin_ia32_selectpd_128((__mmask8) __U,
1918	1719	(__v2df)_mm_cvtepi32_pd(__A),
1919	1720	(__v2df)_mm_setzero_pd());
1920	1721	}
1921	1722
1922		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1923		_mm256_mask_cvtepi32_pd (__m256d __W, __mmask8 __U, __m128i __A) {
	1723	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1724	_mm256_mask_cvtepi32_pd(__m256d __W, __mmask8 __U, __m128i __A) {
1924	1725	return (__m256d)__builtin_ia32_selectpd_256((__mmask8) __U,
1925	1726	(__v4df)_mm256_cvtepi32_pd(__A),
1926	1727	(__v4df)__W);
1927	1728	}
1928	1729
1929		static __inline__ __m256d __DEFAULT_FN_ATTRS256
1930		_mm256_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) {
	1730	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	1731	_mm256_maskz_cvtepi32_pd(__mmask8 __U, __m128i __A) {
1931	1732	return (__m256d)__builtin_ia32_selectpd_256((__mmask8) __U,
1932	1733	(__v4df)_mm256_cvtepi32_pd(__A),
1933	1734	(__v4df)_mm256_setzero_pd());
1934	1735	}
1935	1736
1936		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1937		_mm_mask_cvtepi32_ps (__m128 __W, __mmask8 __U, __m128i __A) {
	1737	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1738	_mm_mask_cvtepi32_ps(__m128 __W, __mmask8 __U, __m128i __A) {
1938	1739	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
1939	1740	(__v4sf)_mm_cvtepi32_ps(__A),
1940	1741	(__v4sf)__W);
1941	1742	}
1942	1743
1943		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1944		_mm_maskz_cvtepi32_ps (__mmask8 __U, __m128i __A) {
	1744	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1745	_mm_maskz_cvtepi32_ps(__mmask8 __U, __m128i __A) {
1945	1746	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
1946	1747	(__v4sf)_mm_cvtepi32_ps(__A),
1947	1748	(__v4sf)_mm_setzero_ps());
1948	1749	}
1949	1750
1950		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1951		_mm256_mask_cvtepi32_ps (__m256 __W, __mmask8 __U, __m256i __A) {
	1751	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1752	_mm256_mask_cvtepi32_ps(__m256 __W, __mmask8 __U, __m256i __A) {
1952	1753	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
1953	1754	(__v8sf)_mm256_cvtepi32_ps(__A),
1954	1755	(__v8sf)__W);
1955	1756	}
1956	1757
1957		static __inline__ __m256 __DEFAULT_FN_ATTRS256
1958		_mm256_maskz_cvtepi32_ps (__mmask8 __U, __m256i __A) {
	1758	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1759	_mm256_maskz_cvtepi32_ps(__mmask8 __U, __m256i __A) {
1959	1760	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
1960	1761	(__v8sf)_mm256_cvtepi32_ps(__A),
1961	1762	(__v8sf)_mm256_setzero_ps());
...	...	@@ -1990,30 +1791,30 @@ _mm256_maskz_cvtpd_epi32 (__mmask8 __U, __m256d __A) {
1990	1791	(__v4si)_mm_setzero_si128());
1991	1792	}
1992	1793
1993		static __inline__ __m128 __DEFAULT_FN_ATTRS128
1994		_mm_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m128d __A) {
	1794	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1795	_mm_mask_cvtpd_ps(__m128 __W, __mmask8 __U, __m128d __A) {
1995	1796	return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A,
1996	1797	(__v4sf) __W,
1997	1798	(__mmask8) __U);
1998	1799	}
1999	1800
2000		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2001		_mm_maskz_cvtpd_ps (__mmask8 __U, __m128d __A) {
	1801	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	1802	_mm_maskz_cvtpd_ps(__mmask8 __U, __m128d __A) {
2002	1803	return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A,
2003	1804	(__v4sf)
2004	1805	_mm_setzero_ps (),
2005	1806	(__mmask8) __U);
2006	1807	}
2007	1808
2008		static __inline__ __m128 __DEFAULT_FN_ATTRS256
2009		_mm256_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m256d __A) {
	1809	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1810	_mm256_mask_cvtpd_ps(__m128 __W, __mmask8 __U, __m256d __A) {
2010	1811	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2011	1812	(__v4sf)_mm256_cvtpd_ps(__A),
2012	1813	(__v4sf)__W);
2013	1814	}
2014	1815
2015		static __inline__ __m128 __DEFAULT_FN_ATTRS256
2016		_mm256_maskz_cvtpd_ps (__mmask8 __U, __m256d __A) {
	1816	static __inline__ __m128 __DEFAULT_FN_ATTRS256_CONSTEXPR
	1817	_mm256_maskz_cvtpd_ps(__mmask8 __U, __m256d __A) {
2017	1818	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2018	1819	(__v4sf)_mm256_cvtpd_ps(__A),
2019	1820	(__v4sf)_mm_setzero_ps());
...	...	@@ -2316,133 +2117,133 @@ _mm256_maskz_cvttps_epu32 (__mmask8 __U, __m256 __A) {
2316	2117	(__mmask8) __U);
2317	2118	}
2318	2119
2319		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2320		_mm_cvtepu32_pd (__m128i __A) {
	2120	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2121	_mm_cvtepu32_pd(__m128i __A) {
2321	2122	return (__m128d) __builtin_convertvector(
2322	2123	__builtin_shufflevector((__v4su)__A, (__v4su)__A, 0, 1), __v2df);
2323	2124	}
2324	2125
2325		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2326		_mm_mask_cvtepu32_pd (__m128d __W, __mmask8 __U, __m128i __A) {
	2126	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2127	_mm_mask_cvtepu32_pd(__m128d __W, __mmask8 __U, __m128i __A) {
2327	2128	return (__m128d)__builtin_ia32_selectpd_128((__mmask8) __U,
2328	2129	(__v2df)_mm_cvtepu32_pd(__A),
2329	2130	(__v2df)__W);
2330	2131	}
2331	2132
2332		static __inline__ __m128d __DEFAULT_FN_ATTRS128
2333		_mm_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) {
	2133	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	2134	_mm_maskz_cvtepu32_pd(__mmask8 __U, __m128i __A) {
2334	2135	return (__m128d)__builtin_ia32_selectpd_128((__mmask8) __U,
2335	2136	(__v2df)_mm_cvtepu32_pd(__A),
2336	2137	(__v2df)_mm_setzero_pd());
2337	2138	}
2338	2139
2339		static __inline__ __m256d __DEFAULT_FN_ATTRS256
2340		_mm256_cvtepu32_pd (__m128i __A) {
	2140	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	2141	_mm256_cvtepu32_pd(__m128i __A) {
2341	2142	return (__m256d)__builtin_convertvector((__v4su)__A, __v4df);
2342	2143	}
2343	2144
2344		static __inline__ __m256d __DEFAULT_FN_ATTRS256
2345		_mm256_mask_cvtepu32_pd (__m256d __W, __mmask8 __U, __m128i __A) {
	2145	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	2146	_mm256_mask_cvtepu32_pd(__m256d __W, __mmask8 __U, __m128i __A) {
2346	2147	return (__m256d)__builtin_ia32_selectpd_256((__mmask8) __U,
2347	2148	(__v4df)_mm256_cvtepu32_pd(__A),
2348	2149	(__v4df)__W);
2349	2150	}
2350	2151
2351		static __inline__ __m256d __DEFAULT_FN_ATTRS256
2352		_mm256_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) {
	2152	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	2153	_mm256_maskz_cvtepu32_pd(__mmask8 __U, __m128i __A) {
2353	2154	return (__m256d)__builtin_ia32_selectpd_256((__mmask8) __U,
2354	2155	(__v4df)_mm256_cvtepu32_pd(__A),
2355	2156	(__v4df)_mm256_setzero_pd());
2356	2157	}
2357	2158
2358		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2359		_mm_cvtepu32_ps (__m128i __A) {
	2159	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2160	_mm_cvtepu32_ps(__m128i __A) {
2360	2161	return (__m128)__builtin_convertvector((__v4su)__A, __v4sf);
2361	2162	}
2362	2163
2363		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2364		_mm_mask_cvtepu32_ps (__m128 __W, __mmask8 __U, __m128i __A) {
	2164	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2165	_mm_mask_cvtepu32_ps(__m128 __W, __mmask8 __U, __m128i __A) {
2365	2166	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2366	2167	(__v4sf)_mm_cvtepu32_ps(__A),
2367	2168	(__v4sf)__W);
2368	2169	}
2369	2170
2370		static __inline__ __m128 __DEFAULT_FN_ATTRS128
2371		_mm_maskz_cvtepu32_ps (__mmask8 __U, __m128i __A) {
	2171	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	2172	_mm_maskz_cvtepu32_ps(__mmask8 __U, __m128i __A) {
2372	2173	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2373	2174	(__v4sf)_mm_cvtepu32_ps(__A),
2374	2175	(__v4sf)_mm_setzero_ps());
2375	2176	}
2376	2177
2377		static __inline__ __m256 __DEFAULT_FN_ATTRS256
2378		_mm256_cvtepu32_ps (__m256i __A) {
	2178	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2179	_mm256_cvtepu32_ps(__m256i __A) {
2379	2180	return (__m256)__builtin_convertvector((__v8su)__A, __v8sf);
2380	2181	}
2381	2182
2382		static __inline__ __m256 __DEFAULT_FN_ATTRS256
2383		_mm256_mask_cvtepu32_ps (__m256 __W, __mmask8 __U, __m256i __A) {
	2183	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2184	_mm256_mask_cvtepu32_ps(__m256 __W, __mmask8 __U, __m256i __A) {
2384	2185	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2385	2186	(__v8sf)_mm256_cvtepu32_ps(__A),
2386	2187	(__v8sf)__W);
2387	2188	}
2388	2189
2389		static __inline__ __m256 __DEFAULT_FN_ATTRS256
2390		_mm256_maskz_cvtepu32_ps (__mmask8 __U, __m256i __A) {
	2190	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	2191	_mm256_maskz_cvtepu32_ps(__mmask8 __U, __m256i __A) {
2391	2192	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2392	2193	(__v8sf)_mm256_cvtepu32_ps(__A),
2393	2194	(__v8sf)_mm256_setzero_ps());
2394	2195	}
2395	2196
2396		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2197	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2397	2198	_mm_mask_div_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2398	2199	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2399	2200	(__v2df)_mm_div_pd(__A, __B),
2400	2201	(__v2df)__W);
2401	2202	}
2402	2203
2403		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2204	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2404	2205	_mm_maskz_div_pd(__mmask8 __U, __m128d __A, __m128d __B) {
2405	2206	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2406	2207	(__v2df)_mm_div_pd(__A, __B),
2407	2208	(__v2df)_mm_setzero_pd());
2408	2209	}
2409	2210
2410		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2211	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2411	2212	_mm256_mask_div_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
2412	2213	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2413	2214	(__v4df)_mm256_div_pd(__A, __B),
2414	2215	(__v4df)__W);
2415	2216	}
2416	2217
2417		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2218	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2418	2219	_mm256_maskz_div_pd(__mmask8 __U, __m256d __A, __m256d __B) {
2419	2220	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2420	2221	(__v4df)_mm256_div_pd(__A, __B),
2421	2222	(__v4df)_mm256_setzero_pd());
2422	2223	}
2423	2224
2424		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2225	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2425	2226	_mm_mask_div_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2426	2227	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2427	2228	(__v4sf)_mm_div_ps(__A, __B),
2428	2229	(__v4sf)__W);
2429	2230	}
2430	2231
2431		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2232	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2432	2233	_mm_maskz_div_ps(__mmask8 __U, __m128 __A, __m128 __B) {
2433	2234	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2434	2235	(__v4sf)_mm_div_ps(__A, __B),
2435	2236	(__v4sf)_mm_setzero_ps());
2436	2237	}
2437	2238
2438		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2239	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2439	2240	_mm256_mask_div_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
2440	2241	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2441	2242	(__v8sf)_mm256_div_ps(__A, __B),
2442	2243	(__v8sf)__W);
2443	2244	}
2444	2245
2445		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2246	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2446	2247	_mm256_maskz_div_ps(__mmask8 __U, __m256 __A, __m256 __B) {
2447	2248	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2448	2249	(__v8sf)_mm256_div_ps(__A, __B),
...	...	@@ -2796,499 +2597,499 @@ _mm256_maskz_getexp_ps (__mmask8 __U, __m256 __A) {
2796	2597	(__mmask8) __U);
2797	2598	}
2798	2599
2799		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2600	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2800	2601	_mm_mask_max_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2801	2602	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2802	2603	(__v2df)_mm_max_pd(__A, __B),
2803	2604	(__v2df)__W);
2804	2605	}
2805	2606
2806		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2607	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2807	2608	_mm_maskz_max_pd(__mmask8 __U, __m128d __A, __m128d __B) {
2808	2609	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2809	2610	(__v2df)_mm_max_pd(__A, __B),
2810	2611	(__v2df)_mm_setzero_pd());
2811	2612	}
2812	2613
2813		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2614	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2814	2615	_mm256_mask_max_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
2815	2616	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2816	2617	(__v4df)_mm256_max_pd(__A, __B),
2817	2618	(__v4df)__W);
2818	2619	}
2819	2620
2820		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2621	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2821	2622	_mm256_maskz_max_pd(__mmask8 __U, __m256d __A, __m256d __B) {
2822	2623	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2823	2624	(__v4df)_mm256_max_pd(__A, __B),
2824	2625	(__v4df)_mm256_setzero_pd());
2825	2626	}
2826	2627
2827		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2628	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2828	2629	_mm_mask_max_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2829	2630	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2830	2631	(__v4sf)_mm_max_ps(__A, __B),
2831	2632	(__v4sf)__W);
2832	2633	}
2833	2634
2834		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2635	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2835	2636	_mm_maskz_max_ps(__mmask8 __U, __m128 __A, __m128 __B) {
2836	2637	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2837	2638	(__v4sf)_mm_max_ps(__A, __B),
2838	2639	(__v4sf)_mm_setzero_ps());
2839	2640	}
2840	2641
2841		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2642	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2842	2643	_mm256_mask_max_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
2843	2644	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2844	2645	(__v8sf)_mm256_max_ps(__A, __B),
2845	2646	(__v8sf)__W);
2846	2647	}
2847	2648
2848		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2649	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2849	2650	_mm256_maskz_max_ps(__mmask8 __U, __m256 __A, __m256 __B) {
2850	2651	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2851	2652	(__v8sf)_mm256_max_ps(__A, __B),
2852	2653	(__v8sf)_mm256_setzero_ps());
2853	2654	}
2854	2655
2855		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2656	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2856	2657	_mm_mask_min_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2857	2658	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2858	2659	(__v2df)_mm_min_pd(__A, __B),
2859	2660	(__v2df)__W);
2860	2661	}
2861	2662
2862		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2663	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2863	2664	_mm_maskz_min_pd(__mmask8 __U, __m128d __A, __m128d __B) {
2864	2665	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2865	2666	(__v2df)_mm_min_pd(__A, __B),
2866	2667	(__v2df)_mm_setzero_pd());
2867	2668	}
2868	2669
2869		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2670	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2870	2671	_mm256_mask_min_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
2871	2672	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2872	2673	(__v4df)_mm256_min_pd(__A, __B),
2873	2674	(__v4df)__W);
2874	2675	}
2875	2676
2876		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2677	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2877	2678	_mm256_maskz_min_pd(__mmask8 __U, __m256d __A, __m256d __B) {
2878	2679	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2879	2680	(__v4df)_mm256_min_pd(__A, __B),
2880	2681	(__v4df)_mm256_setzero_pd());
2881	2682	}
2882	2683
2883		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2684	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2884	2685	_mm_mask_min_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2885	2686	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2886	2687	(__v4sf)_mm_min_ps(__A, __B),
2887	2688	(__v4sf)__W);
2888	2689	}
2889	2690
2890		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2691	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2891	2692	_mm_maskz_min_ps(__mmask8 __U, __m128 __A, __m128 __B) {
2892	2693	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2893	2694	(__v4sf)_mm_min_ps(__A, __B),
2894	2695	(__v4sf)_mm_setzero_ps());
2895	2696	}
2896	2697
2897		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2698	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2898	2699	_mm256_mask_min_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
2899	2700	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2900	2701	(__v8sf)_mm256_min_ps(__A, __B),
2901	2702	(__v8sf)__W);
2902	2703	}
2903	2704
2904		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2705	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2905	2706	_mm256_maskz_min_ps(__mmask8 __U, __m256 __A, __m256 __B) {
2906	2707	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2907	2708	(__v8sf)_mm256_min_ps(__A, __B),
2908	2709	(__v8sf)_mm256_setzero_ps());
2909	2710	}
2910	2711
2911		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2712	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2912	2713	_mm_mask_mul_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
2913	2714	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2914	2715	(__v2df)_mm_mul_pd(__A, __B),
2915	2716	(__v2df)__W);
2916	2717	}
2917	2718
2918		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	2719	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
2919	2720	_mm_maskz_mul_pd(__mmask8 __U, __m128d __A, __m128d __B) {
2920	2721	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
2921	2722	(__v2df)_mm_mul_pd(__A, __B),
2922	2723	(__v2df)_mm_setzero_pd());
2923	2724	}
2924	2725
2925		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2726	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2926	2727	_mm256_mask_mul_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
2927	2728	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2928	2729	(__v4df)_mm256_mul_pd(__A, __B),
2929	2730	(__v4df)__W);
2930	2731	}
2931	2732
2932		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	2733	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
2933	2734	_mm256_maskz_mul_pd(__mmask8 __U, __m256d __A, __m256d __B) {
2934	2735	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
2935	2736	(__v4df)_mm256_mul_pd(__A, __B),
2936	2737	(__v4df)_mm256_setzero_pd());
2937	2738	}
2938	2739
2939		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2740	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2940	2741	_mm_mask_mul_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
2941	2742	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2942	2743	(__v4sf)_mm_mul_ps(__A, __B),
2943	2744	(__v4sf)__W);
2944	2745	}
2945	2746
2946		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	2747	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
2947	2748	_mm_maskz_mul_ps(__mmask8 __U, __m128 __A, __m128 __B) {
2948	2749	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
2949	2750	(__v4sf)_mm_mul_ps(__A, __B),
2950	2751	(__v4sf)_mm_setzero_ps());
2951	2752	}
2952	2753
2953		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2754	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2954	2755	_mm256_mask_mul_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
2955	2756	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2956	2757	(__v8sf)_mm256_mul_ps(__A, __B),
2957	2758	(__v8sf)__W);
2958	2759	}
2959	2760
2960		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	2761	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
2961	2762	_mm256_maskz_mul_ps(__mmask8 __U, __m256 __A, __m256 __B) {
2962	2763	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
2963	2764	(__v8sf)_mm256_mul_ps(__A, __B),
2964	2765	(__v8sf)_mm256_setzero_ps());
2965	2766	}
2966	2767
2967		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2768	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2968	2769	_mm_mask_abs_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
2969	2770	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
2970	2771	(__v4si)_mm_abs_epi32(__A),
2971	2772	(__v4si)__W);
2972	2773	}
2973	2774
2974		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2775	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
2975	2776	_mm_maskz_abs_epi32(__mmask8 __U, __m128i __A) {
2976	2777	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
2977	2778	(__v4si)_mm_abs_epi32(__A),
2978	2779	(__v4si)_mm_setzero_si128());
2979	2780	}
2980	2781
2981		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2782	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2982	2783	_mm256_mask_abs_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
2983	2784	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
2984	2785	(__v8si)_mm256_abs_epi32(__A),
2985	2786	(__v8si)__W);
2986	2787	}
2987	2788
2988		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2789	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
2989	2790	_mm256_maskz_abs_epi32(__mmask8 __U, __m256i __A) {
2990	2791	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
2991	2792	(__v8si)_mm256_abs_epi32(__A),
2992	2793	(__v8si)_mm256_setzero_si256());
2993	2794	}
2994	2795
2995		static __inline__ __m128i __DEFAULT_FN_ATTRS128
2996		_mm_abs_epi64 (__m128i __A) {
	2796	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2797	_mm_abs_epi64(__m128i __A) {
2997	2798	return (__m128i)__builtin_elementwise_abs((__v2di)__A);
2998	2799	}
2999	2800
3000		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3001		_mm_mask_abs_epi64 (__m128i __W, __mmask8 __U, __m128i __A) {
	2801	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2802	_mm_mask_abs_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
3002	2803	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
3003	2804	(__v2di)_mm_abs_epi64(__A),
3004	2805	(__v2di)__W);
3005	2806	}
3006	2807
3007		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3008		_mm_maskz_abs_epi64 (__mmask8 __U, __m128i __A) {
	2808	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2809	_mm_maskz_abs_epi64(__mmask8 __U, __m128i __A) {
3009	2810	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
3010	2811	(__v2di)_mm_abs_epi64(__A),
3011	2812	(__v2di)_mm_setzero_si128());
3012	2813	}
3013	2814
3014		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3015		_mm256_abs_epi64 (__m256i __A) {
	2815	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2816	_mm256_abs_epi64(__m256i __A) {
3016	2817	return (__m256i)__builtin_elementwise_abs((__v4di)__A);
3017	2818	}
3018	2819
3019		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3020		_mm256_mask_abs_epi64 (__m256i __W, __mmask8 __U, __m256i __A) {
	2820	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2821	_mm256_mask_abs_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
3021	2822	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
3022	2823	(__v4di)_mm256_abs_epi64(__A),
3023	2824	(__v4di)__W);
3024	2825	}
3025	2826
3026		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3027		_mm256_maskz_abs_epi64 (__mmask8 __U, __m256i __A) {
	2827	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2828	_mm256_maskz_abs_epi64(__mmask8 __U, __m256i __A) {
3028	2829	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
3029	2830	(__v4di)_mm256_abs_epi64(__A),
3030	2831	(__v4di)_mm256_setzero_si256());
3031	2832	}
3032	2833
3033		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2834	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3034	2835	_mm_maskz_max_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
3035	2836	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3036	2837	(__v4si)_mm_max_epi32(__A, __B),
3037	2838	(__v4si)_mm_setzero_si128());
3038	2839	}
3039	2840
3040		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2841	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3041	2842	_mm_mask_max_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3042	2843	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3043	2844	(__v4si)_mm_max_epi32(__A, __B),
3044	2845	(__v4si)__W);
3045	2846	}
3046	2847
3047		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2848	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3048	2849	_mm256_maskz_max_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
3049	2850	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3050	2851	(__v8si)_mm256_max_epi32(__A, __B),
3051	2852	(__v8si)_mm256_setzero_si256());
3052	2853	}
3053	2854
3054		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2855	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3055	2856	_mm256_mask_max_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3056	2857	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3057	2858	(__v8si)_mm256_max_epi32(__A, __B),
3058	2859	(__v8si)__W);
3059	2860	}
3060	2861
3061		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3062		_mm_max_epi64 (__m128i __A, __m128i __B) {
	2862	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2863	_mm_max_epi64(__m128i __A, __m128i __B) {
3063	2864	return (__m128i)__builtin_elementwise_max((__v2di)__A, (__v2di)__B);
3064	2865	}
3065	2866
3066		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3067		_mm_maskz_max_epi64 (__mmask8 __M, __m128i __A, __m128i __B) {
	2867	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2868	_mm_maskz_max_epi64(__mmask8 __M, __m128i __A, __m128i __B) {
3068	2869	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3069	2870	(__v2di)_mm_max_epi64(__A, __B),
3070	2871	(__v2di)_mm_setzero_si128());
3071	2872	}
3072	2873
3073		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3074		_mm_mask_max_epi64 (__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
	2874	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2875	_mm_mask_max_epi64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3075	2876	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3076	2877	(__v2di)_mm_max_epi64(__A, __B),
3077	2878	(__v2di)__W);
3078	2879	}
3079	2880
3080		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3081		_mm256_max_epi64 (__m256i __A, __m256i __B) {
	2881	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2882	_mm256_max_epi64(__m256i __A, __m256i __B) {
3082	2883	return (__m256i)__builtin_elementwise_max((__v4di)__A, (__v4di)__B);
3083	2884	}
3084	2885
3085		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3086		_mm256_maskz_max_epi64 (__mmask8 __M, __m256i __A, __m256i __B) {
	2886	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2887	_mm256_maskz_max_epi64(__mmask8 __M, __m256i __A, __m256i __B) {
3087	2888	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3088	2889	(__v4di)_mm256_max_epi64(__A, __B),
3089	2890	(__v4di)_mm256_setzero_si256());
3090	2891	}
3091	2892
3092		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3093		_mm256_mask_max_epi64 (__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
	2893	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2894	_mm256_mask_max_epi64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3094	2895	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3095	2896	(__v4di)_mm256_max_epi64(__A, __B),
3096	2897	(__v4di)__W);
3097	2898	}
3098	2899
3099		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2900	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3100	2901	_mm_maskz_max_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
3101	2902	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3102	2903	(__v4si)_mm_max_epu32(__A, __B),
3103	2904	(__v4si)_mm_setzero_si128());
3104	2905	}
3105	2906
3106		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2907	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3107	2908	_mm_mask_max_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3108	2909	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3109	2910	(__v4si)_mm_max_epu32(__A, __B),
3110	2911	(__v4si)__W);
3111	2912	}
3112	2913
3113		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2914	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3114	2915	_mm256_maskz_max_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
3115	2916	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3116	2917	(__v8si)_mm256_max_epu32(__A, __B),
3117	2918	(__v8si)_mm256_setzero_si256());
3118	2919	}
3119	2920
3120		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2921	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3121	2922	_mm256_mask_max_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3122	2923	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3123	2924	(__v8si)_mm256_max_epu32(__A, __B),
3124	2925	(__v8si)__W);
3125	2926	}
3126	2927
3127		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3128		_mm_max_epu64 (__m128i __A, __m128i __B) {
	2928	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2929	_mm_max_epu64(__m128i __A, __m128i __B) {
3129	2930	return (__m128i)__builtin_elementwise_max((__v2du)__A, (__v2du)__B);
3130	2931	}
3131	2932
3132		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3133		_mm_maskz_max_epu64 (__mmask8 __M, __m128i __A, __m128i __B) {
	2933	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2934	_mm_maskz_max_epu64(__mmask8 __M, __m128i __A, __m128i __B) {
3134	2935	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3135	2936	(__v2di)_mm_max_epu64(__A, __B),
3136	2937	(__v2di)_mm_setzero_si128());
3137	2938	}
3138	2939
3139		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3140		_mm_mask_max_epu64 (__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
	2940	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2941	_mm_mask_max_epu64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3141	2942	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3142	2943	(__v2di)_mm_max_epu64(__A, __B),
3143	2944	(__v2di)__W);
3144	2945	}
3145	2946
3146		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3147		_mm256_max_epu64 (__m256i __A, __m256i __B) {
	2947	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2948	_mm256_max_epu64(__m256i __A, __m256i __B) {
3148	2949	return (__m256i)__builtin_elementwise_max((__v4du)__A, (__v4du)__B);
3149	2950	}
3150	2951
3151		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3152		_mm256_maskz_max_epu64 (__mmask8 __M, __m256i __A, __m256i __B) {
	2952	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2953	_mm256_maskz_max_epu64(__mmask8 __M, __m256i __A, __m256i __B) {
3153	2954	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3154	2955	(__v4di)_mm256_max_epu64(__A, __B),
3155	2956	(__v4di)_mm256_setzero_si256());
3156	2957	}
3157	2958
3158		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3159		_mm256_mask_max_epu64 (__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
	2959	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	2960	_mm256_mask_max_epu64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3160	2961	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3161	2962	(__v4di)_mm256_max_epu64(__A, __B),
3162	2963	(__v4di)__W);
3163	2964	}
3164	2965
3165		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2966	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3166	2967	_mm_maskz_min_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
3167	2968	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3168	2969	(__v4si)_mm_min_epi32(__A, __B),
3169	2970	(__v4si)_mm_setzero_si128());
3170	2971	}
3171	2972
3172		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	2973	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3173	2974	_mm_mask_min_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3174	2975	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3175	2976	(__v4si)_mm_min_epi32(__A, __B),
3176	2977	(__v4si)__W);
3177	2978	}
3178	2979
3179		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2980	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3180	2981	_mm256_maskz_min_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
3181	2982	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3182	2983	(__v8si)_mm256_min_epi32(__A, __B),
3183	2984	(__v8si)_mm256_setzero_si256());
3184	2985	}
3185	2986
3186		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	2987	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3187	2988	_mm256_mask_min_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3188	2989	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3189	2990	(__v8si)_mm256_min_epi32(__A, __B),
3190	2991	(__v8si)__W);
3191	2992	}
3192	2993
3193		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3194		_mm_min_epi64 (__m128i __A, __m128i __B) {
	2994	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	2995	_mm_min_epi64(__m128i __A, __m128i __B) {
3195	2996	return (__m128i)__builtin_elementwise_min((__v2di)__A, (__v2di)__B);
3196	2997	}
3197	2998
3198		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3199		_mm_mask_min_epi64 (__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
	2999	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3000	_mm_mask_min_epi64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3200	3001	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3201	3002	(__v2di)_mm_min_epi64(__A, __B),
3202	3003	(__v2di)__W);
3203	3004	}
3204	3005
3205		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3206		_mm_maskz_min_epi64 (__mmask8 __M, __m128i __A, __m128i __B) {
	3006	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3007	_mm_maskz_min_epi64(__mmask8 __M, __m128i __A, __m128i __B) {
3207	3008	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3208	3009	(__v2di)_mm_min_epi64(__A, __B),
3209	3010	(__v2di)_mm_setzero_si128());
3210	3011	}
3211	3012
3212		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3213		_mm256_min_epi64 (__m256i __A, __m256i __B) {
	3013	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3014	_mm256_min_epi64(__m256i __A, __m256i __B) {
3214	3015	return (__m256i)__builtin_elementwise_min((__v4di)__A, (__v4di)__B);
3215	3016	}
3216	3017
3217		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3218		_mm256_mask_min_epi64 (__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
	3018	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3019	_mm256_mask_min_epi64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3219	3020	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3220	3021	(__v4di)_mm256_min_epi64(__A, __B),
3221	3022	(__v4di)__W);
3222	3023	}
3223	3024
3224		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3225		_mm256_maskz_min_epi64 (__mmask8 __M, __m256i __A, __m256i __B) {
	3025	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3026	_mm256_maskz_min_epi64(__mmask8 __M, __m256i __A, __m256i __B) {
3226	3027	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3227	3028	(__v4di)_mm256_min_epi64(__A, __B),
3228	3029	(__v4di)_mm256_setzero_si256());
3229	3030	}
3230	3031
3231		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3032	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3232	3033	_mm_maskz_min_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
3233	3034	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3234	3035	(__v4si)_mm_min_epu32(__A, __B),
3235	3036	(__v4si)_mm_setzero_si128());
3236	3037	}
3237	3038
3238		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3039	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3239	3040	_mm_mask_min_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3240	3041	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
3241	3042	(__v4si)_mm_min_epu32(__A, __B),
3242	3043	(__v4si)__W);
3243	3044	}
3244	3045
3245		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3046	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3246	3047	_mm256_maskz_min_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
3247	3048	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3248	3049	(__v8si)_mm256_min_epu32(__A, __B),
3249	3050	(__v8si)_mm256_setzero_si256());
3250	3051	}
3251	3052
3252		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3053	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3253	3054	_mm256_mask_min_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3254	3055	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
3255	3056	(__v8si)_mm256_min_epu32(__A, __B),
3256	3057	(__v8si)__W);
3257	3058	}
3258	3059
3259		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3260		_mm_min_epu64 (__m128i __A, __m128i __B) {
	3060	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3061	_mm_min_epu64(__m128i __A, __m128i __B) {
3261	3062	return (__m128i)__builtin_elementwise_min((__v2du)__A, (__v2du)__B);
3262	3063	}
3263	3064
3264		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3265		_mm_mask_min_epu64 (__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
	3065	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3066	_mm_mask_min_epu64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
3266	3067	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3267	3068	(__v2di)_mm_min_epu64(__A, __B),
3268	3069	(__v2di)__W);
3269	3070	}
3270	3071
3271		static __inline__ __m128i __DEFAULT_FN_ATTRS128
3272		_mm_maskz_min_epu64 (__mmask8 __M, __m128i __A, __m128i __B) {
	3072	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	3073	_mm_maskz_min_epu64(__mmask8 __M, __m128i __A, __m128i __B) {
3273	3074	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__M,
3274	3075	(__v2di)_mm_min_epu64(__A, __B),
3275	3076	(__v2di)_mm_setzero_si128());
3276	3077	}
3277	3078
3278		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3279		_mm256_min_epu64 (__m256i __A, __m256i __B) {
	3079	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3080	_mm256_min_epu64(__m256i __A, __m256i __B) {
3280	3081	return (__m256i)__builtin_elementwise_min((__v4du)__A, (__v4du)__B);
3281	3082	}
3282	3083
3283		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3284		_mm256_mask_min_epu64 (__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
	3084	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3085	_mm256_mask_min_epu64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
3285	3086	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3286	3087	(__v4di)_mm256_min_epu64(__A, __B),
3287	3088	(__v4di)__W);
3288	3089	}
3289	3090
3290		static __inline__ __m256i __DEFAULT_FN_ATTRS256
3291		_mm256_maskz_min_epu64 (__mmask8 __M, __m256i __A, __m256i __B) {
	3091	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	3092	_mm256_maskz_min_epu64(__mmask8 __M, __m256i __A, __m256i __B) {
3292	3093	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
3293	3094	(__v4di)_mm256_min_epu64(__A, __B),
3294	3095	(__v4di)_mm256_setzero_si256());
...	...	@@ -3691,69 +3492,69 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3691	3492	(__v8sf)_mm256_setzero_ps());
3692	3493	}
3693	3494
3694		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	3495	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
3695	3496	_mm_mask_sub_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
3696	3497	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
3697	3498	(__v2df)_mm_sub_pd(__A, __B),
3698	3499	(__v2df)__W);
3699	3500	}
3700	3501
3701		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	3502	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
3702	3503	_mm_maskz_sub_pd(__mmask8 __U, __m128d __A, __m128d __B) {
3703	3504	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
3704	3505	(__v2df)_mm_sub_pd(__A, __B),
3705	3506	(__v2df)_mm_setzero_pd());
3706	3507	}
3707	3508
3708		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3509	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3709	3510	_mm256_mask_sub_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
3710	3511	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
3711	3512	(__v4df)_mm256_sub_pd(__A, __B),
3712	3513	(__v4df)__W);
3713	3514	}
3714	3515
3715		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3516	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3716	3517	_mm256_maskz_sub_pd(__mmask8 __U, __m256d __A, __m256d __B) {
3717	3518	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
3718	3519	(__v4df)_mm256_sub_pd(__A, __B),
3719	3520	(__v4df)_mm256_setzero_pd());
3720	3521	}
3721	3522
3722		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3523	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3723	3524	_mm_mask_sub_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
3724	3525	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
3725	3526	(__v4sf)_mm_sub_ps(__A, __B),
3726	3527	(__v4sf)__W);
3727	3528	}
3728	3529
3729		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3530	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3730	3531	_mm_maskz_sub_ps(__mmask8 __U, __m128 __A, __m128 __B) {
3731	3532	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
3732	3533	(__v4sf)_mm_sub_ps(__A, __B),
3733	3534	(__v4sf)_mm_setzero_ps());
3734	3535	}
3735	3536
3736		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	3537	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
3737	3538	_mm256_mask_sub_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
3738	3539	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
3739	3540	(__v8sf)_mm256_sub_ps(__A, __B),
3740	3541	(__v8sf)__W);
3741	3542	}
3742	3543
3743		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	3544	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
3744	3545	_mm256_maskz_sub_ps(__mmask8 __U, __m256 __A, __m256 __B) {
3745	3546	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
3746	3547	(__v8sf)_mm256_sub_ps(__A, __B),
3747	3548	(__v8sf)_mm256_setzero_ps());
3748	3549	}
3749	3550
3750		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3551	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3751	3552	_mm_permutex2var_epi32(__m128i __A, __m128i __I, __m128i __B) {
3752	3553	return (__m128i)__builtin_ia32_vpermi2vard128((__v4si) __A, (__v4si)__I,
3753	3554	(__v4si)__B);
3754	3555	}
3755	3556
3756		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3557	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3757	3558	_mm_mask_permutex2var_epi32(__m128i __A, __mmask8 __U, __m128i __I,
3758	3559	__m128i __B) {
3759	3560	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -3761,7 +3562,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3761	3562	(__v4si)__A);
3762	3563	}
3763	3564
3764		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3565	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3765	3566	_mm_mask2_permutex2var_epi32(__m128i __A, __m128i __I, __mmask8 __U,
3766	3567	__m128i __B) {
3767	3568	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -3769,7 +3570,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3769	3570	(__v4si)__I);
3770	3571	}
3771	3572
3772		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3573	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3773	3574	_mm_maskz_permutex2var_epi32(__mmask8 __U, __m128i __A, __m128i __I,
3774	3575	__m128i __B) {
3775	3576	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -3777,13 +3578,13 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3777	3578	(__v4si)_mm_setzero_si128());
3778	3579	}
3779	3580
3780		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3581	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3781	3582	_mm256_permutex2var_epi32(__m256i __A, __m256i __I, __m256i __B) {
3782	3583	return (__m256i)__builtin_ia32_vpermi2vard256((__v8si)__A, (__v8si) __I,
3783	3584	(__v8si) __B);
3784	3585	}
3785	3586
3786		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3587	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3787	3588	_mm256_mask_permutex2var_epi32(__m256i __A, __mmask8 __U, __m256i __I,
3788	3589	__m256i __B) {
3789	3590	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -3791,7 +3592,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3791	3592	(__v8si)__A);
3792	3593	}
3793	3594
3794		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3595	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3795	3596	_mm256_mask2_permutex2var_epi32(__m256i __A, __m256i __I, __mmask8 __U,
3796	3597	__m256i __B) {
3797	3598	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -3799,7 +3600,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3799	3600	(__v8si)__I);
3800	3601	}
3801	3602
3802		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3603	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3803	3604	_mm256_maskz_permutex2var_epi32(__mmask8 __U, __m256i __A, __m256i __I,
3804	3605	__m256i __B) {
3805	3606	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -3807,40 +3608,43 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3807	3608	(__v8si)_mm256_setzero_si256());
3808	3609	}
3809	3610
3810		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	3611	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
3811	3612	_mm_permutex2var_pd(__m128d __A, __m128i __I, __m128d __B) {
3812	3613	return (__m128d)__builtin_ia32_vpermi2varpd128((__v2df)__A, (__v2di)__I,
3813	3614	(__v2df)__B);
3814	3615	}
3815	3616
3816		static __inline__ __m128d __DEFAULT_FN_ATTRS128
3817		_mm_mask_permutex2var_pd(__m128d __A, __mmask8 __U, __m128i __I, __m128d __B) {
	3617	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	3618	_mm_mask_permutex2var_pd(__m128d __A, __mmask8 __U, __m128i __I,
	3619	__m128d __B) {
3818	3620	return (__m128d)__builtin_ia32_selectpd_128(__U,
3819	3621	(__v2df)_mm_permutex2var_pd(__A, __I, __B),
3820	3622	(__v2df)__A);
3821	3623	}
3822	3624
3823		static __inline__ __m128d __DEFAULT_FN_ATTRS128
3824		_mm_mask2_permutex2var_pd(__m128d __A, __m128i __I, __mmask8 __U, __m128d __B) {
	3625	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	3626	_mm_mask2_permutex2var_pd(__m128d __A, __m128i __I, __mmask8 __U,
	3627	__m128d __B) {
3825	3628	return (__m128d)__builtin_ia32_selectpd_128(__U,
3826	3629	(__v2df)_mm_permutex2var_pd(__A, __I, __B),
3827	3630	(__v2df)(__m128d)__I);
3828	3631	}
3829	3632
3830		static __inline__ __m128d __DEFAULT_FN_ATTRS128
3831		_mm_maskz_permutex2var_pd(__mmask8 __U, __m128d __A, __m128i __I, __m128d __B) {
	3633	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	3634	_mm_maskz_permutex2var_pd(__mmask8 __U, __m128d __A, __m128i __I,
	3635	__m128d __B) {
3832	3636	return (__m128d)__builtin_ia32_selectpd_128(__U,
3833	3637	(__v2df)_mm_permutex2var_pd(__A, __I, __B),
3834	3638	(__v2df)_mm_setzero_pd());
3835	3639	}
3836	3640
3837		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3641	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3838	3642	_mm256_permutex2var_pd(__m256d __A, __m256i __I, __m256d __B) {
3839	3643	return (__m256d)__builtin_ia32_vpermi2varpd256((__v4df)__A, (__v4di)__I,
3840	3644	(__v4df)__B);
3841	3645	}
3842	3646
3843		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3647	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3844	3648	_mm256_mask_permutex2var_pd(__m256d __A, __mmask8 __U, __m256i __I,
3845	3649	__m256d __B) {
3846	3650	return (__m256d)__builtin_ia32_selectpd_256(__U,
...	...	@@ -3848,7 +3652,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3848	3652	(__v4df)__A);
3849	3653	}
3850	3654
3851		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3655	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3852	3656	_mm256_mask2_permutex2var_pd(__m256d __A, __m256i __I, __mmask8 __U,
3853	3657	__m256d __B) {
3854	3658	return (__m256d)__builtin_ia32_selectpd_256(__U,
...	...	@@ -3856,7 +3660,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3856	3660	(__v4df)(__m256d)__I);
3857	3661	}
3858	3662
3859		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	3663	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
3860	3664	_mm256_maskz_permutex2var_pd(__mmask8 __U, __m256d __A, __m256i __I,
3861	3665	__m256d __B) {
3862	3666	return (__m256d)__builtin_ia32_selectpd_256(__U,
...	...	@@ -3864,47 +3668,48 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3864	3668	(__v4df)_mm256_setzero_pd());
3865	3669	}
3866	3670
3867		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3671	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3868	3672	_mm_permutex2var_ps(__m128 __A, __m128i __I, __m128 __B) {
3869	3673	return (__m128)__builtin_ia32_vpermi2varps128((__v4sf)__A, (__v4si)__I,
3870	3674	(__v4sf)__B);
3871	3675	}
3872	3676
3873		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3677	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3874	3678	_mm_mask_permutex2var_ps(__m128 __A, __mmask8 __U, __m128i __I, __m128 __B) {
3875	3679	return (__m128)__builtin_ia32_selectps_128(__U,
3876	3680	(__v4sf)_mm_permutex2var_ps(__A, __I, __B),
3877	3681	(__v4sf)__A);
3878	3682	}
3879	3683
3880		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3684	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3881	3685	_mm_mask2_permutex2var_ps(__m128 __A, __m128i __I, __mmask8 __U, __m128 __B) {
3882	3686	return (__m128)__builtin_ia32_selectps_128(__U,
3883	3687	(__v4sf)_mm_permutex2var_ps(__A, __I, __B),
3884	3688	(__v4sf)(__m128)__I);
3885	3689	}
3886	3690
3887		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	3691	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
3888	3692	_mm_maskz_permutex2var_ps(__mmask8 __U, __m128 __A, __m128i __I, __m128 __B) {
3889	3693	return (__m128)__builtin_ia32_selectps_128(__U,
3890	3694	(__v4sf)_mm_permutex2var_ps(__A, __I, __B),
3891	3695	(__v4sf)_mm_setzero_ps());
3892	3696	}
3893	3697
3894		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	3698	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
3895	3699	_mm256_permutex2var_ps(__m256 __A, __m256i __I, __m256 __B) {
3896	3700	return (__m256)__builtin_ia32_vpermi2varps256((__v8sf)__A, (__v8si)__I,
3897	3701	(__v8sf) __B);
3898	3702	}
3899	3703
3900		static __inline__ __m256 __DEFAULT_FN_ATTRS256
3901		_mm256_mask_permutex2var_ps(__m256 __A, __mmask8 __U, __m256i __I, __m256 __B) {
	3704	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	3705	_mm256_mask_permutex2var_ps(__m256 __A, __mmask8 __U, __m256i __I,
	3706	__m256 __B) {
3902	3707	return (__m256)__builtin_ia32_selectps_256(__U,
3903	3708	(__v8sf)_mm256_permutex2var_ps(__A, __I, __B),
3904	3709	(__v8sf)__A);
3905	3710	}
3906	3711
3907		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	3712	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
3908	3713	_mm256_mask2_permutex2var_ps(__m256 __A, __m256i __I, __mmask8 __U,
3909	3714	__m256 __B) {
3910	3715	return (__m256)__builtin_ia32_selectps_256(__U,
...	...	@@ -3912,7 +3717,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3912	3717	(__v8sf)(__m256)__I);
3913	3718	}
3914	3719
3915		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	3720	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
3916	3721	_mm256_maskz_permutex2var_ps(__mmask8 __U, __m256 __A, __m256i __I,
3917	3722	__m256 __B) {
3918	3723	return (__m256)__builtin_ia32_selectps_256(__U,
...	...	@@ -3920,13 +3725,13 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3920	3725	(__v8sf)_mm256_setzero_ps());
3921	3726	}
3922	3727
3923		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3728	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3924	3729	_mm_permutex2var_epi64(__m128i __A, __m128i __I, __m128i __B) {
3925	3730	return (__m128i)__builtin_ia32_vpermi2varq128((__v2di)__A, (__v2di)__I,
3926	3731	(__v2di)__B);
3927	3732	}
3928	3733
3929		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3734	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3930	3735	_mm_mask_permutex2var_epi64(__m128i __A, __mmask8 __U, __m128i __I,
3931	3736	__m128i __B) {
3932	3737	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -3934,7 +3739,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3934	3739	(__v2di)__A);
3935	3740	}
3936	3741
3937		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3742	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3938	3743	_mm_mask2_permutex2var_epi64(__m128i __A, __m128i __I, __mmask8 __U,
3939	3744	__m128i __B) {
3940	3745	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -3942,7 +3747,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3942	3747	(__v2di)__I);
3943	3748	}
3944	3749
3945		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	3750	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
3946	3751	_mm_maskz_permutex2var_epi64(__mmask8 __U, __m128i __A, __m128i __I,
3947	3752	__m128i __B) {
3948	3753	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -3950,14 +3755,13 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3950	3755	(__v2di)_mm_setzero_si128());
3951	3756	}
3952	3757
3953
3954		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3758	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3955	3759	_mm256_permutex2var_epi64(__m256i __A, __m256i __I, __m256i __B) {
3956	3760	return (__m256i)__builtin_ia32_vpermi2varq256((__v4di)__A, (__v4di) __I,
3957	3761	(__v4di) __B);
3958	3762	}
3959	3763
3960		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3764	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3961	3765	_mm256_mask_permutex2var_epi64(__m256i __A, __mmask8 __U, __m256i __I,
3962	3766	__m256i __B) {
3963	3767	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -3965,7 +3769,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3965	3769	(__v4di)__A);
3966	3770	}
3967	3771
3968		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3772	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3969	3773	_mm256_mask2_permutex2var_epi64(__m256i __A, __m256i __I, __mmask8 __U,
3970	3774	__m256i __B) {
3971	3775	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -3973,7 +3777,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
3973	3777	(__v4di)__I);
3974	3778	}
3975	3779
3976		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	3780	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
3977	3781	_mm256_maskz_permutex2var_epi64(__mmask8 __U, __m256i __A, __m256i __I,
3978	3782	__m256i __B) {
3979	3783	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -4355,13 +4159,13 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
4355	4159	(__v4di)_mm256_rol_epi64((a), (b)), \
4356	4160	(__v4di)_mm256_setzero_si256()))
4357	4161
4358		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4162	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4359	4163	_mm_rolv_epi32 (__m128i __A, __m128i __B)
4360	4164	{
4361		return (__m128i)__builtin_ia32_prolvd128((__v4si)__A, (__v4si)__B);
	4165	return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__A, (__v4su)__B);
4362	4166	}
4363	4167
4364		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4168	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4365	4169	_mm_mask_rolv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4366	4170	{
4367	4171	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -4369,7 +4173,7 @@ _mm_mask_rolv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4369	4173	(__v4si)__W);
4370	4174	}
4371	4175
4372		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4176	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4373	4177	_mm_maskz_rolv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
4374	4178	{
4375	4179	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -4377,13 +4181,13 @@ _mm_maskz_rolv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
4377	4181	(__v4si)_mm_setzero_si128());
4378	4182	}
4379	4183
4380		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4184	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4381	4185	_mm256_rolv_epi32 (__m256i __A, __m256i __B)
4382	4186	{
4383		return (__m256i)__builtin_ia32_prolvd256((__v8si)__A, (__v8si)__B);
	4187	return (__m256i)__builtin_elementwise_fshl((__v8su)__A, (__v8su)__A, (__v8su)__B);
4384	4188	}
4385	4189
4386		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4190	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4387	4191	_mm256_mask_rolv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4388	4192	{
4389	4193	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -4391,7 +4195,7 @@ _mm256_mask_rolv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4391	4195	(__v8si)__W);
4392	4196	}
4393	4197
4394		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4198	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4395	4199	_mm256_maskz_rolv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
4396	4200	{
4397	4201	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -4399,13 +4203,13 @@ _mm256_maskz_rolv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
4399	4203	(__v8si)_mm256_setzero_si256());
4400	4204	}
4401	4205
4402		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4206	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4403	4207	_mm_rolv_epi64 (__m128i __A, __m128i __B)
4404	4208	{
4405		return (__m128i)__builtin_ia32_prolvq128((__v2di)__A, (__v2di)__B);
	4209	return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__A, (__v2du)__B);
4406	4210	}
4407	4211
4408		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4212	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4409	4213	_mm_mask_rolv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4410	4214	{
4411	4215	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -4413,7 +4217,7 @@ _mm_mask_rolv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4413	4217	(__v2di)__W);
4414	4218	}
4415	4219
4416		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4220	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4417	4221	_mm_maskz_rolv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
4418	4222	{
4419	4223	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -4421,13 +4225,13 @@ _mm_maskz_rolv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
4421	4225	(__v2di)_mm_setzero_si128());
4422	4226	}
4423	4227
4424		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4228	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4425	4229	_mm256_rolv_epi64 (__m256i __A, __m256i __B)
4426	4230	{
4427		return (__m256i)__builtin_ia32_prolvq256((__v4di)__A, (__v4di)__B);
	4231	return (__m256i)__builtin_elementwise_fshl((__v4du)__A, (__v4du)__A, (__v4du)__B);
4428	4232	}
4429	4233
4430		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4234	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4431	4235	_mm256_mask_rolv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4432	4236	{
4433	4237	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -4435,7 +4239,7 @@ _mm256_mask_rolv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4435	4239	(__v4di)__W);
4436	4240	}
4437	4241
4438		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4242	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4439	4243	_mm256_maskz_rolv_epi64 (__mmask8 __U, __m256i __A, __m256i __B)
4440	4244	{
4441	4245	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -4495,33 +4299,29 @@ _mm256_maskz_rolv_epi64 (__mmask8 __U, __m256i __A, __m256i __B)
4495	4299	(__v4di)_mm256_ror_epi64((a), (b)), \
4496	4300	(__v4di)_mm256_setzero_si256()))
4497	4301
4498		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4499		_mm_mask_sll_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4500		{
	4302	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4303	_mm_mask_sll_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
4501	4304	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
4502	4305	(__v4si)_mm_sll_epi32(__A, __B),
4503	4306	(__v4si)__W);
4504	4307	}
4505	4308
4506		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4507		_mm_maskz_sll_epi32(__mmask8 __U, __m128i __A, __m128i __B)
4508		{
	4309	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4310	_mm_maskz_sll_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
4509	4311	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
4510	4312	(__v4si)_mm_sll_epi32(__A, __B),
4511	4313	(__v4si)_mm_setzero_si128());
4512	4314	}
4513	4315
4514		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4515		_mm256_mask_sll_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
4516		{
	4316	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4317	_mm256_mask_sll_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
4517	4318	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4518	4319	(__v8si)_mm256_sll_epi32(__A, __B),
4519	4320	(__v8si)__W);
4520	4321	}
4521	4322
4522		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4523		_mm256_maskz_sll_epi32(__mmask8 __U, __m256i __A, __m128i __B)
4524		{
	4323	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4324	_mm256_maskz_sll_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
4525	4325	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4526	4326	(__v8si)_mm256_sll_epi32(__A, __B),
4527	4327	(__v8si)_mm256_setzero_si256());
...	...	@@ -4543,49 +4343,44 @@ _mm_maskz_slli_epi32(__mmask8 __U, __m128i __A, unsigned int __B)
4543	4343	(__v4si)_mm_setzero_si128());
4544	4344	}
4545	4345
4546		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4547		_mm256_mask_slli_epi32(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B)
4548		{
	4346	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4347	_mm256_mask_slli_epi32(__m256i __W, __mmask8 __U, __m256i __A,
	4348	unsigned int __B) {
4549	4349	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4550	4350	(__v8si)_mm256_slli_epi32(__A, (int)__B),
4551	4351	(__v8si)__W);
4552	4352	}
4553	4353
4554		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4555		_mm256_maskz_slli_epi32(__mmask8 __U, __m256i __A, unsigned int __B)
4556		{
	4354	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4355	_mm256_maskz_slli_epi32(__mmask8 __U, __m256i __A, unsigned int __B) {
4557	4356	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4558	4357	(__v8si)_mm256_slli_epi32(__A, (int)__B),
4559	4358	(__v8si)_mm256_setzero_si256());
4560	4359	}
4561	4360
4562		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4563		_mm_mask_sll_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4564		{
	4361	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4362	_mm_mask_sll_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
4565	4363	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
4566	4364	(__v2di)_mm_sll_epi64(__A, __B),
4567	4365	(__v2di)__W);
4568	4366	}
4569	4367
4570		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4571		_mm_maskz_sll_epi64(__mmask8 __U, __m128i __A, __m128i __B)
4572		{
	4368	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4369	_mm_maskz_sll_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
4573	4370	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
4574	4371	(__v2di)_mm_sll_epi64(__A, __B),
4575	4372	(__v2di)_mm_setzero_si128());
4576	4373	}
4577	4374
4578		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4579		_mm256_mask_sll_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
4580		{
	4375	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4376	_mm256_mask_sll_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
4581	4377	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4582	4378	(__v4di)_mm256_sll_epi64(__A, __B),
4583	4379	(__v4di)__W);
4584	4380	}
4585	4381
4586		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4587		_mm256_maskz_sll_epi64(__mmask8 __U, __m256i __A, __m128i __B)
4588		{
	4382	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4383	_mm256_maskz_sll_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
4589	4384	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4590	4385	(__v4di)_mm256_sll_epi64(__A, __B),
4591	4386	(__v4di)_mm256_setzero_si256());
...	...	@@ -4607,29 +4402,28 @@ _mm_maskz_slli_epi64(__mmask8 __U, __m128i __A, unsigned int __B)
4607	4402	(__v2di)_mm_setzero_si128());
4608	4403	}
4609	4404
4610		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4611		_mm256_mask_slli_epi64(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B)
4612		{
	4405	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4406	_mm256_mask_slli_epi64(__m256i __W, __mmask8 __U, __m256i __A,
	4407	unsigned int __B) {
4613	4408	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4614	4409	(__v4di)_mm256_slli_epi64(__A, (int)__B),
4615	4410	(__v4di)__W);
4616	4411	}
4617	4412
4618		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4619		_mm256_maskz_slli_epi64(__mmask8 __U, __m256i __A, unsigned int __B)
4620		{
	4413	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4414	_mm256_maskz_slli_epi64(__mmask8 __U, __m256i __A, unsigned int __B) {
4621	4415	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4622	4416	(__v4di)_mm256_slli_epi64(__A, (int)__B),
4623	4417	(__v4di)_mm256_setzero_si256());
4624	4418	}
4625	4419
4626		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4420	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4627	4421	_mm_rorv_epi32 (__m128i __A, __m128i __B)
4628	4422	{
4629		return (__m128i)__builtin_ia32_prorvd128((__v4si)__A, (__v4si)__B);
	4423	return (__m128i)__builtin_elementwise_fshr((__v4su)__A, (__v4su)__A, (__v4su)__B);
4630	4424	}
4631	4425
4632		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4426	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4633	4427	_mm_mask_rorv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4634	4428	{
4635	4429	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -4637,7 +4431,7 @@ _mm_mask_rorv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4637	4431	(__v4si)__W);
4638	4432	}
4639	4433
4640		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4434	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4641	4435	_mm_maskz_rorv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
4642	4436	{
4643	4437	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -4645,13 +4439,13 @@ _mm_maskz_rorv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
4645	4439	(__v4si)_mm_setzero_si128());
4646	4440	}
4647	4441
4648		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4442	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4649	4443	_mm256_rorv_epi32 (__m256i __A, __m256i __B)
4650	4444	{
4651		return (__m256i)__builtin_ia32_prorvd256((__v8si)__A, (__v8si)__B);
	4445	return (__m256i)__builtin_elementwise_fshr((__v8su)__A, (__v8su)__A, (__v8su)__B);
4652	4446	}
4653	4447
4654		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4448	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4655	4449	_mm256_mask_rorv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4656	4450	{
4657	4451	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -4659,7 +4453,7 @@ _mm256_mask_rorv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4659	4453	(__v8si)__W);
4660	4454	}
4661	4455
4662		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4456	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4663	4457	_mm256_maskz_rorv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
4664	4458	{
4665	4459	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -4667,13 +4461,13 @@ _mm256_maskz_rorv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
4667	4461	(__v8si)_mm256_setzero_si256());
4668	4462	}
4669	4463
4670		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4464	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4671	4465	_mm_rorv_epi64 (__m128i __A, __m128i __B)
4672	4466	{
4673		return (__m128i)__builtin_ia32_prorvq128((__v2di)__A, (__v2di)__B);
	4467	return (__m128i)__builtin_elementwise_fshr((__v2du)__A, (__v2du)__A, (__v2du)__B);
4674	4468	}
4675	4469
4676		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4470	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4677	4471	_mm_mask_rorv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4678	4472	{
4679	4473	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -4681,7 +4475,7 @@ _mm_mask_rorv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4681	4475	(__v2di)__W);
4682	4476	}
4683	4477
4684		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4478	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4685	4479	_mm_maskz_rorv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
4686	4480	{
4687	4481	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -4689,13 +4483,13 @@ _mm_maskz_rorv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
4689	4483	(__v2di)_mm_setzero_si128());
4690	4484	}
4691	4485
4692		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4486	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4693	4487	_mm256_rorv_epi64 (__m256i __A, __m256i __B)
4694	4488	{
4695		return (__m256i)__builtin_ia32_prorvq256((__v4di)__A, (__v4di)__B);
	4489	return (__m256i)__builtin_elementwise_fshr((__v4du)__A, (__v4du)__A, (__v4du)__B);
4696	4490	}
4697	4491
4698		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4492	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4699	4493	_mm256_mask_rorv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4700	4494	{
4701	4495	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -4703,7 +4497,7 @@ _mm256_mask_rorv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
4703	4497	(__v4di)__W);
4704	4498	}
4705	4499
4706		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4500	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4707	4501	_mm256_maskz_rorv_epi64 (__mmask8 __U, __m256i __A, __m256i __B)
4708	4502	{
4709	4503	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -4711,7 +4505,7 @@ _mm256_maskz_rorv_epi64 (__mmask8 __U, __m256i __A, __m256i __B)
4711	4505	(__v4di)_mm256_setzero_si256());
4712	4506	}
4713	4507
4714		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4508	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4715	4509	_mm_mask_sllv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4716	4510	{
4717	4511	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -4719,7 +4513,7 @@ _mm_mask_sllv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4719	4513	(__v2di)__W);
4720	4514	}
4721	4515
4722		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4516	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4723	4517	_mm_maskz_sllv_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
4724	4518	{
4725	4519	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -4727,7 +4521,7 @@ _mm_maskz_sllv_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
4727	4521	(__v2di)_mm_setzero_si128());
4728	4522	}
4729	4523
4730		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4524	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4731	4525	_mm256_mask_sllv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4732	4526	{
4733	4527	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -4735,7 +4529,7 @@ _mm256_mask_sllv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4735	4529	(__v4di)__W);
4736	4530	}
4737	4531
4738		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4532	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4739	4533	_mm256_maskz_sllv_epi64(__mmask8 __U, __m256i __X, __m256i __Y)
4740	4534	{
4741	4535	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -4743,7 +4537,7 @@ _mm256_maskz_sllv_epi64(__mmask8 __U, __m256i __X, __m256i __Y)
4743	4537	(__v4di)_mm256_setzero_si256());
4744	4538	}
4745	4539
4746		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4540	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4747	4541	_mm_mask_sllv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4748	4542	{
4749	4543	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4751,7 +4545,7 @@ _mm_mask_sllv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4751	4545	(__v4si)__W);
4752	4546	}
4753	4547
4754		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4548	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4755	4549	_mm_maskz_sllv_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4756	4550	{
4757	4551	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4759,7 +4553,7 @@ _mm_maskz_sllv_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4759	4553	(__v4si)_mm_setzero_si128());
4760	4554	}
4761	4555
4762		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4556	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4763	4557	_mm256_mask_sllv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4764	4558	{
4765	4559	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4767,7 +4561,7 @@ _mm256_mask_sllv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4767	4561	(__v8si)__W);
4768	4562	}
4769	4563
4770		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4564	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4771	4565	_mm256_maskz_sllv_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4772	4566	{
4773	4567	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4775,7 +4569,7 @@ _mm256_maskz_sllv_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4775	4569	(__v8si)_mm256_setzero_si256());
4776	4570	}
4777	4571
4778		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4572	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4779	4573	_mm_mask_srlv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4780	4574	{
4781	4575	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -4783,7 +4577,7 @@ _mm_mask_srlv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4783	4577	(__v2di)__W);
4784	4578	}
4785	4579
4786		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4580	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4787	4581	_mm_maskz_srlv_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
4788	4582	{
4789	4583	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -4791,7 +4585,7 @@ _mm_maskz_srlv_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
4791	4585	(__v2di)_mm_setzero_si128());
4792	4586	}
4793	4587
4794		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4588	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4795	4589	_mm256_mask_srlv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4796	4590	{
4797	4591	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -4799,7 +4593,7 @@ _mm256_mask_srlv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4799	4593	(__v4di)__W);
4800	4594	}
4801	4595
4802		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4596	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4803	4597	_mm256_maskz_srlv_epi64(__mmask8 __U, __m256i __X, __m256i __Y)
4804	4598	{
4805	4599	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -4807,7 +4601,7 @@ _mm256_maskz_srlv_epi64(__mmask8 __U, __m256i __X, __m256i __Y)
4807	4601	(__v4di)_mm256_setzero_si256());
4808	4602	}
4809	4603
4810		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4604	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4811	4605	_mm_mask_srlv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4812	4606	{
4813	4607	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4815,7 +4609,7 @@ _mm_mask_srlv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4815	4609	(__v4si)__W);
4816	4610	}
4817	4611
4818		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4612	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4819	4613	_mm_maskz_srlv_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4820	4614	{
4821	4615	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4823,7 +4617,7 @@ _mm_maskz_srlv_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4823	4617	(__v4si)_mm_setzero_si128());
4824	4618	}
4825	4619
4826		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4620	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4827	4621	_mm256_mask_srlv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4828	4622	{
4829	4623	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4831,7 +4625,7 @@ _mm256_mask_srlv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4831	4625	(__v8si)__W);
4832	4626	}
4833	4627
4834		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4628	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4835	4629	_mm256_maskz_srlv_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4836	4630	{
4837	4631	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4839,33 +4633,29 @@ _mm256_maskz_srlv_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4839	4633	(__v8si)_mm256_setzero_si256());
4840	4634	}
4841	4635
4842		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4843		_mm_mask_srl_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4844		{
	4636	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4637	_mm_mask_srl_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
4845	4638	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
4846	4639	(__v4si)_mm_srl_epi32(__A, __B),
4847	4640	(__v4si)__W);
4848	4641	}
4849	4642
4850		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4851		_mm_maskz_srl_epi32(__mmask8 __U, __m128i __A, __m128i __B)
4852		{
	4643	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4644	_mm_maskz_srl_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
4853	4645	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
4854	4646	(__v4si)_mm_srl_epi32(__A, __B),
4855	4647	(__v4si)_mm_setzero_si128());
4856	4648	}
4857	4649
4858		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4859		_mm256_mask_srl_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
4860		{
	4650	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4651	_mm256_mask_srl_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
4861	4652	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4862	4653	(__v8si)_mm256_srl_epi32(__A, __B),
4863	4654	(__v8si)__W);
4864	4655	}
4865	4656
4866		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4867		_mm256_maskz_srl_epi32(__mmask8 __U, __m256i __A, __m128i __B)
4868		{
	4657	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4658	_mm256_maskz_srl_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
4869	4659	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4870	4660	(__v8si)_mm256_srl_epi32(__A, __B),
4871	4661	(__v8si)_mm256_setzero_si256());
...	...	@@ -4887,49 +4677,44 @@ _mm_maskz_srli_epi32(__mmask8 __U, __m128i __A, unsigned int __B)
4887	4677	(__v4si)_mm_setzero_si128());
4888	4678	}
4889	4679
4890		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4891		_mm256_mask_srli_epi32(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B)
4892		{
	4680	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4681	_mm256_mask_srli_epi32(__m256i __W, __mmask8 __U, __m256i __A,
	4682	unsigned int __B) {
4893	4683	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4894	4684	(__v8si)_mm256_srli_epi32(__A, (int)__B),
4895	4685	(__v8si)__W);
4896	4686	}
4897	4687
4898		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4899		_mm256_maskz_srli_epi32(__mmask8 __U, __m256i __A, unsigned int __B)
4900		{
	4688	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4689	_mm256_maskz_srli_epi32(__mmask8 __U, __m256i __A, unsigned int __B) {
4901	4690	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
4902	4691	(__v8si)_mm256_srli_epi32(__A, (int)__B),
4903	4692	(__v8si)_mm256_setzero_si256());
4904	4693	}
4905	4694
4906		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4907		_mm_mask_srl_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
4908		{
	4695	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4696	_mm_mask_srl_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
4909	4697	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
4910	4698	(__v2di)_mm_srl_epi64(__A, __B),
4911	4699	(__v2di)__W);
4912	4700	}
4913	4701
4914		static __inline__ __m128i __DEFAULT_FN_ATTRS128
4915		_mm_maskz_srl_epi64(__mmask8 __U, __m128i __A, __m128i __B)
4916		{
	4702	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	4703	_mm_maskz_srl_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
4917	4704	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
4918	4705	(__v2di)_mm_srl_epi64(__A, __B),
4919	4706	(__v2di)_mm_setzero_si128());
4920	4707	}
4921	4708
4922		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4923		_mm256_mask_srl_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
4924		{
	4709	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4710	_mm256_mask_srl_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
4925	4711	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4926	4712	(__v4di)_mm256_srl_epi64(__A, __B),
4927	4713	(__v4di)__W);
4928	4714	}
4929	4715
4930		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4931		_mm256_maskz_srl_epi64(__mmask8 __U, __m256i __A, __m128i __B)
4932		{
	4716	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4717	_mm256_maskz_srl_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
4933	4718	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4934	4719	(__v4di)_mm256_srl_epi64(__A, __B),
4935	4720	(__v4di)_mm256_setzero_si256());
...	...	@@ -4951,23 +4736,22 @@ _mm_maskz_srli_epi64(__mmask8 __U, __m128i __A, unsigned int __B)
4951	4736	(__v2di)_mm_setzero_si128());
4952	4737	}
4953	4738
4954		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4955		_mm256_mask_srli_epi64(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B)
4956		{
	4739	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4740	_mm256_mask_srli_epi64(__m256i __W, __mmask8 __U, __m256i __A,
	4741	unsigned int __B) {
4957	4742	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4958	4743	(__v4di)_mm256_srli_epi64(__A, (int)__B),
4959	4744	(__v4di)__W);
4960	4745	}
4961	4746
4962		static __inline__ __m256i __DEFAULT_FN_ATTRS256
4963		_mm256_maskz_srli_epi64(__mmask8 __U, __m256i __A, unsigned int __B)
4964		{
	4747	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	4748	_mm256_maskz_srli_epi64(__mmask8 __U, __m256i __A, unsigned int __B) {
4965	4749	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
4966	4750	(__v4di)_mm256_srli_epi64(__A, (int)__B),
4967	4751	(__v4di)_mm256_setzero_si256());
4968	4752	}
4969	4753
4970		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4754	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4971	4755	_mm_mask_srav_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4972	4756	{
4973	4757	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4975,7 +4759,7 @@ _mm_mask_srav_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
4975	4759	(__v4si)__W);
4976	4760	}
4977	4761
4978		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4762	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
4979	4763	_mm_maskz_srav_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4980	4764	{
4981	4765	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
...	...	@@ -4983,7 +4767,7 @@ _mm_maskz_srav_epi32(__mmask8 __U, __m128i __X, __m128i __Y)
4983	4767	(__v4si)_mm_setzero_si128());
4984	4768	}
4985	4769
4986		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4770	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4987	4771	_mm256_mask_srav_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4988	4772	{
4989	4773	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4991,7 +4775,7 @@ _mm256_mask_srav_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
4991	4775	(__v8si)__W);
4992	4776	}
4993	4777
4994		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4778	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
4995	4779	_mm256_maskz_srav_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4996	4780	{
4997	4781	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
...	...	@@ -4999,13 +4783,13 @@ _mm256_maskz_srav_epi32(__mmask8 __U, __m256i __X, __m256i __Y)
4999	4783	(__v8si)_mm256_setzero_si256());
5000	4784	}
5001	4785
5002		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4786	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
5003	4787	_mm_srav_epi64(__m128i __X, __m128i __Y)
5004	4788	{
5005	4789	return (__m128i)__builtin_ia32_psravq128((__v2di)__X, (__v2di)__Y);
5006	4790	}
5007	4791
5008		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4792	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
5009	4793	_mm_mask_srav_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
5010	4794	{
5011	4795	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -5013,7 +4797,7 @@ _mm_mask_srav_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y)
5013	4797	(__v2di)__W);
5014	4798	}
5015	4799
5016		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	4800	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
5017	4801	_mm_maskz_srav_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
5018	4802	{
5019	4803	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
...	...	@@ -5021,13 +4805,13 @@ _mm_maskz_srav_epi64(__mmask8 __U, __m128i __X, __m128i __Y)
5021	4805	(__v2di)_mm_setzero_si128());
5022	4806	}
5023	4807
5024		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4808	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
5025	4809	_mm256_srav_epi64(__m256i __X, __m256i __Y)
5026	4810	{
5027	4811	return (__m256i)__builtin_ia32_psravq256((__v4di)__X, (__v4di) __Y);
5028	4812	}
5029	4813
5030		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4814	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
5031	4815	_mm256_mask_srav_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
5032	4816	{
5033	4817	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -5035,7 +4819,7 @@ _mm256_mask_srav_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y)
5035	4819	(__v4di)__W);
5036	4820	}
5037	4821
5038		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	4822	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
5039	4823	_mm256_maskz_srav_epi64 (__mmask8 __U, __m256i __X, __m256i __Y)
5040	4824	{
5041	4825	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
...	...	@@ -5296,69 +5080,55 @@ _mm256_maskz_movedup_pd (__mmask8 __U, __m256d __A)
5296	5080	(__v4df)_mm256_setzero_pd());
5297	5081	}
5298	5082
5299		static __inline__ __m128i __DEFAULT_FN_ATTRS128
5300		_mm_mask_set1_epi32(__m128i __O, __mmask8 __M, int __A)
5301		{
5302		return (__m128i)__builtin_ia32_selectd_128(__M,
5303		(__v4si) _mm_set1_epi32(__A),
5304		(__v4si)__O);
	5083	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	5084	_mm_mask_set1_epi32(__m128i __O, __mmask8 __M, int __A) {
	5085	return (__m128i)__builtin_ia32_selectd_128(__M, (__v4si)_mm_set1_epi32(__A),
	5086	(__v4si)__O);
5305	5087	}
5306	5088
5307		static __inline__ __m128i __DEFAULT_FN_ATTRS128
5308		_mm_maskz_set1_epi32( __mmask8 __M, int __A)
5309		{
5310		return (__m128i)__builtin_ia32_selectd_128(__M,
5311		(__v4si) _mm_set1_epi32(__A),
5312		(__v4si)_mm_setzero_si128());
	5089	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	5090	_mm_maskz_set1_epi32(__mmask8 __M, int __A) {
	5091	return (__m128i)__builtin_ia32_selectd_128(__M, (__v4si)_mm_set1_epi32(__A),
	5092	(__v4si)_mm_setzero_si128());
5313	5093	}
5314	5094
5315		static __inline__ __m256i __DEFAULT_FN_ATTRS256
5316		_mm256_mask_set1_epi32(__m256i __O, __mmask8 __M, int __A)
5317		{
5318		return (__m256i)__builtin_ia32_selectd_256(__M,
5319		(__v8si) _mm256_set1_epi32(__A),
5320		(__v8si)__O);
	5095	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	5096	_mm256_mask_set1_epi32(__m256i __O, __mmask8 __M, int __A) {
	5097	return (__m256i)__builtin_ia32_selectd_256(
	5098	__M, (__v8si)_mm256_set1_epi32(__A), (__v8si)__O);
5321	5099	}
5322	5100
5323		static __inline__ __m256i __DEFAULT_FN_ATTRS256
5324		_mm256_maskz_set1_epi32( __mmask8 __M, int __A)
5325		{
5326		return (__m256i)__builtin_ia32_selectd_256(__M,
5327		(__v8si) _mm256_set1_epi32(__A),
5328		(__v8si)_mm256_setzero_si256());
	5101	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	5102	_mm256_maskz_set1_epi32(__mmask8 __M, int __A) {
	5103	return (__m256i)__builtin_ia32_selectd_256(
	5104	__M, (__v8si)_mm256_set1_epi32(__A), (__v8si)_mm256_setzero_si256());
5329	5105	}
5330	5106
5331
5332		static __inline__ __m128i __DEFAULT_FN_ATTRS128
5333		_mm_mask_set1_epi64 (__m128i __O, __mmask8 __M, long long __A)
5334		{
	5107	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	5108	_mm_mask_set1_epi64(__m128i __O, __mmask8 __M, long long __A) {
5335	5109	return (__m128i) __builtin_ia32_selectq_128(__M,
5336	5110	(__v2di) _mm_set1_epi64x(__A),
5337	5111	(__v2di) __O);
5338	5112	}
5339	5113
5340		static __inline__ __m128i __DEFAULT_FN_ATTRS128
5341		_mm_maskz_set1_epi64 (__mmask8 __M, long long __A)
5342		{
	5114	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	5115	_mm_maskz_set1_epi64(__mmask8 __M, long long __A) {
5343	5116	return (__m128i) __builtin_ia32_selectq_128(__M,
5344	5117	(__v2di) _mm_set1_epi64x(__A),
5345	5118	(__v2di) _mm_setzero_si128());
5346	5119	}
5347	5120
5348		static __inline__ __m256i __DEFAULT_FN_ATTRS256
5349		_mm256_mask_set1_epi64 (__m256i __O, __mmask8 __M, long long __A)
5350		{
	5121	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	5122	_mm256_mask_set1_epi64(__m256i __O, __mmask8 __M, long long __A) {
5351	5123	return (__m256i) __builtin_ia32_selectq_256(__M,
5352	5124	(__v4di) _mm256_set1_epi64x(__A),
5353	5125	(__v4di) __O) ;
5354	5126	}
5355	5127
5356		static __inline__ __m256i __DEFAULT_FN_ATTRS256
5357		_mm256_maskz_set1_epi64 (__mmask8 __M, long long __A)
5358		{
5359		return (__m256i) __builtin_ia32_selectq_256(__M,
5360		(__v4di) _mm256_set1_epi64x(__A),
5361		(__v4di) _mm256_setzero_si256());
	5128	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	5129	_mm256_maskz_set1_epi64(__mmask8 __M, long long __A) {
	5130	return (__m256i)__builtin_ia32_selectq_256(
	5131	__M, (__v4di)_mm256_set1_epi64x(__A), (__v4di)_mm256_setzero_si256());
5362	5132	}
5363	5133
5364	5134	#define _mm_fixupimm_pd(A, B, C, imm) \
...	...	@@ -5805,130 +5575,113 @@ _mm256_mask_storeu_ps (void *__P, __mmask8 __U, __m256 __A)
5805	5575	(__mmask8) __U);
5806	5576	}
5807	5577
5808
5809		static __inline__ __m128d __DEFAULT_FN_ATTRS128
5810		_mm_mask_unpackhi_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
5811		{
	5578	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5579	_mm_mask_unpackhi_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
5812	5580	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
5813	5581	(__v2df)_mm_unpackhi_pd(__A, __B),
5814	5582	(__v2df)__W);
5815	5583	}
5816	5584
5817		static __inline__ __m128d __DEFAULT_FN_ATTRS128
5818		_mm_maskz_unpackhi_pd(__mmask8 __U, __m128d __A, __m128d __B)
5819		{
	5585	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5586	_mm_maskz_unpackhi_pd(__mmask8 __U, __m128d __A, __m128d __B) {
5820	5587	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
5821	5588	(__v2df)_mm_unpackhi_pd(__A, __B),
5822	5589	(__v2df)_mm_setzero_pd());
5823	5590	}
5824	5591
5825		static __inline__ __m256d __DEFAULT_FN_ATTRS256
5826		_mm256_mask_unpackhi_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B)
5827		{
	5592	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5593	_mm256_mask_unpackhi_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
5828	5594	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
5829	5595	(__v4df)_mm256_unpackhi_pd(__A, __B),
5830	5596	(__v4df)__W);
5831	5597	}
5832	5598
5833		static __inline__ __m256d __DEFAULT_FN_ATTRS256
5834		_mm256_maskz_unpackhi_pd(__mmask8 __U, __m256d __A, __m256d __B)
5835		{
	5599	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5600	_mm256_maskz_unpackhi_pd(__mmask8 __U, __m256d __A, __m256d __B) {
5836	5601	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
5837	5602	(__v4df)_mm256_unpackhi_pd(__A, __B),
5838	5603	(__v4df)_mm256_setzero_pd());
5839	5604	}
5840	5605
5841		static __inline__ __m128 __DEFAULT_FN_ATTRS128
5842		_mm_mask_unpackhi_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
5843		{
	5606	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5607	_mm_mask_unpackhi_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
5844	5608	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
5845	5609	(__v4sf)_mm_unpackhi_ps(__A, __B),
5846	5610	(__v4sf)__W);
5847	5611	}
5848	5612
5849		static __inline__ __m128 __DEFAULT_FN_ATTRS128
5850		_mm_maskz_unpackhi_ps(__mmask8 __U, __m128 __A, __m128 __B)
5851		{
	5613	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5614	_mm_maskz_unpackhi_ps(__mmask8 __U, __m128 __A, __m128 __B) {
5852	5615	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
5853	5616	(__v4sf)_mm_unpackhi_ps(__A, __B),
5854	5617	(__v4sf)_mm_setzero_ps());
5855	5618	}
5856	5619
5857		static __inline__ __m256 __DEFAULT_FN_ATTRS256
5858		_mm256_mask_unpackhi_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B)
5859		{
	5620	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5621	_mm256_mask_unpackhi_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
5860	5622	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
5861	5623	(__v8sf)_mm256_unpackhi_ps(__A, __B),
5862	5624	(__v8sf)__W);
5863	5625	}
5864	5626
5865		static __inline__ __m256 __DEFAULT_FN_ATTRS256
5866		_mm256_maskz_unpackhi_ps(__mmask8 __U, __m256 __A, __m256 __B)
5867		{
	5627	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5628	_mm256_maskz_unpackhi_ps(__mmask8 __U, __m256 __A, __m256 __B) {
5868	5629	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
5869	5630	(__v8sf)_mm256_unpackhi_ps(__A, __B),
5870	5631	(__v8sf)_mm256_setzero_ps());
5871	5632	}
5872	5633
5873		static __inline__ __m128d __DEFAULT_FN_ATTRS128
5874		_mm_mask_unpacklo_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
5875		{
	5634	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5635	_mm_mask_unpacklo_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
5876	5636	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
5877	5637	(__v2df)_mm_unpacklo_pd(__A, __B),
5878	5638	(__v2df)__W);
5879	5639	}
5880	5640
5881		static __inline__ __m128d __DEFAULT_FN_ATTRS128
5882		_mm_maskz_unpacklo_pd(__mmask8 __U, __m128d __A, __m128d __B)
5883		{
	5641	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5642	_mm_maskz_unpacklo_pd(__mmask8 __U, __m128d __A, __m128d __B) {
5884	5643	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
5885	5644	(__v2df)_mm_unpacklo_pd(__A, __B),
5886	5645	(__v2df)_mm_setzero_pd());
5887	5646	}
5888	5647
5889		static __inline__ __m256d __DEFAULT_FN_ATTRS256
5890		_mm256_mask_unpacklo_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B)
5891		{
	5648	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5649	_mm256_mask_unpacklo_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
5892	5650	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
5893	5651	(__v4df)_mm256_unpacklo_pd(__A, __B),
5894	5652	(__v4df)__W);
5895	5653	}
5896	5654
5897		static __inline__ __m256d __DEFAULT_FN_ATTRS256
5898		_mm256_maskz_unpacklo_pd(__mmask8 __U, __m256d __A, __m256d __B)
5899		{
	5655	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5656	_mm256_maskz_unpacklo_pd(__mmask8 __U, __m256d __A, __m256d __B) {
5900	5657	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
5901	5658	(__v4df)_mm256_unpacklo_pd(__A, __B),
5902	5659	(__v4df)_mm256_setzero_pd());
5903	5660	}
5904	5661
5905		static __inline__ __m128 __DEFAULT_FN_ATTRS128
5906		_mm_mask_unpacklo_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
5907		{
	5662	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5663	_mm_mask_unpacklo_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
5908	5664	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
5909	5665	(__v4sf)_mm_unpacklo_ps(__A, __B),
5910	5666	(__v4sf)__W);
5911	5667	}
5912	5668
5913		static __inline__ __m128 __DEFAULT_FN_ATTRS128
5914		_mm_maskz_unpacklo_ps(__mmask8 __U, __m128 __A, __m128 __B)
5915		{
	5669	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5670	_mm_maskz_unpacklo_ps(__mmask8 __U, __m128 __A, __m128 __B) {
5916	5671	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
5917	5672	(__v4sf)_mm_unpacklo_ps(__A, __B),
5918	5673	(__v4sf)_mm_setzero_ps());
5919	5674	}
5920	5675
5921		static __inline__ __m256 __DEFAULT_FN_ATTRS256
5922		_mm256_mask_unpacklo_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B)
5923		{
	5676	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5677	_mm256_mask_unpacklo_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
5924	5678	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
5925	5679	(__v8sf)_mm256_unpacklo_ps(__A, __B),
5926	5680	(__v8sf)__W);
5927	5681	}
5928	5682
5929		static __inline__ __m256 __DEFAULT_FN_ATTRS256
5930		_mm256_maskz_unpacklo_ps(__mmask8 __U, __m256 __A, __m256 __B)
5931		{
	5683	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5684	_mm256_maskz_unpacklo_ps(__mmask8 __U, __m256 __A, __m256 __B) {
5932	5685	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
5933	5686	(__v8sf)_mm256_unpacklo_ps(__A, __B),
5934	5687	(__v8sf)_mm256_setzero_ps());
...	...	@@ -6078,65 +5831,57 @@ _mm256_maskz_rcp14_ps (__mmask8 __U, __m256 __A)
6078	5831	(__v8sf)_mm256_permute_ps((X), (C)), \
6079	5832	(__v8sf)_mm256_setzero_ps()))
6080	5833
6081		static __inline__ __m128d __DEFAULT_FN_ATTRS128
6082		_mm_mask_permutevar_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128i __C)
6083		{
	5834	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5835	_mm_mask_permutevar_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128i __C) {
6084	5836	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
6085	5837	(__v2df)_mm_permutevar_pd(__A, __C),
6086	5838	(__v2df)__W);
6087	5839	}
6088	5840
6089		static __inline__ __m128d __DEFAULT_FN_ATTRS128
6090		_mm_maskz_permutevar_pd(__mmask8 __U, __m128d __A, __m128i __C)
6091		{
	5841	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	5842	_mm_maskz_permutevar_pd(__mmask8 __U, __m128d __A, __m128i __C) {
6092	5843	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U,
6093	5844	(__v2df)_mm_permutevar_pd(__A, __C),
6094	5845	(__v2df)_mm_setzero_pd());
6095	5846	}
6096	5847
6097		static __inline__ __m256d __DEFAULT_FN_ATTRS256
6098		_mm256_mask_permutevar_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256i __C)
6099		{
	5848	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5849	_mm256_mask_permutevar_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256i __C) {
6100	5850	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
6101	5851	(__v4df)_mm256_permutevar_pd(__A, __C),
6102	5852	(__v4df)__W);
6103	5853	}
6104	5854
6105		static __inline__ __m256d __DEFAULT_FN_ATTRS256
6106		_mm256_maskz_permutevar_pd(__mmask8 __U, __m256d __A, __m256i __C)
6107		{
	5855	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	5856	_mm256_maskz_permutevar_pd(__mmask8 __U, __m256d __A, __m256i __C) {
6108	5857	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
6109	5858	(__v4df)_mm256_permutevar_pd(__A, __C),
6110	5859	(__v4df)_mm256_setzero_pd());
6111	5860	}
6112	5861
6113		static __inline__ __m128 __DEFAULT_FN_ATTRS128
6114		_mm_mask_permutevar_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128i __C)
6115		{
	5862	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5863	_mm_mask_permutevar_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128i __C) {
6116	5864	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
6117	5865	(__v4sf)_mm_permutevar_ps(__A, __C),
6118	5866	(__v4sf)__W);
6119	5867	}
6120	5868
6121		static __inline__ __m128 __DEFAULT_FN_ATTRS128
6122		_mm_maskz_permutevar_ps(__mmask8 __U, __m128 __A, __m128i __C)
6123		{
	5869	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	5870	_mm_maskz_permutevar_ps(__mmask8 __U, __m128 __A, __m128i __C) {
6124	5871	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
6125	5872	(__v4sf)_mm_permutevar_ps(__A, __C),
6126	5873	(__v4sf)_mm_setzero_ps());
6127	5874	}
6128	5875
6129		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6130		_mm256_mask_permutevar_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256i __C)
6131		{
	5876	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5877	_mm256_mask_permutevar_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256i __C) {
6132	5878	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
6133	5879	(__v8sf)_mm256_permutevar_ps(__A, __C),
6134	5880	(__v8sf)__W);
6135	5881	}
6136	5882
6137		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6138		_mm256_maskz_permutevar_ps(__mmask8 __U, __m256 __A, __m256i __C)
6139		{
	5883	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	5884	_mm256_maskz_permutevar_ps(__mmask8 __U, __m256 __A, __m256i __C) {
6140	5885	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
6141	5886	(__v8sf)_mm256_permutevar_ps(__A, __C),
6142	5887	(__v8sf)_mm256_setzero_ps());
...	...	@@ -6250,282 +5995,251 @@ _mm256_mask_testn_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B)
6250	5995	_mm256_setzero_si256());
6251	5996	}
6252	5997
6253		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6254		_mm_mask_unpackhi_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6255		{
	5998	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	5999	_mm_mask_unpackhi_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6256	6000	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6257	6001	(__v4si)_mm_unpackhi_epi32(__A, __B),
6258	6002	(__v4si)__W);
6259	6003	}
6260	6004
6261		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6262		_mm_maskz_unpackhi_epi32(__mmask8 __U, __m128i __A, __m128i __B)
6263		{
	6005	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6006	_mm_maskz_unpackhi_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
6264	6007	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6265	6008	(__v4si)_mm_unpackhi_epi32(__A, __B),
6266	6009	(__v4si)_mm_setzero_si128());
6267	6010	}
6268	6011
6269		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6270		_mm256_mask_unpackhi_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
6271		{
	6012	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6013	_mm256_mask_unpackhi_epi32(__m256i __W, __mmask8 __U, __m256i __A,
	6014	__m256i __B) {
6272	6015	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6273	6016	(__v8si)_mm256_unpackhi_epi32(__A, __B),
6274	6017	(__v8si)__W);
6275	6018	}
6276	6019
6277		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6278		_mm256_maskz_unpackhi_epi32(__mmask8 __U, __m256i __A, __m256i __B)
6279		{
	6020	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6021	_mm256_maskz_unpackhi_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
6280	6022	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6281	6023	(__v8si)_mm256_unpackhi_epi32(__A, __B),
6282	6024	(__v8si)_mm256_setzero_si256());
6283	6025	}
6284	6026
6285		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6286		_mm_mask_unpackhi_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6287		{
	6027	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6028	_mm_mask_unpackhi_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6288	6029	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
6289	6030	(__v2di)_mm_unpackhi_epi64(__A, __B),
6290	6031	(__v2di)__W);
6291	6032	}
6292	6033
6293		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6294		_mm_maskz_unpackhi_epi64(__mmask8 __U, __m128i __A, __m128i __B)
6295		{
	6034	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6035	_mm_maskz_unpackhi_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
6296	6036	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
6297	6037	(__v2di)_mm_unpackhi_epi64(__A, __B),
6298	6038	(__v2di)_mm_setzero_si128());
6299	6039	}
6300	6040
6301		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6302		_mm256_mask_unpackhi_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
6303		{
	6041	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6042	_mm256_mask_unpackhi_epi64(__m256i __W, __mmask8 __U, __m256i __A,
	6043	__m256i __B) {
6304	6044	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
6305	6045	(__v4di)_mm256_unpackhi_epi64(__A, __B),
6306	6046	(__v4di)__W);
6307	6047	}
6308	6048
6309		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6310		_mm256_maskz_unpackhi_epi64(__mmask8 __U, __m256i __A, __m256i __B)
6311		{
	6049	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6050	_mm256_maskz_unpackhi_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
6312	6051	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
6313	6052	(__v4di)_mm256_unpackhi_epi64(__A, __B),
6314	6053	(__v4di)_mm256_setzero_si256());
6315	6054	}
6316	6055
6317		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6318		_mm_mask_unpacklo_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6319		{
	6056	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6057	_mm_mask_unpacklo_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6320	6058	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6321	6059	(__v4si)_mm_unpacklo_epi32(__A, __B),
6322	6060	(__v4si)__W);
6323	6061	}
6324	6062
6325		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6326		_mm_maskz_unpacklo_epi32(__mmask8 __U, __m128i __A, __m128i __B)
6327		{
	6063	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6064	_mm_maskz_unpacklo_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
6328	6065	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6329	6066	(__v4si)_mm_unpacklo_epi32(__A, __B),
6330	6067	(__v4si)_mm_setzero_si128());
6331	6068	}
6332	6069
6333		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6334		_mm256_mask_unpacklo_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
6335		{
	6070	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6071	_mm256_mask_unpacklo_epi32(__m256i __W, __mmask8 __U, __m256i __A,
	6072	__m256i __B) {
6336	6073	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6337	6074	(__v8si)_mm256_unpacklo_epi32(__A, __B),
6338	6075	(__v8si)__W);
6339	6076	}
6340	6077
6341		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6342		_mm256_maskz_unpacklo_epi32(__mmask8 __U, __m256i __A, __m256i __B)
6343		{
	6078	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6079	_mm256_maskz_unpacklo_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
6344	6080	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6345	6081	(__v8si)_mm256_unpacklo_epi32(__A, __B),
6346	6082	(__v8si)_mm256_setzero_si256());
6347	6083	}
6348	6084
6349		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6350		_mm_mask_unpacklo_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6351		{
	6085	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6086	_mm_mask_unpacklo_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6352	6087	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
6353	6088	(__v2di)_mm_unpacklo_epi64(__A, __B),
6354	6089	(__v2di)__W);
6355	6090	}
6356	6091
6357		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6358		_mm_maskz_unpacklo_epi64(__mmask8 __U, __m128i __A, __m128i __B)
6359		{
	6092	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6093	_mm_maskz_unpacklo_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
6360	6094	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U,
6361	6095	(__v2di)_mm_unpacklo_epi64(__A, __B),
6362	6096	(__v2di)_mm_setzero_si128());
6363	6097	}
6364	6098
6365		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6366		_mm256_mask_unpacklo_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B)
6367		{
	6099	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6100	_mm256_mask_unpacklo_epi64(__m256i __W, __mmask8 __U, __m256i __A,
	6101	__m256i __B) {
6368	6102	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
6369	6103	(__v4di)_mm256_unpacklo_epi64(__A, __B),
6370	6104	(__v4di)__W);
6371	6105	}
6372	6106
6373		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6374		_mm256_maskz_unpacklo_epi64(__mmask8 __U, __m256i __A, __m256i __B)
6375		{
	6107	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6108	_mm256_maskz_unpacklo_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
6376	6109	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U,
6377	6110	(__v4di)_mm256_unpacklo_epi64(__A, __B),
6378	6111	(__v4di)_mm256_setzero_si256());
6379	6112	}
6380	6113
6381		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6382		_mm_mask_sra_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6383		{
	6114	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6115	_mm_mask_sra_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6384	6116	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6385	6117	(__v4si)_mm_sra_epi32(__A, __B),
6386	6118	(__v4si)__W);
6387	6119	}
6388	6120
6389		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6390		_mm_maskz_sra_epi32(__mmask8 __U, __m128i __A, __m128i __B)
6391		{
	6121	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6122	_mm_maskz_sra_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
6392	6123	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6393	6124	(__v4si)_mm_sra_epi32(__A, __B),
6394	6125	(__v4si)_mm_setzero_si128());
6395	6126	}
6396	6127
6397		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6398		_mm256_mask_sra_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
6399		{
	6128	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6129	_mm256_mask_sra_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
6400	6130	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6401	6131	(__v8si)_mm256_sra_epi32(__A, __B),
6402	6132	(__v8si)__W);
6403	6133	}
6404	6134
6405		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6406		_mm256_maskz_sra_epi32(__mmask8 __U, __m256i __A, __m128i __B)
6407		{
	6135	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6136	_mm256_maskz_sra_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
6408	6137	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6409	6138	(__v8si)_mm256_sra_epi32(__A, __B),
6410	6139	(__v8si)_mm256_setzero_si256());
6411	6140	}
6412	6141
6413		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6414		_mm_mask_srai_epi32(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B)
6415		{
	6142	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6143	_mm_mask_srai_epi32(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
6416	6144	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6417	6145	(__v4si)_mm_srai_epi32(__A, (int)__B),
6418	6146	(__v4si)__W);
6419	6147	}
6420	6148
6421		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6422		_mm_maskz_srai_epi32(__mmask8 __U, __m128i __A, unsigned int __B)
6423		{
	6149	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6150	_mm_maskz_srai_epi32(__mmask8 __U, __m128i __A, unsigned int __B) {
6424	6151	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__U,
6425	6152	(__v4si)_mm_srai_epi32(__A, (int)__B),
6426	6153	(__v4si)_mm_setzero_si128());
6427	6154	}
6428	6155
6429		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6430		_mm256_mask_srai_epi32(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B)
6431		{
	6156	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6157	_mm256_mask_srai_epi32(__m256i __W, __mmask8 __U, __m256i __A,
	6158	unsigned int __B) {
6432	6159	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6433	6160	(__v8si)_mm256_srai_epi32(__A, (int)__B),
6434	6161	(__v8si)__W);
6435	6162	}
6436	6163
6437		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6438		_mm256_maskz_srai_epi32(__mmask8 __U, __m256i __A, unsigned int __B)
6439		{
	6164	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6165	_mm256_maskz_srai_epi32(__mmask8 __U, __m256i __A, unsigned int __B) {
6440	6166	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__U,
6441	6167	(__v8si)_mm256_srai_epi32(__A, (int)__B),
6442	6168	(__v8si)_mm256_setzero_si256());
6443	6169	}
6444	6170
6445		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6446		_mm_sra_epi64(__m128i __A, __m128i __B)
6447		{
	6171	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6172	_mm_sra_epi64(__m128i __A, __m128i __B) {
6448	6173	return (__m128i)__builtin_ia32_psraq128((__v2di)__A, (__v2di)__B);
6449	6174	}
6450	6175
6451		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6452		_mm_mask_sra_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B)
6453		{
	6176	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6177	_mm_mask_sra_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
6454	6178	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, \
6455	6179	(__v2di)_mm_sra_epi64(__A, __B), \
6456	6180	(__v2di)__W);
6457	6181	}
6458	6182
6459		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6460		_mm_maskz_sra_epi64(__mmask8 __U, __m128i __A, __m128i __B)
6461		{
	6183	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6184	_mm_maskz_sra_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
6462	6185	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, \
6463	6186	(__v2di)_mm_sra_epi64(__A, __B), \
6464	6187	(__v2di)_mm_setzero_si128());
6465	6188	}
6466	6189
6467		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6468		_mm256_sra_epi64(__m256i __A, __m128i __B)
6469		{
	6190	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6191	_mm256_sra_epi64(__m256i __A, __m128i __B) {
6470	6192	return (__m256i)__builtin_ia32_psraq256((__v4di) __A, (__v2di) __B);
6471	6193	}
6472	6194
6473		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6474		_mm256_mask_sra_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B)
6475		{
	6195	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6196	_mm256_mask_sra_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
6476	6197	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, \
6477	6198	(__v4di)_mm256_sra_epi64(__A, __B), \
6478	6199	(__v4di)__W);
6479	6200	}
6480	6201
6481		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6482		_mm256_maskz_sra_epi64(__mmask8 __U, __m256i __A, __m128i __B)
6483		{
	6202	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6203	_mm256_maskz_sra_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
6484	6204	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, \
6485	6205	(__v4di)_mm256_sra_epi64(__A, __B), \
6486	6206	(__v4di)_mm256_setzero_si256());
6487	6207	}
6488	6208
6489		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6490		_mm_srai_epi64(__m128i __A, unsigned int __imm)
6491		{
	6209	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6210	_mm_srai_epi64(__m128i __A, unsigned int __imm) {
6492	6211	return (__m128i)__builtin_ia32_psraqi128((__v2di)__A, (int)__imm);
6493	6212	}
6494	6213
6495		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6496		_mm_mask_srai_epi64(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __imm)
6497		{
	6214	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_mask_srai_epi64(
	6215	__m128i __W, __mmask8 __U, __m128i __A, unsigned int __imm) {
6498	6216	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, \
6499	6217	(__v2di)_mm_srai_epi64(__A, __imm), \
6500	6218	(__v2di)__W);
6501	6219	}
6502	6220
6503		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6504		_mm_maskz_srai_epi64(__mmask8 __U, __m128i __A, unsigned int __imm)
6505		{
	6221	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6222	_mm_maskz_srai_epi64(__mmask8 __U, __m128i __A, unsigned int __imm) {
6506	6223	return (__m128i)__builtin_ia32_selectq_128((__mmask8)__U, \
6507	6224	(__v2di)_mm_srai_epi64(__A, __imm), \
6508	6225	(__v2di)_mm_setzero_si128());
6509	6226	}
6510	6227
6511		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6512		_mm256_srai_epi64(__m256i __A, unsigned int __imm)
6513		{
	6228	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6229	_mm256_srai_epi64(__m256i __A, unsigned int __imm) {
6514	6230	return (__m256i)__builtin_ia32_psraqi256((__v4di)__A, (int)__imm);
6515	6231	}
6516	6232
6517		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	6233	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
6518	6234	_mm256_mask_srai_epi64(__m256i __W, __mmask8 __U, __m256i __A,
6519		unsigned int __imm)
6520		{
	6235	unsigned int __imm) {
6521	6236	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, \
6522	6237	(__v4di)_mm256_srai_epi64(__A, __imm), \
6523	6238	(__v4di)__W);
6524	6239	}
6525	6240
6526		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6527		_mm256_maskz_srai_epi64(__mmask8 __U, __m256i __A, unsigned int __imm)
6528		{
	6241	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6242	_mm256_maskz_srai_epi64(__mmask8 __U, __m256i __A, unsigned int __imm) {
6529	6243	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__U, \
6530	6244	(__v4di)_mm256_srai_epi64(__A, __imm), \
6531	6245	(__v4di)_mm256_setzero_si256());
...	...	@@ -6792,159 +6506,139 @@ _mm256_maskz_rsqrt14_ps (__mmask8 __U, __m256 __A)
6792	6506	(__mmask8) __U);
6793	6507	}
6794	6508
6795		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6796		_mm256_broadcast_f32x4(__m128 __A)
6797		{
	6509	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	6510	_mm256_broadcast_f32x4(__m128 __A) {
6798	6511	return (__m256)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
6799	6512	0, 1, 2, 3, 0, 1, 2, 3);
6800	6513	}
6801	6514
6802		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6803		_mm256_mask_broadcast_f32x4(__m256 __O, __mmask8 __M, __m128 __A)
6804		{
	6515	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	6516	_mm256_mask_broadcast_f32x4(__m256 __O, __mmask8 __M, __m128 __A) {
6805	6517	return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
6806	6518	(__v8sf)_mm256_broadcast_f32x4(__A),
6807	6519	(__v8sf)__O);
6808	6520	}
6809	6521
6810		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6811		_mm256_maskz_broadcast_f32x4 (__mmask8 __M, __m128 __A)
6812		{
	6522	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	6523	_mm256_maskz_broadcast_f32x4(__mmask8 __M, __m128 __A) {
6813	6524	return (__m256)__builtin_ia32_selectps_256((__mmask8)__M,
6814	6525	(__v8sf)_mm256_broadcast_f32x4(__A),
6815	6526	(__v8sf)_mm256_setzero_ps());
6816	6527	}
6817	6528
6818		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6819		_mm256_broadcast_i32x4(__m128i __A)
6820		{
	6529	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6530	_mm256_broadcast_i32x4(__m128i __A) {
6821	6531	return (__m256i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
6822	6532	0, 1, 2, 3, 0, 1, 2, 3);
6823	6533	}
6824	6534
6825		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6826		_mm256_mask_broadcast_i32x4(__m256i __O, __mmask8 __M, __m128i __A)
6827		{
	6535	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6536	_mm256_mask_broadcast_i32x4(__m256i __O, __mmask8 __M, __m128i __A) {
6828	6537	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
6829	6538	(__v8si)_mm256_broadcast_i32x4(__A),
6830	6539	(__v8si)__O);
6831	6540	}
6832	6541
6833		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6834		_mm256_maskz_broadcast_i32x4(__mmask8 __M, __m128i __A)
6835		{
	6542	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6543	_mm256_maskz_broadcast_i32x4(__mmask8 __M, __m128i __A) {
6836	6544	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
6837	6545	(__v8si)_mm256_broadcast_i32x4(__A),
6838	6546	(__v8si)_mm256_setzero_si256());
6839	6547	}
6840	6548
6841		static __inline__ __m256d __DEFAULT_FN_ATTRS256
6842		_mm256_mask_broadcastsd_pd (__m256d __O, __mmask8 __M, __m128d __A)
6843		{
	6549	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	6550	_mm256_mask_broadcastsd_pd(__m256d __O, __mmask8 __M, __m128d __A) {
6844	6551	return (__m256d)__builtin_ia32_selectpd_256(__M,
6845	6552	(__v4df) _mm256_broadcastsd_pd(__A),
6846	6553	(__v4df) __O);
6847	6554	}
6848	6555
6849		static __inline__ __m256d __DEFAULT_FN_ATTRS256
6850		_mm256_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A)
6851		{
	6556	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	6557	_mm256_maskz_broadcastsd_pd(__mmask8 __M, __m128d __A) {
6852	6558	return (__m256d)__builtin_ia32_selectpd_256(__M,
6853	6559	(__v4df) _mm256_broadcastsd_pd(__A),
6854	6560	(__v4df) _mm256_setzero_pd());
6855	6561	}
6856	6562
6857		static __inline__ __m128 __DEFAULT_FN_ATTRS128
6858		_mm_mask_broadcastss_ps (__m128 __O, __mmask8 __M, __m128 __A)
6859		{
	6563	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	6564	_mm_mask_broadcastss_ps(__m128 __O, __mmask8 __M, __m128 __A) {
6860	6565	return (__m128)__builtin_ia32_selectps_128(__M,
6861	6566	(__v4sf) _mm_broadcastss_ps(__A),
6862	6567	(__v4sf) __O);
6863	6568	}
6864	6569
6865		static __inline__ __m128 __DEFAULT_FN_ATTRS128
6866		_mm_maskz_broadcastss_ps (__mmask8 __M, __m128 __A)
6867		{
	6570	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	6571	_mm_maskz_broadcastss_ps(__mmask8 __M, __m128 __A) {
6868	6572	return (__m128)__builtin_ia32_selectps_128(__M,
6869	6573	(__v4sf) _mm_broadcastss_ps(__A),
6870	6574	(__v4sf) _mm_setzero_ps());
6871	6575	}
6872	6576
6873		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6874		_mm256_mask_broadcastss_ps (__m256 __O, __mmask8 __M, __m128 __A)
6875		{
	6577	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	6578	_mm256_mask_broadcastss_ps(__m256 __O, __mmask8 __M, __m128 __A) {
6876	6579	return (__m256)__builtin_ia32_selectps_256(__M,
6877	6580	(__v8sf) _mm256_broadcastss_ps(__A),
6878	6581	(__v8sf) __O);
6879	6582	}
6880	6583
6881		static __inline__ __m256 __DEFAULT_FN_ATTRS256
6882		_mm256_maskz_broadcastss_ps (__mmask8 __M, __m128 __A)
6883		{
	6584	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	6585	_mm256_maskz_broadcastss_ps(__mmask8 __M, __m128 __A) {
6884	6586	return (__m256)__builtin_ia32_selectps_256(__M,
6885	6587	(__v8sf) _mm256_broadcastss_ps(__A),
6886	6588	(__v8sf) _mm256_setzero_ps());
6887	6589	}
6888	6590
6889		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6890		_mm_mask_broadcastd_epi32 (__m128i __O, __mmask8 __M, __m128i __A)
6891		{
	6591	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6592	_mm_mask_broadcastd_epi32(__m128i __O, __mmask8 __M, __m128i __A) {
6892	6593	return (__m128i)__builtin_ia32_selectd_128(__M,
6893	6594	(__v4si) _mm_broadcastd_epi32(__A),
6894	6595	(__v4si) __O);
6895	6596	}
6896	6597
6897		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6898		_mm_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A)
6899		{
	6598	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6599	_mm_maskz_broadcastd_epi32(__mmask8 __M, __m128i __A) {
6900	6600	return (__m128i)__builtin_ia32_selectd_128(__M,
6901	6601	(__v4si) _mm_broadcastd_epi32(__A),
6902	6602	(__v4si) _mm_setzero_si128());
6903	6603	}
6904	6604
6905		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6906		_mm256_mask_broadcastd_epi32 (__m256i __O, __mmask8 __M, __m128i __A)
6907		{
	6605	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6606	_mm256_mask_broadcastd_epi32(__m256i __O, __mmask8 __M, __m128i __A) {
6908	6607	return (__m256i)__builtin_ia32_selectd_256(__M,
6909	6608	(__v8si) _mm256_broadcastd_epi32(__A),
6910	6609	(__v8si) __O);
6911	6610	}
6912	6611
6913		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6914		_mm256_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A)
6915		{
	6612	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6613	_mm256_maskz_broadcastd_epi32(__mmask8 __M, __m128i __A) {
6916	6614	return (__m256i)__builtin_ia32_selectd_256(__M,
6917	6615	(__v8si) _mm256_broadcastd_epi32(__A),
6918	6616	(__v8si) _mm256_setzero_si256());
6919	6617	}
6920	6618
6921		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6922		_mm_mask_broadcastq_epi64 (__m128i __O, __mmask8 __M, __m128i __A)
6923		{
	6619	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6620	_mm_mask_broadcastq_epi64(__m128i __O, __mmask8 __M, __m128i __A) {
6924	6621	return (__m128i)__builtin_ia32_selectq_128(__M,
6925	6622	(__v2di) _mm_broadcastq_epi64(__A),
6926	6623	(__v2di) __O);
6927	6624	}
6928	6625
6929		static __inline__ __m128i __DEFAULT_FN_ATTRS128
6930		_mm_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
6931		{
	6626	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	6627	_mm_maskz_broadcastq_epi64(__mmask8 __M, __m128i __A) {
6932	6628	return (__m128i)__builtin_ia32_selectq_128(__M,
6933	6629	(__v2di) _mm_broadcastq_epi64(__A),
6934	6630	(__v2di) _mm_setzero_si128());
6935	6631	}
6936	6632
6937		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6938		_mm256_mask_broadcastq_epi64 (__m256i __O, __mmask8 __M, __m128i __A)
6939		{
	6633	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6634	_mm256_mask_broadcastq_epi64(__m256i __O, __mmask8 __M, __m128i __A) {
6940	6635	return (__m256i)__builtin_ia32_selectq_256(__M,
6941	6636	(__v4di) _mm256_broadcastq_epi64(__A),
6942	6637	(__v4di) __O);
6943	6638	}
6944	6639
6945		static __inline__ __m256i __DEFAULT_FN_ATTRS256
6946		_mm256_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A)
6947		{
	6640	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	6641	_mm256_maskz_broadcastq_epi64(__mmask8 __M, __m128i __A) {
6948	6642	return (__m256i)__builtin_ia32_selectq_256(__M,
6949	6643	(__v4di) _mm256_broadcastq_epi64(__A),
6950	6644	(__v4di) _mm256_setzero_si256());
...	...	@@ -7536,9 +7230,8 @@ _mm256_mask_cvtusepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A)
7536	7230	__builtin_ia32_pmovusqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M);
7537	7231	}
7538	7232
7539		static __inline__ __m128i __DEFAULT_FN_ATTRS128
7540		_mm_cvtepi32_epi8 (__m128i __A)
7541		{
	7233	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	7234	_mm_cvtepi32_epi8(__m128i __A) {
7542	7235	return (__m128i)__builtin_shufflevector(
7543	7236	__builtin_convertvector((__v4si)__A, __v4qi), (__v4qi){0, 0, 0, 0}, 0, 1,
7544	7237	2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7);
...	...	@@ -7566,9 +7259,8 @@ _mm_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
7566	7259	__builtin_ia32_pmovdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M);
7567	7260	}
7568	7261
7569		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7570		_mm256_cvtepi32_epi8 (__m256i __A)
7571		{
	7262	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7263	_mm256_cvtepi32_epi8(__m256i __A) {
7572	7264	return (__m128i)__builtin_shufflevector(
7573	7265	__builtin_convertvector((__v8si)__A, __v8qi),
7574	7266	(__v8qi){0, 0, 0, 0, 0, 0, 0, 0}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
...	...	@@ -7576,8 +7268,7 @@ _mm256_cvtepi32_epi8 (__m256i __A)
7576	7268	}
7577	7269
7578	7270	static __inline__ __m128i __DEFAULT_FN_ATTRS256
7579		_mm256_mask_cvtepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A)
7580		{
	7271	_mm256_mask_cvtepi32_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
7581	7272	return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A,
7582	7273	(__v16qi) __O, __M);
7583	7274	}
...	...	@@ -7596,9 +7287,8 @@ _mm256_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A)
7596	7287	__builtin_ia32_pmovdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M);
7597	7288	}
7598	7289
7599		static __inline__ __m128i __DEFAULT_FN_ATTRS128
7600		_mm_cvtepi32_epi16 (__m128i __A)
7601		{
	7290	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	7291	_mm_cvtepi32_epi16(__m128i __A) {
7602	7292	return (__m128i)__builtin_shufflevector(
7603	7293	__builtin_convertvector((__v4si)__A, __v4hi), (__v4hi){0, 0, 0, 0}, 0, 1,
7604	7294	2, 3, 4, 5, 6, 7);
...	...	@@ -7625,9 +7315,8 @@ _mm_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A)
7625	7315	__builtin_ia32_pmovdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M);
7626	7316	}
7627	7317
7628		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7629		_mm256_cvtepi32_epi16 (__m256i __A)
7630		{
	7318	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7319	_mm256_cvtepi32_epi16(__m256i __A) {
7631	7320	return (__m128i)__builtin_convertvector((__v8si)__A, __v8hi);
7632	7321	}
7633	7322
...	...	@@ -7652,9 +7341,8 @@ _mm256_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A)
7652	7341	__builtin_ia32_pmovdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M);
7653	7342	}
7654	7343
7655		static __inline__ __m128i __DEFAULT_FN_ATTRS128
7656		_mm_cvtepi64_epi8 (__m128i __A)
7657		{
	7344	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	7345	_mm_cvtepi64_epi8(__m128i __A) {
7658	7346	return (__m128i)__builtin_shufflevector(
7659	7347	__builtin_convertvector((__v2di)__A, __v2qi), (__v2qi){0, 0}, 0, 1, 2, 3,
7660	7348	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3);
...	...	@@ -7681,9 +7369,8 @@ _mm_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
7681	7369	__builtin_ia32_pmovqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M);
7682	7370	}
7683	7371
7684		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7685		_mm256_cvtepi64_epi8 (__m256i __A)
7686		{
	7372	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7373	_mm256_cvtepi64_epi8(__m256i __A) {
7687	7374	return (__m128i)__builtin_shufflevector(
7688	7375	__builtin_convertvector((__v4di)__A, __v4qi), (__v4qi){0, 0, 0, 0}, 0, 1,
7689	7376	2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7);
...	...	@@ -7710,9 +7397,8 @@ _mm256_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A)
7710	7397	__builtin_ia32_pmovqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M);
7711	7398	}
7712	7399
7713		static __inline__ __m128i __DEFAULT_FN_ATTRS128
7714		_mm_cvtepi64_epi32 (__m128i __A)
7715		{
	7400	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	7401	_mm_cvtepi64_epi32(__m128i __A) {
7716	7402	return (__m128i)__builtin_shufflevector(
7717	7403	__builtin_convertvector((__v2di)__A, __v2si), (__v2si){0, 0}, 0, 1, 2, 3);
7718	7404	}
...	...	@@ -7738,23 +7424,20 @@ _mm_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A)
7738	7424	__builtin_ia32_pmovqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M);
7739	7425	}
7740	7426
7741		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7742		_mm256_cvtepi64_epi32 (__m256i __A)
7743		{
	7427	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7428	_mm256_cvtepi64_epi32(__m256i __A) {
7744	7429	return (__m128i)__builtin_convertvector((__v4di)__A, __v4si);
7745	7430	}
7746	7431
7747		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7748		_mm256_mask_cvtepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A)
7749		{
	7432	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7433	_mm256_mask_cvtepi64_epi32(__m128i __O, __mmask8 __M, __m256i __A) {
7750	7434	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
7751	7435	(__v4si)_mm256_cvtepi64_epi32(__A),
7752	7436	(__v4si)__O);
7753	7437	}
7754	7438
7755		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7756		_mm256_maskz_cvtepi64_epi32 (__mmask8 __M, __m256i __A)
7757		{
	7439	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7440	_mm256_maskz_cvtepi64_epi32(__mmask8 __M, __m256i __A) {
7758	7441	return (__m128i)__builtin_ia32_selectd_128((__mmask8)__M,
7759	7442	(__v4si)_mm256_cvtepi64_epi32(__A),
7760	7443	(__v4si)_mm_setzero_si128());
...	...	@@ -7766,9 +7449,8 @@ _mm256_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A)
7766	7449	__builtin_ia32_pmovqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M);
7767	7450	}
7768	7451
7769		static __inline__ __m128i __DEFAULT_FN_ATTRS128
7770		_mm_cvtepi64_epi16 (__m128i __A)
7771		{
	7452	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
	7453	_mm_cvtepi64_epi16(__m128i __A) {
7772	7454	return (__m128i)__builtin_shufflevector(
7773	7455	__builtin_convertvector((__v2di)__A, __v2hi), (__v2hi){0, 0}, 0, 1, 2, 3,
7774	7456	3, 3, 3, 3);
...	...	@@ -7796,9 +7478,8 @@ _mm_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A)
7796	7478	__builtin_ia32_pmovqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M);
7797	7479	}
7798	7480
7799		static __inline__ __m128i __DEFAULT_FN_ATTRS256
7800		_mm256_cvtepi64_epi16 (__m256i __A)
7801		{
	7481	static __inline__ __m128i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7482	_mm256_cvtepi64_epi16(__m256i __A) {
7802	7483	return (__m128i)__builtin_shufflevector(
7803	7484	__builtin_convertvector((__v4di)__A, __v4hi), (__v4hi){0, 0, 0, 0}, 0, 1,
7804	7485	2, 3, 4, 5, 6, 7);
...	...	@@ -7825,11 +7506,10 @@ _mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A)
7825	7506	__builtin_ia32_pmovqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M);
7826	7507	}
7827	7508
7828		#define _mm256_extractf32x4_ps(A, imm) \
7829		((__m128)__builtin_ia32_extractf32x4_256_mask((__v8sf)(__m256)(A), \
7830		(int)(imm), \
7831		(__v4sf)_mm_undefined_ps(), \
7832		(__mmask8)-1))
	7509	#define _mm256_extractf32x4_ps(A, imm) \
	7510	((__m128)__builtin_ia32_extractf32x4_256_mask( \
	7511	(__v8sf)(__m256)(A), (int)(imm), (__v4sf)_mm_setzero_ps(), \
	7512	(__mmask8) - 1))
7833	7513
7834	7514	#define _mm256_mask_extractf32x4_ps(W, U, A, imm) \
7835	7515	((__m128)__builtin_ia32_extractf32x4_256_mask((__v8sf)(__m256)(A), \
...	...	@@ -7843,11 +7523,10 @@ _mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A)
7843	7523	(__v4sf)_mm_setzero_ps(), \
7844	7524	(__mmask8)(U)))
7845	7525
7846		#define _mm256_extracti32x4_epi32(A, imm) \
7847		((__m128i)__builtin_ia32_extracti32x4_256_mask((__v8si)(__m256i)(A), \
7848		(int)(imm), \
7849		(__v4si)_mm_undefined_si128(), \
7850		(__mmask8)-1))
	7526	#define _mm256_extracti32x4_epi32(A, imm) \
	7527	((__m128i)__builtin_ia32_extracti32x4_256_mask( \
	7528	(__v8si)(__m256i)(A), (int)(imm), (__v4si)_mm_setzero_si128(), \
	7529	(__mmask8) - 1))
7851	7530
7852	7531	#define _mm256_mask_extracti32x4_epi32(W, U, A, imm) \
7853	7532	((__m128i)__builtin_ia32_extracti32x4_256_mask((__v8si)(__m256i)(A), \
...	...	@@ -8083,47 +7762,41 @@ _mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A)
8083	7762	(__v4di)_mm256_permutex_epi64((X), (C)), \
8084	7763	(__v4di)_mm256_setzero_si256()))
8085	7764
8086		static __inline__ __m256d __DEFAULT_FN_ATTRS256
8087		_mm256_permutexvar_pd (__m256i __X, __m256d __Y)
8088		{
	7765	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	7766	_mm256_permutexvar_pd(__m256i __X, __m256d __Y) {
8089	7767	return (__m256d)__builtin_ia32_permvardf256((__v4df)__Y, (__v4di)__X);
8090	7768	}
8091	7769
8092		static __inline__ __m256d __DEFAULT_FN_ATTRS256
8093		_mm256_mask_permutexvar_pd (__m256d __W, __mmask8 __U, __m256i __X,
8094		__m256d __Y)
8095		{
	7770	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	7771	_mm256_mask_permutexvar_pd(__m256d __W, __mmask8 __U, __m256i __X,
	7772	__m256d __Y) {
8096	7773	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
8097	7774	(__v4df)_mm256_permutexvar_pd(__X, __Y),
8098	7775	(__v4df)__W);
8099	7776	}
8100	7777
8101		static __inline__ __m256d __DEFAULT_FN_ATTRS256
8102		_mm256_maskz_permutexvar_pd (__mmask8 __U, __m256i __X, __m256d __Y)
8103		{
	7778	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	7779	_mm256_maskz_permutexvar_pd(__mmask8 __U, __m256i __X, __m256d __Y) {
8104	7780	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U,
8105	7781	(__v4df)_mm256_permutexvar_pd(__X, __Y),
8106	7782	(__v4df)_mm256_setzero_pd());
8107	7783	}
8108	7784
8109		static __inline__ __m256i __DEFAULT_FN_ATTRS256
8110		_mm256_permutexvar_epi64 ( __m256i __X, __m256i __Y)
8111		{
	7785	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7786	_mm256_permutexvar_epi64(__m256i __X, __m256i __Y) {
8112	7787	return (__m256i)__builtin_ia32_permvardi256((__v4di) __Y, (__v4di) __X);
8113	7788	}
8114	7789
8115		static __inline__ __m256i __DEFAULT_FN_ATTRS256
8116		_mm256_maskz_permutexvar_epi64 (__mmask8 __M, __m256i __X, __m256i __Y)
8117		{
	7790	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7791	_mm256_maskz_permutexvar_epi64(__mmask8 __M, __m256i __X, __m256i __Y) {
8118	7792	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
8119	7793	(__v4di)_mm256_permutexvar_epi64(__X, __Y),
8120	7794	(__v4di)_mm256_setzero_si256());
8121	7795	}
8122	7796
8123		static __inline__ __m256i __DEFAULT_FN_ATTRS256
8124		_mm256_mask_permutexvar_epi64 (__m256i __W, __mmask8 __M, __m256i __X,
8125		__m256i __Y)
8126		{
	7797	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7798	_mm256_mask_permutexvar_epi64(__m256i __W, __mmask8 __M, __m256i __X,
	7799	__m256i __Y) {
8127	7800	return (__m256i)__builtin_ia32_selectq_256((__mmask8)__M,
8128	7801	(__v4di)_mm256_permutexvar_epi64(__X, __Y),
8129	7802	(__v4di)__W);
...	...	@@ -8131,17 +7804,15 @@ _mm256_mask_permutexvar_epi64 (__m256i __W, __mmask8 __M, __m256i __X,
8131	7804
8132	7805	#define _mm256_permutexvar_ps(A, B) _mm256_permutevar8x32_ps((B), (A))
8133	7806
8134		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8135		_mm256_mask_permutexvar_ps(__m256 __W, __mmask8 __U, __m256i __X, __m256 __Y)
8136		{
	7807	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7808	_mm256_mask_permutexvar_ps(__m256 __W, __mmask8 __U, __m256i __X, __m256 __Y) {
8137	7809	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8138	7810	(__v8sf)_mm256_permutexvar_ps(__X, __Y),
8139	7811	(__v8sf)__W);
8140	7812	}
8141	7813
8142		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8143		_mm256_maskz_permutexvar_ps(__mmask8 __U, __m256i __X, __m256 __Y)
8144		{
	7814	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7815	_mm256_maskz_permutexvar_ps(__mmask8 __U, __m256i __X, __m256 __Y) {
8145	7816	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8146	7817	(__v8sf)_mm256_permutexvar_ps(__X, __Y),
8147	7818	(__v8sf)_mm256_setzero_ps());
...	...	@@ -8149,18 +7820,16 @@ _mm256_maskz_permutexvar_ps(__mmask8 __U, __m256i __X, __m256 __Y)
8149	7820
8150	7821	#define _mm256_permutexvar_epi32(A, B) _mm256_permutevar8x32_epi32((B), (A))
8151	7822
8152		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	7823	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
8153	7824	_mm256_mask_permutexvar_epi32(__m256i __W, __mmask8 __M, __m256i __X,
8154		__m256i __Y)
8155		{
	7825	__m256i __Y) {
8156	7826	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
8157	7827	(__v8si)_mm256_permutexvar_epi32(__X, __Y),
8158	7828	(__v8si)__W);
8159	7829	}
8160	7830
8161		static __inline__ __m256i __DEFAULT_FN_ATTRS256
8162		_mm256_maskz_permutexvar_epi32(__mmask8 __M, __m256i __X, __m256i __Y)
8163		{
	7831	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	7832	_mm256_maskz_permutexvar_epi32(__mmask8 __M, __m256i __X, __m256i __Y) {
8164	7833	return (__m256i)__builtin_ia32_selectd_256((__mmask8)__M,
8165	7834	(__v8si)_mm256_permutexvar_epi32(__X, __Y),
8166	7835	(__v8si)_mm256_setzero_si256());
...	...	@@ -8222,65 +7891,57 @@ _mm256_maskz_permutexvar_epi32(__mmask8 __M, __m256i __X, __m256i __Y)
8222	7891	(__v4di)_mm256_alignr_epi64((A), (B), (imm)), \
8223	7892	(__v4di)_mm256_setzero_si256()))
8224	7893
8225		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8226		_mm_mask_movehdup_ps (__m128 __W, __mmask8 __U, __m128 __A)
8227		{
	7894	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	7895	_mm_mask_movehdup_ps(__m128 __W, __mmask8 __U, __m128 __A) {
8228	7896	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
8229	7897	(__v4sf)_mm_movehdup_ps(__A),
8230	7898	(__v4sf)__W);
8231	7899	}
8232	7900
8233		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8234		_mm_maskz_movehdup_ps (__mmask8 __U, __m128 __A)
8235		{
	7901	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	7902	_mm_maskz_movehdup_ps(__mmask8 __U, __m128 __A) {
8236	7903	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
8237	7904	(__v4sf)_mm_movehdup_ps(__A),
8238	7905	(__v4sf)_mm_setzero_ps());
8239	7906	}
8240	7907
8241		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8242		_mm256_mask_movehdup_ps (__m256 __W, __mmask8 __U, __m256 __A)
8243		{
	7908	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7909	_mm256_mask_movehdup_ps(__m256 __W, __mmask8 __U, __m256 __A) {
8244	7910	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8245	7911	(__v8sf)_mm256_movehdup_ps(__A),
8246	7912	(__v8sf)__W);
8247	7913	}
8248	7914
8249		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8250		_mm256_maskz_movehdup_ps (__mmask8 __U, __m256 __A)
8251		{
	7915	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7916	_mm256_maskz_movehdup_ps(__mmask8 __U, __m256 __A) {
8252	7917	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8253	7918	(__v8sf)_mm256_movehdup_ps(__A),
8254	7919	(__v8sf)_mm256_setzero_ps());
8255	7920	}
8256	7921
8257		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8258		_mm_mask_moveldup_ps (__m128 __W, __mmask8 __U, __m128 __A)
8259		{
	7922	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	7923	_mm_mask_moveldup_ps(__m128 __W, __mmask8 __U, __m128 __A) {
8260	7924	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
8261	7925	(__v4sf)_mm_moveldup_ps(__A),
8262	7926	(__v4sf)__W);
8263	7927	}
8264	7928
8265		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8266		_mm_maskz_moveldup_ps (__mmask8 __U, __m128 __A)
8267		{
	7929	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	7930	_mm_maskz_moveldup_ps(__mmask8 __U, __m128 __A) {
8268	7931	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U,
8269	7932	(__v4sf)_mm_moveldup_ps(__A),
8270	7933	(__v4sf)_mm_setzero_ps());
8271	7934	}
8272	7935
8273		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8274		_mm256_mask_moveldup_ps (__m256 __W, __mmask8 __U, __m256 __A)
8275		{
	7936	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7937	_mm256_mask_moveldup_ps(__m256 __W, __mmask8 __U, __m256 __A) {
8276	7938	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8277	7939	(__v8sf)_mm256_moveldup_ps(__A),
8278	7940	(__v8sf)__W);
8279	7941	}
8280	7942
8281		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8282		_mm256_maskz_moveldup_ps (__mmask8 __U, __m256 __A)
8283		{
	7943	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	7944	_mm256_maskz_moveldup_ps(__mmask8 __U, __m256 __A) {
8284	7945	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U,
8285	7946	(__v8sf)_mm256_moveldup_ps(__A),
8286	7947	(__v8sf)_mm256_setzero_ps());
...	...	@@ -8306,68 +7967,52 @@ _mm256_maskz_moveldup_ps (__mmask8 __U, __m256 __A)
8306	7967	(__v4si)_mm_shuffle_epi32((A), (I)), \
8307	7968	(__v4si)_mm_setzero_si128()))
8308	7969
8309		static __inline__ __m128d __DEFAULT_FN_ATTRS128
8310		_mm_mask_mov_pd (__m128d __W, __mmask8 __U, __m128d __A)
8311		{
8312		return (__m128d) __builtin_ia32_selectpd_128 ((__mmask8) __U,
8313		(__v2df) __A,
8314		(__v2df) __W);
	7970	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	7971	_mm_mask_mov_pd(__m128d __W, __mmask8 __U, __m128d __A) {
	7972	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U, (__v2df)__A,
	7973	(__v2df)__W);
8315	7974	}
8316	7975
8317		static __inline__ __m128d __DEFAULT_FN_ATTRS128
8318		_mm_maskz_mov_pd (__mmask8 __U, __m128d __A)
8319		{
8320		return (__m128d) __builtin_ia32_selectpd_128 ((__mmask8) __U,
8321		(__v2df) __A,
8322		(__v2df) _mm_setzero_pd ());
	7976	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	7977	_mm_maskz_mov_pd(__mmask8 __U, __m128d __A) {
	7978	return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U, (__v2df)__A,
	7979	(__v2df)_mm_setzero_pd());
8323	7980	}
8324	7981
8325		static __inline__ __m256d __DEFAULT_FN_ATTRS256
8326		_mm256_mask_mov_pd (__m256d __W, __mmask8 __U, __m256d __A)
8327		{
8328		return (__m256d) __builtin_ia32_selectpd_256 ((__mmask8) __U,
8329		(__v4df) __A,
8330		(__v4df) __W);
	7982	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	7983	_mm256_mask_mov_pd(__m256d __W, __mmask8 __U, __m256d __A) {
	7984	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U, (__v4df)__A,
	7985	(__v4df)__W);
8331	7986	}
8332	7987
8333		static __inline__ __m256d __DEFAULT_FN_ATTRS256
8334		_mm256_maskz_mov_pd (__mmask8 __U, __m256d __A)
8335		{
8336		return (__m256d) __builtin_ia32_selectpd_256 ((__mmask8) __U,
8337		(__v4df) __A,
8338		(__v4df) _mm256_setzero_pd ());
	7988	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	7989	_mm256_maskz_mov_pd(__mmask8 __U, __m256d __A) {
	7990	return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U, (__v4df)__A,
	7991	(__v4df)_mm256_setzero_pd());
8339	7992	}
8340	7993
8341		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8342		_mm_mask_mov_ps (__m128 __W, __mmask8 __U, __m128 __A)
8343		{
8344		return (__m128) __builtin_ia32_selectps_128 ((__mmask8) __U,
8345		(__v4sf) __A,
8346		(__v4sf) __W);
	7994	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	7995	_mm_mask_mov_ps(__m128 __W, __mmask8 __U, __m128 __A) {
	7996	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, (__v4sf)__A,
	7997	(__v4sf)__W);
8347	7998	}
8348	7999
8349		static __inline__ __m128 __DEFAULT_FN_ATTRS128
8350		_mm_maskz_mov_ps (__mmask8 __U, __m128 __A)
8351		{
8352		return (__m128) __builtin_ia32_selectps_128 ((__mmask8) __U,
8353		(__v4sf) __A,
8354		(__v4sf) _mm_setzero_ps ());
	8000	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	8001	_mm_maskz_mov_ps(__mmask8 __U, __m128 __A) {
	8002	return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, (__v4sf)__A,
	8003	(__v4sf)_mm_setzero_ps());
8355	8004	}
8356	8005
8357		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8358		_mm256_mask_mov_ps (__m256 __W, __mmask8 __U, __m256 __A)
8359		{
8360		return (__m256) __builtin_ia32_selectps_256 ((__mmask8) __U,
8361		(__v8sf) __A,
8362		(__v8sf) __W);
	8006	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	8007	_mm256_mask_mov_ps(__m256 __W, __mmask8 __U, __m256 __A) {
	8008	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, (__v8sf)__A,
	8009	(__v8sf)__W);
8363	8010	}
8364	8011
8365		static __inline__ __m256 __DEFAULT_FN_ATTRS256
8366		_mm256_maskz_mov_ps (__mmask8 __U, __m256 __A)
8367		{
8368		return (__m256) __builtin_ia32_selectps_256 ((__mmask8) __U,
8369		(__v8sf) __A,
8370		(__v8sf) _mm256_setzero_ps ());
	8012	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	8013	_mm256_maskz_mov_ps(__mmask8 __U, __m256 __A) {
	8014	return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, (__v8sf)__A,
	8015	(__v8sf)_mm256_setzero_ps());
8371	8016	}
8372	8017
8373	8018	static __inline__ __m128 __DEFAULT_FN_ATTRS128
...	...	@@ -8430,8 +8075,9 @@ _mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A)
8430	8075	#define _mm256_mask_cvtps_ph _mm256_mask_cvt_roundps_ph
8431	8076	#define _mm256_maskz_cvtps_ph _mm256_maskz_cvt_roundps_ph
8432	8077
8433
8434	8078	#undef __DEFAULT_FN_ATTRS128
8435	8079	#undef __DEFAULT_FN_ATTRS256
	8080	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
	8081	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
8436	8082
8437	8083	#endif /* __AVX512VLINTRIN_H */

lib/include/avx512vlvbmi2intrin.h+78-63

...	...	@@ -17,13 +17,21 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS128 \
19	19	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512vbmi2,no-evex512"), \
	20	__target__("avx512vl,avx512vbmi2"), \
21	21	__min_vector_width__(128)))
22	22	#define __DEFAULT_FN_ATTRS256 \
23	23	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512vbmi2,no-evex512"), \
	24	__target__("avx512vl,avx512vbmi2"), \
25	25	__min_vector_width__(256)))
26	26
	27	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	28	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	29	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	30	#else
	31	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	32	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	33	#endif
	34
27	35	static __inline__ __m128i __DEFAULT_FN_ATTRS128
28	36	_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D)
29	37	{
...	...	@@ -412,14 +420,14 @@ _mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P)
412	420	(__v8hi)_mm_shrdi_epi16((A), (B), (I)), \
413	421	(__v8hi)_mm_setzero_si128()))
414	422
415		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	423	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
416	424	_mm256_shldv_epi64(__m256i __A, __m256i __B, __m256i __C)
417	425	{
418		return (__m256i)__builtin_ia32_vpshldvq256((__v4di)__A, (__v4di)__B,
419		(__v4di)__C);
	426	return (__m256i)__builtin_elementwise_fshl((__v4du)__A, (__v4du)__B,
	427	(__v4du)__C);
420	428	}
421	429
422		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	430	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
423	431	_mm256_mask_shldv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
424	432	{
425	433	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -427,7 +435,7 @@ _mm256_mask_shldv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
427	435	(__v4di)__A);
428	436	}
429	437
430		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	438	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
431	439	_mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
432	440	{
433	441	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -435,14 +443,14 @@ _mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
435	443	(__v4di)_mm256_setzero_si256());
436	444	}
437	445
438		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	446	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
439	447	_mm_shldv_epi64(__m128i __A, __m128i __B, __m128i __C)
440	448	{
441		return (__m128i)__builtin_ia32_vpshldvq128((__v2di)__A, (__v2di)__B,
442		(__v2di)__C);
	449	return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__B,
	450	(__v2du)__C);
443	451	}
444	452
445		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	453	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
446	454	_mm_mask_shldv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
447	455	{
448	456	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -450,7 +458,7 @@ _mm_mask_shldv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
450	458	(__v2di)__A);
451	459	}
452	460
453		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	461	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
454	462	_mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
455	463	{
456	464	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -458,14 +466,14 @@ _mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
458	466	(__v2di)_mm_setzero_si128());
459	467	}
460	468
461		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	469	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
462	470	_mm256_shldv_epi32(__m256i __A, __m256i __B, __m256i __C)
463	471	{
464		return (__m256i)__builtin_ia32_vpshldvd256((__v8si)__A, (__v8si)__B,
465		(__v8si)__C);
	472	return (__m256i)__builtin_elementwise_fshl((__v8su)__A, (__v8su)__B,
	473	(__v8su)__C);
466	474	}
467	475
468		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	476	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
469	477	_mm256_mask_shldv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
470	478	{
471	479	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -473,7 +481,7 @@ _mm256_mask_shldv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
473	481	(__v8si)__A);
474	482	}
475	483
476		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	484	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
477	485	_mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
478	486	{
479	487	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -481,14 +489,14 @@ _mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
481	489	(__v8si)_mm256_setzero_si256());
482	490	}
483	491
484		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	492	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
485	493	_mm_shldv_epi32(__m128i __A, __m128i __B, __m128i __C)
486	494	{
487		return (__m128i)__builtin_ia32_vpshldvd128((__v4si)__A, (__v4si)__B,
488		(__v4si)__C);
	495	return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__B,
	496	(__v4su)__C);
489	497	}
490	498
491		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	499	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
492	500	_mm_mask_shldv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
493	501	{
494	502	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -496,7 +504,7 @@ _mm_mask_shldv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
496	504	(__v4si)__A);
497	505	}
498	506
499		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	507	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
500	508	_mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
501	509	{
502	510	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -504,14 +512,14 @@ _mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
504	512	(__v4si)_mm_setzero_si128());
505	513	}
506	514
507		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	515	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
508	516	_mm256_shldv_epi16(__m256i __A, __m256i __B, __m256i __C)
509	517	{
510		return (__m256i)__builtin_ia32_vpshldvw256((__v16hi)__A, (__v16hi)__B,
511		(__v16hi)__C);
	518	return (__m256i)__builtin_elementwise_fshl((__v16hu)__A, (__v16hu)__B,
	519	(__v16hu)__C);
512	520	}
513	521
514		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	522	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
515	523	_mm256_mask_shldv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
516	524	{
517	525	return (__m256i)__builtin_ia32_selectw_256(__U,
...	...	@@ -519,7 +527,7 @@ _mm256_mask_shldv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
519	527	(__v16hi)__A);
520	528	}
521	529
522		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	530	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
523	531	_mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
524	532	{
525	533	return (__m256i)__builtin_ia32_selectw_256(__U,
...	...	@@ -527,14 +535,14 @@ _mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
527	535	(__v16hi)_mm256_setzero_si256());
528	536	}
529	537
530		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	538	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
531	539	_mm_shldv_epi16(__m128i __A, __m128i __B, __m128i __C)
532	540	{
533		return (__m128i)__builtin_ia32_vpshldvw128((__v8hi)__A, (__v8hi)__B,
534		(__v8hi)__C);
	541	return (__m128i)__builtin_elementwise_fshl((__v8hu)__A, (__v8hu)__B,
	542	(__v8hu)__C);
535	543	}
536	544
537		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	545	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
538	546	_mm_mask_shldv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
539	547	{
540	548	return (__m128i)__builtin_ia32_selectw_128(__U,
...	...	@@ -542,7 +550,7 @@ _mm_mask_shldv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
542	550	(__v8hi)__A);
543	551	}
544	552
545		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	553	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
546	554	_mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
547	555	{
548	556	return (__m128i)__builtin_ia32_selectw_128(__U,
...	...	@@ -550,14 +558,15 @@ _mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
550	558	(__v8hi)_mm_setzero_si128());
551	559	}
552	560
553		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	561	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
554	562	_mm256_shrdv_epi64(__m256i __A, __m256i __B, __m256i __C)
555	563	{
556		return (__m256i)__builtin_ia32_vpshrdvq256((__v4di)__A, (__v4di)__B,
557		(__v4di)__C);
	564	// Ops __A and __B are swapped.
	565	return (__m256i)__builtin_elementwise_fshr((__v4du)__B, (__v4du)__A,
	566	(__v4du)__C);
558	567	}
559	568
560		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	569	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
561	570	_mm256_mask_shrdv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
562	571	{
563	572	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -565,7 +574,7 @@ _mm256_mask_shrdv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
565	574	(__v4di)__A);
566	575	}
567	576
568		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	577	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
569	578	_mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
570	579	{
571	580	return (__m256i)__builtin_ia32_selectq_256(__U,
...	...	@@ -573,14 +582,15 @@ _mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
573	582	(__v4di)_mm256_setzero_si256());
574	583	}
575	584
576		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	585	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
577	586	_mm_shrdv_epi64(__m128i __A, __m128i __B, __m128i __C)
578	587	{
579		return (__m128i)__builtin_ia32_vpshrdvq128((__v2di)__A, (__v2di)__B,
580		(__v2di)__C);
	588	// Ops __A and __B are swapped.
	589	return (__m128i)__builtin_elementwise_fshr((__v2du)__B, (__v2du)__A,
	590	(__v2du)__C);
581	591	}
582	592
583		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	593	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
584	594	_mm_mask_shrdv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
585	595	{
586	596	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -588,7 +598,7 @@ _mm_mask_shrdv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
588	598	(__v2di)__A);
589	599	}
590	600
591		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	601	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
592	602	_mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
593	603	{
594	604	return (__m128i)__builtin_ia32_selectq_128(__U,
...	...	@@ -596,14 +606,15 @@ _mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
596	606	(__v2di)_mm_setzero_si128());
597	607	}
598	608
599		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	609	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
600	610	_mm256_shrdv_epi32(__m256i __A, __m256i __B, __m256i __C)
601	611	{
602		return (__m256i)__builtin_ia32_vpshrdvd256((__v8si)__A, (__v8si)__B,
603		(__v8si)__C);
	612	// Ops __A and __B are swapped.
	613	return (__m256i)__builtin_elementwise_fshr((__v8su)__B, (__v8su)__A,
	614	(__v8su)__C);
604	615	}
605	616
606		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	617	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
607	618	_mm256_mask_shrdv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
608	619	{
609	620	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -611,7 +622,7 @@ _mm256_mask_shrdv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
611	622	(__v8si)__A);
612	623	}
613	624
614		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	625	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
615	626	_mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
616	627	{
617	628	return (__m256i)__builtin_ia32_selectd_256(__U,
...	...	@@ -619,14 +630,15 @@ _mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
619	630	(__v8si)_mm256_setzero_si256());
620	631	}
621	632
622		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	633	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
623	634	_mm_shrdv_epi32(__m128i __A, __m128i __B, __m128i __C)
624	635	{
625		return (__m128i)__builtin_ia32_vpshrdvd128((__v4si)__A, (__v4si)__B,
626		(__v4si)__C);
	636	// Ops __A and __B are swapped.
	637	return (__m128i)__builtin_elementwise_fshr((__v4su)__B, (__v4su)__A,
	638	(__v4su)__C);
627	639	}
628	640
629		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	641	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
630	642	_mm_mask_shrdv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
631	643	{
632	644	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -634,7 +646,7 @@ _mm_mask_shrdv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
634	646	(__v4si)__A);
635	647	}
636	648
637		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	649	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
638	650	_mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
639	651	{
640	652	return (__m128i)__builtin_ia32_selectd_128(__U,
...	...	@@ -642,14 +654,15 @@ _mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
642	654	(__v4si)_mm_setzero_si128());
643	655	}
644	656
645		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	657	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
646	658	_mm256_shrdv_epi16(__m256i __A, __m256i __B, __m256i __C)
647	659	{
648		return (__m256i)__builtin_ia32_vpshrdvw256((__v16hi)__A, (__v16hi)__B,
649		(__v16hi)__C);
	660	// Ops __A and __B are swapped.
	661	return (__m256i)__builtin_elementwise_fshr((__v16hu)__B, (__v16hu)__A,
	662	(__v16hu)__C);
650	663	}
651	664
652		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	665	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
653	666	_mm256_mask_shrdv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
654	667	{
655	668	return (__m256i)__builtin_ia32_selectw_256(__U,
...	...	@@ -657,7 +670,7 @@ _mm256_mask_shrdv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
657	670	(__v16hi)__A);
658	671	}
659	672
660		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	673	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
661	674	_mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
662	675	{
663	676	return (__m256i)__builtin_ia32_selectw_256(__U,
...	...	@@ -665,14 +678,15 @@ _mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
665	678	(__v16hi)_mm256_setzero_si256());
666	679	}
667	680
668		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	681	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
669	682	_mm_shrdv_epi16(__m128i __A, __m128i __B, __m128i __C)
670	683	{
671		return (__m128i)__builtin_ia32_vpshrdvw128((__v8hi)__A, (__v8hi)__B,
672		(__v8hi)__C);
	684	// Ops __A and __B are swapped.
	685	return (__m128i)__builtin_elementwise_fshr((__v8hu)__B, (__v8hu)__A,
	686	(__v8hu)__C);
673	687	}
674	688
675		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	689	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
676	690	_mm_mask_shrdv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
677	691	{
678	692	return (__m128i)__builtin_ia32_selectw_128(__U,
...	...	@@ -680,7 +694,7 @@ _mm_mask_shrdv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
680	694	(__v8hi)__A);
681	695	}
682	696
683		static __inline__ __m128i __DEFAULT_FN_ATTRS128
	697	static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
684	698	_mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
685	699	{
686	700	return (__m128i)__builtin_ia32_selectw_128(__U,
...	...	@@ -688,8 +702,9 @@ _mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
688	702	(__v8hi)_mm_setzero_si128());
689	703	}
690	704
691
692	705	#undef __DEFAULT_FN_ATTRS128
693	706	#undef __DEFAULT_FN_ATTRS256
	707	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	708	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
694	709
695	710	#endif

lib/include/avx512vlvnniintrin.h+21-18

...	...	@@ -17,11 +17,11 @@
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS128 \
19	19	__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vl,avx512vnni,no-evex512"), \
	20	__target__("avx512vl,avx512vnni"), \
21	21	__min_vector_width__(128)))
22	22	#define __DEFAULT_FN_ATTRS256 \
23	23	__attribute__((__always_inline__, __nodebug__, \
24		__target__("avx512vl,avx512vnni,no-evex512"), \
	24	__target__("avx512vl,avx512vnni"), \
25	25	__min_vector_width__(256)))
26	26
27	27	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
...	...	@@ -41,8 +41,8 @@
41	41	/// ENDFOR
42	42	/// DST[MAX:256] := 0
43	43	/// \endcode
44		#define _mm256_dpbusd_epi32(S, A, B) \
45		((__m256i)__builtin_ia32_vpdpbusd256((__v8si)(S), (__v8si)(A), (__v8si)(B)))
	44	#define _mm256_dpbusd_epi32(S, A, B) \
	45	((__m256i)__builtin_ia32_vpdpbusd256((__v8si)(S), (__v32qu)(A), (__v32qi)(B)))
46	46
47	47	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
48	48	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
...	...	@@ -61,8 +61,9 @@
61	61	/// ENDFOR
62	62	/// DST[MAX:256] := 0
63	63	/// \endcode
64		#define _mm256_dpbusds_epi32(S, A, B) \
65		((__m256i)__builtin_ia32_vpdpbusds256((__v8si)(S), (__v8si)(A), (__v8si)(B)))
	64	#define _mm256_dpbusds_epi32(S, A, B) \
	65	((__m256i)__builtin_ia32_vpdpbusds256((__v8si)(S), (__v32qu)(A), \
	66	(__v32qi)(B)))
66	67
67	68	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
68	69	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
...	...	@@ -79,8 +80,8 @@
79	80	/// ENDFOR
80	81	/// DST[MAX:256] := 0
81	82	/// \endcode
82		#define _mm256_dpwssd_epi32(S, A, B) \
83		((__m256i)__builtin_ia32_vpdpwssd256((__v8si)(S), (__v8si)(A), (__v8si)(B)))
	83	#define _mm256_dpwssd_epi32(S, A, B) \
	84	((__m256i)__builtin_ia32_vpdpwssd256((__v8si)(S), (__v16hi)(A), (__v16hi)(B)))
84	85
85	86	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
86	87	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
...	...	@@ -97,8 +98,9 @@
97	98	/// ENDFOR
98	99	/// DST[MAX:256] := 0
99	100	/// \endcode
100		#define _mm256_dpwssds_epi32(S, A, B) \
101		((__m256i)__builtin_ia32_vpdpwssds256((__v8si)(S), (__v8si)(A), (__v8si)(B)))
	101	#define _mm256_dpwssds_epi32(S, A, B) \
	102	((__m256i)__builtin_ia32_vpdpwssds256((__v8si)(S), (__v16hi)(A), \
	103	(__v16hi)(B)))
102	104
103	105	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
104	106	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
...	...	@@ -117,8 +119,8 @@
117	119	/// ENDFOR
118	120	/// DST[MAX:128] := 0
119	121	/// \endcode
120		#define _mm_dpbusd_epi32(S, A, B) \
121		((__m128i)__builtin_ia32_vpdpbusd128((__v4si)(S), (__v4si)(A), (__v4si)(B)))
	122	#define _mm_dpbusd_epi32(S, A, B) \
	123	((__m128i)__builtin_ia32_vpdpbusd128((__v4si)(S), (__v16qu)(A), (__v16qi)(B)))
122	124
123	125	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a A with
124	126	/// corresponding signed 8-bit integers in \a B, producing 4 intermediate signed
...	...	@@ -137,8 +139,9 @@
137	139	/// ENDFOR
138	140	/// DST[MAX:128] := 0
139	141	/// \endcode
140		#define _mm_dpbusds_epi32(S, A, B) \
141		((__m128i)__builtin_ia32_vpdpbusds128((__v4si)(S), (__v4si)(A), (__v4si)(B)))
	142	#define _mm_dpbusds_epi32(S, A, B) \
	143	((__m128i)__builtin_ia32_vpdpbusds128((__v4si)(S), (__v16qu)(A), \
	144	(__v16qi)(B)))
142	145
143	146	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
144	147	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
...	...	@@ -155,8 +158,8 @@
155	158	/// ENDFOR
156	159	/// DST[MAX:128] := 0
157	160	/// \endcode
158		#define _mm_dpwssd_epi32(S, A, B) \
159		((__m128i)__builtin_ia32_vpdpwssd128((__v4si)(S), (__v4si)(A), (__v4si)(B)))
	161	#define _mm_dpwssd_epi32(S, A, B) \
	162	((__m128i)__builtin_ia32_vpdpwssd128((__v4si)(S), (__v8hi)(A), (__v8hi)(B)))
160	163
161	164	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a A with
162	165	/// corresponding 16-bit integers in \a B, producing 2 intermediate signed 32-bit
...	...	@@ -173,8 +176,8 @@
173	176	/// ENDFOR
174	177	/// DST[MAX:128] := 0
175	178	/// \endcode
176		#define _mm_dpwssds_epi32(S, A, B) \
177		((__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v4si)(A), (__v4si)(B)))
	179	#define _mm_dpwssds_epi32(S, A, B) \
	180	((__m128i)__builtin_ia32_vpdpwssds128((__v4si)(S), (__v8hi)(A), (__v8hi)(B)))
178	181
179	182	static __inline__ __m256i __DEFAULT_FN_ATTRS256
180	183	_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B)

lib/include/avx512vlvp2intersectintrin.h+2-2

...	...	@@ -30,12 +30,12 @@
30	30
31	31	#define __DEFAULT_FN_ATTRS128 \
32	32	__attribute__((__always_inline__, __nodebug__, \
33		__target__("avx512vl,avx512vp2intersect,no-evex512"), \
	33	__target__("avx512vl,avx512vp2intersect"), \
34	34	__min_vector_width__(128)))
35	35
36	36	#define __DEFAULT_FN_ATTRS256 \
37	37	__attribute__((__always_inline__, __nodebug__, \
38		__target__("avx512vl,avx512vp2intersect,no-evex512"), \
	38	__target__("avx512vl,avx512vp2intersect"), \
39	39	__min_vector_width__(256)))
40	40	/// Store, in an even/odd pair of mask registers, the indicators of the
41	41	/// locations of value matches between dwords in operands __a and __b.

lib/include/avx512vnniintrin.h+10-10

...	...	@@ -16,14 +16,14 @@
16	16
17	17	/* Define the default attributes for the functions in this file. */
18	18	#define __DEFAULT_FN_ATTRS \
19		__attribute__((__always_inline__, __nodebug__, \
20		__target__("avx512vnni,evex512"), __min_vector_width__(512)))
	19	__attribute__((__always_inline__, __nodebug__, __target__("avx512vnni"), \
	20	__min_vector_width__(512)))
21	21
22	22	static __inline__ __m512i __DEFAULT_FN_ATTRS
23	23	_mm512_dpbusd_epi32(__m512i __S, __m512i __A, __m512i __B)
24	24	{
25		return (__m512i)__builtin_ia32_vpdpbusd512((__v16si)__S, (__v16si)__A,
26		(__v16si)__B);
	25	return (__m512i)__builtin_ia32_vpdpbusd512((__v16si)__S, (__v64qu)__A,
	26	(__v64qi)__B);
27	27	}
28	28
29	29	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -45,8 +45,8 @@ _mm512_maskz_dpbusd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
45	45	static __inline__ __m512i __DEFAULT_FN_ATTRS
46	46	_mm512_dpbusds_epi32(__m512i __S, __m512i __A, __m512i __B)
47	47	{
48		return (__m512i)__builtin_ia32_vpdpbusds512((__v16si)__S, (__v16si)__A,
49		(__v16si)__B);
	48	return (__m512i)__builtin_ia32_vpdpbusds512((__v16si)__S, (__v64qu)__A,
	49	(__v64qi)__B);
50	50	}
51	51
52	52	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -68,8 +68,8 @@ _mm512_maskz_dpbusds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
68	68	static __inline__ __m512i __DEFAULT_FN_ATTRS
69	69	_mm512_dpwssd_epi32(__m512i __S, __m512i __A, __m512i __B)
70	70	{
71		return (__m512i)__builtin_ia32_vpdpwssd512((__v16si)__S, (__v16si)__A,
72		(__v16si)__B);
	71	return (__m512i)__builtin_ia32_vpdpwssd512((__v16si)__S, (__v32hi)__A,
	72	(__v32hi)__B);
73	73	}
74	74
75	75	static __inline__ __m512i __DEFAULT_FN_ATTRS
...	...	@@ -91,8 +91,8 @@ _mm512_maskz_dpwssd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B)
91	91	static __inline__ __m512i __DEFAULT_FN_ATTRS
92	92	_mm512_dpwssds_epi32(__m512i __S, __m512i __A, __m512i __B)
93	93	{
94		return (__m512i)__builtin_ia32_vpdpwssds512((__v16si)__S, (__v16si)__A,
95		(__v16si)__B);
	94	return (__m512i)__builtin_ia32_vpdpwssds512((__v16si)__S, (__v32hi)__A,
	95	(__v32hi)__B);
96	96	}
97	97
98	98	static __inline__ __m512i __DEFAULT_FN_ATTRS

lib/include/avx512vp2intersectintrin.h+1-2

...	...	@@ -30,8 +30,7 @@
30	30
31	31	#define __DEFAULT_FN_ATTRS \
32	32	__attribute__((__always_inline__, __nodebug__, \
33		__target__("avx512vp2intersect,evex512"), \
34		__min_vector_width__(512)))
	33	__target__("avx512vp2intersect"), __min_vector_width__(512)))
35	34
36	35	/// Store, in an even/odd pair of mask registers, the indicators of the
37	36	/// locations of value matches between dwords in operands __a and __b.

lib/include/avx512vpopcntdqintrin.h+8-10

...	...	@@ -16,19 +16,18 @@
16	16	#define __AVX512VPOPCNTDQINTRIN_H
17	17
18	18	/* Define the default attributes for the functions in this file. */
	19	#if defined(__cplusplus) && (__cplusplus >= 201103L)
19	20	#define __DEFAULT_FN_ATTRS \
20	21	__attribute__((__always_inline__, __nodebug__, \
21		__target__("avx512vpopcntdq,evex512"), \
22		__min_vector_width__(512)))
23
24		#if defined(__cplusplus) && (__cplusplus >= 201103L)
25		#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	22	__target__("avx512vpopcntdq"), \
	23	__min_vector_width__(512))) constexpr
26	24	#else
27		#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	25	#define __DEFAULT_FN_ATTRS \
	26	__attribute__((__always_inline__, __nodebug__, \
	27	__target__("avx512vpopcntdq"), __min_vector_width__(512)))
28	28	#endif
29	29
30		static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
31		_mm512_popcnt_epi64(__m512i __A) {
	30	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi64(__m512i __A) {
32	31	return (__m512i)__builtin_elementwise_popcount((__v8du)__A);
33	32	}
34	33
...	...	@@ -43,8 +42,7 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
43	42	return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);
44	43	}
45	44
46		static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
47		_mm512_popcnt_epi32(__m512i __A) {
	45	static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi32(__m512i __A) {
48	46	return (__m512i)__builtin_elementwise_popcount((__v16su)__A);
49	47	}
50	48

lib/include/avx512vpopcntdqvlintrin.h+17-15

...	...	@@ -16,25 +16,28 @@
16	16	#define __AVX512VPOPCNTDQVLINTRIN_H
17	17
18	18	/* Define the default attributes for the functions in this file. */
	19
	20	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	21	#define __DEFAULT_FN_ATTRS128 \
	22	__attribute__((__always_inline__, __nodebug__, \
	23	__target__("avx512vpopcntdq,avx512vl"), \
	24	__min_vector_width__(128))) constexpr
	25	#define __DEFAULT_FN_ATTRS256 \
	26	__attribute__((__always_inline__, __nodebug__, \
	27	__target__("avx512vpopcntdq,avx512vl"), \
	28	__min_vector_width__(256))) constexpr
	29	#else
19	30	#define __DEFAULT_FN_ATTRS128 \
20	31	__attribute__((__always_inline__, __nodebug__, \
21		__target__("avx512vpopcntdq,avx512vl,no-evex512"), \
	32	__target__("avx512vpopcntdq,avx512vl"), \
22	33	__min_vector_width__(128)))
23	34	#define __DEFAULT_FN_ATTRS256 \
24	35	__attribute__((__always_inline__, __nodebug__, \
25		__target__("avx512vpopcntdq,avx512vl,no-evex512"), \
	36	__target__("avx512vpopcntdq,avx512vl"), \
26	37	__min_vector_width__(256)))
27
28		#if defined(__cplusplus) && (__cplusplus >= 201103L)
29		#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
30		#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
31		#else
32		#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
33		#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
34	38	#endif
35	39
36		static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
37		_mm_popcnt_epi64(__m128i __A) {
	40	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi64(__m128i __A) {
38	41	return (__m128i)__builtin_elementwise_popcount((__v2du)__A);
39	42	}
40	43
...	...	@@ -49,8 +52,7 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
49	52	return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);
50	53	}
51	54
52		static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
53		_mm_popcnt_epi32(__m128i __A) {
	55	static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi32(__m128i __A) {
54	56	return (__m128i)__builtin_elementwise_popcount((__v4su)__A);
55	57	}
56	58
...	...	@@ -65,7 +67,7 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
65	67	return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);
66	68	}
67	69
68		static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	70	static __inline__ __m256i __DEFAULT_FN_ATTRS256
69	71	_mm256_popcnt_epi64(__m256i __A) {
70	72	return (__m256i)__builtin_elementwise_popcount((__v4du)__A);
71	73	}
...	...	@@ -81,7 +83,7 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
81	83	return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);
82	84	}
83	85
84		static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
	86	static __inline__ __m256i __DEFAULT_FN_ATTRS256
85	87	_mm256_popcnt_epi32(__m256i __A) {
86	88	return (__m256i)__builtin_elementwise_popcount((__v8su)__A);
87	89	}

lib/include/avxifmaintrin.h+16

...	...	@@ -15,12 +15,28 @@
15	15	#define __AVXIFMAINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	19	#define __DEFAULT_FN_ATTRS128 \
	20	__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
	21	__min_vector_width__(128))) constexpr
	22	#define __DEFAULT_FN_ATTRS256 \
	23	__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
	24	__min_vector_width__(256))) constexpr
	25	#else
18	26	#define __DEFAULT_FN_ATTRS128 \
19	27	__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
20	28	__min_vector_width__(128)))
21	29	#define __DEFAULT_FN_ATTRS256 \
22	30	__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
23	31	__min_vector_width__(256)))
	32	#endif
	33
	34	#if !defined(__AVX512IFMA__) && defined(__AVXIFMA__)
	35	#define _mm_madd52hi_epu64(X, Y, Z) _mm_madd52hi_avx_epu64(X, Y, Z)
	36	#define _mm_madd52lo_epu64(X, Y, Z) _mm_madd52lo_avx_epu64(X, Y, Z)
	37	#define _mm256_madd52hi_epu64(X, Y, Z) _mm256_madd52hi_avx_epu64(X, Y, Z)
	38	#define _mm256_madd52lo_epu64(X, Y, Z) _mm256_madd52lo_avx_epu64(X, Y, Z)
	39	#endif
24	40
25	41	// must vex-encoding
26	42

lib/include/avxintrin.h+165-241

...	...	@@ -50,28 +50,19 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
50	50	#endif
51	51
52	52	/* Define the default attributes for the functions in this file. */
53		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
54		#define __DEFAULT_FN_ATTRS \
55		__attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
56		__min_vector_width__(256)))
57		#define __DEFAULT_FN_ATTRS128 \
58		__attribute__((__always_inline__, __nodebug__, __target__("avx,no-evex512"), \
59		__min_vector_width__(128)))
60		#else
61	53	#define __DEFAULT_FN_ATTRS \
62	54	__attribute__((__always_inline__, __nodebug__, __target__("avx"), \
63	55	__min_vector_width__(256)))
64	56	#define __DEFAULT_FN_ATTRS128 \
65	57	__attribute__((__always_inline__, __nodebug__, __target__("avx"), \
66	58	__min_vector_width__(128)))
67		#endif
68	59
69	60	#if defined(__cplusplus) && (__cplusplus >= 201103L)
70	61	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
71	62	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
72	63	#else
73		#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
74		#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
	64	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	65	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
75	66	#endif
76	67
77	68	/* Arithmetic */
...	...	@@ -87,9 +78,8 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
87	78	/// A 256-bit vector of [4 x double] containing one of the source operands.
88	79	/// \returns A 256-bit vector of [4 x double] containing the sums of both
89	80	/// operands.
90		static __inline __m256d __DEFAULT_FN_ATTRS
91		_mm256_add_pd(__m256d __a, __m256d __b)
92		{
	81	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	82	_mm256_add_pd(__m256d __a, __m256d __b) {
93	83	return (__m256d)((__v4df)__a+(__v4df)__b);
94	84	}
95	85
...	...	@@ -105,9 +95,8 @@ _mm256_add_pd(__m256d __a, __m256d __b)
105	95	/// A 256-bit vector of [8 x float] containing one of the source operands.
106	96	/// \returns A 256-bit vector of [8 x float] containing the sums of both
107	97	/// operands.
108		static __inline __m256 __DEFAULT_FN_ATTRS
109		_mm256_add_ps(__m256 __a, __m256 __b)
110		{
	98	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_add_ps(__m256 __a,
	99	__m256 __b) {
111	100	return (__m256)((__v8sf)__a+(__v8sf)__b);
112	101	}
113	102
...	...	@@ -123,9 +112,8 @@ _mm256_add_ps(__m256 __a, __m256 __b)
123	112	/// A 256-bit vector of [4 x double] containing the subtrahend.
124	113	/// \returns A 256-bit vector of [4 x double] containing the differences between
125	114	/// both operands.
126		static __inline __m256d __DEFAULT_FN_ATTRS
127		_mm256_sub_pd(__m256d __a, __m256d __b)
128		{
	115	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	116	_mm256_sub_pd(__m256d __a, __m256d __b) {
129	117	return (__m256d)((__v4df)__a-(__v4df)__b);
130	118	}
131	119
...	...	@@ -141,9 +129,8 @@ _mm256_sub_pd(__m256d __a, __m256d __b)
141	129	/// A 256-bit vector of [8 x float] containing the subtrahend.
142	130	/// \returns A 256-bit vector of [8 x float] containing the differences between
143	131	/// both operands.
144		static __inline __m256 __DEFAULT_FN_ATTRS
145		_mm256_sub_ps(__m256 __a, __m256 __b)
146		{
	132	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_sub_ps(__m256 __a,
	133	__m256 __b) {
147	134	return (__m256)((__v8sf)__a-(__v8sf)__b);
148	135	}
149	136
...	...	@@ -160,9 +147,8 @@ _mm256_sub_ps(__m256 __a, __m256 __b)
160	147	/// A 256-bit vector of [4 x double] containing the right source operand.
161	148	/// \returns A 256-bit vector of [4 x double] containing the alternating sums
162	149	/// and differences between both operands.
163		static __inline __m256d __DEFAULT_FN_ATTRS
164		_mm256_addsub_pd(__m256d __a, __m256d __b)
165		{
	150	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	151	_mm256_addsub_pd(__m256d __a, __m256d __b) {
166	152	return (__m256d)__builtin_ia32_addsubpd256((__v4df)__a, (__v4df)__b);
167	153	}
168	154
...	...	@@ -179,9 +165,8 @@ _mm256_addsub_pd(__m256d __a, __m256d __b)
179	165	/// A 256-bit vector of [8 x float] containing the right source operand.
180	166	/// \returns A 256-bit vector of [8 x float] containing the alternating sums and
181	167	/// differences between both operands.
182		static __inline __m256 __DEFAULT_FN_ATTRS
183		_mm256_addsub_ps(__m256 __a, __m256 __b)
184		{
	168	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	169	_mm256_addsub_ps(__m256 __a, __m256 __b) {
185	170	return (__m256)__builtin_ia32_addsubps256((__v8sf)__a, (__v8sf)__b);
186	171	}
187	172
...	...	@@ -197,9 +182,8 @@ _mm256_addsub_ps(__m256 __a, __m256 __b)
197	182	/// A 256-bit vector of [4 x double] containing the divisor.
198	183	/// \returns A 256-bit vector of [4 x double] containing the quotients of both
199	184	/// operands.
200		static __inline __m256d __DEFAULT_FN_ATTRS
201		_mm256_div_pd(__m256d __a, __m256d __b)
202		{
	185	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	186	_mm256_div_pd(__m256d __a, __m256d __b) {
203	187	return (__m256d)((__v4df)__a/(__v4df)__b);
204	188	}
205	189
...	...	@@ -215,9 +199,8 @@ _mm256_div_pd(__m256d __a, __m256d __b)
215	199	/// A 256-bit vector of [8 x float] containing the divisor.
216	200	/// \returns A 256-bit vector of [8 x float] containing the quotients of both
217	201	/// operands.
218		static __inline __m256 __DEFAULT_FN_ATTRS
219		_mm256_div_ps(__m256 __a, __m256 __b)
220		{
	202	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_div_ps(__m256 __a,
	203	__m256 __b) {
221	204	return (__m256)((__v8sf)__a/(__v8sf)__b);
222	205	}
223	206
...	...	@@ -236,9 +219,8 @@ _mm256_div_ps(__m256 __a, __m256 __b)
236	219	/// A 256-bit vector of [4 x double] containing one of the operands.
237	220	/// \returns A 256-bit vector of [4 x double] containing the maximum values
238	221	/// between both operands.
239		static __inline __m256d __DEFAULT_FN_ATTRS
240		_mm256_max_pd(__m256d __a, __m256d __b)
241		{
	222	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	223	_mm256_max_pd(__m256d __a, __m256d __b) {
242	224	return (__m256d)__builtin_ia32_maxpd256((__v4df)__a, (__v4df)__b);
243	225	}
244	226
...	...	@@ -257,9 +239,8 @@ _mm256_max_pd(__m256d __a, __m256d __b)
257	239	/// A 256-bit vector of [8 x float] containing one of the operands.
258	240	/// \returns A 256-bit vector of [8 x float] containing the maximum values
259	241	/// between both operands.
260		static __inline __m256 __DEFAULT_FN_ATTRS
261		_mm256_max_ps(__m256 __a, __m256 __b)
262		{
	242	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_max_ps(__m256 __a,
	243	__m256 __b) {
263	244	return (__m256)__builtin_ia32_maxps256((__v8sf)__a, (__v8sf)__b);
264	245	}
265	246
...	...	@@ -278,9 +259,8 @@ _mm256_max_ps(__m256 __a, __m256 __b)
278	259	/// A 256-bit vector of [4 x double] containing one of the operands.
279	260	/// \returns A 256-bit vector of [4 x double] containing the minimum values
280	261	/// between both operands.
281		static __inline __m256d __DEFAULT_FN_ATTRS
282		_mm256_min_pd(__m256d __a, __m256d __b)
283		{
	262	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	263	_mm256_min_pd(__m256d __a, __m256d __b) {
284	264	return (__m256d)__builtin_ia32_minpd256((__v4df)__a, (__v4df)__b);
285	265	}
286	266
...	...	@@ -299,9 +279,8 @@ _mm256_min_pd(__m256d __a, __m256d __b)
299	279	/// A 256-bit vector of [8 x float] containing one of the operands.
300	280	/// \returns A 256-bit vector of [8 x float] containing the minimum values
301	281	/// between both operands.
302		static __inline __m256 __DEFAULT_FN_ATTRS
303		_mm256_min_ps(__m256 __a, __m256 __b)
304		{
	282	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_min_ps(__m256 __a,
	283	__m256 __b) {
305	284	return (__m256)__builtin_ia32_minps256((__v8sf)__a, (__v8sf)__b);
306	285	}
307	286
...	...	@@ -317,9 +296,8 @@ _mm256_min_ps(__m256 __a, __m256 __b)
317	296	/// A 256-bit vector of [4 x double] containing one of the operands.
318	297	/// \returns A 256-bit vector of [4 x double] containing the products of both
319	298	/// operands.
320		static __inline __m256d __DEFAULT_FN_ATTRS
321		_mm256_mul_pd(__m256d __a, __m256d __b)
322		{
	299	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	300	_mm256_mul_pd(__m256d __a, __m256d __b) {
323	301	return (__m256d)((__v4df)__a * (__v4df)__b);
324	302	}
325	303
...	...	@@ -335,9 +313,8 @@ _mm256_mul_pd(__m256d __a, __m256d __b)
335	313	/// A 256-bit vector of [8 x float] containing one of the operands.
336	314	/// \returns A 256-bit vector of [8 x float] containing the products of both
337	315	/// operands.
338		static __inline __m256 __DEFAULT_FN_ATTRS
339		_mm256_mul_ps(__m256 __a, __m256 __b)
340		{
	316	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_mul_ps(__m256 __a,
	317	__m256 __b) {
341	318	return (__m256)((__v8sf)__a * (__v8sf)__b);
342	319	}
343	320
...	...	@@ -352,10 +329,8 @@ _mm256_mul_ps(__m256 __a, __m256 __b)
352	329	/// A 256-bit vector of [4 x double].
353	330	/// \returns A 256-bit vector of [4 x double] containing the square roots of the
354	331	/// values in the operand.
355		static __inline __m256d __DEFAULT_FN_ATTRS
356		_mm256_sqrt_pd(__m256d __a)
357		{
358		return (__m256d)__builtin_ia32_sqrtpd256((__v4df)__a);
	332	static __inline __m256d __DEFAULT_FN_ATTRS _mm256_sqrt_pd(__m256d __a) {
	333	return __builtin_elementwise_sqrt(__a);
359	334	}
360	335
361	336	/// Calculates the square roots of the values in a 256-bit vector of
...	...	@@ -369,10 +344,8 @@ _mm256_sqrt_pd(__m256d __a)
369	344	/// A 256-bit vector of [8 x float].
370	345	/// \returns A 256-bit vector of [8 x float] containing the square roots of the
371	346	/// values in the operand.
372		static __inline __m256 __DEFAULT_FN_ATTRS
373		_mm256_sqrt_ps(__m256 __a)
374		{
375		return (__m256)__builtin_ia32_sqrtps256((__v8sf)__a);
	347	static __inline __m256 __DEFAULT_FN_ATTRS _mm256_sqrt_ps(__m256 __a) {
	348	return __builtin_elementwise_sqrt(__a);
376	349	}
377	350
378	351	/// Calculates the reciprocal square roots of the values in a 256-bit
...	...	@@ -555,7 +528,7 @@ _mm256_rcp_ps(__m256 __a)
555	528	/// A 256-bit vector of [4 x double] containing one of the source operands.
556	529	/// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the
557	530	/// values between both operands.
558		static __inline __m256d __DEFAULT_FN_ATTRS
	531	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
559	532	_mm256_and_pd(__m256d __a, __m256d __b)
560	533	{
561	534	return (__m256d)((__v4du)__a & (__v4du)__b);
...	...	@@ -573,7 +546,7 @@ _mm256_and_pd(__m256d __a, __m256d __b)
573	546	/// A 256-bit vector of [8 x float] containing one of the source operands.
574	547	/// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the
575	548	/// values between both operands.
576		static __inline __m256 __DEFAULT_FN_ATTRS
	549	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
577	550	_mm256_and_ps(__m256 __a, __m256 __b)
578	551	{
579	552	return (__m256)((__v8su)__a & (__v8su)__b);
...	...	@@ -594,7 +567,7 @@ _mm256_and_ps(__m256 __a, __m256 __b)
594	567	/// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the
595	568	/// values of the second operand and the one's complement of the first
596	569	/// operand.
597		static __inline __m256d __DEFAULT_FN_ATTRS
	570	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
598	571	_mm256_andnot_pd(__m256d __a, __m256d __b)
599	572	{
600	573	return (__m256d)(~(__v4du)__a & (__v4du)__b);
...	...	@@ -615,7 +588,7 @@ _mm256_andnot_pd(__m256d __a, __m256d __b)
615	588	/// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the
616	589	/// values of the second operand and the one's complement of the first
617	590	/// operand.
618		static __inline __m256 __DEFAULT_FN_ATTRS
	591	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
619	592	_mm256_andnot_ps(__m256 __a, __m256 __b)
620	593	{
621	594	return (__m256)(~(__v8su)__a & (__v8su)__b);
...	...	@@ -633,7 +606,7 @@ _mm256_andnot_ps(__m256 __a, __m256 __b)
633	606	/// A 256-bit vector of [4 x double] containing one of the source operands.
634	607	/// \returns A 256-bit vector of [4 x double] containing the bitwise OR of the
635	608	/// values between both operands.
636		static __inline __m256d __DEFAULT_FN_ATTRS
	609	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
637	610	_mm256_or_pd(__m256d __a, __m256d __b)
638	611	{
639	612	return (__m256d)((__v4du)__a \| (__v4du)__b);
...	...	@@ -651,7 +624,7 @@ _mm256_or_pd(__m256d __a, __m256d __b)
651	624	/// A 256-bit vector of [8 x float] containing one of the source operands.
652	625	/// \returns A 256-bit vector of [8 x float] containing the bitwise OR of the
653	626	/// values between both operands.
654		static __inline __m256 __DEFAULT_FN_ATTRS
	627	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
655	628	_mm256_or_ps(__m256 __a, __m256 __b)
656	629	{
657	630	return (__m256)((__v8su)__a \| (__v8su)__b);
...	...	@@ -669,7 +642,7 @@ _mm256_or_ps(__m256 __a, __m256 __b)
669	642	/// A 256-bit vector of [4 x double] containing one of the source operands.
670	643	/// \returns A 256-bit vector of [4 x double] containing the bitwise XOR of the
671	644	/// values between both operands.
672		static __inline __m256d __DEFAULT_FN_ATTRS
	645	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
673	646	_mm256_xor_pd(__m256d __a, __m256d __b)
674	647	{
675	648	return (__m256d)((__v4du)__a ^ (__v4du)__b);
...	...	@@ -687,7 +660,7 @@ _mm256_xor_pd(__m256d __a, __m256d __b)
687	660	/// A 256-bit vector of [8 x float] containing one of the source operands.
688	661	/// \returns A 256-bit vector of [8 x float] containing the bitwise XOR of the
689	662	/// values between both operands.
690		static __inline __m256 __DEFAULT_FN_ATTRS
	663	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
691	664	_mm256_xor_ps(__m256 __a, __m256 __b)
692	665	{
693	666	return (__m256)((__v8su)__a ^ (__v8su)__b);
...	...	@@ -711,9 +684,8 @@ _mm256_xor_ps(__m256 __a, __m256 __b)
711	684	/// elements of a vector of [4 x double].
712	685	/// \returns A 256-bit vector of [4 x double] containing the horizontal sums of
713	686	/// both operands.
714		static __inline __m256d __DEFAULT_FN_ATTRS
715		_mm256_hadd_pd(__m256d __a, __m256d __b)
716		{
	687	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	688	_mm256_hadd_pd(__m256d __a, __m256d __b) {
717	689	return (__m256d)__builtin_ia32_haddpd256((__v4df)__a, (__v4df)__b);
718	690	}
719	691
...	...	@@ -734,9 +706,8 @@ _mm256_hadd_pd(__m256d __a, __m256d __b)
734	706	/// index 2, 3, 6, 7 of a vector of [8 x float].
735	707	/// \returns A 256-bit vector of [8 x float] containing the horizontal sums of
736	708	/// both operands.
737		static __inline __m256 __DEFAULT_FN_ATTRS
738		_mm256_hadd_ps(__m256 __a, __m256 __b)
739		{
	709	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_hadd_ps(__m256 __a,
	710	__m256 __b) {
740	711	return (__m256)__builtin_ia32_haddps256((__v8sf)__a, (__v8sf)__b);
741	712	}
742	713
...	...	@@ -757,9 +728,8 @@ _mm256_hadd_ps(__m256 __a, __m256 __b)
757	728	/// odd-indexed elements of a vector of [4 x double].
758	729	/// \returns A 256-bit vector of [4 x double] containing the horizontal
759	730	/// differences of both operands.
760		static __inline __m256d __DEFAULT_FN_ATTRS
761		_mm256_hsub_pd(__m256d __a, __m256d __b)
762		{
	731	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	732	_mm256_hsub_pd(__m256d __a, __m256d __b) {
763	733	return (__m256d)__builtin_ia32_hsubpd256((__v4df)__a, (__v4df)__b);
764	734	}
765	735
...	...	@@ -780,9 +750,8 @@ _mm256_hsub_pd(__m256d __a, __m256d __b)
780	750	/// elements with index 2, 3, 6, 7 of a vector of [8 x float].
781	751	/// \returns A 256-bit vector of [8 x float] containing the horizontal
782	752	/// differences of both operands.
783		static __inline __m256 __DEFAULT_FN_ATTRS
784		_mm256_hsub_ps(__m256 __a, __m256 __b)
785		{
	753	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_hsub_ps(__m256 __a,
	754	__m256 __b) {
786	755	return (__m256)__builtin_ia32_hsubps256((__v8sf)__a, (__v8sf)__b);
787	756	}
788	757
...	...	@@ -810,9 +779,8 @@ _mm256_hsub_ps(__m256 __a, __m256 __b)
810	779	/// 1: Bits [127:64] of the source are copied to bits [127:64] of the
811	780	/// returned vector.
812	781	/// \returns A 128-bit vector of [2 x double] containing the copied values.
813		static __inline __m128d __DEFAULT_FN_ATTRS128
814		_mm_permutevar_pd(__m128d __a, __m128i __c)
815		{
	782	static __inline __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	783	_mm_permutevar_pd(__m128d __a, __m128i __c) {
816	784	return (__m128d)__builtin_ia32_vpermilvarpd((__v2df)__a, (__v2di)__c);
817	785	}
818	786
...	...	@@ -849,9 +817,8 @@ _mm_permutevar_pd(__m128d __a, __m128i __c)
849	817	/// 1: Bits [255:192] of the source are copied to bits [255:192] of the
850	818	/// returned vector.
851	819	/// \returns A 256-bit vector of [4 x double] containing the copied values.
852		static __inline __m256d __DEFAULT_FN_ATTRS
853		_mm256_permutevar_pd(__m256d __a, __m256i __c)
854		{
	820	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	821	_mm256_permutevar_pd(__m256d __a, __m256i __c) {
855	822	return (__m256d)__builtin_ia32_vpermilvarpd256((__v4df)__a, (__v4di)__c);
856	823	}
857	824
...	...	@@ -904,9 +871,8 @@ _mm256_permutevar_pd(__m256d __a, __m256i __c)
904	871	/// 11: Bits [127:96] of the source are copied to bits [127:96] of the
905	872	/// returned vector.
906	873	/// \returns A 128-bit vector of [4 x float] containing the copied values.
907		static __inline __m128 __DEFAULT_FN_ATTRS128
908		_mm_permutevar_ps(__m128 __a, __m128i __c)
909		{
	874	static __inline __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	875	_mm_permutevar_ps(__m128 __a, __m128i __c) {
910	876	return (__m128)__builtin_ia32_vpermilvarps((__v4sf)__a, (__v4si)__c);
911	877	}
912	878
...	...	@@ -995,9 +961,8 @@ _mm_permutevar_ps(__m128 __a, __m128i __c)
995	961	/// 11: Bits [255:224] of the source are copied to bits [255:224] of the
996	962	/// returned vector.
997	963	/// \returns A 256-bit vector of [8 x float] containing the copied values.
998		static __inline __m256 __DEFAULT_FN_ATTRS
999		_mm256_permutevar_ps(__m256 __a, __m256i __c)
1000		{
	964	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	965	_mm256_permutevar_ps(__m256 __a, __m256i __c) {
1001	966	return (__m256)__builtin_ia32_vpermilvarps256((__v8sf)__a, (__v8si)__c);
1002	967	}
1003	968
...	...	@@ -1419,9 +1384,8 @@ _mm256_permutevar_ps(__m256 __a, __m256i __c)
1419	1384	/// 64-bit element in operand \a __b is copied to the same position in the
1420	1385	/// destination.
1421	1386	/// \returns A 256-bit vector of [4 x double] containing the copied values.
1422		static __inline __m256d __DEFAULT_FN_ATTRS
1423		_mm256_blendv_pd(__m256d __a, __m256d __b, __m256d __c)
1424		{
	1387	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	1388	_mm256_blendv_pd(__m256d __a, __m256d __b, __m256d __c) {
1425	1389	return (__m256d)__builtin_ia32_blendvpd256(
1426	1390	(__v4df)__a, (__v4df)__b, (__v4df)__c);
1427	1391	}
...	...	@@ -1447,9 +1411,8 @@ _mm256_blendv_pd(__m256d __a, __m256d __b, __m256d __c)
1447	1411	/// corresponding 32-bit element in operand \a __b is copied to the same
1448	1412	/// position in the destination.
1449	1413	/// \returns A 256-bit vector of [8 x float] containing the copied values.
1450		static __inline __m256 __DEFAULT_FN_ATTRS
1451		_mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c)
1452		{
	1414	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	1415	_mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) {
1453	1416	return (__m256)__builtin_ia32_blendvps256(
1454	1417	(__v8sf)__a, (__v8sf)__b, (__v8sf)__c);
1455	1418	}
...	...	@@ -2190,9 +2153,8 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c)
2190	2153	/// \param __a
2191	2154	/// A 128-bit integer vector of [4 x i32].
2192	2155	/// \returns A 256-bit vector of [4 x double] containing the converted values.
2193		static __inline __m256d __DEFAULT_FN_ATTRS
2194		_mm256_cvtepi32_pd(__m128i __a)
2195		{
	2156	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	2157	_mm256_cvtepi32_pd(__m128i __a) {
2196	2158	return (__m256d)__builtin_convertvector((__v4si)__a, __v4df);
2197	2159	}
2198	2160
...	...	@@ -2205,9 +2167,8 @@ _mm256_cvtepi32_pd(__m128i __a)
2205	2167	/// \param __a
2206	2168	/// A 256-bit integer vector.
2207	2169	/// \returns A 256-bit vector of [8 x float] containing the converted values.
2208		static __inline __m256 __DEFAULT_FN_ATTRS
2209		_mm256_cvtepi32_ps(__m256i __a)
2210		{
	2170	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	2171	_mm256_cvtepi32_ps(__m256i __a) {
2211	2172	return (__m256)__builtin_convertvector((__v8si)__a, __v8sf);
2212	2173	}
2213	2174
...	...	@@ -2221,9 +2182,8 @@ _mm256_cvtepi32_ps(__m256i __a)
2221	2182	/// \param __a
2222	2183	/// A 256-bit vector of [4 x double].
2223	2184	/// \returns A 128-bit vector of [4 x float] containing the converted values.
2224		static __inline __m128 __DEFAULT_FN_ATTRS
2225		_mm256_cvtpd_ps(__m256d __a)
2226		{
	2185	static __inline __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
	2186	_mm256_cvtpd_ps(__m256d __a) {
2227	2187	return (__m128)__builtin_ia32_cvtpd2ps256((__v4df) __a);
2228	2188	}
2229	2189
...	...	@@ -2256,9 +2216,8 @@ _mm256_cvtps_epi32(__m256 __a)
2256	2216	/// \param __a
2257	2217	/// A 128-bit vector of [4 x float].
2258	2218	/// \returns A 256-bit vector of [4 x double] containing the converted values.
2259		static __inline __m256d __DEFAULT_FN_ATTRS
2260		_mm256_cvtps_pd(__m128 __a)
2261		{
	2219	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	2220	_mm256_cvtps_pd(__m128 __a) {
2262	2221	return (__m256d)__builtin_convertvector((__v4sf)__a, __v4df);
2263	2222	}
2264	2223
...	...	@@ -2333,10 +2292,9 @@ _mm256_cvttps_epi32(__m256 __a)
2333	2292	/// \param __a
2334	2293	/// A 256-bit vector of [4 x double].
2335	2294	/// \returns A 64 bit double containing the first element of the input vector.
2336		static __inline double __DEFAULT_FN_ATTRS
2337		_mm256_cvtsd_f64(__m256d __a)
2338		{
2339		return __a[0];
	2295	static __inline double __DEFAULT_FN_ATTRS_CONSTEXPR
	2296	_mm256_cvtsd_f64(__m256d __a) {
	2297	return __a[0];
2340	2298	}
2341	2299
2342	2300	/// Returns the first element of the input vector of [8 x i32].
...	...	@@ -2349,11 +2307,10 @@ _mm256_cvtsd_f64(__m256d __a)
2349	2307	/// \param __a
2350	2308	/// A 256-bit vector of [8 x i32].
2351	2309	/// \returns A 32 bit integer containing the first element of the input vector.
2352		static __inline int __DEFAULT_FN_ATTRS
2353		_mm256_cvtsi256_si32(__m256i __a)
2354		{
2355		__v8si __b = (__v8si)__a;
2356		return __b[0];
	2310	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2311	_mm256_cvtsi256_si32(__m256i __a) {
	2312	__v8si __b = (__v8si)__a;
	2313	return __b[0];
2357	2314	}
2358	2315
2359	2316	/// Returns the first element of the input vector of [8 x float].
...	...	@@ -2366,10 +2323,9 @@ _mm256_cvtsi256_si32(__m256i __a)
2366	2323	/// \param __a
2367	2324	/// A 256-bit vector of [8 x float].
2368	2325	/// \returns A 32 bit float containing the first element of the input vector.
2369		static __inline float __DEFAULT_FN_ATTRS
2370		_mm256_cvtss_f32(__m256 __a)
2371		{
2372		return __a[0];
	2326	static __inline float __DEFAULT_FN_ATTRS_CONSTEXPR
	2327	_mm256_cvtss_f32(__m256 __a) {
	2328	return __a[0];
2373	2329	}
2374	2330
2375	2331	/* Vector replicate */
...	...	@@ -2392,7 +2348,7 @@ _mm256_cvtss_f32(__m256 __a)
2392	2348	/// return value.
2393	2349	/// \returns A 256-bit vector of [8 x float] containing the moved and duplicated
2394	2350	/// values.
2395		static __inline __m256 __DEFAULT_FN_ATTRS
	2351	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
2396	2352	_mm256_movehdup_ps(__m256 __a)
2397	2353	{
2398	2354	return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 1, 1, 3, 3, 5, 5, 7, 7);
...	...	@@ -2417,7 +2373,7 @@ _mm256_movehdup_ps(__m256 __a)
2417	2373	/// return value.
2418	2374	/// \returns A 256-bit vector of [8 x float] containing the moved and duplicated
2419	2375	/// values.
2420		static __inline __m256 __DEFAULT_FN_ATTRS
	2376	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
2421	2377	_mm256_moveldup_ps(__m256 __a)
2422	2378	{
2423	2379	return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 0, 2, 2, 4, 4, 6, 6);
...	...	@@ -2439,7 +2395,7 @@ _mm256_moveldup_ps(__m256 __a)
2439	2395	/// the return value.
2440	2396	/// \returns A 256-bit vector of [4 x double] containing the moved and
2441	2397	/// duplicated values.
2442		static __inline __m256d __DEFAULT_FN_ATTRS
	2398	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
2443	2399	_mm256_movedup_pd(__m256d __a)
2444	2400	{
2445	2401	return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 0, 2, 2);
...	...	@@ -2462,9 +2418,8 @@ _mm256_movedup_pd(__m256d __a)
2462	2418	/// Bits [127:64] are written to bits [127:64] of the return value. \n
2463	2419	/// Bits [255:192] are written to bits [255:192] of the return value. \n
2464	2420	/// \returns A 256-bit vector of [4 x double] containing the interleaved values.
2465		static __inline __m256d __DEFAULT_FN_ATTRS
2466		_mm256_unpackhi_pd(__m256d __a, __m256d __b)
2467		{
	2421	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	2422	_mm256_unpackhi_pd(__m256d __a, __m256d __b) {
2468	2423	return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 1, 5, 1+2, 5+2);
2469	2424	}
2470	2425
...	...	@@ -2484,9 +2439,8 @@ _mm256_unpackhi_pd(__m256d __a, __m256d __b)
2484	2439	/// Bits [63:0] are written to bits [127:64] of the return value. \n
2485	2440	/// Bits [191:128] are written to bits [255:192] of the return value. \n
2486	2441	/// \returns A 256-bit vector of [4 x double] containing the interleaved values.
2487		static __inline __m256d __DEFAULT_FN_ATTRS
2488		_mm256_unpacklo_pd(__m256d __a, __m256d __b)
2489		{
	2442	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	2443	_mm256_unpacklo_pd(__m256d __a, __m256d __b) {
2490	2444	return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 0, 4, 0+2, 4+2);
2491	2445	}
2492	2446
...	...	@@ -2511,9 +2465,8 @@ _mm256_unpacklo_pd(__m256d __a, __m256d __b)
2511	2465	/// Bits [223:192] are written to bits [191:160] of the return value. \n
2512	2466	/// Bits [255:224] are written to bits [255:224] of the return value.
2513	2467	/// \returns A 256-bit vector of [8 x float] containing the interleaved values.
2514		static __inline __m256 __DEFAULT_FN_ATTRS
2515		_mm256_unpackhi_ps(__m256 __a, __m256 __b)
2516		{
	2468	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	2469	_mm256_unpackhi_ps(__m256 __a, __m256 __b) {
2517	2470	return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1);
2518	2471	}
2519	2472
...	...	@@ -2538,9 +2491,8 @@ _mm256_unpackhi_ps(__m256 __a, __m256 __b)
2538	2491	/// Bits [159:128] are written to bits [191:160] of the return value. \n
2539	2492	/// Bits [191:160] are written to bits [255:224] of the return value.
2540	2493	/// \returns A 256-bit vector of [8 x float] containing the interleaved values.
2541		static __inline __m256 __DEFAULT_FN_ATTRS
2542		_mm256_unpacklo_ps(__m256 __a, __m256 __b)
2543		{
	2494	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	2495	_mm256_unpacklo_ps(__m256 __a, __m256 __b) {
2544	2496	return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1);
2545	2497	}
2546	2498
...	...	@@ -2568,9 +2520,8 @@ _mm256_unpacklo_ps(__m256 __a, __m256 __b)
2568	2520	/// \param __b
2569	2521	/// A 128-bit vector of [2 x double].
2570	2522	/// \returns the ZF flag in the EFLAGS register.
2571		static __inline int __DEFAULT_FN_ATTRS128
2572		_mm_testz_pd(__m128d __a, __m128d __b)
2573		{
	2523	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_testz_pd(__m128d __a,
	2524	__m128d __b) {
2574	2525	return __builtin_ia32_vtestzpd((__v2df)__a, (__v2df)__b);
2575	2526	}
2576	2527
...	...	@@ -2597,9 +2548,8 @@ _mm_testz_pd(__m128d __a, __m128d __b)
2597	2548	/// \param __b
2598	2549	/// A 128-bit vector of [2 x double].
2599	2550	/// \returns the CF flag in the EFLAGS register.
2600		static __inline int __DEFAULT_FN_ATTRS128
2601		_mm_testc_pd(__m128d __a, __m128d __b)
2602		{
	2551	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_testc_pd(__m128d __a,
	2552	__m128d __b) {
2603	2553	return __builtin_ia32_vtestcpd((__v2df)__a, (__v2df)__b);
2604	2554	}
2605	2555
...	...	@@ -2627,9 +2577,8 @@ _mm_testc_pd(__m128d __a, __m128d __b)
2627	2577	/// \param __b
2628	2578	/// A 128-bit vector of [2 x double].
2629	2579	/// \returns 1 if both the ZF and CF flags are set to 0, otherwise returns 0.
2630		static __inline int __DEFAULT_FN_ATTRS128
2631		_mm_testnzc_pd(__m128d __a, __m128d __b)
2632		{
	2580	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR
	2581	_mm_testnzc_pd(__m128d __a, __m128d __b) {
2633	2582	return __builtin_ia32_vtestnzcpd((__v2df)__a, (__v2df)__b);
2634	2583	}
2635	2584
...	...	@@ -2656,9 +2605,8 @@ _mm_testnzc_pd(__m128d __a, __m128d __b)
2656	2605	/// \param __b
2657	2606	/// A 128-bit vector of [4 x float].
2658	2607	/// \returns the ZF flag.
2659		static __inline int __DEFAULT_FN_ATTRS128
2660		_mm_testz_ps(__m128 __a, __m128 __b)
2661		{
	2608	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_testz_ps(__m128 __a,
	2609	__m128 __b) {
2662	2610	return __builtin_ia32_vtestzps((__v4sf)__a, (__v4sf)__b);
2663	2611	}
2664	2612
...	...	@@ -2685,9 +2633,8 @@ _mm_testz_ps(__m128 __a, __m128 __b)
2685	2633	/// \param __b
2686	2634	/// A 128-bit vector of [4 x float].
2687	2635	/// \returns the CF flag.
2688		static __inline int __DEFAULT_FN_ATTRS128
2689		_mm_testc_ps(__m128 __a, __m128 __b)
2690		{
	2636	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_testc_ps(__m128 __a,
	2637	__m128 __b) {
2691	2638	return __builtin_ia32_vtestcps((__v4sf)__a, (__v4sf)__b);
2692	2639	}
2693	2640
...	...	@@ -2715,9 +2662,8 @@ _mm_testc_ps(__m128 __a, __m128 __b)
2715	2662	/// \param __b
2716	2663	/// A 128-bit vector of [4 x float].
2717	2664	/// \returns 1 if both the ZF and CF flags are set to 0, otherwise returns 0.
2718		static __inline int __DEFAULT_FN_ATTRS128
2719		_mm_testnzc_ps(__m128 __a, __m128 __b)
2720		{
	2665	static __inline int __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_testnzc_ps(__m128 __a,
	2666	__m128 __b) {
2721	2667	return __builtin_ia32_vtestnzcps((__v4sf)__a, (__v4sf)__b);
2722	2668	}
2723	2669
...	...	@@ -2744,9 +2690,8 @@ _mm_testnzc_ps(__m128 __a, __m128 __b)
2744	2690	/// \param __b
2745	2691	/// A 256-bit vector of [4 x double].
2746	2692	/// \returns the ZF flag.
2747		static __inline int __DEFAULT_FN_ATTRS
2748		_mm256_testz_pd(__m256d __a, __m256d __b)
2749		{
	2693	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_testz_pd(__m256d __a,
	2694	__m256d __b) {
2750	2695	return __builtin_ia32_vtestzpd256((__v4df)__a, (__v4df)__b);
2751	2696	}
2752	2697
...	...	@@ -2773,9 +2718,8 @@ _mm256_testz_pd(__m256d __a, __m256d __b)
2773	2718	/// \param __b
2774	2719	/// A 256-bit vector of [4 x double].
2775	2720	/// \returns the CF flag.
2776		static __inline int __DEFAULT_FN_ATTRS
2777		_mm256_testc_pd(__m256d __a, __m256d __b)
2778		{
	2721	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_testc_pd(__m256d __a,
	2722	__m256d __b) {
2779	2723	return __builtin_ia32_vtestcpd256((__v4df)__a, (__v4df)__b);
2780	2724	}
2781	2725
...	...	@@ -2803,9 +2747,8 @@ _mm256_testc_pd(__m256d __a, __m256d __b)
2803	2747	/// \param __b
2804	2748	/// A 256-bit vector of [4 x double].
2805	2749	/// \returns 1 if both the ZF and CF flags are set to 0, otherwise returns 0.
2806		static __inline int __DEFAULT_FN_ATTRS
2807		_mm256_testnzc_pd(__m256d __a, __m256d __b)
2808		{
	2750	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2751	_mm256_testnzc_pd(__m256d __a, __m256d __b) {
2809	2752	return __builtin_ia32_vtestnzcpd256((__v4df)__a, (__v4df)__b);
2810	2753	}
2811	2754
...	...	@@ -2832,9 +2775,8 @@ _mm256_testnzc_pd(__m256d __a, __m256d __b)
2832	2775	/// \param __b
2833	2776	/// A 256-bit vector of [8 x float].
2834	2777	/// \returns the ZF flag.
2835		static __inline int __DEFAULT_FN_ATTRS
2836		_mm256_testz_ps(__m256 __a, __m256 __b)
2837		{
	2778	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_testz_ps(__m256 __a,
	2779	__m256 __b) {
2838	2780	return __builtin_ia32_vtestzps256((__v8sf)__a, (__v8sf)__b);
2839	2781	}
2840	2782
...	...	@@ -2861,9 +2803,8 @@ _mm256_testz_ps(__m256 __a, __m256 __b)
2861	2803	/// \param __b
2862	2804	/// A 256-bit vector of [8 x float].
2863	2805	/// \returns the CF flag.
2864		static __inline int __DEFAULT_FN_ATTRS
2865		_mm256_testc_ps(__m256 __a, __m256 __b)
2866		{
	2806	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_testc_ps(__m256 __a,
	2807	__m256 __b) {
2867	2808	return __builtin_ia32_vtestcps256((__v8sf)__a, (__v8sf)__b);
2868	2809	}
2869	2810
...	...	@@ -2891,9 +2832,8 @@ _mm256_testc_ps(__m256 __a, __m256 __b)
2891	2832	/// \param __b
2892	2833	/// A 256-bit vector of [8 x float].
2893	2834	/// \returns 1 if both the ZF and CF flags are set to 0, otherwise returns 0.
2894		static __inline int __DEFAULT_FN_ATTRS
2895		_mm256_testnzc_ps(__m256 __a, __m256 __b)
2896		{
	2835	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_testnzc_ps(__m256 __a,
	2836	__m256 __b) {
2897	2837	return __builtin_ia32_vtestnzcps256((__v8sf)__a, (__v8sf)__b);
2898	2838	}
2899	2839
...	...	@@ -2917,9 +2857,8 @@ _mm256_testnzc_ps(__m256 __a, __m256 __b)
2917	2857	/// \param __b
2918	2858	/// A 256-bit integer vector.
2919	2859	/// \returns the ZF flag.
2920		static __inline int __DEFAULT_FN_ATTRS
2921		_mm256_testz_si256(__m256i __a, __m256i __b)
2922		{
	2860	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2861	_mm256_testz_si256(__m256i __a, __m256i __b) {
2923	2862	return __builtin_ia32_ptestz256((__v4di)__a, (__v4di)__b);
2924	2863	}
2925	2864
...	...	@@ -2943,9 +2882,8 @@ _mm256_testz_si256(__m256i __a, __m256i __b)
2943	2882	/// \param __b
2944	2883	/// A 256-bit integer vector.
2945	2884	/// \returns the CF flag.
2946		static __inline int __DEFAULT_FN_ATTRS
2947		_mm256_testc_si256(__m256i __a, __m256i __b)
2948		{
	2885	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2886	_mm256_testc_si256(__m256i __a, __m256i __b) {
2949	2887	return __builtin_ia32_ptestc256((__v4di)__a, (__v4di)__b);
2950	2888	}
2951	2889
...	...	@@ -2970,9 +2908,8 @@ _mm256_testc_si256(__m256i __a, __m256i __b)
2970	2908	/// \param __b
2971	2909	/// A 256-bit integer vector.
2972	2910	/// \returns 1 if both the ZF and CF flags are set to 0, otherwise returns 0.
2973		static __inline int __DEFAULT_FN_ATTRS
2974		_mm256_testnzc_si256(__m256i __a, __m256i __b)
2975		{
	2911	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2912	_mm256_testnzc_si256(__m256i __a, __m256i __b) {
2976	2913	return __builtin_ia32_ptestnzc256((__v4di)__a, (__v4di)__b);
2977	2914	}
2978	2915
...	...	@@ -2989,9 +2926,8 @@ _mm256_testnzc_si256(__m256i __a, __m256i __b)
2989	2926	/// A 256-bit vector of [4 x double] containing the double-precision
2990	2927	/// floating point values with sign bits to be extracted.
2991	2928	/// \returns The sign bits from the operand, written to bits [3:0].
2992		static __inline int __DEFAULT_FN_ATTRS
2993		_mm256_movemask_pd(__m256d __a)
2994		{
	2929	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2930	_mm256_movemask_pd(__m256d __a) {
2995	2931	return __builtin_ia32_movmskpd256((__v4df)__a);
2996	2932	}
2997	2933
...	...	@@ -3007,9 +2943,8 @@ _mm256_movemask_pd(__m256d __a)
3007	2943	/// A 256-bit vector of [8 x float] containing the single-precision floating
3008	2944	/// point values with sign bits to be extracted.
3009	2945	/// \returns The sign bits from the operand, written to bits [7:0].
3010		static __inline int __DEFAULT_FN_ATTRS
3011		_mm256_movemask_ps(__m256 __a)
3012		{
	2946	static __inline int __DEFAULT_FN_ATTRS_CONSTEXPR
	2947	_mm256_movemask_ps(__m256 __a) {
3013	2948	return __builtin_ia32_movmskps256((__v8sf)__a);
3014	2949	}
3015	2950
...	...	@@ -3666,9 +3601,7 @@ _mm256_undefined_pd(void)
3666	3601	/// This intrinsic has no corresponding instruction.
3667	3602	///
3668	3603	/// \returns A 256-bit vector of [8 x float] containing undefined values.
3669		static __inline__ __m256 __DEFAULT_FN_ATTRS
3670		_mm256_undefined_ps(void)
3671		{
	3604	static __inline__ __m256 __DEFAULT_FN_ATTRS _mm256_undefined_ps(void) {
3672	3605	return (__m256)__builtin_ia32_undef256();
3673	3606	}
3674	3607
...	...	@@ -3777,7 +3710,7 @@ _mm256_set_ps(float __a, float __b, float __c, float __d,
3777	3710	/// \param __i7
3778	3711	/// A 32-bit integral value used to initialize bits [31:0] of the result.
3779	3712	/// \returns An initialized 256-bit integer vector.
3780		static __inline __m256i __DEFAULT_FN_ATTRS
	3713	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
3781	3714	_mm256_set_epi32(int __i0, int __i1, int __i2, int __i3,
3782	3715	int __i4, int __i5, int __i6, int __i7)
3783	3716	{
...	...	@@ -3825,7 +3758,7 @@ _mm256_set_epi32(int __i0, int __i1, int __i2, int __i3,
3825	3758	/// \param __w00
3826	3759	/// A 16-bit integral value used to initialize bits [15:0] of the result.
3827	3760	/// \returns An initialized 256-bit integer vector.
3828		static __inline __m256i __DEFAULT_FN_ATTRS
	3761	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
3829	3762	_mm256_set_epi16(short __w15, short __w14, short __w13, short __w12,
3830	3763	short __w11, short __w10, short __w09, short __w08,
3831	3764	short __w07, short __w06, short __w05, short __w04,
...	...	@@ -3908,7 +3841,7 @@ _mm256_set_epi16(short __w15, short __w14, short __w13, short __w12,
3908	3841	/// \param __b00
3909	3842	/// An 8-bit integral value used to initialize bits [7:0] of the result.
3910	3843	/// \returns An initialized 256-bit integer vector.
3911		static __inline __m256i __DEFAULT_FN_ATTRS
	3844	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
3912	3845	_mm256_set_epi8(char __b31, char __b30, char __b29, char __b28,
3913	3846	char __b27, char __b26, char __b25, char __b24,
3914	3847	char __b23, char __b22, char __b21, char __b20,
...	...	@@ -3943,7 +3876,7 @@ _mm256_set_epi8(char __b31, char __b30, char __b29, char __b28,
3943	3876	/// \param __d
3944	3877	/// A 64-bit integral value used to initialize bits [63:0] of the result.
3945	3878	/// \returns An initialized 256-bit integer vector.
3946		static __inline __m256i __DEFAULT_FN_ATTRS
	3879	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
3947	3880	_mm256_set_epi64x(long long __a, long long __b, long long __c, long long __d)
3948	3881	{
3949	3882	return __extension__ (__m256i)(__v4di){ __d, __c, __b, __a };
...	...	@@ -4044,7 +3977,7 @@ _mm256_setr_ps(float __a, float __b, float __c, float __d,
4044	3977	/// \param __i7
4045	3978	/// A 32-bit integral value used to initialize bits [255:224] of the result.
4046	3979	/// \returns An initialized 256-bit integer vector.
4047		static __inline __m256i __DEFAULT_FN_ATTRS
	3980	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4048	3981	_mm256_setr_epi32(int __i0, int __i1, int __i2, int __i3,
4049	3982	int __i4, int __i5, int __i6, int __i7)
4050	3983	{
...	...	@@ -4092,7 +4025,7 @@ _mm256_setr_epi32(int __i0, int __i1, int __i2, int __i3,
4092	4025	/// \param __w00
4093	4026	/// A 16-bit integral value used to initialize bits [255:240] of the result.
4094	4027	/// \returns An initialized 256-bit integer vector.
4095		static __inline __m256i __DEFAULT_FN_ATTRS
	4028	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4096	4029	_mm256_setr_epi16(short __w15, short __w14, short __w13, short __w12,
4097	4030	short __w11, short __w10, short __w09, short __w08,
4098	4031	short __w07, short __w06, short __w05, short __w04,
...	...	@@ -4177,7 +4110,7 @@ _mm256_setr_epi16(short __w15, short __w14, short __w13, short __w12,
4177	4110	/// \param __b00
4178	4111	/// An 8-bit integral value used to initialize bits [255:248] of the result.
4179	4112	/// \returns An initialized 256-bit integer vector.
4180		static __inline __m256i __DEFAULT_FN_ATTRS
	4113	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4181	4114	_mm256_setr_epi8(char __b31, char __b30, char __b29, char __b28,
4182	4115	char __b27, char __b26, char __b25, char __b24,
4183	4116	char __b23, char __b22, char __b21, char __b20,
...	...	@@ -4210,7 +4143,7 @@ _mm256_setr_epi8(char __b31, char __b30, char __b29, char __b28,
4210	4143	/// \param __d
4211	4144	/// A 64-bit integral value used to initialize bits [255:192] of the result.
4212	4145	/// \returns An initialized 256-bit integer vector.
4213		static __inline __m256i __DEFAULT_FN_ATTRS
	4146	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4214	4147	_mm256_setr_epi64x(long long __a, long long __b, long long __c, long long __d)
4215	4148	{
4216	4149	return _mm256_set_epi64x(__d, __c, __b, __a);
...	...	@@ -4267,7 +4200,7 @@ _mm256_set1_ps(float __w)
4267	4200	/// A 32-bit integral value used to initialize each vector element of the
4268	4201	/// result.
4269	4202	/// \returns An initialized 256-bit integer vector of [8 x i32].
4270		static __inline __m256i __DEFAULT_FN_ATTRS
	4203	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4271	4204	_mm256_set1_epi32(int __i)
4272	4205	{
4273	4206	return _mm256_set_epi32(__i, __i, __i, __i, __i, __i, __i, __i);
...	...	@@ -4285,7 +4218,7 @@ _mm256_set1_epi32(int __i)
4285	4218	/// A 16-bit integral value used to initialize each vector element of the
4286	4219	/// result.
4287	4220	/// \returns An initialized 256-bit integer vector of [16 x i16].
4288		static __inline __m256i __DEFAULT_FN_ATTRS
	4221	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4289	4222	_mm256_set1_epi16(short __w)
4290	4223	{
4291	4224	return _mm256_set_epi16(__w, __w, __w, __w, __w, __w, __w, __w,
...	...	@@ -4303,7 +4236,7 @@ _mm256_set1_epi16(short __w)
4303	4236	/// An 8-bit integral value used to initialize each vector element of the
4304	4237	/// result.
4305	4238	/// \returns An initialized 256-bit integer vector of [32 x i8].
4306		static __inline __m256i __DEFAULT_FN_ATTRS
	4239	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4307	4240	_mm256_set1_epi8(char __b)
4308	4241	{
4309	4242	return _mm256_set_epi8(__b, __b, __b, __b, __b, __b, __b, __b,
...	...	@@ -4324,7 +4257,7 @@ _mm256_set1_epi8(char __b)
4324	4257	/// A 64-bit integral value used to initialize each vector element of the
4325	4258	/// result.
4326	4259	/// \returns An initialized 256-bit integer vector of [4 x i64].
4327		static __inline __m256i __DEFAULT_FN_ATTRS
	4260	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4328	4261	_mm256_set1_epi64x(long long __q)
4329	4262	{
4330	4263	return _mm256_set_epi64x(__q, __q, __q, __q);
...	...	@@ -4379,7 +4312,7 @@ _mm256_setzero_si256(void) {
4379	4312	/// A 256-bit floating-point vector of [4 x double].
4380	4313	/// \returns A 256-bit floating-point vector of [8 x float] containing the same
4381	4314	/// bitwise pattern as the parameter.
4382		static __inline __m256 __DEFAULT_FN_ATTRS
	4315	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
4383	4316	_mm256_castpd_ps(__m256d __a)
4384	4317	{
4385	4318	return (__m256)__a;
...	...	@@ -4396,7 +4329,7 @@ _mm256_castpd_ps(__m256d __a)
4396	4329	/// A 256-bit floating-point vector of [4 x double].
4397	4330	/// \returns A 256-bit integer vector containing the same bitwise pattern as the
4398	4331	/// parameter.
4399		static __inline __m256i __DEFAULT_FN_ATTRS
	4332	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4400	4333	_mm256_castpd_si256(__m256d __a)
4401	4334	{
4402	4335	return (__m256i)__a;
...	...	@@ -4413,7 +4346,7 @@ _mm256_castpd_si256(__m256d __a)
4413	4346	/// A 256-bit floating-point vector of [8 x float].
4414	4347	/// \returns A 256-bit floating-point vector of [4 x double] containing the same
4415	4348	/// bitwise pattern as the parameter.
4416		static __inline __m256d __DEFAULT_FN_ATTRS
	4349	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
4417	4350	_mm256_castps_pd(__m256 __a)
4418	4351	{
4419	4352	return (__m256d)__a;
...	...	@@ -4430,7 +4363,7 @@ _mm256_castps_pd(__m256 __a)
4430	4363	/// A 256-bit floating-point vector of [8 x float].
4431	4364	/// \returns A 256-bit integer vector containing the same bitwise pattern as the
4432	4365	/// parameter.
4433		static __inline __m256i __DEFAULT_FN_ATTRS
	4366	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
4434	4367	_mm256_castps_si256(__m256 __a)
4435	4368	{
4436	4369	return (__m256i)__a;
...	...	@@ -4447,7 +4380,7 @@ _mm256_castps_si256(__m256 __a)
4447	4380	/// A 256-bit integer vector.
4448	4381	/// \returns A 256-bit floating-point vector of [8 x float] containing the same
4449	4382	/// bitwise pattern as the parameter.
4450		static __inline __m256 __DEFAULT_FN_ATTRS
	4383	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
4451	4384	_mm256_castsi256_ps(__m256i __a)
4452	4385	{
4453	4386	return (__m256)__a;
...	...	@@ -4464,7 +4397,7 @@ _mm256_castsi256_ps(__m256i __a)
4464	4397	/// A 256-bit integer vector.
4465	4398	/// \returns A 256-bit floating-point vector of [4 x double] containing the same
4466	4399	/// bitwise pattern as the parameter.
4467		static __inline __m256d __DEFAULT_FN_ATTRS
	4400	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
4468	4401	_mm256_castsi256_pd(__m256i __a)
4469	4402	{
4470	4403	return (__m256d)__a;
...	...	@@ -4481,7 +4414,7 @@ _mm256_castsi256_pd(__m256i __a)
4481	4414	/// A 256-bit floating-point vector of [4 x double].
4482	4415	/// \returns A 128-bit floating-point vector of [2 x double] containing the
4483	4416	/// lower 128 bits of the parameter.
4484		static __inline __m128d __DEFAULT_FN_ATTRS
	4417	static __inline __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
4485	4418	_mm256_castpd256_pd128(__m256d __a)
4486	4419	{
4487	4420	return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 1);
...	...	@@ -4498,7 +4431,7 @@ _mm256_castpd256_pd128(__m256d __a)
4498	4431	/// A 256-bit floating-point vector of [8 x float].
4499	4432	/// \returns A 128-bit floating-point vector of [4 x float] containing the
4500	4433	/// lower 128 bits of the parameter.
4501		static __inline __m128 __DEFAULT_FN_ATTRS
	4434	static __inline __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
4502	4435	_mm256_castps256_ps128(__m256 __a)
4503	4436	{
4504	4437	return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 1, 2, 3);
...	...	@@ -4514,7 +4447,7 @@ _mm256_castps256_ps128(__m256 __a)
4514	4447	/// A 256-bit integer vector.
4515	4448	/// \returns A 128-bit integer vector containing the lower 128 bits of the
4516	4449	/// parameter.
4517		static __inline __m128i __DEFAULT_FN_ATTRS
	4450	static __inline __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
4518	4451	_mm256_castsi256_si128(__m256i __a)
4519	4452	{
4520	4453	return __builtin_shufflevector((__v4di)__a, (__v4di)__a, 0, 1);
...	...	@@ -4598,9 +4531,8 @@ _mm256_castsi128_si256(__m128i __a)
4598	4531	/// A 128-bit vector of [2 x double].
4599	4532	/// \returns A 256-bit floating-point vector of [4 x double]. The lower 128 bits
4600	4533	/// contain the value of the parameter. The upper 128 bits are set to zero.
4601		static __inline __m256d __DEFAULT_FN_ATTRS
4602		_mm256_zextpd128_pd256(__m128d __a)
4603		{
	4534	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	4535	_mm256_zextpd128_pd256(__m128d __a) {
4604	4536	return __builtin_shufflevector((__v2df)__a, (__v2df)_mm_setzero_pd(), 0, 1, 2, 3);
4605	4537	}
4606	4538
...	...	@@ -4616,9 +4548,8 @@ _mm256_zextpd128_pd256(__m128d __a)
4616	4548	/// A 128-bit vector of [4 x float].
4617	4549	/// \returns A 256-bit floating-point vector of [8 x float]. The lower 128 bits
4618	4550	/// contain the value of the parameter. The upper 128 bits are set to zero.
4619		static __inline __m256 __DEFAULT_FN_ATTRS
4620		_mm256_zextps128_ps256(__m128 __a)
4621		{
	4551	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	4552	_mm256_zextps128_ps256(__m128 __a) {
4622	4553	return __builtin_shufflevector((__v4sf)__a, (__v4sf)_mm_setzero_ps(), 0, 1, 2, 3, 4, 5, 6, 7);
4623	4554	}
4624	4555
...	...	@@ -4634,9 +4565,8 @@ _mm256_zextps128_ps256(__m128 __a)
4634	4565	/// A 128-bit integer vector.
4635	4566	/// \returns A 256-bit integer vector. The lower 128 bits contain the value of
4636	4567	/// the parameter. The upper 128 bits are set to zero.
4637		static __inline __m256i __DEFAULT_FN_ATTRS
4638		_mm256_zextsi128_si256(__m128i __a)
4639		{
	4568	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
	4569	_mm256_zextsi128_si256(__m128i __a) {
4640	4570	return __builtin_shufflevector((__v2di)__a, (__v2di)_mm_setzero_si128(), 0, 1, 2, 3);
4641	4571	}
4642	4572
...	...	@@ -4851,9 +4781,8 @@ _mm256_zextsi128_si256(__m128i __a)
4851	4781	/// 128 bits of the result.
4852	4782	/// \returns A 256-bit floating-point vector of [8 x float] containing the
4853	4783	/// concatenated result.
4854		static __inline __m256 __DEFAULT_FN_ATTRS
4855		_mm256_set_m128 (__m128 __hi, __m128 __lo)
4856		{
	4784	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	4785	_mm256_set_m128(__m128 __hi, __m128 __lo) {
4857	4786	return (__m256) __builtin_shufflevector((__v4sf)__lo, (__v4sf)__hi, 0, 1, 2, 3, 4, 5, 6, 7);
4858	4787	}
4859	4788
...	...	@@ -4872,9 +4801,8 @@ _mm256_set_m128 (__m128 __hi, __m128 __lo)
4872	4801	/// 128 bits of the result.
4873	4802	/// \returns A 256-bit floating-point vector of [4 x double] containing the
4874	4803	/// concatenated result.
4875		static __inline __m256d __DEFAULT_FN_ATTRS
4876		_mm256_set_m128d (__m128d __hi, __m128d __lo)
4877		{
	4804	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	4805	_mm256_set_m128d(__m128d __hi, __m128d __lo) {
4878	4806	return (__m256d) __builtin_shufflevector((__v2df)__lo, (__v2df)__hi, 0, 1, 2, 3);
4879	4807	}
4880	4808
...	...	@@ -4892,9 +4820,8 @@ _mm256_set_m128d (__m128d __hi, __m128d __lo)
4892	4820	/// A 128-bit integer vector to be copied to the lower 128 bits of the
4893	4821	/// result.
4894	4822	/// \returns A 256-bit integer vector containing the concatenated result.
4895		static __inline __m256i __DEFAULT_FN_ATTRS
4896		_mm256_set_m128i (__m128i __hi, __m128i __lo)
4897		{
	4823	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
	4824	_mm256_set_m128i(__m128i __hi, __m128i __lo) {
4898	4825	return (__m256i) __builtin_shufflevector((__v2di)__lo, (__v2di)__hi, 0, 1, 2, 3);
4899	4826	}
4900	4827
...	...	@@ -4915,9 +4842,8 @@ _mm256_set_m128i (__m128i __hi, __m128i __lo)
4915	4842	/// 128 bits of the result.
4916	4843	/// \returns A 256-bit floating-point vector of [8 x float] containing the
4917	4844	/// concatenated result.
4918		static __inline __m256 __DEFAULT_FN_ATTRS
4919		_mm256_setr_m128 (__m128 __lo, __m128 __hi)
4920		{
	4845	static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
	4846	_mm256_setr_m128(__m128 __lo, __m128 __hi) {
4921	4847	return _mm256_set_m128(__hi, __lo);
4922	4848	}
4923	4849
...	...	@@ -4938,9 +4864,8 @@ _mm256_setr_m128 (__m128 __lo, __m128 __hi)
4938	4864	/// 128 bits of the result.
4939	4865	/// \returns A 256-bit floating-point vector of [4 x double] containing the
4940	4866	/// concatenated result.
4941		static __inline __m256d __DEFAULT_FN_ATTRS
4942		_mm256_setr_m128d (__m128d __lo, __m128d __hi)
4943		{
	4867	static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
	4868	_mm256_setr_m128d(__m128d __lo, __m128d __hi) {
4944	4869	return (__m256d)_mm256_set_m128d(__hi, __lo);
4945	4870	}
4946	4871
...	...	@@ -4959,9 +4884,8 @@ _mm256_setr_m128d (__m128d __lo, __m128d __hi)
4959	4884	/// A 128-bit integer vector to be copied to the upper 128 bits of the
4960	4885	/// result.
4961	4886	/// \returns A 256-bit integer vector containing the concatenated result.
4962		static __inline __m256i __DEFAULT_FN_ATTRS
4963		_mm256_setr_m128i (__m128i __lo, __m128i __hi)
4964		{
	4887	static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
	4888	_mm256_setr_m128i(__m128i __lo, __m128i __hi) {
4965	4889	return (__m256i)_mm256_set_m128i(__hi, __lo);
4966	4890	}
4967	4891

lib/include/avxvnniint16intrin.h+118-99

...	...	@@ -16,9 +16,10 @@
16	16	#define __AVXVNNIINT16INTRIN_H
17	17
18	18	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
19		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
20		/// signed 16-bit results. Sum these 2 results with the corresponding
21		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	19	/// corresponding unsigned 16-bit integers in \a __B, producing 2
	20	/// intermediate signed 16-bit results. Sum these 2 results with the
	21	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	22	/// results in \a dst.
22	23	///
23	24	/// \headerfile <immintrin.h>
24	25	///
...	...	@@ -40,19 +41,21 @@
40	41	/// \code{.operation}
41	42	/// FOR j := 0 to 3
42	43	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
43		/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	44	/// tmp2.dword :=
	45	/// SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
44	46	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
45	47	/// ENDFOR
46	48	/// dst[MAX:128] := 0
47	49	/// \endcode
48	50	#define _mm_dpwsud_epi32(__W, __A, __B) \
49		((__m128i)__builtin_ia32_vpdpwsud128((__v4si)(__W), (__v4si)(__A), \
50		(__v4si)(__B)))
	51	((__m128i)__builtin_ia32_vpdpwsud128((__v4si)(__W), (__v8hi)(__A), \
	52	(__v8hu)(__B)))
51	53
52	54	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
53		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
54		/// signed 16-bit results. Sum these 2 results with the corresponding
55		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	55	/// corresponding unsigned 16-bit integers in \a __B, producing 2
	56	/// intermediate signed 16-bit results. Sum these 2 results with the
	57	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	58	/// results in \a dst.
56	59	///
57	60	/// \headerfile <immintrin.h>
58	61	///
...	...	@@ -74,20 +77,21 @@
74	77	/// \code{.operation}
75	78	/// FOR j := 0 to 7
76	79	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
77		/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
78		/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
	80	/// tmp2.dword :=
	81	/// SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	82	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
79	83	/// ENDFOR
80	84	/// dst[MAX:256] := 0
81	85	/// \endcode
82	86	#define _mm256_dpwsud_epi32(__W, __A, __B) \
83		((__m256i)__builtin_ia32_vpdpwsud256((__v8si)(__W), (__v8si)(__A), \
84		(__v8si)(__B)))
	87	((__m256i)__builtin_ia32_vpdpwsud256((__v8si)(__W), (__v16hi)(__A), \
	88	(__v16hu)(__B)))
85	89
86	90	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
87		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
88		/// signed 16-bit results. Sum these 2 results with the corresponding
89		/// 32-bit integer in \a __W with signed saturation, and store the packed
90		/// 32-bit results in \a dst.
	91	/// corresponding unsigned 16-bit integers in \a __B, producing 2
	92	/// intermediate signed 16-bit results. Sum these 2 results with the
	93	/// corresponding 32-bit integer in \a __W with signed saturation, and store
	94	/// the packed 32-bit results in \a dst.
91	95	///
92	96	/// \headerfile <immintrin.h>
93	97	///
...	...	@@ -109,20 +113,22 @@
109	113	/// \code{.operation}
110	114	/// FOR j := 0 to 3
111	115	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
112		/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	116	/// tmp2.dword :=
	117	/// SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
113	118	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
114	119	/// ENDFOR
115	120	/// dst[MAX:128] := 0
116	121	/// \endcode
	122	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
117	123	#define _mm_dpwsuds_epi32(__W, __A, __B) \
118		((__m128i)__builtin_ia32_vpdpwsuds128((__v4si)(__W), (__v4si)(__A), \
119		(__v4si)(__B)))
	124	((__m128i)__builtin_ia32_vpdpwsuds128((__v4si)(__W), (__v8hi)(__A), \
	125	(__v8hu)(__B)))
120	126
121	127	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
122		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
123		/// signed 16-bit results. Sum these 2 results with the corresponding
124		/// 32-bit integer in \a __W with signed saturation, and store the packed
125		/// 32-bit results in \a dst.
	128	/// corresponding unsigned 16-bit integers in \a __B, producing 2
	129	/// intermediate signed 16-bit results. Sum these 2 results with the
	130	/// corresponding 32-bit integer in \a __W with signed saturation, and store
	131	/// the packed 32-bit results in \a dst.
126	132	///
127	133	/// \headerfile <immintrin.h>
128	134	///
...	...	@@ -144,19 +150,21 @@
144	150	/// \code{.operation}
145	151	/// FOR j := 0 to 7
146	152	/// tmp1.dword := SignExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
147		/// tmp2.dword := SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	153	/// tmp2.dword :=
	154	/// SignExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
148	155	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
149	156	/// ENDFOR
150	157	/// dst[MAX:256] := 0
151	158	/// \endcode
152	159	#define _mm256_dpwsuds_epi32(__W, __A, __B) \
153		((__m256i)__builtin_ia32_vpdpwsuds256((__v8si)(__W), (__v8si)(__A), \
154		(__v8si)(__B)))
	160	((__m256i)__builtin_ia32_vpdpwsuds256((__v8si)(__W), (__v16hi)(__A), \
	161	(__v16hu)(__B)))
155	162
156		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
157		/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
158		/// signed 16-bit results. Sum these 2 results with the corresponding
159		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	163	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	164	/// with corresponding signed 16-bit integers in \a __B, producing 2
	165	/// intermediate signed 16-bit results. Sum these 2 results with the
	166	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	167	/// results in \a dst.
160	168	///
161	169	/// \headerfile <immintrin.h>
162	170	///
...	...	@@ -178,19 +186,21 @@
178	186	/// \code{.operation}
179	187	/// FOR j := 0 to 3
180	188	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
181		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	189	/// tmp2.dword :=
	190	/// ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
182	191	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
183	192	/// ENDFOR
184	193	/// dst[MAX:128] := 0
185	194	/// \endcode
186	195	#define _mm_dpwusd_epi32(__W, __A, __B) \
187		((__m128i)__builtin_ia32_vpdpwusd128((__v4si)(__W), (__v4si)(__A), \
188		(__v4si)(__B)))
	196	((__m128i)__builtin_ia32_vpdpwusd128((__v4si)(__W), (__v8hu)(__A), \
	197	(__v8hi)(__B)))
189	198
190		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
191		/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
192		/// signed 16-bit results. Sum these 2 results with the corresponding
193		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	199	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	200	/// with corresponding signed 16-bit integers in \a __B, producing 2
	201	/// intermediate signed 16-bit results. Sum these 2 results with the
	202	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	203	/// results in \a dst.
194	204	///
195	205	/// \headerfile <immintrin.h>
196	206	///
...	...	@@ -212,20 +222,21 @@
212	222	/// \code{.operation}
213	223	/// FOR j := 0 to 7
214	224	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
215		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	225	/// tmp2.dword :=
	226	/// ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
216	227	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
217	228	/// ENDFOR
218	229	/// dst[MAX:256] := 0
219	230	/// \endcode
220	231	#define _mm256_dpwusd_epi32(__W, __A, __B) \
221		((__m256i)__builtin_ia32_vpdpwusd256((__v8si)(__W), (__v8si)(__A), \
222		(__v8si)(__B)))
	232	((__m256i)__builtin_ia32_vpdpwusd256((__v8si)(__W), (__v16hu)(__A), \
	233	(__v16hi)(__B)))
223	234
224		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
225		/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
226		/// signed 16-bit results. Sum these 2 results with the corresponding
227		/// 32-bit integer in \a __W with signed saturation, and store the packed
228		/// 32-bit results in \a dst.
	235	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	236	/// with corresponding signed 16-bit integers in \a __B, producing 2
	237	/// intermediate signed 16-bit results. Sum these 2 results with the
	238	/// corresponding 32-bit integer in \a __W with signed saturation, and
	239	/// store the packed 32-bit results in \a dst.
229	240	///
230	241	/// \headerfile <immintrin.h>
231	242	///
...	...	@@ -233,7 +244,7 @@
233	244	/// __m128i _mm_dpwusds_epi32(__m128i __W, __m128i __A, __m128i __B)
234	245	/// \endcode
235	246	///
236		/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	247	/// This intrinsic corresponds to the \c VPDPWUSDS instruction.
237	248	///
238	249	/// \param __W
239	250	/// A 128-bit vector of [4 x int].
...	...	@@ -247,20 +258,21 @@
247	258	/// \code{.operation}
248	259	/// FOR j := 0 to 3
249	260	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
250		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	261	/// tmp2.dword :=
	262	/// ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
251	263	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
252	264	/// ENDFOR
253	265	/// dst[MAX:128] := 0
254	266	/// \endcode
255	267	#define _mm_dpwusds_epi32(__W, __A, __B) \
256		((__m128i)__builtin_ia32_vpdpwusds128((__v4si)(__W), (__v4si)(__A), \
257		(__v4si)(__B)))
	268	((__m128i)__builtin_ia32_vpdpwusds128((__v4si)(__W), (__v8hu)(__A), \
	269	(__v8hi)(__B)))
258	270
259		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
260		/// corresponding signed 16-bit integers in \a __B, producing 2 intermediate
261		/// signed 16-bit results. Sum these 2 results with the corresponding
262		/// 32-bit integer in \a __W with signed saturation, and store the packed
263		/// 32-bit results in \a dst.
	271	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	272	/// with corresponding signed 16-bit integers in \a __B, producing 2
	273	/// intermediate signed 16-bit results. Sum these 2 results with the
	274	/// corresponding 32-bit integer in \a __W with signed saturation, and
	275	/// store the packed 32-bit results in \a dst.
264	276	///
265	277	/// \headerfile <immintrin.h>
266	278	///
...	...	@@ -268,7 +280,7 @@
268	280	/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)
269	281	/// \endcode
270	282	///
271		/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	283	/// This intrinsic corresponds to the \c VPDPWUSDS instruction.
272	284	///
273	285	/// \param __W
274	286	/// A 256-bit vector of [8 x int].
...	...	@@ -282,19 +294,21 @@
282	294	/// \code{.operation}
283	295	/// FOR j := 0 to 7
284	296	/// tmp1.dword := ZeroExtend32(__A.word[2j]) SignExtend32(__B.word[2*j])
285		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
	297	/// tmp2.dword :=
	298	/// ZeroExtend32(__A.word[2j+1]) SignExtend32(__B.word[2*j+1])
286	299	/// dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
287	300	/// ENDFOR
288	301	/// dst[MAX:256] := 0
289	302	/// \endcode
290	303	#define _mm256_dpwusds_epi32(__W, __A, __B) \
291		((__m256i)__builtin_ia32_vpdpwusds256((__v8si)(__W), (__v8si)(__A), \
292		(__v8si)(__B)))
	304	((__m256i)__builtin_ia32_vpdpwusds256((__v8si)(__W), (__v16hu)(__A), \
	305	(__v16hi)(__B)))
293	306
294		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
295		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
296		/// signed 16-bit results. Sum these 2 results with the corresponding
297		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	307	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	308	/// with corresponding unsigned 16-bit integers in \a __B, producing 2
	309	/// intermediate signed 16-bit results. Sum these 2 results with the
	310	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	311	/// results in \a dst.
298	312	///
299	313	/// \headerfile <immintrin.h>
300	314	///
...	...	@@ -305,30 +319,32 @@
305	319	/// This intrinsic corresponds to the \c VPDPWUUD instruction.
306	320	///
307	321	/// \param __W
308		/// A 128-bit vector of [4 x unsigned int].
	322	/// A 128-bit vector of [4 x int].
309	323	/// \param __A
310	324	/// A 128-bit vector of [8 x unsigned short].
311	325	/// \param __B
312	326	/// A 128-bit vector of [8 x unsigned short].
313	327	/// \returns
314		/// A 128-bit vector of [4 x unsigned int].
	328	/// A 128-bit vector of [4 x int].
315	329	///
316	330	/// \code{.operation}
317	331	/// FOR j := 0 to 3
318	332	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
319		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	333	/// tmp2.dword :=
	334	/// ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
320	335	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
321	336	/// ENDFOR
322	337	/// dst[MAX:128] := 0
323	338	/// \endcode
324	339	#define _mm_dpwuud_epi32(__W, __A, __B) \
325		((__m128i)__builtin_ia32_vpdpwuud128((__v4si)(__W), (__v4si)(__A), \
326		(__v4si)(__B)))
	340	((__m128i)__builtin_ia32_vpdpwuud128((__v4si)(__W), (__v8hu)(__A), \
	341	(__v8hu)(__B)))
327	342
328		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
329		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
330		/// signed 16-bit results. Sum these 2 results with the corresponding
331		/// 32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
	343	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	344	/// with corresponding unsigned 16-bit integers in \a __B, producing 2
	345	/// intermediate signed 16-bit results. Sum these 2 results with the
	346	/// corresponding 32-bit integer in \a __W, and store the packed 32-bit
	347	/// results in \a dst.
332	348	///
333	349	/// \headerfile <immintrin.h>
334	350	///
...	...	@@ -339,31 +355,32 @@
339	355	/// This intrinsic corresponds to the \c VPDPWUUD instruction.
340	356	///
341	357	/// \param __W
342		/// A 256-bit vector of [8 x unsigned int].
	358	/// A 256-bit vector of [8 x int].
343	359	/// \param __A
344	360	/// A 256-bit vector of [16 x unsigned short].
345	361	/// \param __B
346	362	/// A 256-bit vector of [16 x unsigned short].
347	363	/// \returns
348		/// A 256-bit vector of [8 x unsigned int].
	364	/// A 256-bit vector of [8 x int].
349	365	///
350	366	/// \code{.operation}
351	367	/// FOR j := 0 to 7
352	368	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
353		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	369	/// tmp2.dword :=
	370	/// ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
354	371	/// dst.dword[j] := __W.dword[j] + tmp1 + tmp2
355	372	/// ENDFOR
356	373	/// dst[MAX:256] := 0
357	374	/// \endcode
358	375	#define _mm256_dpwuud_epi32(__W, __A, __B) \
359		((__m256i)__builtin_ia32_vpdpwuud256((__v8si)(__W), (__v8si)(__A), \
360		(__v8si)(__B)))
	376	((__m256i)__builtin_ia32_vpdpwuud256((__v8si)(__W), (__v16hu)(__A), \
	377	(__v16hu)(__B)))
361	378
362		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
363		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
364		/// signed 16-bit results. Sum these 2 results with the corresponding
365		/// 32-bit integer in \a __W with signed saturation, and store the packed
366		/// 32-bit results in \a dst.
	379	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	380	/// with corresponding unsigned 16-bit integers in \a __B, producing 2
	381	/// intermediate signed 16-bit results. Sum these 2 results with the
	382	/// corresponding 32-bit integer in \a __W with signed saturation, and store
	383	/// the packed 32-bit results in \a dst.
367	384	///
368	385	/// \headerfile <immintrin.h>
369	386	///
...	...	@@ -371,34 +388,35 @@
371	388	/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)
372	389	/// \endcode
373	390	///
374		/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	391	/// This intrinsic corresponds to the \c VPDPWUUDS instruction.
375	392	///
376	393	/// \param __W
377		/// A 128-bit vector of [4 x unsigned int].
	394	/// A 128-bit vector of [4 x int].
378	395	/// \param __A
379	396	/// A 128-bit vector of [8 x unsigned short].
380	397	/// \param __B
381	398	/// A 128-bit vector of [8 x unsigned short].
382	399	/// \returns
383		/// A 128-bit vector of [4 x unsigned int].
	400	/// A 128-bit vector of [4 x int].
384	401	///
385	402	/// \code{.operation}
386	403	/// FOR j := 0 to 3
387	404	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
388		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	405	/// tmp2.dword :=
	406	/// ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
389	407	/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
390	408	/// ENDFOR
391	409	/// dst[MAX:128] := 0
392	410	/// \endcode
393	411	#define _mm_dpwuuds_epi32(__W, __A, __B) \
394		((__m128i)__builtin_ia32_vpdpwuuds128((__v4si)(__W), (__v4si)(__A), \
395		(__v4si)(__B)))
	412	((__m128i)__builtin_ia32_vpdpwuuds128((__v4si)(__W), (__v8hu)(__A), \
	413	(__v8hu)(__B)))
396	414
397		/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
398		/// corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
399		/// signed 16-bit results. Sum these 2 results with the corresponding
400		/// 32-bit integer in \a __W with signed saturation, and store the packed
401		/// 32-bit results in \a dst.
	415	/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A
	416	/// with corresponding unsigned 16-bit integers in \a __B, producing 2
	417	/// intermediate signed 16-bit results. Sum these 2 results with the
	418	/// corresponding 32-bit integer in \a __W with signed saturation, and store
	419	/// the packed 32-bit results in \a dst.
402	420	///
403	421	/// \headerfile <immintrin.h>
404	422	///
...	...	@@ -406,27 +424,28 @@
406	424	/// __m256i _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B)
407	425	/// \endcode
408	426	///
409		/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
	427	/// This intrinsic corresponds to the \c VPDPWUUDS instruction.
410	428	///
411	429	/// \param __W
412		/// A 256-bit vector of [8 x unsigned int].
	430	/// A 256-bit vector of [8 x int].
413	431	/// \param __A
414	432	/// A 256-bit vector of [16 x unsigned short].
415	433	/// \param __B
416	434	/// A 256-bit vector of [16 x unsigned short].
417	435	/// \returns
418		/// A 256-bit vector of [8 x unsigned int].
	436	/// A 256-bit vector of [8 x int].
419	437	///
420	438	/// \code{.operation}
421	439	/// FOR j := 0 to 7
422	440	/// tmp1.dword := ZeroExtend32(__A.word[2j]) ZeroExtend32(__B.word[2*j])
423		/// tmp2.dword := ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
	441	/// tmp2.dword :=
	442	/// ZeroExtend32(__A.word[2j+1]) ZeroExtend32(__B.word[2*j+1])
424	443	/// dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
425	444	/// ENDFOR
426	445	/// dst[MAX:256] := 0
427	446	/// \endcode
428	447	#define _mm256_dpwuuds_epi32(__W, __A, __B) \
429		((__m256i)__builtin_ia32_vpdpwuuds256((__v8si)(__W), (__v8si)(__A), \
430		(__v8si)(__B)))
	448	((__m256i)__builtin_ia32_vpdpwuuds256((__v8si)(__W), (__v16hu)(__A), \
	449	(__v16hu)(__B)))
431	450
432	451	#endif // __AVXVNNIINT16INTRIN_H

lib/include/avxvnniint8intrin.h+56-32

...	...	@@ -14,6 +14,7 @@
14	14	#ifndef __AVXVNNIINT8INTRIN_H
15	15	#define __AVXVNNIINT8INTRIN_H
16	16
	17	// clang-format off
17	18	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
18	19	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
19	20	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -44,10 +45,12 @@
44	45	/// ENDFOR
45	46	/// dst[MAX:128] := 0
46	47	/// \endcode
	48	// clang-format on
47	49	#define _mm_dpbssd_epi32(__W, __A, __B) \
48		((__m128i)__builtin_ia32_vpdpbssd128((__v4si)(__W), (__v4si)(__A), \
49		(__v4si)(__B)))
	50	((__m128i)__builtin_ia32_vpdpbssd128((__v4si)(__W), (__v16qi)(__A), \
	51	(__v16qi)(__B)))
50	52
	53	// clang-format off
51	54	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
52	55	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
53	56	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -78,10 +81,12 @@
78	81	/// ENDFOR
79	82	/// dst[MAX:256] := 0
80	83	/// \endcode
	84	// clang-format on
81	85	#define _mm256_dpbssd_epi32(__W, __A, __B) \
82		((__m256i)__builtin_ia32_vpdpbssd256((__v8si)(__W), (__v8si)(__A), \
83		(__v8si)(__B)))
	86	((__m256i)__builtin_ia32_vpdpbssd256((__v8si)(__W), (__v32qi)(__A), \
	87	(__v32qi)(__B)))
84	88
	89	// clang-format off
85	90	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
86	91	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
87	92	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -94,7 +99,7 @@
94	99	/// _mm_dpbssds_epi32( __m128i __W, __m128i __A, __m128i __B);
95	100	/// \endcode
96	101	///
97		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	102	/// This intrinsic corresponds to the \c VPDPBSSDS instruction.
98	103	///
99	104	/// \param __A
100	105	/// A 128-bit vector of [16 x char].
...	...	@@ -113,10 +118,12 @@
113	118	/// ENDFOR
114	119	/// dst[MAX:128] := 0
115	120	/// \endcode
	121	// clang-format on
116	122	#define _mm_dpbssds_epi32(__W, __A, __B) \
117		((__m128i)__builtin_ia32_vpdpbssds128((__v4si)(__W), (__v4si)(__A), \
118		(__v4si)(__B)))
	123	((__m128i)__builtin_ia32_vpdpbssds128((__v4si)(__W), (__v16qi)(__A), \
	124	(__v16qi)(__B)))
119	125
	126	// clang-format off
120	127	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
121	128	/// corresponding signed 8-bit integers in \a __B, producing 4 intermediate
122	129	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -129,7 +136,7 @@
129	136	/// _mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B);
130	137	/// \endcode
131	138	///
132		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	139	/// This intrinsic corresponds to the \c VPDPBSSDS instruction.
133	140	///
134	141	/// \param __A
135	142	/// A 256-bit vector of [32 x char].
...	...	@@ -148,10 +155,12 @@
148	155	/// ENDFOR
149	156	/// dst[MAX:256] := 0
150	157	/// \endcode
	158	// clang-format on
151	159	#define _mm256_dpbssds_epi32(__W, __A, __B) \
152		((__m256i)__builtin_ia32_vpdpbssds256((__v8si)(__W), (__v8si)(__A), \
153		(__v8si)(__B)))
	160	((__m256i)__builtin_ia32_vpdpbssds256((__v8si)(__W), (__v32qi)(__A), \
	161	(__v32qi)(__B)))
154	162
	163	// clang-format off
155	164	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
156	165	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
157	166	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -163,7 +172,7 @@
163	172	/// _mm_dpbsud_epi32(__m128i __W, __m128i __A, __m128i __B);
164	173	/// \endcode
165	174	///
166		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	175	/// This intrinsic corresponds to the \c VPDPBSUD instruction.
167	176	///
168	177	/// \param __A
169	178	/// A 128-bit vector of [16 x char].
...	...	@@ -182,10 +191,12 @@
182	191	/// ENDFOR
183	192	/// dst[MAX:128] := 0
184	193	/// \endcode
	194	// clang-format on
185	195	#define _mm_dpbsud_epi32(__W, __A, __B) \
186		((__m128i)__builtin_ia32_vpdpbsud128((__v4si)(__W), (__v4si)(__A), \
187		(__v4si)(__B)))
	196	((__m128i)__builtin_ia32_vpdpbsud128((__v4si)(__W), (__v16qi)(__A), \
	197	(__v16qu)(__B)))
188	198
	199	// clang-format off
189	200	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
190	201	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
191	202	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -197,7 +208,7 @@
197	208	/// _mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B);
198	209	/// \endcode
199	210	///
200		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	211	/// This intrinsic corresponds to the \c VPDPBSUD instruction.
201	212	///
202	213	/// \param __A
203	214	/// A 256-bit vector of [32 x char].
...	...	@@ -216,10 +227,12 @@
216	227	/// ENDFOR
217	228	/// dst[MAX:256] := 0
218	229	/// \endcode
	230	// clang-format on
219	231	#define _mm256_dpbsud_epi32(__W, __A, __B) \
220		((__m256i)__builtin_ia32_vpdpbsud256((__v8si)(__W), (__v8si)(__A), \
221		(__v8si)(__B)))
	232	((__m256i)__builtin_ia32_vpdpbsud256((__v8si)(__W), (__v32qi)(__A), \
	233	(__v32qu)(__B)))
222	234
	235	// clang-format off
223	236	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
224	237	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
225	238	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -232,7 +245,7 @@
232	245	/// _mm_dpbsuds_epi32( __m128i __W, __m128i __A, __m128i __B);
233	246	/// \endcode
234	247	///
235		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	248	/// This intrinsic corresponds to the \c VPDPBSUDS instruction.
236	249	///
237	250	/// \param __A
238	251	/// A 128-bit vector of [16 x char].
...	...	@@ -251,10 +264,12 @@
251	264	/// ENDFOR
252	265	/// dst[MAX:128] := 0
253	266	/// \endcode
	267	// clang-format on
254	268	#define _mm_dpbsuds_epi32(__W, __A, __B) \
255		((__m128i)__builtin_ia32_vpdpbsuds128((__v4si)(__W), (__v4si)(__A), \
256		(__v4si)(__B)))
	269	((__m128i)__builtin_ia32_vpdpbsuds128((__v4si)(__W), (__v16qi)(__A), \
	270	(__v16qu)(__B)))
257	271
	272	// clang-format off
258	273	/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in \a __A with
259	274	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
260	275	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -267,7 +282,7 @@
267	282	/// _mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B);
268	283	/// \endcode
269	284	///
270		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	285	/// This intrinsic corresponds to the \c VPDPBSUDS instruction.
271	286	///
272	287	/// \param __A
273	288	/// A 256-bit vector of [32 x char].
...	...	@@ -286,10 +301,12 @@
286	301	/// ENDFOR
287	302	/// dst[MAX:256] := 0
288	303	/// \endcode
	304	// clang-format on
289	305	#define _mm256_dpbsuds_epi32(__W, __A, __B) \
290		((__m256i)__builtin_ia32_vpdpbsuds256((__v8si)(__W), (__v8si)(__A), \
291		(__v8si)(__B)))
	306	((__m256i)__builtin_ia32_vpdpbsuds256((__v8si)(__W), (__v32qi)(__A), \
	307	(__v32qu)(__B)))
292	308
	309	// clang-format off
293	310	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
294	311	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
295	312	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -301,7 +318,7 @@
301	318	/// _mm_dpbuud_epi32(__m128i __W, __m128i __A, __m128i __B);
302	319	/// \endcode
303	320	///
304		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	321	/// This intrinsic corresponds to the \c VPDPBUUD instruction.
305	322	///
306	323	/// \param __A
307	324	/// A 128-bit vector of [16 x unsigned char].
...	...	@@ -320,10 +337,12 @@
320	337	/// ENDFOR
321	338	/// dst[MAX:128] := 0
322	339	/// \endcode
	340	// clang-format on
323	341	#define _mm_dpbuud_epi32(__W, __A, __B) \
324		((__m128i)__builtin_ia32_vpdpbuud128((__v4si)(__W), (__v4si)(__A), \
325		(__v4si)(__B)))
	342	((__m128i)__builtin_ia32_vpdpbuud128((__v4si)(__W), (__v16qu)(__A), \
	343	(__v16qu)(__B)))
326	344
	345	// clang-format off
327	346	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
328	347	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
329	348	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -335,7 +354,7 @@
335	354	/// _mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B);
336	355	/// \endcode
337	356	///
338		/// This intrinsic corresponds to the \c VPDPBSSD instruction.
	357	/// This intrinsic corresponds to the \c VPDPBUUD instruction.
339	358	///
340	359	/// \param __A
341	360	/// A 256-bit vector of [32 x unsigned char].
...	...	@@ -354,10 +373,12 @@
354	373	/// ENDFOR
355	374	/// dst[MAX:256] := 0
356	375	/// \endcode
	376	// clang-format on
357	377	#define _mm256_dpbuud_epi32(__W, __A, __B) \
358		((__m256i)__builtin_ia32_vpdpbuud256((__v8si)(__W), (__v8si)(__A), \
359		(__v8si)(__B)))
	378	((__m256i)__builtin_ia32_vpdpbuud256((__v8si)(__W), (__v32qu)(__A), \
	379	(__v32qu)(__B)))
360	380
	381	// clang-format off
361	382	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
362	383	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
363	384	/// signed 16-bit results. Sum these 4 results with the corresponding
...	...	@@ -389,10 +410,12 @@
389	410	/// ENDFOR
390	411	/// dst[MAX:128] := 0
391	412	/// \endcode
	413	// clang-format on
392	414	#define _mm_dpbuuds_epi32(__W, __A, __B) \
393		((__m128i)__builtin_ia32_vpdpbuuds128((__v4si)(__W), (__v4si)(__A), \
394		(__v4si)(__B)))
	415	((__m128i)__builtin_ia32_vpdpbuuds128((__v4si)(__W), (__v16qu)(__A), \
	416	(__v16qu)(__B)))
395	417
	418	// clang-format off
396	419	/// corresponding unsigned 8-bit integers in \a __B, producing 4 intermediate
397	420	/// signed 16-bit results. Sum these 4 results with the corresponding
398	421	/// 32-bit integer in \a __W with signed saturation, and store the packed
...	...	@@ -423,8 +446,9 @@
423	446	/// ENDFOR
424	447	/// dst[MAX:256] := 0
425	448	/// \endcode
	449	// clang-format on
426	450	#define _mm256_dpbuuds_epi32(__W, __A, __B) \
427		((__m256i)__builtin_ia32_vpdpbuuds256((__v8si)(__W), (__v8si)(__A), \
428		(__v8si)(__B)))
	451	((__m256i)__builtin_ia32_vpdpbuuds256((__v8si)(__W), (__v32qu)(__A), \
	452	(__v32qu)(__B)))
429	453
430	454	#endif // __AVXVNNIINT8INTRIN_H

lib/include/avxvnniintrin.h+16-8

...	...	@@ -63,7 +63,8 @@
63	63	static __inline__ __m256i __DEFAULT_FN_ATTRS256
64	64	_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
65	65	{
66		return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v8si)__A, (__v8si)__B);
	66	return (__m256i)__builtin_ia32_vpdpbusd256((__v8si)__S, (__v32qu)__A,
	67	(__v32qi)__B);
67	68	}
68	69
69	70	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
...	...	@@ -86,7 +87,8 @@ _mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
86	87	static __inline__ __m256i __DEFAULT_FN_ATTRS256
87	88	_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
88	89	{
89		return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v8si)__A, (__v8si)__B);
	90	return (__m256i)__builtin_ia32_vpdpbusds256((__v8si)__S, (__v32qu)__A,
	91	(__v32qi)__B);
90	92	}
91	93
92	94	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
...	...	@@ -107,7 +109,8 @@ _mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
107	109	static __inline__ __m256i __DEFAULT_FN_ATTRS256
108	110	_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
109	111	{
110		return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v8si)__A, (__v8si)__B);
	112	return (__m256i)__builtin_ia32_vpdpwssd256((__v8si)__S, (__v16hi)__A,
	113	(__v16hi)__B);
111	114	}
112	115
113	116	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
...	...	@@ -128,7 +131,8 @@ _mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
128	131	static __inline__ __m256i __DEFAULT_FN_ATTRS256
129	132	_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
130	133	{
131		return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v8si)__A, (__v8si)__B);
	134	return (__m256i)__builtin_ia32_vpdpwssds256((__v8si)__S, (__v16hi)__A,
	135	(__v16hi)__B);
132	136	}
133	137
134	138	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
...	...	@@ -151,7 +155,8 @@ _mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B)
151	155	static __inline__ __m128i __DEFAULT_FN_ATTRS128
152	156	_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
153	157	{
154		return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v4si)__A, (__v4si)__B);
	158	return (__m128i)__builtin_ia32_vpdpbusd128((__v4si)__S, (__v16qu)__A,
	159	(__v16qi)__B);
155	160	}
156	161
157	162	/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in \a __A with
...	...	@@ -174,7 +179,8 @@ _mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
174	179	static __inline__ __m128i __DEFAULT_FN_ATTRS128
175	180	_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
176	181	{
177		return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v4si)__A, (__v4si)__B);
	182	return (__m128i)__builtin_ia32_vpdpbusds128((__v4si)__S, (__v16qu)__A,
	183	(__v16qi)__B);
178	184	}
179	185
180	186	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
...	...	@@ -195,7 +201,8 @@ _mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
195	201	static __inline__ __m128i __DEFAULT_FN_ATTRS128
196	202	_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
197	203	{
198		return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v4si)__A, (__v4si)__B);
	204	return (__m128i)__builtin_ia32_vpdpwssd128((__v4si)__S, (__v8hi)__A,
	205	(__v8hi)__B);
199	206	}
200	207
201	208	/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
...	...	@@ -216,7 +223,8 @@ _mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
216	223	static __inline__ __m128i __DEFAULT_FN_ATTRS128
217	224	_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B)
218	225	{
219		return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v4si)__A, (__v4si)__B);
	226	return (__m128i)__builtin_ia32_vpdpwssds128((__v4si)__S, (__v8hi)__A,
	227	(__v8hi)__B);
220	228	}
221	229
222	230	#undef __DEFAULT_FN_ATTRS128

lib/include/cpuid.h+101-4

...	...	@@ -253,10 +253,6 @@
253	253	#define bit_RDPRU 0x00000010
254	254	#define bit_WBNOINVD 0x00000200
255	255
256		/* Features in %ebx for leaf 0x24 */
257		#define bit_AVX10_256 0x00020000
258		#define bit_AVX10_512 0x00040000
259
260	256	#ifdef __i386__
261	257	#define __cpuid(__leaf, __eax, __ebx, __ecx, __edx) \
262	258	__asm("cpuid" : "=a"(__eax), "=b" (__ebx), "=c"(__ecx), "=d"(__edx) \
...	...	@@ -282,6 +278,24 @@
282	278	: "0"(__leaf), "2"(__count))
283	279	#endif
284	280
	281	/// Queries the processor to determine the highest supported \c CPUID leaf.
	282	/// This intrinsic is only available on x86 and x64.
	283	///
	284	/// \headerfile <cpuid.h>
	285	///
	286	/// This intrinsic corresponds to the <c> CPUID </c> instruction.
	287	///
	288	/// \param __leaf
	289	/// \a __leaf can be either 0x0 or 0x8000000. If \a __leaf == 0x0, the
	290	/// highest supported value for basic \c CPUID information is returned.
	291	/// If \a __leaf == 0x8000000, the highest supported value for extended
	292	/// \c CPUID information is returned.
	293	/// \param __sig
	294	/// If the \a __sig pointer is non-null, the first four bytes of the
	295	/// signature (as found in the \c EBX register) are returned in the
	296	/// location pointed to by \a __sig.
	297	/// \returns Returns 0 if \c CPUID is supported; otherwise returns the value
	298	/// that \c CPUID returns in the \c EAX register.
285	299	static __inline unsigned int __get_cpuid_max (unsigned int __leaf,
286	300	unsigned int *__sig)
287	301	{
...	...	@@ -315,6 +329,32 @@ static __inline unsigned int __get_cpuid_max (unsigned int __leaf,
315	329	return __eax;
316	330	}
317	331
	332	/// For the requested \c CPUID leaf, queries the processor for information
	333	/// about the CPU type and CPU features (such as processor vendor, supported
	334	/// instruction sets, CPU capabilities, cache sizes, CPU model and family, and
	335	/// other hardware details). This intrinsic is only available on x86 and x64.
	336	///
	337	/// \headerfile <cpuid.h>
	338	///
	339	/// This intrinsic corresponds to the <c> CPUID </c> instruction.
	340	///
	341	/// \param __leaf
	342	/// An unsigned integer that identifies the level (also called "leaf") at
	343	/// which the \c CPUID instruction will be executed.
	344	/// \param __eax
	345	/// A pointer to an integer that corresponds to the \c EAX register where
	346	/// \c CPUID stores output results.
	347	/// \param __ebx
	348	/// A pointer to an integer that corresponds to the \c EBX register where
	349	/// \c CPUID stores output results.
	350	/// \param __ecx
	351	/// A pointer to an integer that corresponds to the \c ECX register where
	352	/// \c CPUID stores output results.
	353	/// \param __edx
	354	/// A pointer to an integer that corresponds to the \c EDX register where
	355	/// \c CPUID stores output results.
	356	/// \returns Returns 1 if the requested \c CPUID leaf is supported; otherwise
	357	/// returns 0.
318	358	static __inline int __get_cpuid (unsigned int __leaf, unsigned int *__eax,
319	359	unsigned int __ebx, unsigned int __ecx,
320	360	unsigned int *__edx)
...	...	@@ -328,6 +368,36 @@ static __inline int __get_cpuid (unsigned int __leaf, unsigned int *__eax,
328	368	return 1;
329	369	}
330	370
	371	/// For the requested \c CPUID leaf and subleaf, queries the processor for
	372	/// information about the CPU type and CPU features (such as processor vendor,
	373	/// supported instruction sets, CPU capabilities, cache sizes, CPU model and
	374	/// family, and other hardware details). This intrinsic is only available on
	375	/// x86 and x64.
	376	///
	377	/// \headerfile <cpuid.h>
	378	///
	379	/// This intrinsic corresponds to the <c> CPUID </c> instruction.
	380	///
	381	/// \param __leaf
	382	/// An unsigned integer that identifies the level (also called "leaf") at
	383	/// which the \c CPUID instruction will be executed.
	384	/// \param __subleaf
	385	/// An unsigned integer that identifies the sublevel (also called
	386	/// "subleaf") at which the \c CPUID instruction will be executed.
	387	/// \param __eax
	388	/// A pointer to an integer that corresponds to the \c EAX register where
	389	/// \c CPUID stores output results.
	390	/// \param __ebx
	391	/// A pointer to an integer that corresponds to the \c EBX register where
	392	/// \c CPUID stores output results.
	393	/// \param __ecx
	394	/// A pointer to an integer that corresponds to the \c ECX register where
	395	/// \c CPUID stores output results.
	396	/// \param __edx
	397	/// A pointer to an integer that corresponds to the \c EDX register where
	398	/// \c CPUID stores output results.
	399	/// \returns Returns 1 if the requested \c CPUID leaf is supported; otherwise
	400	/// returns 0.
331	401	static __inline int __get_cpuid_count (unsigned int __leaf,
332	402	unsigned int __subleaf,
333	403	unsigned int __eax, unsigned int __ebx,
...	...	@@ -345,10 +415,37 @@ static __inline int __get_cpuid_count (unsigned int __leaf,
345	415	// In some configurations, __cpuidex is defined as a builtin (primarily
346	416	// -fms-extensions) which will conflict with the __cpuidex definition below.
347	417	#if !(__has_builtin(__cpuidex))
	418	// In some cases, offloading will set the host as the aux triple and define the
	419	// builtin. Given __has_builtin does not detect builtins on aux triples, we need
	420	// to explicitly check for some offloading cases.
	421	#if !defined(__NVPTX__) && !defined(__AMDGPU__) && !defined(__SPIRV__)
	422	/// Executes the \c CPUID instruction with the specified leaf and subleaf
	423	/// values, and returns the results from the CPU's registers. This intrinsic
	424	/// is only available on x86 and x64.
	425	///
	426	/// \headerfile <cpuid.h>
	427	///
	428	/// This intrinsic corresponds to the <c> CPUID </c> instruction.
	429	///
	430	/// \param __cpu_info
	431	/// An output array of four integers:
	432	/// <ul>
	433	/// <li>\a __cpuInfo[0] receives the value of the \c EAX register.</li>
	434	/// <li>\a __cpuInfo[1] receives the value of the \c EBX register.</li>
	435	/// <li>\a __cpuInfo[2] receives the value of the \c ECX register.</li>
	436	/// <li>\a __cpuInfo[3] receives the value of the \c EDX register.</li>
	437	/// </ul>
	438	/// \param __leaf
	439	/// An unsigned integer that identifies the level (also called the "leaf")
	440	/// at which the \c CPUID instruction will be executed.
	441	/// \param __subleaf
	442	/// An unsigned integer that identifies the sublevel (also called the
	443	/// "subleaf") at which the \c CPUID instruction will be executed.
348	444	static __inline void __cpuidex(int __cpu_info[4], int __leaf, int __subleaf) {
349	445	__cpuid_count(__leaf, __subleaf, __cpu_info[0], __cpu_info[1], __cpu_info[2],
350	446	__cpu_info[3]);
351	447	}
352	448	#endif
	449	#endif
353	450
354	451	#endif /* __CPUID_H */

lib/include/emmintrin.h+168-168

...	...	@@ -17,7 +17,6 @@
17	17	#include <xmmintrin.h>
18	18
19	19	typedef double __m128d __attribute__((__vector_size__(16), __aligned__(16)));
20		typedef long long __m128i __attribute__((__vector_size__(16), __aligned__(16)));
21	20
22	21	typedef double __m128d_u __attribute__((__vector_size__(16), __aligned__(1)));
23	22	typedef long long __m128i_u
...	...	@@ -25,14 +24,9 @@ typedef long long __m128i_u
25	24
26	25	/* Type defines. */
27	26	typedef double __v2df __attribute__((__vector_size__(16)));
28		typedef long long __v2di __attribute__((__vector_size__(16)));
29		typedef short __v8hi __attribute__((__vector_size__(16)));
30		typedef char __v16qi __attribute__((__vector_size__(16)));
31	27
32	28	/* Unsigned types */
33	29	typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
34		typedef unsigned short __v8hu __attribute__((__vector_size__(16)));
35		typedef unsigned char __v16qu __attribute__((__vector_size__(16)));
36	30
37	31	/* We need an explicitly signed variant for char. Note that this shouldn't
38	32	* appear in the interface though. */
...	...	@@ -49,15 +43,9 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
49	43	#endif
50	44
51	45	/* Define the default attributes for the functions in this file. */
52		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
53		#define __DEFAULT_FN_ATTRS \
54		__attribute__((__always_inline__, __nodebug__, \
55		__target__("sse2,no-evex512"), __min_vector_width__(128)))
56		#else
57	46	#define __DEFAULT_FN_ATTRS \
58	47	__attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
59	48	__min_vector_width__(128)))
60		#endif
61	49
62	50	#if defined(__cplusplus) && (__cplusplus >= 201103L)
63	51	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
...	...	@@ -67,6 +55,9 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
67	55
68	56	#define __trunc64(x) \
69	57	(__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
	58	#define __zext128(x) \
	59	(__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
	60	1, 2, 3)
70	61	#define __anyext128(x) \
71	62	(__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
72	63	1, -1, -1)
...	...	@@ -250,8 +241,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_div_pd(__m128d __a,
250	241	/// bits are copied from the upper 64 bits of operand \a __a.
251	242	static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_sd(__m128d __a,
252	243	__m128d __b) {
253		__m128d __c = __builtin_ia32_sqrtsd((__v2df)__b);
254		return __extension__(__m128d){__c[0], __a[1]};
	244	return __extension__(__m128d){__builtin_elementwise_sqrt(__b[0]), __a[1]};
255	245	}
256	246
257	247	/// Calculates the square root of the each of two values stored in a
...	...	@@ -266,7 +256,7 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_sd(__m128d __a,
266	256	/// \returns A 128-bit vector of [2 x double] containing the square roots of the
267	257	/// values in the operand.
268	258	static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_pd(__m128d __a) {
269		return __builtin_ia32_sqrtpd((__v2df)__a);
	259	return __builtin_elementwise_sqrt(__a);
270	260	}
271	261
272	262	/// Compares lower 64-bit double-precision values of both operands, and
...	...	@@ -310,8 +300,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_sd(__m128d __a,
310	300	/// A 128-bit vector of [2 x double] containing one of the operands.
311	301	/// \returns A 128-bit vector of [2 x double] containing the minimum values
312	302	/// between both operands.
313		static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_pd(__m128d __a,
314		__m128d __b) {
	303	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_pd(__m128d __a,
	304	__m128d __b) {
315	305	return __builtin_ia32_minpd((__v2df)__a, (__v2df)__b);
316	306	}
317	307
...	...	@@ -356,8 +346,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_sd(__m128d __a,
356	346	/// A 128-bit vector of [2 x double] containing one of the operands.
357	347	/// \returns A 128-bit vector of [2 x double] containing the maximum values
358	348	/// between both operands.
359		static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_pd(__m128d __a,
360		__m128d __b) {
	349	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_pd(__m128d __a,
	350	__m128d __b) {
361	351	return __builtin_ia32_maxpd((__v2df)__a, (__v2df)__b);
362	352	}
363	353
...	...	@@ -1288,7 +1278,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomineq_sd(__m128d __a,
1288	1278	/// A 128-bit vector of [2 x double].
1289	1279	/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
1290	1280	/// converted values. The upper 64 bits are set to zero.
1291		static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtpd_ps(__m128d __a) {
	1281	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
	1282	_mm_cvtpd_ps(__m128d __a) {
1292	1283	return __builtin_ia32_cvtpd2ps((__v2df)__a);
1293	1284	}
1294	1285
...	...	@@ -1393,8 +1384,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvtsd_si32(__m128d __a) {
1393	1384	/// \returns A 128-bit vector of [4 x float]. The lower 32 bits contain the
1394	1385	/// converted value from the second parameter. The upper 96 bits are copied
1395	1386	/// from the upper 96 bits of the first parameter.
1396		static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_cvtsd_ss(__m128 __a,
1397		__m128d __b) {
	1387	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
	1388	_mm_cvtsd_ss(__m128 __a, __m128d __b) {
1398	1389	return (__m128)__builtin_ia32_cvtsd2ss((__v4sf)__a, (__v2df)__b);
1399	1390	}
1400	1391
...	...	@@ -2068,8 +2059,8 @@ static __inline__ void __DEFAULT_FN_ATTRS _mm_storel_pd(double *__dp,
2068	2059	/// A 128-bit vector of [16 x i8].
2069	2060	/// \returns A 128-bit vector of [16 x i8] containing the sums of both
2070	2061	/// parameters.
2071		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi8(__m128i __a,
2072		__m128i __b) {
	2062	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2063	_mm_add_epi8(__m128i __a, __m128i __b) {
2073	2064	return (__m128i)((__v16qu)__a + (__v16qu)__b);
2074	2065	}
2075	2066
...	...	@@ -2089,8 +2080,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi8(__m128i __a,
2089	2080	/// A 128-bit vector of [8 x i16].
2090	2081	/// \returns A 128-bit vector of [8 x i16] containing the sums of both
2091	2082	/// parameters.
2092		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_add_epi16(__m128i __a,
2093		__m128i __b) {
	2083	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2084	_mm_add_epi16(__m128i __a, __m128i __b) {
2094	2085	return (__m128i)((__v8hu)__a + (__v8hu)__b);
2095	2086	}
2096	2087
...	...	@@ -2127,8 +2118,9 @@ _mm_add_epi32(__m128i __a, __m128i __b) {
2127	2118	/// \param __b
2128	2119	/// A 64-bit integer.
2129	2120	/// \returns A 64-bit integer containing the sum of both parameters.
2130		static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) {
2131		return (__m64)(((unsigned long long)__a) + ((unsigned long long)__b));
	2121	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_si64(__m64 __a,
	2122	__m64 __b) {
	2123	return (__m64)(((__v1du)__a)[0] + ((__v1du)__b)[0]);
2132	2124	}
2133	2125
2134	2126	/// Adds the corresponding elements of two 128-bit vectors of [2 x i64],
...	...	@@ -2169,8 +2161,8 @@ _mm_add_epi64(__m128i __a, __m128i __b) {
2169	2161	/// A 128-bit signed [16 x i8] vector.
2170	2162	/// \returns A 128-bit signed [16 x i8] vector containing the saturated sums of
2171	2163	/// both parameters.
2172		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi8(__m128i __a,
2173		__m128i __b) {
	2164	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2165	_mm_adds_epi8(__m128i __a, __m128i __b) {
2174	2166	return (__m128i)__builtin_elementwise_add_sat((__v16qs)__a, (__v16qs)__b);
2175	2167	}
2176	2168
...	...	@@ -2191,8 +2183,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi8(__m128i __a,
2191	2183	/// A 128-bit signed [8 x i16] vector.
2192	2184	/// \returns A 128-bit signed [8 x i16] vector containing the saturated sums of
2193	2185	/// both parameters.
2194		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi16(__m128i __a,
2195		__m128i __b) {
	2186	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2187	_mm_adds_epi16(__m128i __a, __m128i __b) {
2196	2188	return (__m128i)__builtin_elementwise_add_sat((__v8hi)__a, (__v8hi)__b);
2197	2189	}
2198	2190
...	...	@@ -2213,8 +2205,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi16(__m128i __a,
2213	2205	/// A 128-bit unsigned [16 x i8] vector.
2214	2206	/// \returns A 128-bit unsigned [16 x i8] vector containing the saturated sums
2215	2207	/// of both parameters.
2216		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu8(__m128i __a,
2217		__m128i __b) {
	2208	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2209	_mm_adds_epu8(__m128i __a, __m128i __b) {
2218	2210	return (__m128i)__builtin_elementwise_add_sat((__v16qu)__a, (__v16qu)__b);
2219	2211	}
2220	2212
...	...	@@ -2235,8 +2227,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu8(__m128i __a,
2235	2227	/// A 128-bit unsigned [8 x i16] vector.
2236	2228	/// \returns A 128-bit unsigned [8 x i16] vector containing the saturated sums
2237	2229	/// of both parameters.
2238		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu16(__m128i __a,
2239		__m128i __b) {
	2230	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2231	_mm_adds_epu16(__m128i __a, __m128i __b) {
2240	2232	return (__m128i)__builtin_elementwise_add_sat((__v8hu)__a, (__v8hu)__b);
2241	2233	}
2242	2234
...	...	@@ -2254,9 +2246,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu16(__m128i __a,
2254	2246	/// A 128-bit unsigned [16 x i8] vector.
2255	2247	/// \returns A 128-bit unsigned [16 x i8] vector containing the rounded
2256	2248	/// averages of both parameters.
2257		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_avg_epu8(__m128i __a,
2258		__m128i __b) {
2259		return (__m128i)__builtin_ia32_pavgb128((__v16qi)__a, (__v16qi)__b);
	2249	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2250	_mm_avg_epu8(__m128i __a, __m128i __b) {
	2251	return (__m128i)__builtin_ia32_pavgb128((__v16qu)__a, (__v16qu)__b);
2260	2252	}
2261	2253
2262	2254	/// Computes the rounded averages of corresponding elements of two
...	...	@@ -2273,9 +2265,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_avg_epu8(__m128i __a,
2273	2265	/// A 128-bit unsigned [8 x i16] vector.
2274	2266	/// \returns A 128-bit unsigned [8 x i16] vector containing the rounded
2275	2267	/// averages of both parameters.
2276		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_avg_epu16(__m128i __a,
2277		__m128i __b) {
2278		return (__m128i)__builtin_ia32_pavgw128((__v8hi)__a, (__v8hi)__b);
	2268	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2269	_mm_avg_epu16(__m128i __a, __m128i __b) {
	2270	return (__m128i)__builtin_ia32_pavgw128((__v8hu)__a, (__v8hu)__b);
2279	2271	}
2280	2272
2281	2273	/// Multiplies the corresponding elements of two 128-bit signed [8 x i16]
...	...	@@ -2298,8 +2290,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_avg_epu16(__m128i __a,
2298	2290	/// A 128-bit signed [8 x i16] vector.
2299	2291	/// \returns A 128-bit signed [4 x i32] vector containing the sums of products
2300	2292	/// of both parameters.
2301		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_madd_epi16(__m128i __a,
2302		__m128i __b) {
	2293	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2294	_mm_madd_epi16(__m128i __a, __m128i __b) {
2303	2295	return (__m128i)__builtin_ia32_pmaddwd128((__v8hi)__a, (__v8hi)__b);
2304	2296	}
2305	2297
...	...	@@ -2317,8 +2309,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_madd_epi16(__m128i __a,
2317	2309	/// A 128-bit signed [8 x i16] vector.
2318	2310	/// \returns A 128-bit signed [8 x i16] vector containing the greater value of
2319	2311	/// each comparison.
2320		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi16(__m128i __a,
2321		__m128i __b) {
	2312	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2313	_mm_max_epi16(__m128i __a, __m128i __b) {
2322	2314	return (__m128i)__builtin_elementwise_max((__v8hi)__a, (__v8hi)__b);
2323	2315	}
2324	2316
...	...	@@ -2336,8 +2328,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi16(__m128i __a,
2336	2328	/// A 128-bit unsigned [16 x i8] vector.
2337	2329	/// \returns A 128-bit unsigned [16 x i8] vector containing the greater value of
2338	2330	/// each comparison.
2339		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu8(__m128i __a,
2340		__m128i __b) {
	2331	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2332	_mm_max_epu8(__m128i __a, __m128i __b) {
2341	2333	return (__m128i)__builtin_elementwise_max((__v16qu)__a, (__v16qu)__b);
2342	2334	}
2343	2335
...	...	@@ -2355,8 +2347,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu8(__m128i __a,
2355	2347	/// A 128-bit signed [8 x i16] vector.
2356	2348	/// \returns A 128-bit signed [8 x i16] vector containing the smaller value of
2357	2349	/// each comparison.
2358		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi16(__m128i __a,
2359		__m128i __b) {
	2350	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2351	_mm_min_epi16(__m128i __a, __m128i __b) {
2360	2352	return (__m128i)__builtin_elementwise_min((__v8hi)__a, (__v8hi)__b);
2361	2353	}
2362	2354
...	...	@@ -2374,8 +2366,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi16(__m128i __a,
2374	2366	/// A 128-bit unsigned [16 x i8] vector.
2375	2367	/// \returns A 128-bit unsigned [16 x i8] vector containing the smaller value of
2376	2368	/// each comparison.
2377		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu8(__m128i __a,
2378		__m128i __b) {
	2369	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2370	_mm_min_epu8(__m128i __a, __m128i __b) {
2379	2371	return (__m128i)__builtin_elementwise_min((__v16qu)__a, (__v16qu)__b);
2380	2372	}
2381	2373
...	...	@@ -2393,8 +2385,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu8(__m128i __a,
2393	2385	/// A 128-bit signed [8 x i16] vector.
2394	2386	/// \returns A 128-bit signed [8 x i16] vector containing the upper 16 bits of
2395	2387	/// each of the eight 32-bit products.
2396		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epi16(__m128i __a,
2397		__m128i __b) {
	2388	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2389	_mm_mulhi_epi16(__m128i __a, __m128i __b) {
2398	2390	return (__m128i)__builtin_ia32_pmulhw128((__v8hi)__a, (__v8hi)__b);
2399	2391	}
2400	2392
...	...	@@ -2412,9 +2404,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epi16(__m128i __a,
2412	2404	/// A 128-bit unsigned [8 x i16] vector.
2413	2405	/// \returns A 128-bit unsigned [8 x i16] vector containing the upper 16 bits
2414	2406	/// of each of the eight 32-bit products.
2415		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epu16(__m128i __a,
2416		__m128i __b) {
2417		return (__m128i)__builtin_ia32_pmulhuw128((__v8hi)__a, (__v8hi)__b);
	2407	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2408	_mm_mulhi_epu16(__m128i __a, __m128i __b) {
	2409	return (__m128i)__builtin_ia32_pmulhuw128((__v8hu)__a, (__v8hu)__b);
2418	2410	}
2419	2411
2420	2412	/// Multiplies the corresponding elements of two signed [8 x i16]
...	...	@@ -2431,8 +2423,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epu16(__m128i __a,
2431	2423	/// A 128-bit signed [8 x i16] vector.
2432	2424	/// \returns A 128-bit signed [8 x i16] vector containing the lower 16 bits of
2433	2425	/// each of the eight 32-bit products.
2434		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
2435		__m128i __b) {
	2426	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2427	_mm_mullo_epi16(__m128i __a, __m128i __b) {
2436	2428	return (__m128i)((__v8hu)__a * (__v8hu)__b);
2437	2429	}
2438	2430
...	...	@@ -2449,9 +2441,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
2449	2441	/// \param __b
2450	2442	/// A 64-bit integer containing one of the source operands.
2451	2443	/// \returns A 64-bit integer vector containing the product of both operands.
2452		static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
2453		return __trunc64(__builtin_ia32_pmuludq128((__v4si)__anyext128(__a),
2454		(__v4si)__anyext128(__b)));
	2444	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_su32(__m64 __a,
	2445	__m64 __b) {
	2446	return __trunc64(__builtin_ia32_pmuludq128((__v4si)__zext128(__a),
	2447	(__v4si)__zext128(__b)));
2455	2448	}
2456	2449
2457	2450	/// Multiplies 32-bit unsigned integer values contained in the lower
...	...	@@ -2467,8 +2460,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
2467	2460	/// \param __b
2468	2461	/// A [2 x i64] vector containing one of the source operands.
2469	2462	/// \returns A [2 x i64] vector containing the product of both operands.
2470		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mul_epu32(__m128i __a,
2471		__m128i __b) {
	2463	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2464	_mm_mul_epu32(__m128i __a, __m128i __b) {
2472	2465	return __builtin_ia32_pmuludq128((__v4si)__a, (__v4si)__b);
2473	2466	}
2474	2467
...	...	@@ -2505,8 +2498,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sad_epu8(__m128i __a,
2505	2498	/// A 128-bit integer vector containing the subtrahends.
2506	2499	/// \returns A 128-bit integer vector containing the differences of the values
2507	2500	/// in the operands.
2508		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi8(__m128i __a,
2509		__m128i __b) {
	2501	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2502	_mm_sub_epi8(__m128i __a, __m128i __b) {
2510	2503	return (__m128i)((__v16qu)__a - (__v16qu)__b);
2511	2504	}
2512	2505
...	...	@@ -2522,8 +2515,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi8(__m128i __a,
2522	2515	/// A 128-bit integer vector containing the subtrahends.
2523	2516	/// \returns A 128-bit integer vector containing the differences of the values
2524	2517	/// in the operands.
2525		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sub_epi16(__m128i __a,
2526		__m128i __b) {
	2518	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2519	_mm_sub_epi16(__m128i __a, __m128i __b) {
2527	2520	return (__m128i)((__v8hu)__a - (__v8hu)__b);
2528	2521	}
2529	2522
...	...	@@ -2557,8 +2550,9 @@ _mm_sub_epi32(__m128i __a, __m128i __b) {
2557	2550	/// A 64-bit integer vector containing the subtrahend.
2558	2551	/// \returns A 64-bit integer vector containing the difference of the values in
2559	2552	/// the operands.
2560		static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) {
2561		return (__m64)((unsigned long long)__a - (unsigned long long)__b);
	2553	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_si64(__m64 __a,
	2554	__m64 __b) {
	2555	return (__m64)(((__v1du)__a)[0] - ((__v1du)__b)[0]);
2562	2556	}
2563	2557
2564	2558	/// Subtracts the corresponding elements of two [2 x i64] vectors.
...	...	@@ -2595,8 +2589,8 @@ _mm_sub_epi64(__m128i __a, __m128i __b) {
2595	2589	/// A 128-bit integer vector containing the subtrahends.
2596	2590	/// \returns A 128-bit integer vector containing the differences of the values
2597	2591	/// in the operands.
2598		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi8(__m128i __a,
2599		__m128i __b) {
	2592	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2593	_mm_subs_epi8(__m128i __a, __m128i __b) {
2600	2594	return (__m128i)__builtin_elementwise_sub_sat((__v16qs)__a, (__v16qs)__b);
2601	2595	}
2602	2596
...	...	@@ -2617,8 +2611,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi8(__m128i __a,
2617	2611	/// A 128-bit integer vector containing the subtrahends.
2618	2612	/// \returns A 128-bit integer vector containing the differences of the values
2619	2613	/// in the operands.
2620		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi16(__m128i __a,
2621		__m128i __b) {
	2614	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2615	_mm_subs_epi16(__m128i __a, __m128i __b) {
2622	2616	return (__m128i)__builtin_elementwise_sub_sat((__v8hi)__a, (__v8hi)__b);
2623	2617	}
2624	2618
...	...	@@ -2638,8 +2632,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi16(__m128i __a,
2638	2632	/// A 128-bit integer vector containing the subtrahends.
2639	2633	/// \returns A 128-bit integer vector containing the unsigned integer
2640	2634	/// differences of the values in the operands.
2641		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu8(__m128i __a,
2642		__m128i __b) {
	2635	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2636	_mm_subs_epu8(__m128i __a, __m128i __b) {
2643	2637	return (__m128i)__builtin_elementwise_sub_sat((__v16qu)__a, (__v16qu)__b);
2644	2638	}
2645	2639
...	...	@@ -2659,8 +2653,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu8(__m128i __a,
2659	2653	/// A 128-bit integer vector containing the subtrahends.
2660	2654	/// \returns A 128-bit integer vector containing the unsigned integer
2661	2655	/// differences of the values in the operands.
2662		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu16(__m128i __a,
2663		__m128i __b) {
	2656	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2657	_mm_subs_epu16(__m128i __a, __m128i __b) {
2664	2658	return (__m128i)__builtin_elementwise_sub_sat((__v8hu)__a, (__v8hu)__b);
2665	2659	}
2666	2660
...	...	@@ -2676,8 +2670,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu16(__m128i __a,
2676	2670	/// A 128-bit integer vector containing one of the source operands.
2677	2671	/// \returns A 128-bit integer vector containing the bitwise AND of the values
2678	2672	/// in both operands.
2679		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a,
2680		__m128i __b) {
	2673	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2674	_mm_and_si128(__m128i __a, __m128i __b) {
2681	2675	return (__m128i)((__v2du)__a & (__v2du)__b);
2682	2676	}
2683	2677
...	...	@@ -2695,8 +2689,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a,
2695	2689	/// A 128-bit vector containing the right source operand.
2696	2690	/// \returns A 128-bit integer vector containing the bitwise AND of the one's
2697	2691	/// complement of the first operand and the values in the second operand.
2698		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a,
2699		__m128i __b) {
	2692	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2693	_mm_andnot_si128(__m128i __a, __m128i __b) {
2700	2694	return (__m128i)(~(__v2du)__a & (__v2du)__b);
2701	2695	}
2702	2696	/// Performs a bitwise OR of two 128-bit integer vectors.
...	...	@@ -2711,8 +2705,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a,
2711	2705	/// A 128-bit integer vector containing one of the source operands.
2712	2706	/// \returns A 128-bit integer vector containing the bitwise OR of the values
2713	2707	/// in both operands.
2714		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a,
2715		__m128i __b) {
	2708	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2709	_mm_or_si128(__m128i __a, __m128i __b) {
2716	2710	return (__m128i)((__v2du)__a \| (__v2du)__b);
2717	2711	}
2718	2712
...	...	@@ -2728,8 +2722,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a,
2728	2722	/// A 128-bit integer vector containing one of the source operands.
2729	2723	/// \returns A 128-bit integer vector containing the bitwise exclusive OR of the
2730	2724	/// values in both operands.
2731		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_xor_si128(__m128i __a,
2732		__m128i __b) {
	2725	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2726	_mm_xor_si128(__m128i __a, __m128i __b) {
2733	2727	return (__m128i)((__v2du)__a ^ (__v2du)__b);
2734	2728	}
2735	2729
...	...	@@ -2751,11 +2745,11 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_xor_si128(__m128i __a,
2751	2745	/// \a a.
2752	2746	/// \returns A 128-bit integer vector containing the left-shifted value.
2753	2747	#define _mm_slli_si128(a, imm) \
2754		((__m128i)__builtin_ia32_pslldqi128_byteshift((__v2di)(__m128i)(a), \
	2748	((__m128i)__builtin_ia32_pslldqi128_byteshift((__v16qi)(__m128i)(a), \
2755	2749	(int)(imm)))
2756	2750
2757	2751	#define _mm_bslli_si128(a, imm) \
2758		((__m128i)__builtin_ia32_pslldqi128_byteshift((__v2di)(__m128i)(a), \
	2752	((__m128i)__builtin_ia32_pslldqi128_byteshift((__v16qi)(__m128i)(a), \
2759	2753	(int)(imm)))
2760	2754
2761	2755	/// Left-shifts each 16-bit value in the 128-bit integer vector operand
...	...	@@ -2771,8 +2765,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_xor_si128(__m128i __a,
2771	2765	/// An integer value specifying the number of bits to left-shift each value
2772	2766	/// in operand \a __a.
2773	2767	/// \returns A 128-bit integer vector containing the left-shifted values.
2774		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi16(__m128i __a,
2775		int __count) {
	2768	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2769	_mm_slli_epi16(__m128i __a, int __count) {
2776	2770	return (__m128i)__builtin_ia32_psllwi128((__v8hi)__a, __count);
2777	2771	}
2778	2772
...	...	@@ -2789,8 +2783,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi16(__m128i __a,
2789	2783	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2790	2784	/// to left-shift each value in operand \a __a.
2791	2785	/// \returns A 128-bit integer vector containing the left-shifted values.
2792		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi16(__m128i __a,
2793		__m128i __count) {
	2786	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2787	_mm_sll_epi16(__m128i __a, __m128i __count) {
2794	2788	return (__m128i)__builtin_ia32_psllw128((__v8hi)__a, (__v8hi)__count);
2795	2789	}
2796	2790
...	...	@@ -2807,8 +2801,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi16(__m128i __a,
2807	2801	/// An integer value specifying the number of bits to left-shift each value
2808	2802	/// in operand \a __a.
2809	2803	/// \returns A 128-bit integer vector containing the left-shifted values.
2810		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi32(__m128i __a,
2811		int __count) {
	2804	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2805	_mm_slli_epi32(__m128i __a, int __count) {
2812	2806	return (__m128i)__builtin_ia32_pslldi128((__v4si)__a, __count);
2813	2807	}
2814	2808
...	...	@@ -2825,8 +2819,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi32(__m128i __a,
2825	2819	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2826	2820	/// to left-shift each value in operand \a __a.
2827	2821	/// \returns A 128-bit integer vector containing the left-shifted values.
2828		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi32(__m128i __a,
2829		__m128i __count) {
	2822	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2823	_mm_sll_epi32(__m128i __a, __m128i __count) {
2830	2824	return (__m128i)__builtin_ia32_pslld128((__v4si)__a, (__v4si)__count);
2831	2825	}
2832	2826
...	...	@@ -2843,8 +2837,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi32(__m128i __a,
2843	2837	/// An integer value specifying the number of bits to left-shift each value
2844	2838	/// in operand \a __a.
2845	2839	/// \returns A 128-bit integer vector containing the left-shifted values.
2846		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi64(__m128i __a,
2847		int __count) {
	2840	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2841	_mm_slli_epi64(__m128i __a, int __count) {
2848	2842	return __builtin_ia32_psllqi128((__v2di)__a, __count);
2849	2843	}
2850	2844
...	...	@@ -2861,8 +2855,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_slli_epi64(__m128i __a,
2861	2855	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2862	2856	/// to left-shift each value in operand \a __a.
2863	2857	/// \returns A 128-bit integer vector containing the left-shifted values.
2864		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi64(__m128i __a,
2865		__m128i __count) {
	2858	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2859	_mm_sll_epi64(__m128i __a, __m128i __count) {
2866	2860	return __builtin_ia32_psllq128((__v2di)__a, (__v2di)__count);
2867	2861	}
2868	2862
...	...	@@ -2880,8 +2874,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sll_epi64(__m128i __a,
2880	2874	/// An integer value specifying the number of bits to right-shift each value
2881	2875	/// in operand \a __a.
2882	2876	/// \returns A 128-bit integer vector containing the right-shifted values.
2883		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srai_epi16(__m128i __a,
2884		int __count) {
	2877	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2878	_mm_srai_epi16(__m128i __a, int __count) {
2885	2879	return (__m128i)__builtin_ia32_psrawi128((__v8hi)__a, __count);
2886	2880	}
2887	2881
...	...	@@ -2899,8 +2893,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srai_epi16(__m128i __a,
2899	2893	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2900	2894	/// to right-shift each value in operand \a __a.
2901	2895	/// \returns A 128-bit integer vector containing the right-shifted values.
2902		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sra_epi16(__m128i __a,
2903		__m128i __count) {
	2896	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2897	_mm_sra_epi16(__m128i __a, __m128i __count) {
2904	2898	return (__m128i)__builtin_ia32_psraw128((__v8hi)__a, (__v8hi)__count);
2905	2899	}
2906	2900
...	...	@@ -2918,8 +2912,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sra_epi16(__m128i __a,
2918	2912	/// An integer value specifying the number of bits to right-shift each value
2919	2913	/// in operand \a __a.
2920	2914	/// \returns A 128-bit integer vector containing the right-shifted values.
2921		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srai_epi32(__m128i __a,
2922		int __count) {
	2915	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2916	_mm_srai_epi32(__m128i __a, int __count) {
2923	2917	return (__m128i)__builtin_ia32_psradi128((__v4si)__a, __count);
2924	2918	}
2925	2919
...	...	@@ -2937,8 +2931,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srai_epi32(__m128i __a,
2937	2931	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2938	2932	/// to right-shift each value in operand \a __a.
2939	2933	/// \returns A 128-bit integer vector containing the right-shifted values.
2940		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sra_epi32(__m128i __a,
2941		__m128i __count) {
	2934	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2935	_mm_sra_epi32(__m128i __a, __m128i __count) {
2942	2936	return (__m128i)__builtin_ia32_psrad128((__v4si)__a, (__v4si)__count);
2943	2937	}
2944	2938
...	...	@@ -2960,11 +2954,11 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sra_epi32(__m128i __a,
2960	2954	/// \a a.
2961	2955	/// \returns A 128-bit integer vector containing the right-shifted value.
2962	2956	#define _mm_srli_si128(a, imm) \
2963		((__m128i)__builtin_ia32_psrldqi128_byteshift((__v2di)(__m128i)(a), \
	2957	((__m128i)__builtin_ia32_psrldqi128_byteshift((__v16qi)(__m128i)(a), \
2964	2958	(int)(imm)))
2965	2959
2966	2960	#define _mm_bsrli_si128(a, imm) \
2967		((__m128i)__builtin_ia32_psrldqi128_byteshift((__v2di)(__m128i)(a), \
	2961	((__m128i)__builtin_ia32_psrldqi128_byteshift((__v16qi)(__m128i)(a), \
2968	2962	(int)(imm)))
2969	2963
2970	2964	/// Right-shifts each of 16-bit values in the 128-bit integer vector
...	...	@@ -2980,8 +2974,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_sra_epi32(__m128i __a,
2980	2974	/// An integer value specifying the number of bits to right-shift each value
2981	2975	/// in operand \a __a.
2982	2976	/// \returns A 128-bit integer vector containing the right-shifted values.
2983		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi16(__m128i __a,
2984		int __count) {
	2977	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2978	_mm_srli_epi16(__m128i __a, int __count) {
2985	2979	return (__m128i)__builtin_ia32_psrlwi128((__v8hi)__a, __count);
2986	2980	}
2987	2981
...	...	@@ -2998,8 +2992,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi16(__m128i __a,
2998	2992	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
2999	2993	/// to right-shift each value in operand \a __a.
3000	2994	/// \returns A 128-bit integer vector containing the right-shifted values.
3001		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi16(__m128i __a,
3002		__m128i __count) {
	2995	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2996	_mm_srl_epi16(__m128i __a, __m128i __count) {
3003	2997	return (__m128i)__builtin_ia32_psrlw128((__v8hi)__a, (__v8hi)__count);
3004	2998	}
3005	2999
...	...	@@ -3016,8 +3010,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi16(__m128i __a,
3016	3010	/// An integer value specifying the number of bits to right-shift each value
3017	3011	/// in operand \a __a.
3018	3012	/// \returns A 128-bit integer vector containing the right-shifted values.
3019		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi32(__m128i __a,
3020		int __count) {
	3013	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3014	_mm_srli_epi32(__m128i __a, int __count) {
3021	3015	return (__m128i)__builtin_ia32_psrldi128((__v4si)__a, __count);
3022	3016	}
3023	3017
...	...	@@ -3034,8 +3028,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi32(__m128i __a,
3034	3028	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
3035	3029	/// to right-shift each value in operand \a __a.
3036	3030	/// \returns A 128-bit integer vector containing the right-shifted values.
3037		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi32(__m128i __a,
3038		__m128i __count) {
	3031	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3032	_mm_srl_epi32(__m128i __a, __m128i __count) {
3039	3033	return (__m128i)__builtin_ia32_psrld128((__v4si)__a, (__v4si)__count);
3040	3034	}
3041	3035
...	...	@@ -3052,8 +3046,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi32(__m128i __a,
3052	3046	/// An integer value specifying the number of bits to right-shift each value
3053	3047	/// in operand \a __a.
3054	3048	/// \returns A 128-bit integer vector containing the right-shifted values.
3055		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi64(__m128i __a,
3056		int __count) {
	3049	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3050	_mm_srli_epi64(__m128i __a, int __count) {
3057	3051	return __builtin_ia32_psrlqi128((__v2di)__a, __count);
3058	3052	}
3059	3053
...	...	@@ -3070,8 +3064,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srli_epi64(__m128i __a,
3070	3064	/// A 128-bit integer vector in which bits [63:0] specify the number of bits
3071	3065	/// to right-shift each value in operand \a __a.
3072	3066	/// \returns A 128-bit integer vector containing the right-shifted values.
3073		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi64(__m128i __a,
3074		__m128i __count) {
	3067	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3068	_mm_srl_epi64(__m128i __a, __m128i __count) {
3075	3069	return __builtin_ia32_psrlq128((__v2di)__a, (__v2di)__count);
3076	3070	}
3077	3071
...	...	@@ -3089,8 +3083,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi64(__m128i __a,
3089	3083	/// \param __b
3090	3084	/// A 128-bit integer vector.
3091	3085	/// \returns A 128-bit integer vector containing the comparison results.
3092		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi8(__m128i __a,
3093		__m128i __b) {
	3086	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3087	_mm_cmpeq_epi8(__m128i __a, __m128i __b) {
3094	3088	return (__m128i)((__v16qi)__a == (__v16qi)__b);
3095	3089	}
3096	3090
...	...	@@ -3108,8 +3102,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi8(__m128i __a,
3108	3102	/// \param __b
3109	3103	/// A 128-bit integer vector.
3110	3104	/// \returns A 128-bit integer vector containing the comparison results.
3111		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi16(__m128i __a,
3112		__m128i __b) {
	3105	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3106	_mm_cmpeq_epi16(__m128i __a, __m128i __b) {
3113	3107	return (__m128i)((__v8hi)__a == (__v8hi)__b);
3114	3108	}
3115	3109
...	...	@@ -3127,8 +3121,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi16(__m128i __a,
3127	3121	/// \param __b
3128	3122	/// A 128-bit integer vector.
3129	3123	/// \returns A 128-bit integer vector containing the comparison results.
3130		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi32(__m128i __a,
3131		__m128i __b) {
	3124	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3125	_mm_cmpeq_epi32(__m128i __a, __m128i __b) {
3132	3126	return (__m128i)((__v4si)__a == (__v4si)__b);
3133	3127	}
3134	3128
...	...	@@ -3147,8 +3141,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi32(__m128i __a,
3147	3141	/// \param __b
3148	3142	/// A 128-bit integer vector.
3149	3143	/// \returns A 128-bit integer vector containing the comparison results.
3150		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi8(__m128i __a,
3151		__m128i __b) {
	3144	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3145	_mm_cmpgt_epi8(__m128i __a, __m128i __b) {
3152	3146	/* This function always performs a signed comparison, but __v16qi is a char
3153	3147	which may be signed or unsigned, so use __v16qs. */
3154	3148	return (__m128i)((__v16qs)__a > (__v16qs)__b);
...	...	@@ -3169,8 +3163,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi8(__m128i __a,
3169	3163	/// \param __b
3170	3164	/// A 128-bit integer vector.
3171	3165	/// \returns A 128-bit integer vector containing the comparison results.
3172		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi16(__m128i __a,
3173		__m128i __b) {
	3166	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3167	_mm_cmpgt_epi16(__m128i __a, __m128i __b) {
3174	3168	return (__m128i)((__v8hi)__a > (__v8hi)__b);
3175	3169	}
3176	3170
...	...	@@ -3189,8 +3183,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi16(__m128i __a,
3189	3183	/// \param __b
3190	3184	/// A 128-bit integer vector.
3191	3185	/// \returns A 128-bit integer vector containing the comparison results.
3192		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi32(__m128i __a,
3193		__m128i __b) {
	3186	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3187	_mm_cmpgt_epi32(__m128i __a, __m128i __b) {
3194	3188	return (__m128i)((__v4si)__a > (__v4si)__b);
3195	3189	}
3196	3190
...	...	@@ -3209,8 +3203,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi32(__m128i __a,
3209	3203	/// \param __b
3210	3204	/// A 128-bit integer vector.
3211	3205	/// \returns A 128-bit integer vector containing the comparison results.
3212		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi8(__m128i __a,
3213		__m128i __b) {
	3206	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3207	_mm_cmplt_epi8(__m128i __a, __m128i __b) {
3214	3208	return _mm_cmpgt_epi8(__b, __a);
3215	3209	}
3216	3210
...	...	@@ -3229,8 +3223,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi8(__m128i __a,
3229	3223	/// \param __b
3230	3224	/// A 128-bit integer vector.
3231	3225	/// \returns A 128-bit integer vector containing the comparison results.
3232		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi16(__m128i __a,
3233		__m128i __b) {
	3226	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3227	_mm_cmplt_epi16(__m128i __a, __m128i __b) {
3234	3228	return _mm_cmpgt_epi16(__b, __a);
3235	3229	}
3236	3230
...	...	@@ -3249,8 +3243,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi16(__m128i __a,
3249	3243	/// \param __b
3250	3244	/// A 128-bit integer vector.
3251	3245	/// \returns A 128-bit integer vector containing the comparison results.
3252		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi32(__m128i __a,
3253		__m128i __b) {
	3246	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3247	_mm_cmplt_epi32(__m128i __a, __m128i __b) {
3254	3248	return _mm_cmpgt_epi32(__b, __a);
3255	3249	}
3256	3250
...	...	@@ -3379,7 +3373,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvttps_epi32(__m128 __a) {
3379	3373	/// \param __a
3380	3374	/// A 32-bit signed integer operand.
3381	3375	/// \returns A 128-bit vector of [4 x i32].
3382		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi32_si128(int __a) {
	3376	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3377	_mm_cvtsi32_si128(int __a) {
3383	3378	return __extension__(__m128i)(__v4si){__a, 0, 0, 0};
3384	3379	}
3385	3380
...	...	@@ -3394,7 +3389,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi32_si128(int __a) {
3394	3389	/// \param __a
3395	3390	/// A 64-bit signed integer operand containing the value to be converted.
3396	3391	/// \returns A 128-bit vector of [2 x i64] containing the converted value.
3397		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi64_si128(long long __a) {
	3392	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	3393	_mm_cvtsi64_si128(long long __a) {
3398	3394	return __extension__(__m128i)(__v2di){__a, 0};
3399	3395	}
3400	3396
...	...	@@ -3409,7 +3405,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi64_si128(long long __a) {
3409	3405	/// A vector of [4 x i32]. The least significant 32 bits are moved to the
3410	3406	/// destination.
3411	3407	/// \returns A 32-bit signed integer containing the moved value.
3412		static __inline__ int __DEFAULT_FN_ATTRS _mm_cvtsi128_si32(__m128i __a) {
	3408	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	3409	_mm_cvtsi128_si32(__m128i __a) {
3413	3410	__v4si __b = (__v4si)__a;
3414	3411	return __b[0];
3415	3412	}
...	...	@@ -3425,7 +3422,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvtsi128_si32(__m128i __a) {
3425	3422	/// A vector of [2 x i64]. The least significant 64 bits are moved to the
3426	3423	/// destination.
3427	3424	/// \returns A 64-bit signed integer containing the moved value.
3428		static __inline__ long long __DEFAULT_FN_ATTRS _mm_cvtsi128_si64(__m128i __a) {
	3425	static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
	3426	_mm_cvtsi128_si64(__m128i __a) {
3429	3427	return __a[0];
3430	3428	}
3431	3429
...	...	@@ -4161,8 +4159,8 @@ void _mm_mfence(void);
4161	4159	/// A 128-bit integer vector of [8 x i16]. The converted [8 x i8] values are
4162	4160	/// written to the higher 64 bits of the result.
4163	4161	/// \returns A 128-bit vector of [16 x i8] containing the converted values.
4164		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packs_epi16(__m128i __a,
4165		__m128i __b) {
	4162	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4163	_mm_packs_epi16(__m128i __a, __m128i __b) {
4166	4164	return (__m128i)__builtin_ia32_packsswb128((__v8hi)__a, (__v8hi)__b);
4167	4165	}
4168	4166
...	...	@@ -4184,8 +4182,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packs_epi16(__m128i __a,
4184	4182	/// A 128-bit integer vector of [4 x i32]. The converted [4 x i16] values
4185	4183	/// are written to the higher 64 bits of the result.
4186	4184	/// \returns A 128-bit vector of [8 x i16] containing the converted values.
4187		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packs_epi32(__m128i __a,
4188		__m128i __b) {
	4185	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4186	_mm_packs_epi32(__m128i __a, __m128i __b) {
4189	4187	return (__m128i)__builtin_ia32_packssdw128((__v4si)__a, (__v4si)__b);
4190	4188	}
4191	4189
...	...	@@ -4207,8 +4205,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packs_epi32(__m128i __a,
4207	4205	/// A 128-bit integer vector of [8 x i16]. The converted [8 x i8] values are
4208	4206	/// written to the higher 64 bits of the result.
4209	4207	/// \returns A 128-bit vector of [16 x i8] containing the converted values.
4210		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packus_epi16(__m128i __a,
4211		__m128i __b) {
	4208	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4209	_mm_packus_epi16(__m128i __a, __m128i __b) {
4212	4210	return (__m128i)__builtin_ia32_packuswb128((__v8hi)__a, (__v8hi)__b);
4213	4211	}
4214	4212
...	...	@@ -4282,7 +4280,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packus_epi16(__m128i __a,
4282	4280	/// A 128-bit integer vector containing the values with bits to be extracted.
4283	4281	/// \returns The most significant bits from each 8-bit element in \a __a,
4284	4282	/// written to bits [15:0]. The other bits are assigned zeros.
4285		static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_epi8(__m128i __a) {
	4283	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	4284	_mm_movemask_epi8(__m128i __a) {
4286	4285	return __builtin_ia32_pmovmskb128((__v16qi)__a);
4287	4286	}
4288	4287
...	...	@@ -4415,8 +4414,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_epi8(__m128i __a) {
4415	4414	/// Bits [119:112] are written to bits [111:104] of the result. \n
4416	4415	/// Bits [127:120] are written to bits [127:120] of the result.
4417	4416	/// \returns A 128-bit vector of [16 x i8] containing the interleaved values.
4418		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi8(__m128i __a,
4419		__m128i __b) {
	4417	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4418	_mm_unpackhi_epi8(__m128i __a, __m128i __b) {
4420	4419	return (__m128i)__builtin_shufflevector(
4421	4420	(__v16qi)__a, (__v16qi)__b, 8, 16 + 8, 9, 16 + 9, 10, 16 + 10, 11,
4422	4421	16 + 11, 12, 16 + 12, 13, 16 + 13, 14, 16 + 14, 15, 16 + 15);
...	...	@@ -4443,8 +4442,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi8(__m128i __a,
4443	4442	/// Bits [111:96] are written to bits [95:80] of the result. \n
4444	4443	/// Bits [127:112] are written to bits [127:112] of the result.
4445	4444	/// \returns A 128-bit vector of [8 x i16] containing the interleaved values.
4446		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi16(__m128i __a,
4447		__m128i __b) {
	4445	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4446	_mm_unpackhi_epi16(__m128i __a, __m128i __b) {
4448	4447	return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 4, 8 + 4, 5,
4449	4448	8 + 5, 6, 8 + 6, 7, 8 + 7);
4450	4449	}
...	...	@@ -4466,8 +4465,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi16(__m128i __a,
4466	4465	/// Bits [95:64] are written to bits [64:32] of the destination. \n
4467	4466	/// Bits [127:96] are written to bits [127:96] of the destination.
4468	4467	/// \returns A 128-bit vector of [4 x i32] containing the interleaved values.
4469		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi32(__m128i __a,
4470		__m128i __b) {
	4468	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4469	_mm_unpackhi_epi32(__m128i __a, __m128i __b) {
4471	4470	return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 2, 4 + 2, 3,
4472	4471	4 + 3);
4473	4472	}
...	...	@@ -4487,8 +4486,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi32(__m128i __a,
4487	4486	/// A 128-bit vector of [2 x i64]. \n
4488	4487	/// Bits [127:64] are written to bits [127:64] of the destination.
4489	4488	/// \returns A 128-bit vector of [2 x i64] containing the interleaved values.
4490		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi64(__m128i __a,
4491		__m128i __b) {
	4489	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4490	_mm_unpackhi_epi64(__m128i __a, __m128i __b) {
4492	4491	return (__m128i)__builtin_shufflevector((__v2di)__a, (__v2di)__b, 1, 2 + 1);
4493	4492	}
4494	4493
...	...	@@ -4521,8 +4520,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi64(__m128i __a,
4521	4520	/// Bits [55:48] are written to bits [111:104] of the result. \n
4522	4521	/// Bits [63:56] are written to bits [127:120] of the result.
4523	4522	/// \returns A 128-bit vector of [16 x i8] containing the interleaved values.
4524		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi8(__m128i __a,
4525		__m128i __b) {
	4523	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4524	_mm_unpacklo_epi8(__m128i __a, __m128i __b) {
4526	4525	return (__m128i)__builtin_shufflevector(
4527	4526	(__v16qi)__a, (__v16qi)__b, 0, 16 + 0, 1, 16 + 1, 2, 16 + 2, 3, 16 + 3, 4,
4528	4527	16 + 4, 5, 16 + 5, 6, 16 + 6, 7, 16 + 7);
...	...	@@ -4550,8 +4549,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi8(__m128i __a,
4550	4549	/// Bits [47:32] are written to bits [95:80] of the result. \n
4551	4550	/// Bits [63:48] are written to bits [127:112] of the result.
4552	4551	/// \returns A 128-bit vector of [8 x i16] containing the interleaved values.
4553		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi16(__m128i __a,
4554		__m128i __b) {
	4552	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4553	_mm_unpacklo_epi16(__m128i __a, __m128i __b) {
4555	4554	return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 0, 8 + 0, 1,
4556	4555	8 + 1, 2, 8 + 2, 3, 8 + 3);
4557	4556	}
...	...	@@ -4573,8 +4572,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi16(__m128i __a,
4573	4572	/// Bits [31:0] are written to bits [64:32] of the destination. \n
4574	4573	/// Bits [63:32] are written to bits [127:96] of the destination.
4575	4574	/// \returns A 128-bit vector of [4 x i32] containing the interleaved values.
4576		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi32(__m128i __a,
4577		__m128i __b) {
	4575	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4576	_mm_unpacklo_epi32(__m128i __a, __m128i __b) {
4578	4577	return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 0, 4 + 0, 1,
4579	4578	4 + 1);
4580	4579	}
...	...	@@ -4594,8 +4593,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi32(__m128i __a,
4594	4593	/// A 128-bit vector of [2 x i64]. \n
4595	4594	/// Bits [63:0] are written to bits [127:64] of the destination. \n
4596	4595	/// \returns A 128-bit vector of [2 x i64] containing the interleaved values.
4597		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi64(__m128i __a,
4598		__m128i __b) {
	4596	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	4597	_mm_unpacklo_epi64(__m128i __a, __m128i __b) {
4599	4598	return (__m128i)__builtin_shufflevector((__v2di)__a, (__v2di)__b, 0, 2 + 0);
4600	4599	}
4601	4600
...	...	@@ -4701,7 +4700,8 @@ _mm_unpacklo_pd(__m128d __a, __m128d __b) {
4701	4700	/// be extracted.
4702	4701	/// \returns The sign bits from each of the double-precision elements in \a __a,
4703	4702	/// written to bits [1:0]. The remaining bits are assigned values of zero.
4704		static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_pd(__m128d __a) {
	4703	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	4704	_mm_movemask_pd(__m128d __a) {
4705	4705	return __builtin_ia32_movmskpd((__v2df)__a);
4706	4706	}
4707	4707

lib/include/f16cintrin.h+23-9

...	...	@@ -15,10 +15,21 @@
15	15	#define __F16CINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
18		#define __DEFAULT_FN_ATTRS128 \
19		__attribute__((__always_inline__, __nodebug__, __target__("f16c"), __min_vector_width__(128)))
20		#define __DEFAULT_FN_ATTRS256 \
21		__attribute__((__always_inline__, __nodebug__, __target__("f16c"), __min_vector_width__(256)))
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	19	#define __DEFAULT_FN_ATTRS128 \
	20	__attribute__((__always_inline__, __nodebug__, __target__("f16c"), \
	21	__min_vector_width__(128))) constexpr
	22	#define __DEFAULT_FN_ATTRS256 \
	23	__attribute__((__always_inline__, __nodebug__, __target__("f16c"), \
	24	__min_vector_width__(256))) constexpr
	25	#else
	26	#define __DEFAULT_FN_ATTRS128 \
	27	__attribute__((__always_inline__, __nodebug__, __target__("f16c"), \
	28	__min_vector_width__(128)))
	29	#define __DEFAULT_FN_ATTRS256 \
	30	__attribute__((__always_inline__, __nodebug__, __target__("f16c"), \
	31	__min_vector_width__(256)))
	32	#endif
22	33
23	34	/* NOTE: Intel documents the 128-bit versions of these as being in emmintrin.h,
24	35	* but that's because icc can emulate these without f16c using a library call.
...	...	@@ -38,9 +49,7 @@
38	49	static __inline float __DEFAULT_FN_ATTRS128
39	50	_cvtsh_ss(unsigned short __a)
40	51	{
41		__v8hi __v = {(short)__a, 0, 0, 0, 0, 0, 0, 0};
42		__v4sf __r = __builtin_ia32_vcvtph2ps(__v);
43		return __r[0];
	52	return (float)__builtin_bit_cast(__fp16, __a);
44	53	}
45	54
46	55	/// Converts a 32-bit single-precision float value to a 16-bit
...	...	@@ -109,7 +118,10 @@ _cvtsh_ss(unsigned short __a)
109	118	static __inline __m128 __DEFAULT_FN_ATTRS128
110	119	_mm_cvtph_ps(__m128i __a)
111	120	{
112		return (__m128)__builtin_ia32_vcvtph2ps((__v8hi)__a);
	121	typedef __fp16 __v4fp16 __attribute__((__vector_size__(8)));
	122
	123	__v4hi __v = __builtin_shufflevector((__v8hi)__a, (__v8hi)__a, 0, 1, 2, 3);
	124	return (__m128) __builtin_convertvector((__v4fp16)__v, __v4sf);
113	125	}
114	126
115	127	/// Converts a 256-bit vector of [8 x float] into a 128-bit vector
...	...	@@ -153,7 +165,9 @@ _mm_cvtph_ps(__m128i __a)
153	165	static __inline __m256 __DEFAULT_FN_ATTRS256
154	166	_mm256_cvtph_ps(__m128i __a)
155	167	{
156		return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)__a);
	168	typedef __fp16 __v8fp16 __attribute__((__vector_size__(16), __aligned__(16)));
	169
	170	return (__m256) __builtin_convertvector((__v8fp16)__a, __v8sf);
157	171	}
158	172
159	173	#undef __DEFAULT_FN_ATTRS128

lib/include/float.h+16-155

...	...	@@ -7,13 +7,21 @@
7	7	*===-----------------------------------------------------------------------===
8	8	*/
9	9
10		#ifndef __CLANG_FLOAT_H
11		#define __CLANG_FLOAT_H
12
13	10	#if defined(__MVS__) && __has_include_next(<float.h>)
	11	#include <__float_header_macro.h>
14	12	#include_next <float.h>
15	13	#else
16	14
	15	#if !defined(__need_infinity_nan)
	16	#define __need_float_float
	17	#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
	18	!defined(__STRICT_ANSI__)
	19	#define __need_infinity_nan
	20	#endif
	21	#include <__float_header_macro.h>
	22	#endif
	23
	24	#ifdef __need_float_float
17	25	/* If we're on MinGW, fall back to the system's float.h, which might have
18	26	* additional definitions provided for Windows.
19	27	* For more details see http://msdn.microsoft.com/en-us/library/y0ybw9fy.aspx
...	...	@@ -26,162 +34,15 @@
26	34
27	35	# include_next <float.h>
28	36
29		/* Undefine anything that we'll be redefining below. */
30		# undef FLT_EVAL_METHOD
31		# undef FLT_ROUNDS
32		# undef FLT_RADIX
33		# undef FLT_MANT_DIG
34		# undef DBL_MANT_DIG
35		# undef LDBL_MANT_DIG
36		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
37		!defined(__STRICT_ANSI__) \|\| \
38		(defined(__cplusplus) && __cplusplus >= 201103L) \|\| \
39		(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
40		# undef DECIMAL_DIG
41		# endif
42		# undef FLT_DIG
43		# undef DBL_DIG
44		# undef LDBL_DIG
45		# undef FLT_MIN_EXP
46		# undef DBL_MIN_EXP
47		# undef LDBL_MIN_EXP
48		# undef FLT_MIN_10_EXP
49		# undef DBL_MIN_10_EXP
50		# undef LDBL_MIN_10_EXP
51		# undef FLT_MAX_EXP
52		# undef DBL_MAX_EXP
53		# undef LDBL_MAX_EXP
54		# undef FLT_MAX_10_EXP
55		# undef DBL_MAX_10_EXP
56		# undef LDBL_MAX_10_EXP
57		# undef FLT_MAX
58		# undef DBL_MAX
59		# undef LDBL_MAX
60		# undef FLT_EPSILON
61		# undef DBL_EPSILON
62		# undef LDBL_EPSILON
63		# undef FLT_MIN
64		# undef DBL_MIN
65		# undef LDBL_MIN
66		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \|\| \
67		!defined(__STRICT_ANSI__) \|\| \
68		(defined(__cplusplus) && __cplusplus >= 201703L) \|\| \
69		(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
70		# undef FLT_TRUE_MIN
71		# undef DBL_TRUE_MIN
72		# undef LDBL_TRUE_MIN
73		# undef FLT_DECIMAL_DIG
74		# undef DBL_DECIMAL_DIG
75		# undef LDBL_DECIMAL_DIG
76		# undef FLT_HAS_SUBNORM
77		# undef DBL_HAS_SUBNORM
78		# undef LDBL_HAS_SUBNORM
79		# endif
80		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
81		!defined(__STRICT_ANSI__)
82		# undef FLT_NORM_MAX
83		# undef DBL_NORM_MAX
84		# undef LDBL_NORM_MAX
85		#endif
86		#endif
87
88		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
89		!defined(__STRICT_ANSI__)
90		# undef INFINITY
91		# undef NAN
92		#endif
93
94		/* Characteristics of floating point types, C99 5.2.4.2.2 */
95
96		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
97		(defined(__cplusplus) && __cplusplus >= 201103L)
98		#define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
99	37	#endif
100		#define FLT_ROUNDS (__builtin_flt_rounds())
101		#define FLT_RADIX __FLT_RADIX__
102
103		#define FLT_MANT_DIG __FLT_MANT_DIG__
104		#define DBL_MANT_DIG __DBL_MANT_DIG__
105		#define LDBL_MANT_DIG __LDBL_MANT_DIG__
106	38
107		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \|\| \
108		!defined(__STRICT_ANSI__) \|\| \
109		(defined(__cplusplus) && __cplusplus >= 201103L) \|\| \
110		(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
111		# define DECIMAL_DIG __DECIMAL_DIG__
	39	#include <__float_float.h>
	40	#undef __need_float_float
112	41	#endif
113	42
114		#define FLT_DIG __FLT_DIG__
115		#define DBL_DIG __DBL_DIG__
116		#define LDBL_DIG __LDBL_DIG__
117
118		#define FLT_MIN_EXP __FLT_MIN_EXP__
119		#define DBL_MIN_EXP __DBL_MIN_EXP__
120		#define LDBL_MIN_EXP __LDBL_MIN_EXP__
121
122		#define FLT_MIN_10_EXP __FLT_MIN_10_EXP__
123		#define DBL_MIN_10_EXP __DBL_MIN_10_EXP__
124		#define LDBL_MIN_10_EXP __LDBL_MIN_10_EXP__
125
126		#define FLT_MAX_EXP __FLT_MAX_EXP__
127		#define DBL_MAX_EXP __DBL_MAX_EXP__
128		#define LDBL_MAX_EXP __LDBL_MAX_EXP__
129
130		#define FLT_MAX_10_EXP __FLT_MAX_10_EXP__
131		#define DBL_MAX_10_EXP __DBL_MAX_10_EXP__
132		#define LDBL_MAX_10_EXP __LDBL_MAX_10_EXP__
133
134		#define FLT_MAX __FLT_MAX__
135		#define DBL_MAX __DBL_MAX__
136		#define LDBL_MAX __LDBL_MAX__
137
138		#define FLT_EPSILON __FLT_EPSILON__
139		#define DBL_EPSILON __DBL_EPSILON__
140		#define LDBL_EPSILON __LDBL_EPSILON__
141
142		#define FLT_MIN __FLT_MIN__
143		#define DBL_MIN __DBL_MIN__
144		#define LDBL_MIN __LDBL_MIN__
145
146		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \|\| \
147		!defined(__STRICT_ANSI__) \|\| \
148		(defined(__cplusplus) && __cplusplus >= 201703L) \|\| \
149		(__STDC_HOSTED__ && defined(_AIX) && defined(_ALL_SOURCE))
150		# define FLT_TRUE_MIN __FLT_DENORM_MIN__
151		# define DBL_TRUE_MIN __DBL_DENORM_MIN__
152		# define LDBL_TRUE_MIN __LDBL_DENORM_MIN__
153		# define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__
154		# define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__
155		# define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__
156		# define FLT_HAS_SUBNORM __FLT_HAS_DENORM__
157		# define DBL_HAS_SUBNORM __DBL_HAS_DENORM__
158		# define LDBL_HAS_SUBNORM __LDBL_HAS_DENORM__
	43	#ifdef __need_infinity_nan
	44	#include <__float_infinity_nan.h>
	45	#undef __need_infinity_nan
159	46	#endif
160	47
161		#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \|\| \
162		!defined(__STRICT_ANSI__)
163		/* C23 5.2.5.3.3p29-30 */
164		# define INFINITY (__builtin_inff())
165		# define NAN (__builtin_nanf(""))
166		/* C23 5.2.5.3.3p32 */
167		# define FLT_NORM_MAX __FLT_NORM_MAX__
168		# define DBL_NORM_MAX __DBL_NORM_MAX__
169		# define LDBL_NORM_MAX __LDBL_NORM_MAX__
170		#endif
171
172		#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__
173		# define FLT16_MANT_DIG __FLT16_MANT_DIG__
174		# define FLT16_DECIMAL_DIG __FLT16_DECIMAL_DIG__
175		# define FLT16_DIG __FLT16_DIG__
176		# define FLT16_MIN_EXP __FLT16_MIN_EXP__
177		# define FLT16_MIN_10_EXP __FLT16_MIN_10_EXP__
178		# define FLT16_MAX_EXP __FLT16_MAX_EXP__
179		# define FLT16_MAX_10_EXP __FLT16_MAX_10_EXP__
180		# define FLT16_MAX __FLT16_MAX__
181		# define FLT16_EPSILON __FLT16_EPSILON__
182		# define FLT16_MIN __FLT16_MIN__
183		# define FLT16_TRUE_MIN __FLT16_TRUE_MIN__
184		#endif /* __STDC_WANT_IEC_60559_TYPES_EXT__ */
185
186	48	#endif /* __MVS__ */
187		#endif /* __CLANG_FLOAT_H */

lib/include/fma4intrin.h+98-96

...	...	@@ -20,100 +20,100 @@
20	20	#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma4"), __min_vector_width__(128)))
21	21	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma4"), __min_vector_width__(256)))
22	22
23		static __inline__ __m128 __DEFAULT_FN_ATTRS128
24		_mm_macc_ps(__m128 __A, __m128 __B, __m128 __C)
25		{
26		return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	23	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	24	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	25	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	26	#else
	27	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	28	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	29	#endif
	30
	31	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	32	_mm_macc_ps(__m128 __A, __m128 __B, __m128 __C) {
	33	return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
	34	(__v4sf)__C);
27	35	}
28	36
29		static __inline__ __m128d __DEFAULT_FN_ATTRS128
30		_mm_macc_pd(__m128d __A, __m128d __B, __m128d __C)
31		{
32		return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
	37	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	38	_mm_macc_pd(__m128d __A, __m128d __B, __m128d __C) {
	39	return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
	40	(__v2df)__C);
33	41	}
34	42
35		static __inline__ __m128 __DEFAULT_FN_ATTRS128
36		_mm_macc_ss(__m128 __A, __m128 __B, __m128 __C)
37		{
38		return (__m128)__builtin_ia32_vfmaddss((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	43	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	44	_mm_macc_ss(__m128 __A, __m128 __B, __m128 __C) {
	45	return _mm_set_ss(__builtin_elementwise_fma(__A[0], __B[0], __C[0]));
39	46	}
40	47
41		static __inline__ __m128d __DEFAULT_FN_ATTRS128
42		_mm_macc_sd(__m128d __A, __m128d __B, __m128d __C)
43		{
44		return (__m128d)__builtin_ia32_vfmaddsd((__v2df)__A, (__v2df)__B, (__v2df)__C);
	48	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	49	_mm_macc_sd(__m128d __A, __m128d __B, __m128d __C) {
	50	return _mm_set_sd(__builtin_elementwise_fma(__A[0], __B[0], __C[0]));
45	51	}
46	52
47		static __inline__ __m128 __DEFAULT_FN_ATTRS128
48		_mm_msub_ps(__m128 __A, __m128 __B, __m128 __C)
49		{
50		return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	53	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	54	_mm_msub_ps(__m128 __A, __m128 __B, __m128 __C) {
	55	return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
	56	-(__v4sf)__C);
51	57	}
52	58
53		static __inline__ __m128d __DEFAULT_FN_ATTRS128
54		_mm_msub_pd(__m128d __A, __m128d __B, __m128d __C)
55		{
56		return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
	59	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	60	_mm_msub_pd(__m128d __A, __m128d __B, __m128d __C) {
	61	return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
	62	-(__v2df)__C);
57	63	}
58	64
59		static __inline__ __m128 __DEFAULT_FN_ATTRS128
60		_mm_msub_ss(__m128 __A, __m128 __B, __m128 __C)
61		{
62		return (__m128)__builtin_ia32_vfmaddss((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	65	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	66	_mm_msub_ss(__m128 __A, __m128 __B, __m128 __C) {
	67	return _mm_set_ss(__builtin_elementwise_fma(__A[0], __B[0], -__C[0]));
63	68	}
64	69
65		static __inline__ __m128d __DEFAULT_FN_ATTRS128
66		_mm_msub_sd(__m128d __A, __m128d __B, __m128d __C)
67		{
68		return (__m128d)__builtin_ia32_vfmaddsd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
	70	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	71	_mm_msub_sd(__m128d __A, __m128d __B, __m128d __C) {
	72	return _mm_set_sd(__builtin_elementwise_fma(__A[0], __B[0], -__C[0]));
69	73	}
70	74
71		static __inline__ __m128 __DEFAULT_FN_ATTRS128
72		_mm_nmacc_ps(__m128 __A, __m128 __B, __m128 __C)
73		{
74		return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	75	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	76	_mm_nmacc_ps(__m128 __A, __m128 __B, __m128 __C) {
	77	return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
	78	(__v4sf)__C);
75	79	}
76	80
77		static __inline__ __m128d __DEFAULT_FN_ATTRS128
78		_mm_nmacc_pd(__m128d __A, __m128d __B, __m128d __C)
79		{
80		return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
	81	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	82	_mm_nmacc_pd(__m128d __A, __m128d __B, __m128d __C) {
	83	return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
	84	(__v2df)__C);
81	85	}
82	86
83		static __inline__ __m128 __DEFAULT_FN_ATTRS128
84		_mm_nmacc_ss(__m128 __A, __m128 __B, __m128 __C)
85		{
86		return (__m128)__builtin_ia32_vfmaddss(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	87	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	88	_mm_nmacc_ss(__m128 __A, __m128 __B, __m128 __C) {
	89	return _mm_set_ss(__builtin_elementwise_fma(-__A[0], __B[0], __C[0]));
87	90	}
88	91
89		static __inline__ __m128d __DEFAULT_FN_ATTRS128
90		_mm_nmacc_sd(__m128d __A, __m128d __B, __m128d __C)
91		{
92		return (__m128d)__builtin_ia32_vfmaddsd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
	92	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	93	_mm_nmacc_sd(__m128d __A, __m128d __B, __m128d __C) {
	94	return _mm_set_sd(__builtin_elementwise_fma(-__A[0], __B[0], __C[0]));
93	95	}
94	96
95		static __inline__ __m128 __DEFAULT_FN_ATTRS128
96		_mm_nmsub_ps(__m128 __A, __m128 __B, __m128 __C)
97		{
98		return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	97	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	98	_mm_nmsub_ps(__m128 __A, __m128 __B, __m128 __C) {
	99	return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
	100	-(__v4sf)__C);
99	101	}
100	102
101		static __inline__ __m128d __DEFAULT_FN_ATTRS128
102		_mm_nmsub_pd(__m128d __A, __m128d __B, __m128d __C)
103		{
104		return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
	103	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	104	_mm_nmsub_pd(__m128d __A, __m128d __B, __m128d __C) {
	105	return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
	106	-(__v2df)__C);
105	107	}
106	108
107		static __inline__ __m128 __DEFAULT_FN_ATTRS128
108		_mm_nmsub_ss(__m128 __A, __m128 __B, __m128 __C)
109		{
110		return (__m128)__builtin_ia32_vfmaddss(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	109	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	110	_mm_nmsub_ss(__m128 __A, __m128 __B, __m128 __C) {
	111	return _mm_set_ss(__builtin_elementwise_fma(-__A[0], __B[0], -__C[0]));
111	112	}
112	113
113		static __inline__ __m128d __DEFAULT_FN_ATTRS128
114		_mm_nmsub_sd(__m128d __A, __m128d __B, __m128d __C)
115		{
116		return (__m128d)__builtin_ia32_vfmaddsd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
	114	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	115	_mm_nmsub_sd(__m128d __A, __m128d __B, __m128d __C) {
	116	return _mm_set_sd(__builtin_elementwise_fma(-__A[0], __B[0], -__C[0]));
117	117	}
118	118
119	119	static __inline__ __m128 __DEFAULT_FN_ATTRS128
...	...	@@ -140,52 +140,52 @@ _mm_msubadd_pd(__m128d __A, __m128d __B, __m128d __C)
140	140	return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
141	141	}
142	142
143		static __inline__ __m256 __DEFAULT_FN_ATTRS256
144		_mm256_macc_ps(__m256 __A, __m256 __B, __m256 __C)
145		{
146		return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
	143	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	144	_mm256_macc_ps(__m256 __A, __m256 __B, __m256 __C) {
	145	return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
	146	(__v8sf)__C);
147	147	}
148	148
149		static __inline__ __m256d __DEFAULT_FN_ATTRS256
150		_mm256_macc_pd(__m256d __A, __m256d __B, __m256d __C)
151		{
152		return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
	149	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	150	_mm256_macc_pd(__m256d __A, __m256d __B, __m256d __C) {
	151	return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
	152	(__v4df)__C);
153	153	}
154	154
155		static __inline__ __m256 __DEFAULT_FN_ATTRS256
156		_mm256_msub_ps(__m256 __A, __m256 __B, __m256 __C)
157		{
158		return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
	155	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	156	_mm256_msub_ps(__m256 __A, __m256 __B, __m256 __C) {
	157	return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
	158	-(__v8sf)__C);
159	159	}
160	160
161		static __inline__ __m256d __DEFAULT_FN_ATTRS256
162		_mm256_msub_pd(__m256d __A, __m256d __B, __m256d __C)
163		{
164		return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
	161	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	162	_mm256_msub_pd(__m256d __A, __m256d __B, __m256d __C) {
	163	return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
	164	-(__v4df)__C);
165	165	}
166	166
167		static __inline__ __m256 __DEFAULT_FN_ATTRS256
168		_mm256_nmacc_ps(__m256 __A, __m256 __B, __m256 __C)
169		{
170		return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
	167	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	168	_mm256_nmacc_ps(__m256 __A, __m256 __B, __m256 __C) {
	169	return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
	170	(__v8sf)__C);
171	171	}
172	172
173		static __inline__ __m256d __DEFAULT_FN_ATTRS256
174		_mm256_nmacc_pd(__m256d __A, __m256d __B, __m256d __C)
175		{
176		return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
	173	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	174	_mm256_nmacc_pd(__m256d __A, __m256d __B, __m256d __C) {
	175	return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
	176	(__v4df)__C);
177	177	}
178	178
179		static __inline__ __m256 __DEFAULT_FN_ATTRS256
180		_mm256_nmsub_ps(__m256 __A, __m256 __B, __m256 __C)
181		{
182		return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
	179	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
	180	_mm256_nmsub_ps(__m256 __A, __m256 __B, __m256 __C) {
	181	return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
	182	-(__v8sf)__C);
183	183	}
184	184
185		static __inline__ __m256d __DEFAULT_FN_ATTRS256
186		_mm256_nmsub_pd(__m256d __A, __m256d __B, __m256d __C)
187		{
188		return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
	185	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
	186	_mm256_nmsub_pd(__m256d __A, __m256d __B, __m256d __C) {
	187	return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
	188	-(__v4df)__C);
189	189	}
190	190
191	191	static __inline__ __m256 __DEFAULT_FN_ATTRS256
...	...	@@ -214,5 +214,7 @@ _mm256_msubadd_pd(__m256d __A, __m256d __B, __m256d __C)
214	214
215	215	#undef __DEFAULT_FN_ATTRS128
216	216	#undef __DEFAULT_FN_ATTRS256
	217	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	218	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
217	219
218	220	#endif /* __FMA4INTRIN_H */

lib/include/fmaintrin.h+96-66

...	...	@@ -15,8 +15,20 @@
15	15	#define __FMAINTRIN_H
16	16
17	17	/* Define the default attributes for the functions in this file. */
18		#define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128)))
19		#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256)))
	18	#define __DEFAULT_FN_ATTRS128 \
	19	__attribute__((__always_inline__, __nodebug__, __target__("fma"), \
	20	__min_vector_width__(128)))
	21	#define __DEFAULT_FN_ATTRS256 \
	22	__attribute__((__always_inline__, __nodebug__, __target__("fma"), \
	23	__min_vector_width__(256)))
	24
	25	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	26	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
	27	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	28	#else
	29	#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
	30	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	31	#endif
20	32
21	33	/// Computes a multiply-add of 128-bit vectors of [4 x float].
22	34	/// For each element, computes <c> (__A * __B) + __C </c>.
...	...	@@ -32,10 +44,11 @@
32	44	/// \param __C
33	45	/// A 128-bit vector of [4 x float] containing the addend.
34	46	/// \returns A 128-bit vector of [4 x float] containing the result.
35		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	47	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
36	48	_mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
37	49	{
38		return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	50	return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
	51	(__v4sf)__C);
39	52	}
40	53
41	54	/// Computes a multiply-add of 128-bit vectors of [2 x double].
...	...	@@ -52,10 +65,11 @@ _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
52	65	/// \param __C
53	66	/// A 128-bit vector of [2 x double] containing the addend.
54	67	/// \returns A 128-bit [2 x double] vector containing the result.
55		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	68	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
56	69	_mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C)
57	70	{
58		return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
	71	return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
	72	(__v2df)__C);
59	73	}
60	74
61	75	/// Computes a scalar multiply-add of the single-precision values in the
...	...	@@ -81,10 +95,10 @@ _mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C)
81	95	/// 32 bits.
82	96	/// \returns A 128-bit vector of [4 x float] containing the result in the low
83	97	/// 32 bits and a copy of \a __A[127:32] in the upper 96 bits.
84		static __inline__ __m128 __DEFAULT_FN_ATTRS128
85		_mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C)
86		{
87		return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	98	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	99	_mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C) {
	100	__A[0] = __builtin_elementwise_fma(__A[0], __B[0], __C[0]);
	101	return __A;
88	102	}
89	103
90	104	/// Computes a scalar multiply-add of the double-precision values in the
...	...	@@ -110,10 +124,10 @@ _mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C)
110	124	/// 64 bits.
111	125	/// \returns A 128-bit vector of [2 x double] containing the result in the low
112	126	/// 64 bits and a copy of \a __A[127:64] in the upper 64 bits.
113		static __inline__ __m128d __DEFAULT_FN_ATTRS128
114		_mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C)
115		{
116		return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, (__v2df)__C);
	127	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	128	_mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C) {
	129	__A[0] = __builtin_elementwise_fma(__A[0], __B[0], __C[0]);
	130	return __A;
117	131	}
118	132
119	133	/// Computes a multiply-subtract of 128-bit vectors of [4 x float].
...	...	@@ -130,10 +144,11 @@ _mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C)
130	144	/// \param __C
131	145	/// A 128-bit vector of [4 x float] containing the subtrahend.
132	146	/// \returns A 128-bit vector of [4 x float] containing the result.
133		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	147	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
134	148	_mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
135	149	{
136		return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	150	return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
	151	-(__v4sf)__C);
137	152	}
138	153
139	154	/// Computes a multiply-subtract of 128-bit vectors of [2 x double].
...	...	@@ -150,10 +165,11 @@ _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
150	165	/// \param __C
151	166	/// A 128-bit vector of [2 x double] containing the addend.
152	167	/// \returns A 128-bit vector of [2 x double] containing the result.
153		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	168	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
154	169	_mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C)
155	170	{
156		return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
	171	return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
	172	-(__v2df)__C);
157	173	}
158	174
159	175	/// Computes a scalar multiply-subtract of the single-precision values in
...	...	@@ -179,10 +195,10 @@ _mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C)
179	195	/// 32 bits.
180	196	/// \returns A 128-bit vector of [4 x float] containing the result in the low
181	197	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
182		static __inline__ __m128 __DEFAULT_FN_ATTRS128
183		_mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C)
184		{
185		return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	198	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	199	_mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C) {
	200	__A[0] = __builtin_elementwise_fma(__A[0], __B[0], -__C[0]);
	201	return __A;
186	202	}
187	203
188	204	/// Computes a scalar multiply-subtract of the double-precision values in
...	...	@@ -208,10 +224,10 @@ _mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C)
208	224	/// 64 bits.
209	225	/// \returns A 128-bit vector of [2 x double] containing the result in the low
210	226	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
211		static __inline__ __m128d __DEFAULT_FN_ATTRS128
212		_mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C)
213		{
214		return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, -(__v2df)__C);
	227	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	228	_mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C) {
	229	__A[0] = __builtin_elementwise_fma(__A[0], __B[0], -__C[0]);
	230	return __A;
215	231	}
216	232
217	233	/// Computes a negated multiply-add of 128-bit vectors of [4 x float].
...	...	@@ -228,10 +244,11 @@ _mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C)
228	244	/// \param __C
229	245	/// A 128-bit vector of [4 x float] containing the addend.
230	246	/// \returns A 128-bit [4 x float] vector containing the result.
231		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	247	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
232	248	_mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
233	249	{
234		return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
	250	return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
	251	(__v4sf)__C);
235	252	}
236	253
237	254	/// Computes a negated multiply-add of 128-bit vectors of [2 x double].
...	...	@@ -248,10 +265,11 @@ _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
248	265	/// \param __C
249	266	/// A 128-bit vector of [2 x double] containing the addend.
250	267	/// \returns A 128-bit vector of [2 x double] containing the result.
251		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	268	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
252	269	_mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C)
253	270	{
254		return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
	271	return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
	272	(__v2df)__C);
255	273	}
256	274
257	275	/// Computes a scalar negated multiply-add of the single-precision values in
...	...	@@ -277,10 +295,10 @@ _mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C)
277	295	/// 32 bits.
278	296	/// \returns A 128-bit vector of [4 x float] containing the result in the low
279	297	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
280		static __inline__ __m128 __DEFAULT_FN_ATTRS128
281		_mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C)
282		{
283		return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C);
	298	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	299	_mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C) {
	300	__A[0] = __builtin_elementwise_fma(__A[0], -__B[0], __C[0]);
	301	return __A;
284	302	}
285	303
286	304	/// Computes a scalar negated multiply-add of the double-precision values
...	...	@@ -306,10 +324,10 @@ _mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C)
306	324	/// 64 bits.
307	325	/// \returns A 128-bit vector of [2 x double] containing the result in the low
308	326	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
309		static __inline__ __m128d __DEFAULT_FN_ATTRS128
310		_mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C)
311		{
312		return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, (__v2df)__C);
	327	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	328	_mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C) {
	329	__A[0] = __builtin_elementwise_fma(__A[0], -__B[0], __C[0]);
	330	return __A;
313	331	}
314	332
315	333	/// Computes a negated multiply-subtract of 128-bit vectors of [4 x float].
...	...	@@ -326,10 +344,11 @@ _mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C)
326	344	/// \param __C
327	345	/// A 128-bit vector of [4 x float] containing the subtrahend.
328	346	/// \returns A 128-bit vector of [4 x float] containing the result.
329		static __inline__ __m128 __DEFAULT_FN_ATTRS128
	347	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
330	348	_mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
331	349	{
332		return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
	350	return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
	351	-(__v4sf)__C);
333	352	}
334	353
335	354	/// Computes a negated multiply-subtract of 128-bit vectors of [2 x double].
...	...	@@ -346,10 +365,11 @@ _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
346	365	/// \param __C
347	366	/// A 128-bit vector of [2 x double] containing the subtrahend.
348	367	/// \returns A 128-bit vector of [2 x double] containing the result.
349		static __inline__ __m128d __DEFAULT_FN_ATTRS128
	368	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
350	369	_mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C)
351	370	{
352		return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
	371	return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
	372	-(__v2df)__C);
353	373	}
354	374
355	375	/// Computes a scalar negated multiply-subtract of the single-precision
...	...	@@ -375,10 +395,10 @@ _mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C)
375	395	/// 32 bits.
376	396	/// \returns A 128-bit vector of [4 x float] containing the result in the low
377	397	/// 32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
378		static __inline__ __m128 __DEFAULT_FN_ATTRS128
379		_mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C)
380		{
381		return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C);
	398	static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
	399	_mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C) {
	400	__A[0] = __builtin_elementwise_fma(__A[0], -__B[0], -__C[0]);
	401	return __A;
382	402	}
383	403
384	404	/// Computes a scalar negated multiply-subtract of the double-precision
...	...	@@ -404,10 +424,10 @@ _mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C)
404	424	/// 64 bits.
405	425	/// \returns A 128-bit vector of [2 x double] containing the result in the low
406	426	/// 64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
407		static __inline__ __m128d __DEFAULT_FN_ATTRS128
408		_mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C)
409		{
410		return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, -(__v2df)__C);
	427	static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
	428	_mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C) {
	429	__A[0] = __builtin_elementwise_fma(__A[0], -__B[0], -__C[0]);
	430	return __A;
411	431	}
412	432
413	433	/// Computes a multiply with alternating add/subtract of 128-bit vectors of
...	...	@@ -528,10 +548,11 @@ _mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C)
528	548	/// \param __C
529	549	/// A 256-bit vector of [8 x float] containing the addend.
530	550	/// \returns A 256-bit vector of [8 x float] containing the result.
531		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	551	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
532	552	_mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
533	553	{
534		return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
	554	return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
	555	(__v8sf)__C);
535	556	}
536	557
537	558	/// Computes a multiply-add of 256-bit vectors of [4 x double].
...	...	@@ -548,10 +569,11 @@ _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
548	569	/// \param __C
549	570	/// A 256-bit vector of [4 x double] containing the addend.
550	571	/// \returns A 256-bit vector of [4 x double] containing the result.
551		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	572	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
552	573	_mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
553	574	{
554		return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
	575	return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
	576	(__v4df)__C);
555	577	}
556	578
557	579	/// Computes a multiply-subtract of 256-bit vectors of [8 x float].
...	...	@@ -568,10 +590,11 @@ _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
568	590	/// \param __C
569	591	/// A 256-bit vector of [8 x float] containing the subtrahend.
570	592	/// \returns A 256-bit vector of [8 x float] containing the result.
571		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	593	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
572	594	_mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
573	595	{
574		return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
	596	return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
	597	-(__v8sf)__C);
575	598	}
576	599
577	600	/// Computes a multiply-subtract of 256-bit vectors of [4 x double].
...	...	@@ -588,10 +611,11 @@ _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
588	611	/// \param __C
589	612	/// A 256-bit vector of [4 x double] containing the subtrahend.
590	613	/// \returns A 256-bit vector of [4 x double] containing the result.
591		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	614	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
592	615	_mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
593	616	{
594		return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
	617	return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
	618	-(__v4df)__C);
595	619	}
596	620
597	621	/// Computes a negated multiply-add of 256-bit vectors of [8 x float].
...	...	@@ -608,10 +632,11 @@ _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
608	632	/// \param __C
609	633	/// A 256-bit vector of [8 x float] containing the addend.
610	634	/// \returns A 256-bit vector of [8 x float] containing the result.
611		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	635	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
612	636	_mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
613	637	{
614		return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
	638	return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
	639	(__v8sf)__C);
615	640	}
616	641
617	642	/// Computes a negated multiply-add of 256-bit vectors of [4 x double].
...	...	@@ -628,10 +653,11 @@ _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
628	653	/// \param __C
629	654	/// A 256-bit vector of [4 x double] containing the addend.
630	655	/// \returns A 256-bit vector of [4 x double] containing the result.
631		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	656	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
632	657	_mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
633	658	{
634		return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
	659	return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
	660	(__v4df)__C);
635	661	}
636	662
637	663	/// Computes a negated multiply-subtract of 256-bit vectors of [8 x float].
...	...	@@ -648,10 +674,11 @@ _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
648	674	/// \param __C
649	675	/// A 256-bit vector of [8 x float] containing the subtrahend.
650	676	/// \returns A 256-bit vector of [8 x float] containing the result.
651		static __inline__ __m256 __DEFAULT_FN_ATTRS256
	677	static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
652	678	_mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
653	679	{
654		return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
	680	return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
	681	-(__v8sf)__C);
655	682	}
656	683
657	684	/// Computes a negated multiply-subtract of 256-bit vectors of [4 x double].
...	...	@@ -668,10 +695,11 @@ _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
668	695	/// \param __C
669	696	/// A 256-bit vector of [4 x double] containing the subtrahend.
670	697	/// \returns A 256-bit vector of [4 x double] containing the result.
671		static __inline__ __m256d __DEFAULT_FN_ATTRS256
	698	static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
672	699	_mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C)
673	700	{
674		return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
	701	return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
	702	-(__v4df)__C);
675	703	}
676	704
677	705	/// Computes a multiply with alternating add/subtract of 256-bit vectors of
...	...	@@ -792,5 +820,7 @@ _mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C)
792	820
793	821	#undef __DEFAULT_FN_ATTRS128
794	822	#undef __DEFAULT_FN_ATTRS256
	823	#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
	824	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
795	825
796	826	#endif /* __FMAINTRIN_H */

lib/include/gfniintrin.h+21-16

...	...	@@ -14,29 +14,36 @@
14	14	#ifndef __GFNIINTRIN_H
15	15	#define __GFNIINTRIN_H
16	16
17		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
18	17	/* Default attributes for simple form (no masking). */
	18	#if defined(__cplusplus) && (__cplusplus >= 201103L)
19	19	#define __DEFAULT_FN_ATTRS \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("gfni,no-evex512"), __min_vector_width__(128)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("gfni"), \
	21	__min_vector_width__(128))) constexpr
22	22
23	23	/* Default attributes for YMM unmasked form. */
24	24	#define __DEFAULT_FN_ATTRS_Y \
25		__attribute__((__always_inline__, __nodebug__, \
26		__target__("avx,gfni,no-evex512"), \
27		__min_vector_width__(256)))
	25	__attribute__((__always_inline__, __nodebug__, __target__("avx,gfni"), \
	26	__min_vector_width__(256))) constexpr
28	27
29	28	/* Default attributes for VLX masked forms. */
30	29	#define __DEFAULT_FN_ATTRS_VL128 \
31	30	__attribute__((__always_inline__, __nodebug__, \
32		__target__("avx512bw,avx512vl,gfni,no-evex512"), \
33		__min_vector_width__(128)))
	31	__target__("avx512bw,avx512vl,gfni"), \
	32	__min_vector_width__(128))) constexpr
34	33	#define __DEFAULT_FN_ATTRS_VL256 \
35	34	__attribute__((__always_inline__, __nodebug__, \
36		__target__("avx512bw,avx512vl,gfni,no-evex512"), \
37		__min_vector_width__(256)))
	35	__target__("avx512bw,avx512vl,gfni"), \
	36	__min_vector_width__(256))) constexpr
	37
	38	/* Default attributes for ZMM unmasked forms. */
	39	#define __DEFAULT_FN_ATTRS_Z \
	40	__attribute__((__always_inline__, __nodebug__, __target__("avx512f,gfni"), \
	41	__min_vector_width__(512))) constexpr
	42	/* Default attributes for ZMM masked forms. */
	43	#define __DEFAULT_FN_ATTRS_Z_MASK \
	44	__attribute__((__always_inline__, __nodebug__, __target__("avx512bw,gfni"), \
	45	__min_vector_width__(512))) constexpr
38	46	#else
39		/* Default attributes for simple form (no masking). */
40	47	#define __DEFAULT_FN_ATTRS \
41	48	__attribute__((__always_inline__, __nodebug__, __target__("gfni"), \
42	49	__min_vector_width__(128)))
...	...	@@ -55,18 +62,16 @@
55	62	__attribute__((__always_inline__, __nodebug__, \
56	63	__target__("avx512bw,avx512vl,gfni"), \
57	64	__min_vector_width__(256)))
58		#endif
59	65
60	66	/* Default attributes for ZMM unmasked forms. */
61	67	#define __DEFAULT_FN_ATTRS_Z \
62		__attribute__((__always_inline__, __nodebug__, \
63		__target__("avx512f,evex512,gfni"), \
	68	__attribute__((__always_inline__, __nodebug__, __target__("avx512f,gfni"), \
64	69	__min_vector_width__(512)))
65	70	/* Default attributes for ZMM masked forms. */
66	71	#define __DEFAULT_FN_ATTRS_Z_MASK \
67		__attribute__((__always_inline__, __nodebug__, \
68		__target__("avx512bw,evex512,gfni"), \
	72	__attribute__((__always_inline__, __nodebug__, __target__("avx512bw,gfni"), \
69	73	__min_vector_width__(512)))
	74	#endif
70	75
71	76	#define _mm_gf2p8affineinv_epi64_epi8(A, B, I) \
72	77	((__m128i)__builtin_ia32_vgf2p8affineinvqb_v16qi((__v16qi)(__m128i)(A), \

lib/include/hexagon_types.h+23-20

...	...	@@ -11,6 +11,11 @@
11	11
12	12	#include <hexagon_protos.h>
13	13
	14	// Save and undefine B0 to avoid conflicts with POSIX termios.h which
	15	// defines B0 as a macro for baud rate 0.
	16	#pragma push_macro("B0")
	17	#undef B0
	18
14	19	/* Hexagon names */
15	20	#define HEXAGON_Vect HEXAGON_Vect64
16	21	#define HEXAGON_V_GET_D HEXAGON_V64_GET_D
...	...	@@ -697,9 +702,8 @@ public:
697	702	};
698	703
699	704	// Extract byte methods
700		signed char B0(void) {
701		return HEXAGON_V64_GET_B0(data);
702		};
	705	signed char b0(void) { return HEXAGON_V64_GET_B0(data); };
	706	signed char B0(void) { return b0(); };
703	707	signed char B1(void) {
704	708	return HEXAGON_V64_GET_B1(data);
705	709	};
...	...	@@ -776,9 +780,10 @@ public:
776	780	};
777	781
778	782	// Set byte methods
779		HEXAGON_Vect64C B0(signed char b) {
	783	HEXAGON_Vect64C b0(signed char b) {
780	784	return HEXAGON_Vect64C(HEXAGON_V64_PUT_B0(data, b));
781	785	};
	786	HEXAGON_Vect64C B0(signed char b) { return b0(b); };
782	787	HEXAGON_Vect64C B1(signed char b) {
783	788	return HEXAGON_Vect64C(HEXAGON_V64_PUT_B1(data, b));
784	789	};
...	...	@@ -1121,9 +1126,8 @@ public:
1121	1126	};
1122	1127
1123	1128	// Extract byte methods
1124		signed char B0(void) {
1125		return HEXAGON_V32_GET_B0(data);
1126		};
	1129	signed char b0(void) { return HEXAGON_V32_GET_B0(data); };
	1130	signed char B0(void) { return b0(); };
1127	1131	signed char B1(void) {
1128	1132	return HEXAGON_V32_GET_B1(data);
1129	1133	};
...	...	@@ -1162,9 +1166,10 @@ public:
1162	1166	};
1163	1167
1164	1168	// Set byte methods
1165		HEXAGON_Vect32C B0(signed char b) {
	1169	HEXAGON_Vect32C b0(signed char b) {
1166	1170	return HEXAGON_Vect32C(HEXAGON_V32_PUT_B0(data, b));
1167	1171	};
	1172	HEXAGON_Vect32C B0(signed char b) { return b0(b); };
1168	1173	HEXAGON_Vect32C B1(signed char b) {
1169	1174	return HEXAGON_Vect32C(HEXAGON_V32_PUT_B1(data, b));
1170	1175	};
...	...	@@ -1924,9 +1929,8 @@ public:
1924	1929	};
1925	1930
1926	1931	// Extract byte methods
1927		signed char B0(void) {
1928		return Q6V64_GET_B0(data);
1929		};
	1932	signed char b0(void) { return Q6V64_GET_B0(data); };
	1933	signed char B0(void) { return b0(); };
1930	1934	signed char B1(void) {
1931	1935	return Q6V64_GET_B1(data);
1932	1936	};
...	...	@@ -2003,9 +2007,8 @@ public:
2003	2007	};
2004	2008
2005	2009	// Set byte methods
2006		Q6Vect64C B0(signed char b) {
2007		return Q6Vect64C(Q6V64_PUT_B0(data, b));
2008		};
	2010	Q6Vect64C b0(signed char b) { return Q6Vect64C(Q6V64_PUT_B0(data, b)); };
	2011	Q6Vect64C B0(signed char b) { return b0(b); };
2009	2012	Q6Vect64C B1(signed char b) {
2010	2013	return Q6Vect64C(Q6V64_PUT_B1(data, b));
2011	2014	};
...	...	@@ -2348,9 +2351,8 @@ public:
2348	2351	};
2349	2352
2350	2353	// Extract byte methods
2351		signed char B0(void) {
2352		return Q6V32_GET_B0(data);
2353		};
	2354	signed char b0(void) { return Q6V32_GET_B0(data); };
	2355	signed char B0(void) { return b0(); };
2354	2356	signed char B1(void) {
2355	2357	return Q6V32_GET_B1(data);
2356	2358	};
...	...	@@ -2389,9 +2391,8 @@ public:
2389	2391	};
2390	2392
2391	2393	// Set byte methods
2392		Q6Vect32C B0(signed char b) {
2393		return Q6Vect32C(Q6V32_PUT_B0(data, b));
2394		};
	2394	Q6Vect32C b0(signed char b) { return Q6Vect32C(Q6V32_PUT_B0(data, b)); };
	2395	Q6Vect32C B0(signed char b) { return b0(b); };
2395	2396	Q6Vect32C B1(signed char b) {
2396	2397	return Q6Vect32C(Q6V32_PUT_B1(data, b));
2397	2398	};
...	...	@@ -2622,4 +2623,6 @@ typedef struct hexagon_udma_descriptor_type1_s
2622	2623	unsigned int dstwidthoffset:16;
2623	2624	} hexagon_udma_descriptor_type1_t;
2624	2625
	2626	#pragma pop_macro("B0")
	2627
2625	2628	#endif /* !HEXAGON_TYPES_H */

lib/include/hvx_hexagon_protos.h+649-860

...	...	@@ -19,7 +19,6 @@
19	19	#define __BUILTIN_VECTOR_WRAP(a) a
20	20	#endif
21	21
22		#if __HVX_ARCH__ >= 60
23	22	/* ==========================================================================
24	23	Assembly Syntax: Rd32=vextract(Vu32,Rs32)
25	24	C Intrinsic Prototype: Word32 Q6_R_vextract_VR(HVX_Vector Vu, Word32 Rs)
...	...	@@ -28,9 +27,7 @@
28	27	========================================================================== */
29	28
30	29	#define Q6_R_vextract_VR(Vu,Rs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_extractw)(Vu,Rs)
31		#endif /* __HEXAGON_ARCH___ >= 60 */
32	30
33		#if __HVX_ARCH__ >= 60
34	31	/* ==========================================================================
35	32	Assembly Syntax: Vd32=hi(Vss32)
36	33	C Intrinsic Prototype: HVX_Vector Q6_V_hi_W(HVX_VectorPair Vss)
...	...	@@ -39,9 +36,7 @@
39	36	========================================================================== */
40	37
41	38	#define Q6_V_hi_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_hi)(Vss)
42		#endif /* __HEXAGON_ARCH___ >= 60 */
43	39
44		#if __HVX_ARCH__ >= 60
45	40	/* ==========================================================================
46	41	Assembly Syntax: Vd32=lo(Vss32)
47	42	C Intrinsic Prototype: HVX_Vector Q6_V_lo_W(HVX_VectorPair Vss)
...	...	@@ -50,9 +45,7 @@
50	45	========================================================================== */
51	46
52	47	#define Q6_V_lo_W(Vss) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lo)(Vss)
53		#endif /* __HEXAGON_ARCH___ >= 60 */
54	48
55		#if __HVX_ARCH__ >= 60
56	49	/* ==========================================================================
57	50	Assembly Syntax: Vd32=vsplat(Rt32)
58	51	C Intrinsic Prototype: HVX_Vector Q6_V_vsplat_R(Word32 Rt)
...	...	@@ -61,9 +54,7 @@
61	54	========================================================================== */
62	55
63	56	#define Q6_V_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatw)(Rt)
64		#endif /* __HEXAGON_ARCH___ >= 60 */
65	57
66		#if __HVX_ARCH__ >= 60
67	58	/* ==========================================================================
68	59	Assembly Syntax: Qd4=and(Qs4,Qt4)
69	60	C Intrinsic Prototype: HVX_VectorPred Q6_Q_and_QQ(HVX_VectorPred Qs, HVX_VectorPred Qt)
...	...	@@ -72,9 +63,7 @@
72	63	========================================================================== */
73	64
74	65	#define Q6_Q_and_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
75		#endif /* __HEXAGON_ARCH___ >= 60 */
76	66
77		#if __HVX_ARCH__ >= 60
78	67	/* ==========================================================================
79	68	Assembly Syntax: Qd4=and(Qs4,!Qt4)
80	69	C Intrinsic Prototype: HVX_VectorPred Q6_Q_and_QQn(HVX_VectorPred Qs, HVX_VectorPred Qt)
...	...	@@ -83,9 +72,7 @@
83	72	========================================================================== */
84	73
85	74	#define Q6_Q_and_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_and_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
86		#endif /* __HEXAGON_ARCH___ >= 60 */
87	75
88		#if __HVX_ARCH__ >= 60
89	76	/* ==========================================================================
90	77	Assembly Syntax: Qd4=not(Qs4)
91	78	C Intrinsic Prototype: HVX_VectorPred Q6_Q_not_Q(HVX_VectorPred Qs)
...	...	@@ -94,9 +81,7 @@
94	81	========================================================================== */
95	82
96	83	#define Q6_Q_not_Q(Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_not)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1))),-1)
97		#endif /* __HEXAGON_ARCH___ >= 60 */
98	84
99		#if __HVX_ARCH__ >= 60
100	85	/* ==========================================================================
101	86	Assembly Syntax: Qd4=or(Qs4,Qt4)
102	87	C Intrinsic Prototype: HVX_VectorPred Q6_Q_or_QQ(HVX_VectorPred Qs, HVX_VectorPred Qt)
...	...	@@ -105,9 +90,7 @@
105	90	========================================================================== */
106	91
107	92	#define Q6_Q_or_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
108		#endif /* __HEXAGON_ARCH___ >= 60 */
109	93
110		#if __HVX_ARCH__ >= 60
111	94	/* ==========================================================================
112	95	Assembly Syntax: Qd4=or(Qs4,!Qt4)
113	96	C Intrinsic Prototype: HVX_VectorPred Q6_Q_or_QQn(HVX_VectorPred Qs, HVX_VectorPred Qt)
...	...	@@ -116,9 +99,7 @@
116	99	========================================================================== */
117	100
118	101	#define Q6_Q_or_QQn(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_or_n)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
119		#endif /* __HEXAGON_ARCH___ >= 60 */
120	102
121		#if __HVX_ARCH__ >= 60
122	103	/* ==========================================================================
123	104	Assembly Syntax: Qd4=vsetq(Rt32)
124	105	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vsetq_R(Word32 Rt)
...	...	@@ -127,9 +108,7 @@
127	108	========================================================================== */
128	109
129	110	#define Q6_Q_vsetq_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2)(Rt)),-1)
130		#endif /* __HEXAGON_ARCH___ >= 60 */
131	111
132		#if __HVX_ARCH__ >= 60
133	112	/* ==========================================================================
134	113	Assembly Syntax: Qd4=xor(Qs4,Qt4)
135	114	C Intrinsic Prototype: HVX_VectorPred Q6_Q_xor_QQ(HVX_VectorPred Qs, HVX_VectorPred Qt)
...	...	@@ -138,9 +117,7 @@
138	117	========================================================================== */
139	118
140	119	#define Q6_Q_xor_QQ(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
141		#endif /* __HEXAGON_ARCH___ >= 60 */
142	120
143		#if __HVX_ARCH__ >= 60
144	121	/* ==========================================================================
145	122	Assembly Syntax: if (!Qv4) vmem(Rt32+#s4)=Vs32
146	123	C Intrinsic Prototype: void Q6_vmem_QnRIV(HVX_VectorPred Qv, HVX_Vector* Rt, HVX_Vector Vs)
...	...	@@ -149,9 +126,7 @@
149	126	========================================================================== */
150	127
151	128	#define Q6_vmem_QnRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
152		#endif /* __HEXAGON_ARCH___ >= 60 */
153	129
154		#if __HVX_ARCH__ >= 60
155	130	/* ==========================================================================
156	131	Assembly Syntax: if (!Qv4) vmem(Rt32+#s4):nt=Vs32
157	132	C Intrinsic Prototype: void Q6_vmem_QnRIV_nt(HVX_VectorPred Qv, HVX_Vector* Rt, HVX_Vector Vs)
...	...	@@ -160,9 +135,7 @@
160	135	========================================================================== */
161	136
162	137	#define Q6_vmem_QnRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_nqpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
163		#endif /* __HEXAGON_ARCH___ >= 60 */
164	138
165		#if __HVX_ARCH__ >= 60
166	139	/* ==========================================================================
167	140	Assembly Syntax: if (Qv4) vmem(Rt32+#s4):nt=Vs32
168	141	C Intrinsic Prototype: void Q6_vmem_QRIV_nt(HVX_VectorPred Qv, HVX_Vector* Rt, HVX_Vector Vs)
...	...	@@ -171,9 +144,7 @@
171	144	========================================================================== */
172	145
173	146	#define Q6_vmem_QRIV_nt(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_nt_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
174		#endif /* __HEXAGON_ARCH___ >= 60 */
175	147
176		#if __HVX_ARCH__ >= 60
177	148	/* ==========================================================================
178	149	Assembly Syntax: if (Qv4) vmem(Rt32+#s4)=Vs32
179	150	C Intrinsic Prototype: void Q6_vmem_QRIV(HVX_VectorPred Qv, HVX_Vector* Rt, HVX_Vector Vs)
...	...	@@ -182,9 +153,7 @@
182	153	========================================================================== */
183	154
184	155	#define Q6_vmem_QRIV(Qv,Rt,Vs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vS32b_qpred_ai)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Rt,Vs)
185		#endif /* __HEXAGON_ARCH___ >= 60 */
186	156
187		#if __HVX_ARCH__ >= 60
188	157	/* ==========================================================================
189	158	Assembly Syntax: Vd32.uh=vabsdiff(Vu32.h,Vv32.h)
190	159	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vabsdiff_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -193,9 +162,7 @@
193	162	========================================================================== */
194	163
195	164	#define Q6_Vuh_vabsdiff_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffh)(Vu,Vv)
196		#endif /* __HEXAGON_ARCH___ >= 60 */
197	165
198		#if __HVX_ARCH__ >= 60
199	166	/* ==========================================================================
200	167	Assembly Syntax: Vd32.ub=vabsdiff(Vu32.ub,Vv32.ub)
201	168	C Intrinsic Prototype: HVX_Vector Q6_Vub_vabsdiff_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -204,9 +171,7 @@
204	171	========================================================================== */
205	172
206	173	#define Q6_Vub_vabsdiff_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffub)(Vu,Vv)
207		#endif /* __HEXAGON_ARCH___ >= 60 */
208	174
209		#if __HVX_ARCH__ >= 60
210	175	/* ==========================================================================
211	176	Assembly Syntax: Vd32.uh=vabsdiff(Vu32.uh,Vv32.uh)
212	177	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vabsdiff_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -215,9 +180,7 @@
215	180	========================================================================== */
216	181
217	182	#define Q6_Vuh_vabsdiff_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffuh)(Vu,Vv)
218		#endif /* __HEXAGON_ARCH___ >= 60 */
219	183
220		#if __HVX_ARCH__ >= 60
221	184	/* ==========================================================================
222	185	Assembly Syntax: Vd32.uw=vabsdiff(Vu32.w,Vv32.w)
223	186	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vabsdiff_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -226,9 +189,7 @@
226	189	========================================================================== */
227	190
228	191	#define Q6_Vuw_vabsdiff_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsdiffw)(Vu,Vv)
229		#endif /* __HEXAGON_ARCH___ >= 60 */
230	192
231		#if __HVX_ARCH__ >= 60
232	193	/* ==========================================================================
233	194	Assembly Syntax: Vd32.h=vabs(Vu32.h)
234	195	C Intrinsic Prototype: HVX_Vector Q6_Vh_vabs_Vh(HVX_Vector Vu)
...	...	@@ -237,9 +198,7 @@
237	198	========================================================================== */
238	199
239	200	#define Q6_Vh_vabs_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh)(Vu)
240		#endif /* __HEXAGON_ARCH___ >= 60 */
241	201
242		#if __HVX_ARCH__ >= 60
243	202	/* ==========================================================================
244	203	Assembly Syntax: Vd32.h=vabs(Vu32.h):sat
245	204	C Intrinsic Prototype: HVX_Vector Q6_Vh_vabs_Vh_sat(HVX_Vector Vu)
...	...	@@ -248,9 +207,7 @@
248	207	========================================================================== */
249	208
250	209	#define Q6_Vh_vabs_Vh_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsh_sat)(Vu)
251		#endif /* __HEXAGON_ARCH___ >= 60 */
252	210
253		#if __HVX_ARCH__ >= 60
254	211	/* ==========================================================================
255	212	Assembly Syntax: Vd32.w=vabs(Vu32.w)
256	213	C Intrinsic Prototype: HVX_Vector Q6_Vw_vabs_Vw(HVX_Vector Vu)
...	...	@@ -259,9 +216,7 @@
259	216	========================================================================== */
260	217
261	218	#define Q6_Vw_vabs_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw)(Vu)
262		#endif /* __HEXAGON_ARCH___ >= 60 */
263	219
264		#if __HVX_ARCH__ >= 60
265	220	/* ==========================================================================
266	221	Assembly Syntax: Vd32.w=vabs(Vu32.w):sat
267	222	C Intrinsic Prototype: HVX_Vector Q6_Vw_vabs_Vw_sat(HVX_Vector Vu)
...	...	@@ -270,9 +225,7 @@
270	225	========================================================================== */
271	226
272	227	#define Q6_Vw_vabs_Vw_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsw_sat)(Vu)
273		#endif /* __HEXAGON_ARCH___ >= 60 */
274	228
275		#if __HVX_ARCH__ >= 60
276	229	/* ==========================================================================
277	230	Assembly Syntax: Vd32.b=vadd(Vu32.b,Vv32.b)
278	231	C Intrinsic Prototype: HVX_Vector Q6_Vb_vadd_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -281,9 +234,7 @@
281	234	========================================================================== */
282	235
283	236	#define Q6_Vb_vadd_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb)(Vu,Vv)
284		#endif /* __HEXAGON_ARCH___ >= 60 */
285	237
286		#if __HVX_ARCH__ >= 60
287	238	/* ==========================================================================
288	239	Assembly Syntax: Vdd32.b=vadd(Vuu32.b,Vvv32.b)
289	240	C Intrinsic Prototype: HVX_VectorPair Q6_Wb_vadd_WbWb(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -292,9 +243,7 @@
292	243	========================================================================== */
293	244
294	245	#define Q6_Wb_vadd_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddb_dv)(Vuu,Vvv)
295		#endif /* __HEXAGON_ARCH___ >= 60 */
296	246
297		#if __HVX_ARCH__ >= 60
298	247	/* ==========================================================================
299	248	Assembly Syntax: if (!Qv4) Vx32.b+=Vu32.b
300	249	C Intrinsic Prototype: HVX_Vector Q6_Vb_condacc_QnVbVb(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -303,9 +252,7 @@
303	252	========================================================================== */
304	253
305	254	#define Q6_Vb_condacc_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
306		#endif /* __HEXAGON_ARCH___ >= 60 */
307	255
308		#if __HVX_ARCH__ >= 60
309	256	/* ==========================================================================
310	257	Assembly Syntax: if (Qv4) Vx32.b+=Vu32.b
311	258	C Intrinsic Prototype: HVX_Vector Q6_Vb_condacc_QVbVb(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -314,9 +261,7 @@
314	261	========================================================================== */
315	262
316	263	#define Q6_Vb_condacc_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
317		#endif /* __HEXAGON_ARCH___ >= 60 */
318	264
319		#if __HVX_ARCH__ >= 60
320	265	/* ==========================================================================
321	266	Assembly Syntax: Vd32.h=vadd(Vu32.h,Vv32.h)
322	267	C Intrinsic Prototype: HVX_Vector Q6_Vh_vadd_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -325,9 +270,7 @@
325	270	========================================================================== */
326	271
327	272	#define Q6_Vh_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh)(Vu,Vv)
328		#endif /* __HEXAGON_ARCH___ >= 60 */
329	273
330		#if __HVX_ARCH__ >= 60
331	274	/* ==========================================================================
332	275	Assembly Syntax: Vdd32.h=vadd(Vuu32.h,Vvv32.h)
333	276	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vadd_WhWh(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -336,9 +279,7 @@
336	279	========================================================================== */
337	280
338	281	#define Q6_Wh_vadd_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddh_dv)(Vuu,Vvv)
339		#endif /* __HEXAGON_ARCH___ >= 60 */
340	282
341		#if __HVX_ARCH__ >= 60
342	283	/* ==========================================================================
343	284	Assembly Syntax: if (!Qv4) Vx32.h+=Vu32.h
344	285	C Intrinsic Prototype: HVX_Vector Q6_Vh_condacc_QnVhVh(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -347,9 +288,7 @@
347	288	========================================================================== */
348	289
349	290	#define Q6_Vh_condacc_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
350		#endif /* __HEXAGON_ARCH___ >= 60 */
351	291
352		#if __HVX_ARCH__ >= 60
353	292	/* ==========================================================================
354	293	Assembly Syntax: if (Qv4) Vx32.h+=Vu32.h
355	294	C Intrinsic Prototype: HVX_Vector Q6_Vh_condacc_QVhVh(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -358,9 +297,7 @@
358	297	========================================================================== */
359	298
360	299	#define Q6_Vh_condacc_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
361		#endif /* __HEXAGON_ARCH___ >= 60 */
362	300
363		#if __HVX_ARCH__ >= 60
364	301	/* ==========================================================================
365	302	Assembly Syntax: Vd32.h=vadd(Vu32.h,Vv32.h):sat
366	303	C Intrinsic Prototype: HVX_Vector Q6_Vh_vadd_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -369,9 +306,7 @@
369	306	========================================================================== */
370	307
371	308	#define Q6_Vh_vadd_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat)(Vu,Vv)
372		#endif /* __HEXAGON_ARCH___ >= 60 */
373	309
374		#if __HVX_ARCH__ >= 60
375	310	/* ==========================================================================
376	311	Assembly Syntax: Vdd32.h=vadd(Vuu32.h,Vvv32.h):sat
377	312	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vadd_WhWh_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -380,9 +315,7 @@
380	315	========================================================================== */
381	316
382	317	#define Q6_Wh_vadd_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhsat_dv)(Vuu,Vvv)
383		#endif /* __HEXAGON_ARCH___ >= 60 */
384	318
385		#if __HVX_ARCH__ >= 60
386	319	/* ==========================================================================
387	320	Assembly Syntax: Vdd32.w=vadd(Vu32.h,Vv32.h)
388	321	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vadd_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -391,9 +324,7 @@
391	324	========================================================================== */
392	325
393	326	#define Q6_Ww_vadd_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw)(Vu,Vv)
394		#endif /* __HEXAGON_ARCH___ >= 60 */
395	327
396		#if __HVX_ARCH__ >= 60
397	328	/* ==========================================================================
398	329	Assembly Syntax: Vdd32.h=vadd(Vu32.ub,Vv32.ub)
399	330	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vadd_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -402,9 +333,7 @@
402	333	========================================================================== */
403	334
404	335	#define Q6_Wh_vadd_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh)(Vu,Vv)
405		#endif /* __HEXAGON_ARCH___ >= 60 */
406	336
407		#if __HVX_ARCH__ >= 60
408	337	/* ==========================================================================
409	338	Assembly Syntax: Vd32.ub=vadd(Vu32.ub,Vv32.ub):sat
410	339	C Intrinsic Prototype: HVX_Vector Q6_Vub_vadd_VubVub_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -413,9 +342,7 @@
413	342	========================================================================== */
414	343
415	344	#define Q6_Vub_vadd_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat)(Vu,Vv)
416		#endif /* __HEXAGON_ARCH___ >= 60 */
417	345
418		#if __HVX_ARCH__ >= 60
419	346	/* ==========================================================================
420	347	Assembly Syntax: Vdd32.ub=vadd(Vuu32.ub,Vvv32.ub):sat
421	348	C Intrinsic Prototype: HVX_VectorPair Q6_Wub_vadd_WubWub_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -424,9 +351,7 @@
424	351	========================================================================== */
425	352
426	353	#define Q6_Wub_vadd_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubsat_dv)(Vuu,Vvv)
427		#endif /* __HEXAGON_ARCH___ >= 60 */
428	354
429		#if __HVX_ARCH__ >= 60
430	355	/* ==========================================================================
431	356	Assembly Syntax: Vd32.uh=vadd(Vu32.uh,Vv32.uh):sat
432	357	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vadd_VuhVuh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -435,9 +360,7 @@
435	360	========================================================================== */
436	361
437	362	#define Q6_Vuh_vadd_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat)(Vu,Vv)
438		#endif /* __HEXAGON_ARCH___ >= 60 */
439	363
440		#if __HVX_ARCH__ >= 60
441	364	/* ==========================================================================
442	365	Assembly Syntax: Vdd32.uh=vadd(Vuu32.uh,Vvv32.uh):sat
443	366	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vadd_WuhWuh_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -446,9 +369,7 @@
446	369	========================================================================== */
447	370
448	371	#define Q6_Wuh_vadd_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhsat_dv)(Vuu,Vvv)
449		#endif /* __HEXAGON_ARCH___ >= 60 */
450	372
451		#if __HVX_ARCH__ >= 60
452	373	/* ==========================================================================
453	374	Assembly Syntax: Vdd32.w=vadd(Vu32.uh,Vv32.uh)
454	375	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vadd_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -457,9 +378,7 @@
457	378	========================================================================== */
458	379
459	380	#define Q6_Ww_vadd_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw)(Vu,Vv)
460		#endif /* __HEXAGON_ARCH___ >= 60 */
461	381
462		#if __HVX_ARCH__ >= 60
463	382	/* ==========================================================================
464	383	Assembly Syntax: Vd32.w=vadd(Vu32.w,Vv32.w)
465	384	C Intrinsic Prototype: HVX_Vector Q6_Vw_vadd_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -468,9 +387,7 @@
468	387	========================================================================== */
469	388
470	389	#define Q6_Vw_vadd_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw)(Vu,Vv)
471		#endif /* __HEXAGON_ARCH___ >= 60 */
472	390
473		#if __HVX_ARCH__ >= 60
474	391	/* ==========================================================================
475	392	Assembly Syntax: Vdd32.w=vadd(Vuu32.w,Vvv32.w)
476	393	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vadd_WwWw(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -479,9 +396,7 @@
479	396	========================================================================== */
480	397
481	398	#define Q6_Ww_vadd_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddw_dv)(Vuu,Vvv)
482		#endif /* __HEXAGON_ARCH___ >= 60 */
483	399
484		#if __HVX_ARCH__ >= 60
485	400	/* ==========================================================================
486	401	Assembly Syntax: if (!Qv4) Vx32.w+=Vu32.w
487	402	C Intrinsic Prototype: HVX_Vector Q6_Vw_condacc_QnVwVw(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -490,9 +405,7 @@
490	405	========================================================================== */
491	406
492	407	#define Q6_Vw_condacc_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
493		#endif /* __HEXAGON_ARCH___ >= 60 */
494	408
495		#if __HVX_ARCH__ >= 60
496	409	/* ==========================================================================
497	410	Assembly Syntax: if (Qv4) Vx32.w+=Vu32.w
498	411	C Intrinsic Prototype: HVX_Vector Q6_Vw_condacc_QVwVw(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -501,9 +414,7 @@
501	414	========================================================================== */
502	415
503	416	#define Q6_Vw_condacc_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
504		#endif /* __HEXAGON_ARCH___ >= 60 */
505	417
506		#if __HVX_ARCH__ >= 60
507	418	/* ==========================================================================
508	419	Assembly Syntax: Vd32.w=vadd(Vu32.w,Vv32.w):sat
509	420	C Intrinsic Prototype: HVX_Vector Q6_Vw_vadd_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -512,9 +423,7 @@
512	423	========================================================================== */
513	424
514	425	#define Q6_Vw_vadd_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat)(Vu,Vv)
515		#endif /* __HEXAGON_ARCH___ >= 60 */
516	426
517		#if __HVX_ARCH__ >= 60
518	427	/* ==========================================================================
519	428	Assembly Syntax: Vdd32.w=vadd(Vuu32.w,Vvv32.w):sat
520	429	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vadd_WwWw_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -523,9 +432,7 @@
523	432	========================================================================== */
524	433
525	434	#define Q6_Ww_vadd_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddwsat_dv)(Vuu,Vvv)
526		#endif /* __HEXAGON_ARCH___ >= 60 */
527	435
528		#if __HVX_ARCH__ >= 60
529	436	/* ==========================================================================
530	437	Assembly Syntax: Vd32=valign(Vu32,Vv32,Rt8)
531	438	C Intrinsic Prototype: HVX_Vector Q6_V_valign_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -534,9 +441,7 @@
534	441	========================================================================== */
535	442
536	443	#define Q6_V_valign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignb)(Vu,Vv,Rt)
537		#endif /* __HEXAGON_ARCH___ >= 60 */
538	444
539		#if __HVX_ARCH__ >= 60
540	445	/* ==========================================================================
541	446	Assembly Syntax: Vd32=valign(Vu32,Vv32,#u3)
542	447	C Intrinsic Prototype: HVX_Vector Q6_V_valign_VVI(HVX_Vector Vu, HVX_Vector Vv, Word32 Iu3)
...	...	@@ -545,9 +450,7 @@
545	450	========================================================================== */
546	451
547	452	#define Q6_V_valign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valignbi)(Vu,Vv,Iu3)
548		#endif /* __HEXAGON_ARCH___ >= 60 */
549	453
550		#if __HVX_ARCH__ >= 60
551	454	/* ==========================================================================
552	455	Assembly Syntax: Vd32=vand(Vu32,Vv32)
553	456	C Intrinsic Prototype: HVX_Vector Q6_V_vand_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -556,9 +459,7 @@
556	459	========================================================================== */
557	460
558	461	#define Q6_V_vand_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vand)(Vu,Vv)
559		#endif /* __HEXAGON_ARCH___ >= 60 */
560	462
561		#if __HVX_ARCH__ >= 60
562	463	/* ==========================================================================
563	464	Assembly Syntax: Vd32=vand(Qu4,Rt32)
564	465	C Intrinsic Prototype: HVX_Vector Q6_V_vand_QR(HVX_VectorPred Qu, Word32 Rt)
...	...	@@ -567,9 +468,7 @@
567	468	========================================================================== */
568	469
569	470	#define Q6_V_vand_QR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
570		#endif /* __HEXAGON_ARCH___ >= 60 */
571	471
572		#if __HVX_ARCH__ >= 60
573	472	/* ==========================================================================
574	473	Assembly Syntax: Vx32\|=vand(Qu4,Rt32)
575	474	C Intrinsic Prototype: HVX_Vector Q6_V_vandor_VQR(HVX_Vector Vx, HVX_VectorPred Qu, Word32 Rt)
...	...	@@ -578,9 +477,7 @@
578	477	========================================================================== */
579	478
580	479	#define Q6_V_vandor_VQR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
581		#endif /* __HEXAGON_ARCH___ >= 60 */
582	480
583		#if __HVX_ARCH__ >= 60
584	481	/* ==========================================================================
585	482	Assembly Syntax: Qd4=vand(Vu32,Rt32)
586	483	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vand_VR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -589,9 +486,7 @@
589	486	========================================================================== */
590	487
591	488	#define Q6_Q_vand_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)(Vu,Rt)),-1)
592		#endif /* __HEXAGON_ARCH___ >= 60 */
593	489
594		#if __HVX_ARCH__ >= 60
595	490	/* ==========================================================================
596	491	Assembly Syntax: Qx4\|=vand(Vu32,Rt32)
597	492	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vandor_QVR(HVX_VectorPred Qx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -600,9 +495,7 @@
600	495	========================================================================== */
601	496
602	497	#define Q6_Q_vandor_QVR(Qx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt_acc)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Rt)),-1)
603		#endif /* __HEXAGON_ARCH___ >= 60 */
604	498
605		#if __HVX_ARCH__ >= 60
606	499	/* ==========================================================================
607	500	Assembly Syntax: Vd32.h=vasl(Vu32.h,Rt32)
608	501	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasl_VhR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -611,9 +504,7 @@
611	504	========================================================================== */
612	505
613	506	#define Q6_Vh_vasl_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh)(Vu,Rt)
614		#endif /* __HEXAGON_ARCH___ >= 60 */
615	507
616		#if __HVX_ARCH__ >= 60
617	508	/* ==========================================================================
618	509	Assembly Syntax: Vd32.h=vasl(Vu32.h,Vv32.h)
619	510	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasl_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -622,9 +513,7 @@
622	513	========================================================================== */
623	514
624	515	#define Q6_Vh_vasl_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslhv)(Vu,Vv)
625		#endif /* __HEXAGON_ARCH___ >= 60 */
626	516
627		#if __HVX_ARCH__ >= 60
628	517	/* ==========================================================================
629	518	Assembly Syntax: Vd32.w=vasl(Vu32.w,Rt32)
630	519	C Intrinsic Prototype: HVX_Vector Q6_Vw_vasl_VwR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -633,9 +522,7 @@
633	522	========================================================================== */
634	523
635	524	#define Q6_Vw_vasl_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw)(Vu,Rt)
636		#endif /* __HEXAGON_ARCH___ >= 60 */
637	525
638		#if __HVX_ARCH__ >= 60
639	526	/* ==========================================================================
640	527	Assembly Syntax: Vx32.w+=vasl(Vu32.w,Rt32)
641	528	C Intrinsic Prototype: HVX_Vector Q6_Vw_vaslacc_VwVwR(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -644,9 +531,7 @@
644	531	========================================================================== */
645	532
646	533	#define Q6_Vw_vaslacc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslw_acc)(Vx,Vu,Rt)
647		#endif /* __HEXAGON_ARCH___ >= 60 */
648	534
649		#if __HVX_ARCH__ >= 60
650	535	/* ==========================================================================
651	536	Assembly Syntax: Vd32.w=vasl(Vu32.w,Vv32.w)
652	537	C Intrinsic Prototype: HVX_Vector Q6_Vw_vasl_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -655,9 +540,7 @@
655	540	========================================================================== */
656	541
657	542	#define Q6_Vw_vasl_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslwv)(Vu,Vv)
658		#endif /* __HEXAGON_ARCH___ >= 60 */
659	543
660		#if __HVX_ARCH__ >= 60
661	544	/* ==========================================================================
662	545	Assembly Syntax: Vd32.h=vasr(Vu32.h,Rt32)
663	546	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasr_VhR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -666,9 +549,7 @@
666	549	========================================================================== */
667	550
668	551	#define Q6_Vh_vasr_VhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh)(Vu,Rt)
669		#endif /* __HEXAGON_ARCH___ >= 60 */
670	552
671		#if __HVX_ARCH__ >= 60
672	553	/* ==========================================================================
673	554	Assembly Syntax: Vd32.b=vasr(Vu32.h,Vv32.h,Rt8):rnd:sat
674	555	C Intrinsic Prototype: HVX_Vector Q6_Vb_vasr_VhVhR_rnd_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -677,9 +558,7 @@
677	558	========================================================================== */
678	559
679	560	#define Q6_Vb_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbrndsat)(Vu,Vv,Rt)
680		#endif /* __HEXAGON_ARCH___ >= 60 */
681	561
682		#if __HVX_ARCH__ >= 60
683	562	/* ==========================================================================
684	563	Assembly Syntax: Vd32.ub=vasr(Vu32.h,Vv32.h,Rt8):rnd:sat
685	564	C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_VhVhR_rnd_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -688,9 +567,7 @@
688	567	========================================================================== */
689	568
690	569	#define Q6_Vub_vasr_VhVhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubrndsat)(Vu,Vv,Rt)
691		#endif /* __HEXAGON_ARCH___ >= 60 */
692	570
693		#if __HVX_ARCH__ >= 60
694	571	/* ==========================================================================
695	572	Assembly Syntax: Vd32.ub=vasr(Vu32.h,Vv32.h,Rt8):sat
696	573	C Intrinsic Prototype: HVX_Vector Q6_Vub_vasr_VhVhR_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -699,9 +576,7 @@
699	576	========================================================================== */
700	577
701	578	#define Q6_Vub_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhubsat)(Vu,Vv,Rt)
702		#endif /* __HEXAGON_ARCH___ >= 60 */
703	579
704		#if __HVX_ARCH__ >= 60
705	580	/* ==========================================================================
706	581	Assembly Syntax: Vd32.h=vasr(Vu32.h,Vv32.h)
707	582	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasr_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -710,9 +585,7 @@
710	585	========================================================================== */
711	586
712	587	#define Q6_Vh_vasr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhv)(Vu,Vv)
713		#endif /* __HEXAGON_ARCH___ >= 60 */
714	588
715		#if __HVX_ARCH__ >= 60
716	589	/* ==========================================================================
717	590	Assembly Syntax: Vd32.w=vasr(Vu32.w,Rt32)
718	591	C Intrinsic Prototype: HVX_Vector Q6_Vw_vasr_VwR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -721,9 +594,7 @@
721	594	========================================================================== */
722	595
723	596	#define Q6_Vw_vasr_VwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw)(Vu,Rt)
724		#endif /* __HEXAGON_ARCH___ >= 60 */
725	597
726		#if __HVX_ARCH__ >= 60
727	598	/* ==========================================================================
728	599	Assembly Syntax: Vx32.w+=vasr(Vu32.w,Rt32)
729	600	C Intrinsic Prototype: HVX_Vector Q6_Vw_vasracc_VwVwR(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -732,9 +603,7 @@
732	603	========================================================================== */
733	604
734	605	#define Q6_Vw_vasracc_VwVwR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrw_acc)(Vx,Vu,Rt)
735		#endif /* __HEXAGON_ARCH___ >= 60 */
736	606
737		#if __HVX_ARCH__ >= 60
738	607	/* ==========================================================================
739	608	Assembly Syntax: Vd32.h=vasr(Vu32.w,Vv32.w,Rt8)
740	609	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasr_VwVwR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -743,9 +612,7 @@
743	612	========================================================================== */
744	613
745	614	#define Q6_Vh_vasr_VwVwR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwh)(Vu,Vv,Rt)
746		#endif /* __HEXAGON_ARCH___ >= 60 */
747	615
748		#if __HVX_ARCH__ >= 60
749	616	/* ==========================================================================
750	617	Assembly Syntax: Vd32.h=vasr(Vu32.w,Vv32.w,Rt8):rnd:sat
751	618	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasr_VwVwR_rnd_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -754,9 +621,7 @@
754	621	========================================================================== */
755	622
756	623	#define Q6_Vh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhrndsat)(Vu,Vv,Rt)
757		#endif /* __HEXAGON_ARCH___ >= 60 */
758	624
759		#if __HVX_ARCH__ >= 60
760	625	/* ==========================================================================
761	626	Assembly Syntax: Vd32.h=vasr(Vu32.w,Vv32.w,Rt8):sat
762	627	C Intrinsic Prototype: HVX_Vector Q6_Vh_vasr_VwVwR_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -765,9 +630,7 @@
765	630	========================================================================== */
766	631
767	632	#define Q6_Vh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwhsat)(Vu,Vv,Rt)
768		#endif /* __HEXAGON_ARCH___ >= 60 */
769	633
770		#if __HVX_ARCH__ >= 60
771	634	/* ==========================================================================
772	635	Assembly Syntax: Vd32.uh=vasr(Vu32.w,Vv32.w,Rt8):sat
773	636	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vasr_VwVwR_sat(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -776,9 +639,7 @@
776	639	========================================================================== */
777	640
778	641	#define Q6_Vuh_vasr_VwVwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhsat)(Vu,Vv,Rt)
779		#endif /* __HEXAGON_ARCH___ >= 60 */
780	642
781		#if __HVX_ARCH__ >= 60
782	643	/* ==========================================================================
783	644	Assembly Syntax: Vd32.w=vasr(Vu32.w,Vv32.w)
784	645	C Intrinsic Prototype: HVX_Vector Q6_Vw_vasr_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -787,9 +648,7 @@
787	648	========================================================================== */
788	649
789	650	#define Q6_Vw_vasr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwv)(Vu,Vv)
790		#endif /* __HEXAGON_ARCH___ >= 60 */
791	651
792		#if __HVX_ARCH__ >= 60
793	652	/* ==========================================================================
794	653	Assembly Syntax: Vd32=Vu32
795	654	C Intrinsic Prototype: HVX_Vector Q6_V_equals_V(HVX_Vector Vu)
...	...	@@ -798,9 +657,7 @@
798	657	========================================================================== */
799	658
800	659	#define Q6_V_equals_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign)(Vu)
801		#endif /* __HEXAGON_ARCH___ >= 60 */
802	660
803		#if __HVX_ARCH__ >= 60
804	661	/* ==========================================================================
805	662	Assembly Syntax: Vdd32=Vuu32
806	663	C Intrinsic Prototype: HVX_VectorPair Q6_W_equals_W(HVX_VectorPair Vuu)
...	...	@@ -809,9 +666,7 @@
809	666	========================================================================== */
810	667
811	668	#define Q6_W_equals_W(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassignp)(Vuu)
812		#endif /* __HEXAGON_ARCH___ >= 60 */
813	669
814		#if __HVX_ARCH__ >= 60
815	670	/* ==========================================================================
816	671	Assembly Syntax: Vd32.h=vavg(Vu32.h,Vv32.h)
817	672	C Intrinsic Prototype: HVX_Vector Q6_Vh_vavg_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -820,9 +675,7 @@
820	675	========================================================================== */
821	676
822	677	#define Q6_Vh_vavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgh)(Vu,Vv)
823		#endif /* __HEXAGON_ARCH___ >= 60 */
824	678
825		#if __HVX_ARCH__ >= 60
826	679	/* ==========================================================================
827	680	Assembly Syntax: Vd32.h=vavg(Vu32.h,Vv32.h):rnd
828	681	C Intrinsic Prototype: HVX_Vector Q6_Vh_vavg_VhVh_rnd(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -831,9 +684,7 @@
831	684	========================================================================== */
832	685
833	686	#define Q6_Vh_vavg_VhVh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavghrnd)(Vu,Vv)
834		#endif /* __HEXAGON_ARCH___ >= 60 */
835	687
836		#if __HVX_ARCH__ >= 60
837	688	/* ==========================================================================
838	689	Assembly Syntax: Vd32.ub=vavg(Vu32.ub,Vv32.ub)
839	690	C Intrinsic Prototype: HVX_Vector Q6_Vub_vavg_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -842,9 +693,7 @@
842	693	========================================================================== */
843	694
844	695	#define Q6_Vub_vavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgub)(Vu,Vv)
845		#endif /* __HEXAGON_ARCH___ >= 60 */
846	696
847		#if __HVX_ARCH__ >= 60
848	697	/* ==========================================================================
849	698	Assembly Syntax: Vd32.ub=vavg(Vu32.ub,Vv32.ub):rnd
850	699	C Intrinsic Prototype: HVX_Vector Q6_Vub_vavg_VubVub_rnd(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -853,9 +702,7 @@
853	702	========================================================================== */
854	703
855	704	#define Q6_Vub_vavg_VubVub_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgubrnd)(Vu,Vv)
856		#endif /* __HEXAGON_ARCH___ >= 60 */
857	705
858		#if __HVX_ARCH__ >= 60
859	706	/* ==========================================================================
860	707	Assembly Syntax: Vd32.uh=vavg(Vu32.uh,Vv32.uh)
861	708	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vavg_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -864,9 +711,7 @@
864	711	========================================================================== */
865	712
866	713	#define Q6_Vuh_vavg_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguh)(Vu,Vv)
867		#endif /* __HEXAGON_ARCH___ >= 60 */
868	714
869		#if __HVX_ARCH__ >= 60
870	715	/* ==========================================================================
871	716	Assembly Syntax: Vd32.uh=vavg(Vu32.uh,Vv32.uh):rnd
872	717	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vavg_VuhVuh_rnd(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -875,9 +720,7 @@
875	720	========================================================================== */
876	721
877	722	#define Q6_Vuh_vavg_VuhVuh_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguhrnd)(Vu,Vv)
878		#endif /* __HEXAGON_ARCH___ >= 60 */
879	723
880		#if __HVX_ARCH__ >= 60
881	724	/* ==========================================================================
882	725	Assembly Syntax: Vd32.w=vavg(Vu32.w,Vv32.w)
883	726	C Intrinsic Prototype: HVX_Vector Q6_Vw_vavg_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -886,9 +729,7 @@
886	729	========================================================================== */
887	730
888	731	#define Q6_Vw_vavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgw)(Vu,Vv)
889		#endif /* __HEXAGON_ARCH___ >= 60 */
890	732
891		#if __HVX_ARCH__ >= 60
892	733	/* ==========================================================================
893	734	Assembly Syntax: Vd32.w=vavg(Vu32.w,Vv32.w):rnd
894	735	C Intrinsic Prototype: HVX_Vector Q6_Vw_vavg_VwVw_rnd(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -897,9 +738,7 @@
897	738	========================================================================== */
898	739
899	740	#define Q6_Vw_vavg_VwVw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgwrnd)(Vu,Vv)
900		#endif /* __HEXAGON_ARCH___ >= 60 */
901	741
902		#if __HVX_ARCH__ >= 60
903	742	/* ==========================================================================
904	743	Assembly Syntax: Vd32.uh=vcl0(Vu32.uh)
905	744	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vcl0_Vuh(HVX_Vector Vu)
...	...	@@ -908,9 +747,7 @@
908	747	========================================================================== */
909	748
910	749	#define Q6_Vuh_vcl0_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0h)(Vu)
911		#endif /* __HEXAGON_ARCH___ >= 60 */
912	750
913		#if __HVX_ARCH__ >= 60
914	751	/* ==========================================================================
915	752	Assembly Syntax: Vd32.uw=vcl0(Vu32.uw)
916	753	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vcl0_Vuw(HVX_Vector Vu)
...	...	@@ -919,9 +756,7 @@
919	756	========================================================================== */
920	757
921	758	#define Q6_Vuw_vcl0_Vuw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcl0w)(Vu)
922		#endif /* __HEXAGON_ARCH___ >= 60 */
923	759
924		#if __HVX_ARCH__ >= 60
925	760	/* ==========================================================================
926	761	Assembly Syntax: Vdd32=vcombine(Vu32,Vv32)
927	762	C Intrinsic Prototype: HVX_VectorPair Q6_W_vcombine_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -930,9 +765,7 @@
930	765	========================================================================== */
931	766
932	767	#define Q6_W_vcombine_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcombine)(Vu,Vv)
933		#endif /* __HEXAGON_ARCH___ >= 60 */
934	768
935		#if __HVX_ARCH__ >= 60
936	769	/* ==========================================================================
937	770	Assembly Syntax: Vd32=#0
938	771	C Intrinsic Prototype: HVX_Vector Q6_V_vzero()
...	...	@@ -941,9 +774,7 @@
941	774	========================================================================== */
942	775
943	776	#define Q6_V_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vd0)()
944		#endif /* __HEXAGON_ARCH___ >= 60 */
945	777
946		#if __HVX_ARCH__ >= 60
947	778	/* ==========================================================================
948	779	Assembly Syntax: Vd32.b=vdeal(Vu32.b)
949	780	C Intrinsic Prototype: HVX_Vector Q6_Vb_vdeal_Vb(HVX_Vector Vu)
...	...	@@ -952,9 +783,7 @@
952	783	========================================================================== */
953	784
954	785	#define Q6_Vb_vdeal_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb)(Vu)
955		#endif /* __HEXAGON_ARCH___ >= 60 */
956	786
957		#if __HVX_ARCH__ >= 60
958	787	/* ==========================================================================
959	788	Assembly Syntax: Vd32.b=vdeale(Vu32.b,Vv32.b)
960	789	C Intrinsic Prototype: HVX_Vector Q6_Vb_vdeale_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -963,9 +792,7 @@
963	792	========================================================================== */
964	793
965	794	#define Q6_Vb_vdeale_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealb4w)(Vu,Vv)
966		#endif /* __HEXAGON_ARCH___ >= 60 */
967	795
968		#if __HVX_ARCH__ >= 60
969	796	/* ==========================================================================
970	797	Assembly Syntax: Vd32.h=vdeal(Vu32.h)
971	798	C Intrinsic Prototype: HVX_Vector Q6_Vh_vdeal_Vh(HVX_Vector Vu)
...	...	@@ -974,9 +801,7 @@
974	801	========================================================================== */
975	802
976	803	#define Q6_Vh_vdeal_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealh)(Vu)
977		#endif /* __HEXAGON_ARCH___ >= 60 */
978	804
979		#if __HVX_ARCH__ >= 60
980	805	/* ==========================================================================
981	806	Assembly Syntax: Vdd32=vdeal(Vu32,Vv32,Rt8)
982	807	C Intrinsic Prototype: HVX_VectorPair Q6_W_vdeal_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -985,9 +810,7 @@
985	810	========================================================================== */
986	811
987	812	#define Q6_W_vdeal_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdealvdd)(Vu,Vv,Rt)
988		#endif /* __HEXAGON_ARCH___ >= 60 */
989	813
990		#if __HVX_ARCH__ >= 60
991	814	/* ==========================================================================
992	815	Assembly Syntax: Vd32=vdelta(Vu32,Vv32)
993	816	C Intrinsic Prototype: HVX_Vector Q6_V_vdelta_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -996,9 +819,7 @@
996	819	========================================================================== */
997	820
998	821	#define Q6_V_vdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdelta)(Vu,Vv)
999		#endif /* __HEXAGON_ARCH___ >= 60 */
1000	822
1001		#if __HVX_ARCH__ >= 60
1002	823	/* ==========================================================================
1003	824	Assembly Syntax: Vd32.h=vdmpy(Vu32.ub,Rt32.b)
1004	825	C Intrinsic Prototype: HVX_Vector Q6_Vh_vdmpy_VubRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1007,9 +828,7 @@
1007	828	========================================================================== */
1008	829
1009	830	#define Q6_Vh_vdmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus)(Vu,Rt)
1010		#endif /* __HEXAGON_ARCH___ >= 60 */
1011	831
1012		#if __HVX_ARCH__ >= 60
1013	832	/* ==========================================================================
1014	833	Assembly Syntax: Vx32.h+=vdmpy(Vu32.ub,Rt32.b)
1015	834	C Intrinsic Prototype: HVX_Vector Q6_Vh_vdmpyacc_VhVubRb(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1018,9 +837,7 @@
1018	837	========================================================================== */
1019	838
1020	839	#define Q6_Vh_vdmpyacc_VhVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_acc)(Vx,Vu,Rt)
1021		#endif /* __HEXAGON_ARCH___ >= 60 */
1022	840
1023		#if __HVX_ARCH__ >= 60
1024	841	/* ==========================================================================
1025	842	Assembly Syntax: Vdd32.h=vdmpy(Vuu32.ub,Rt32.b)
1026	843	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vdmpy_WubRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1029,9 +846,7 @@
1029	846	========================================================================== */
1030	847
1031	848	#define Q6_Wh_vdmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv)(Vuu,Rt)
1032		#endif /* __HEXAGON_ARCH___ >= 60 */
1033	849
1034		#if __HVX_ARCH__ >= 60
1035	850	/* ==========================================================================
1036	851	Assembly Syntax: Vxx32.h+=vdmpy(Vuu32.ub,Rt32.b)
1037	852	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vdmpyacc_WhWubRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1040,9 +855,7 @@
1040	855	========================================================================== */
1041	856
1042	857	#define Q6_Wh_vdmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpybus_dv_acc)(Vxx,Vuu,Rt)
1043		#endif /* __HEXAGON_ARCH___ >= 60 */
1044	858
1045		#if __HVX_ARCH__ >= 60
1046	859	/* ==========================================================================
1047	860	Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Rt32.b)
1048	861	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1051,9 +864,7 @@
1051	864	========================================================================== */
1052	865
1053	866	#define Q6_Vw_vdmpy_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb)(Vu,Rt)
1054		#endif /* __HEXAGON_ARCH___ >= 60 */
1055	867
1056		#if __HVX_ARCH__ >= 60
1057	868	/* ==========================================================================
1058	869	Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Rt32.b)
1059	870	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRb(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1062,9 +873,7 @@
1062	873	========================================================================== */
1063	874
1064	875	#define Q6_Vw_vdmpyacc_VwVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_acc)(Vx,Vu,Rt)
1065		#endif /* __HEXAGON_ARCH___ >= 60 */
1066	876
1067		#if __HVX_ARCH__ >= 60
1068	877	/* ==========================================================================
1069	878	Assembly Syntax: Vdd32.w=vdmpy(Vuu32.h,Rt32.b)
1070	879	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vdmpy_WhRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1073,9 +882,7 @@
1073	882	========================================================================== */
1074	883
1075	884	#define Q6_Ww_vdmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv)(Vuu,Rt)
1076		#endif /* __HEXAGON_ARCH___ >= 60 */
1077	885
1078		#if __HVX_ARCH__ >= 60
1079	886	/* ==========================================================================
1080	887	Assembly Syntax: Vxx32.w+=vdmpy(Vuu32.h,Rt32.b)
1081	888	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vdmpyacc_WwWhRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1084,9 +891,7 @@
1084	891	========================================================================== */
1085	892
1086	893	#define Q6_Ww_vdmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhb_dv_acc)(Vxx,Vuu,Rt)
1087		#endif /* __HEXAGON_ARCH___ >= 60 */
1088	894
1089		#if __HVX_ARCH__ >= 60
1090	895	/* ==========================================================================
1091	896	Assembly Syntax: Vd32.w=vdmpy(Vuu32.h,Rt32.h):sat
1092	897	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_WhRh_sat(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1095,9 +900,7 @@
1095	900	========================================================================== */
1096	901
1097	902	#define Q6_Vw_vdmpy_WhRh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat)(Vuu,Rt)
1098		#endif /* __HEXAGON_ARCH___ >= 60 */
1099	903
1100		#if __HVX_ARCH__ >= 60
1101	904	/* ==========================================================================
1102	905	Assembly Syntax: Vx32.w+=vdmpy(Vuu32.h,Rt32.h):sat
1103	906	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwWhRh_sat(HVX_Vector Vx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1106,9 +909,7 @@
1106	909	========================================================================== */
1107	910
1108	911	#define Q6_Vw_vdmpyacc_VwWhRh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhisat_acc)(Vx,Vuu,Rt)
1109		#endif /* __HEXAGON_ARCH___ >= 60 */
1110	912
1111		#if __HVX_ARCH__ >= 60
1112	913	/* ==========================================================================
1113	914	Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Rt32.h):sat
1114	915	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRh_sat(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1117,9 +918,7 @@
1117	918	========================================================================== */
1118	919
1119	920	#define Q6_Vw_vdmpy_VhRh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat)(Vu,Rt)
1120		#endif /* __HEXAGON_ARCH___ >= 60 */
1121	921
1122		#if __HVX_ARCH__ >= 60
1123	922	/* ==========================================================================
1124	923	Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Rt32.h):sat
1125	924	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1128,9 +927,7 @@
1128	927	========================================================================== */
1129	928
1130	929	#define Q6_Vw_vdmpyacc_VwVhRh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsat_acc)(Vx,Vu,Rt)
1131		#endif /* __HEXAGON_ARCH___ >= 60 */
1132	930
1133		#if __HVX_ARCH__ >= 60
1134	931	/* ==========================================================================
1135	932	Assembly Syntax: Vd32.w=vdmpy(Vuu32.h,Rt32.uh,#1):sat
1136	933	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_WhRuh_sat(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1139,9 +936,7 @@
1139	936	========================================================================== */
1140	937
1141	938	#define Q6_Vw_vdmpy_WhRuh_sat(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat)(Vuu,Rt)
1142		#endif /* __HEXAGON_ARCH___ >= 60 */
1143	939
1144		#if __HVX_ARCH__ >= 60
1145	940	/* ==========================================================================
1146	941	Assembly Syntax: Vx32.w+=vdmpy(Vuu32.h,Rt32.uh,#1):sat
1147	942	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwWhRuh_sat(HVX_Vector Vx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1150,9 +945,7 @@
1150	945	========================================================================== */
1151	946
1152	947	#define Q6_Vw_vdmpyacc_VwWhRuh_sat(Vx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsuisat_acc)(Vx,Vuu,Rt)
1153		#endif /* __HEXAGON_ARCH___ >= 60 */
1154	948
1155		#if __HVX_ARCH__ >= 60
1156	949	/* ==========================================================================
1157	950	Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Rt32.uh):sat
1158	951	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhRuh_sat(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1161,9 +954,7 @@
1161	954	========================================================================== */
1162	955
1163	956	#define Q6_Vw_vdmpy_VhRuh_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat)(Vu,Rt)
1164		#endif /* __HEXAGON_ARCH___ >= 60 */
1165	957
1166		#if __HVX_ARCH__ >= 60
1167	958	/* ==========================================================================
1168	959	Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Rt32.uh):sat
1169	960	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhRuh_sat(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1172,9 +963,7 @@
1172	963	========================================================================== */
1173	964
1174	965	#define Q6_Vw_vdmpyacc_VwVhRuh_sat(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhsusat_acc)(Vx,Vu,Rt)
1175		#endif /* __HEXAGON_ARCH___ >= 60 */
1176	966
1177		#if __HVX_ARCH__ >= 60
1178	967	/* ==========================================================================
1179	968	Assembly Syntax: Vd32.w=vdmpy(Vu32.h,Vv32.h):sat
1180	969	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpy_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1183,9 +972,7 @@
1183	972	========================================================================== */
1184	973
1185	974	#define Q6_Vw_vdmpy_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat)(Vu,Vv)
1186		#endif /* __HEXAGON_ARCH___ >= 60 */
1187	975
1188		#if __HVX_ARCH__ >= 60
1189	976	/* ==========================================================================
1190	977	Assembly Syntax: Vx32.w+=vdmpy(Vu32.h,Vv32.h):sat
1191	978	C Intrinsic Prototype: HVX_Vector Q6_Vw_vdmpyacc_VwVhVh_sat(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1194,9 +981,7 @@
1194	981	========================================================================== */
1195	982
1196	983	#define Q6_Vw_vdmpyacc_VwVhVh_sat(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpyhvsat_acc)(Vx,Vu,Vv)
1197		#endif /* __HEXAGON_ARCH___ >= 60 */
1198	984
1199		#if __HVX_ARCH__ >= 60
1200	985	/* ==========================================================================
1201	986	Assembly Syntax: Vdd32.uw=vdsad(Vuu32.uh,Rt32.uh)
1202	987	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vdsad_WuhRuh(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1205,9 +990,7 @@
1205	990	========================================================================== */
1206	991
1207	992	#define Q6_Wuw_vdsad_WuhRuh(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh)(Vuu,Rt)
1208		#endif /* __HEXAGON_ARCH___ >= 60 */
1209	993
1210		#if __HVX_ARCH__ >= 60
1211	994	/* ==========================================================================
1212	995	Assembly Syntax: Vxx32.uw+=vdsad(Vuu32.uh,Rt32.uh)
1213	996	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vdsadacc_WuwWuhRuh(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1216,9 +999,7 @@
1216	999	========================================================================== */
1217	1000
1218	1001	#define Q6_Wuw_vdsadacc_WuwWuhRuh(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdsaduh_acc)(Vxx,Vuu,Rt)
1219		#endif /* __HEXAGON_ARCH___ >= 60 */
1220	1002
1221		#if __HVX_ARCH__ >= 60
1222	1003	/* ==========================================================================
1223	1004	Assembly Syntax: Qd4=vcmp.eq(Vu32.b,Vv32.b)
1224	1005	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1227,9 +1008,7 @@
1227	1008	========================================================================== */
1228	1009
1229	1010	#define Q6_Q_vcmp_eq_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb)(Vu,Vv)),-1)
1230		#endif /* __HEXAGON_ARCH___ >= 60 */
1231	1011
1232		#if __HVX_ARCH__ >= 60
1233	1012	/* ==========================================================================
1234	1013	Assembly Syntax: Qx4&=vcmp.eq(Vu32.b,Vv32.b)
1235	1014	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1238,9 +1017,7 @@
1238	1017	========================================================================== */
1239	1018
1240	1019	#define Q6_Q_vcmp_eqand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1241		#endif /* __HEXAGON_ARCH___ >= 60 */
1242	1020
1243		#if __HVX_ARCH__ >= 60
1244	1021	/* ==========================================================================
1245	1022	Assembly Syntax: Qx4\|=vcmp.eq(Vu32.b,Vv32.b)
1246	1023	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1249,9 +1026,7 @@
1249	1026	========================================================================== */
1250	1027
1251	1028	#define Q6_Q_vcmp_eqor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1252		#endif /* __HEXAGON_ARCH___ >= 60 */
1253	1029
1254		#if __HVX_ARCH__ >= 60
1255	1030	/* ==========================================================================
1256	1031	Assembly Syntax: Qx4^=vcmp.eq(Vu32.b,Vv32.b)
1257	1032	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1260,9 +1035,7 @@
1260	1035	========================================================================== */
1261	1036
1262	1037	#define Q6_Q_vcmp_eqxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1263		#endif /* __HEXAGON_ARCH___ >= 60 */
1264	1038
1265		#if __HVX_ARCH__ >= 60
1266	1039	/* ==========================================================================
1267	1040	Assembly Syntax: Qd4=vcmp.eq(Vu32.h,Vv32.h)
1268	1041	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1271,9 +1044,7 @@
1271	1044	========================================================================== */
1272	1045
1273	1046	#define Q6_Q_vcmp_eq_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh)(Vu,Vv)),-1)
1274		#endif /* __HEXAGON_ARCH___ >= 60 */
1275	1047
1276		#if __HVX_ARCH__ >= 60
1277	1048	/* ==========================================================================
1278	1049	Assembly Syntax: Qx4&=vcmp.eq(Vu32.h,Vv32.h)
1279	1050	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1282,9 +1053,7 @@
1282	1053	========================================================================== */
1283	1054
1284	1055	#define Q6_Q_vcmp_eqand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1285		#endif /* __HEXAGON_ARCH___ >= 60 */
1286	1056
1287		#if __HVX_ARCH__ >= 60
1288	1057	/* ==========================================================================
1289	1058	Assembly Syntax: Qx4\|=vcmp.eq(Vu32.h,Vv32.h)
1290	1059	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1293,9 +1062,7 @@
1293	1062	========================================================================== */
1294	1063
1295	1064	#define Q6_Q_vcmp_eqor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1296		#endif /* __HEXAGON_ARCH___ >= 60 */
1297	1065
1298		#if __HVX_ARCH__ >= 60
1299	1066	/* ==========================================================================
1300	1067	Assembly Syntax: Qx4^=vcmp.eq(Vu32.h,Vv32.h)
1301	1068	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1304,9 +1071,7 @@
1304	1071	========================================================================== */
1305	1072
1306	1073	#define Q6_Q_vcmp_eqxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1307		#endif /* __HEXAGON_ARCH___ >= 60 */
1308	1074
1309		#if __HVX_ARCH__ >= 60
1310	1075	/* ==========================================================================
1311	1076	Assembly Syntax: Qd4=vcmp.eq(Vu32.w,Vv32.w)
1312	1077	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1315,9 +1080,7 @@
1315	1080	========================================================================== */
1316	1081
1317	1082	#define Q6_Q_vcmp_eq_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw)(Vu,Vv)),-1)
1318		#endif /* __HEXAGON_ARCH___ >= 60 */
1319	1083
1320		#if __HVX_ARCH__ >= 60
1321	1084	/* ==========================================================================
1322	1085	Assembly Syntax: Qx4&=vcmp.eq(Vu32.w,Vv32.w)
1323	1086	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1326,9 +1089,7 @@
1326	1089	========================================================================== */
1327	1090
1328	1091	#define Q6_Q_vcmp_eqand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1329		#endif /* __HEXAGON_ARCH___ >= 60 */
1330	1092
1331		#if __HVX_ARCH__ >= 60
1332	1093	/* ==========================================================================
1333	1094	Assembly Syntax: Qx4\|=vcmp.eq(Vu32.w,Vv32.w)
1334	1095	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1337,9 +1098,7 @@
1337	1098	========================================================================== */
1338	1099
1339	1100	#define Q6_Q_vcmp_eqor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1340		#endif /* __HEXAGON_ARCH___ >= 60 */
1341	1101
1342		#if __HVX_ARCH__ >= 60
1343	1102	/* ==========================================================================
1344	1103	Assembly Syntax: Qx4^=vcmp.eq(Vu32.w,Vv32.w)
1345	1104	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1348,9 +1107,7 @@
1348	1107	========================================================================== */
1349	1108
1350	1109	#define Q6_Q_vcmp_eqxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1351		#endif /* __HEXAGON_ARCH___ >= 60 */
1352	1110
1353		#if __HVX_ARCH__ >= 60
1354	1111	/* ==========================================================================
1355	1112	Assembly Syntax: Qd4=vcmp.gt(Vu32.b,Vv32.b)
1356	1113	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1359,9 +1116,7 @@
1359	1116	========================================================================== */
1360	1117
1361	1118	#define Q6_Q_vcmp_gt_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb)(Vu,Vv)),-1)
1362		#endif /* __HEXAGON_ARCH___ >= 60 */
1363	1119
1364		#if __HVX_ARCH__ >= 60
1365	1120	/* ==========================================================================
1366	1121	Assembly Syntax: Qx4&=vcmp.gt(Vu32.b,Vv32.b)
1367	1122	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1370,9 +1125,7 @@
1370	1125	========================================================================== */
1371	1126
1372	1127	#define Q6_Q_vcmp_gtand_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1373		#endif /* __HEXAGON_ARCH___ >= 60 */
1374	1128
1375		#if __HVX_ARCH__ >= 60
1376	1129	/* ==========================================================================
1377	1130	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.b,Vv32.b)
1378	1131	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1381,9 +1134,7 @@
1381	1134	========================================================================== */
1382	1135
1383	1136	#define Q6_Q_vcmp_gtor_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1384		#endif /* __HEXAGON_ARCH___ >= 60 */
1385	1137
1386		#if __HVX_ARCH__ >= 60
1387	1138	/* ==========================================================================
1388	1139	Assembly Syntax: Qx4^=vcmp.gt(Vu32.b,Vv32.b)
1389	1140	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVbVb(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1392,9 +1143,7 @@
1392	1143	========================================================================== */
1393	1144
1394	1145	#define Q6_Q_vcmp_gtxacc_QVbVb(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtb_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1395		#endif /* __HEXAGON_ARCH___ >= 60 */
1396	1146
1397		#if __HVX_ARCH__ >= 60
1398	1147	/* ==========================================================================
1399	1148	Assembly Syntax: Qd4=vcmp.gt(Vu32.h,Vv32.h)
1400	1149	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1403,9 +1152,7 @@
1403	1152	========================================================================== */
1404	1153
1405	1154	#define Q6_Q_vcmp_gt_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth)(Vu,Vv)),-1)
1406		#endif /* __HEXAGON_ARCH___ >= 60 */
1407	1155
1408		#if __HVX_ARCH__ >= 60
1409	1156	/* ==========================================================================
1410	1157	Assembly Syntax: Qx4&=vcmp.gt(Vu32.h,Vv32.h)
1411	1158	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1414,9 +1161,7 @@
1414	1161	========================================================================== */
1415	1162
1416	1163	#define Q6_Q_vcmp_gtand_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1417		#endif /* __HEXAGON_ARCH___ >= 60 */
1418	1164
1419		#if __HVX_ARCH__ >= 60
1420	1165	/* ==========================================================================
1421	1166	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.h,Vv32.h)
1422	1167	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1425,9 +1170,7 @@
1425	1170	========================================================================== */
1426	1171
1427	1172	#define Q6_Q_vcmp_gtor_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1428		#endif /* __HEXAGON_ARCH___ >= 60 */
1429	1173
1430		#if __HVX_ARCH__ >= 60
1431	1174	/* ==========================================================================
1432	1175	Assembly Syntax: Qx4^=vcmp.gt(Vu32.h,Vv32.h)
1433	1176	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVhVh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1436,9 +1179,7 @@
1436	1179	========================================================================== */
1437	1180
1438	1181	#define Q6_Q_vcmp_gtxacc_QVhVh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgth_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1439		#endif /* __HEXAGON_ARCH___ >= 60 */
1440	1182
1441		#if __HVX_ARCH__ >= 60
1442	1183	/* ==========================================================================
1443	1184	Assembly Syntax: Qd4=vcmp.gt(Vu32.ub,Vv32.ub)
1444	1185	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1447,9 +1188,7 @@
1447	1188	========================================================================== */
1448	1189
1449	1190	#define Q6_Q_vcmp_gt_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub)(Vu,Vv)),-1)
1450		#endif /* __HEXAGON_ARCH___ >= 60 */
1451	1191
1452		#if __HVX_ARCH__ >= 60
1453	1192	/* ==========================================================================
1454	1193	Assembly Syntax: Qx4&=vcmp.gt(Vu32.ub,Vv32.ub)
1455	1194	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVubVub(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1458,9 +1197,7 @@
1458	1197	========================================================================== */
1459	1198
1460	1199	#define Q6_Q_vcmp_gtand_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1461		#endif /* __HEXAGON_ARCH___ >= 60 */
1462	1200
1463		#if __HVX_ARCH__ >= 60
1464	1201	/* ==========================================================================
1465	1202	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.ub,Vv32.ub)
1466	1203	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVubVub(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1469,9 +1206,7 @@
1469	1206	========================================================================== */
1470	1207
1471	1208	#define Q6_Q_vcmp_gtor_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1472		#endif /* __HEXAGON_ARCH___ >= 60 */
1473	1209
1474		#if __HVX_ARCH__ >= 60
1475	1210	/* ==========================================================================
1476	1211	Assembly Syntax: Qx4^=vcmp.gt(Vu32.ub,Vv32.ub)
1477	1212	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVubVub(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1480,9 +1215,7 @@
1480	1215	========================================================================== */
1481	1216
1482	1217	#define Q6_Q_vcmp_gtxacc_QVubVub(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtub_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1483		#endif /* __HEXAGON_ARCH___ >= 60 */
1484	1218
1485		#if __HVX_ARCH__ >= 60
1486	1219	/* ==========================================================================
1487	1220	Assembly Syntax: Qd4=vcmp.gt(Vu32.uh,Vv32.uh)
1488	1221	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1491,9 +1224,7 @@
1491	1224	========================================================================== */
1492	1225
1493	1226	#define Q6_Q_vcmp_gt_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh)(Vu,Vv)),-1)
1494		#endif /* __HEXAGON_ARCH___ >= 60 */
1495	1227
1496		#if __HVX_ARCH__ >= 60
1497	1228	/* ==========================================================================
1498	1229	Assembly Syntax: Qx4&=vcmp.gt(Vu32.uh,Vv32.uh)
1499	1230	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVuhVuh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1502,9 +1233,7 @@
1502	1233	========================================================================== */
1503	1234
1504	1235	#define Q6_Q_vcmp_gtand_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1505		#endif /* __HEXAGON_ARCH___ >= 60 */
1506	1236
1507		#if __HVX_ARCH__ >= 60
1508	1237	/* ==========================================================================
1509	1238	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.uh,Vv32.uh)
1510	1239	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVuhVuh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1513,9 +1242,7 @@
1513	1242	========================================================================== */
1514	1243
1515	1244	#define Q6_Q_vcmp_gtor_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1516		#endif /* __HEXAGON_ARCH___ >= 60 */
1517	1245
1518		#if __HVX_ARCH__ >= 60
1519	1246	/* ==========================================================================
1520	1247	Assembly Syntax: Qx4^=vcmp.gt(Vu32.uh,Vv32.uh)
1521	1248	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVuhVuh(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1524,9 +1251,7 @@
1524	1251	========================================================================== */
1525	1252
1526	1253	#define Q6_Q_vcmp_gtxacc_QVuhVuh(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuh_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1527		#endif /* __HEXAGON_ARCH___ >= 60 */
1528	1254
1529		#if __HVX_ARCH__ >= 60
1530	1255	/* ==========================================================================
1531	1256	Assembly Syntax: Qd4=vcmp.gt(Vu32.uw,Vv32.uw)
1532	1257	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VuwVuw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1535,9 +1260,7 @@
1535	1260	========================================================================== */
1536	1261
1537	1262	#define Q6_Q_vcmp_gt_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw)(Vu,Vv)),-1)
1538		#endif /* __HEXAGON_ARCH___ >= 60 */
1539	1263
1540		#if __HVX_ARCH__ >= 60
1541	1264	/* ==========================================================================
1542	1265	Assembly Syntax: Qx4&=vcmp.gt(Vu32.uw,Vv32.uw)
1543	1266	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVuwVuw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1546,9 +1269,7 @@
1546	1269	========================================================================== */
1547	1270
1548	1271	#define Q6_Q_vcmp_gtand_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1549		#endif /* __HEXAGON_ARCH___ >= 60 */
1550	1272
1551		#if __HVX_ARCH__ >= 60
1552	1273	/* ==========================================================================
1553	1274	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.uw,Vv32.uw)
1554	1275	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVuwVuw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1557,9 +1278,7 @@
1557	1278	========================================================================== */
1558	1279
1559	1280	#define Q6_Q_vcmp_gtor_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1560		#endif /* __HEXAGON_ARCH___ >= 60 */
1561	1281
1562		#if __HVX_ARCH__ >= 60
1563	1282	/* ==========================================================================
1564	1283	Assembly Syntax: Qx4^=vcmp.gt(Vu32.uw,Vv32.uw)
1565	1284	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVuwVuw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1568,9 +1287,7 @@
1568	1287	========================================================================== */
1569	1288
1570	1289	#define Q6_Q_vcmp_gtxacc_QVuwVuw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtuw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1571		#endif /* __HEXAGON_ARCH___ >= 60 */
1572	1290
1573		#if __HVX_ARCH__ >= 60
1574	1291	/* ==========================================================================
1575	1292	Assembly Syntax: Qd4=vcmp.gt(Vu32.w,Vv32.w)
1576	1293	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gt_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1579,9 +1296,7 @@
1579	1296	========================================================================== */
1580	1297
1581	1298	#define Q6_Q_vcmp_gt_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw)(Vu,Vv)),-1)
1582		#endif /* __HEXAGON_ARCH___ >= 60 */
1583	1299
1584		#if __HVX_ARCH__ >= 60
1585	1300	/* ==========================================================================
1586	1301	Assembly Syntax: Qx4&=vcmp.gt(Vu32.w,Vv32.w)
1587	1302	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtand_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1590,9 +1305,7 @@
1590	1305	========================================================================== */
1591	1306
1592	1307	#define Q6_Q_vcmp_gtand_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1593		#endif /* __HEXAGON_ARCH___ >= 60 */
1594	1308
1595		#if __HVX_ARCH__ >= 60
1596	1309	/* ==========================================================================
1597	1310	Assembly Syntax: Qx4\|=vcmp.gt(Vu32.w,Vv32.w)
1598	1311	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtor_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1601,9 +1314,7 @@
1601	1314	========================================================================== */
1602	1315
1603	1316	#define Q6_Q_vcmp_gtor_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1604		#endif /* __HEXAGON_ARCH___ >= 60 */
1605	1317
1606		#if __HVX_ARCH__ >= 60
1607	1318	/* ==========================================================================
1608	1319	Assembly Syntax: Qx4^=vcmp.gt(Vu32.w,Vv32.w)
1609	1320	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_gtxacc_QVwVw(HVX_VectorPred Qx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1612,9 +1323,7 @@
1612	1323	========================================================================== */
1613	1324
1614	1325	#define Q6_Q_vcmp_gtxacc_QVwVw(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtw_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
1615		#endif /* __HEXAGON_ARCH___ >= 60 */
1616	1326
1617		#if __HVX_ARCH__ >= 60
1618	1327	/* ==========================================================================
1619	1328	Assembly Syntax: Vx32.w=vinsert(Rt32)
1620	1329	C Intrinsic Prototype: HVX_Vector Q6_Vw_vinsert_VwR(HVX_Vector Vx, Word32 Rt)
...	...	@@ -1623,9 +1332,7 @@
1623	1332	========================================================================== */
1624	1333
1625	1334	#define Q6_Vw_vinsert_VwR(Vx,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vinsertwr)(Vx,Rt)
1626		#endif /* __HEXAGON_ARCH___ >= 60 */
1627	1335
1628		#if __HVX_ARCH__ >= 60
1629	1336	/* ==========================================================================
1630	1337	Assembly Syntax: Vd32=vlalign(Vu32,Vv32,Rt8)
1631	1338	C Intrinsic Prototype: HVX_Vector Q6_V_vlalign_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -1634,9 +1341,7 @@
1634	1341	========================================================================== */
1635	1342
1636	1343	#define Q6_V_vlalign_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignb)(Vu,Vv,Rt)
1637		#endif /* __HEXAGON_ARCH___ >= 60 */
1638	1344
1639		#if __HVX_ARCH__ >= 60
1640	1345	/* ==========================================================================
1641	1346	Assembly Syntax: Vd32=vlalign(Vu32,Vv32,#u3)
1642	1347	C Intrinsic Prototype: HVX_Vector Q6_V_vlalign_VVI(HVX_Vector Vu, HVX_Vector Vv, Word32 Iu3)
...	...	@@ -1645,9 +1350,7 @@
1645	1350	========================================================================== */
1646	1351
1647	1352	#define Q6_V_vlalign_VVI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlalignbi)(Vu,Vv,Iu3)
1648		#endif /* __HEXAGON_ARCH___ >= 60 */
1649	1353
1650		#if __HVX_ARCH__ >= 60
1651	1354	/* ==========================================================================
1652	1355	Assembly Syntax: Vd32.uh=vlsr(Vu32.uh,Rt32)
1653	1356	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vlsr_VuhR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1656,9 +1359,7 @@
1656	1359	========================================================================== */
1657	1360
1658	1361	#define Q6_Vuh_vlsr_VuhR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrh)(Vu,Rt)
1659		#endif /* __HEXAGON_ARCH___ >= 60 */
1660	1362
1661		#if __HVX_ARCH__ >= 60
1662	1363	/* ==========================================================================
1663	1364	Assembly Syntax: Vd32.h=vlsr(Vu32.h,Vv32.h)
1664	1365	C Intrinsic Prototype: HVX_Vector Q6_Vh_vlsr_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1667,9 +1368,7 @@
1667	1368	========================================================================== */
1668	1369
1669	1370	#define Q6_Vh_vlsr_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrhv)(Vu,Vv)
1670		#endif /* __HEXAGON_ARCH___ >= 60 */
1671	1371
1672		#if __HVX_ARCH__ >= 60
1673	1372	/* ==========================================================================
1674	1373	Assembly Syntax: Vd32.uw=vlsr(Vu32.uw,Rt32)
1675	1374	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vlsr_VuwR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1678,9 +1377,7 @@
1678	1377	========================================================================== */
1679	1378
1680	1379	#define Q6_Vuw_vlsr_VuwR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrw)(Vu,Rt)
1681		#endif /* __HEXAGON_ARCH___ >= 60 */
1682	1380
1683		#if __HVX_ARCH__ >= 60
1684	1381	/* ==========================================================================
1685	1382	Assembly Syntax: Vd32.w=vlsr(Vu32.w,Vv32.w)
1686	1383	C Intrinsic Prototype: HVX_Vector Q6_Vw_vlsr_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1689,9 +1386,7 @@
1689	1386	========================================================================== */
1690	1387
1691	1388	#define Q6_Vw_vlsr_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrwv)(Vu,Vv)
1692		#endif /* __HEXAGON_ARCH___ >= 60 */
1693	1389
1694		#if __HVX_ARCH__ >= 60
1695	1390	/* ==========================================================================
1696	1391	Assembly Syntax: Vd32.b=vlut32(Vu32.b,Vv32.b,Rt8)
1697	1392	C Intrinsic Prototype: HVX_Vector Q6_Vb_vlut32_VbVbR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -1700,9 +1395,7 @@
1700	1395	========================================================================== */
1701	1396
1702	1397	#define Q6_Vb_vlut32_VbVbR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb)(Vu,Vv,Rt)
1703		#endif /* __HEXAGON_ARCH___ >= 60 */
1704	1398
1705		#if __HVX_ARCH__ >= 60
1706	1399	/* ==========================================================================
1707	1400	Assembly Syntax: Vx32.b\|=vlut32(Vu32.b,Vv32.b,Rt8)
1708	1401	C Intrinsic Prototype: HVX_Vector Q6_Vb_vlut32or_VbVbVbR(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -1711,9 +1404,7 @@
1711	1404	========================================================================== */
1712	1405
1713	1406	#define Q6_Vb_vlut32or_VbVbVbR(Vx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracc)(Vx,Vu,Vv,Rt)
1714		#endif /* __HEXAGON_ARCH___ >= 60 */
1715	1407
1716		#if __HVX_ARCH__ >= 60
1717	1408	/* ==========================================================================
1718	1409	Assembly Syntax: Vdd32.h=vlut16(Vu32.b,Vv32.h,Rt8)
1719	1410	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vlut16_VbVhR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -1722,9 +1413,7 @@
1722	1413	========================================================================== */
1723	1414
1724	1415	#define Q6_Wh_vlut16_VbVhR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh)(Vu,Vv,Rt)
1725		#endif /* __HEXAGON_ARCH___ >= 60 */
1726	1416
1727		#if __HVX_ARCH__ >= 60
1728	1417	/* ==========================================================================
1729	1418	Assembly Syntax: Vxx32.h\|=vlut16(Vu32.b,Vv32.h,Rt8)
1730	1419	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vlut16or_WhVbVhR(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -1733,9 +1422,7 @@
1733	1422	========================================================================== */
1734	1423
1735	1424	#define Q6_Wh_vlut16or_WhVbVhR(Vxx,Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracc)(Vxx,Vu,Vv,Rt)
1736		#endif /* __HEXAGON_ARCH___ >= 60 */
1737	1425
1738		#if __HVX_ARCH__ >= 60
1739	1426	/* ==========================================================================
1740	1427	Assembly Syntax: Vd32.h=vmax(Vu32.h,Vv32.h)
1741	1428	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmax_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1744,9 +1431,7 @@
1744	1431	========================================================================== */
1745	1432
1746	1433	#define Q6_Vh_vmax_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxh)(Vu,Vv)
1747		#endif /* __HEXAGON_ARCH___ >= 60 */
1748	1434
1749		#if __HVX_ARCH__ >= 60
1750	1435	/* ==========================================================================
1751	1436	Assembly Syntax: Vd32.ub=vmax(Vu32.ub,Vv32.ub)
1752	1437	C Intrinsic Prototype: HVX_Vector Q6_Vub_vmax_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1755,9 +1440,7 @@
1755	1440	========================================================================== */
1756	1441
1757	1442	#define Q6_Vub_vmax_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxub)(Vu,Vv)
1758		#endif /* __HEXAGON_ARCH___ >= 60 */
1759	1443
1760		#if __HVX_ARCH__ >= 60
1761	1444	/* ==========================================================================
1762	1445	Assembly Syntax: Vd32.uh=vmax(Vu32.uh,Vv32.uh)
1763	1446	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vmax_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1766,9 +1449,7 @@
1766	1449	========================================================================== */
1767	1450
1768	1451	#define Q6_Vuh_vmax_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxuh)(Vu,Vv)
1769		#endif /* __HEXAGON_ARCH___ >= 60 */
1770	1452
1771		#if __HVX_ARCH__ >= 60
1772	1453	/* ==========================================================================
1773	1454	Assembly Syntax: Vd32.w=vmax(Vu32.w,Vv32.w)
1774	1455	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmax_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1777,9 +1458,7 @@
1777	1458	========================================================================== */
1778	1459
1779	1460	#define Q6_Vw_vmax_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxw)(Vu,Vv)
1780		#endif /* __HEXAGON_ARCH___ >= 60 */
1781	1461
1782		#if __HVX_ARCH__ >= 60
1783	1462	/* ==========================================================================
1784	1463	Assembly Syntax: Vd32.h=vmin(Vu32.h,Vv32.h)
1785	1464	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmin_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1788,9 +1467,7 @@
1788	1467	========================================================================== */
1789	1468
1790	1469	#define Q6_Vh_vmin_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminh)(Vu,Vv)
1791		#endif /* __HEXAGON_ARCH___ >= 60 */
1792	1470
1793		#if __HVX_ARCH__ >= 60
1794	1471	/* ==========================================================================
1795	1472	Assembly Syntax: Vd32.ub=vmin(Vu32.ub,Vv32.ub)
1796	1473	C Intrinsic Prototype: HVX_Vector Q6_Vub_vmin_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1799,9 +1476,7 @@
1799	1476	========================================================================== */
1800	1477
1801	1478	#define Q6_Vub_vmin_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminub)(Vu,Vv)
1802		#endif /* __HEXAGON_ARCH___ >= 60 */
1803	1479
1804		#if __HVX_ARCH__ >= 60
1805	1480	/* ==========================================================================
1806	1481	Assembly Syntax: Vd32.uh=vmin(Vu32.uh,Vv32.uh)
1807	1482	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vmin_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1810,9 +1485,7 @@
1810	1485	========================================================================== */
1811	1486
1812	1487	#define Q6_Vuh_vmin_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminuh)(Vu,Vv)
1813		#endif /* __HEXAGON_ARCH___ >= 60 */
1814	1488
1815		#if __HVX_ARCH__ >= 60
1816	1489	/* ==========================================================================
1817	1490	Assembly Syntax: Vd32.w=vmin(Vu32.w,Vv32.w)
1818	1491	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmin_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1821,9 +1494,7 @@
1821	1494	========================================================================== */
1822	1495
1823	1496	#define Q6_Vw_vmin_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminw)(Vu,Vv)
1824		#endif /* __HEXAGON_ARCH___ >= 60 */
1825	1497
1826		#if __HVX_ARCH__ >= 60
1827	1498	/* ==========================================================================
1828	1499	Assembly Syntax: Vdd32.h=vmpa(Vuu32.ub,Rt32.b)
1829	1500	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpa_WubRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1832,9 +1503,7 @@
1832	1503	========================================================================== */
1833	1504
1834	1505	#define Q6_Wh_vmpa_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus)(Vuu,Rt)
1835		#endif /* __HEXAGON_ARCH___ >= 60 */
1836	1506
1837		#if __HVX_ARCH__ >= 60
1838	1507	/* ==========================================================================
1839	1508	Assembly Syntax: Vxx32.h+=vmpa(Vuu32.ub,Rt32.b)
1840	1509	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpaacc_WhWubRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1843,9 +1512,7 @@
1843	1512	========================================================================== */
1844	1513
1845	1514	#define Q6_Wh_vmpaacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabus_acc)(Vxx,Vuu,Rt)
1846		#endif /* __HEXAGON_ARCH___ >= 60 */
1847	1515
1848		#if __HVX_ARCH__ >= 60
1849	1516	/* ==========================================================================
1850	1517	Assembly Syntax: Vdd32.h=vmpa(Vuu32.ub,Vvv32.b)
1851	1518	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpa_WubWb(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -1854,9 +1521,7 @@
1854	1521	========================================================================== */
1855	1522
1856	1523	#define Q6_Wh_vmpa_WubWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabusv)(Vuu,Vvv)
1857		#endif /* __HEXAGON_ARCH___ >= 60 */
1858	1524
1859		#if __HVX_ARCH__ >= 60
1860	1525	/* ==========================================================================
1861	1526	Assembly Syntax: Vdd32.h=vmpa(Vuu32.ub,Vvv32.ub)
1862	1527	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpa_WubWub(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -1865,9 +1530,7 @@
1865	1530	========================================================================== */
1866	1531
1867	1532	#define Q6_Wh_vmpa_WubWub(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuuv)(Vuu,Vvv)
1868		#endif /* __HEXAGON_ARCH___ >= 60 */
1869	1533
1870		#if __HVX_ARCH__ >= 60
1871	1534	/* ==========================================================================
1872	1535	Assembly Syntax: Vdd32.w=vmpa(Vuu32.h,Rt32.b)
1873	1536	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpa_WhRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1876,9 +1539,7 @@
1876	1539	========================================================================== */
1877	1540
1878	1541	#define Q6_Ww_vmpa_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb)(Vuu,Rt)
1879		#endif /* __HEXAGON_ARCH___ >= 60 */
1880	1542
1881		#if __HVX_ARCH__ >= 60
1882	1543	/* ==========================================================================
1883	1544	Assembly Syntax: Vxx32.w+=vmpa(Vuu32.h,Rt32.b)
1884	1545	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpaacc_WwWhRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -1887,9 +1548,7 @@
1887	1548	========================================================================== */
1888	1549
1889	1550	#define Q6_Ww_vmpaacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahb_acc)(Vxx,Vuu,Rt)
1890		#endif /* __HEXAGON_ARCH___ >= 60 */
1891	1551
1892		#if __HVX_ARCH__ >= 60
1893	1552	/* ==========================================================================
1894	1553	Assembly Syntax: Vdd32.h=vmpy(Vu32.ub,Rt32.b)
1895	1554	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpy_VubRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1898,9 +1557,7 @@
1898	1557	========================================================================== */
1899	1558
1900	1559	#define Q6_Wh_vmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus)(Vu,Rt)
1901		#endif /* __HEXAGON_ARCH___ >= 60 */
1902	1560
1903		#if __HVX_ARCH__ >= 60
1904	1561	/* ==========================================================================
1905	1562	Assembly Syntax: Vxx32.h+=vmpy(Vu32.ub,Rt32.b)
1906	1563	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpyacc_WhVubRb(HVX_VectorPair Vxx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1909,9 +1566,7 @@
1909	1566	========================================================================== */
1910	1567
1911	1568	#define Q6_Wh_vmpyacc_WhVubRb(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybus_acc)(Vxx,Vu,Rt)
1912		#endif /* __HEXAGON_ARCH___ >= 60 */
1913	1569
1914		#if __HVX_ARCH__ >= 60
1915	1570	/* ==========================================================================
1916	1571	Assembly Syntax: Vdd32.h=vmpy(Vu32.ub,Vv32.b)
1917	1572	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpy_VubVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1920,9 +1575,7 @@
1920	1575	========================================================================== */
1921	1576
1922	1577	#define Q6_Wh_vmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv)(Vu,Vv)
1923		#endif /* __HEXAGON_ARCH___ >= 60 */
1924	1578
1925		#if __HVX_ARCH__ >= 60
1926	1579	/* ==========================================================================
1927	1580	Assembly Syntax: Vxx32.h+=vmpy(Vu32.ub,Vv32.b)
1928	1581	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpyacc_WhVubVb(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1931,9 +1584,7 @@
1931	1584	========================================================================== */
1932	1585
1933	1586	#define Q6_Wh_vmpyacc_WhVubVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybusv_acc)(Vxx,Vu,Vv)
1934		#endif /* __HEXAGON_ARCH___ >= 60 */
1935	1587
1936		#if __HVX_ARCH__ >= 60
1937	1588	/* ==========================================================================
1938	1589	Assembly Syntax: Vdd32.h=vmpy(Vu32.b,Vv32.b)
1939	1590	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpy_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1942,9 +1593,7 @@
1942	1593	========================================================================== */
1943	1594
1944	1595	#define Q6_Wh_vmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv)(Vu,Vv)
1945		#endif /* __HEXAGON_ARCH___ >= 60 */
1946	1596
1947		#if __HVX_ARCH__ >= 60
1948	1597	/* ==========================================================================
1949	1598	Assembly Syntax: Vxx32.h+=vmpy(Vu32.b,Vv32.b)
1950	1599	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vmpyacc_WhVbVb(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1953,9 +1602,7 @@
1953	1602	========================================================================== */
1954	1603
1955	1604	#define Q6_Wh_vmpyacc_WhVbVb(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpybv_acc)(Vxx,Vu,Vv)
1956		#endif /* __HEXAGON_ARCH___ >= 60 */
1957	1605
1958		#if __HVX_ARCH__ >= 60
1959	1606	/* ==========================================================================
1960	1607	Assembly Syntax: Vd32.w=vmpye(Vu32.w,Vv32.uh)
1961	1608	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpye_VwVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -1964,9 +1611,7 @@
1964	1611	========================================================================== */
1965	1612
1966	1613	#define Q6_Vw_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh)(Vu,Vv)
1967		#endif /* __HEXAGON_ARCH___ >= 60 */
1968	1614
1969		#if __HVX_ARCH__ >= 60
1970	1615	/* ==========================================================================
1971	1616	Assembly Syntax: Vdd32.w=vmpy(Vu32.h,Rt32.h)
1972	1617	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpy_VhRh(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1975,9 +1620,7 @@
1975	1620	========================================================================== */
1976	1621
1977	1622	#define Q6_Ww_vmpy_VhRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh)(Vu,Rt)
1978		#endif /* __HEXAGON_ARCH___ >= 60 */
1979	1623
1980		#if __HVX_ARCH__ >= 60
1981	1624	/* ==========================================================================
1982	1625	Assembly Syntax: Vxx32.w+=vmpy(Vu32.h,Rt32.h):sat
1983	1626	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpyacc_WwVhRh_sat(HVX_VectorPair Vxx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -1986,9 +1629,7 @@
1986	1629	========================================================================== */
1987	1630
1988	1631	#define Q6_Ww_vmpyacc_WwVhRh_sat(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsat_acc)(Vxx,Vu,Rt)
1989		#endif /* __HEXAGON_ARCH___ >= 60 */
1990	1632
1991		#if __HVX_ARCH__ >= 60
1992	1633	/* ==========================================================================
1993	1634	Assembly Syntax: Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:rnd:sat
1994	1635	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_rnd_sat(HVX_Vector Vu, Word32 Rt)
...	...	@@ -1997,9 +1638,7 @@
1997	1638	========================================================================== */
1998	1639
1999	1640	#define Q6_Vh_vmpy_VhRh_s1_rnd_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhsrs)(Vu,Rt)
2000		#endif /* __HEXAGON_ARCH___ >= 60 */
2001	1641
2002		#if __HVX_ARCH__ >= 60
2003	1642	/* ==========================================================================
2004	1643	Assembly Syntax: Vd32.h=vmpy(Vu32.h,Rt32.h):<<1:sat
2005	1644	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhRh_s1_sat(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2008,9 +1647,7 @@
2008	1647	========================================================================== */
2009	1648
2010	1649	#define Q6_Vh_vmpy_VhRh_s1_sat(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhss)(Vu,Rt)
2011		#endif /* __HEXAGON_ARCH___ >= 60 */
2012	1650
2013		#if __HVX_ARCH__ >= 60
2014	1651	/* ==========================================================================
2015	1652	Assembly Syntax: Vdd32.w=vmpy(Vu32.h,Vv32.uh)
2016	1653	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpy_VhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2019,9 +1656,7 @@
2019	1656	========================================================================== */
2020	1657
2021	1658	#define Q6_Ww_vmpy_VhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus)(Vu,Vv)
2022		#endif /* __HEXAGON_ARCH___ >= 60 */
2023	1659
2024		#if __HVX_ARCH__ >= 60
2025	1660	/* ==========================================================================
2026	1661	Assembly Syntax: Vxx32.w+=vmpy(Vu32.h,Vv32.uh)
2027	1662	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpyacc_WwVhVuh(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2030,9 +1665,7 @@
2030	1665	========================================================================== */
2031	1666
2032	1667	#define Q6_Ww_vmpyacc_WwVhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhus_acc)(Vxx,Vu,Vv)
2033		#endif /* __HEXAGON_ARCH___ >= 60 */
2034	1668
2035		#if __HVX_ARCH__ >= 60
2036	1669	/* ==========================================================================
2037	1670	Assembly Syntax: Vdd32.w=vmpy(Vu32.h,Vv32.h)
2038	1671	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpy_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2041,9 +1674,7 @@
2041	1674	========================================================================== */
2042	1675
2043	1676	#define Q6_Ww_vmpy_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv)(Vu,Vv)
2044		#endif /* __HEXAGON_ARCH___ >= 60 */
2045	1677
2046		#if __HVX_ARCH__ >= 60
2047	1678	/* ==========================================================================
2048	1679	Assembly Syntax: Vxx32.w+=vmpy(Vu32.h,Vv32.h)
2049	1680	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vmpyacc_WwVhVh(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2052,9 +1683,7 @@
2052	1683	========================================================================== */
2053	1684
2054	1685	#define Q6_Ww_vmpyacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhv_acc)(Vxx,Vu,Vv)
2055		#endif /* __HEXAGON_ARCH___ >= 60 */
2056	1686
2057		#if __HVX_ARCH__ >= 60
2058	1687	/* ==========================================================================
2059	1688	Assembly Syntax: Vd32.h=vmpy(Vu32.h,Vv32.h):<<1:rnd:sat
2060	1689	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpy_VhVh_s1_rnd_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2063,9 +1692,7 @@
2063	1692	========================================================================== */
2064	1693
2065	1694	#define Q6_Vh_vmpy_VhVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyhvsrs)(Vu,Vv)
2066		#endif /* __HEXAGON_ARCH___ >= 60 */
2067	1695
2068		#if __HVX_ARCH__ >= 60
2069	1696	/* ==========================================================================
2070	1697	Assembly Syntax: Vd32.w=vmpyieo(Vu32.h,Vv32.h)
2071	1698	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyieo_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2074,9 +1701,7 @@
2074	1701	========================================================================== */
2075	1702
2076	1703	#define Q6_Vw_vmpyieo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyieoh)(Vu,Vv)
2077		#endif /* __HEXAGON_ARCH___ >= 60 */
2078	1704
2079		#if __HVX_ARCH__ >= 60
2080	1705	/* ==========================================================================
2081	1706	Assembly Syntax: Vx32.w+=vmpyie(Vu32.w,Vv32.h)
2082	1707	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyieacc_VwVwVh(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2085,9 +1710,7 @@
2085	1710	========================================================================== */
2086	1711
2087	1712	#define Q6_Vw_vmpyieacc_VwVwVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewh_acc)(Vx,Vu,Vv)
2088		#endif /* __HEXAGON_ARCH___ >= 60 */
2089	1713
2090		#if __HVX_ARCH__ >= 60
2091	1714	/* ==========================================================================
2092	1715	Assembly Syntax: Vd32.w=vmpyie(Vu32.w,Vv32.uh)
2093	1716	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyie_VwVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2096,9 +1719,7 @@
2096	1719	========================================================================== */
2097	1720
2098	1721	#define Q6_Vw_vmpyie_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh)(Vu,Vv)
2099		#endif /* __HEXAGON_ARCH___ >= 60 */
2100	1722
2101		#if __HVX_ARCH__ >= 60
2102	1723	/* ==========================================================================
2103	1724	Assembly Syntax: Vx32.w+=vmpyie(Vu32.w,Vv32.uh)
2104	1725	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyieacc_VwVwVuh(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2107,9 +1728,7 @@
2107	1728	========================================================================== */
2108	1729
2109	1730	#define Q6_Vw_vmpyieacc_VwVwVuh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiewuh_acc)(Vx,Vu,Vv)
2110		#endif /* __HEXAGON_ARCH___ >= 60 */
2111	1731
2112		#if __HVX_ARCH__ >= 60
2113	1732	/* ==========================================================================
2114	1733	Assembly Syntax: Vd32.h=vmpyi(Vu32.h,Vv32.h)
2115	1734	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpyi_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2118,9 +1737,7 @@
2118	1737	========================================================================== */
2119	1738
2120	1739	#define Q6_Vh_vmpyi_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih)(Vu,Vv)
2121		#endif /* __HEXAGON_ARCH___ >= 60 */
2122	1740
2123		#if __HVX_ARCH__ >= 60
2124	1741	/* ==========================================================================
2125	1742	Assembly Syntax: Vx32.h+=vmpyi(Vu32.h,Vv32.h)
2126	1743	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpyiacc_VhVhVh(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2129,9 +1746,7 @@
2129	1746	========================================================================== */
2130	1747
2131	1748	#define Q6_Vh_vmpyiacc_VhVhVh(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyih_acc)(Vx,Vu,Vv)
2132		#endif /* __HEXAGON_ARCH___ >= 60 */
2133	1749
2134		#if __HVX_ARCH__ >= 60
2135	1750	/* ==========================================================================
2136	1751	Assembly Syntax: Vd32.h=vmpyi(Vu32.h,Rt32.b)
2137	1752	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpyi_VhRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2140,9 +1755,7 @@
2140	1755	========================================================================== */
2141	1756
2142	1757	#define Q6_Vh_vmpyi_VhRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb)(Vu,Rt)
2143		#endif /* __HEXAGON_ARCH___ >= 60 */
2144	1758
2145		#if __HVX_ARCH__ >= 60
2146	1759	/* ==========================================================================
2147	1760	Assembly Syntax: Vx32.h+=vmpyi(Vu32.h,Rt32.b)
2148	1761	C Intrinsic Prototype: HVX_Vector Q6_Vh_vmpyiacc_VhVhRb(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2151,9 +1764,7 @@
2151	1764	========================================================================== */
2152	1765
2153	1766	#define Q6_Vh_vmpyiacc_VhVhRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyihb_acc)(Vx,Vu,Rt)
2154		#endif /* __HEXAGON_ARCH___ >= 60 */
2155	1767
2156		#if __HVX_ARCH__ >= 60
2157	1768	/* ==========================================================================
2158	1769	Assembly Syntax: Vd32.w=vmpyio(Vu32.w,Vv32.h)
2159	1770	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyio_VwVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2162,9 +1773,7 @@
2162	1773	========================================================================== */
2163	1774
2164	1775	#define Q6_Vw_vmpyio_VwVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiowh)(Vu,Vv)
2165		#endif /* __HEXAGON_ARCH___ >= 60 */
2166	1776
2167		#if __HVX_ARCH__ >= 60
2168	1777	/* ==========================================================================
2169	1778	Assembly Syntax: Vd32.w=vmpyi(Vu32.w,Rt32.b)
2170	1779	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyi_VwRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2173,9 +1782,7 @@
2173	1782	========================================================================== */
2174	1783
2175	1784	#define Q6_Vw_vmpyi_VwRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb)(Vu,Rt)
2176		#endif /* __HEXAGON_ARCH___ >= 60 */
2177	1785
2178		#if __HVX_ARCH__ >= 60
2179	1786	/* ==========================================================================
2180	1787	Assembly Syntax: Vx32.w+=vmpyi(Vu32.w,Rt32.b)
2181	1788	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyiacc_VwVwRb(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2184,9 +1791,7 @@
2184	1791	========================================================================== */
2185	1792
2186	1793	#define Q6_Vw_vmpyiacc_VwVwRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwb_acc)(Vx,Vu,Rt)
2187		#endif /* __HEXAGON_ARCH___ >= 60 */
2188	1794
2189		#if __HVX_ARCH__ >= 60
2190	1795	/* ==========================================================================
2191	1796	Assembly Syntax: Vd32.w=vmpyi(Vu32.w,Rt32.h)
2192	1797	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyi_VwRh(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2195,9 +1800,7 @@
2195	1800	========================================================================== */
2196	1801
2197	1802	#define Q6_Vw_vmpyi_VwRh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh)(Vu,Rt)
2198		#endif /* __HEXAGON_ARCH___ >= 60 */
2199	1803
2200		#if __HVX_ARCH__ >= 60
2201	1804	/* ==========================================================================
2202	1805	Assembly Syntax: Vx32.w+=vmpyi(Vu32.w,Rt32.h)
2203	1806	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyiacc_VwVwRh(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2206,9 +1809,7 @@
2206	1809	========================================================================== */
2207	1810
2208	1811	#define Q6_Vw_vmpyiacc_VwVwRh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwh_acc)(Vx,Vu,Rt)
2209		#endif /* __HEXAGON_ARCH___ >= 60 */
2210	1812
2211		#if __HVX_ARCH__ >= 60
2212	1813	/* ==========================================================================
2213	1814	Assembly Syntax: Vd32.w=vmpyo(Vu32.w,Vv32.h):<<1:sat
2214	1815	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyo_VwVh_s1_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2217,9 +1818,7 @@
2217	1818	========================================================================== */
2218	1819
2219	1820	#define Q6_Vw_vmpyo_VwVh_s1_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh)(Vu,Vv)
2220		#endif /* __HEXAGON_ARCH___ >= 60 */
2221	1821
2222		#if __HVX_ARCH__ >= 60
2223	1822	/* ==========================================================================
2224	1823	Assembly Syntax: Vd32.w=vmpyo(Vu32.w,Vv32.h):<<1:rnd:sat
2225	1824	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyo_VwVh_s1_rnd_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2228,9 +1827,7 @@
2228	1827	========================================================================== */
2229	1828
2230	1829	#define Q6_Vw_vmpyo_VwVh_s1_rnd_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd)(Vu,Vv)
2231		#endif /* __HEXAGON_ARCH___ >= 60 */
2232	1830
2233		#if __HVX_ARCH__ >= 60
2234	1831	/* ==========================================================================
2235	1832	Assembly Syntax: Vx32.w+=vmpyo(Vu32.w,Vv32.h):<<1:rnd:sat:shift
2236	1833	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2239,9 +1836,7 @@
2239	1836	========================================================================== */
2240	1837
2241	1838	#define Q6_Vw_vmpyoacc_VwVwVh_s1_rnd_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_rnd_sacc)(Vx,Vu,Vv)
2242		#endif /* __HEXAGON_ARCH___ >= 60 */
2243	1839
2244		#if __HVX_ARCH__ >= 60
2245	1840	/* ==========================================================================
2246	1841	Assembly Syntax: Vx32.w+=vmpyo(Vu32.w,Vv32.h):<<1:sat:shift
2247	1842	C Intrinsic Prototype: HVX_Vector Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2250,9 +1845,7 @@
2250	1845	========================================================================== */
2251	1846
2252	1847	#define Q6_Vw_vmpyoacc_VwVwVh_s1_sat_shift(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_sacc)(Vx,Vu,Vv)
2253		#endif /* __HEXAGON_ARCH___ >= 60 */
2254	1848
2255		#if __HVX_ARCH__ >= 60
2256	1849	/* ==========================================================================
2257	1850	Assembly Syntax: Vdd32.uh=vmpy(Vu32.ub,Rt32.ub)
2258	1851	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vmpy_VubRub(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2261,9 +1854,7 @@
2261	1854	========================================================================== */
2262	1855
2263	1856	#define Q6_Wuh_vmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub)(Vu,Rt)
2264		#endif /* __HEXAGON_ARCH___ >= 60 */
2265	1857
2266		#if __HVX_ARCH__ >= 60
2267	1858	/* ==========================================================================
2268	1859	Assembly Syntax: Vxx32.uh+=vmpy(Vu32.ub,Rt32.ub)
2269	1860	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vmpyacc_WuhVubRub(HVX_VectorPair Vxx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2272,9 +1863,7 @@
2272	1863	========================================================================== */
2273	1864
2274	1865	#define Q6_Wuh_vmpyacc_WuhVubRub(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyub_acc)(Vxx,Vu,Rt)
2275		#endif /* __HEXAGON_ARCH___ >= 60 */
2276	1866
2277		#if __HVX_ARCH__ >= 60
2278	1867	/* ==========================================================================
2279	1868	Assembly Syntax: Vdd32.uh=vmpy(Vu32.ub,Vv32.ub)
2280	1869	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vmpy_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2283,9 +1872,7 @@
2283	1872	========================================================================== */
2284	1873
2285	1874	#define Q6_Wuh_vmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv)(Vu,Vv)
2286		#endif /* __HEXAGON_ARCH___ >= 60 */
2287	1875
2288		#if __HVX_ARCH__ >= 60
2289	1876	/* ==========================================================================
2290	1877	Assembly Syntax: Vxx32.uh+=vmpy(Vu32.ub,Vv32.ub)
2291	1878	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vmpyacc_WuhVubVub(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2294,9 +1881,7 @@
2294	1881	========================================================================== */
2295	1882
2296	1883	#define Q6_Wuh_vmpyacc_WuhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyubv_acc)(Vxx,Vu,Vv)
2297		#endif /* __HEXAGON_ARCH___ >= 60 */
2298	1884
2299		#if __HVX_ARCH__ >= 60
2300	1885	/* ==========================================================================
2301	1886	Assembly Syntax: Vdd32.uw=vmpy(Vu32.uh,Rt32.uh)
2302	1887	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vmpy_VuhRuh(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2305,9 +1890,7 @@
2305	1890	========================================================================== */
2306	1891
2307	1892	#define Q6_Wuw_vmpy_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh)(Vu,Rt)
2308		#endif /* __HEXAGON_ARCH___ >= 60 */
2309	1893
2310		#if __HVX_ARCH__ >= 60
2311	1894	/* ==========================================================================
2312	1895	Assembly Syntax: Vxx32.uw+=vmpy(Vu32.uh,Rt32.uh)
2313	1896	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vmpyacc_WuwVuhRuh(HVX_VectorPair Vxx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2316,9 +1899,7 @@
2316	1899	========================================================================== */
2317	1900
2318	1901	#define Q6_Wuw_vmpyacc_WuwVuhRuh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuh_acc)(Vxx,Vu,Rt)
2319		#endif /* __HEXAGON_ARCH___ >= 60 */
2320	1902
2321		#if __HVX_ARCH__ >= 60
2322	1903	/* ==========================================================================
2323	1904	Assembly Syntax: Vdd32.uw=vmpy(Vu32.uh,Vv32.uh)
2324	1905	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vmpy_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2327,9 +1908,7 @@
2327	1908	========================================================================== */
2328	1909
2329	1910	#define Q6_Wuw_vmpy_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv)(Vu,Vv)
2330		#endif /* __HEXAGON_ARCH___ >= 60 */
2331	1911
2332		#if __HVX_ARCH__ >= 60
2333	1912	/* ==========================================================================
2334	1913	Assembly Syntax: Vxx32.uw+=vmpy(Vu32.uh,Vv32.uh)
2335	1914	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vmpyacc_WuwVuhVuh(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2338,9 +1917,7 @@
2338	1917	========================================================================== */
2339	1918
2340	1919	#define Q6_Wuw_vmpyacc_WuwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhv_acc)(Vxx,Vu,Vv)
2341		#endif /* __HEXAGON_ARCH___ >= 60 */
2342	1920
2343		#if __HVX_ARCH__ >= 60
2344	1921	/* ==========================================================================
2345	1922	Assembly Syntax: Vd32=vmux(Qt4,Vu32,Vv32)
2346	1923	C Intrinsic Prototype: HVX_Vector Q6_V_vmux_QVV(HVX_VectorPred Qt, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2349,9 +1926,7 @@
2349	1926	========================================================================== */
2350	1927
2351	1928	#define Q6_V_vmux_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmux)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv)
2352		#endif /* __HEXAGON_ARCH___ >= 60 */
2353	1929
2354		#if __HVX_ARCH__ >= 60
2355	1930	/* ==========================================================================
2356	1931	Assembly Syntax: Vd32.h=vnavg(Vu32.h,Vv32.h)
2357	1932	C Intrinsic Prototype: HVX_Vector Q6_Vh_vnavg_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2360,9 +1935,7 @@
2360	1935	========================================================================== */
2361	1936
2362	1937	#define Q6_Vh_vnavg_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgh)(Vu,Vv)
2363		#endif /* __HEXAGON_ARCH___ >= 60 */
2364	1938
2365		#if __HVX_ARCH__ >= 60
2366	1939	/* ==========================================================================
2367	1940	Assembly Syntax: Vd32.b=vnavg(Vu32.ub,Vv32.ub)
2368	1941	C Intrinsic Prototype: HVX_Vector Q6_Vb_vnavg_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2371,9 +1944,7 @@
2371	1944	========================================================================== */
2372	1945
2373	1946	#define Q6_Vb_vnavg_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgub)(Vu,Vv)
2374		#endif /* __HEXAGON_ARCH___ >= 60 */
2375	1947
2376		#if __HVX_ARCH__ >= 60
2377	1948	/* ==========================================================================
2378	1949	Assembly Syntax: Vd32.w=vnavg(Vu32.w,Vv32.w)
2379	1950	C Intrinsic Prototype: HVX_Vector Q6_Vw_vnavg_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2382,9 +1953,7 @@
2382	1953	========================================================================== */
2383	1954
2384	1955	#define Q6_Vw_vnavg_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgw)(Vu,Vv)
2385		#endif /* __HEXAGON_ARCH___ >= 60 */
2386	1956
2387		#if __HVX_ARCH__ >= 60
2388	1957	/* ==========================================================================
2389	1958	Assembly Syntax: Vd32.h=vnormamt(Vu32.h)
2390	1959	C Intrinsic Prototype: HVX_Vector Q6_Vh_vnormamt_Vh(HVX_Vector Vu)
...	...	@@ -2393,9 +1962,7 @@
2393	1962	========================================================================== */
2394	1963
2395	1964	#define Q6_Vh_vnormamt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamth)(Vu)
2396		#endif /* __HEXAGON_ARCH___ >= 60 */
2397	1965
2398		#if __HVX_ARCH__ >= 60
2399	1966	/* ==========================================================================
2400	1967	Assembly Syntax: Vd32.w=vnormamt(Vu32.w)
2401	1968	C Intrinsic Prototype: HVX_Vector Q6_Vw_vnormamt_Vw(HVX_Vector Vu)
...	...	@@ -2404,9 +1971,7 @@
2404	1971	========================================================================== */
2405	1972
2406	1973	#define Q6_Vw_vnormamt_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnormamtw)(Vu)
2407		#endif /* __HEXAGON_ARCH___ >= 60 */
2408	1974
2409		#if __HVX_ARCH__ >= 60
2410	1975	/* ==========================================================================
2411	1976	Assembly Syntax: Vd32=vnot(Vu32)
2412	1977	C Intrinsic Prototype: HVX_Vector Q6_V_vnot_V(HVX_Vector Vu)
...	...	@@ -2415,9 +1980,7 @@
2415	1980	========================================================================== */
2416	1981
2417	1982	#define Q6_V_vnot_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnot)(Vu)
2418		#endif /* __HEXAGON_ARCH___ >= 60 */
2419	1983
2420		#if __HVX_ARCH__ >= 60
2421	1984	/* ==========================================================================
2422	1985	Assembly Syntax: Vd32=vor(Vu32,Vv32)
2423	1986	C Intrinsic Prototype: HVX_Vector Q6_V_vor_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2426,9 +1989,7 @@
2426	1989	========================================================================== */
2427	1990
2428	1991	#define Q6_V_vor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vor)(Vu,Vv)
2429		#endif /* __HEXAGON_ARCH___ >= 60 */
2430	1992
2431		#if __HVX_ARCH__ >= 60
2432	1993	/* ==========================================================================
2433	1994	Assembly Syntax: Vd32.b=vpacke(Vu32.h,Vv32.h)
2434	1995	C Intrinsic Prototype: HVX_Vector Q6_Vb_vpacke_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2437,9 +1998,7 @@
2437	1998	========================================================================== */
2438	1999
2439	2000	#define Q6_Vb_vpacke_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeb)(Vu,Vv)
2440		#endif /* __HEXAGON_ARCH___ >= 60 */
2441	2001
2442		#if __HVX_ARCH__ >= 60
2443	2002	/* ==========================================================================
2444	2003	Assembly Syntax: Vd32.h=vpacke(Vu32.w,Vv32.w)
2445	2004	C Intrinsic Prototype: HVX_Vector Q6_Vh_vpacke_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2448,9 +2007,7 @@
2448	2007	========================================================================== */
2449	2008
2450	2009	#define Q6_Vh_vpacke_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackeh)(Vu,Vv)
2451		#endif /* __HEXAGON_ARCH___ >= 60 */
2452	2010
2453		#if __HVX_ARCH__ >= 60
2454	2011	/* ==========================================================================
2455	2012	Assembly Syntax: Vd32.b=vpack(Vu32.h,Vv32.h):sat
2456	2013	C Intrinsic Prototype: HVX_Vector Q6_Vb_vpack_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2459,9 +2016,7 @@
2459	2016	========================================================================== */
2460	2017
2461	2018	#define Q6_Vb_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhb_sat)(Vu,Vv)
2462		#endif /* __HEXAGON_ARCH___ >= 60 */
2463	2019
2464		#if __HVX_ARCH__ >= 60
2465	2020	/* ==========================================================================
2466	2021	Assembly Syntax: Vd32.ub=vpack(Vu32.h,Vv32.h):sat
2467	2022	C Intrinsic Prototype: HVX_Vector Q6_Vub_vpack_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2470,9 +2025,7 @@
2470	2025	========================================================================== */
2471	2026
2472	2027	#define Q6_Vub_vpack_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackhub_sat)(Vu,Vv)
2473		#endif /* __HEXAGON_ARCH___ >= 60 */
2474	2028
2475		#if __HVX_ARCH__ >= 60
2476	2029	/* ==========================================================================
2477	2030	Assembly Syntax: Vd32.b=vpacko(Vu32.h,Vv32.h)
2478	2031	C Intrinsic Prototype: HVX_Vector Q6_Vb_vpacko_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2481,9 +2034,7 @@
2481	2034	========================================================================== */
2482	2035
2483	2036	#define Q6_Vb_vpacko_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackob)(Vu,Vv)
2484		#endif /* __HEXAGON_ARCH___ >= 60 */
2485	2037
2486		#if __HVX_ARCH__ >= 60
2487	2038	/* ==========================================================================
2488	2039	Assembly Syntax: Vd32.h=vpacko(Vu32.w,Vv32.w)
2489	2040	C Intrinsic Prototype: HVX_Vector Q6_Vh_vpacko_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2492,9 +2043,7 @@
2492	2043	========================================================================== */
2493	2044
2494	2045	#define Q6_Vh_vpacko_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackoh)(Vu,Vv)
2495		#endif /* __HEXAGON_ARCH___ >= 60 */
2496	2046
2497		#if __HVX_ARCH__ >= 60
2498	2047	/* ==========================================================================
2499	2048	Assembly Syntax: Vd32.h=vpack(Vu32.w,Vv32.w):sat
2500	2049	C Intrinsic Prototype: HVX_Vector Q6_Vh_vpack_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2503,9 +2052,7 @@
2503	2052	========================================================================== */
2504	2053
2505	2054	#define Q6_Vh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwh_sat)(Vu,Vv)
2506		#endif /* __HEXAGON_ARCH___ >= 60 */
2507	2055
2508		#if __HVX_ARCH__ >= 60
2509	2056	/* ==========================================================================
2510	2057	Assembly Syntax: Vd32.uh=vpack(Vu32.w,Vv32.w):sat
2511	2058	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vpack_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2514,9 +2061,7 @@
2514	2061	========================================================================== */
2515	2062
2516	2063	#define Q6_Vuh_vpack_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpackwuh_sat)(Vu,Vv)
2517		#endif /* __HEXAGON_ARCH___ >= 60 */
2518	2064
2519		#if __HVX_ARCH__ >= 60
2520	2065	/* ==========================================================================
2521	2066	Assembly Syntax: Vd32.h=vpopcount(Vu32.h)
2522	2067	C Intrinsic Prototype: HVX_Vector Q6_Vh_vpopcount_Vh(HVX_Vector Vu)
...	...	@@ -2525,9 +2070,7 @@
2525	2070	========================================================================== */
2526	2071
2527	2072	#define Q6_Vh_vpopcount_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vpopcounth)(Vu)
2528		#endif /* __HEXAGON_ARCH___ >= 60 */
2529	2073
2530		#if __HVX_ARCH__ >= 60
2531	2074	/* ==========================================================================
2532	2075	Assembly Syntax: Vd32=vrdelta(Vu32,Vv32)
2533	2076	C Intrinsic Prototype: HVX_Vector Q6_V_vrdelta_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2536,9 +2079,7 @@
2536	2079	========================================================================== */
2537	2080
2538	2081	#define Q6_V_vrdelta_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrdelta)(Vu,Vv)
2539		#endif /* __HEXAGON_ARCH___ >= 60 */
2540	2082
2541		#if __HVX_ARCH__ >= 60
2542	2083	/* ==========================================================================
2543	2084	Assembly Syntax: Vd32.w=vrmpy(Vu32.ub,Rt32.b)
2544	2085	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpy_VubRb(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2547,9 +2088,7 @@
2547	2088	========================================================================== */
2548	2089
2549	2090	#define Q6_Vw_vrmpy_VubRb(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus)(Vu,Rt)
2550		#endif /* __HEXAGON_ARCH___ >= 60 */
2551	2091
2552		#if __HVX_ARCH__ >= 60
2553	2092	/* ==========================================================================
2554	2093	Assembly Syntax: Vx32.w+=vrmpy(Vu32.ub,Rt32.b)
2555	2094	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVubRb(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2558,9 +2097,7 @@
2558	2097	========================================================================== */
2559	2098
2560	2099	#define Q6_Vw_vrmpyacc_VwVubRb(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybus_acc)(Vx,Vu,Rt)
2561		#endif /* __HEXAGON_ARCH___ >= 60 */
2562	2100
2563		#if __HVX_ARCH__ >= 60
2564	2101	/* ==========================================================================
2565	2102	Assembly Syntax: Vdd32.w=vrmpy(Vuu32.ub,Rt32.b,#u1)
2566	2103	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vrmpy_WubRbI(HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2569,9 +2106,7 @@
2569	2106	========================================================================== */
2570	2107
2571	2108	#define Q6_Ww_vrmpy_WubRbI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi)(Vuu,Rt,Iu1)
2572		#endif /* __HEXAGON_ARCH___ >= 60 */
2573	2109
2574		#if __HVX_ARCH__ >= 60
2575	2110	/* ==========================================================================
2576	2111	Assembly Syntax: Vxx32.w+=vrmpy(Vuu32.ub,Rt32.b,#u1)
2577	2112	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vrmpyacc_WwWubRbI(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2580,9 +2115,7 @@
2580	2115	========================================================================== */
2581	2116
2582	2117	#define Q6_Ww_vrmpyacc_WwWubRbI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusi_acc)(Vxx,Vuu,Rt,Iu1)
2583		#endif /* __HEXAGON_ARCH___ >= 60 */
2584	2118
2585		#if __HVX_ARCH__ >= 60
2586	2119	/* ==========================================================================
2587	2120	Assembly Syntax: Vd32.w=vrmpy(Vu32.ub,Vv32.b)
2588	2121	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpy_VubVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2591,9 +2124,7 @@
2591	2124	========================================================================== */
2592	2125
2593	2126	#define Q6_Vw_vrmpy_VubVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv)(Vu,Vv)
2594		#endif /* __HEXAGON_ARCH___ >= 60 */
2595	2127
2596		#if __HVX_ARCH__ >= 60
2597	2128	/* ==========================================================================
2598	2129	Assembly Syntax: Vx32.w+=vrmpy(Vu32.ub,Vv32.b)
2599	2130	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVubVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2602,9 +2133,7 @@
2602	2133	========================================================================== */
2603	2134
2604	2135	#define Q6_Vw_vrmpyacc_VwVubVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybusv_acc)(Vx,Vu,Vv)
2605		#endif /* __HEXAGON_ARCH___ >= 60 */
2606	2136
2607		#if __HVX_ARCH__ >= 60
2608	2137	/* ==========================================================================
2609	2138	Assembly Syntax: Vd32.w=vrmpy(Vu32.b,Vv32.b)
2610	2139	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpy_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2613,9 +2142,7 @@
2613	2142	========================================================================== */
2614	2143
2615	2144	#define Q6_Vw_vrmpy_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv)(Vu,Vv)
2616		#endif /* __HEXAGON_ARCH___ >= 60 */
2617	2145
2618		#if __HVX_ARCH__ >= 60
2619	2146	/* ==========================================================================
2620	2147	Assembly Syntax: Vx32.w+=vrmpy(Vu32.b,Vv32.b)
2621	2148	C Intrinsic Prototype: HVX_Vector Q6_Vw_vrmpyacc_VwVbVb(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2624,9 +2151,7 @@
2624	2151	========================================================================== */
2625	2152
2626	2153	#define Q6_Vw_vrmpyacc_VwVbVb(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpybv_acc)(Vx,Vu,Vv)
2627		#endif /* __HEXAGON_ARCH___ >= 60 */
2628	2154
2629		#if __HVX_ARCH__ >= 60
2630	2155	/* ==========================================================================
2631	2156	Assembly Syntax: Vd32.uw=vrmpy(Vu32.ub,Rt32.ub)
2632	2157	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpy_VubRub(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2635,9 +2160,7 @@
2635	2160	========================================================================== */
2636	2161
2637	2162	#define Q6_Vuw_vrmpy_VubRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub)(Vu,Rt)
2638		#endif /* __HEXAGON_ARCH___ >= 60 */
2639	2163
2640		#if __HVX_ARCH__ >= 60
2641	2164	/* ==========================================================================
2642	2165	Assembly Syntax: Vx32.uw+=vrmpy(Vu32.ub,Rt32.ub)
2643	2166	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpyacc_VuwVubRub(HVX_Vector Vx, HVX_Vector Vu, Word32 Rt)
...	...	@@ -2646,9 +2169,7 @@
2646	2169	========================================================================== */
2647	2170
2648	2171	#define Q6_Vuw_vrmpyacc_VuwVubRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyub_acc)(Vx,Vu,Rt)
2649		#endif /* __HEXAGON_ARCH___ >= 60 */
2650	2172
2651		#if __HVX_ARCH__ >= 60
2652	2173	/* ==========================================================================
2653	2174	Assembly Syntax: Vdd32.uw=vrmpy(Vuu32.ub,Rt32.ub,#u1)
2654	2175	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vrmpy_WubRubI(HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2657,9 +2178,7 @@
2657	2178	========================================================================== */
2658	2179
2659	2180	#define Q6_Wuw_vrmpy_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi)(Vuu,Rt,Iu1)
2660		#endif /* __HEXAGON_ARCH___ >= 60 */
2661	2181
2662		#if __HVX_ARCH__ >= 60
2663	2182	/* ==========================================================================
2664	2183	Assembly Syntax: Vxx32.uw+=vrmpy(Vuu32.ub,Rt32.ub,#u1)
2665	2184	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vrmpyacc_WuwWubRubI(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2668,9 +2187,7 @@
2668	2187	========================================================================== */
2669	2188
2670	2189	#define Q6_Wuw_vrmpyacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubi_acc)(Vxx,Vuu,Rt,Iu1)
2671		#endif /* __HEXAGON_ARCH___ >= 60 */
2672	2190
2673		#if __HVX_ARCH__ >= 60
2674	2191	/* ==========================================================================
2675	2192	Assembly Syntax: Vd32.uw=vrmpy(Vu32.ub,Vv32.ub)
2676	2193	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpy_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2679,9 +2196,7 @@
2679	2196	========================================================================== */
2680	2197
2681	2198	#define Q6_Vuw_vrmpy_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv)(Vu,Vv)
2682		#endif /* __HEXAGON_ARCH___ >= 60 */
2683	2199
2684		#if __HVX_ARCH__ >= 60
2685	2200	/* ==========================================================================
2686	2201	Assembly Syntax: Vx32.uw+=vrmpy(Vu32.ub,Vv32.ub)
2687	2202	C Intrinsic Prototype: HVX_Vector Q6_Vuw_vrmpyacc_VuwVubVub(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2690,9 +2205,7 @@
2690	2205	========================================================================== */
2691	2206
2692	2207	#define Q6_Vuw_vrmpyacc_VuwVubVub(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrmpyubv_acc)(Vx,Vu,Vv)
2693		#endif /* __HEXAGON_ARCH___ >= 60 */
2694	2208
2695		#if __HVX_ARCH__ >= 60
2696	2209	/* ==========================================================================
2697	2210	Assembly Syntax: Vd32=vror(Vu32,Rt32)
2698	2211	C Intrinsic Prototype: HVX_Vector Q6_V_vror_VR(HVX_Vector Vu, Word32 Rt)
...	...	@@ -2701,9 +2214,7 @@
2701	2214	========================================================================== */
2702	2215
2703	2216	#define Q6_V_vror_VR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vror)(Vu,Rt)
2704		#endif /* __HEXAGON_ARCH___ >= 60 */
2705	2217
2706		#if __HVX_ARCH__ >= 60
2707	2218	/* ==========================================================================
2708	2219	Assembly Syntax: Vd32.b=vround(Vu32.h,Vv32.h):sat
2709	2220	C Intrinsic Prototype: HVX_Vector Q6_Vb_vround_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2712,9 +2223,7 @@
2712	2223	========================================================================== */
2713	2224
2714	2225	#define Q6_Vb_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhb)(Vu,Vv)
2715		#endif /* __HEXAGON_ARCH___ >= 60 */
2716	2226
2717		#if __HVX_ARCH__ >= 60
2718	2227	/* ==========================================================================
2719	2228	Assembly Syntax: Vd32.ub=vround(Vu32.h,Vv32.h):sat
2720	2229	C Intrinsic Prototype: HVX_Vector Q6_Vub_vround_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2723,9 +2232,7 @@
2723	2232	========================================================================== */
2724	2233
2725	2234	#define Q6_Vub_vround_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundhub)(Vu,Vv)
2726		#endif /* __HEXAGON_ARCH___ >= 60 */
2727	2235
2728		#if __HVX_ARCH__ >= 60
2729	2236	/* ==========================================================================
2730	2237	Assembly Syntax: Vd32.h=vround(Vu32.w,Vv32.w):sat
2731	2238	C Intrinsic Prototype: HVX_Vector Q6_Vh_vround_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2734,9 +2241,7 @@
2734	2241	========================================================================== */
2735	2242
2736	2243	#define Q6_Vh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwh)(Vu,Vv)
2737		#endif /* __HEXAGON_ARCH___ >= 60 */
2738	2244
2739		#if __HVX_ARCH__ >= 60
2740	2245	/* ==========================================================================
2741	2246	Assembly Syntax: Vd32.uh=vround(Vu32.w,Vv32.w):sat
2742	2247	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vround_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2745,9 +2250,7 @@
2745	2250	========================================================================== */
2746	2251
2747	2252	#define Q6_Vuh_vround_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vroundwuh)(Vu,Vv)
2748		#endif /* __HEXAGON_ARCH___ >= 60 */
2749	2253
2750		#if __HVX_ARCH__ >= 60
2751	2254	/* ==========================================================================
2752	2255	Assembly Syntax: Vdd32.uw=vrsad(Vuu32.ub,Rt32.ub,#u1)
2753	2256	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vrsad_WubRubI(HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2756,9 +2259,7 @@
2756	2259	========================================================================== */
2757	2260
2758	2261	#define Q6_Wuw_vrsad_WubRubI(Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi)(Vuu,Rt,Iu1)
2759		#endif /* __HEXAGON_ARCH___ >= 60 */
2760	2262
2761		#if __HVX_ARCH__ >= 60
2762	2263	/* ==========================================================================
2763	2264	Assembly Syntax: Vxx32.uw+=vrsad(Vuu32.ub,Rt32.ub,#u1)
2764	2265	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vrsadacc_WuwWubRubI(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt, Word32 Iu1)
...	...	@@ -2767,9 +2268,7 @@
2767	2268	========================================================================== */
2768	2269
2769	2270	#define Q6_Wuw_vrsadacc_WuwWubRubI(Vxx,Vuu,Rt,Iu1) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrsadubi_acc)(Vxx,Vuu,Rt,Iu1)
2770		#endif /* __HEXAGON_ARCH___ >= 60 */
2771	2271
2772		#if __HVX_ARCH__ >= 60
2773	2272	/* ==========================================================================
2774	2273	Assembly Syntax: Vd32.ub=vsat(Vu32.h,Vv32.h)
2775	2274	C Intrinsic Prototype: HVX_Vector Q6_Vub_vsat_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2778,9 +2277,7 @@
2778	2277	========================================================================== */
2779	2278
2780	2279	#define Q6_Vub_vsat_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsathub)(Vu,Vv)
2781		#endif /* __HEXAGON_ARCH___ >= 60 */
2782	2280
2783		#if __HVX_ARCH__ >= 60
2784	2281	/* ==========================================================================
2785	2282	Assembly Syntax: Vd32.h=vsat(Vu32.w,Vv32.w)
2786	2283	C Intrinsic Prototype: HVX_Vector Q6_Vh_vsat_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2789,9 +2286,7 @@
2789	2286	========================================================================== */
2790	2287
2791	2288	#define Q6_Vh_vsat_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatwh)(Vu,Vv)
2792		#endif /* __HEXAGON_ARCH___ >= 60 */
2793	2289
2794		#if __HVX_ARCH__ >= 60
2795	2290	/* ==========================================================================
2796	2291	Assembly Syntax: Vdd32.h=vsxt(Vu32.b)
2797	2292	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vsxt_Vb(HVX_Vector Vu)
...	...	@@ -2800,9 +2295,7 @@
2800	2295	========================================================================== */
2801	2296
2802	2297	#define Q6_Wh_vsxt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsb)(Vu)
2803		#endif /* __HEXAGON_ARCH___ >= 60 */
2804	2298
2805		#if __HVX_ARCH__ >= 60
2806	2299	/* ==========================================================================
2807	2300	Assembly Syntax: Vdd32.w=vsxt(Vu32.h)
2808	2301	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vsxt_Vh(HVX_Vector Vu)
...	...	@@ -2811,9 +2304,7 @@
2811	2304	========================================================================== */
2812	2305
2813	2306	#define Q6_Ww_vsxt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsh)(Vu)
2814		#endif /* __HEXAGON_ARCH___ >= 60 */
2815	2307
2816		#if __HVX_ARCH__ >= 60
2817	2308	/* ==========================================================================
2818	2309	Assembly Syntax: Vd32.h=vshuffe(Vu32.h,Vv32.h)
2819	2310	C Intrinsic Prototype: HVX_Vector Q6_Vh_vshuffe_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2822,9 +2313,7 @@
2822	2313	========================================================================== */
2823	2314
2824	2315	#define Q6_Vh_vshuffe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufeh)(Vu,Vv)
2825		#endif /* __HEXAGON_ARCH___ >= 60 */
2826	2316
2827		#if __HVX_ARCH__ >= 60
2828	2317	/* ==========================================================================
2829	2318	Assembly Syntax: Vd32.b=vshuff(Vu32.b)
2830	2319	C Intrinsic Prototype: HVX_Vector Q6_Vb_vshuff_Vb(HVX_Vector Vu)
...	...	@@ -2833,9 +2322,7 @@
2833	2322	========================================================================== */
2834	2323
2835	2324	#define Q6_Vb_vshuff_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffb)(Vu)
2836		#endif /* __HEXAGON_ARCH___ >= 60 */
2837	2325
2838		#if __HVX_ARCH__ >= 60
2839	2326	/* ==========================================================================
2840	2327	Assembly Syntax: Vd32.b=vshuffe(Vu32.b,Vv32.b)
2841	2328	C Intrinsic Prototype: HVX_Vector Q6_Vb_vshuffe_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2844,9 +2331,7 @@
2844	2331	========================================================================== */
2845	2332
2846	2333	#define Q6_Vb_vshuffe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffeb)(Vu,Vv)
2847		#endif /* __HEXAGON_ARCH___ >= 60 */
2848	2334
2849		#if __HVX_ARCH__ >= 60
2850	2335	/* ==========================================================================
2851	2336	Assembly Syntax: Vd32.h=vshuff(Vu32.h)
2852	2337	C Intrinsic Prototype: HVX_Vector Q6_Vh_vshuff_Vh(HVX_Vector Vu)
...	...	@@ -2855,9 +2340,7 @@
2855	2340	========================================================================== */
2856	2341
2857	2342	#define Q6_Vh_vshuff_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffh)(Vu)
2858		#endif /* __HEXAGON_ARCH___ >= 60 */
2859	2343
2860		#if __HVX_ARCH__ >= 60
2861	2344	/* ==========================================================================
2862	2345	Assembly Syntax: Vd32.b=vshuffo(Vu32.b,Vv32.b)
2863	2346	C Intrinsic Prototype: HVX_Vector Q6_Vb_vshuffo_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2866,9 +2349,7 @@
2866	2349	========================================================================== */
2867	2350
2868	2351	#define Q6_Vb_vshuffo_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffob)(Vu,Vv)
2869		#endif /* __HEXAGON_ARCH___ >= 60 */
2870	2352
2871		#if __HVX_ARCH__ >= 60
2872	2353	/* ==========================================================================
2873	2354	Assembly Syntax: Vdd32=vshuff(Vu32,Vv32,Rt8)
2874	2355	C Intrinsic Prototype: HVX_VectorPair Q6_W_vshuff_VVR(HVX_Vector Vu, HVX_Vector Vv, Word32 Rt)
...	...	@@ -2877,9 +2358,7 @@
2877	2358	========================================================================== */
2878	2359
2879	2360	#define Q6_W_vshuff_VVR(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshuffvdd)(Vu,Vv,Rt)
2880		#endif /* __HEXAGON_ARCH___ >= 60 */
2881	2361
2882		#if __HVX_ARCH__ >= 60
2883	2362	/* ==========================================================================
2884	2363	Assembly Syntax: Vdd32.b=vshuffoe(Vu32.b,Vv32.b)
2885	2364	C Intrinsic Prototype: HVX_VectorPair Q6_Wb_vshuffoe_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2888,9 +2367,7 @@
2888	2367	========================================================================== */
2889	2368
2890	2369	#define Q6_Wb_vshuffoe_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeb)(Vu,Vv)
2891		#endif /* __HEXAGON_ARCH___ >= 60 */
2892	2370
2893		#if __HVX_ARCH__ >= 60
2894	2371	/* ==========================================================================
2895	2372	Assembly Syntax: Vdd32.h=vshuffoe(Vu32.h,Vv32.h)
2896	2373	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vshuffoe_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2899,9 +2376,7 @@
2899	2376	========================================================================== */
2900	2377
2901	2378	#define Q6_Wh_vshuffoe_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoeh)(Vu,Vv)
2902		#endif /* __HEXAGON_ARCH___ >= 60 */
2903	2379
2904		#if __HVX_ARCH__ >= 60
2905	2380	/* ==========================================================================
2906	2381	Assembly Syntax: Vd32.h=vshuffo(Vu32.h,Vv32.h)
2907	2382	C Intrinsic Prototype: HVX_Vector Q6_Vh_vshuffo_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2910,9 +2385,7 @@
2910	2385	========================================================================== */
2911	2386
2912	2387	#define Q6_Vh_vshuffo_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vshufoh)(Vu,Vv)
2913		#endif /* __HEXAGON_ARCH___ >= 60 */
2914	2388
2915		#if __HVX_ARCH__ >= 60
2916	2389	/* ==========================================================================
2917	2390	Assembly Syntax: Vd32.b=vsub(Vu32.b,Vv32.b)
2918	2391	C Intrinsic Prototype: HVX_Vector Q6_Vb_vsub_VbVb(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2921,9 +2394,7 @@
2921	2394	========================================================================== */
2922	2395
2923	2396	#define Q6_Vb_vsub_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb)(Vu,Vv)
2924		#endif /* __HEXAGON_ARCH___ >= 60 */
2925	2397
2926		#if __HVX_ARCH__ >= 60
2927	2398	/* ==========================================================================
2928	2399	Assembly Syntax: Vdd32.b=vsub(Vuu32.b,Vvv32.b)
2929	2400	C Intrinsic Prototype: HVX_VectorPair Q6_Wb_vsub_WbWb(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -2932,9 +2403,7 @@
2932	2403	========================================================================== */
2933	2404
2934	2405	#define Q6_Wb_vsub_WbWb(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubb_dv)(Vuu,Vvv)
2935		#endif /* __HEXAGON_ARCH___ >= 60 */
2936	2406
2937		#if __HVX_ARCH__ >= 60
2938	2407	/* ==========================================================================
2939	2408	Assembly Syntax: if (!Qv4) Vx32.b-=Vu32.b
2940	2409	C Intrinsic Prototype: HVX_Vector Q6_Vb_condnac_QnVbVb(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -2943,9 +2412,7 @@
2943	2412	========================================================================== */
2944	2413
2945	2414	#define Q6_Vb_condnac_QnVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
2946		#endif /* __HEXAGON_ARCH___ >= 60 */
2947	2415
2948		#if __HVX_ARCH__ >= 60
2949	2416	/* ==========================================================================
2950	2417	Assembly Syntax: if (Qv4) Vx32.b-=Vu32.b
2951	2418	C Intrinsic Prototype: HVX_Vector Q6_Vb_condnac_QVbVb(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -2954,9 +2421,7 @@
2954	2421	========================================================================== */
2955	2422
2956	2423	#define Q6_Vb_condnac_QVbVb(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
2957		#endif /* __HEXAGON_ARCH___ >= 60 */
2958	2424
2959		#if __HVX_ARCH__ >= 60
2960	2425	/* ==========================================================================
2961	2426	Assembly Syntax: Vd32.h=vsub(Vu32.h,Vv32.h)
2962	2427	C Intrinsic Prototype: HVX_Vector Q6_Vh_vsub_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -2965,9 +2430,7 @@
2965	2430	========================================================================== */
2966	2431
2967	2432	#define Q6_Vh_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh)(Vu,Vv)
2968		#endif /* __HEXAGON_ARCH___ >= 60 */
2969	2433
2970		#if __HVX_ARCH__ >= 60
2971	2434	/* ==========================================================================
2972	2435	Assembly Syntax: Vdd32.h=vsub(Vuu32.h,Vvv32.h)
2973	2436	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vsub_WhWh(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -2976,9 +2439,7 @@
2976	2439	========================================================================== */
2977	2440
2978	2441	#define Q6_Wh_vsub_WhWh(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubh_dv)(Vuu,Vvv)
2979		#endif /* __HEXAGON_ARCH___ >= 60 */
2980	2442
2981		#if __HVX_ARCH__ >= 60
2982	2443	/* ==========================================================================
2983	2444	Assembly Syntax: if (!Qv4) Vx32.h-=Vu32.h
2984	2445	C Intrinsic Prototype: HVX_Vector Q6_Vh_condnac_QnVhVh(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -2987,9 +2448,7 @@
2987	2448	========================================================================== */
2988	2449
2989	2450	#define Q6_Vh_condnac_QnVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
2990		#endif /* __HEXAGON_ARCH___ >= 60 */
2991	2451
2992		#if __HVX_ARCH__ >= 60
2993	2452	/* ==========================================================================
2994	2453	Assembly Syntax: if (Qv4) Vx32.h-=Vu32.h
2995	2454	C Intrinsic Prototype: HVX_Vector Q6_Vh_condnac_QVhVh(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -2998,9 +2457,7 @@
2998	2457	========================================================================== */
2999	2458
3000	2459	#define Q6_Vh_condnac_QVhVh(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
3001		#endif /* __HEXAGON_ARCH___ >= 60 */
3002	2460
3003		#if __HVX_ARCH__ >= 60
3004	2461	/* ==========================================================================
3005	2462	Assembly Syntax: Vd32.h=vsub(Vu32.h,Vv32.h):sat
3006	2463	C Intrinsic Prototype: HVX_Vector Q6_Vh_vsub_VhVh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3009,9 +2466,7 @@
3009	2466	========================================================================== */
3010	2467
3011	2468	#define Q6_Vh_vsub_VhVh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat)(Vu,Vv)
3012		#endif /* __HEXAGON_ARCH___ >= 60 */
3013	2469
3014		#if __HVX_ARCH__ >= 60
3015	2470	/* ==========================================================================
3016	2471	Assembly Syntax: Vdd32.h=vsub(Vuu32.h,Vvv32.h):sat
3017	2472	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vsub_WhWh_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -3020,9 +2475,7 @@
3020	2475	========================================================================== */
3021	2476
3022	2477	#define Q6_Wh_vsub_WhWh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhsat_dv)(Vuu,Vvv)
3023		#endif /* __HEXAGON_ARCH___ >= 60 */
3024	2478
3025		#if __HVX_ARCH__ >= 60
3026	2479	/* ==========================================================================
3027	2480	Assembly Syntax: Vdd32.w=vsub(Vu32.h,Vv32.h)
3028	2481	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vsub_VhVh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3031,9 +2484,7 @@
3031	2484	========================================================================== */
3032	2485
3033	2486	#define Q6_Ww_vsub_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubhw)(Vu,Vv)
3034		#endif /* __HEXAGON_ARCH___ >= 60 */
3035	2487
3036		#if __HVX_ARCH__ >= 60
3037	2488	/* ==========================================================================
3038	2489	Assembly Syntax: Vdd32.h=vsub(Vu32.ub,Vv32.ub)
3039	2490	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vsub_VubVub(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3042,9 +2493,7 @@
3042	2493	========================================================================== */
3043	2494
3044	2495	#define Q6_Wh_vsub_VubVub(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububh)(Vu,Vv)
3045		#endif /* __HEXAGON_ARCH___ >= 60 */
3046	2496
3047		#if __HVX_ARCH__ >= 60
3048	2497	/* ==========================================================================
3049	2498	Assembly Syntax: Vd32.ub=vsub(Vu32.ub,Vv32.ub):sat
3050	2499	C Intrinsic Prototype: HVX_Vector Q6_Vub_vsub_VubVub_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3053,9 +2502,7 @@
3053	2502	========================================================================== */
3054	2503
3055	2504	#define Q6_Vub_vsub_VubVub_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat)(Vu,Vv)
3056		#endif /* __HEXAGON_ARCH___ >= 60 */
3057	2505
3058		#if __HVX_ARCH__ >= 60
3059	2506	/* ==========================================================================
3060	2507	Assembly Syntax: Vdd32.ub=vsub(Vuu32.ub,Vvv32.ub):sat
3061	2508	C Intrinsic Prototype: HVX_VectorPair Q6_Wub_vsub_WubWub_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -3064,9 +2511,7 @@
3064	2511	========================================================================== */
3065	2512
3066	2513	#define Q6_Wub_vsub_WubWub_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsububsat_dv)(Vuu,Vvv)
3067		#endif /* __HEXAGON_ARCH___ >= 60 */
3068	2514
3069		#if __HVX_ARCH__ >= 60
3070	2515	/* ==========================================================================
3071	2516	Assembly Syntax: Vd32.uh=vsub(Vu32.uh,Vv32.uh):sat
3072	2517	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vsub_VuhVuh_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3075,9 +2520,7 @@
3075	2520	========================================================================== */
3076	2521
3077	2522	#define Q6_Vuh_vsub_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat)(Vu,Vv)
3078		#endif /* __HEXAGON_ARCH___ >= 60 */
3079	2523
3080		#if __HVX_ARCH__ >= 60
3081	2524	/* ==========================================================================
3082	2525	Assembly Syntax: Vdd32.uh=vsub(Vuu32.uh,Vvv32.uh):sat
3083	2526	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vsub_WuhWuh_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -3086,9 +2529,7 @@
3086	2529	========================================================================== */
3087	2530
3088	2531	#define Q6_Wuh_vsub_WuhWuh_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhsat_dv)(Vuu,Vvv)
3089		#endif /* __HEXAGON_ARCH___ >= 60 */
3090	2532
3091		#if __HVX_ARCH__ >= 60
3092	2533	/* ==========================================================================
3093	2534	Assembly Syntax: Vdd32.w=vsub(Vu32.uh,Vv32.uh)
3094	2535	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vsub_VuhVuh(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3097,9 +2538,7 @@
3097	2538	========================================================================== */
3098	2539
3099	2540	#define Q6_Ww_vsub_VuhVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuhw)(Vu,Vv)
3100		#endif /* __HEXAGON_ARCH___ >= 60 */
3101	2541
3102		#if __HVX_ARCH__ >= 60
3103	2542	/* ==========================================================================
3104	2543	Assembly Syntax: Vd32.w=vsub(Vu32.w,Vv32.w)
3105	2544	C Intrinsic Prototype: HVX_Vector Q6_Vw_vsub_VwVw(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3108,9 +2547,7 @@
3108	2547	========================================================================== */
3109	2548
3110	2549	#define Q6_Vw_vsub_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw)(Vu,Vv)
3111		#endif /* __HEXAGON_ARCH___ >= 60 */
3112	2550
3113		#if __HVX_ARCH__ >= 60
3114	2551	/* ==========================================================================
3115	2552	Assembly Syntax: Vdd32.w=vsub(Vuu32.w,Vvv32.w)
3116	2553	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vsub_WwWw(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -3119,9 +2556,7 @@
3119	2556	========================================================================== */
3120	2557
3121	2558	#define Q6_Ww_vsub_WwWw(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubw_dv)(Vuu,Vvv)
3122		#endif /* __HEXAGON_ARCH___ >= 60 */
3123	2559
3124		#if __HVX_ARCH__ >= 60
3125	2560	/* ==========================================================================
3126	2561	Assembly Syntax: if (!Qv4) Vx32.w-=Vu32.w
3127	2562	C Intrinsic Prototype: HVX_Vector Q6_Vw_condnac_QnVwVw(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -3130,9 +2565,7 @@
3130	2565	========================================================================== */
3131	2566
3132	2567	#define Q6_Vw_condnac_QnVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwnq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
3133		#endif /* __HEXAGON_ARCH___ >= 60 */
3134	2568
3135		#if __HVX_ARCH__ >= 60
3136	2569	/* ==========================================================================
3137	2570	Assembly Syntax: if (Qv4) Vx32.w-=Vu32.w
3138	2571	C Intrinsic Prototype: HVX_Vector Q6_Vw_condnac_QVwVw(HVX_VectorPred Qv, HVX_Vector Vx, HVX_Vector Vu)
...	...	@@ -3141,9 +2574,7 @@
3141	2574	========================================================================== */
3142	2575
3143	2576	#define Q6_Vw_condnac_QVwVw(Qv,Vx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vx,Vu)
3144		#endif /* __HEXAGON_ARCH___ >= 60 */
3145	2577
3146		#if __HVX_ARCH__ >= 60
3147	2578	/* ==========================================================================
3148	2579	Assembly Syntax: Vd32.w=vsub(Vu32.w,Vv32.w):sat
3149	2580	C Intrinsic Prototype: HVX_Vector Q6_Vw_vsub_VwVw_sat(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3152,9 +2583,7 @@
3152	2583	========================================================================== */
3153	2584
3154	2585	#define Q6_Vw_vsub_VwVw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat)(Vu,Vv)
3155		#endif /* __HEXAGON_ARCH___ >= 60 */
3156	2586
3157		#if __HVX_ARCH__ >= 60
3158	2587	/* ==========================================================================
3159	2588	Assembly Syntax: Vdd32.w=vsub(Vuu32.w,Vvv32.w):sat
3160	2589	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vsub_WwWw_sat(HVX_VectorPair Vuu, HVX_VectorPair Vvv)
...	...	@@ -3163,9 +2592,7 @@
3163	2592	========================================================================== */
3164	2593
3165	2594	#define Q6_Ww_vsub_WwWw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubwsat_dv)(Vuu,Vvv)
3166		#endif /* __HEXAGON_ARCH___ >= 60 */
3167	2595
3168		#if __HVX_ARCH__ >= 60
3169	2596	/* ==========================================================================
3170	2597	Assembly Syntax: Vdd32=vswap(Qt4,Vu32,Vv32)
3171	2598	C Intrinsic Prototype: HVX_VectorPair Q6_W_vswap_QVV(HVX_VectorPred Qt, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3174,9 +2601,7 @@
3174	2601	========================================================================== */
3175	2602
3176	2603	#define Q6_W_vswap_QVV(Qt,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vswap)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1),Vu,Vv)
3177		#endif /* __HEXAGON_ARCH___ >= 60 */
3178	2604
3179		#if __HVX_ARCH__ >= 60
3180	2605	/* ==========================================================================
3181	2606	Assembly Syntax: Vdd32.h=vtmpy(Vuu32.b,Rt32.b)
3182	2607	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vtmpy_WbRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3185,9 +2610,7 @@
3185	2610	========================================================================== */
3186	2611
3187	2612	#define Q6_Wh_vtmpy_WbRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb)(Vuu,Rt)
3188		#endif /* __HEXAGON_ARCH___ >= 60 */
3189	2613
3190		#if __HVX_ARCH__ >= 60
3191	2614	/* ==========================================================================
3192	2615	Assembly Syntax: Vxx32.h+=vtmpy(Vuu32.b,Rt32.b)
3193	2616	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vtmpyacc_WhWbRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3196,9 +2619,7 @@
3196	2619	========================================================================== */
3197	2620
3198	2621	#define Q6_Wh_vtmpyacc_WhWbRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyb_acc)(Vxx,Vuu,Rt)
3199		#endif /* __HEXAGON_ARCH___ >= 60 */
3200	2622
3201		#if __HVX_ARCH__ >= 60
3202	2623	/* ==========================================================================
3203	2624	Assembly Syntax: Vdd32.h=vtmpy(Vuu32.ub,Rt32.b)
3204	2625	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vtmpy_WubRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3207,9 +2628,7 @@
3207	2628	========================================================================== */
3208	2629
3209	2630	#define Q6_Wh_vtmpy_WubRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus)(Vuu,Rt)
3210		#endif /* __HEXAGON_ARCH___ >= 60 */
3211	2631
3212		#if __HVX_ARCH__ >= 60
3213	2632	/* ==========================================================================
3214	2633	Assembly Syntax: Vxx32.h+=vtmpy(Vuu32.ub,Rt32.b)
3215	2634	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vtmpyacc_WhWubRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3218,9 +2637,7 @@
3218	2637	========================================================================== */
3219	2638
3220	2639	#define Q6_Wh_vtmpyacc_WhWubRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpybus_acc)(Vxx,Vuu,Rt)
3221		#endif /* __HEXAGON_ARCH___ >= 60 */
3222	2640
3223		#if __HVX_ARCH__ >= 60
3224	2641	/* ==========================================================================
3225	2642	Assembly Syntax: Vdd32.w=vtmpy(Vuu32.h,Rt32.b)
3226	2643	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vtmpy_WhRb(HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3229,9 +2646,7 @@
3229	2646	========================================================================== */
3230	2647
3231	2648	#define Q6_Ww_vtmpy_WhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb)(Vuu,Rt)
3232		#endif /* __HEXAGON_ARCH___ >= 60 */
3233	2649
3234		#if __HVX_ARCH__ >= 60
3235	2650	/* ==========================================================================
3236	2651	Assembly Syntax: Vxx32.w+=vtmpy(Vuu32.h,Rt32.b)
3237	2652	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vtmpyacc_WwWhRb(HVX_VectorPair Vxx, HVX_VectorPair Vuu, Word32 Rt)
...	...	@@ -3240,9 +2655,7 @@
3240	2655	========================================================================== */
3241	2656
3242	2657	#define Q6_Ww_vtmpyacc_WwWhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vtmpyhb_acc)(Vxx,Vuu,Rt)
3243		#endif /* __HEXAGON_ARCH___ >= 60 */
3244	2658
3245		#if __HVX_ARCH__ >= 60
3246	2659	/* ==========================================================================
3247	2660	Assembly Syntax: Vdd32.h=vunpack(Vu32.b)
3248	2661	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vunpack_Vb(HVX_Vector Vu)
...	...	@@ -3251,9 +2664,7 @@
3251	2664	========================================================================== */
3252	2665
3253	2666	#define Q6_Wh_vunpack_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackb)(Vu)
3254		#endif /* __HEXAGON_ARCH___ >= 60 */
3255	2667
3256		#if __HVX_ARCH__ >= 60
3257	2668	/* ==========================================================================
3258	2669	Assembly Syntax: Vdd32.w=vunpack(Vu32.h)
3259	2670	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vunpack_Vh(HVX_Vector Vu)
...	...	@@ -3262,9 +2673,7 @@
3262	2673	========================================================================== */
3263	2674
3264	2675	#define Q6_Ww_vunpack_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackh)(Vu)
3265		#endif /* __HEXAGON_ARCH___ >= 60 */
3266	2676
3267		#if __HVX_ARCH__ >= 60
3268	2677	/* ==========================================================================
3269	2678	Assembly Syntax: Vxx32.h\|=vunpacko(Vu32.b)
3270	2679	C Intrinsic Prototype: HVX_VectorPair Q6_Wh_vunpackoor_WhVb(HVX_VectorPair Vxx, HVX_Vector Vu)
...	...	@@ -3273,9 +2682,7 @@
3273	2682	========================================================================== */
3274	2683
3275	2684	#define Q6_Wh_vunpackoor_WhVb(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackob)(Vxx,Vu)
3276		#endif /* __HEXAGON_ARCH___ >= 60 */
3277	2685
3278		#if __HVX_ARCH__ >= 60
3279	2686	/* ==========================================================================
3280	2687	Assembly Syntax: Vxx32.w\|=vunpacko(Vu32.h)
3281	2688	C Intrinsic Prototype: HVX_VectorPair Q6_Ww_vunpackoor_WwVh(HVX_VectorPair Vxx, HVX_Vector Vu)
...	...	@@ -3284,9 +2691,7 @@
3284	2691	========================================================================== */
3285	2692
3286	2693	#define Q6_Ww_vunpackoor_WwVh(Vxx,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackoh)(Vxx,Vu)
3287		#endif /* __HEXAGON_ARCH___ >= 60 */
3288	2694
3289		#if __HVX_ARCH__ >= 60
3290	2695	/* ==========================================================================
3291	2696	Assembly Syntax: Vdd32.uh=vunpack(Vu32.ub)
3292	2697	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vunpack_Vub(HVX_Vector Vu)
...	...	@@ -3295,9 +2700,7 @@
3295	2700	========================================================================== */
3296	2701
3297	2702	#define Q6_Wuh_vunpack_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackub)(Vu)
3298		#endif /* __HEXAGON_ARCH___ >= 60 */
3299	2703
3300		#if __HVX_ARCH__ >= 60
3301	2704	/* ==========================================================================
3302	2705	Assembly Syntax: Vdd32.uw=vunpack(Vu32.uh)
3303	2706	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vunpack_Vuh(HVX_Vector Vu)
...	...	@@ -3306,9 +2709,7 @@
3306	2709	========================================================================== */
3307	2710
3308	2711	#define Q6_Wuw_vunpack_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vunpackuh)(Vu)
3309		#endif /* __HEXAGON_ARCH___ >= 60 */
3310	2712
3311		#if __HVX_ARCH__ >= 60
3312	2713	/* ==========================================================================
3313	2714	Assembly Syntax: Vd32=vxor(Vu32,Vv32)
3314	2715	C Intrinsic Prototype: HVX_Vector Q6_V_vxor_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -3317,9 +2718,7 @@
3317	2718	========================================================================== */
3318	2719
3319	2720	#define Q6_V_vxor_VV(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vxor)(Vu,Vv)
3320		#endif /* __HEXAGON_ARCH___ >= 60 */
3321	2721
3322		#if __HVX_ARCH__ >= 60
3323	2722	/* ==========================================================================
3324	2723	Assembly Syntax: Vdd32.uh=vzxt(Vu32.ub)
3325	2724	C Intrinsic Prototype: HVX_VectorPair Q6_Wuh_vzxt_Vub(HVX_Vector Vu)
...	...	@@ -3328,9 +2727,7 @@
3328	2727	========================================================================== */
3329	2728
3330	2729	#define Q6_Wuh_vzxt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzb)(Vu)
3331		#endif /* __HEXAGON_ARCH___ >= 60 */
3332	2730
3333		#if __HVX_ARCH__ >= 60
3334	2731	/* ==========================================================================
3335	2732	Assembly Syntax: Vdd32.uw=vzxt(Vu32.uh)
3336	2733	C Intrinsic Prototype: HVX_VectorPair Q6_Wuw_vzxt_Vuh(HVX_Vector Vu)
...	...	@@ -3339,7 +2736,6 @@
3339	2736	========================================================================== */
3340	2737
3341	2738	#define Q6_Wuw_vzxt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vzh)(Vu)
3342		#endif /* __HEXAGON_ARCH___ >= 60 */
3343	2739
3344	2740	#if __HVX_ARCH__ >= 62
3345	2741	/* ==========================================================================
...	...	@@ -3350,7 +2746,7 @@
3350	2746	========================================================================== */
3351	2747
3352	2748	#define Q6_Vb_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplatb)(Rt)
3353		#endif /* __HEXAGON_ARCH___ >= 62 */
	2749	#endif
3354	2750
3355	2751	#if __HVX_ARCH__ >= 62
3356	2752	/* ==========================================================================
...	...	@@ -3361,7 +2757,7 @@
3361	2757	========================================================================== */
3362	2758
3363	2759	#define Q6_Vh_vsplat_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_lvsplath)(Rt)
3364		#endif /* __HEXAGON_ARCH___ >= 62 */
	2760	#endif
3365	2761
3366	2762	#if __HVX_ARCH__ >= 62
3367	2763	/* ==========================================================================
...	...	@@ -3372,7 +2768,7 @@
3372	2768	========================================================================== */
3373	2769
3374	2770	#define Q6_Q_vsetq2_R(Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_pred_scalar2v2)(Rt)),-1)
3375		#endif /* __HEXAGON_ARCH___ >= 62 */
	2771	#endif
3376	2772
3377	2773	#if __HVX_ARCH__ >= 62
3378	2774	/* ==========================================================================
...	...	@@ -3383,7 +2779,7 @@
3383	2779	========================================================================== */
3384	2780
3385	2781	#define Q6_Qb_vshuffe_QhQh(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
3386		#endif /* __HEXAGON_ARCH___ >= 62 */
	2782	#endif
3387	2783
3388	2784	#if __HVX_ARCH__ >= 62
3389	2785	/* ==========================================================================
...	...	@@ -3394,7 +2790,7 @@
3394	2790	========================================================================== */
3395	2791
3396	2792	#define Q6_Qh_vshuffe_QwQw(Qs,Qt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_shuffeqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qt),-1))),-1)
3397		#endif /* __HEXAGON_ARCH___ >= 62 */
	2793	#endif
3398	2794
3399	2795	#if __HVX_ARCH__ >= 62
3400	2796	/* ==========================================================================
...	...	@@ -3405,7 +2801,7 @@
3405	2801	========================================================================== */
3406	2802
3407	2803	#define Q6_Vb_vadd_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat)(Vu,Vv)
3408		#endif /* __HEXAGON_ARCH___ >= 62 */
	2804	#endif
3409	2805
3410	2806	#if __HVX_ARCH__ >= 62
3411	2807	/* ==========================================================================
...	...	@@ -3416,7 +2812,7 @@
3416	2812	========================================================================== */
3417	2813
3418	2814	#define Q6_Wb_vadd_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddbsat_dv)(Vuu,Vvv)
3419		#endif /* __HEXAGON_ARCH___ >= 62 */
	2815	#endif
3420	2816
3421	2817	#if __HVX_ARCH__ >= 62
3422	2818	/* ==========================================================================
...	...	@@ -3427,7 +2823,7 @@
3427	2823	========================================================================== */
3428	2824
3429	2825	#define Q6_Vw_vadd_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarry)(Vu,Vv,Qx)
3430		#endif /* __HEXAGON_ARCH___ >= 62 */
	2826	#endif
3431	2827
3432	2828	#if __HVX_ARCH__ >= 62
3433	2829	/* ==========================================================================
...	...	@@ -3438,7 +2834,7 @@
3438	2834	========================================================================== */
3439	2835
3440	2836	#define Q6_Vh_vadd_vclb_VhVh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbh)(Vu,Vv)
3441		#endif /* __HEXAGON_ARCH___ >= 62 */
	2837	#endif
3442	2838
3443	2839	#if __HVX_ARCH__ >= 62
3444	2840	/* ==========================================================================
...	...	@@ -3449,7 +2845,7 @@
3449	2845	========================================================================== */
3450	2846
3451	2847	#define Q6_Vw_vadd_vclb_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddclbw)(Vu,Vv)
3452		#endif /* __HEXAGON_ARCH___ >= 62 */
	2848	#endif
3453	2849
3454	2850	#if __HVX_ARCH__ >= 62
3455	2851	/* ==========================================================================
...	...	@@ -3460,7 +2856,7 @@
3460	2856	========================================================================== */
3461	2857
3462	2858	#define Q6_Ww_vaddacc_WwVhVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddhw_acc)(Vxx,Vu,Vv)
3463		#endif /* __HEXAGON_ARCH___ >= 62 */
	2859	#endif
3464	2860
3465	2861	#if __HVX_ARCH__ >= 62
3466	2862	/* ==========================================================================
...	...	@@ -3471,7 +2867,7 @@
3471	2867	========================================================================== */
3472	2868
3473	2869	#define Q6_Wh_vaddacc_WhVubVub(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddubh_acc)(Vxx,Vu,Vv)
3474		#endif /* __HEXAGON_ARCH___ >= 62 */
	2870	#endif
3475	2871
3476	2872	#if __HVX_ARCH__ >= 62
3477	2873	/* ==========================================================================
...	...	@@ -3482,7 +2878,7 @@
3482	2878	========================================================================== */
3483	2879
3484	2880	#define Q6_Vub_vadd_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddububb_sat)(Vu,Vv)
3485		#endif /* __HEXAGON_ARCH___ >= 62 */
	2881	#endif
3486	2882
3487	2883	#if __HVX_ARCH__ >= 62
3488	2884	/* ==========================================================================
...	...	@@ -3493,7 +2889,7 @@
3493	2889	========================================================================== */
3494	2890
3495	2891	#define Q6_Ww_vaddacc_WwVuhVuh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduhw_acc)(Vxx,Vu,Vv)
3496		#endif /* __HEXAGON_ARCH___ >= 62 */
	2892	#endif
3497	2893
3498	2894	#if __HVX_ARCH__ >= 62
3499	2895	/* ==========================================================================
...	...	@@ -3504,7 +2900,7 @@
3504	2900	========================================================================== */
3505	2901
3506	2902	#define Q6_Vuw_vadd_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat)(Vu,Vv)
3507		#endif /* __HEXAGON_ARCH___ >= 62 */
	2903	#endif
3508	2904
3509	2905	#if __HVX_ARCH__ >= 62
3510	2906	/* ==========================================================================
...	...	@@ -3515,7 +2911,7 @@
3515	2911	========================================================================== */
3516	2912
3517	2913	#define Q6_Wuw_vadd_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadduwsat_dv)(Vuu,Vvv)
3518		#endif /* __HEXAGON_ARCH___ >= 62 */
	2914	#endif
3519	2915
3520	2916	#if __HVX_ARCH__ >= 62
3521	2917	/* ==========================================================================
...	...	@@ -3526,7 +2922,7 @@
3526	2922	========================================================================== */
3527	2923
3528	2924	#define Q6_V_vand_QnR(Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
3529		#endif /* __HEXAGON_ARCH___ >= 62 */
	2925	#endif
3530	2926
3531	2927	#if __HVX_ARCH__ >= 62
3532	2928	/* ==========================================================================
...	...	@@ -3537,7 +2933,7 @@
3537	2933	========================================================================== */
3538	2934
3539	2935	#define Q6_V_vandor_VQnR(Vx,Qu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandnqrt_acc)(Vx,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qu),-1),Rt)
3540		#endif /* __HEXAGON_ARCH___ >= 62 */
	2936	#endif
3541	2937
3542	2938	#if __HVX_ARCH__ >= 62
3543	2939	/* ==========================================================================
...	...	@@ -3548,7 +2944,7 @@
3548	2944	========================================================================== */
3549	2945
3550	2946	#define Q6_V_vand_QnV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvnqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu)
3551		#endif /* __HEXAGON_ARCH___ >= 62 */
	2947	#endif
3552	2948
3553	2949	#if __HVX_ARCH__ >= 62
3554	2950	/* ==========================================================================
...	...	@@ -3559,7 +2955,7 @@
3559	2955	========================================================================== */
3560	2956
3561	2957	#define Q6_V_vand_QV(Qv,Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvqv)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1),Vu)
3562		#endif /* __HEXAGON_ARCH___ >= 62 */
	2958	#endif
3563	2959
3564	2960	#if __HVX_ARCH__ >= 62
3565	2961	/* ==========================================================================
...	...	@@ -3570,7 +2966,7 @@
3570	2966	========================================================================== */
3571	2967
3572	2968	#define Q6_Vb_vasr_VhVhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrhbsat)(Vu,Vv,Rt)
3573		#endif /* __HEXAGON_ARCH___ >= 62 */
	2969	#endif
3574	2970
3575	2971	#if __HVX_ARCH__ >= 62
3576	2972	/* ==========================================================================
...	...	@@ -3581,7 +2977,7 @@
3581	2977	========================================================================== */
3582	2978
3583	2979	#define Q6_Vuh_vasr_VuwVuwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhrndsat)(Vu,Vv,Rt)
3584		#endif /* __HEXAGON_ARCH___ >= 62 */
	2980	#endif
3585	2981
3586	2982	#if __HVX_ARCH__ >= 62
3587	2983	/* ==========================================================================
...	...	@@ -3592,7 +2988,7 @@
3592	2988	========================================================================== */
3593	2989
3594	2990	#define Q6_Vuh_vasr_VwVwR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrwuhrndsat)(Vu,Vv,Rt)
3595		#endif /* __HEXAGON_ARCH___ >= 62 */
	2991	#endif
3596	2992
3597	2993	#if __HVX_ARCH__ >= 62
3598	2994	/* ==========================================================================
...	...	@@ -3603,7 +2999,7 @@
3603	2999	========================================================================== */
3604	3000
3605	3001	#define Q6_Vub_vlsr_VubR(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlsrb)(Vu,Rt)
3606		#endif /* __HEXAGON_ARCH___ >= 62 */
	3002	#endif
3607	3003
3608	3004	#if __HVX_ARCH__ >= 62
3609	3005	/* ==========================================================================
...	...	@@ -3614,7 +3010,7 @@
3614	3010	========================================================================== */
3615	3011
3616	3012	#define Q6_Vb_vlut32_VbVbR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_nm)(Vu,Vv,Rt)
3617		#endif /* __HEXAGON_ARCH___ >= 62 */
	3013	#endif
3618	3014
3619	3015	#if __HVX_ARCH__ >= 62
3620	3016	/* ==========================================================================
...	...	@@ -3625,7 +3021,7 @@
3625	3021	========================================================================== */
3626	3022
3627	3023	#define Q6_Vb_vlut32or_VbVbVbI(Vx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvb_oracci)(Vx,Vu,Vv,Iu3)
3628		#endif /* __HEXAGON_ARCH___ >= 62 */
	3024	#endif
3629	3025
3630	3026	#if __HVX_ARCH__ >= 62
3631	3027	/* ==========================================================================
...	...	@@ -3636,7 +3032,7 @@
3636	3032	========================================================================== */
3637	3033
3638	3034	#define Q6_Vb_vlut32_VbVbI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvvbi)(Vu,Vv,Iu3)
3639		#endif /* __HEXAGON_ARCH___ >= 62 */
	3035	#endif
3640	3036
3641	3037	#if __HVX_ARCH__ >= 62
3642	3038	/* ==========================================================================
...	...	@@ -3647,7 +3043,7 @@
3647	3043	========================================================================== */
3648	3044
3649	3045	#define Q6_Wh_vlut16_VbVhR_nomatch(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_nm)(Vu,Vv,Rt)
3650		#endif /* __HEXAGON_ARCH___ >= 62 */
	3046	#endif
3651	3047
3652	3048	#if __HVX_ARCH__ >= 62
3653	3049	/* ==========================================================================
...	...	@@ -3658,7 +3054,7 @@
3658	3054	========================================================================== */
3659	3055
3660	3056	#define Q6_Wh_vlut16or_WhVbVhI(Vxx,Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwh_oracci)(Vxx,Vu,Vv,Iu3)
3661		#endif /* __HEXAGON_ARCH___ >= 62 */
	3057	#endif
3662	3058
3663	3059	#if __HVX_ARCH__ >= 62
3664	3060	/* ==========================================================================
...	...	@@ -3669,7 +3065,7 @@
3669	3065	========================================================================== */
3670	3066
3671	3067	#define Q6_Wh_vlut16_VbVhI(Vu,Vv,Iu3) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlutvwhi)(Vu,Vv,Iu3)
3672		#endif /* __HEXAGON_ARCH___ >= 62 */
	3068	#endif
3673	3069
3674	3070	#if __HVX_ARCH__ >= 62
3675	3071	/* ==========================================================================
...	...	@@ -3680,7 +3076,7 @@
3680	3076	========================================================================== */
3681	3077
3682	3078	#define Q6_Vb_vmax_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmaxb)(Vu,Vv)
3683		#endif /* __HEXAGON_ARCH___ >= 62 */
	3079	#endif
3684	3080
3685	3081	#if __HVX_ARCH__ >= 62
3686	3082	/* ==========================================================================
...	...	@@ -3691,7 +3087,7 @@
3691	3087	========================================================================== */
3692	3088
3693	3089	#define Q6_Vb_vmin_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vminb)(Vu,Vv)
3694		#endif /* __HEXAGON_ARCH___ >= 62 */
	3090	#endif
3695	3091
3696	3092	#if __HVX_ARCH__ >= 62
3697	3093	/* ==========================================================================
...	...	@@ -3702,7 +3098,7 @@
3702	3098	========================================================================== */
3703	3099
3704	3100	#define Q6_Ww_vmpa_WuhRb(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb)(Vuu,Rt)
3705		#endif /* __HEXAGON_ARCH___ >= 62 */
	3101	#endif
3706	3102
3707	3103	#if __HVX_ARCH__ >= 62
3708	3104	/* ==========================================================================
...	...	@@ -3713,7 +3109,7 @@
3713	3109	========================================================================== */
3714	3110
3715	3111	#define Q6_Ww_vmpaacc_WwWuhRb(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhb_acc)(Vxx,Vuu,Rt)
3716		#endif /* __HEXAGON_ARCH___ >= 62 */
	3112	#endif
3717	3113
3718	3114	#if __HVX_ARCH__ >= 62
3719	3115	/* ==========================================================================
...	...	@@ -3724,7 +3120,7 @@
3724	3120	========================================================================== */
3725	3121
3726	3122	#define Q6_W_vmpye_VwVuh(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyewuh_64)(Vu,Vv)
3727		#endif /* __HEXAGON_ARCH___ >= 62 */
	3123	#endif
3728	3124
3729	3125	#if __HVX_ARCH__ >= 62
3730	3126	/* ==========================================================================
...	...	@@ -3735,7 +3131,7 @@
3735	3131	========================================================================== */
3736	3132
3737	3133	#define Q6_Vw_vmpyi_VwRub(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub)(Vu,Rt)
3738		#endif /* __HEXAGON_ARCH___ >= 62 */
	3134	#endif
3739	3135
3740	3136	#if __HVX_ARCH__ >= 62
3741	3137	/* ==========================================================================
...	...	@@ -3746,7 +3142,7 @@
3746	3142	========================================================================== */
3747	3143
3748	3144	#define Q6_Vw_vmpyiacc_VwVwRub(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyiwub_acc)(Vx,Vu,Rt)
3749		#endif /* __HEXAGON_ARCH___ >= 62 */
	3145	#endif
3750	3146
3751	3147	#if __HVX_ARCH__ >= 62
3752	3148	/* ==========================================================================
...	...	@@ -3757,7 +3153,7 @@
3757	3153	========================================================================== */
3758	3154
3759	3155	#define Q6_W_vmpyoacc_WVwVh(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyowh_64_acc)(Vxx,Vu,Vv)
3760		#endif /* __HEXAGON_ARCH___ >= 62 */
	3156	#endif
3761	3157
3762	3158	#if __HVX_ARCH__ >= 62
3763	3159	/* ==========================================================================
...	...	@@ -3768,7 +3164,7 @@
3768	3164	========================================================================== */
3769	3165
3770	3166	#define Q6_Vub_vround_VuhVuh_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduhub)(Vu,Vv)
3771		#endif /* __HEXAGON_ARCH___ >= 62 */
	3167	#endif
3772	3168
3773	3169	#if __HVX_ARCH__ >= 62
3774	3170	/* ==========================================================================
...	...	@@ -3779,7 +3175,7 @@
3779	3175	========================================================================== */
3780	3176
3781	3177	#define Q6_Vuh_vround_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrounduwuh)(Vu,Vv)
3782		#endif /* __HEXAGON_ARCH___ >= 62 */
	3178	#endif
3783	3179
3784	3180	#if __HVX_ARCH__ >= 62
3785	3181	/* ==========================================================================
...	...	@@ -3790,7 +3186,7 @@
3790	3186	========================================================================== */
3791	3187
3792	3188	#define Q6_Vuh_vsat_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatuwuh)(Vu,Vv)
3793		#endif /* __HEXAGON_ARCH___ >= 62 */
	3189	#endif
3794	3190
3795	3191	#if __HVX_ARCH__ >= 62
3796	3192	/* ==========================================================================
...	...	@@ -3801,7 +3197,7 @@
3801	3197	========================================================================== */
3802	3198
3803	3199	#define Q6_Vb_vsub_VbVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat)(Vu,Vv)
3804		#endif /* __HEXAGON_ARCH___ >= 62 */
	3200	#endif
3805	3201
3806	3202	#if __HVX_ARCH__ >= 62
3807	3203	/* ==========================================================================
...	...	@@ -3812,7 +3208,7 @@
3812	3208	========================================================================== */
3813	3209
3814	3210	#define Q6_Wb_vsub_WbWb_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubbsat_dv)(Vuu,Vvv)
3815		#endif /* __HEXAGON_ARCH___ >= 62 */
	3211	#endif
3816	3212
3817	3213	#if __HVX_ARCH__ >= 62
3818	3214	/* ==========================================================================
...	...	@@ -3823,7 +3219,7 @@
3823	3219	========================================================================== */
3824	3220
3825	3221	#define Q6_Vw_vsub_VwVwQ_carry(Vu,Vv,Qx) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubcarry)(Vu,Vv,Qx)
3826		#endif /* __HEXAGON_ARCH___ >= 62 */
	3222	#endif
3827	3223
3828	3224	#if __HVX_ARCH__ >= 62
3829	3225	/* ==========================================================================
...	...	@@ -3834,7 +3230,7 @@
3834	3230	========================================================================== */
3835	3231
3836	3232	#define Q6_Vub_vsub_VubVb_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubububb_sat)(Vu,Vv)
3837		#endif /* __HEXAGON_ARCH___ >= 62 */
	3233	#endif
3838	3234
3839	3235	#if __HVX_ARCH__ >= 62
3840	3236	/* ==========================================================================
...	...	@@ -3845,7 +3241,7 @@
3845	3241	========================================================================== */
3846	3242
3847	3243	#define Q6_Vuw_vsub_VuwVuw_sat(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat)(Vu,Vv)
3848		#endif /* __HEXAGON_ARCH___ >= 62 */
	3244	#endif
3849	3245
3850	3246	#if __HVX_ARCH__ >= 62
3851	3247	/* ==========================================================================
...	...	@@ -3856,7 +3252,7 @@
3856	3252	========================================================================== */
3857	3253
3858	3254	#define Q6_Wuw_vsub_WuwWuw_sat(Vuu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsubuwsat_dv)(Vuu,Vvv)
3859		#endif /* __HEXAGON_ARCH___ >= 62 */
	3255	#endif
3860	3256
3861	3257	#if __HVX_ARCH__ >= 65
3862	3258	/* ==========================================================================
...	...	@@ -3867,7 +3263,7 @@
3867	3263	========================================================================== */
3868	3264
3869	3265	#define Q6_Vb_vabs_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb)(Vu)
3870		#endif /* __HEXAGON_ARCH___ >= 65 */
	3266	#endif
3871	3267
3872	3268	#if __HVX_ARCH__ >= 65
3873	3269	/* ==========================================================================
...	...	@@ -3878,7 +3274,7 @@
3878	3274	========================================================================== */
3879	3275
3880	3276	#define Q6_Vb_vabs_Vb_sat(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabsb_sat)(Vu)
3881		#endif /* __HEXAGON_ARCH___ >= 65 */
	3277	#endif
3882	3278
3883	3279	#if __HVX_ARCH__ >= 65
3884	3280	/* ==========================================================================
...	...	@@ -3889,7 +3285,7 @@
3889	3285	========================================================================== */
3890	3286
3891	3287	#define Q6_Vh_vaslacc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaslh_acc)(Vx,Vu,Rt)
3892		#endif /* __HEXAGON_ARCH___ >= 65 */
	3288	#endif
3893	3289
3894	3290	#if __HVX_ARCH__ >= 65
3895	3291	/* ==========================================================================
...	...	@@ -3900,7 +3296,7 @@
3900	3296	========================================================================== */
3901	3297
3902	3298	#define Q6_Vh_vasracc_VhVhR(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrh_acc)(Vx,Vu,Rt)
3903		#endif /* __HEXAGON_ARCH___ >= 65 */
	3299	#endif
3904	3300
3905	3301	#if __HVX_ARCH__ >= 65
3906	3302	/* ==========================================================================
...	...	@@ -3911,7 +3307,7 @@
3911	3307	========================================================================== */
3912	3308
3913	3309	#define Q6_Vub_vasr_VuhVuhR_rnd_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubrndsat)(Vu,Vv,Rt)
3914		#endif /* __HEXAGON_ARCH___ >= 65 */
	3310	#endif
3915	3311
3916	3312	#if __HVX_ARCH__ >= 65
3917	3313	/* ==========================================================================
...	...	@@ -3922,7 +3318,7 @@
3922	3318	========================================================================== */
3923	3319
3924	3320	#define Q6_Vub_vasr_VuhVuhR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruhubsat)(Vu,Vv,Rt)
3925		#endif /* __HEXAGON_ARCH___ >= 65 */
	3321	#endif
3926	3322
3927	3323	#if __HVX_ARCH__ >= 65
3928	3324	/* ==========================================================================
...	...	@@ -3933,7 +3329,7 @@
3933	3329	========================================================================== */
3934	3330
3935	3331	#define Q6_Vuh_vasr_VuwVuwR_sat(Vu,Vv,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasruwuhsat)(Vu,Vv,Rt)
3936		#endif /* __HEXAGON_ARCH___ >= 65 */
	3332	#endif
3937	3333
3938	3334	#if __HVX_ARCH__ >= 65
3939	3335	/* ==========================================================================
...	...	@@ -3944,7 +3340,7 @@
3944	3340	========================================================================== */
3945	3341
3946	3342	#define Q6_Vb_vavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgb)(Vu,Vv)
3947		#endif /* __HEXAGON_ARCH___ >= 65 */
	3343	#endif
3948	3344
3949	3345	#if __HVX_ARCH__ >= 65
3950	3346	/* ==========================================================================
...	...	@@ -3955,7 +3351,7 @@
3955	3351	========================================================================== */
3956	3352
3957	3353	#define Q6_Vb_vavg_VbVb_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavgbrnd)(Vu,Vv)
3958		#endif /* __HEXAGON_ARCH___ >= 65 */
	3354	#endif
3959	3355
3960	3356	#if __HVX_ARCH__ >= 65
3961	3357	/* ==========================================================================
...	...	@@ -3966,7 +3362,7 @@
3966	3362	========================================================================== */
3967	3363
3968	3364	#define Q6_Vuw_vavg_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguw)(Vu,Vv)
3969		#endif /* __HEXAGON_ARCH___ >= 65 */
	3365	#endif
3970	3366
3971	3367	#if __HVX_ARCH__ >= 65
3972	3368	/* ==========================================================================
...	...	@@ -3977,7 +3373,7 @@
3977	3373	========================================================================== */
3978	3374
3979	3375	#define Q6_Vuw_vavg_VuwVuw_rnd(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vavguwrnd)(Vu,Vv)
3980		#endif /* __HEXAGON_ARCH___ >= 65 */
	3376	#endif
3981	3377
3982	3378	#if __HVX_ARCH__ >= 65
3983	3379	/* ==========================================================================
...	...	@@ -3988,7 +3384,7 @@
3988	3384	========================================================================== */
3989	3385
3990	3386	#define Q6_W_vzero() __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdd0)()
3991		#endif /* __HEXAGON_ARCH___ >= 65 */
	3387	#endif
3992	3388
3993	3389	#if __HVX_ARCH__ >= 65
3994	3390	/* ==========================================================================
...	...	@@ -3999,7 +3395,7 @@
3999	3395	========================================================================== */
4000	3396
4001	3397	#define Q6_vgather_ARMVh(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermh)(Rs,Rt,Mu,Vv)
4002		#endif /* __HEXAGON_ARCH___ >= 65 */
	3398	#endif
4003	3399
4004	3400	#if __HVX_ARCH__ >= 65
4005	3401	/* ==========================================================================
...	...	@@ -4010,7 +3406,7 @@
4010	3406	========================================================================== */
4011	3407
4012	3408	#define Q6_vgather_AQRMVh(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv)
4013		#endif /* __HEXAGON_ARCH___ >= 65 */
	3409	#endif
4014	3410
4015	3411	#if __HVX_ARCH__ >= 65
4016	3412	/* ==========================================================================
...	...	@@ -4021,7 +3417,7 @@
4021	3417	========================================================================== */
4022	3418
4023	3419	#define Q6_vgather_ARMWw(Rs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhw)(Rs,Rt,Mu,Vvv)
4024		#endif /* __HEXAGON_ARCH___ >= 65 */
	3420	#endif
4025	3421
4026	3422	#if __HVX_ARCH__ >= 65
4027	3423	/* ==========================================================================
...	...	@@ -4032,7 +3428,7 @@
4032	3428	========================================================================== */
4033	3429
4034	3430	#define Q6_vgather_AQRMWw(Rs,Qs,Rt,Mu,Vvv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermhwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv)
4035		#endif /* __HEXAGON_ARCH___ >= 65 */
	3431	#endif
4036	3432
4037	3433	#if __HVX_ARCH__ >= 65
4038	3434	/* ==========================================================================
...	...	@@ -4043,7 +3439,7 @@
4043	3439	========================================================================== */
4044	3440
4045	3441	#define Q6_vgather_ARMVw(Rs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermw)(Rs,Rt,Mu,Vv)
4046		#endif /* __HEXAGON_ARCH___ >= 65 */
	3442	#endif
4047	3443
4048	3444	#if __HVX_ARCH__ >= 65
4049	3445	/* ==========================================================================
...	...	@@ -4054,7 +3450,7 @@
4054	3450	========================================================================== */
4055	3451
4056	3452	#define Q6_vgather_AQRMVw(Rs,Qs,Rt,Mu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgathermwq)(Rs,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv)
4057		#endif /* __HEXAGON_ARCH___ >= 65 */
	3453	#endif
4058	3454
4059	3455	#if __HVX_ARCH__ >= 65
4060	3456	/* ==========================================================================
...	...	@@ -4065,7 +3461,7 @@
4065	3461	========================================================================== */
4066	3462
4067	3463	#define Q6_Vh_vlut4_VuhPh(Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vlut4)(Vu,Rtt)
4068		#endif /* __HEXAGON_ARCH___ >= 65 */
	3464	#endif
4069	3465
4070	3466	#if __HVX_ARCH__ >= 65
4071	3467	/* ==========================================================================
...	...	@@ -4076,7 +3472,7 @@
4076	3472	========================================================================== */
4077	3473
4078	3474	#define Q6_Wh_vmpa_WubRub(Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu)(Vuu,Rt)
4079		#endif /* __HEXAGON_ARCH___ >= 65 */
	3475	#endif
4080	3476
4081	3477	#if __HVX_ARCH__ >= 65
4082	3478	/* ==========================================================================
...	...	@@ -4087,7 +3483,7 @@
4087	3483	========================================================================== */
4088	3484
4089	3485	#define Q6_Wh_vmpaacc_WhWubRub(Vxx,Vuu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpabuu_acc)(Vxx,Vuu,Rt)
4090		#endif /* __HEXAGON_ARCH___ >= 65 */
	3486	#endif
4091	3487
4092	3488	#if __HVX_ARCH__ >= 65
4093	3489	/* ==========================================================================
...	...	@@ -4098,7 +3494,7 @@
4098	3494	========================================================================== */
4099	3495
4100	3496	#define Q6_Vh_vmpa_VhVhVhPh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpahhsat)(Vx,Vu,Rtt)
4101		#endif /* __HEXAGON_ARCH___ >= 65 */
	3497	#endif
4102	3498
4103	3499	#if __HVX_ARCH__ >= 65
4104	3500	/* ==========================================================================
...	...	@@ -4109,7 +3505,7 @@
4109	3505	========================================================================== */
4110	3506
4111	3507	#define Q6_Vh_vmpa_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpauhuhsat)(Vx,Vu,Rtt)
4112		#endif /* __HEXAGON_ARCH___ >= 65 */
	3508	#endif
4113	3509
4114	3510	#if __HVX_ARCH__ >= 65
4115	3511	/* ==========================================================================
...	...	@@ -4120,7 +3516,7 @@
4120	3516	========================================================================== */
4121	3517
4122	3518	#define Q6_Vh_vmps_VhVhVuhPuh_sat(Vx,Vu,Rtt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpsuhuhsat)(Vx,Vu,Rtt)
4123		#endif /* __HEXAGON_ARCH___ >= 65 */
	3519	#endif
4124	3520
4125	3521	#if __HVX_ARCH__ >= 65
4126	3522	/* ==========================================================================
...	...	@@ -4131,7 +3527,7 @@
4131	3527	========================================================================== */
4132	3528
4133	3529	#define Q6_Ww_vmpyacc_WwVhRh(Vxx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyh_acc)(Vxx,Vu,Rt)
4134		#endif /* __HEXAGON_ARCH___ >= 65 */
	3530	#endif
4135	3531
4136	3532	#if __HVX_ARCH__ >= 65
4137	3533	/* ==========================================================================
...	...	@@ -4142,7 +3538,7 @@
4142	3538	========================================================================== */
4143	3539
4144	3540	#define Q6_Vuw_vmpye_VuhRuh(Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe)(Vu,Rt)
4145		#endif /* __HEXAGON_ARCH___ >= 65 */
	3541	#endif
4146	3542
4147	3543	#if __HVX_ARCH__ >= 65
4148	3544	/* ==========================================================================
...	...	@@ -4153,7 +3549,7 @@
4153	3549	========================================================================== */
4154	3550
4155	3551	#define Q6_Vuw_vmpyeacc_VuwVuhRuh(Vx,Vu,Rt) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhe_acc)(Vx,Vu,Rt)
4156		#endif /* __HEXAGON_ARCH___ >= 65 */
	3552	#endif
4157	3553
4158	3554	#if __HVX_ARCH__ >= 65
4159	3555	/* ==========================================================================
...	...	@@ -4164,7 +3560,7 @@
4164	3560	========================================================================== */
4165	3561
4166	3562	#define Q6_Vb_vnavg_VbVb(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vnavgb)(Vu,Vv)
4167		#endif /* __HEXAGON_ARCH___ >= 65 */
	3563	#endif
4168	3564
4169	3565	#if __HVX_ARCH__ >= 65
4170	3566	/* ==========================================================================
...	...	@@ -4175,7 +3571,7 @@
4175	3571	========================================================================== */
4176	3572
4177	3573	#define Q6_Vb_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqb)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
4178		#endif /* __HEXAGON_ARCH___ >= 65 */
	3574	#endif
4179	3575
4180	3576	#if __HVX_ARCH__ >= 65
4181	3577	/* ==========================================================================
...	...	@@ -4186,7 +3582,7 @@
4186	3582	========================================================================== */
4187	3583
4188	3584	#define Q6_Vh_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqh)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
4189		#endif /* __HEXAGON_ARCH___ >= 65 */
	3585	#endif
4190	3586
4191	3587	#if __HVX_ARCH__ >= 65
4192	3588	/* ==========================================================================
...	...	@@ -4197,7 +3593,7 @@
4197	3593	========================================================================== */
4198	3594
4199	3595	#define Q6_Vw_prefixsum_Q(Qv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vprefixqw)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qv),-1))
4200		#endif /* __HEXAGON_ARCH___ >= 65 */
	3596	#endif
4201	3597
4202	3598	#if __HVX_ARCH__ >= 65
4203	3599	/* ==========================================================================
...	...	@@ -4208,7 +3604,7 @@
4208	3604	========================================================================== */
4209	3605
4210	3606	#define Q6_vscatter_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh)(Rt,Mu,Vv,Vw)
4211		#endif /* __HEXAGON_ARCH___ >= 65 */
	3607	#endif
4212	3608
4213	3609	#if __HVX_ARCH__ >= 65
4214	3610	/* ==========================================================================
...	...	@@ -4219,7 +3615,7 @@
4219	3615	========================================================================== */
4220	3616
4221	3617	#define Q6_vscatteracc_RMVhV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermh_add)(Rt,Mu,Vv,Vw)
4222		#endif /* __HEXAGON_ARCH___ >= 65 */
	3618	#endif
4223	3619
4224	3620	#if __HVX_ARCH__ >= 65
4225	3621	/* ==========================================================================
...	...	@@ -4230,7 +3626,7 @@
4230	3626	========================================================================== */
4231	3627
4232	3628	#define Q6_vscatter_QRMVhV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw)
4233		#endif /* __HEXAGON_ARCH___ >= 65 */
	3629	#endif
4234	3630
4235	3631	#if __HVX_ARCH__ >= 65
4236	3632	/* ==========================================================================
...	...	@@ -4241,7 +3637,7 @@
4241	3637	========================================================================== */
4242	3638
4243	3639	#define Q6_vscatter_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw)(Rt,Mu,Vvv,Vw)
4244		#endif /* __HEXAGON_ARCH___ >= 65 */
	3640	#endif
4245	3641
4246	3642	#if __HVX_ARCH__ >= 65
4247	3643	/* ==========================================================================
...	...	@@ -4252,7 +3648,7 @@
4252	3648	========================================================================== */
4253	3649
4254	3650	#define Q6_vscatteracc_RMWwV(Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhw_add)(Rt,Mu,Vvv,Vw)
4255		#endif /* __HEXAGON_ARCH___ >= 65 */
	3651	#endif
4256	3652
4257	3653	#if __HVX_ARCH__ >= 65
4258	3654	/* ==========================================================================
...	...	@@ -4263,7 +3659,7 @@
4263	3659	========================================================================== */
4264	3660
4265	3661	#define Q6_vscatter_QRMWwV(Qs,Rt,Mu,Vvv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermhwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vvv,Vw)
4266		#endif /* __HEXAGON_ARCH___ >= 65 */
	3662	#endif
4267	3663
4268	3664	#if __HVX_ARCH__ >= 65
4269	3665	/* ==========================================================================
...	...	@@ -4274,7 +3670,7 @@
4274	3670	========================================================================== */
4275	3671
4276	3672	#define Q6_vscatter_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw)(Rt,Mu,Vv,Vw)
4277		#endif /* __HEXAGON_ARCH___ >= 65 */
	3673	#endif
4278	3674
4279	3675	#if __HVX_ARCH__ >= 65
4280	3676	/* ==========================================================================
...	...	@@ -4285,7 +3681,7 @@
4285	3681	========================================================================== */
4286	3682
4287	3683	#define Q6_vscatteracc_RMVwV(Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermw_add)(Rt,Mu,Vv,Vw)
4288		#endif /* __HEXAGON_ARCH___ >= 65 */
	3684	#endif
4289	3685
4290	3686	#if __HVX_ARCH__ >= 65
4291	3687	/* ==========================================================================
...	...	@@ -4296,7 +3692,7 @@
4296	3692	========================================================================== */
4297	3693
4298	3694	#define Q6_vscatter_QRMVwV(Qs,Rt,Mu,Vv,Vw) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vscattermwq)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1),Rt,Mu,Vv,Vw)
4299		#endif /* __HEXAGON_ARCH___ >= 65 */
	3695	#endif
4300	3696
4301	3697	#if __HVX_ARCH__ >= 66
4302	3698	/* ==========================================================================
...	...	@@ -4307,7 +3703,7 @@
4307	3703	========================================================================== */
4308	3704
4309	3705	#define Q6_Vw_vadd_VwVwQ_carry_sat(Vu,Vv,Qs) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vaddcarrysat)(Vu,Vv,__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qs),-1))
4310		#endif /* __HEXAGON_ARCH___ >= 66 */
	3706	#endif
4311	3707
4312	3708	#if __HVX_ARCH__ >= 66
4313	3709	/* ==========================================================================
...	...	@@ -4318,7 +3714,7 @@
4318	3714	========================================================================== */
4319	3715
4320	3716	#define Q6_Ww_vasrinto_WwVwVw(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasr_into)(Vxx,Vu,Vv)
4321		#endif /* __HEXAGON_ARCH___ >= 66 */
	3717	#endif
4322	3718
4323	3719	#if __HVX_ARCH__ >= 66
4324	3720	/* ==========================================================================
...	...	@@ -4329,7 +3725,7 @@
4329	3725	========================================================================== */
4330	3726
4331	3727	#define Q6_Vuw_vrotr_VuwVuw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vrotr)(Vu,Vv)
4332		#endif /* __HEXAGON_ARCH___ >= 66 */
	3728	#endif
4333	3729
4334	3730	#if __HVX_ARCH__ >= 66
4335	3731	/* ==========================================================================
...	...	@@ -4340,7 +3736,7 @@
4340	3736	========================================================================== */
4341	3737
4342	3738	#define Q6_Vw_vsatdw_VwVw(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsatdw)(Vu,Vv)
4343		#endif /* __HEXAGON_ARCH___ >= 66 */
	3739	#endif
4344	3740
4345	3741	#if __HVX_ARCH__ >= 68
4346	3742	/* ==========================================================================
...	...	@@ -4351,7 +3747,7 @@
4351	3747	========================================================================== */
4352	3748
4353	3749	#define Q6_Ww_v6mpy_WubWbI_h(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10)(Vuu,Vvv,Iu2)
4354		#endif /* __HEXAGON_ARCH___ >= 68 */
	3750	#endif
4355	3751
4356	3752	#if __HVX_ARCH__ >= 68
4357	3753	/* ==========================================================================
...	...	@@ -4362,7 +3758,7 @@
4362	3758	========================================================================== */
4363	3759
4364	3760	#define Q6_Ww_v6mpyacc_WwWubWbI_h(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyhubs10_vxx)(Vxx,Vuu,Vvv,Iu2)
4365		#endif /* __HEXAGON_ARCH___ >= 68 */
	3761	#endif
4366	3762
4367	3763	#if __HVX_ARCH__ >= 68
4368	3764	/* ==========================================================================
...	...	@@ -4373,7 +3769,7 @@
4373	3769	========================================================================== */
4374	3770
4375	3771	#define Q6_Ww_v6mpy_WubWbI_v(Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10)(Vuu,Vvv,Iu2)
4376		#endif /* __HEXAGON_ARCH___ >= 68 */
	3772	#endif
4377	3773
4378	3774	#if __HVX_ARCH__ >= 68
4379	3775	/* ==========================================================================
...	...	@@ -4384,9 +3780,9 @@
4384	3780	========================================================================== */
4385	3781
4386	3782	#define Q6_Ww_v6mpyacc_WwWubWbI_v(Vxx,Vuu,Vvv,Iu2) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_v6mpyvubs10_vxx)(Vxx,Vuu,Vvv,Iu2)
4387		#endif /* __HEXAGON_ARCH___ >= 68 */
	3783	#endif
4388	3784
4389		#if __HVX_ARCH__ >= 68
	3785	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4390	3786	/* ==========================================================================
4391	3787	Assembly Syntax: Vd32.hf=vabs(Vu32.hf)
4392	3788	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vabs_Vhf(HVX_Vector Vu)
...	...	@@ -4395,9 +3791,9 @@
4395	3791	========================================================================== */
4396	3792
4397	3793	#define Q6_Vhf_vabs_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_hf)(Vu)
4398		#endif /* __HEXAGON_ARCH___ >= 68 */
	3794	#endif
4399	3795
4400		#if __HVX_ARCH__ >= 68
	3796	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4401	3797	/* ==========================================================================
4402	3798	Assembly Syntax: Vd32.sf=vabs(Vu32.sf)
4403	3799	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vabs_Vsf(HVX_Vector Vu)
...	...	@@ -4406,7 +3802,7 @@
4406	3802	========================================================================== */
4407	3803
4408	3804	#define Q6_Vsf_vabs_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_sf)(Vu)
4409		#endif /* __HEXAGON_ARCH___ >= 68 */
	3805	#endif
4410	3806
4411	3807	#if __HVX_ARCH__ >= 68
4412	3808	/* ==========================================================================
...	...	@@ -4417,9 +3813,9 @@
4417	3813	========================================================================== */
4418	3814
4419	3815	#define Q6_Vqf16_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf)(Vu,Vv)
4420		#endif /* __HEXAGON_ARCH___ >= 68 */
	3816	#endif
4421	3817
4422		#if __HVX_ARCH__ >= 68
	3818	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4423	3819	/* ==========================================================================
4424	3820	Assembly Syntax: Vd32.hf=vadd(Vu32.hf,Vv32.hf)
4425	3821	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4428,7 +3824,7 @@
4428	3824	========================================================================== */
4429	3825
4430	3826	#define Q6_Vhf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_hf)(Vu,Vv)
4431		#endif /* __HEXAGON_ARCH___ >= 68 */
	3827	#endif
4432	3828
4433	3829	#if __HVX_ARCH__ >= 68
4434	3830	/* ==========================================================================
...	...	@@ -4439,7 +3835,7 @@
4439	3835	========================================================================== */
4440	3836
4441	3837	#define Q6_Vqf16_vadd_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16)(Vu,Vv)
4442		#endif /* __HEXAGON_ARCH___ >= 68 */
	3838	#endif
4443	3839
4444	3840	#if __HVX_ARCH__ >= 68
4445	3841	/* ==========================================================================
...	...	@@ -4450,7 +3846,7 @@
4450	3846	========================================================================== */
4451	3847
4452	3848	#define Q6_Vqf16_vadd_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf16_mix)(Vu,Vv)
4453		#endif /* __HEXAGON_ARCH___ >= 68 */
	3849	#endif
4454	3850
4455	3851	#if __HVX_ARCH__ >= 68
4456	3852	/* ==========================================================================
...	...	@@ -4461,7 +3857,7 @@
4461	3857	========================================================================== */
4462	3858
4463	3859	#define Q6_Vqf32_vadd_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32)(Vu,Vv)
4464		#endif /* __HEXAGON_ARCH___ >= 68 */
	3860	#endif
4465	3861
4466	3862	#if __HVX_ARCH__ >= 68
4467	3863	/* ==========================================================================
...	...	@@ -4472,7 +3868,7 @@
4472	3868	========================================================================== */
4473	3869
4474	3870	#define Q6_Vqf32_vadd_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_qf32_mix)(Vu,Vv)
4475		#endif /* __HEXAGON_ARCH___ >= 68 */
	3871	#endif
4476	3872
4477	3873	#if __HVX_ARCH__ >= 68
4478	3874	/* ==========================================================================
...	...	@@ -4483,9 +3879,9 @@
4483	3879	========================================================================== */
4484	3880
4485	3881	#define Q6_Vqf32_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf)(Vu,Vv)
4486		#endif /* __HEXAGON_ARCH___ >= 68 */
	3882	#endif
4487	3883
4488		#if __HVX_ARCH__ >= 68
	3884	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4489	3885	/* ==========================================================================
4490	3886	Assembly Syntax: Vdd32.sf=vadd(Vu32.hf,Vv32.hf)
4491	3887	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4494,9 +3890,9 @@
4494	3890	========================================================================== */
4495	3891
4496	3892	#define Q6_Wsf_vadd_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_hf)(Vu,Vv)
4497		#endif /* __HEXAGON_ARCH___ >= 68 */
	3893	#endif
4498	3894
4499		#if __HVX_ARCH__ >= 68
	3895	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4500	3896	/* ==========================================================================
4501	3897	Assembly Syntax: Vd32.sf=vadd(Vu32.sf,Vv32.sf)
4502	3898	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vadd_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4505,9 +3901,9 @@
4505	3901	========================================================================== */
4506	3902
4507	3903	#define Q6_Vsf_vadd_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_sf)(Vu,Vv)
4508		#endif /* __HEXAGON_ARCH___ >= 68 */
	3904	#endif
4509	3905
4510		#if __HVX_ARCH__ >= 68
	3906	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4511	3907	/* ==========================================================================
4512	3908	Assembly Syntax: Vd32.w=vfmv(Vu32.w)
4513	3909	C Intrinsic Prototype: HVX_Vector Q6_Vw_vfmv_Vw(HVX_Vector Vu)
...	...	@@ -4516,7 +3912,7 @@
4516	3912	========================================================================== */
4517	3913
4518	3914	#define Q6_Vw_vfmv_Vw(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vassign_fp)(Vu)
4519		#endif /* __HEXAGON_ARCH___ >= 68 */
	3915	#endif
4520	3916
4521	3917	#if __HVX_ARCH__ >= 68
4522	3918	/* ==========================================================================
...	...	@@ -4527,7 +3923,7 @@
4527	3923	========================================================================== */
4528	3924
4529	3925	#define Q6_Vhf_equals_Vqf16(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf16)(Vu)
4530		#endif /* __HEXAGON_ARCH___ >= 68 */
	3926	#endif
4531	3927
4532	3928	#if __HVX_ARCH__ >= 68
4533	3929	/* ==========================================================================
...	...	@@ -4538,7 +3934,7 @@
4538	3934	========================================================================== */
4539	3935
4540	3936	#define Q6_Vhf_equals_Wqf32(Vuu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_qf32)(Vuu)
4541		#endif /* __HEXAGON_ARCH___ >= 68 */
	3937	#endif
4542	3938
4543	3939	#if __HVX_ARCH__ >= 68
4544	3940	/* ==========================================================================
...	...	@@ -4549,9 +3945,9 @@
4549	3945	========================================================================== */
4550	3946
4551	3947	#define Q6_Vsf_equals_Vqf32(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_qf32)(Vu)
4552		#endif /* __HEXAGON_ARCH___ >= 68 */
	3948	#endif
4553	3949
4554		#if __HVX_ARCH__ >= 68
	3950	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4555	3951	/* ==========================================================================
4556	3952	Assembly Syntax: Vd32.b=vcvt(Vu32.hf,Vv32.hf)
4557	3953	C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4560,9 +3956,9 @@
4560	3956	========================================================================== */
4561	3957
4562	3958	#define Q6_Vb_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_b_hf)(Vu,Vv)
4563		#endif /* __HEXAGON_ARCH___ >= 68 */
	3959	#endif
4564	3960
4565		#if __HVX_ARCH__ >= 68
	3961	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4566	3962	/* ==========================================================================
4567	3963	Assembly Syntax: Vd32.h=vcvt(Vu32.hf)
4568	3964	C Intrinsic Prototype: HVX_Vector Q6_Vh_vcvt_Vhf(HVX_Vector Vu)
...	...	@@ -4571,9 +3967,9 @@
4571	3967	========================================================================== */
4572	3968
4573	3969	#define Q6_Vh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_h_hf)(Vu)
4574		#endif /* __HEXAGON_ARCH___ >= 68 */
	3970	#endif
4575	3971
4576		#if __HVX_ARCH__ >= 68
	3972	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4577	3973	/* ==========================================================================
4578	3974	Assembly Syntax: Vdd32.hf=vcvt(Vu32.b)
4579	3975	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vb(HVX_Vector Vu)
...	...	@@ -4582,9 +3978,9 @@
4582	3978	========================================================================== */
4583	3979
4584	3980	#define Q6_Whf_vcvt_Vb(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_b)(Vu)
4585		#endif /* __HEXAGON_ARCH___ >= 68 */
	3981	#endif
4586	3982
4587		#if __HVX_ARCH__ >= 68
	3983	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4588	3984	/* ==========================================================================
4589	3985	Assembly Syntax: Vd32.hf=vcvt(Vu32.h)
4590	3986	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vh(HVX_Vector Vu)
...	...	@@ -4593,9 +3989,9 @@
4593	3989	========================================================================== */
4594	3990
4595	3991	#define Q6_Vhf_vcvt_Vh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_h)(Vu)
4596		#endif /* __HEXAGON_ARCH___ >= 68 */
	3992	#endif
4597	3993
4598		#if __HVX_ARCH__ >= 68
	3994	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4599	3995	/* ==========================================================================
4600	3996	Assembly Syntax: Vd32.hf=vcvt(Vu32.sf,Vv32.sf)
4601	3997	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4604,9 +4000,9 @@
4604	4000	========================================================================== */
4605	4001
4606	4002	#define Q6_Vhf_vcvt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_sf)(Vu,Vv)
4607		#endif /* __HEXAGON_ARCH___ >= 68 */
	4003	#endif
4608	4004
4609		#if __HVX_ARCH__ >= 68
	4005	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4610	4006	/* ==========================================================================
4611	4007	Assembly Syntax: Vdd32.hf=vcvt(Vu32.ub)
4612	4008	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_Vub(HVX_Vector Vu)
...	...	@@ -4615,9 +4011,9 @@
4615	4011	========================================================================== */
4616	4012
4617	4013	#define Q6_Whf_vcvt_Vub(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_ub)(Vu)
4618		#endif /* __HEXAGON_ARCH___ >= 68 */
	4014	#endif
4619	4015
4620		#if __HVX_ARCH__ >= 68
	4016	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4621	4017	/* ==========================================================================
4622	4018	Assembly Syntax: Vd32.hf=vcvt(Vu32.uh)
4623	4019	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vcvt_Vuh(HVX_Vector Vu)
...	...	@@ -4626,9 +4022,9 @@
4626	4022	========================================================================== */
4627	4023
4628	4024	#define Q6_Vhf_vcvt_Vuh(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_uh)(Vu)
4629		#endif /* __HEXAGON_ARCH___ >= 68 */
	4025	#endif
4630	4026
4631		#if __HVX_ARCH__ >= 68
	4027	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4632	4028	/* ==========================================================================
4633	4029	Assembly Syntax: Vdd32.sf=vcvt(Vu32.hf)
4634	4030	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vcvt_Vhf(HVX_Vector Vu)
...	...	@@ -4637,9 +4033,9 @@
4637	4033	========================================================================== */
4638	4034
4639	4035	#define Q6_Wsf_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_sf_hf)(Vu)
4640		#endif /* __HEXAGON_ARCH___ >= 68 */
	4036	#endif
4641	4037
4642		#if __HVX_ARCH__ >= 68
	4038	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4643	4039	/* ==========================================================================
4644	4040	Assembly Syntax: Vd32.ub=vcvt(Vu32.hf,Vv32.hf)
4645	4041	C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4648,9 +4044,9 @@
4648	4044	========================================================================== */
4649	4045
4650	4046	#define Q6_Vub_vcvt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_ub_hf)(Vu,Vv)
4651		#endif /* __HEXAGON_ARCH___ >= 68 */
	4047	#endif
4652	4048
4653		#if __HVX_ARCH__ >= 68
	4049	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4654	4050	/* ==========================================================================
4655	4051	Assembly Syntax: Vd32.uh=vcvt(Vu32.hf)
4656	4052	C Intrinsic Prototype: HVX_Vector Q6_Vuh_vcvt_Vhf(HVX_Vector Vu)
...	...	@@ -4659,9 +4055,9 @@
4659	4055	========================================================================== */
4660	4056
4661	4057	#define Q6_Vuh_vcvt_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_uh_hf)(Vu)
4662		#endif /* __HEXAGON_ARCH___ >= 68 */
	4058	#endif
4663	4059
4664		#if __HVX_ARCH__ >= 68
	4060	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4665	4061	/* ==========================================================================
4666	4062	Assembly Syntax: Vd32.sf=vdmpy(Vu32.hf,Vv32.hf)
4667	4063	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4670,9 +4066,9 @@
4670	4066	========================================================================== */
4671	4067
4672	4068	#define Q6_Vsf_vdmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf)(Vu,Vv)
4673		#endif /* __HEXAGON_ARCH___ >= 68 */
	4069	#endif
4674	4070
4675		#if __HVX_ARCH__ >= 68
	4071	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4676	4072	/* ==========================================================================
4677	4073	Assembly Syntax: Vx32.sf+=vdmpy(Vu32.hf,Vv32.hf)
4678	4074	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vdmpyacc_VsfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4681,9 +4077,9 @@
4681	4077	========================================================================== */
4682	4078
4683	4079	#define Q6_Vsf_vdmpyacc_VsfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vdmpy_sf_hf_acc)(Vx,Vu,Vv)
4684		#endif /* __HEXAGON_ARCH___ >= 68 */
	4080	#endif
4685	4081
4686		#if __HVX_ARCH__ >= 68
	4082	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4687	4083	/* ==========================================================================
4688	4084	Assembly Syntax: Vd32.hf=vfmax(Vu32.hf,Vv32.hf)
4689	4085	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmax_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4692,9 +4088,9 @@
4692	4088	========================================================================== */
4693	4089
4694	4090	#define Q6_Vhf_vfmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_hf)(Vu,Vv)
4695		#endif /* __HEXAGON_ARCH___ >= 68 */
	4091	#endif
4696	4092
4697		#if __HVX_ARCH__ >= 68
	4093	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4698	4094	/* ==========================================================================
4699	4095	Assembly Syntax: Vd32.sf=vfmax(Vu32.sf,Vv32.sf)
4700	4096	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmax_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4703,9 +4099,9 @@
4703	4099	========================================================================== */
4704	4100
4705	4101	#define Q6_Vsf_vfmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_sf)(Vu,Vv)
4706		#endif /* __HEXAGON_ARCH___ >= 68 */
	4102	#endif
4707	4103
4708		#if __HVX_ARCH__ >= 68
	4104	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4709	4105	/* ==========================================================================
4710	4106	Assembly Syntax: Vd32.hf=vfmin(Vu32.hf,Vv32.hf)
4711	4107	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfmin_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4714,9 +4110,9 @@
4714	4110	========================================================================== */
4715	4111
4716	4112	#define Q6_Vhf_vfmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_hf)(Vu,Vv)
4717		#endif /* __HEXAGON_ARCH___ >= 68 */
	4113	#endif
4718	4114
4719		#if __HVX_ARCH__ >= 68
	4115	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4720	4116	/* ==========================================================================
4721	4117	Assembly Syntax: Vd32.sf=vfmin(Vu32.sf,Vv32.sf)
4722	4118	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfmin_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4725,9 +4121,9 @@
4725	4121	========================================================================== */
4726	4122
4727	4123	#define Q6_Vsf_vfmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_sf)(Vu,Vv)
4728		#endif /* __HEXAGON_ARCH___ >= 68 */
	4124	#endif
4729	4125
4730		#if __HVX_ARCH__ >= 68
	4126	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4731	4127	/* ==========================================================================
4732	4128	Assembly Syntax: Vd32.hf=vfneg(Vu32.hf)
4733	4129	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vfneg_Vhf(HVX_Vector Vu)
...	...	@@ -4736,9 +4132,9 @@
4736	4132	========================================================================== */
4737	4133
4738	4134	#define Q6_Vhf_vfneg_Vhf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_hf)(Vu)
4739		#endif /* __HEXAGON_ARCH___ >= 68 */
	4135	#endif
4740	4136
4741		#if __HVX_ARCH__ >= 68
	4137	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4742	4138	/* ==========================================================================
4743	4139	Assembly Syntax: Vd32.sf=vfneg(Vu32.sf)
4744	4140	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vfneg_Vsf(HVX_Vector Vu)
...	...	@@ -4747,7 +4143,7 @@
4747	4143	========================================================================== */
4748	4144
4749	4145	#define Q6_Vsf_vfneg_Vsf(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_sf)(Vu)
4750		#endif /* __HEXAGON_ARCH___ >= 68 */
	4146	#endif
4751	4147
4752	4148	#if __HVX_ARCH__ >= 68
4753	4149	/* ==========================================================================
...	...	@@ -4758,7 +4154,7 @@
4758	4154	========================================================================== */
4759	4155
4760	4156	#define Q6_Q_vcmp_gt_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf)(Vu,Vv)),-1)
4761		#endif /* __HEXAGON_ARCH___ >= 68 */
	4157	#endif
4762	4158
4763	4159	#if __HVX_ARCH__ >= 68
4764	4160	/* ==========================================================================
...	...	@@ -4769,7 +4165,7 @@
4769	4165	========================================================================== */
4770	4166
4771	4167	#define Q6_Q_vcmp_gtand_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4772		#endif /* __HEXAGON_ARCH___ >= 68 */
	4168	#endif
4773	4169
4774	4170	#if __HVX_ARCH__ >= 68
4775	4171	/* ==========================================================================
...	...	@@ -4780,7 +4176,7 @@
4780	4176	========================================================================== */
4781	4177
4782	4178	#define Q6_Q_vcmp_gtor_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4783		#endif /* __HEXAGON_ARCH___ >= 68 */
	4179	#endif
4784	4180
4785	4181	#if __HVX_ARCH__ >= 68
4786	4182	/* ==========================================================================
...	...	@@ -4791,7 +4187,7 @@
4791	4187	========================================================================== */
4792	4188
4793	4189	#define Q6_Q_vcmp_gtxacc_QVhfVhf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgthf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4794		#endif /* __HEXAGON_ARCH___ >= 68 */
	4190	#endif
4795	4191
4796	4192	#if __HVX_ARCH__ >= 68
4797	4193	/* ==========================================================================
...	...	@@ -4802,7 +4198,7 @@
4802	4198	========================================================================== */
4803	4199
4804	4200	#define Q6_Q_vcmp_gt_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf)(Vu,Vv)),-1)
4805		#endif /* __HEXAGON_ARCH___ >= 68 */
	4201	#endif
4806	4202
4807	4203	#if __HVX_ARCH__ >= 68
4808	4204	/* ==========================================================================
...	...	@@ -4813,7 +4209,7 @@
4813	4209	========================================================================== */
4814	4210
4815	4211	#define Q6_Q_vcmp_gtand_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_and)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4816		#endif /* __HEXAGON_ARCH___ >= 68 */
	4212	#endif
4817	4213
4818	4214	#if __HVX_ARCH__ >= 68
4819	4215	/* ==========================================================================
...	...	@@ -4824,7 +4220,7 @@
4824	4220	========================================================================== */
4825	4221
4826	4222	#define Q6_Q_vcmp_gtor_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_or)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4827		#endif /* __HEXAGON_ARCH___ >= 68 */
	4223	#endif
4828	4224
4829	4225	#if __HVX_ARCH__ >= 68
4830	4226	/* ==========================================================================
...	...	@@ -4835,7 +4231,7 @@
4835	4231	========================================================================== */
4836	4232
4837	4233	#define Q6_Q_vcmp_gtxacc_QVsfVsf(Qx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtsf_xor)(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx),-1),Vu,Vv)),-1)
4838		#endif /* __HEXAGON_ARCH___ >= 68 */
	4234	#endif
4839	4235
4840	4236	#if __HVX_ARCH__ >= 68
4841	4237	/* ==========================================================================
...	...	@@ -4846,7 +4242,7 @@
4846	4242	========================================================================== */
4847	4243
4848	4244	#define Q6_Vhf_vmax_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_hf)(Vu,Vv)
4849		#endif /* __HEXAGON_ARCH___ >= 68 */
	4245	#endif
4850	4246
4851	4247	#if __HVX_ARCH__ >= 68
4852	4248	/* ==========================================================================
...	...	@@ -4857,7 +4253,7 @@
4857	4253	========================================================================== */
4858	4254
4859	4255	#define Q6_Vsf_vmax_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_sf)(Vu,Vv)
4860		#endif /* __HEXAGON_ARCH___ >= 68 */
	4256	#endif
4861	4257
4862	4258	#if __HVX_ARCH__ >= 68
4863	4259	/* ==========================================================================
...	...	@@ -4868,7 +4264,7 @@
4868	4264	========================================================================== */
4869	4265
4870	4266	#define Q6_Vhf_vmin_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_hf)(Vu,Vv)
4871		#endif /* __HEXAGON_ARCH___ >= 68 */
	4267	#endif
4872	4268
4873	4269	#if __HVX_ARCH__ >= 68
4874	4270	/* ==========================================================================
...	...	@@ -4879,9 +4275,9 @@
4879	4275	========================================================================== */
4880	4276
4881	4277	#define Q6_Vsf_vmin_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_sf)(Vu,Vv)
4882		#endif /* __HEXAGON_ARCH___ >= 68 */
	4278	#endif
4883	4279
4884		#if __HVX_ARCH__ >= 68
	4280	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4885	4281	/* ==========================================================================
4886	4282	Assembly Syntax: Vd32.hf=vmpy(Vu32.hf,Vv32.hf)
4887	4283	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4890,9 +4286,9 @@
4890	4286	========================================================================== */
4891	4287
4892	4288	#define Q6_Vhf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf)(Vu,Vv)
4893		#endif /* __HEXAGON_ARCH___ >= 68 */
	4289	#endif
4894	4290
4895		#if __HVX_ARCH__ >= 68
	4291	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4896	4292	/* ==========================================================================
4897	4293	Assembly Syntax: Vx32.hf+=vmpy(Vu32.hf,Vv32.hf)
4898	4294	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vmpyacc_VhfVhfVhf(HVX_Vector Vx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -4901,7 +4297,7 @@
4901	4297	========================================================================== */
4902	4298
4903	4299	#define Q6_Vhf_vmpyacc_VhfVhfVhf(Vx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_hf_acc)(Vx,Vu,Vv)
4904		#endif /* __HEXAGON_ARCH___ >= 68 */
	4300	#endif
4905	4301
4906	4302	#if __HVX_ARCH__ >= 68
4907	4303	/* ==========================================================================
...	...	@@ -4912,7 +4308,7 @@
4912	4308	========================================================================== */
4913	4309
4914	4310	#define Q6_Vqf16_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16)(Vu,Vv)
4915		#endif /* __HEXAGON_ARCH___ >= 68 */
	4311	#endif
4916	4312
4917	4313	#if __HVX_ARCH__ >= 68
4918	4314	/* ==========================================================================
...	...	@@ -4923,7 +4319,7 @@
4923	4319	========================================================================== */
4924	4320
4925	4321	#define Q6_Vqf16_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_hf)(Vu,Vv)
4926		#endif /* __HEXAGON_ARCH___ >= 68 */
	4322	#endif
4927	4323
4928	4324	#if __HVX_ARCH__ >= 68
4929	4325	/* ==========================================================================
...	...	@@ -4934,7 +4330,7 @@
4934	4330	========================================================================== */
4935	4331
4936	4332	#define Q6_Vqf16_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf16_mix_hf)(Vu,Vv)
4937		#endif /* __HEXAGON_ARCH___ >= 68 */
	4333	#endif
4938	4334
4939	4335	#if __HVX_ARCH__ >= 68
4940	4336	/* ==========================================================================
...	...	@@ -4945,7 +4341,7 @@
4945	4341	========================================================================== */
4946	4342
4947	4343	#define Q6_Vqf32_vmpy_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32)(Vu,Vv)
4948		#endif /* __HEXAGON_ARCH___ >= 68 */
	4344	#endif
4949	4345
4950	4346	#if __HVX_ARCH__ >= 68
4951	4347	/* ==========================================================================
...	...	@@ -4956,7 +4352,7 @@
4956	4352	========================================================================== */
4957	4353
4958	4354	#define Q6_Wqf32_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_hf)(Vu,Vv)
4959		#endif /* __HEXAGON_ARCH___ >= 68 */
	4355	#endif
4960	4356
4961	4357	#if __HVX_ARCH__ >= 68
4962	4358	/* ==========================================================================
...	...	@@ -4967,7 +4363,7 @@
4967	4363	========================================================================== */
4968	4364
4969	4365	#define Q6_Wqf32_vmpy_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_mix_hf)(Vu,Vv)
4970		#endif /* __HEXAGON_ARCH___ >= 68 */
	4366	#endif
4971	4367
4972	4368	#if __HVX_ARCH__ >= 68
4973	4369	/* ==========================================================================
...	...	@@ -4978,7 +4374,7 @@
4978	4374	========================================================================== */
4979	4375
4980	4376	#define Q6_Wqf32_vmpy_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_qf16)(Vu,Vv)
4981		#endif /* __HEXAGON_ARCH___ >= 68 */
	4377	#endif
4982	4378
4983	4379	#if __HVX_ARCH__ >= 68
4984	4380	/* ==========================================================================
...	...	@@ -4989,9 +4385,9 @@
4989	4385	========================================================================== */
4990	4386
4991	4387	#define Q6_Vqf32_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_qf32_sf)(Vu,Vv)
4992		#endif /* __HEXAGON_ARCH___ >= 68 */
	4388	#endif
4993	4389
4994		#if __HVX_ARCH__ >= 68
	4390	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
4995	4391	/* ==========================================================================
4996	4392	Assembly Syntax: Vdd32.sf=vmpy(Vu32.hf,Vv32.hf)
4997	4393	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5000,9 +4396,9 @@
5000	4396	========================================================================== */
5001	4397
5002	4398	#define Q6_Wsf_vmpy_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf)(Vu,Vv)
5003		#endif /* __HEXAGON_ARCH___ >= 68 */
	4399	#endif
5004	4400
5005		#if __HVX_ARCH__ >= 68
	4401	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
5006	4402	/* ==========================================================================
5007	4403	Assembly Syntax: Vxx32.sf+=vmpy(Vu32.hf,Vv32.hf)
5008	4404	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVhfVhf(HVX_VectorPair Vxx, HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5011,9 +4407,9 @@
5011	4407	========================================================================== */
5012	4408
5013	4409	#define Q6_Wsf_vmpyacc_WsfVhfVhf(Vxx,Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_hf_acc)(Vxx,Vu,Vv)
5014		#endif /* __HEXAGON_ARCH___ >= 68 */
	4410	#endif
5015	4411
5016		#if __HVX_ARCH__ >= 68
	4412	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
5017	4413	/* ==========================================================================
5018	4414	Assembly Syntax: Vd32.sf=vmpy(Vu32.sf,Vv32.sf)
5019	4415	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vmpy_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5022,7 +4418,7 @@
5022	4418	========================================================================== */
5023	4419
5024	4420	#define Q6_Vsf_vmpy_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_sf)(Vu,Vv)
5025		#endif /* __HEXAGON_ARCH___ >= 68 */
	4421	#endif
5026	4422
5027	4423	#if __HVX_ARCH__ >= 68
5028	4424	/* ==========================================================================
...	...	@@ -5033,9 +4429,9 @@
5033	4429	========================================================================== */
5034	4430
5035	4431	#define Q6_Vqf16_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf)(Vu,Vv)
5036		#endif /* __HEXAGON_ARCH___ >= 68 */
	4432	#endif
5037	4433
5038		#if __HVX_ARCH__ >= 68
	4434	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
5039	4435	/* ==========================================================================
5040	4436	Assembly Syntax: Vd32.hf=vsub(Vu32.hf,Vv32.hf)
5041	4437	C Intrinsic Prototype: HVX_Vector Q6_Vhf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5044,7 +4440,7 @@
5044	4440	========================================================================== */
5045	4441
5046	4442	#define Q6_Vhf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_hf)(Vu,Vv)
5047		#endif /* __HEXAGON_ARCH___ >= 68 */
	4443	#endif
5048	4444
5049	4445	#if __HVX_ARCH__ >= 68
5050	4446	/* ==========================================================================
...	...	@@ -5055,7 +4451,7 @@
5055	4451	========================================================================== */
5056	4452
5057	4453	#define Q6_Vqf16_vsub_Vqf16Vqf16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16)(Vu,Vv)
5058		#endif /* __HEXAGON_ARCH___ >= 68 */
	4454	#endif
5059	4455
5060	4456	#if __HVX_ARCH__ >= 68
5061	4457	/* ==========================================================================
...	...	@@ -5066,7 +4462,7 @@
5066	4462	========================================================================== */
5067	4463
5068	4464	#define Q6_Vqf16_vsub_Vqf16Vhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf16_mix)(Vu,Vv)
5069		#endif /* __HEXAGON_ARCH___ >= 68 */
	4465	#endif
5070	4466
5071	4467	#if __HVX_ARCH__ >= 68
5072	4468	/* ==========================================================================
...	...	@@ -5077,7 +4473,7 @@
5077	4473	========================================================================== */
5078	4474
5079	4475	#define Q6_Vqf32_vsub_Vqf32Vqf32(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32)(Vu,Vv)
5080		#endif /* __HEXAGON_ARCH___ >= 68 */
	4476	#endif
5081	4477
5082	4478	#if __HVX_ARCH__ >= 68
5083	4479	/* ==========================================================================
...	...	@@ -5088,7 +4484,7 @@
5088	4484	========================================================================== */
5089	4485
5090	4486	#define Q6_Vqf32_vsub_Vqf32Vsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_qf32_mix)(Vu,Vv)
5091		#endif /* __HEXAGON_ARCH___ >= 68 */
	4487	#endif
5092	4488
5093	4489	#if __HVX_ARCH__ >= 68
5094	4490	/* ==========================================================================
...	...	@@ -5099,9 +4495,9 @@
5099	4495	========================================================================== */
5100	4496
5101	4497	#define Q6_Vqf32_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf)(Vu,Vv)
5102		#endif /* __HEXAGON_ARCH___ >= 68 */
	4498	#endif
5103	4499
5104		#if __HVX_ARCH__ >= 68
	4500	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
5105	4501	/* ==========================================================================
5106	4502	Assembly Syntax: Vdd32.sf=vsub(Vu32.hf,Vv32.hf)
5107	4503	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VhfVhf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5110,9 +4506,9 @@
5110	4506	========================================================================== */
5111	4507
5112	4508	#define Q6_Wsf_vsub_VhfVhf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_hf)(Vu,Vv)
5113		#endif /* __HEXAGON_ARCH___ >= 68 */
	4509	#endif
5114	4510
5115		#if __HVX_ARCH__ >= 68
	4511	#if __HVX_ARCH__ >= 68 && defined __HVX_IEEE_FP__
5116	4512	/* ==========================================================================
5117	4513	Assembly Syntax: Vd32.sf=vsub(Vu32.sf,Vv32.sf)
5118	4514	C Intrinsic Prototype: HVX_Vector Q6_Vsf_vsub_VsfVsf(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5121,7 +4517,7 @@
5121	4517	========================================================================== */
5122	4518
5123	4519	#define Q6_Vsf_vsub_VsfVsf(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_sf)(Vu,Vv)
5124		#endif /* __HEXAGON_ARCH___ >= 68 */
	4520	#endif
5125	4521
5126	4522	#if __HVX_ARCH__ >= 69
5127	4523	/* ==========================================================================
...	...	@@ -5132,7 +4528,7 @@
5132	4528	========================================================================== */
5133	4529
5134	4530	#define Q6_Vub_vasr_WuhVub_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubrndsat)(Vuu,Vv)
5135		#endif /* __HEXAGON_ARCH___ >= 69 */
	4531	#endif
5136	4532
5137	4533	#if __HVX_ARCH__ >= 69
5138	4534	/* ==========================================================================
...	...	@@ -5143,7 +4539,7 @@
5143	4539	========================================================================== */
5144	4540
5145	4541	#define Q6_Vub_vasr_WuhVub_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvuhubsat)(Vuu,Vv)
5146		#endif /* __HEXAGON_ARCH___ >= 69 */
	4542	#endif
5147	4543
5148	4544	#if __HVX_ARCH__ >= 69
5149	4545	/* ==========================================================================
...	...	@@ -5154,7 +4550,7 @@
5154	4550	========================================================================== */
5155	4551
5156	4552	#define Q6_Vuh_vasr_WwVuh_rnd_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhrndsat)(Vuu,Vv)
5157		#endif /* __HEXAGON_ARCH___ >= 69 */
	4553	#endif
5158	4554
5159	4555	#if __HVX_ARCH__ >= 69
5160	4556	/* ==========================================================================
...	...	@@ -5165,7 +4561,7 @@
5165	4561	========================================================================== */
5166	4562
5167	4563	#define Q6_Vuh_vasr_WwVuh_sat(Vuu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vasrvwuhsat)(Vuu,Vv)
5168		#endif /* __HEXAGON_ARCH___ >= 69 */
	4564	#endif
5169	4565
5170	4566	#if __HVX_ARCH__ >= 69
5171	4567	/* ==========================================================================
...	...	@@ -5176,9 +4572,9 @@
5176	4572	========================================================================== */
5177	4573
5178	4574	#define Q6_Vuh_vmpy_VuhVuh_rs16(Vu,Vv) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpyuhvs)(Vu,Vv)
5179		#endif /* __HEXAGON_ARCH___ >= 69 */
	4575	#endif
5180	4576
5181		#if __HVX_ARCH__ >= 73
	4577	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5182	4578	/* ==========================================================================
5183	4579	Assembly Syntax: Vdd32.sf=vadd(Vu32.bf,Vv32.bf)
5184	4580	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vadd_VbfVbf(HVX_Vector Vu,
...	...	@@ -5187,7 +4583,7 @@
5187	4583
5188	4584	#define Q6_Wsf_vadd_VbfVbf(Vu, Vv) \
5189	4585	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_sf_bf)(Vu, Vv)
5190		#endif /* __HEXAGON_ARCH___ >= 73 */
	4586	#endif
5191	4587
5192	4588	#if __HVX_ARCH__ >= 73
5193	4589	/* ==========================================================================
...	...	@@ -5199,7 +4595,7 @@
5199	4595
5200	4596	#define Q6_Vh_equals_Vhf(Vu) \
5201	4597	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_h_hf)(Vu)
5202		#endif /* __HEXAGON_ARCH___ >= 73 */
	4598	#endif
5203	4599
5204	4600	#if __HVX_ARCH__ >= 73
5205	4601	/* ==========================================================================
...	...	@@ -5211,7 +4607,7 @@
5211	4607
5212	4608	#define Q6_Vhf_equals_Vh(Vu) \
5213	4609	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_hf_h)(Vu)
5214		#endif /* __HEXAGON_ARCH___ >= 73 */
	4610	#endif
5215	4611
5216	4612	#if __HVX_ARCH__ >= 73
5217	4613	/* ==========================================================================
...	...	@@ -5223,7 +4619,7 @@
5223	4619
5224	4620	#define Q6_Vsf_equals_Vw(Vu) \
5225	4621	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_sf_w)(Vu)
5226		#endif /* __HEXAGON_ARCH___ >= 73 */
	4622	#endif
5227	4623
5228	4624	#if __HVX_ARCH__ >= 73
5229	4625	/* ==========================================================================
...	...	@@ -5235,9 +4631,9 @@
5235	4631
5236	4632	#define Q6_Vw_equals_Vsf(Vu) \
5237	4633	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_w_sf)(Vu)
5238		#endif /* __HEXAGON_ARCH___ >= 73 */
	4634	#endif
5239	4635
5240		#if __HVX_ARCH__ >= 73
	4636	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5241	4637	/* ==========================================================================
5242	4638	Assembly Syntax: Vd32.bf=vcvt(Vu32.sf,Vv32.sf)
5243	4639	C Intrinsic Prototype: HVX_Vector Q6_Vbf_vcvt_VsfVsf(HVX_Vector Vu,
...	...	@@ -5246,7 +4642,7 @@
5246	4642
5247	4643	#define Q6_Vbf_vcvt_VsfVsf(Vu, Vv) \
5248	4644	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_bf_sf)(Vu, Vv)
5249		#endif /* __HEXAGON_ARCH___ >= 73 */
	4645	#endif
5250	4646
5251	4647	#if __HVX_ARCH__ >= 73
5252	4648	/* ==========================================================================
...	...	@@ -5258,7 +4654,7 @@
5258	4654	#define Q6_Q_vcmp_gt_VbfVbf(Vu, Vv) \
5259	4655	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt) \
5260	4656	((__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vgtbf)(Vu, Vv)), -1)
5261		#endif /* __HEXAGON_ARCH___ >= 73 */
	4657	#endif
5262	4658
5263	4659	#if __HVX_ARCH__ >= 73
5264	4660	/* ==========================================================================
...	...	@@ -5274,7 +4670,7 @@
5274	4670	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5275	4671	Vv)), \
5276	4672	-1)
5277		#endif /* __HEXAGON_ARCH___ >= 73 */
	4673	#endif
5278	4674
5279	4675	#if __HVX_ARCH__ >= 73
5280	4676	/* ==========================================================================
...	...	@@ -5290,7 +4686,7 @@
5290	4686	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5291	4687	Vv)), \
5292	4688	-1)
5293		#endif /* __HEXAGON_ARCH___ >= 73 */
	4689	#endif
5294	4690
5295	4691	#if __HVX_ARCH__ >= 73
5296	4692	/* ==========================================================================
...	...	@@ -5306,9 +4702,9 @@
5306	4702	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
5307	4703	Vv)), \
5308	4704	-1)
5309		#endif /* __HEXAGON_ARCH___ >= 73 */
	4705	#endif
5310	4706
5311		#if __HVX_ARCH__ >= 73
	4707	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5312	4708	/* ==========================================================================
5313	4709	Assembly Syntax: Vd32.bf=vmax(Vu32.bf,Vv32.bf)
5314	4710	C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmax_VbfVbf(HVX_Vector Vu,
...	...	@@ -5317,9 +4713,9 @@
5317	4713
5318	4714	#define Q6_Vbf_vmax_VbfVbf(Vu, Vv) \
5319	4715	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmax_bf)(Vu, Vv)
5320		#endif /* __HEXAGON_ARCH___ >= 73 */
	4716	#endif
5321	4717
5322		#if __HVX_ARCH__ >= 73
	4718	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5323	4719	/* ==========================================================================
5324	4720	Assembly Syntax: Vd32.bf=vmin(Vu32.bf,Vv32.bf)
5325	4721	C Intrinsic Prototype: HVX_Vector Q6_Vbf_vmin_VbfVbf(HVX_Vector Vu,
...	...	@@ -5328,9 +4724,9 @@
5328	4724
5329	4725	#define Q6_Vbf_vmin_VbfVbf(Vu, Vv) \
5330	4726	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmin_bf)(Vu, Vv)
5331		#endif /* __HEXAGON_ARCH___ >= 73 */
	4727	#endif
5332	4728
5333		#if __HVX_ARCH__ >= 73
	4729	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5334	4730	/* ==========================================================================
5335	4731	Assembly Syntax: Vdd32.sf=vmpy(Vu32.bf,Vv32.bf)
5336	4732	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpy_VbfVbf(HVX_Vector Vu,
...	...	@@ -5339,9 +4735,9 @@
5339	4735
5340	4736	#define Q6_Wsf_vmpy_VbfVbf(Vu, Vv) \
5341	4737	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf)(Vu, Vv)
5342		#endif /* __HEXAGON_ARCH___ >= 73 */
	4738	#endif
5343	4739
5344		#if __HVX_ARCH__ >= 73
	4740	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5345	4741	/* ==========================================================================
5346	4742	Assembly Syntax: Vxx32.sf+=vmpy(Vu32.bf,Vv32.bf)
5347	4743	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vmpyacc_WsfVbfVbf(HVX_VectorPair
...	...	@@ -5351,9 +4747,9 @@
5351	4747
5352	4748	#define Q6_Wsf_vmpyacc_WsfVbfVbf(Vxx, Vu, Vv) \
5353	4749	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_sf_bf_acc)(Vxx, Vu, Vv)
5354		#endif /* __HEXAGON_ARCH___ >= 73 */
	4750	#endif
5355	4751
5356		#if __HVX_ARCH__ >= 73
	4752	#if __HVX_ARCH__ >= 73 && defined __HVX_IEEE_FP__
5357	4753	/* ==========================================================================
5358	4754	Assembly Syntax: Vdd32.sf=vsub(Vu32.bf,Vv32.bf)
5359	4755	C Intrinsic Prototype: HVX_VectorPair Q6_Wsf_vsub_VbfVbf(HVX_Vector Vu,
...	...	@@ -5362,7 +4758,7 @@
5362	4758
5363	4759	#define Q6_Wsf_vsub_VbfVbf(Vu, Vv) \
5364	4760	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_bf)(Vu, Vv)
5365		#endif /* __HEXAGON_ARCH___ >= 73 */
	4761	#endif
5366	4762
5367	4763	#if __HVX_ARCH__ >= 79
5368	4764	/* ==========================================================================
...	...	@@ -5374,7 +4770,7 @@
5374	4770
5375	4771	#define Q6_V_vgetqfext_VR(Vu, Rt) \
5376	4772	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext)(Vu, Rt)
5377		#endif /* __HEXAGON_ARCH___ >= 79 */
	4773	#endif
5378	4774
5379	4775	#if __HVX_ARCH__ >= 79
5380	4776	/* ==========================================================================
...	...	@@ -5386,7 +4782,7 @@
5386	4782
5387	4783	#define Q6_V_vgetqfextor_VVR(Vx, Vu, Rt) \
5388	4784	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_get_qfext_oracc)(Vx, Vu, Rt)
5389		#endif /* __HEXAGON_ARCH___ >= 79 */
	4785	#endif
5390	4786
5391	4787	#if __HVX_ARCH__ >= 79
5392	4788	/* ==========================================================================
...	...	@@ -5398,9 +4794,9 @@
5398	4794
5399	4795	#define Q6_V_vsetqfext_VR(Vu, Rt) \
5400	4796	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_set_qfext)(Vu, Rt)
5401		#endif /* __HEXAGON_ARCH___ >= 79 */
	4797	#endif
5402	4798
5403		#if __HVX_ARCH__ >= 79
	4799	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5404	4800	/* ==========================================================================
5405	4801	Assembly Syntax: Vd32.f8=vabs(Vu32.f8)
5406	4802	C Intrinsic Prototype: HVX_Vector Q6_V_vabs_V(HVX_Vector Vu)
...	...	@@ -5409,9 +4805,9 @@
5409	4805	========================================================================== */
5410	4806
5411	4807	#define Q6_V_vabs_V(Vu) __BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_f8)(Vu)
5412		#endif /* __HEXAGON_ARCH___ >= 79 */
	4808	#endif
5413	4809
5414		#if __HVX_ARCH__ >= 79
	4810	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5415	4811	/* ==========================================================================
5416	4812	Assembly Syntax: Vdd32.hf=vadd(Vu32.f8,Vv32.f8)
5417	4813	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vadd_VV(HVX_Vector Vu,
...	...	@@ -5420,9 +4816,9 @@
5420	4816
5421	4817	#define Q6_Whf_vadd_VV(Vu, Vv) \
5422	4818	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vadd_hf_f8)(Vu, Vv)
5423		#endif /* __HEXAGON_ARCH___ >= 79 */
	4819	#endif
5424	4820
5425		#if __HVX_ARCH__ >= 79
	4821	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5426	4822	/* ==========================================================================
5427	4823	Assembly Syntax: Vd32.b=vcvt2(Vu32.hf,Vv32.hf)
5428	4824	C Intrinsic Prototype: HVX_Vector Q6_Vb_vcvt2_VhfVhf(HVX_Vector Vu,
...	...	@@ -5431,9 +4827,9 @@
5431	4827
5432	4828	#define Q6_Vb_vcvt2_VhfVhf(Vu, Vv) \
5433	4829	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_b_hf)(Vu, Vv)
5434		#endif /* __HEXAGON_ARCH___ >= 79 */
	4830	#endif
5435	4831
5436		#if __HVX_ARCH__ >= 79
	4832	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5437	4833	/* ==========================================================================
5438	4834	Assembly Syntax: Vdd32.hf=vcvt2(Vu32.b)
5439	4835	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vb(HVX_Vector Vu)
...	...	@@ -5443,9 +4839,9 @@
5443	4839
5444	4840	#define Q6_Whf_vcvt2_Vb(Vu) \
5445	4841	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_b)(Vu)
5446		#endif /* __HEXAGON_ARCH___ >= 79 */
	4842	#endif
5447	4843
5448		#if __HVX_ARCH__ >= 79
	4844	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5449	4845	/* ==========================================================================
5450	4846	Assembly Syntax: Vdd32.hf=vcvt2(Vu32.ub)
5451	4847	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt2_Vub(HVX_Vector Vu)
...	...	@@ -5455,9 +4851,9 @@
5455	4851
5456	4852	#define Q6_Whf_vcvt2_Vub(Vu) \
5457	4853	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_hf_ub)(Vu)
5458		#endif /* __HEXAGON_ARCH___ >= 79 */
	4854	#endif
5459	4855
5460		#if __HVX_ARCH__ >= 79
	4856	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5461	4857	/* ==========================================================================
5462	4858	Assembly Syntax: Vd32.ub=vcvt2(Vu32.hf,Vv32.hf)
5463	4859	C Intrinsic Prototype: HVX_Vector Q6_Vub_vcvt2_VhfVhf(HVX_Vector Vu,
...	...	@@ -5466,9 +4862,9 @@
5466	4862
5467	4863	#define Q6_Vub_vcvt2_VhfVhf(Vu, Vv) \
5468	4864	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt2_ub_hf)(Vu, Vv)
5469		#endif /* __HEXAGON_ARCH___ >= 79 */
	4865	#endif
5470	4866
5471		#if __HVX_ARCH__ >= 79
	4867	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5472	4868	/* ==========================================================================
5473	4869	Assembly Syntax: Vd32.f8=vcvt(Vu32.hf,Vv32.hf)
5474	4870	C Intrinsic Prototype: HVX_Vector Q6_V_vcvt_VhfVhf(HVX_Vector Vu, HVX_Vector
...	...	@@ -5477,9 +4873,9 @@
5477	4873
5478	4874	#define Q6_V_vcvt_VhfVhf(Vu, Vv) \
5479	4875	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_f8_hf)(Vu, Vv)
5480		#endif /* __HEXAGON_ARCH___ >= 79 */
	4876	#endif
5481	4877
5482		#if __HVX_ARCH__ >= 79
	4878	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5483	4879	/* ==========================================================================
5484	4880	Assembly Syntax: Vdd32.hf=vcvt(Vu32.f8)
5485	4881	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vcvt_V(HVX_Vector Vu)
...	...	@@ -5489,9 +4885,9 @@
5489	4885
5490	4886	#define Q6_Whf_vcvt_V(Vu) \
5491	4887	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vcvt_hf_f8)(Vu)
5492		#endif /* __HEXAGON_ARCH___ >= 79 */
	4888	#endif
5493	4889
5494		#if __HVX_ARCH__ >= 79
	4890	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5495	4891	/* ==========================================================================
5496	4892	Assembly Syntax: Vd32.f8=vfmax(Vu32.f8,Vv32.f8)
5497	4893	C Intrinsic Prototype: HVX_Vector Q6_V_vfmax_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5501,9 +4897,9 @@
5501	4897
5502	4898	#define Q6_V_vfmax_VV(Vu, Vv) \
5503	4899	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmax_f8)(Vu, Vv)
5504		#endif /* __HEXAGON_ARCH___ >= 79 */
	4900	#endif
5505	4901
5506		#if __HVX_ARCH__ >= 79
	4902	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5507	4903	/* ==========================================================================
5508	4904	Assembly Syntax: Vd32.f8=vfmin(Vu32.f8,Vv32.f8)
5509	4905	C Intrinsic Prototype: HVX_Vector Q6_V_vfmin_VV(HVX_Vector Vu, HVX_Vector Vv)
...	...	@@ -5513,9 +4909,9 @@
5513	4909
5514	4910	#define Q6_V_vfmin_VV(Vu, Vv) \
5515	4911	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfmin_f8)(Vu, Vv)
5516		#endif /* __HEXAGON_ARCH___ >= 79 */
	4912	#endif
5517	4913
5518		#if __HVX_ARCH__ >= 79
	4914	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5519	4915	/* ==========================================================================
5520	4916	Assembly Syntax: Vd32.f8=vfneg(Vu32.f8)
5521	4917	C Intrinsic Prototype: HVX_Vector Q6_V_vfneg_V(HVX_Vector Vu)
...	...	@@ -5525,7 +4921,7 @@
5525	4921
5526	4922	#define Q6_V_vfneg_V(Vu) \
5527	4923	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vfneg_f8)(Vu)
5528		#endif /* __HEXAGON_ARCH___ >= 79 */
	4924	#endif
5529	4925
5530	4926	#if __HVX_ARCH__ >= 79
5531	4927	/* ==========================================================================
...	...	@@ -5536,9 +4932,9 @@
5536	4932
5537	4933	#define Q6_V_vmerge_VVw(Vu, Vv) \
5538	4934	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmerge_qf)(Vu, Vv)
5539		#endif /* __HEXAGON_ARCH___ >= 79 */
	4935	#endif
5540	4936
5541		#if __HVX_ARCH__ >= 79
	4937	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5542	4938	/* ==========================================================================
5543	4939	Assembly Syntax: Vdd32.hf=vmpy(Vu32.f8,Vv32.f8)
5544	4940	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpy_VV(HVX_Vector Vu,
...	...	@@ -5547,9 +4943,9 @@
5547	4943
5548	4944	#define Q6_Whf_vmpy_VV(Vu, Vv) \
5549	4945	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8)(Vu, Vv)
5550		#endif /* __HEXAGON_ARCH___ >= 79 */
	4946	#endif
5551	4947
5552		#if __HVX_ARCH__ >= 79
	4948	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5553	4949	/* ==========================================================================
5554	4950	Assembly Syntax: Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8)
5555	4951	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vmpyacc_WhfVV(HVX_VectorPair
...	...	@@ -5559,7 +4955,7 @@
5559	4955
5560	4956	#define Q6_Whf_vmpyacc_WhfVV(Vxx, Vu, Vv) \
5561	4957	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_hf_f8_acc)(Vxx, Vu, Vv)
5562		#endif /* __HEXAGON_ARCH___ >= 79 */
	4958	#endif
5563	4959
5564	4960	#if __HVX_ARCH__ >= 79
5565	4961	/* ==========================================================================
...	...	@@ -5570,7 +4966,7 @@
5570	4966
5571	4967	#define Q6_Vqf16_vmpy_VhfRhf(Vu, Rt) \
5572	4968	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_hf)(Vu, Rt)
5573		#endif /* __HEXAGON_ARCH___ >= 79 */
	4969	#endif
5574	4970
5575	4971	#if __HVX_ARCH__ >= 79
5576	4972	/* ==========================================================================
...	...	@@ -5581,7 +4977,7 @@
5581	4977
5582	4978	#define Q6_Vqf16_vmpy_Vqf16Rhf(Vu, Rt) \
5583	4979	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_qf16)(Vu, Rt)
5584		#endif /* __HEXAGON_ARCH___ >= 79 */
	4980	#endif
5585	4981
5586	4982	#if __HVX_ARCH__ >= 79
5587	4983	/* ==========================================================================
...	...	@@ -5592,9 +4988,9 @@
5592	4988
5593	4989	#define Q6_Vqf32_vmpy_VsfRsf(Vu, Rt) \
5594	4990	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vmpy_rt_sf)(Vu, Rt)
5595		#endif /* __HEXAGON_ARCH___ >= 79 */
	4991	#endif
5596	4992
5597		#if __HVX_ARCH__ >= 79
	4993	#if __HVX_ARCH__ >= 79 && defined __HVX_IEEE_FP__
5598	4994	/* ==========================================================================
5599	4995	Assembly Syntax: Vdd32.hf=vsub(Vu32.f8,Vv32.f8)
5600	4996	C Intrinsic Prototype: HVX_VectorPair Q6_Whf_vsub_VV(HVX_Vector Vu,
...	...	@@ -5603,7 +4999,400 @@
5603	4999
5604	5000	#define Q6_Whf_vsub_VV(Vu, Vv) \
5605	5001	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_f8)(Vu, Vv)
5606		#endif /* __HEXAGON_ARCH___ >= 79 */
	5002	#endif
	5003
	5004	#if __HVX_ARCH__ >= 81
	5005	/* ==========================================================================
	5006	Assembly Syntax: Vd32.qf16=vabs(Vu32.hf)
	5007	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vabs_Vhf(HVX_Vector Vu)
	5008	Instruction Type: CVI_VS
	5009	Execution Slots: SLOT0123
	5010	========================================================================== */
	5011
	5012	#define Q6_Vqf16_vabs_Vhf(Vu) \
	5013	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_qf16_hf)(Vu)
	5014	#endif
	5015
	5016	#if __HVX_ARCH__ >= 81
	5017	/* ==========================================================================
	5018	Assembly Syntax: Vd32.qf16=vabs(Vu32.qf16)
	5019	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vabs_Vqf16(HVX_Vector Vu)
	5020	Instruction Type: CVI_VS
	5021	Execution Slots: SLOT0123
	5022	========================================================================== */
	5023
	5024	#define Q6_Vqf16_vabs_Vqf16(Vu) \
	5025	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_qf16_qf16)(Vu)
	5026	#endif
	5027
	5028	#if __HVX_ARCH__ >= 81
	5029	/* ==========================================================================
	5030	Assembly Syntax: Vd32.qf32=vabs(Vu32.qf32)
	5031	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vabs_Vqf32(HVX_Vector Vu)
	5032	Instruction Type: CVI_VS
	5033	Execution Slots: SLOT0123
	5034	========================================================================== */
	5035
	5036	#define Q6_Vqf32_vabs_Vqf32(Vu) \
	5037	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_qf32_qf32)(Vu)
	5038	#endif
	5039
	5040	#if __HVX_ARCH__ >= 81
	5041	/* ==========================================================================
	5042	Assembly Syntax: Vd32.qf32=vabs(Vu32.sf)
	5043	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vabs_Vsf(HVX_Vector Vu)
	5044	Instruction Type: CVI_VS
	5045	Execution Slots: SLOT0123
	5046	========================================================================== */
	5047
	5048	#define Q6_Vqf32_vabs_Vsf(Vu) \
	5049	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vabs_qf32_sf)(Vu)
	5050	#endif
	5051
	5052	#if __HVX_ARCH__ >= 81
	5053	/* ==========================================================================
	5054	Assembly Syntax: Vd32=valign4(Vu32,Vv32,Rt8)
	5055	C Intrinsic Prototype: HVX_Vector Q6_V_valign4_VVR(HVX_Vector Vu, HVX_Vector
	5056	Vv, Word32 Rt) Instruction Type: CVI_VA Execution Slots: SLOT0123
	5057	========================================================================== */
	5058
	5059	#define Q6_V_valign4_VVR(Vu, Vv, Rt) \
	5060	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_valign4)(Vu, Vv, Rt)
	5061	#endif
	5062
	5063	#if __HVX_ARCH__ >= 81
	5064	/* ==========================================================================
	5065	Assembly Syntax: Vd32.bf=Vuu32.qf32
	5066	C Intrinsic Prototype: HVX_Vector Q6_Vbf_equals_Wqf32(HVX_VectorPair Vuu)
	5067	Instruction Type: CVI_VS
	5068	Execution Slots: SLOT0123
	5069	========================================================================== */
	5070
	5071	#define Q6_Vbf_equals_Wqf32(Vuu) \
	5072	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_bf_qf32)(Vuu)
	5073	#endif
	5074
	5075	#if __HVX_ARCH__ >= 81
	5076	/* ==========================================================================
	5077	Assembly Syntax: Vd32.f8=Vu32.qf16
	5078	C Intrinsic Prototype: HVX_Vector Q6_V_equals_Vqf16(HVX_Vector Vu)
	5079	Instruction Type: CVI_VS
	5080	Execution Slots: SLOT0123
	5081	========================================================================== */
	5082
	5083	#define Q6_V_equals_Vqf16(Vu) \
	5084	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_f8_qf16)(Vu)
	5085	#endif
	5086
	5087	#if __HVX_ARCH__ >= 81
	5088	/* ==========================================================================
	5089	Assembly Syntax: Vd32.h=Vu32.hf:rnd
	5090	C Intrinsic Prototype: HVX_Vector Q6_Vh_equals_Vhf_rnd(HVX_Vector Vu)
	5091	Instruction Type: CVI_VS
	5092	Execution Slots: SLOT0123
	5093	========================================================================== */
	5094
	5095	#define Q6_Vh_equals_Vhf_rnd(Vu) \
	5096	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_h_hf_rnd)(Vu)
	5097	#endif
	5098
	5099	#if __HVX_ARCH__ >= 81
	5100	/* ==========================================================================
	5101	Assembly Syntax: Vdd32.qf16=Vu32.f8
	5102	C Intrinsic Prototype: HVX_VectorPair Q6_Wqf16_equals_V(HVX_Vector Vu)
	5103	Instruction Type: CVI_VP_VS
	5104	Execution Slots: SLOT0123
	5105	========================================================================== */
	5106
	5107	#define Q6_Wqf16_equals_V(Vu) \
	5108	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_qf16_f8)(Vu)
	5109	#endif
	5110
	5111	#if __HVX_ARCH__ >= 81
	5112	/* ==========================================================================
	5113	Assembly Syntax: Vd32.qf16=Vu32.hf
	5114	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_equals_Vhf(HVX_Vector Vu)
	5115	Instruction Type: CVI_VS
	5116	Execution Slots: SLOT0123
	5117	========================================================================== */
	5118
	5119	#define Q6_Vqf16_equals_Vhf(Vu) \
	5120	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_qf16_hf)(Vu)
	5121	#endif
	5122
	5123	#if __HVX_ARCH__ >= 81
	5124	/* ==========================================================================
	5125	Assembly Syntax: Vd32.qf16=Vu32.qf16
	5126	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_equals_Vqf16(HVX_Vector Vu)
	5127	Instruction Type: CVI_VS
	5128	Execution Slots: SLOT0123
	5129	========================================================================== */
	5130
	5131	#define Q6_Vqf16_equals_Vqf16(Vu) \
	5132	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_qf16_qf16)(Vu)
	5133	#endif
	5134
	5135	#if __HVX_ARCH__ >= 81
	5136	/* ==========================================================================
	5137	Assembly Syntax: Vd32.qf32=Vu32.qf32
	5138	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_equals_Vqf32(HVX_Vector Vu)
	5139	Instruction Type: CVI_VS
	5140	Execution Slots: SLOT0123
	5141	========================================================================== */
	5142
	5143	#define Q6_Vqf32_equals_Vqf32(Vu) \
	5144	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_qf32_qf32)(Vu)
	5145	#endif
	5146
	5147	#if __HVX_ARCH__ >= 81
	5148	/* ==========================================================================
	5149	Assembly Syntax: Vd32.qf32=Vu32.sf
	5150	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_equals_Vsf(HVX_Vector Vu)
	5151	Instruction Type: CVI_VS
	5152	Execution Slots: SLOT0123
	5153	========================================================================== */
	5154
	5155	#define Q6_Vqf32_equals_Vsf(Vu) \
	5156	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vconv_qf32_sf)(Vu)
	5157	#endif
	5158
	5159	#if __HVX_ARCH__ >= 81
	5160	/* ==========================================================================
	5161	Assembly Syntax: Qd4=vcmp.eq(Vu32.hf,Vv32.hf)
	5162	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VhfVhf(HVX_Vector Vu,
	5163	HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
	5164	========================================================================== */
	5165
	5166	#define Q6_Q_vcmp_eq_VhfVhf(Vu, Vv) \
	5167	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5168	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqhf)(Vu, Vv)), -1)
	5169	#endif
	5170
	5171	#if __HVX_ARCH__ >= 81
	5172	/* ==========================================================================
	5173	Assembly Syntax: Qx4&=vcmp.eq(Vu32.hf,Vv32.hf)
	5174	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVhfVhf(HVX_VectorPred
	5175	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5176	Slots: SLOT0123
	5177	========================================================================== */
	5178
	5179	#define Q6_Q_vcmp_eqand_QVhfVhf(Qx, Vu, Vv) \
	5180	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5181	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqhf_and)( \
	5182	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5183	Vv)), \
	5184	-1)
	5185	#endif
	5186
	5187	#if __HVX_ARCH__ >= 81
	5188	/* ==========================================================================
	5189	Assembly Syntax: Qx4\|=vcmp.eq(Vu32.hf,Vv32.hf)
	5190	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVhfVhf(HVX_VectorPred
	5191	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5192	Slots: SLOT0123
	5193	========================================================================== */
	5194
	5195	#define Q6_Q_vcmp_eqor_QVhfVhf(Qx, Vu, Vv) \
	5196	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5197	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqhf_or)( \
	5198	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5199	Vv)), \
	5200	-1)
	5201	#endif
	5202
	5203	#if __HVX_ARCH__ >= 81
	5204	/* ==========================================================================
	5205	Assembly Syntax: Qx4^=vcmp.eq(Vu32.hf,Vv32.hf)
	5206	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVhfVhf(HVX_VectorPred
	5207	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5208	Slots: SLOT0123
	5209	========================================================================== */
	5210
	5211	#define Q6_Q_vcmp_eqxacc_QVhfVhf(Qx, Vu, Vv) \
	5212	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5213	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqhf_xor)( \
	5214	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5215	Vv)), \
	5216	-1)
	5217	#endif
	5218
	5219	#if __HVX_ARCH__ >= 81
	5220	/* ==========================================================================
	5221	Assembly Syntax: Qd4=vcmp.eq(Vu32.sf,Vv32.sf)
	5222	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eq_VsfVsf(HVX_Vector Vu,
	5223	HVX_Vector Vv) Instruction Type: CVI_VA Execution Slots: SLOT0123
	5224	========================================================================== */
	5225
	5226	#define Q6_Q_vcmp_eq_VsfVsf(Vu, Vv) \
	5227	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5228	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqsf)(Vu, Vv)), -1)
	5229	#endif
	5230
	5231	#if __HVX_ARCH__ >= 81
	5232	/* ==========================================================================
	5233	Assembly Syntax: Qx4&=vcmp.eq(Vu32.sf,Vv32.sf)
	5234	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqand_QVsfVsf(HVX_VectorPred
	5235	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5236	Slots: SLOT0123
	5237	========================================================================== */
	5238
	5239	#define Q6_Q_vcmp_eqand_QVsfVsf(Qx, Vu, Vv) \
	5240	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5241	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqsf_and)( \
	5242	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5243	Vv)), \
	5244	-1)
	5245	#endif
	5246
	5247	#if __HVX_ARCH__ >= 81
	5248	/* ==========================================================================
	5249	Assembly Syntax: Qx4\|=vcmp.eq(Vu32.sf,Vv32.sf)
	5250	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqor_QVsfVsf(HVX_VectorPred
	5251	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5252	Slots: SLOT0123
	5253	========================================================================== */
	5254
	5255	#define Q6_Q_vcmp_eqor_QVsfVsf(Qx, Vu, Vv) \
	5256	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5257	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqsf_or)( \
	5258	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5259	Vv)), \
	5260	-1)
	5261	#endif
	5262
	5263	#if __HVX_ARCH__ >= 81
	5264	/* ==========================================================================
	5265	Assembly Syntax: Qx4^=vcmp.eq(Vu32.sf,Vv32.sf)
	5266	C Intrinsic Prototype: HVX_VectorPred Q6_Q_vcmp_eqxacc_QVsfVsf(HVX_VectorPred
	5267	Qx, HVX_Vector Vu, HVX_Vector Vv) Instruction Type: CVI_VA Execution
	5268	Slots: SLOT0123
	5269	========================================================================== */
	5270
	5271	#define Q6_Q_vcmp_eqxacc_QVsfVsf(Qx, Vu, Vv) \
	5272	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandqrt)( \
	5273	(__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_veqsf_xor)( \
	5274	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vandvrt)((Qx), -1), Vu, \
	5275	Vv)), \
	5276	-1)
	5277	#endif
	5278
	5279	#if __HVX_ARCH__ >= 81
	5280	/* ==========================================================================
	5281	Assembly Syntax: Vd32.w=vilog2(Vu32.hf)
	5282	C Intrinsic Prototype: HVX_Vector Q6_Vw_vilog2_Vhf(HVX_Vector Vu)
	5283	Instruction Type: CVI_VS
	5284	Execution Slots: SLOT0123
	5285	========================================================================== */
	5286
	5287	#define Q6_Vw_vilog2_Vhf(Vu) \
	5288	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vilog2_hf)(Vu)
	5289	#endif
	5290
	5291	#if __HVX_ARCH__ >= 81
	5292	/* ==========================================================================
	5293	Assembly Syntax: Vd32.w=vilog2(Vu32.qf16)
	5294	C Intrinsic Prototype: HVX_Vector Q6_Vw_vilog2_Vqf16(HVX_Vector Vu)
	5295	Instruction Type: CVI_VS
	5296	Execution Slots: SLOT0123
	5297	========================================================================== */
	5298
	5299	#define Q6_Vw_vilog2_Vqf16(Vu) \
	5300	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vilog2_qf16)(Vu)
	5301	#endif
	5302
	5303	#if __HVX_ARCH__ >= 81
	5304	/* ==========================================================================
	5305	Assembly Syntax: Vd32.w=vilog2(Vu32.qf32)
	5306	C Intrinsic Prototype: HVX_Vector Q6_Vw_vilog2_Vqf32(HVX_Vector Vu)
	5307	Instruction Type: CVI_VS
	5308	Execution Slots: SLOT0123
	5309	========================================================================== */
	5310
	5311	#define Q6_Vw_vilog2_Vqf32(Vu) \
	5312	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vilog2_qf32)(Vu)
	5313	#endif
	5314
	5315	#if __HVX_ARCH__ >= 81
	5316	/* ==========================================================================
	5317	Assembly Syntax: Vd32.w=vilog2(Vu32.sf)
	5318	C Intrinsic Prototype: HVX_Vector Q6_Vw_vilog2_Vsf(HVX_Vector Vu)
	5319	Instruction Type: CVI_VS
	5320	Execution Slots: SLOT0123
	5321	========================================================================== */
	5322
	5323	#define Q6_Vw_vilog2_Vsf(Vu) \
	5324	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vilog2_sf)(Vu)
	5325	#endif
	5326
	5327	#if __HVX_ARCH__ >= 81
	5328	/* ==========================================================================
	5329	Assembly Syntax: Vd32.qf16=vneg(Vu32.hf)
	5330	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vneg_Vhf(HVX_Vector Vu)
	5331	Instruction Type: CVI_VS
	5332	Execution Slots: SLOT0123
	5333	========================================================================== */
	5334
	5335	#define Q6_Vqf16_vneg_Vhf(Vu) \
	5336	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vneg_qf16_hf)(Vu)
	5337	#endif
	5338
	5339	#if __HVX_ARCH__ >= 81
	5340	/* ==========================================================================
	5341	Assembly Syntax: Vd32.qf16=vneg(Vu32.qf16)
	5342	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vneg_Vqf16(HVX_Vector Vu)
	5343	Instruction Type: CVI_VS
	5344	Execution Slots: SLOT0123
	5345	========================================================================== */
	5346
	5347	#define Q6_Vqf16_vneg_Vqf16(Vu) \
	5348	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vneg_qf16_qf16)(Vu)
	5349	#endif
	5350
	5351	#if __HVX_ARCH__ >= 81
	5352	/* ==========================================================================
	5353	Assembly Syntax: Vd32.qf32=vneg(Vu32.qf32)
	5354	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vneg_Vqf32(HVX_Vector Vu)
	5355	Instruction Type: CVI_VS
	5356	Execution Slots: SLOT0123
	5357	========================================================================== */
	5358
	5359	#define Q6_Vqf32_vneg_Vqf32(Vu) \
	5360	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vneg_qf32_qf32)(Vu)
	5361	#endif
	5362
	5363	#if __HVX_ARCH__ >= 81
	5364	/* ==========================================================================
	5365	Assembly Syntax: Vd32.qf32=vneg(Vu32.sf)
	5366	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vneg_Vsf(HVX_Vector Vu)
	5367	Instruction Type: CVI_VS
	5368	Execution Slots: SLOT0123
	5369	========================================================================== */
	5370
	5371	#define Q6_Vqf32_vneg_Vsf(Vu) \
	5372	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vneg_qf32_sf)(Vu)
	5373	#endif
	5374
	5375	#if __HVX_ARCH__ >= 81
	5376	/* ==========================================================================
	5377	Assembly Syntax: Vd32.qf16=vsub(Vu32.hf,Vv32.qf16)
	5378	C Intrinsic Prototype: HVX_Vector Q6_Vqf16_vsub_VhfVqf16(HVX_Vector Vu,
	5379	HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
	5380	========================================================================== */
	5381
	5382	#define Q6_Vqf16_vsub_VhfVqf16(Vu, Vv) \
	5383	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_hf_mix)(Vu, Vv)
	5384	#endif
	5385
	5386	#if __HVX_ARCH__ >= 81
	5387	/* ==========================================================================
	5388	Assembly Syntax: Vd32.qf32=vsub(Vu32.sf,Vv32.qf32)
	5389	C Intrinsic Prototype: HVX_Vector Q6_Vqf32_vsub_VsfVqf32(HVX_Vector Vu,
	5390	HVX_Vector Vv) Instruction Type: CVI_VS Execution Slots: SLOT0123
	5391	========================================================================== */
	5392
	5393	#define Q6_Vqf32_vsub_VsfVqf32(Vu, Vv) \
	5394	__BUILTIN_VECTOR_WRAP(__builtin_HEXAGON_V6_vsub_sf_mix)(Vu, Vv)
	5395	#endif
5607	5396
5608	5397	#endif /* __HVX__ */
5609	5398

lib/include/immintrin.h-12

...	...	@@ -475,24 +475,12 @@ _storebe_i64(void * __P, long long __D) {
475	475
476	476	#include <amxfp8intrin.h>
477	477
478		#include <amxtransposeintrin.h>
479
480	478	#include <amxmovrsintrin.h>
481	479
482		#include <amxmovrstransposeintrin.h>
483
484	480	#include <amxavx512intrin.h>
485	481
486	482	#include <amxtf32intrin.h>
487	483
488		#include <amxtf32transposeintrin.h>
489
490		#include <amxbf16transposeintrin.h>
491
492		#include <amxfp16transposeintrin.h>
493
494		#include <amxcomplextransposeintrin.h>
495
496	484	#include <avx512vp2intersectintrin.h>
497	485
498	486	#include <avx512vlvp2intersectintrin.h>

lib/include/intrin.h+4

...	...	@@ -30,6 +30,10 @@
30	30	#include <arm64intr.h>
31	31	#endif
32	32
	33	#if defined(__ARM_ACLE)
	34	#include <arm_acle.h>
	35	#endif
	36
33	37	/* For the definition of jmp_buf. */
34	38	#if __STDC_HOSTED__
35	39	#include <setjmp.h>

lib/include/lasxintrin.h+113

...	...	@@ -10,6 +10,8 @@
10	10	#ifndef _LOONGSON_ASXINTRIN_H
11	11	#define _LOONGSON_ASXINTRIN_H 1
12	12
	13	#include <lsxintrin.h>
	14
13	15	#if defined(__loongarch_asx)
14	16
15	17	typedef signed char v32i8 __attribute__((vector_size(32), aligned(32)));
...	...	@@ -3882,5 +3884,116 @@ extern __inline
3882	3884
3883	3885	#define __lasx_xvrepli_w(/si10/ _1) ((__m256i)__builtin_lasx_xvrepli_w((_1)))
3884	3886
	3887	#if defined(__loongarch_asx_sx_conv)
	3888
	3889	extern __inline
	3890	__attribute__((__gnu_inline__, __always_inline__,
	3891	__artificial__)) __m256 __lasx_cast_128_s(__m128 _1) {
	3892	return (__m256)__builtin_lasx_cast_128_s((v4f32)_1);
	3893	}
	3894
	3895	extern __inline
	3896	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
	3897	__lasx_cast_128_d(__m128d _1) {
	3898	return (__m256d)__builtin_lasx_cast_128_d((v2f64)_1);
	3899	}
	3900
	3901	extern __inline
	3902	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
	3903	__lasx_cast_128(__m128i _1) {
	3904	return (__m256i)__builtin_lasx_cast_128((v2i64)_1);
	3905	}
	3906
	3907	extern __inline
	3908	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
	3909	__lasx_concat_128_s(__m128 _1, __m128 _2) {
	3910	return (__m256)__builtin_lasx_concat_128_s((v4f32)_1, (v4f32)_2);
	3911	}
	3912
	3913	extern __inline
	3914	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
	3915	__lasx_concat_128_d(__m128d _1, __m128d _2) {
	3916	return (__m256d)__builtin_lasx_concat_128_d((v2f64)_1, (v2f64)_2);
	3917	}
	3918
	3919	extern __inline
	3920	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
	3921	__lasx_concat_128(__m128i _1, __m128i _2) {
	3922	return (__m256i)__builtin_lasx_concat_128((v2i64)_1, (v2i64)_2);
	3923	}
	3924
	3925	extern __inline
	3926	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
	3927	__lasx_extract_128_lo_s(__m256 _1) {
	3928	return (__m128)__builtin_lasx_extract_128_lo_s((v8f32)_1);
	3929	}
	3930
	3931	extern __inline
	3932	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
	3933	__lasx_extract_128_lo_d(__m256d _1) {
	3934	return (__m128d)__builtin_lasx_extract_128_lo_d((v4f64)_1);
	3935	}
	3936
	3937	extern __inline
	3938	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128i
	3939	__lasx_extract_128_lo(__m256i _1) {
	3940	return (__m128i)__builtin_lasx_extract_128_lo((v4i64)_1);
	3941	}
	3942
	3943	extern __inline
	3944	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128
	3945	__lasx_extract_128_hi_s(__m256 _1) {
	3946	return (__m128)__builtin_lasx_extract_128_hi_s((v8f32)_1);
	3947	}
	3948
	3949	extern __inline
	3950	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128d
	3951	__lasx_extract_128_hi_d(__m256d _1) {
	3952	return (__m128d)__builtin_lasx_extract_128_hi_d((v4f64)_1);
	3953	}
	3954
	3955	extern __inline
	3956	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m128i
	3957	__lasx_extract_128_hi(__m256i _1) {
	3958	return (__m128i)__builtin_lasx_extract_128_hi((v4i64)_1);
	3959	}
	3960
	3961	extern __inline
	3962	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
	3963	__lasx_insert_128_lo_s(__m256 _1, __m128 _2) {
	3964	return (__m256)__builtin_lasx_insert_128_lo_s((v8f32)_1, (v4f32)_2);
	3965	}
	3966
	3967	extern __inline
	3968	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
	3969	__lasx_insert_128_lo_d(__m256d _1, __m128d _2) {
	3970	return (__m256d)__builtin_lasx_insert_128_lo_d((v4f64)_1, (v2f64)_2);
	3971	}
	3972
	3973	extern __inline
	3974	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
	3975	__lasx_insert_128_lo(__m256i _1, __m128i _2) {
	3976	return (__m256i)__builtin_lasx_insert_128_lo((v4i64)_1, (v2i64)_2);
	3977	}
	3978
	3979	extern __inline
	3980	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256
	3981	__lasx_insert_128_hi_s(__m256 _1, __m128 _2) {
	3982	return (__m256)__builtin_lasx_insert_128_hi_s((v8f32)_1, (v4f32)_2);
	3983	}
	3984
	3985	extern __inline
	3986	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256d
	3987	__lasx_insert_128_hi_d(__m256d _1, __m128d _2) {
	3988	return (__m256d)__builtin_lasx_insert_128_hi_d((v4f64)_1, (v2f64)_2);
	3989	}
	3990
	3991	extern __inline
	3992	__attribute__((__gnu_inline__, __always_inline__, __artificial__)) __m256i
	3993	__lasx_insert_128_hi(__m256i _1, __m128i _2) {
	3994	return (__m256i)__builtin_lasx_insert_128_hi((v4i64)_1, (v2i64)_2);
	3995	}
	3996
	3997	#endif /* defined(__loongarch_asx_sx_conv). */
3885	3998	#endif /* defined(__loongarch_asx). */
3886	3999	#endif /* _LOONGSON_ASXINTRIN_H. */

lib/include/llvm_libc_wrappers/assert.h+3-5

...	...	@@ -19,13 +19,11 @@
19	19
20	20	#if defined(__HIP__) \|\| defined(__CUDA__)
21	21	#define __LIBC_ATTRS __attribute__((device))
	22	#else
	23	#define __LIBC_ATTRS
22	24	#endif
23	25
24		#pragma omp begin declare target
25
26		#include <llvm-libc-decls/assert.h>
27
28		#pragma omp end declare target
	26	// TODO: Define these for CUDA / HIP.
29	27
30	28	#undef __LIBC_ATTRS
31	29

lib/include/llvm_libc_wrappers/ctype.h+3-115

...	...	@@ -13,128 +13,16 @@
13	13	#error "This file is for GPU offloading compilation only"
14	14	#endif
15	15
16		// The GNU headers like to define 'toupper' and 'tolower' redundantly. This is
17		// necessary to prevent it from doing that and remapping our implementation.
18		#if (defined(__NVPTX__) \|\| defined(__AMDGPU__)) && defined(__GLIBC__)
19		#pragma push_macro("__USE_EXTERN_INLINES")
20		#undef __USE_EXTERN_INLINES
21		#endif
22
23	16	#include_next <ctype.h>
24	17
25		#if (defined(__NVPTX__) \|\| defined(__AMDGPU__)) && defined(__GLIBC__)
26		#pragma pop_macro("__USE_EXTERN_INLINES")
27		#endif
28
29		#if __has_include(<llvm-libc-decls/ctype.h>)
30
31	18	#if defined(__HIP__) \|\| defined(__CUDA__)
32	19	#define __LIBC_ATTRS __attribute__((device))
	20	#else
	21	#define __LIBC_ATTRS
33	22	#endif
34	23
35		// The GNU headers like to provide these as macros, we need to undefine them so
36		// they do not conflict with the following definitions for the GPU.
37
38		#pragma push_macro("isalnum")
39		#pragma push_macro("isalpha")
40		#pragma push_macro("isascii")
41		#pragma push_macro("isblank")
42		#pragma push_macro("iscntrl")
43		#pragma push_macro("isdigit")
44		#pragma push_macro("isgraph")
45		#pragma push_macro("islower")
46		#pragma push_macro("isprint")
47		#pragma push_macro("ispunct")
48		#pragma push_macro("isspace")
49		#pragma push_macro("isupper")
50		#pragma push_macro("isxdigit")
51		#pragma push_macro("toascii")
52		#pragma push_macro("tolower")
53		#pragma push_macro("toupper")
54		#pragma push_macro("isalnum_l")
55		#pragma push_macro("isalpha_l")
56		#pragma push_macro("isascii_l")
57		#pragma push_macro("isblank_l")
58		#pragma push_macro("iscntrl_l")
59		#pragma push_macro("isdigit_l")
60		#pragma push_macro("isgraph_l")
61		#pragma push_macro("islower_l")
62		#pragma push_macro("isprint_l")
63		#pragma push_macro("ispunct_l")
64		#pragma push_macro("isspace_l")
65		#pragma push_macro("isupper_l")
66		#pragma push_macro("isxdigit_l")
67
68		#undef isalnum
69		#undef isalpha
70		#undef isascii
71		#undef iscntrl
72		#undef isdigit
73		#undef islower
74		#undef isgraph
75		#undef isprint
76		#undef ispunct
77		#undef isspace
78		#undef isupper
79		#undef isblank
80		#undef isxdigit
81		#undef toascii
82		#undef tolower
83		#undef toupper
84		#undef isalnum_l
85		#undef isalpha_l
86		#undef iscntrl_l
87		#undef isdigit_l
88		#undef islower_l
89		#undef isgraph_l
90		#undef isprint_l
91		#undef ispunct_l
92		#undef isspace_l
93		#undef isupper_l
94		#undef isblank_l
95		#undef isxdigit_l
96
97		#pragma omp begin declare target
98
99		#include <llvm-libc-decls/ctype.h>
100
101		#pragma omp end declare target
102
103		// Restore the original macros when compiling on the host.
104		#if !defined(__NVPTX__) && !defined(__AMDGPU__)
105		#pragma pop_macro("isalnum")
106		#pragma pop_macro("isalpha")
107		#pragma pop_macro("isascii")
108		#pragma pop_macro("isblank")
109		#pragma pop_macro("iscntrl")
110		#pragma pop_macro("isdigit")
111		#pragma pop_macro("isgraph")
112		#pragma pop_macro("islower")
113		#pragma pop_macro("isprint")
114		#pragma pop_macro("ispunct")
115		#pragma pop_macro("isspace")
116		#pragma pop_macro("isupper")
117		#pragma pop_macro("isxdigit")
118		#pragma pop_macro("toascii")
119		#pragma pop_macro("tolower")
120		#pragma pop_macro("toupper")
121		#pragma pop_macro("isalnum_l")
122		#pragma pop_macro("isalpha_l")
123		#pragma pop_macro("isascii_l")
124		#pragma pop_macro("isblank_l")
125		#pragma pop_macro("iscntrl_l")
126		#pragma pop_macro("isdigit_l")
127		#pragma pop_macro("isgraph_l")
128		#pragma pop_macro("islower_l")
129		#pragma pop_macro("isprint_l")
130		#pragma pop_macro("ispunct_l")
131		#pragma pop_macro("isspace_l")
132		#pragma pop_macro("isupper_l")
133		#pragma pop_macro("isxdigit_l")
134		#endif
	24	// TODO: Define these for CUDA / HIP.
135	25
136	26	#undef __LIBC_ATTRS
137	27
138		#endif
139
140	28	#endif // __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__

lib/include/llvm_libc_wrappers/inttypes.h+3-5

...	...	@@ -19,13 +19,11 @@
19	19
20	20	#if defined(__HIP__) \|\| defined(__CUDA__)
21	21	#define __LIBC_ATTRS __attribute__((device))
	22	#else
	23	#define __LIBC_ATTRS
22	24	#endif
23	25
24		#pragma omp begin declare target
25
26		#include <llvm-libc-decls/inttypes.h>
27
28		#pragma omp end declare target
	26	// TODO: Define these for CUDA / HIP.
29	27
30	28	#undef __LIBC_ATTRS
31	29

lib/include/llvm_libc_wrappers/stdio.h+10-38

...	...	@@ -6,45 +6,19 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8
	9	#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
	10	#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
	11
9	12	#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
10	13	#error "This file is for GPU offloading compilation only"
11	14	#endif
12	15
13	16	#include_next <stdio.h>
14	17
15		// In some old versions of glibc, other standard headers sometimes define
16		// special macros (e.g., __need_FILE) before including stdio.h to cause stdio.h
17		// to produce special definitions. Future includes of stdio.h when those
18		// special macros are undefined are expected to produce the normal definitions
19		// from stdio.h.
20		//
21		// We do not apply our include guard (__CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__)
22		// unconditionally to the above include_next. Otherwise, after an occurrence of
23		// the first glibc stdio.h use case described above, the include_next would be
24		// skipped for remaining includes of stdio.h, leaving required symbols
25		// undefined.
26		//
27		// We make the following assumptions to handle all use cases:
28		//
29		// 1. If the above include_next produces special glibc definitions, then (a) it
30		// does not produce the normal definitions that we must intercept below, (b)
31		// the current file was included from a glibc header that already defined
32		// __GLIBC__ (usually by including glibc's <features.h>), and (c) the above
33		// include_next does not define _STDIO_H. In that case, we skip the rest of
34		// the current file and don't guard against future includes.
35		// 2. If the above include_next produces the normal stdio.h definitions, then
36		// either (a) __GLIBC__ is not defined because C headers are from some other
37		// libc implementation or (b) the above include_next defines _STDIO_H to
38		// prevent the above include_next from having any effect in the future.
39		#if !defined(__GLIBC__) \|\| defined(_STDIO_H)
40
41		#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
42		#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
43
44		#if __has_include(<llvm-libc-decls/stdio.h>)
45
46	18	#if defined(__HIP__) \|\| defined(__CUDA__)
47	19	#define __LIBC_ATTRS __attribute__((device))
	20	#else
	21	#define __LIBC_ATTRS
48	22	#endif
49	23
50	24	// Some headers provide these as macros. Temporarily undefine them so they do
...	...	@@ -60,21 +34,19 @@
60	34
61	35	#pragma omp begin declare target
62	36
63		#include <llvm-libc-decls/stdio.h>
	37	__LIBC_ATTRS extern FILE *stderr;
	38	__LIBC_ATTRS extern FILE *stdin;
	39	__LIBC_ATTRS extern FILE *stdout;
64	40
65	41	#pragma omp end declare target
66	42
67		#undef __LIBC_ATTRS
68
69	43	// Restore the original macros when compiling on the host.
70	44	#if !defined(__NVPTX__) && !defined(__AMDGPU__)
71		#pragma pop_macro("stdout")
72	45	#pragma pop_macro("stderr")
73	46	#pragma pop_macro("stdin")
	47	#pragma pop_macro("stdout")
74	48	#endif
75	49
76		#endif
	50	#undef __LIBC_ATTRS
77	51
78	52	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
79
80		#endif

lib/include/llvm_libc_wrappers/stdlib.h+3-24

...	...	@@ -15,39 +15,18 @@
15	15
16	16	#include_next <stdlib.h>
17	17
18		#if __has_include(<llvm-libc-decls/stdlib.h>)
19
20	18	#if defined(__HIP__) \|\| defined(__CUDA__)
21	19	#define __LIBC_ATTRS __attribute__((device))
	20	#else
	21	#define __LIBC_ATTRS
22	22	#endif
23	23
24	24	#pragma omp begin declare target
25	25
26		// The LLVM C library uses these named types so we forward declare them.
27		typedef void (*__atexithandler_t)(void);
28		typedef int (__search_compare_t)(const void , const void *);
29		typedef int (__qsortcompare_t)(const void , const void *);
30		typedef int (__qsortrcompare_t)(const void , const void , void );
31
32		// Enforce ABI compatibility with the structs used by the LLVM C library.
33		_Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");
34		_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");
35		_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");
36
37		#if defined(__GLIBC__) && __cplusplus >= 201703L
38		#define at_quick_exit atexit
39		#endif
40
41		#include <llvm-libc-decls/stdlib.h>
42
43		#if defined(__GLIBC__) && __cplusplus >= 201703L
44		#undef at_quick_exit
45		#endif
	26	// TODO: Define these for CUDA / HIP.
46	27
47	28	#pragma omp end declare target
48	29
49	30	#undef __LIBC_ATTRS
50	31
51		#endif
52
53	32	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__

lib/include/llvm_libc_wrappers/string.h+2-70

...	...	@@ -15,82 +15,14 @@
15	15
16	16	#include_next <string.h>
17	17
18		#if __has_include(<llvm-libc-decls/string.h>)
19
20	18	#if defined(__HIP__) \|\| defined(__CUDA__)
21	19	#define __LIBC_ATTRS __attribute__((device))
22		#endif
23
24		#pragma omp begin declare target
25
26		// The GNU headers provide C++ standard compliant headers when in C++ mode and
27		// the LLVM libc does not. We need to manually provide the definitions using the
28		// same prototypes.
29		#if defined(__cplusplus) && defined(__GLIBC__) && \
30		defined(__CORRECT_ISO_CPP_STRING_H_PROTO)
31
32		#ifndef __LIBC_ATTRS
33		#define __LIBC_ATTRS
34		#endif
35
36		extern "C" {
37		void memccpy(void __restrict, const void *__restrict, int,
38		size_t) __LIBC_ATTRS;
39		int memcmp(const void , const void , size_t) __LIBC_ATTRS;
40		void memcpy(void __restrict, const void *__restrict, size_t) __LIBC_ATTRS;
41		void memmem(const void , size_t, const void *, size_t) __LIBC_ATTRS;
42		void memmove(void , const void *, size_t) __LIBC_ATTRS;
43		void mempcpy(void __restrict, const void *__restrict, size_t) __LIBC_ATTRS;
44		void memset(void , int, size_t) __LIBC_ATTRS;
45		char stpcpy(char __restrict, const char *__restrict) __LIBC_ATTRS;
46		char stpncpy(char __restrict, const char *__restrict, size_t) __LIBC_ATTRS;
47		char strcat(char __restrict, const char *__restrict) __LIBC_ATTRS;
48		int strcmp(const char , const char ) __LIBC_ATTRS;
49		int strcoll(const char , const char ) __LIBC_ATTRS;
50		char strcpy(char __restrict, const char *__restrict) __LIBC_ATTRS;
51		size_t strcspn(const char , const char ) __LIBC_ATTRS;
52		char strdup(const char ) __LIBC_ATTRS;
53		size_t strlen(const char *) __LIBC_ATTRS;
54		char strncat(char __restrict, const char *__restrict, size_t) __LIBC_ATTRS;
55		int strncmp(const char , const char , size_t) __LIBC_ATTRS;
56		char strncpy(char __restrict, const char *__restrict, size_t) __LIBC_ATTRS;
57		char strndup(const char , size_t) __LIBC_ATTRS;
58		size_t strnlen(const char *, size_t) __LIBC_ATTRS;
59		size_t strspn(const char , const char ) __LIBC_ATTRS;
60		char strtok(char __restrict, const char *__restrict) __LIBC_ATTRS;
61		char strtok_r(char __restrict, const char *__restrict,
62		char **__restrict) __LIBC_ATTRS;
63		size_t strxfrm(char __restrict, const char __restrict, size_t) __LIBC_ATTRS;
64		}
65
66		extern "C++" {
67		char strstr(char , const char *) noexcept __LIBC_ATTRS;
68		const char strstr(const char , const char *) noexcept __LIBC_ATTRS;
69		char strpbrk(char , const char *) noexcept __LIBC_ATTRS;
70		const char strpbrk(const char , const char *) noexcept __LIBC_ATTRS;
71		char strrchr(char , int) noexcept __LIBC_ATTRS;
72		const char strrchr(const char , int) noexcept __LIBC_ATTRS;
73		char strchr(char , int) noexcept __LIBC_ATTRS;
74		const char strchr(const char , int) noexcept __LIBC_ATTRS;
75		char strchrnul(char , int) noexcept __LIBC_ATTRS;
76		const char strchrnul(const char , int) noexcept __LIBC_ATTRS;
77		char strcasestr(char , const char *) noexcept __LIBC_ATTRS;
78		const char strcasestr(const char , const char *) noexcept __LIBC_ATTRS;
79		void memrchr(void __s, int __c, size_t __n) noexcept __LIBC_ATTRS;
80		const void memrchr(const void __s, int __c, size_t __n) noexcept __LIBC_ATTRS;
81		void memchr(void __s, int __c, size_t __n) noexcept __LIBC_ATTRS;
82		const void memchr(const void __s, int __c, size_t __n) noexcept __LIBC_ATTRS;
83		}
84
85	20	#else
86		#include <llvm-libc-decls/string.h>
87
	21	#define __LIBC_ATTRS
88	22	#endif
89	23
90		#pragma omp end declare target
	24	// TODO: Define these for CUDA / HIP.
91	25
92	26	#undef __LIBC_ATTRS
93	27
94		#endif
95
96	28	#endif // __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__

lib/include/llvm_libc_wrappers/time.h+4-10

...	...	@@ -15,20 +15,14 @@
15	15
16	16	#include_next <time.h>
17	17
18		#if __has_include(<llvm-libc-decls/time.h>)
19
20	18	#if defined(__HIP__) \|\| defined(__CUDA__)
21	19	#define __LIBC_ATTRS __attribute__((device))
	20	#else
	21	#define __LIBC_ATTRS
22	22	#endif
23	23
24		#pragma omp begin declare target
25
26		_Static_assert(sizeof(clock_t) == sizeof(long), "ABI mismatch!");
27
28		#include <llvm-libc-decls/time.h>
	24	// TODO: Define these for CUDA / HIP.
29	25
30		#pragma omp end declare target
31
32		#endif
	26	#undef __LIBC_ATTRS
33	27
34	28	#endif // __CLANG_LLVM_LIBC_WRAPPERS_TIME_H__

lib/include/mmintrin.h+188-257

...	...	@@ -39,27 +39,21 @@ typedef short __v8hi __attribute__((__vector_size__(16)));
39	39	typedef char __v16qi __attribute__((__vector_size__(16)));
40	40
41	41	/* Define the default attributes for the functions in this file. */
42		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
	42	#if defined(__cplusplus) && (__cplusplus >= 201103L)
43	43	#define __DEFAULT_FN_ATTRS_SSE2 \
44		__attribute__((__always_inline__, __nodebug__, \
45		__target__("sse2,no-evex512"), __min_vector_width__(128)))
	44	__attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
	45	__min_vector_width__(128))) constexpr
46	46	#else
47	47	#define __DEFAULT_FN_ATTRS_SSE2 \
48	48	__attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
49	49	__min_vector_width__(128)))
50	50	#endif
51	51
52		#if defined(__cplusplus) && (__cplusplus >= 201103L)
53		#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2 constexpr
54		#else
55		#define __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR __DEFAULT_FN_ATTRS_SSE2
56		#endif
57
58	52	#define __trunc64(x) \
59	53	(__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
60		#define __anyext128(x) \
	54	#define __zext128(x) \
61	55	(__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
62		1, -1, -1)
	56	1, 2, 3)
63	57
64	58	/// Clears the MMX state by setting the state of the x87 stack registers
65	59	/// to empty.
...	...	@@ -68,9 +62,9 @@ typedef char __v16qi __attribute__((__vector_size__(16)));
68	62	///
69	63	/// This intrinsic corresponds to the <c> EMMS </c> instruction.
70	64	///
71		static __inline__ void __attribute__((__always_inline__, __nodebug__,
72		__target__("mmx,no-evex512")))
73		_mm_empty(void) {
	65	static __inline__ void
	66	__attribute__((__always_inline__, __nodebug__, __target__("mmx")))
	67	_mm_empty(void) {
74	68	__builtin_ia32_emms();
75	69	}
76	70
...	...	@@ -85,10 +79,8 @@ _mm_empty(void) {
85	79	/// A 32-bit integer value.
86	80	/// \returns A 64-bit integer vector. The lower 32 bits contain the value of the
87	81	/// parameter. The upper 32 bits are set to 0.
88		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
89		_mm_cvtsi32_si64(int __i)
90		{
91		return __extension__ (__m64)(__v2si){__i, 0};
	82	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cvtsi32_si64(int __i) {
	83	return __extension__(__m64)(__v2si){__i, 0};
92	84	}
93	85
94	86	/// Returns the lower 32 bits of a 64-bit integer vector as a 32-bit
...	...	@@ -102,10 +94,8 @@ _mm_cvtsi32_si64(int __i)
102	94	/// A 64-bit integer vector.
103	95	/// \returns A 32-bit signed integer value containing the lower 32 bits of the
104	96	/// parameter.
105		static __inline__ int __DEFAULT_FN_ATTRS_SSE2
106		_mm_cvtsi64_si32(__m64 __m)
107		{
108		return ((__v2si)__m)[0];
	97	static __inline__ int __DEFAULT_FN_ATTRS_SSE2 _mm_cvtsi64_si32(__m64 __m) {
	98	return ((__v2si)__m)[0];
109	99	}
110	100
111	101	/// Casts a 64-bit signed integer value into a 64-bit integer vector.
...	...	@@ -118,10 +108,8 @@ _mm_cvtsi64_si32(__m64 __m)
118	108	/// A 64-bit signed integer.
119	109	/// \returns A 64-bit integer vector containing the same bitwise pattern as the
120	110	/// parameter.
121		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
122		_mm_cvtsi64_m64(long long __i)
123		{
124		return (__m64)__i;
	111	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cvtsi64_m64(long long __i) {
	112	return __extension__(__m64)(__v1di){__i};
125	113	}
126	114
127	115	/// Casts a 64-bit integer vector into a 64-bit signed integer value.
...	...	@@ -134,10 +122,8 @@ _mm_cvtsi64_m64(long long __i)
134	122	/// A 64-bit integer vector.
135	123	/// \returns A 64-bit signed integer containing the same bitwise pattern as the
136	124	/// parameter.
137		static __inline__ long long __DEFAULT_FN_ATTRS_SSE2
138		_mm_cvtm64_si64(__m64 __m)
139		{
140		return (long long)__m;
	125	static __inline__ long long __DEFAULT_FN_ATTRS_SSE2 _mm_cvtm64_si64(__m64 __m) {
	126	return ((__v1di)__m)[0];
141	127	}
142	128
143	129	/// Converts, with saturation, 16-bit signed integers from both 64-bit integer
...	...	@@ -159,11 +145,10 @@ _mm_cvtm64_si64(__m64 __m)
159	145	/// written to the upper 32 bits of the result.
160	146	/// \returns A 64-bit integer vector of [8 x i8] containing the converted
161	147	/// values.
162		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
163		_mm_packs_pi16(__m64 __m1, __m64 __m2)
164		{
165		return __trunc64(__builtin_ia32_packsswb128(
166		(__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
	148	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_packs_pi16(__m64 __m1,
	149	__m64 __m2) {
	150	return __trunc64(__builtin_ia32_packsswb128(
	151	(__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
167	152	}
168	153
169	154	/// Converts, with saturation, 32-bit signed integers from both 64-bit integer
...	...	@@ -185,11 +170,10 @@ _mm_packs_pi16(__m64 __m1, __m64 __m2)
185	170	/// written to the upper 32 bits of the result.
186	171	/// \returns A 64-bit integer vector of [4 x i16] containing the converted
187	172	/// values.
188		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
189		_mm_packs_pi32(__m64 __m1, __m64 __m2)
190		{
191		return __trunc64(__builtin_ia32_packssdw128(
192		(__v4si)__builtin_shufflevector(__m1, __m2, 0, 1), (__v4si){}));
	173	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_packs_pi32(__m64 __m1,
	174	__m64 __m2) {
	175	return __trunc64(__builtin_ia32_packssdw128(
	176	(__v4si)__builtin_shufflevector(__m1, __m2, 0, 1), (__v4si){}));
193	177	}
194	178
195	179	/// Converts, with saturation, 16-bit signed integers from both 64-bit integer
...	...	@@ -211,11 +195,10 @@ _mm_packs_pi32(__m64 __m1, __m64 __m2)
211	195	/// written to the upper 32 bits of the result.
212	196	/// \returns A 64-bit integer vector of [8 x i8] containing the converted
213	197	/// values.
214		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
215		_mm_packs_pu16(__m64 __m1, __m64 __m2)
216		{
217		return __trunc64(__builtin_ia32_packuswb128(
218		(__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
	198	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_packs_pu16(__m64 __m1,
	199	__m64 __m2) {
	200	return __trunc64(__builtin_ia32_packuswb128(
	201	(__v8hi)__builtin_shufflevector(__m1, __m2, 0, 1), (__v8hi){}));
219	202	}
220	203
221	204	/// Unpacks the upper 32 bits from two 64-bit integer vectors of [8 x i8]
...	...	@@ -239,11 +222,10 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)
239	222	/// Bits [63:56] are written to bits [63:56] of the result.
240	223	/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
241	224	/// values.
242		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
243		_mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
244		{
245		return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
246		4, 12, 5, 13, 6, 14, 7, 15);
	225	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpackhi_pi8(__m64 __m1,
	226	__m64 __m2) {
	227	return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 4, 12, 5,
	228	13, 6, 14, 7, 15);
247	229	}
248	230
249	231	/// Unpacks the upper 32 bits from two 64-bit integer vectors of
...	...	@@ -263,11 +245,9 @@ _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
263	245	/// Bits [63:48] are written to bits [63:48] of the result.
264	246	/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
265	247	/// values.
266		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
267		_mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
268		{
269		return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
270		2, 6, 3, 7);
	248	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpackhi_pi16(__m64 __m1,
	249	__m64 __m2) {
	250	return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 2, 6, 3, 7);
271	251	}
272	252
273	253	/// Unpacks the upper 32 bits from two 64-bit integer vectors of
...	...	@@ -285,10 +265,9 @@ _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
285	265	/// the upper 32 bits of the result.
286	266	/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
287	267	/// values.
288		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
289		_mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
290		{
291		return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
	268	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpackhi_pi32(__m64 __m1,
	269	__m64 __m2) {
	270	return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
292	271	}
293	272
294	273	/// Unpacks the lower 32 bits from two 64-bit integer vectors of [8 x i8]
...	...	@@ -312,11 +291,10 @@ _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
312	291	/// Bits [31:24] are written to bits [63:56] of the result.
313	292	/// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
314	293	/// values.
315		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
316		_mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
317		{
318		return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
319		0, 8, 1, 9, 2, 10, 3, 11);
	294	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpacklo_pi8(__m64 __m1,
	295	__m64 __m2) {
	296	return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 0, 8, 1, 9,
	297	2, 10, 3, 11);
320	298	}
321	299
322	300	/// Unpacks the lower 32 bits from two 64-bit integer vectors of
...	...	@@ -336,11 +314,9 @@ _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
336	314	/// Bits [31:16] are written to bits [63:48] of the result.
337	315	/// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
338	316	/// values.
339		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
340		_mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
341		{
342		return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
343		0, 4, 1, 5);
	317	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpacklo_pi16(__m64 __m1,
	318	__m64 __m2) {
	319	return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 0, 4, 1, 5);
344	320	}
345	321
346	322	/// Unpacks the lower 32 bits from two 64-bit integer vectors of
...	...	@@ -358,10 +334,9 @@ _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
358	334	/// the upper 32 bits of the result.
359	335	/// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
360	336	/// values.
361		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
362		_mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
363		{
364		return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
	337	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_unpacklo_pi32(__m64 __m1,
	338	__m64 __m2) {
	339	return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
365	340	}
366	341
367	342	/// Adds each 8-bit integer element of the first 64-bit integer vector
...	...	@@ -379,10 +354,9 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
379	354	/// A 64-bit integer vector of [8 x i8].
380	355	/// \returns A 64-bit integer vector of [8 x i8] containing the sums of both
381	356	/// parameters.
382		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
383		_mm_add_pi8(__m64 __m1, __m64 __m2)
384		{
385		return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2));
	357	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_add_pi8(__m64 __m1,
	358	__m64 __m2) {
	359	return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2));
386	360	}
387	361
388	362	/// Adds each 16-bit integer element of the first 64-bit integer vector
...	...	@@ -400,10 +374,9 @@ _mm_add_pi8(__m64 __m1, __m64 __m2)
400	374	/// A 64-bit integer vector of [4 x i16].
401	375	/// \returns A 64-bit integer vector of [4 x i16] containing the sums of both
402	376	/// parameters.
403		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
404		_mm_add_pi16(__m64 __m1, __m64 __m2)
405		{
406		return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2));
	377	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_add_pi16(__m64 __m1,
	378	__m64 __m2) {
	379	return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2));
407	380	}
408	381
409	382	/// Adds each 32-bit integer element of the first 64-bit integer vector
...	...	@@ -421,10 +394,9 @@ _mm_add_pi16(__m64 __m1, __m64 __m2)
421	394	/// A 64-bit integer vector of [2 x i32].
422	395	/// \returns A 64-bit integer vector of [2 x i32] containing the sums of both
423	396	/// parameters.
424		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
425		_mm_add_pi32(__m64 __m1, __m64 __m2)
426		{
427		return (__m64)(((__v2su)__m1) + ((__v2su)__m2));
	397	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_add_pi32(__m64 __m1,
	398	__m64 __m2) {
	399	return (__m64)(((__v2su)__m1) + ((__v2su)__m2));
428	400	}
429	401
430	402	/// Adds, with saturation, each 8-bit signed integer element of the first
...	...	@@ -445,10 +417,9 @@ _mm_add_pi32(__m64 __m1, __m64 __m2)
445	417	/// A 64-bit integer vector of [8 x i8].
446	418	/// \returns A 64-bit integer vector of [8 x i8] containing the saturated sums
447	419	/// of both parameters.
448		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
449		_mm_adds_pi8(__m64 __m1, __m64 __m2)
450		{
451		return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
	420	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_adds_pi8(__m64 __m1,
	421	__m64 __m2) {
	422	return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
452	423	}
453	424
454	425	/// Adds, with saturation, each 16-bit signed integer element of the first
...	...	@@ -469,10 +440,9 @@ _mm_adds_pi8(__m64 __m1, __m64 __m2)
469	440	/// A 64-bit integer vector of [4 x i16].
470	441	/// \returns A 64-bit integer vector of [4 x i16] containing the saturated sums
471	442	/// of both parameters.
472		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
473		_mm_adds_pi16(__m64 __m1, __m64 __m2)
474		{
475		return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
	443	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_adds_pi16(__m64 __m1,
	444	__m64 __m2) {
	445	return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
476	446	}
477	447
478	448	/// Adds, with saturation, each 8-bit unsigned integer element of the first
...	...	@@ -492,10 +462,9 @@ _mm_adds_pi16(__m64 __m1, __m64 __m2)
492	462	/// A 64-bit integer vector of [8 x i8].
493	463	/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
494	464	/// unsigned sums of both parameters.
495		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
496		_mm_adds_pu8(__m64 __m1, __m64 __m2)
497		{
498		return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
	465	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_adds_pu8(__m64 __m1,
	466	__m64 __m2) {
	467	return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
499	468	}
500	469
501	470	/// Adds, with saturation, each 16-bit unsigned integer element of the first
...	...	@@ -515,10 +484,9 @@ _mm_adds_pu8(__m64 __m1, __m64 __m2)
515	484	/// A 64-bit integer vector of [4 x i16].
516	485	/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
517	486	/// unsigned sums of both parameters.
518		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
519		_mm_adds_pu16(__m64 __m1, __m64 __m2)
520		{
521		return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
	487	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_adds_pu16(__m64 __m1,
	488	__m64 __m2) {
	489	return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
522	490	}
523	491
524	492	/// Subtracts each 8-bit integer element of the second 64-bit integer
...	...	@@ -536,10 +504,9 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)
536	504	/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
537	505	/// \returns A 64-bit integer vector of [8 x i8] containing the differences of
538	506	/// both parameters.
539		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
540		_mm_sub_pi8(__m64 __m1, __m64 __m2)
541		{
542		return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2));
	507	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_sub_pi8(__m64 __m1,
	508	__m64 __m2) {
	509	return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2));
543	510	}
544	511
545	512	/// Subtracts each 16-bit integer element of the second 64-bit integer
...	...	@@ -557,10 +524,9 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2)
557	524	/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
558	525	/// \returns A 64-bit integer vector of [4 x i16] containing the differences of
559	526	/// both parameters.
560		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
561		_mm_sub_pi16(__m64 __m1, __m64 __m2)
562		{
563		return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2));
	527	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_sub_pi16(__m64 __m1,
	528	__m64 __m2) {
	529	return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2));
564	530	}
565	531
566	532	/// Subtracts each 32-bit integer element of the second 64-bit integer
...	...	@@ -578,10 +544,9 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2)
578	544	/// A 64-bit integer vector of [2 x i32] containing the subtrahends.
579	545	/// \returns A 64-bit integer vector of [2 x i32] containing the differences of
580	546	/// both parameters.
581		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
582		_mm_sub_pi32(__m64 __m1, __m64 __m2)
583		{
584		return (__m64)(((__v2su)__m1) - ((__v2su)__m2));
	547	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_sub_pi32(__m64 __m1,
	548	__m64 __m2) {
	549	return (__m64)(((__v2su)__m1) - ((__v2su)__m2));
585	550	}
586	551
587	552	/// Subtracts, with saturation, each 8-bit signed integer element of the second
...	...	@@ -602,10 +567,9 @@ _mm_sub_pi32(__m64 __m1, __m64 __m2)
602	567	/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
603	568	/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
604	569	/// differences of both parameters.
605		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
606		_mm_subs_pi8(__m64 __m1, __m64 __m2)
607		{
608		return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
	570	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_subs_pi8(__m64 __m1,
	571	__m64 __m2) {
	572	return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
609	573	}
610	574
611	575	/// Subtracts, with saturation, each 16-bit signed integer element of the
...	...	@@ -626,10 +590,9 @@ _mm_subs_pi8(__m64 __m1, __m64 __m2)
626	590	/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
627	591	/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
628	592	/// differences of both parameters.
629		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
630		_mm_subs_pi16(__m64 __m1, __m64 __m2)
631		{
632		return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
	593	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_subs_pi16(__m64 __m1,
	594	__m64 __m2) {
	595	return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
633	596	}
634	597
635	598	/// Subtracts each 8-bit unsigned integer element of the second 64-bit
...	...	@@ -650,10 +613,9 @@ _mm_subs_pi16(__m64 __m1, __m64 __m2)
650	613	/// A 64-bit integer vector of [8 x i8] containing the subtrahends.
651	614	/// \returns A 64-bit integer vector of [8 x i8] containing the saturated
652	615	/// differences of both parameters.
653		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
654		_mm_subs_pu8(__m64 __m1, __m64 __m2)
655		{
656		return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
	616	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_subs_pu8(__m64 __m1,
	617	__m64 __m2) {
	618	return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
657	619	}
658	620
659	621	/// Subtracts each 16-bit unsigned integer element of the second 64-bit
...	...	@@ -674,10 +636,9 @@ _mm_subs_pu8(__m64 __m1, __m64 __m2)
674	636	/// A 64-bit integer vector of [4 x i16] containing the subtrahends.
675	637	/// \returns A 64-bit integer vector of [4 x i16] containing the saturated
676	638	/// differences of both parameters.
677		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
678		_mm_subs_pu16(__m64 __m1, __m64 __m2)
679		{
680		return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
	639	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_subs_pu16(__m64 __m1,
	640	__m64 __m2) {
	641	return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
681	642	}
682	643
683	644	/// Multiplies each 16-bit signed integer element of the first 64-bit
...	...	@@ -701,11 +662,10 @@ _mm_subs_pu16(__m64 __m1, __m64 __m2)
701	662	/// A 64-bit integer vector of [4 x i16].
702	663	/// \returns A 64-bit integer vector of [2 x i32] containing the sums of
703	664	/// products of both parameters.
704		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
705		_mm_madd_pi16(__m64 __m1, __m64 __m2)
706		{
707		return __trunc64(__builtin_ia32_pmaddwd128((__v8hi)__anyext128(__m1),
708		(__v8hi)__anyext128(__m2)));
	665	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_madd_pi16(__m64 __m1,
	666	__m64 __m2) {
	667	return __trunc64(__builtin_ia32_pmaddwd128((__v8hi)__zext128(__m1),
	668	(__v8hi)__zext128(__m2)));
709	669	}
710	670
711	671	/// Multiplies each 16-bit signed integer element of the first 64-bit
...	...	@@ -723,11 +683,10 @@ _mm_madd_pi16(__m64 __m1, __m64 __m2)
723	683	/// A 64-bit integer vector of [4 x i16].
724	684	/// \returns A 64-bit integer vector of [4 x i16] containing the upper 16 bits
725	685	/// of the products of both parameters.
726		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
727		_mm_mulhi_pi16(__m64 __m1, __m64 __m2)
728		{
729		return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__anyext128(__m1),
730		(__v8hi)__anyext128(__m2)));
	686	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_mulhi_pi16(__m64 __m1,
	687	__m64 __m2) {
	688	return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__zext128(__m1),
	689	(__v8hi)__zext128(__m2)));
731	690	}
732	691
733	692	/// Multiplies each 16-bit signed integer element of the first 64-bit
...	...	@@ -745,10 +704,9 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
745	704	/// A 64-bit integer vector of [4 x i16].
746	705	/// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits
747	706	/// of the products of both parameters.
748		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
749		_mm_mullo_pi16(__m64 __m1, __m64 __m2)
750		{
751		return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2));
	707	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_mullo_pi16(__m64 __m1,
	708	__m64 __m2) {
	709	return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2));
752	710	}
753	711
754	712	/// Left-shifts each 16-bit signed integer element of the first
...	...	@@ -771,8 +729,8 @@ _mm_mullo_pi16(__m64 __m1, __m64 __m2)
771	729	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
772	730	_mm_sll_pi16(__m64 __m, __m64 __count)
773	731	{
774		return __trunc64(__builtin_ia32_psllw128((__v8hi)__anyext128(__m),
775		(__v8hi)__anyext128(__count)));
	732	return __trunc64(__builtin_ia32_psllw128((__v8hi)__zext128(__m),
	733	(__v8hi)__zext128(__count)));
776	734	}
777	735
778	736	/// Left-shifts each 16-bit signed integer element of a 64-bit integer
...	...	@@ -791,11 +749,9 @@ _mm_sll_pi16(__m64 __m, __m64 __count)
791	749	/// \returns A 64-bit integer vector of [4 x i16] containing the left-shifted
792	750	/// values. If \a __count is greater or equal to 16, the result is set to all
793	751	/// 0.
794		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
795		_mm_slli_pi16(__m64 __m, int __count)
796		{
797		return __trunc64(__builtin_ia32_psllwi128((__v8hi)__anyext128(__m),
798		__count));
	752	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_slli_pi16(__m64 __m,
	753	int __count) {
	754	return __trunc64(__builtin_ia32_psllwi128((__v8hi)__zext128(__m), __count));
799	755	}
800	756
801	757	/// Left-shifts each 32-bit signed integer element of the first
...	...	@@ -818,8 +774,8 @@ _mm_slli_pi16(__m64 __m, int __count)
818	774	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
819	775	_mm_sll_pi32(__m64 __m, __m64 __count)
820	776	{
821		return __trunc64(__builtin_ia32_pslld128((__v4si)__anyext128(__m),
822		(__v4si)__anyext128(__count)));
	777	return __trunc64(__builtin_ia32_pslld128((__v4si)__zext128(__m),
	778	(__v4si)__zext128(__count)));
823	779	}
824	780
825	781	/// Left-shifts each 32-bit signed integer element of a 64-bit integer
...	...	@@ -838,11 +794,9 @@ _mm_sll_pi32(__m64 __m, __m64 __count)
838	794	/// \returns A 64-bit integer vector of [2 x i32] containing the left-shifted
839	795	/// values. If \a __count is greater or equal to 32, the result is set to all
840	796	/// 0.
841		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
842		_mm_slli_pi32(__m64 __m, int __count)
843		{
844		return __trunc64(__builtin_ia32_pslldi128((__v4si)__anyext128(__m),
845		__count));
	797	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_slli_pi32(__m64 __m,
	798	int __count) {
	799	return __trunc64(__builtin_ia32_pslldi128((__v4si)__zext128(__m), __count));
846	800	}
847	801
848	802	/// Left-shifts the first 64-bit integer parameter by the number of bits
...	...	@@ -862,8 +816,8 @@ _mm_slli_pi32(__m64 __m, int __count)
862	816	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
863	817	_mm_sll_si64(__m64 __m, __m64 __count)
864	818	{
865		return __trunc64(__builtin_ia32_psllq128((__v2di)__anyext128(__m),
866		(__v2di)__anyext128(__count)));
	819	return __trunc64(__builtin_ia32_psllq128((__v2di)__zext128(__m),
	820	(__v2di)__zext128(__count)));
867	821	}
868	822
869	823	/// Left-shifts the first parameter, which is a 64-bit integer, by the
...	...	@@ -880,11 +834,9 @@ _mm_sll_si64(__m64 __m, __m64 __count)
880	834	/// A 32-bit integer value.
881	835	/// \returns A 64-bit integer vector containing the left-shifted value. If
882	836	/// \a __count is greater or equal to 64, the result is set to 0.
883		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
884		_mm_slli_si64(__m64 __m, int __count)
885		{
886		return __trunc64(__builtin_ia32_psllqi128((__v2di)__anyext128(__m),
887		__count));
	837	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_slli_si64(__m64 __m,
	838	int __count) {
	839	return __trunc64(__builtin_ia32_psllqi128((__v2di)__zext128(__m), __count));
888	840	}
889	841
890	842	/// Right-shifts each 16-bit integer element of the first parameter,
...	...	@@ -908,8 +860,8 @@ _mm_slli_si64(__m64 __m, int __count)
908	860	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
909	861	_mm_sra_pi16(__m64 __m, __m64 __count)
910	862	{
911		return __trunc64(__builtin_ia32_psraw128((__v8hi)__anyext128(__m),
912		(__v8hi)__anyext128(__count)));
	863	return __trunc64(__builtin_ia32_psraw128((__v8hi)__zext128(__m),
	864	(__v8hi)__zext128(__count)));
913	865	}
914	866
915	867	/// Right-shifts each 16-bit integer element of a 64-bit integer vector
...	...	@@ -929,11 +881,9 @@ _mm_sra_pi16(__m64 __m, __m64 __count)
929	881	/// A 32-bit integer value.
930	882	/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
931	883	/// values.
932		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
933		_mm_srai_pi16(__m64 __m, int __count)
934		{
935		return __trunc64(__builtin_ia32_psrawi128((__v8hi)__anyext128(__m),
936		__count));
	884	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_srai_pi16(__m64 __m,
	885	int __count) {
	886	return __trunc64(__builtin_ia32_psrawi128((__v8hi)__zext128(__m), __count));
937	887	}
938	888
939	889	/// Right-shifts each 32-bit integer element of the first parameter,
...	...	@@ -957,8 +907,8 @@ _mm_srai_pi16(__m64 __m, int __count)
957	907	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
958	908	_mm_sra_pi32(__m64 __m, __m64 __count)
959	909	{
960		return __trunc64(__builtin_ia32_psrad128((__v4si)__anyext128(__m),
961		(__v4si)__anyext128(__count)));
	910	return __trunc64(__builtin_ia32_psrad128((__v4si)__zext128(__m),
	911	(__v4si)__zext128(__count)));
962	912	}
963	913
964	914	/// Right-shifts each 32-bit integer element of a 64-bit integer vector
...	...	@@ -978,11 +928,9 @@ _mm_sra_pi32(__m64 __m, __m64 __count)
978	928	/// A 32-bit integer value.
979	929	/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
980	930	/// values.
981		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
982		_mm_srai_pi32(__m64 __m, int __count)
983		{
984		return __trunc64(__builtin_ia32_psradi128((__v4si)__anyext128(__m),
985		__count));
	931	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_srai_pi32(__m64 __m,
	932	int __count) {
	933	return __trunc64(__builtin_ia32_psradi128((__v4si)__zext128(__m), __count));
986	934	}
987	935
988	936	/// Right-shifts each 16-bit integer element of the first parameter,
...	...	@@ -1005,8 +953,8 @@ _mm_srai_pi32(__m64 __m, int __count)
1005	953	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1006	954	_mm_srl_pi16(__m64 __m, __m64 __count)
1007	955	{
1008		return __trunc64(__builtin_ia32_psrlw128((__v8hi)__anyext128(__m),
1009		(__v8hi)__anyext128(__count)));
	956	return __trunc64(__builtin_ia32_psrlw128((__v8hi)__zext128(__m),
	957	(__v8hi)__zext128(__count)));
1010	958	}
1011	959
1012	960	/// Right-shifts each 16-bit integer element of a 64-bit integer vector
...	...	@@ -1025,11 +973,9 @@ _mm_srl_pi16(__m64 __m, __m64 __count)
1025	973	/// A 32-bit integer value.
1026	974	/// \returns A 64-bit integer vector of [4 x i16] containing the right-shifted
1027	975	/// values.
1028		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1029		_mm_srli_pi16(__m64 __m, int __count)
1030		{
1031		return __trunc64(__builtin_ia32_psrlwi128((__v8hi)__anyext128(__m),
1032		__count));
	976	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_srli_pi16(__m64 __m,
	977	int __count) {
	978	return __trunc64(__builtin_ia32_psrlwi128((__v8hi)__zext128(__m), __count));
1033	979	}
1034	980
1035	981	/// Right-shifts each 32-bit integer element of the first parameter,
...	...	@@ -1052,8 +998,8 @@ _mm_srli_pi16(__m64 __m, int __count)
1052	998	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1053	999	_mm_srl_pi32(__m64 __m, __m64 __count)
1054	1000	{
1055		return __trunc64(__builtin_ia32_psrld128((__v4si)__anyext128(__m),
1056		(__v4si)__anyext128(__count)));
	1001	return __trunc64(__builtin_ia32_psrld128((__v4si)__zext128(__m),
	1002	(__v4si)__zext128(__count)));
1057	1003	}
1058	1004
1059	1005	/// Right-shifts each 32-bit integer element of a 64-bit integer vector
...	...	@@ -1072,11 +1018,9 @@ _mm_srl_pi32(__m64 __m, __m64 __count)
1072	1018	/// A 32-bit integer value.
1073	1019	/// \returns A 64-bit integer vector of [2 x i32] containing the right-shifted
1074	1020	/// values.
1075		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1076		_mm_srli_pi32(__m64 __m, int __count)
1077		{
1078		return __trunc64(__builtin_ia32_psrldi128((__v4si)__anyext128(__m),
1079		__count));
	1021	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_srli_pi32(__m64 __m,
	1022	int __count) {
	1023	return __trunc64(__builtin_ia32_psrldi128((__v4si)__zext128(__m), __count));
1080	1024	}
1081	1025
1082	1026	/// Right-shifts the first 64-bit integer parameter by the number of bits
...	...	@@ -1096,8 +1040,8 @@ _mm_srli_pi32(__m64 __m, int __count)
1096	1040	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1097	1041	_mm_srl_si64(__m64 __m, __m64 __count)
1098	1042	{
1099		return __trunc64(__builtin_ia32_psrlq128((__v2di)__anyext128(__m),
1100		(__v2di)__anyext128(__count)));
	1043	return __trunc64(__builtin_ia32_psrlq128((__v2di)__zext128(__m),
	1044	(__v2di)__zext128(__count)));
1101	1045	}
1102	1046
1103	1047	/// Right-shifts the first parameter, which is a 64-bit integer, by the
...	...	@@ -1115,11 +1059,9 @@ _mm_srl_si64(__m64 __m, __m64 __count)
1115	1059	/// \param __count
1116	1060	/// A 32-bit integer value.
1117	1061	/// \returns A 64-bit integer vector containing the right-shifted value.
1118		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1119		_mm_srli_si64(__m64 __m, int __count)
1120		{
1121		return __trunc64(__builtin_ia32_psrlqi128((__v2di)__anyext128(__m),
1122		__count));
	1062	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_srli_si64(__m64 __m,
	1063	int __count) {
	1064	return __trunc64(__builtin_ia32_psrlqi128((__v2di)__zext128(__m), __count));
1123	1065	}
1124	1066
1125	1067	/// Performs a bitwise AND of two 64-bit integer vectors.
...	...	@@ -1134,10 +1076,9 @@ _mm_srli_si64(__m64 __m, int __count)
1134	1076	/// A 64-bit integer vector.
1135	1077	/// \returns A 64-bit integer vector containing the bitwise AND of both
1136	1078	/// parameters.
1137		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1138		_mm_and_si64(__m64 __m1, __m64 __m2)
1139		{
1140		return (__m64)(((__v1du)__m1) & ((__v1du)__m2));
	1079	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_and_si64(__m64 __m1,
	1080	__m64 __m2) {
	1081	return (__m64)(((__v1du)__m1) & ((__v1du)__m2));
1141	1082	}
1142	1083
1143	1084	/// Performs a bitwise NOT of the first 64-bit integer vector, and then
...	...	@@ -1155,10 +1096,9 @@ _mm_and_si64(__m64 __m1, __m64 __m2)
1155	1096	/// A 64-bit integer vector.
1156	1097	/// \returns A 64-bit integer vector containing the bitwise AND of the second
1157	1098	/// parameter and the one's complement of the first parameter.
1158		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1159		_mm_andnot_si64(__m64 __m1, __m64 __m2)
1160		{
1161		return (__m64)(~((__v1du)__m1) & ((__v1du)__m2));
	1099	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_andnot_si64(__m64 __m1,
	1100	__m64 __m2) {
	1101	return (__m64)(~((__v1du)__m1) & ((__v1du)__m2));
1162	1102	}
1163	1103
1164	1104	/// Performs a bitwise OR of two 64-bit integer vectors.
...	...	@@ -1173,10 +1113,9 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2)
1173	1113	/// A 64-bit integer vector.
1174	1114	/// \returns A 64-bit integer vector containing the bitwise OR of both
1175	1115	/// parameters.
1176		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1177		_mm_or_si64(__m64 __m1, __m64 __m2)
1178		{
1179		return (__m64)(((__v1du)__m1) \| ((__v1du)__m2));
	1116	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_or_si64(__m64 __m1,
	1117	__m64 __m2) {
	1118	return (__m64)(((__v1du)__m1) \| ((__v1du)__m2));
1180	1119	}
1181	1120
1182	1121	/// Performs a bitwise exclusive OR of two 64-bit integer vectors.
...	...	@@ -1191,10 +1130,9 @@ _mm_or_si64(__m64 __m1, __m64 __m2)
1191	1130	/// A 64-bit integer vector.
1192	1131	/// \returns A 64-bit integer vector containing the bitwise exclusive OR of both
1193	1132	/// parameters.
1194		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1195		_mm_xor_si64(__m64 __m1, __m64 __m2)
1196		{
1197		return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2));
	1133	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_xor_si64(__m64 __m1,
	1134	__m64 __m2) {
	1135	return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2));
1198	1136	}
1199	1137
1200	1138	/// Compares the 8-bit integer elements of two 64-bit integer vectors of
...	...	@@ -1213,10 +1151,9 @@ _mm_xor_si64(__m64 __m1, __m64 __m2)
1213	1151	/// A 64-bit integer vector of [8 x i8].
1214	1152	/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
1215	1153	/// results.
1216		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1217		_mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
1218		{
1219		return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2));
	1154	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpeq_pi8(__m64 __m1,
	1155	__m64 __m2) {
	1156	return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2));
1220	1157	}
1221	1158
1222	1159	/// Compares the 16-bit integer elements of two 64-bit integer vectors of
...	...	@@ -1235,10 +1172,9 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
1235	1172	/// A 64-bit integer vector of [4 x i16].
1236	1173	/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
1237	1174	/// results.
1238		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1239		_mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
1240		{
1241		return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2));
	1175	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpeq_pi16(__m64 __m1,
	1176	__m64 __m2) {
	1177	return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2));
1242	1178	}
1243	1179
1244	1180	/// Compares the 32-bit integer elements of two 64-bit integer vectors of
...	...	@@ -1257,10 +1193,9 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
1257	1193	/// A 64-bit integer vector of [2 x i32].
1258	1194	/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
1259	1195	/// results.
1260		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1261		_mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
1262		{
1263		return (__m64)(((__v2si)__m1) == ((__v2si)__m2));
	1196	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpeq_pi32(__m64 __m1,
	1197	__m64 __m2) {
	1198	return (__m64)(((__v2si)__m1) == ((__v2si)__m2));
1264	1199	}
1265	1200
1266	1201	/// Compares the 8-bit integer elements of two 64-bit integer vectors of
...	...	@@ -1279,9 +1214,8 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
1279	1214	/// A 64-bit integer vector of [8 x i8].
1280	1215	/// \returns A 64-bit integer vector of [8 x i8] containing the comparison
1281	1216	/// results.
1282		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1283		_mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
1284		{
	1217	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpgt_pi8(__m64 __m1,
	1218	__m64 __m2) {
1285	1219	/* This function always performs a signed comparison, but __v8qi is a char
1286	1220	which may be signed or unsigned, so use __v8qs. */
1287	1221	return (__m64)((__v8qs)__m1 > (__v8qs)__m2);
...	...	@@ -1303,10 +1237,9 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
1303	1237	/// A 64-bit integer vector of [4 x i16].
1304	1238	/// \returns A 64-bit integer vector of [4 x i16] containing the comparison
1305	1239	/// results.
1306		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1307		_mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
1308		{
1309		return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
	1240	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpgt_pi16(__m64 __m1,
	1241	__m64 __m2) {
	1242	return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
1310	1243	}
1311	1244
1312	1245	/// Compares the 32-bit integer elements of two 64-bit integer vectors of
...	...	@@ -1325,10 +1258,9 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
1325	1258	/// A 64-bit integer vector of [2 x i32].
1326	1259	/// \returns A 64-bit integer vector of [2 x i32] containing the comparison
1327	1260	/// results.
1328		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1329		_mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
1330		{
1331		return (__m64)((__v2si)__m1 > (__v2si)__m2);
	1261	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_cmpgt_pi32(__m64 __m1,
	1262	__m64 __m2) {
	1263	return (__m64)((__v2si)__m1 > (__v2si)__m2);
1332	1264	}
1333	1265
1334	1266	/// Constructs a 64-bit integer vector initialized to zero.
...	...	@@ -1338,8 +1270,7 @@ _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
1338	1270	/// This intrinsic corresponds to the <c> PXOR </c> instruction.
1339	1271	///
1340	1272	/// \returns An initialized 64-bit integer vector with all elements set to zero.
1341		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1342		_mm_setzero_si64(void) {
	1273	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_setzero_si64(void) {
1343	1274	return __extension__(__m64){0LL};
1344	1275	}
1345	1276
...	...	@@ -1358,8 +1289,8 @@ _mm_setzero_si64(void) {
1358	1289	/// A 32-bit integer value used to initialize the lower 32 bits of the
1359	1290	/// result.
1360	1291	/// \returns An initialized 64-bit integer vector.
1361		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1362		_mm_set_pi32(int __i1, int __i0) {
	1292	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_set_pi32(int __i1,
	1293	int __i0) {
1363	1294	return __extension__(__m64)(__v2si){__i0, __i1};
1364	1295	}
1365	1296
...	...	@@ -1380,8 +1311,10 @@ _mm_set_pi32(int __i1, int __i0) {
1380	1311	/// \param __s0
1381	1312	/// A 16-bit integer value used to initialize bits [15:0] of the result.
1382	1313	/// \returns An initialized 64-bit integer vector.
1383		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1384		_mm_set_pi16(short __s3, short __s2, short __s1, short __s0) {
	1314	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_set_pi16(short __s3,
	1315	short __s2,
	1316	short __s1,
	1317	short __s0) {
1385	1318	return __extension__(__m64)(__v4hi){__s0, __s1, __s2, __s3};
1386	1319	}
1387	1320
...	...	@@ -1410,7 +1343,7 @@ _mm_set_pi16(short __s3, short __s2, short __s1, short __s0) {
1410	1343	/// \param __b0
1411	1344	/// An 8-bit integer value used to initialize bits [7:0] of the result.
1412	1345	/// \returns An initialized 64-bit integer vector.
1413		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	1346	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1414	1347	_mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
1415	1348	char __b1, char __b0) {
1416	1349	return __extension__(__m64)(__v8qi){__b0, __b1, __b2, __b3,
...	...	@@ -1430,8 +1363,7 @@ _mm_set_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
1430	1363	/// A 32-bit integer value used to initialize each vector element of the
1431	1364	/// result.
1432	1365	/// \returns An initialized 64-bit integer vector of [2 x i32].
1433		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1434		_mm_set1_pi32(int __i) {
	1366	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_set1_pi32(int __i) {
1435	1367	return _mm_set_pi32(__i, __i);
1436	1368	}
1437	1369
...	...	@@ -1448,8 +1380,7 @@ _mm_set1_pi32(int __i) {
1448	1380	/// A 16-bit integer value used to initialize each vector element of the
1449	1381	/// result.
1450	1382	/// \returns An initialized 64-bit integer vector of [4 x i16].
1451		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1452		_mm_set1_pi16(short __w) {
	1383	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_set1_pi16(short __w) {
1453	1384	return _mm_set_pi16(__w, __w, __w, __w);
1454	1385	}
1455	1386
...	...	@@ -1465,8 +1396,7 @@ _mm_set1_pi16(short __w) {
1465	1396	/// An 8-bit integer value used to initialize each vector element of the
1466	1397	/// result.
1467	1398	/// \returns An initialized 64-bit integer vector of [8 x i8].
1468		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1469		_mm_set1_pi8(char __b) {
	1399	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_set1_pi8(char __b) {
1470	1400	return _mm_set_pi8(__b, __b, __b, __b, __b, __b, __b, __b);
1471	1401	}
1472	1402
...	...	@@ -1485,8 +1415,8 @@ _mm_set1_pi8(char __b) {
1485	1415	/// A 32-bit integer value used to initialize the upper 32 bits of the
1486	1416	/// result.
1487	1417	/// \returns An initialized 64-bit integer vector.
1488		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1489		_mm_setr_pi32(int __i0, int __i1) {
	1418	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_setr_pi32(int __i0,
	1419	int __i1) {
1490	1420	return _mm_set_pi32(__i1, __i0);
1491	1421	}
1492	1422
...	...	@@ -1507,8 +1437,10 @@ _mm_setr_pi32(int __i0, int __i1) {
1507	1437	/// \param __w3
1508	1438	/// A 16-bit integer value used to initialize bits [63:48] of the result.
1509	1439	/// \returns An initialized 64-bit integer vector.
1510		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1511		_mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
	1440	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2 _mm_setr_pi16(short __w0,
	1441	short __w1,
	1442	short __w2,
	1443	short __w3) {
1512	1444	return _mm_set_pi16(__w3, __w2, __w1, __w0);
1513	1445	}
1514	1446
...	...	@@ -1537,13 +1469,12 @@ _mm_setr_pi16(short __w0, short __w1, short __w2, short __w3) {
1537	1469	/// \param __b7
1538	1470	/// An 8-bit integer value used to initialize bits [63:56] of the result.
1539	1471	/// \returns An initialized 64-bit integer vector.
1540		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	1472	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
1541	1473	_mm_setr_pi8(char __b0, char __b1, char __b2, char __b3, char __b4, char __b5,
1542	1474	char __b6, char __b7) {
1543	1475	return _mm_set_pi8(__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
1544	1476	}
1545	1477
1546		#undef __anyext128
1547	1478	#undef __trunc64
1548	1479	#undef __DEFAULT_FN_ATTRS_SSE2
1549	1480

lib/include/module.modulemap+23-4

...	...	@@ -171,8 +171,22 @@ module _Builtin_intrinsics [system] [extern_c] {
171	171	// that module. The system float.h (if present) will be treated
172	172	// as a textual header in the sytem module.
173	173	module _Builtin_float [system] {
174		header "float.h"
175		export *
	174	textual header "float.h"
	175
	176	explicit module float {
	177	header "__float_float.h"
	178	export *
	179	}
	180
	181	explicit module header_macro {
	182	header "__float_header_macro.h"
	183	export *
	184	}
	185
	186	explicit module infinity_nan {
	187	header "__float_infinity_nan.h"
	188	export *
	189	}
176	190	}
177	191
178	192	module _Builtin_inttypes [system] {
...	...	@@ -239,6 +253,11 @@ module _Builtin_stdbool [system] {
239	253	export *
240	254	}
241	255
	256	module _Builtin_stdckdint [system] {
	257	header "stdckdint.h"
	258	export *
	259	}
	260
242	261	module _Builtin_stdcountof [system] {
243	262	header "stdcountof.h"
244	263	export *
...	...	@@ -329,13 +348,13 @@ module _Builtin_unwind [system] {
329	348	}
330	349	// End -fbuiltin-headers-in-system-modules affected modules
331	350
332		module opencl_c {
	351	module opencl_c [system] {
333	352	requires opencl
334	353	header "opencl-c.h"
335	354	header "opencl-c-base.h"
336	355	}
337	356
338		module ptrauth {
	357	module ptrauth [system] {
339	358	header "ptrauth.h"
340	359	export *
341	360	}

lib/include/movrs_avx10_2_512intrin.h+2-2

...	...	@@ -17,8 +17,8 @@
17	17
18	18	/* Define the default attributes for the functions in this file. */
19	19	#define __DEFAULT_FN_ATTRS512 \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("movrs, avx10.2-512"), __min_vector_width__(512)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("movrs, avx10.2"), \
	21	__min_vector_width__(512)))
22	22
23	23	static __inline__ __m512i __DEFAULT_FN_ATTRS512
24	24	_mm512_loadrs_epi8(void const *__A) {

lib/include/movrs_avx10_2intrin.h+4-4

...	...	@@ -17,11 +17,11 @@
17	17
18	18	/* Define the default attributes for the functions in this file. */
19	19	#define __DEFAULT_FN_ATTRS128 \
20		__attribute__((__always_inline__, __nodebug__, \
21		__target__("movrs,avx10.2-256"), __min_vector_width__(128)))
	20	__attribute__((__always_inline__, __nodebug__, __target__("movrs,avx10.2"), \
	21	__min_vector_width__(128)))
22	22	#define __DEFAULT_FN_ATTRS256 \
23		__attribute__((__always_inline__, __nodebug__, \
24		__target__("movrs,avx10.2-256"), __min_vector_width__(256)))
	23	__attribute__((__always_inline__, __nodebug__, __target__("movrs,avx10.2"), \
	24	__min_vector_width__(256)))
25	25
26	26	static __inline__ __m128i __DEFAULT_FN_ATTRS128
27	27	_mm_loadrs_epi8(void const *__A) {

lib/include/pmmintrin.h+12-24

...	...	@@ -17,15 +17,9 @@
17	17	#include <emmintrin.h>
18	18
19	19	/* Define the default attributes for the functions in this file. */
20		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
21		#define __DEFAULT_FN_ATTRS \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("sse3,no-evex512"), __min_vector_width__(128)))
24		#else
25	20	#define __DEFAULT_FN_ATTRS \
26	21	__attribute__((__always_inline__, __nodebug__, __target__("sse3"), \
27	22	__min_vector_width__(128)))
28		#endif
29	23
30	24	#if defined(__cplusplus) && (__cplusplus >= 201103L)
31	25	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
...	...	@@ -66,9 +60,8 @@ _mm_lddqu_si128(__m128i_u const *__p)
66	60	/// A 128-bit vector of [4 x float] containing the right source operand.
67	61	/// \returns A 128-bit vector of [4 x float] containing the alternating sums and
68	62	/// differences of both operands.
69		static __inline__ __m128 __DEFAULT_FN_ATTRS
70		_mm_addsub_ps(__m128 __a, __m128 __b)
71		{
	63	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
	64	_mm_addsub_ps(__m128 __a, __m128 __b) {
72	65	return __builtin_ia32_addsubps((__v4sf)__a, (__v4sf)__b);
73	66	}
74	67
...	...	@@ -89,9 +82,8 @@ _mm_addsub_ps(__m128 __a, __m128 __b)
89	82	/// destination.
90	83	/// \returns A 128-bit vector of [4 x float] containing the horizontal sums of
91	84	/// both operands.
92		static __inline__ __m128 __DEFAULT_FN_ATTRS
93		_mm_hadd_ps(__m128 __a, __m128 __b)
94		{
	85	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hadd_ps(__m128 __a,
	86	__m128 __b) {
95	87	return __builtin_ia32_haddps((__v4sf)__a, (__v4sf)__b);
96	88	}
97	89
...	...	@@ -112,9 +104,8 @@ _mm_hadd_ps(__m128 __a, __m128 __b)
112	104	/// bits of the destination.
113	105	/// \returns A 128-bit vector of [4 x float] containing the horizontal
114	106	/// differences of both operands.
115		static __inline__ __m128 __DEFAULT_FN_ATTRS
116		_mm_hsub_ps(__m128 __a, __m128 __b)
117		{
	107	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hsub_ps(__m128 __a,
	108	__m128 __b) {
118	109	return __builtin_ia32_hsubps((__v4sf)__a, (__v4sf)__b);
119	110	}
120	111
...	...	@@ -174,9 +165,8 @@ _mm_moveldup_ps(__m128 __a)
174	165	/// A 128-bit vector of [2 x double] containing the right source operand.
175	166	/// \returns A 128-bit vector of [2 x double] containing the alternating sums
176	167	/// and differences of both operands.
177		static __inline__ __m128d __DEFAULT_FN_ATTRS
178		_mm_addsub_pd(__m128d __a, __m128d __b)
179		{
	168	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
	169	_mm_addsub_pd(__m128d __a, __m128d __b) {
180	170	return __builtin_ia32_addsubpd((__v2df)__a, (__v2df)__b);
181	171	}
182	172
...	...	@@ -197,9 +187,8 @@ _mm_addsub_pd(__m128d __a, __m128d __b)
197	187	/// destination.
198	188	/// \returns A 128-bit vector of [2 x double] containing the horizontal sums of
199	189	/// both operands.
200		static __inline__ __m128d __DEFAULT_FN_ATTRS
201		_mm_hadd_pd(__m128d __a, __m128d __b)
202		{
	190	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
	191	_mm_hadd_pd(__m128d __a, __m128d __b) {
203	192	return __builtin_ia32_haddpd((__v2df)__a, (__v2df)__b);
204	193	}
205	194
...	...	@@ -220,9 +209,8 @@ _mm_hadd_pd(__m128d __a, __m128d __b)
220	209	/// the destination.
221	210	/// \returns A 128-bit vector of [2 x double] containing the horizontal
222	211	/// differences of both operands.
223		static __inline__ __m128d __DEFAULT_FN_ATTRS
224		_mm_hsub_pd(__m128d __a, __m128d __b)
225		{
	212	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
	213	_mm_hsub_pd(__m128d __a, __m128d __b) {
226	214	return __builtin_ia32_hsubpd((__v2df)__a, (__v2df)__b);
227	215	}
228	216

lib/include/ptrauth.h+16-2

...	...	@@ -95,7 +95,7 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
95	95	__ptrauth qualifier; the compiler will perform this check
96	96	automatically. */
97	97
98		#if __has_feature(ptrauth_intrinsics)
	98	#if __has_feature(ptrauth_intrinsics) \|\| defined(__PTRAUTH__)
99	99
100	100	/* Strip the signature from a value without authenticating it.
101	101
...	...	@@ -241,6 +241,18 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
241	241	#define ptrauth_type_discriminator(__type) \
242	242	__builtin_ptrauth_type_discriminator(__type)
243	243
	244	/* Compute the constant discriminator used by Clang to sign pointers with the
	245	given C function pointer type.
	246
	247	A call to this function is an integer constant expression. */
	248	#if __has_feature(ptrauth_function_pointer_type_discrimination)
	249	#define ptrauth_function_pointer_type_discriminator(__type) \
	250	__builtin_ptrauth_type_discriminator(__type)
	251	#else
	252	#define ptrauth_function_pointer_type_discriminator(__type) \
	253	((ptrauth_extra_data_t)0)
	254	#endif
	255
244	256	/* Compute a signature for the given pair of pointer-sized values.
245	257	The order of the arguments is significant.
246	258
...	...	@@ -372,6 +384,8 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
372	384	})
373	385
374	386	#define ptrauth_type_discriminator(__type) ((ptrauth_extra_data_t)0)
	387	#define ptrauth_function_pointer_type_discriminator(__type) \
	388	((ptrauth_extra_data_t)0)
375	389
376	390	#define ptrauth_sign_generic_data(__value, __data) \
377	391	({ \
...	...	@@ -388,6 +402,6 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
388	402	#define __ptrauth_objc_isa_uintptr
389	403	#define __ptrauth_objc_super_pointer
390	404
391		#endif /* __has_feature(ptrauth_intrinsics) */
	405	#endif /* __has_feature(ptrauth_intrinsics) \|\| defined(__PTRAUTH__) */
392	406
393	407	#endif /* __PTRAUTH_H */

lib/include/riscv_mips.h created+34

...	...	@@ -0,0 +1,34 @@
	1	//===----- riscv_mips.h - RISC-V MIPS Intrinsic definitions
	2	//----------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef __RISCV_MIPS_H
	11	#define __RISCV_MIPS_H
	12
	13	#if !defined(__riscv)
	14	#error "This header is only meant to be used on riscv architecture"
	15	#endif
	16
	17	#define __DEFAULT_FN_ATTRS \
	18	__attribute__((__always_inline__, __nodebug__, __target__("xmipsexectl")))
	19
	20	static __inline__ void __DEFAULT_FN_ATTRS __mips_pause() {
	21	__builtin_riscv_mips_pause();
	22	}
	23
	24	static __inline__ void __DEFAULT_FN_ATTRS __mips_ehb() {
	25	__builtin_riscv_mips_ehb();
	26	}
	27
	28	static __inline__ void __DEFAULT_FN_ATTRS __mips_ihb() {
	29	__builtin_riscv_mips_ihb();
	30	}
	31
	32	#undef __DEFAULT_FN_ATTRS
	33
	34	#endif

lib/include/riscv_nds.h created+89

...	...	@@ -0,0 +1,89 @@
	1	/*===---- riscv_nds.h - Andes intrinsics -----------------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __RISCV_NDS_H
	11	#define __RISCV_NDS_H
	12
	13	#include <stdint.h>
	14
	15	#if defined(__cplusplus)
	16	extern "C" {
	17	#endif
	18
	19	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
	20
	21	#if defined(__riscv_xandesperf)
	22
	23	#if __riscv_xlen == 32
	24
	25	static __inline__ int32_t __DEFAULT_FN_ATTRS __riscv_nds_ffb_32(uint32_t __a,
	26	uint32_t __b) {
	27	return __builtin_riscv_nds_ffb_32(__a, __b);
	28	}
	29
	30	static __inline__ int32_t __DEFAULT_FN_ATTRS
	31	__riscv_nds_ffzmism_32(uint32_t __a, uint32_t __b) {
	32	return __builtin_riscv_nds_ffzmism_32(__a, __b);
	33	}
	34
	35	static __inline__ int32_t __DEFAULT_FN_ATTRS
	36	__riscv_nds_ffmism_32(uint32_t __a, uint32_t __b) {
	37	return __builtin_riscv_nds_ffmism_32(__a, __b);
	38	}
	39
	40	static __inline__ int32_t __DEFAULT_FN_ATTRS
	41	__riscv_nds_flmism_32(uint32_t __a, uint32_t __b) {
	42	return __builtin_riscv_nds_flmism_32(__a, __b);
	43	}
	44
	45	#endif
	46
	47	#if __riscv_xlen == 64
	48
	49	static __inline__ int64_t __DEFAULT_FN_ATTRS __riscv_nds_ffb_64(uint64_t __a,
	50	uint64_t __b) {
	51	return __builtin_riscv_nds_ffb_64(__a, __b);
	52	}
	53
	54	static __inline__ int64_t __DEFAULT_FN_ATTRS
	55	__riscv_nds_ffzmism_64(uint64_t __a, uint64_t __b) {
	56	return __builtin_riscv_nds_ffzmism_64(__a, __b);
	57	}
	58
	59	static __inline__ int64_t __DEFAULT_FN_ATTRS
	60	__riscv_nds_ffmism_64(uint64_t __a, uint64_t __b) {
	61	return __builtin_riscv_nds_ffmism_64(__a, __b);
	62	}
	63
	64	static __inline__ int64_t __DEFAULT_FN_ATTRS
	65	__riscv_nds_flmism_64(uint64_t __a, uint64_t __b) {
	66	return __builtin_riscv_nds_flmism_64(__a, __b);
	67	}
	68
	69	#endif
	70
	71	#endif // defined(__riscv_xandesperf)
	72
	73	#if defined(__riscv_xandesbfhcvt)
	74
	75	static __inline__ float __DEFAULT_FN_ATTRS __riscv_nds_fcvt_s_bf16(__bf16 bf) {
	76	return __builtin_riscv_nds_fcvt_s_bf16(bf);
	77	}
	78
	79	static __inline__ __bf16 __DEFAULT_FN_ATTRS __riscv_nds_fcvt_bf16_s(float sf) {
	80	return __builtin_riscv_nds_fcvt_bf16_s(sf);
	81	}
	82
	83	#endif // defined(__riscv_xandesbfhcvt)
	84
	85	#if defined(__cplusplus)
	86	}
	87	#endif
	88
	89	#endif // define __RISCV_NDS_H

lib/include/sifive_vector.h+58-2

...	...	@@ -47,9 +47,9 @@
47	47	__riscv_sf_vc_x_se(p27_26, p24_20, p11_7, (uint32_t)rs1, 32, 3, vl)
48	48
49	49	#define __riscv_sf_vc_i_se_u8mf4(p27_26, p24_20, p11_7, simm5, vl) \
50		__riscv_sf_vc_i_se(p27_26, p24_20, p11_7, simm5, 8, 7, vl)
51		#define __riscv_sf_vc_i_se_u8mf2(p27_26, p24_20, p11_7, simm5, vl) \
52	50	__riscv_sf_vc_i_se(p27_26, p24_20, p11_7, simm5, 8, 6, vl)
	51	#define __riscv_sf_vc_i_se_u8mf2(p27_26, p24_20, p11_7, simm5, vl) \
	52	__riscv_sf_vc_i_se(p27_26, p24_20, p11_7, simm5, 8, 7, vl)
53	53	#define __riscv_sf_vc_i_se_u8m1(p27_26, p24_20, p11_7, simm5, vl) \
54	54	__riscv_sf_vc_i_se(p27_26, p24_20, p11_7, simm5, 8, 0, vl)
55	55	#define __riscv_sf_vc_i_se_u8m2(p27_26, p24_20, p11_7, simm5, vl) \
...	...	@@ -115,4 +115,60 @@
115	115	#endif
116	116	#endif
117	117
	118	#define __riscv_sf_vsettnt_e8w1(atn) __riscv_sf_vsettnt(atn, 0, 1);
	119	#define __riscv_sf_vsettnt_e8w2(atn) __riscv_sf_vsettnt(atn, 0, 2);
	120	#define __riscv_sf_vsettnt_e8w4(atn) __riscv_sf_vsettnt(atn, 0, 3);
	121	#define __riscv_sf_vsettnt_e16w1(atn) __riscv_sf_vsettnt(atn, 1, 1);
	122	#define __riscv_sf_vsettnt_e16w2(atn) __riscv_sf_vsettnt(atn, 1, 2);
	123	#define __riscv_sf_vsettnt_e16w4(atn) __riscv_sf_vsettnt(atn, 1, 3);
	124	#define __riscv_sf_vsettnt_e32w1(atn) __riscv_sf_vsettnt(atn, 2, 1);
	125	#define __riscv_sf_vsettnt_e32w2(atn) __riscv_sf_vsettnt(atn, 2, 2);
	126	#define __riscv_sf_vsettm_e8w1(atm) __riscv_sf_vsettm(atm, 0, 1);
	127	#define __riscv_sf_vsettm_e8w2(atm) __riscv_sf_vsettm(atm, 0, 2);
	128	#define __riscv_sf_vsettm_e8w4(atm) __riscv_sf_vsettm(atm, 0, 3);
	129	#define __riscv_sf_vsettm_e16w1(atm) __riscv_sf_vsettm(atm, 1, 1);
	130	#define __riscv_sf_vsettm_e16w2(atm) __riscv_sf_vsettm(atm, 1, 2);
	131	#define __riscv_sf_vsettm_e16w4(atm) __riscv_sf_vsettm(atm, 1, 3);
	132	#define __riscv_sf_vsettm_e32w1(atm) __riscv_sf_vsettm(atm, 2, 1);
	133	#define __riscv_sf_vsettm_e32w2(atm) __riscv_sf_vsettm(atm, 2, 2);
	134	#define __riscv_sf_vsettn_e8w1(atn) __riscv_sf_vsettn(atn, 0, 1);
	135	#define __riscv_sf_vsettn_e8w2(atn) __riscv_sf_vsettn(atn, 0, 2);
	136	#define __riscv_sf_vsettn_e8w4(atn) __riscv_sf_vsettn(atn, 0, 3);
	137	#define __riscv_sf_vsettn_e16w1(atn) __riscv_sf_vsettn(atn, 1, 1);
	138	#define __riscv_sf_vsettn_e16w2(atn) __riscv_sf_vsettn(atn, 1, 2);
	139	#define __riscv_sf_vsettn_e16w4(atn) __riscv_sf_vsettn(atn, 1, 3);
	140	#define __riscv_sf_vsettn_e32w1(atn) __riscv_sf_vsettn(atn, 2, 1);
	141	#define __riscv_sf_vsettn_e32w2(atn) __riscv_sf_vsettn(atn, 2, 2);
	142	#define __riscv_sf_vsettk_e8w1(atk) __riscv_sf_vsettk(atk, 0, 1);
	143	#define __riscv_sf_vsettk_e8w2(atk) __riscv_sf_vsettk(atk, 0, 2);
	144	#define __riscv_sf_vsettk_e8w4(atk) __riscv_sf_vsettk(atk, 0, 3);
	145	#define __riscv_sf_vsettk_e16w1(atk) __riscv_sf_vsettk(atk, 1, 1);
	146	#define __riscv_sf_vsettk_e16w2(atk) __riscv_sf_vsettk(atk, 1, 2);
	147	#define __riscv_sf_vsettk_e16w4(atk) __riscv_sf_vsettk(atk, 1, 3);
	148	#define __riscv_sf_vsettk_e32w1(atk) __riscv_sf_vsettk(atk, 2, 1);
	149	#define __riscv_sf_vsettk_e32w2(atk) __riscv_sf_vsettk(atk, 2, 2);
	150	#define __riscv_sf_vtzero_t_e8w1(tile, atm, atn) \
	151	__riscv_sf_vtzero_t(tile, atm, atn, 3, 1);
	152	#define __riscv_sf_vtzero_t_e8w2(tile, atm, atn) \
	153	__riscv_sf_vtzero_t(tile, atm, atn, 3, 2);
	154	#define __riscv_sf_vtzero_t_e8w4(tile, atm, atn) \
	155	__riscv_sf_vtzero_t(tile, atm, atn, 3, 4);
	156	#define __riscv_sf_vtzero_t_e16w1(tile, atm, atn) \
	157	__riscv_sf_vtzero_t(tile, atm, atn, 4, 1);
	158	#define __riscv_sf_vtzero_t_e16w2(tile, atm, atn) \
	159	__riscv_sf_vtzero_t(tile, atm, atn, 4, 2);
	160	#define __riscv_sf_vtzero_t_e16w4(tile, atm, atn) \
	161	__riscv_sf_vtzero_t(tile, atm, atn, 4, 4);
	162	#define __riscv_sf_vtzero_t_e32w1(tile, atm, atn) \
	163	__riscv_sf_vtzero_t(tile, atm, atn, 5, 1);
	164	#define __riscv_sf_vtzero_t_e32w2(tile, atm, atn) \
	165	__riscv_sf_vtzero_t(tile, atm, atn, 5, 2);
	166	#if __riscv_v_elen >= 64
	167	#define __riscv_sf_vsettnt_e64w1(atn) __riscv_sf_vsettnt(atn, 3, 1);
	168	#define __riscv_sf_vsettm_e64w1(atm) __riscv_sf_vsettm(atm, 3, 1);
	169	#define __riscv_sf_vsettn_e64w1(atn) __riscv_sf_vsettn(atn, 3, 1);
	170	#define __riscv_sf_vsettk_e64w1(atk) __riscv_sf_vsettk(atk, 3, 1);
	171	#define __riscv_sf_vtzero_t_e64w1(tile, atm, atn) \
	172	__riscv_sf_vtzero_t(tile, atm, atn, 6, 1);
	173	#endif
118	174	#endif //_SIFIVE_VECTOR_H_

lib/include/sm4evexintrin.h+2-2

...	...	@@ -14,8 +14,8 @@
14	14	#define __SM4EVEXINTRIN_H
15	15
16	16	#define __DEFAULT_FN_ATTRS512 \
17		__attribute__((__always_inline__, __nodebug__, \
18		__target__("sm4,avx10.2-512"), __min_vector_width__(512)))
	17	__attribute__((__always_inline__, __nodebug__, __target__("sm4,avx10.2"), \
	18	__min_vector_width__(512)))
19	19
20	20	static __inline__ __m512i __DEFAULT_FN_ATTRS512
21	21	_mm512_sm4key4_epi32(__m512i __A, __m512i __B) {

lib/include/smmintrin.h+77-59

...	...	@@ -17,14 +17,14 @@
17	17	#include <tmmintrin.h>
18	18
19	19	/* Define the default attributes for the functions in this file. */
20		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
21		#define __DEFAULT_FN_ATTRS \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("sse4.1,no-evex512"), __min_vector_width__(128)))
24		#else
25	20	#define __DEFAULT_FN_ATTRS \
26	21	__attribute__((__always_inline__, __nodebug__, __target__("sse4.1"), \
27	22	__min_vector_width__(128)))
	23
	24	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	25	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	26	#else
	27	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
28	28	#endif
29	29
30	30	/* SSE4 Rounding macros. */
...	...	@@ -439,9 +439,8 @@
439	439	/// position in the result. When a mask bit is 1, the corresponding 64-bit
440	440	/// element in operand \a __V2 is copied to the same position in the result.
441	441	/// \returns A 128-bit vector of [2 x double] containing the copied values.
442		static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_blendv_pd(__m128d __V1,
443		__m128d __V2,
444		__m128d __M) {
	442	static __inline__ __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
	443	_mm_blendv_pd(__m128d __V1, __m128d __V2, __m128d __M) {
445	444	return (__m128d)__builtin_ia32_blendvpd((__v2df)__V1, (__v2df)__V2,
446	445	(__v2df)__M);
447	446	}
...	...	@@ -466,9 +465,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_blendv_pd(__m128d __V1,
466	465	/// position in the result. When a mask bit is 1, the corresponding 32-bit
467	466	/// element in operand \a __V2 is copied to the same position in the result.
468	467	/// \returns A 128-bit vector of [4 x float] containing the copied values.
469		static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_blendv_ps(__m128 __V1,
470		__m128 __V2,
471		__m128 __M) {
	468	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
	469	_mm_blendv_ps(__m128 __V1, __m128 __V2, __m128 __M) {
472	470	return (__m128)__builtin_ia32_blendvps((__v4sf)__V1, (__v4sf)__V2,
473	471	(__v4sf)__M);
474	472	}
...	...	@@ -493,9 +491,8 @@ static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_blendv_ps(__m128 __V1,
493	491	/// position in the result. When a mask bit is 1, the corresponding 8-bit
494	492	/// element in operand \a __V2 is copied to the same position in the result.
495	493	/// \returns A 128-bit vector of [16 x i8] containing the copied values.
496		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_blendv_epi8(__m128i __V1,
497		__m128i __V2,
498		__m128i __M) {
	494	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	495	_mm_blendv_epi8(__m128i __V1, __m128i __V2, __m128i __M) {
499	496	return (__m128i)__builtin_ia32_pblendvb128((__v16qi)__V1, (__v16qi)__V2,
500	497	(__v16qi)__M);
501	498	}
...	...	@@ -542,8 +539,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_blendv_epi8(__m128i __V1,
542	539	/// \param __V2
543	540	/// A 128-bit integer vector.
544	541	/// \returns A 128-bit integer vector containing the products of both operands.
545		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi32(__m128i __V1,
546		__m128i __V2) {
	542	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	543	_mm_mullo_epi32(__m128i __V1, __m128i __V2) {
547	544	return (__m128i)((__v4su)__V1 * (__v4su)__V2);
548	545	}
549	546
...	...	@@ -561,8 +558,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi32(__m128i __V1,
561	558	/// A 128-bit vector of [4 x i32].
562	559	/// \returns A 128-bit vector of [2 x i64] containing the products of both
563	560	/// operands.
564		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mul_epi32(__m128i __V1,
565		__m128i __V2) {
	561	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	562	_mm_mul_epi32(__m128i __V1, __m128i __V2) {
566	563	return (__m128i)__builtin_ia32_pmuldq128((__v4si)__V1, (__v4si)__V2);
567	564	}
568	565
...	...	@@ -669,8 +666,8 @@ _mm_stream_load_si128(const void *__V) {
669	666	/// \param __V2
670	667	/// A 128-bit vector of [16 x i8]
671	668	/// \returns A 128-bit vector of [16 x i8] containing the lesser values.
672		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi8(__m128i __V1,
673		__m128i __V2) {
	669	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	670	_mm_min_epi8(__m128i __V1, __m128i __V2) {
674	671	return (__m128i)__builtin_elementwise_min((__v16qs)__V1, (__v16qs)__V2);
675	672	}
676	673
...	...	@@ -687,8 +684,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi8(__m128i __V1,
687	684	/// \param __V2
688	685	/// A 128-bit vector of [16 x i8].
689	686	/// \returns A 128-bit vector of [16 x i8] containing the greater values.
690		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi8(__m128i __V1,
691		__m128i __V2) {
	687	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	688	_mm_max_epi8(__m128i __V1, __m128i __V2) {
692	689	return (__m128i)__builtin_elementwise_max((__v16qs)__V1, (__v16qs)__V2);
693	690	}
694	691
...	...	@@ -705,8 +702,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi8(__m128i __V1,
705	702	/// \param __V2
706	703	/// A 128-bit vector of [8 x u16].
707	704	/// \returns A 128-bit vector of [8 x u16] containing the lesser values.
708		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu16(__m128i __V1,
709		__m128i __V2) {
	705	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	706	_mm_min_epu16(__m128i __V1, __m128i __V2) {
710	707	return (__m128i)__builtin_elementwise_min((__v8hu)__V1, (__v8hu)__V2);
711	708	}
712	709
...	...	@@ -723,8 +720,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu16(__m128i __V1,
723	720	/// \param __V2
724	721	/// A 128-bit vector of [8 x u16].
725	722	/// \returns A 128-bit vector of [8 x u16] containing the greater values.
726		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu16(__m128i __V1,
727		__m128i __V2) {
	723	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	724	_mm_max_epu16(__m128i __V1, __m128i __V2) {
728	725	return (__m128i)__builtin_elementwise_max((__v8hu)__V1, (__v8hu)__V2);
729	726	}
730	727
...	...	@@ -741,8 +738,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu16(__m128i __V1,
741	738	/// \param __V2
742	739	/// A 128-bit vector of [4 x i32].
743	740	/// \returns A 128-bit vector of [4 x i32] containing the lesser values.
744		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi32(__m128i __V1,
745		__m128i __V2) {
	741	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	742	_mm_min_epi32(__m128i __V1, __m128i __V2) {
746	743	return (__m128i)__builtin_elementwise_min((__v4si)__V1, (__v4si)__V2);
747	744	}
748	745
...	...	@@ -759,8 +756,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi32(__m128i __V1,
759	756	/// \param __V2
760	757	/// A 128-bit vector of [4 x i32].
761	758	/// \returns A 128-bit vector of [4 x i32] containing the greater values.
762		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi32(__m128i __V1,
763		__m128i __V2) {
	759	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	760	_mm_max_epi32(__m128i __V1, __m128i __V2) {
764	761	return (__m128i)__builtin_elementwise_max((__v4si)__V1, (__v4si)__V2);
765	762	}
766	763
...	...	@@ -777,8 +774,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi32(__m128i __V1,
777	774	/// \param __V2
778	775	/// A 128-bit vector of [4 x u32].
779	776	/// \returns A 128-bit vector of [4 x u32] containing the lesser values.
780		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu32(__m128i __V1,
781		__m128i __V2) {
	777	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	778	_mm_min_epu32(__m128i __V1, __m128i __V2) {
782	779	return (__m128i)__builtin_elementwise_min((__v4su)__V1, (__v4su)__V2);
783	780	}
784	781
...	...	@@ -795,8 +792,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu32(__m128i __V1,
795	792	/// \param __V2
796	793	/// A 128-bit vector of [4 x u32].
797	794	/// \returns A 128-bit vector of [4 x u32] containing the greater values.
798		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu32(__m128i __V1,
799		__m128i __V2) {
	795	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	796	_mm_max_epu32(__m128i __V1, __m128i __V2) {
800	797	return (__m128i)__builtin_elementwise_max((__v4su)__V1, (__v4su)__V2);
801	798	}
802	799
...	...	@@ -1096,8 +1093,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu32(__m128i __V1,
1096	1093	/// \param __V
1097	1094	/// A 128-bit integer vector selecting which bits to test in operand \a __M.
1098	1095	/// \returns TRUE if the specified bits are all zeros; FALSE otherwise.
1099		static __inline__ int __DEFAULT_FN_ATTRS _mm_testz_si128(__m128i __M,
1100		__m128i __V) {
	1096	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	1097	_mm_testz_si128(__m128i __M, __m128i __V) {
1101	1098	return __builtin_ia32_ptestz128((__v2di)__M, (__v2di)__V);
1102	1099	}
1103	1100
...	...	@@ -1113,8 +1110,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testz_si128(__m128i __M,
1113	1110	/// \param __V
1114	1111	/// A 128-bit integer vector selecting which bits to test in operand \a __M.
1115	1112	/// \returns TRUE if the specified bits are all ones; FALSE otherwise.
1116		static __inline__ int __DEFAULT_FN_ATTRS _mm_testc_si128(__m128i __M,
1117		__m128i __V) {
	1113	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	1114	_mm_testc_si128(__m128i __M, __m128i __V) {
1118	1115	return __builtin_ia32_ptestc128((__v2di)__M, (__v2di)__V);
1119	1116	}
1120	1117
...	...	@@ -1131,8 +1128,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testc_si128(__m128i __M,
1131	1128	/// A 128-bit integer vector selecting which bits to test in operand \a __M.
1132	1129	/// \returns TRUE if the specified bits are neither all zeros nor all ones;
1133	1130	/// FALSE otherwise.
1134		static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M,
1135		__m128i __V) {
	1131	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
	1132	_mm_testnzc_si128(__m128i __M, __m128i __V) {
1136	1133	return __builtin_ia32_ptestnzc128((__v2di)__M, (__v2di)__V);
1137	1134	}
1138	1135
...	...	@@ -1205,8 +1202,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M,
1205	1202	/// \param __V2
1206	1203	/// A 128-bit integer vector.
1207	1204	/// \returns A 128-bit integer vector containing the comparison results.
1208		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi64(__m128i __V1,
1209		__m128i __V2) {
	1205	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1206	_mm_cmpeq_epi64(__m128i __V1, __m128i __V2) {
1210	1207	return (__m128i)((__v2di)__V1 == (__v2di)__V2);
1211	1208	}
1212	1209
...	...	@@ -1224,7 +1221,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi64(__m128i __V1,
1224	1221	/// A 128-bit vector of [16 x i8]. The lower eight 8-bit elements are
1225	1222	/// sign-extended to 16-bit values.
1226	1223	/// \returns A 128-bit vector of [8 x i16] containing the sign-extended values.
1227		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi16(__m128i __V) {
	1224	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1225	_mm_cvtepi8_epi16(__m128i __V) {
1228	1226	/* This function always performs a signed extension, but __v16qi is a char
1229	1227	which may be signed or unsigned, so use __v16qs. */
1230	1228	return (__m128i) __builtin_convertvector(
...	...	@@ -1246,7 +1244,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi16(__m128i __V) {
1246	1244	/// A 128-bit vector of [16 x i8]. The lower four 8-bit elements are
1247	1245	/// sign-extended to 32-bit values.
1248	1246	/// \returns A 128-bit vector of [4 x i32] containing the sign-extended values.
1249		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi32(__m128i __V) {
	1247	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1248	_mm_cvtepi8_epi32(__m128i __V) {
1250	1249	/* This function always performs a signed extension, but __v16qi is a char
1251	1250	which may be signed or unsigned, so use __v16qs. */
1252	1251	return (__m128i) __builtin_convertvector(
...	...	@@ -1266,7 +1265,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi32(__m128i __V) {
1266	1265	/// A 128-bit vector of [16 x i8]. The lower two 8-bit elements are
1267	1266	/// sign-extended to 64-bit values.
1268	1267	/// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
1269		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi64(__m128i __V) {
	1268	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1269	_mm_cvtepi8_epi64(__m128i __V) {
1270	1270	/* This function always performs a signed extension, but __v16qi is a char
1271	1271	which may be signed or unsigned, so use __v16qs. */
1272	1272	return (__m128i) __builtin_convertvector(
...	...	@@ -1286,7 +1286,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi64(__m128i __V) {
1286	1286	/// A 128-bit vector of [8 x i16]. The lower four 16-bit elements are
1287	1287	/// sign-extended to 32-bit values.
1288	1288	/// \returns A 128-bit vector of [4 x i32] containing the sign-extended values.
1289		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi32(__m128i __V) {
	1289	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1290	_mm_cvtepi16_epi32(__m128i __V) {
1290	1291	return (__m128i) __builtin_convertvector(
1291	1292	__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4si);
1292	1293	}
...	...	@@ -1304,7 +1305,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi32(__m128i __V) {
1304	1305	/// A 128-bit vector of [8 x i16]. The lower two 16-bit elements are
1305	1306	/// sign-extended to 64-bit values.
1306	1307	/// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
1307		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi64(__m128i __V) {
	1308	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1309	_mm_cvtepi16_epi64(__m128i __V) {
1308	1310	return (__m128i) __builtin_convertvector(
1309	1311	__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1), __v2di);
1310	1312	}
...	...	@@ -1322,7 +1324,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi64(__m128i __V) {
1322	1324	/// A 128-bit vector of [4 x i32]. The lower two 32-bit elements are
1323	1325	/// sign-extended to 64-bit values.
1324	1326	/// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
1325		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi32_epi64(__m128i __V) {
	1327	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1328	_mm_cvtepi32_epi64(__m128i __V) {
1326	1329	return (__m128i) __builtin_convertvector(
1327	1330	__builtin_shufflevector((__v4si)__V, (__v4si)__V, 0, 1), __v2di);
1328	1331	}
...	...	@@ -1341,7 +1344,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi32_epi64(__m128i __V) {
1341	1344	/// A 128-bit vector of [16 x i8]. The lower eight 8-bit elements are
1342	1345	/// zero-extended to 16-bit values.
1343	1346	/// \returns A 128-bit vector of [8 x i16] containing the zero-extended values.
1344		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi16(__m128i __V) {
	1347	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1348	_mm_cvtepu8_epi16(__m128i __V) {
1345	1349	return (__m128i) __builtin_convertvector(
1346	1350	__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6,
1347	1351	7),
...	...	@@ -1361,7 +1365,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi16(__m128i __V) {
1361	1365	/// A 128-bit vector of [16 x i8]. The lower four 8-bit elements are
1362	1366	/// zero-extended to 32-bit values.
1363	1367	/// \returns A 128-bit vector of [4 x i32] containing the zero-extended values.
1364		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi32(__m128i __V) {
	1368	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1369	_mm_cvtepu8_epi32(__m128i __V) {
1365	1370	return (__m128i) __builtin_convertvector(
1366	1371	__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4si);
1367	1372	}
...	...	@@ -1379,7 +1384,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi32(__m128i __V) {
1379	1384	/// A 128-bit vector of [16 x i8]. The lower two 8-bit elements are
1380	1385	/// zero-extended to 64-bit values.
1381	1386	/// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
1382		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi64(__m128i __V) {
	1387	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1388	_mm_cvtepu8_epi64(__m128i __V) {
1383	1389	return (__m128i) __builtin_convertvector(
1384	1390	__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1), __v2di);
1385	1391	}
...	...	@@ -1397,7 +1403,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi64(__m128i __V) {
1397	1403	/// A 128-bit vector of [8 x i16]. The lower four 16-bit elements are
1398	1404	/// zero-extended to 32-bit values.
1399	1405	/// \returns A 128-bit vector of [4 x i32] containing the zero-extended values.
1400		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi32(__m128i __V) {
	1406	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1407	_mm_cvtepu16_epi32(__m128i __V) {
1401	1408	return (__m128i) __builtin_convertvector(
1402	1409	__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4si);
1403	1410	}
...	...	@@ -1415,7 +1422,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi32(__m128i __V) {
1415	1422	/// A 128-bit vector of [8 x i16]. The lower two 16-bit elements are
1416	1423	/// zero-extended to 64-bit values.
1417	1424	/// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
1418		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi64(__m128i __V) {
	1425	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1426	_mm_cvtepu16_epi64(__m128i __V) {
1419	1427	return (__m128i) __builtin_convertvector(
1420	1428	__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1), __v2di);
1421	1429	}
...	...	@@ -1433,7 +1441,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi64(__m128i __V) {
1433	1441	/// A 128-bit vector of [4 x i32]. The lower two 32-bit elements are
1434	1442	/// zero-extended to 64-bit values.
1435	1443	/// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
1436		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu32_epi64(__m128i __V) {
	1444	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1445	_mm_cvtepu32_epi64(__m128i __V) {
1437	1446	return (__m128i) __builtin_convertvector(
1438	1447	__builtin_shufflevector((__v4su)__V, (__v4su)__V, 0, 1), __v2di);
1439	1448	}
...	...	@@ -1457,8 +1466,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu32_epi64(__m128i __V) {
1457	1466	/// A 128-bit vector of [4 x i32]. The converted [4 x i16] values are
1458	1467	/// written to the higher 64 bits of the result.
1459	1468	/// \returns A 128-bit vector of [8 x i16] containing the converted values.
1460		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packus_epi32(__m128i __V1,
1461		__m128i __V2) {
	1469	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1470	_mm_packus_epi32(__m128i __V1, __m128i __V2) {
1462	1471	return (__m128i)__builtin_ia32_packusdw128((__v4si)__V1, (__v4si)__V2);
1463	1472	}
1464	1473
...	...	@@ -1515,7 +1524,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packus_epi32(__m128i __V1,
1515	1524	/// \returns A 128-bit value where bits [15:0] contain the minimum value found
1516	1525	/// in parameter \a __V, bits [18:16] contain the index of the minimum value
1517	1526	/// and the remaining bits are set to 0.
1518		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) {
	1527	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	1528	_mm_minpos_epu16(__m128i __V) {
1519	1529	return (__m128i)__builtin_ia32_phminposuw128((__v8hi)__V);
1520	1530	}
1521	1531
...	...	@@ -1525,9 +1535,16 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) {
1525	1535	so we'll do the same. */
1526	1536
1527	1537	#undef __DEFAULT_FN_ATTRS
	1538	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
1528	1539	#define __DEFAULT_FN_ATTRS \
1529	1540	__attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
1530	1541
	1542	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	1543	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	1544	#else
	1545	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	1546	#endif
	1547
1531	1548	/* These specify the type of data that we're comparing. */
1532	1549	#define _SIDD_UBYTE_OPS 0x00
1533	1550	#define _SIDD_UWORD_OPS 0x01
...	...	@@ -2320,12 +2337,13 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) {
2320	2337	/// \param __V2
2321	2338	/// A 128-bit integer vector.
2322	2339	/// \returns A 128-bit integer vector containing the comparison results.
2323		static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi64(__m128i __V1,
2324		__m128i __V2) {
	2340	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	2341	_mm_cmpgt_epi64(__m128i __V1, __m128i __V2) {
2325	2342	return (__m128i)((__v2di)__V1 > (__v2di)__V2);
2326	2343	}
2327	2344
2328	2345	#undef __DEFAULT_FN_ATTRS
	2346	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
2329	2347
2330	2348	#include <popcntintrin.h>
2331	2349

lib/include/spirvintrin.h created+194

...	...	@@ -0,0 +1,194 @@
	1	//===-- spirvintrin.h - SPIR-V intrinsic functions ------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef __SPIRVINTRIN_H
	10	#define __SPIRVINTRIN_H
	11
	12	#ifndef __SPIRV__
	13	#error "This file is intended for SPIR-V targets or offloading to SPIR-V"
	14	#endif
	15
	16	#ifndef __GPUINTRIN_H
	17	#error "Never use <spirvintrin.h> directly; include <gpuintrin.h> instead"
	18	#endif
	19
	20	_Pragma("omp begin declare target device_type(nohost)");
	21	_Pragma("omp begin declare variant match(device = {arch(spirv64)})");
	22
	23	// Type aliases to the address spaces used by the SPIR-V backend.
	24	#define __gpu_private __attribute__((address_space(0)))
	25	#define __gpu_constant __attribute__((address_space(2)))
	26	#define __gpu_local __attribute__((address_space(3)))
	27	#define __gpu_global __attribute__((address_space(1)))
	28	#define __gpu_generic __attribute__((address_space(4)))
	29
	30	// Attribute to declare a function as a kernel.
	31	#define __gpu_kernel __attribute__((device_kernel, visibility("protected")))
	32
	33	// Returns the number of workgroups in the 'x' dimension of the grid.
	34	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_x(void) {
	35	return __builtin_spirv_num_workgroups(0);
	36	}
	37
	38	// Returns the number of workgroups in the 'y' dimension of the grid.
	39	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_y(void) {
	40	return __builtin_spirv_num_workgroups(1);
	41	}
	42
	43	// Returns the number of workgroups in the 'z' dimension of the grid.
	44	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_blocks_z(void) {
	45	return __builtin_spirv_num_workgroups(2);
	46	}
	47
	48	// Returns the 'x' dimension of the current workgroup's id.
	49	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_x(void) {
	50	return __builtin_spirv_workgroup_id(0);
	51	}
	52
	53	// Returns the 'y' dimension of the current workgroup's id.
	54	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_y(void) {
	55	return __builtin_spirv_workgroup_id(1);
	56	}
	57
	58	// Returns the 'z' dimension of the current workgroup's id.
	59	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_block_id_z(void) {
	60	return __builtin_spirv_workgroup_id(2);
	61	}
	62
	63	// Returns the number of workitems in the 'x' dimension.
	64	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_x(void) {
	65	return __builtin_spirv_workgroup_size(0);
	66	}
	67
	68	// Returns the number of workitems in the 'y' dimension.
	69	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_y(void) {
	70	return __builtin_spirv_workgroup_size(1);
	71	}
	72
	73	// Returns the number of workitems in the 'z' dimension.
	74	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_threads_z(void) {
	75	return __builtin_spirv_workgroup_size(2);
	76	}
	77
	78	// Returns the 'x' dimension id of the workitem in the current workgroup.
	79	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_x(void) {
	80	return __builtin_spirv_local_invocation_id(0);
	81	}
	82
	83	// Returns the 'y' dimension id of the workitem in the current workgroup.
	84	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_y(void) {
	85	return __builtin_spirv_local_invocation_id(1);
	86	}
	87
	88	// Returns the 'z' dimension id of the workitem in the current workgroup.
	89	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_thread_id_z(void) {
	90	return __builtin_spirv_local_invocation_id(2);
	91	}
	92
	93	// Returns the size of an wavefront, either 32 or 64 depending on hardware
	94	// and compilation options.
	95	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_num_lanes(void) {
	96	return __builtin_spirv_subgroup_size();
	97	}
	98
	99	// Returns the id of the thread inside of an wavefront executing together.
	100	_DEFAULT_FN_ATTRS static __inline__ uint32_t __gpu_lane_id(void) {
	101	return __builtin_spirv_subgroup_local_invocation_id();
	102	}
	103
	104	// Returns the bit-mask of active threads in the current wavefront. This
	105	// implementation is incorrect if the target uses more than 64 lanes.
	106	_DEFAULT_FN_ATTRS static __inline__ uint64_t __gpu_lane_mask(void) {
	107	uint32_t [[clang::ext_vector_type(4)]] __mask =
	108	__builtin_spirv_subgroup_ballot(1);
	109	return __builtin_bit_cast(uint64_t,
	110	__builtin_shufflevector(__mask, __mask, 0, 1));
	111	}
	112
	113	// Copies the value from the first active thread in the wavefront to the rest.
	114	_DEFAULT_FN_ATTRS static __inline__ uint32_t
	115	__gpu_read_first_lane_u32(uint64_t __lane_mask, uint32_t __x) {
	116	return __builtin_spirv_subgroup_shuffle(__x,
	117	__builtin_ctzg(__gpu_lane_mask()));
	118	}
	119
	120	// Returns a bitmask of threads in the current lane for which \p x is true. This
	121	// implementation is incorrect if the target uses more than 64 lanes.
	122	_DEFAULT_FN_ATTRS static __inline__ uint64_t __gpu_ballot(uint64_t __lane_mask,
	123	bool __x) {
	124	// The lane_mask & gives the nvptx semantics when lane_mask is a subset of
	125	// the active threads.
	126	uint32_t [[clang::ext_vector_type(4)]] __mask =
	127	__builtin_spirv_subgroup_ballot(__x);
	128	return __lane_mask & __builtin_bit_cast(uint64_t, __builtin_shufflevector(
	129	__mask, __mask, 0, 1));
	130	}
	131
	132	// Waits for all the threads in the block to converge and issues a fence.
	133	_DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_threads(void) {
	134	__builtin_spirv_group_barrier();
	135	}
	136
	137	// Wait for all threads in the wavefront to converge, this is a noop on SPIR-V.
	138	_DEFAULT_FN_ATTRS static __inline__ void __gpu_sync_lane(uint64_t __lane_mask) {
	139	}
	140
	141	// Shuffles the the lanes inside the wavefront according to the given index.
	142	_DEFAULT_FN_ATTRS static __inline__ uint32_t
	143	__gpu_shuffle_idx_u32(uint64_t __lane_mask, uint32_t __idx, uint32_t __x,
	144	uint32_t __width) {
	145	uint32_t __lane = __idx + (__gpu_lane_id() & ~(__width - 1));
	146	return __builtin_spirv_subgroup_shuffle(__x, __lane);
	147	}
	148
	149	// Returns a bitmask marking all lanes that have the same value of __x.
	150	_DEFAULT_FN_ATTRS static __inline__ uint64_t
	151	__gpu_match_any_u32(uint64_t __lane_mask, uint32_t __x) {
	152	return __gpu_match_any_u32_impl(__lane_mask, __x);
	153	}
	154
	155	// Returns a bitmask marking all lanes that have the same value of __x.
	156	_DEFAULT_FN_ATTRS static __inline__ uint64_t
	157	__gpu_match_any_u64(uint64_t __lane_mask, uint64_t __x) {
	158	return __gpu_match_any_u64_impl(__lane_mask, __x);
	159	}
	160
	161	// Returns the current lane mask if every lane contains __x.
	162	_DEFAULT_FN_ATTRS static __inline__ uint64_t
	163	__gpu_match_all_u32(uint64_t __lane_mask, uint32_t __x) {
	164	return __gpu_match_all_u32_impl(__lane_mask, __x);
	165	}
	166
	167	// Returns the current lane mask if every lane contains __x.
	168	_DEFAULT_FN_ATTRS static __inline__ uint64_t
	169	__gpu_match_all_u64(uint64_t __lane_mask, uint64_t __x) {
	170	return __gpu_match_all_u64_impl(__lane_mask, __x);
	171	}
	172
	173	// SPIR-V does not expose this, always return false.
	174	_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_local(void *ptr) {
	175	return 0;
	176	}
	177
	178	// SPIR-V does not expose this, always return false.
	179	_DEFAULT_FN_ATTRS static __inline__ bool __gpu_is_ptr_private(void *ptr) {
	180	return 0;
	181	}
	182
	183	// SPIR-V only supports 'OpTerminateInvocation' in fragment shaders.
	184	_DEFAULT_FN_ATTRS [[noreturn]] static __inline__ void __gpu_exit(void) {
	185	__builtin_trap();
	186	}
	187
	188	// This is a no-op as SPIR-V does not support it.
	189	_DEFAULT_FN_ATTRS static __inline__ void __gpu_thread_suspend(void) {}
	190
	191	_Pragma("omp end declare variant");
	192	_Pragma("omp end declare target");
	193
	194	#endif // __SPIRVINTRIN_H

lib/include/stddefer.h created+19

...	...	@@ -0,0 +1,19 @@
	1	/*===---- stddefer.h - Standard header for 'defer' -------------------------===
	2	*
	3	* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	* See https://llvm.org/LICENSE.txt for license information.
	5	* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	*
	7	*===-----------------------------------------------------------------------===
	8	*/
	9
	10	#ifndef __CLANG_STDDEFER_H
	11	#define __CLANG_STDDEFER_H
	12
	13	/* Provide 'defer' if '_Defer' is supported. */
	14	#ifdef __STDC_DEFER_TS25755__
	15	#define __STDC_VERSION_STDDEFER_H__ 202602L
	16	#define defer _Defer
	17	#endif
	18
	19	#endif /* __CLANG_STDDEFER_H */

lib/include/tmmintrin.h+116-147

...	...	@@ -17,21 +17,21 @@
17	17	#include <pmmintrin.h>
18	18
19	19	/* Define the default attributes for the functions in this file. */
20		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
21		#define __DEFAULT_FN_ATTRS \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("ssse3,no-evex512"), __min_vector_width__(128)))
24		#else
25	20	#define __DEFAULT_FN_ATTRS \
26	21	__attribute__((__always_inline__, __nodebug__, __target__("ssse3"), \
27	22	__min_vector_width__(128)))
28		#endif
29	23
30	24	#define __trunc64(x) \
31	25	(__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
32		#define __anyext128(x) \
	26	#define __zext128(x) \
33	27	(__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0, \
34		1, -1, -1)
	28	1, 2, 3)
	29
	30	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	31	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	32	#else
	33	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	34	#endif
35	35
36	36	/// Computes the absolute value of each of the packed 8-bit signed
37	37	/// integers in the source operand and stores the 8-bit unsigned integer
...	...	@@ -45,9 +45,7 @@
45	45	/// A 64-bit vector of [8 x i8].
46	46	/// \returns A 64-bit integer vector containing the absolute values of the
47	47	/// elements in the operand.
48		static __inline__ __m64 __DEFAULT_FN_ATTRS
49		_mm_abs_pi8(__m64 __a)
50		{
	48	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_abs_pi8(__m64 __a) {
51	49	return (__m64)__builtin_elementwise_abs((__v8qs)__a);
52	50	}
53	51
...	...	@@ -63,10 +61,9 @@ _mm_abs_pi8(__m64 __a)
63	61	/// A 128-bit vector of [16 x i8].
64	62	/// \returns A 128-bit integer vector containing the absolute values of the
65	63	/// elements in the operand.
66		static __inline__ __m128i __DEFAULT_FN_ATTRS
67		_mm_abs_epi8(__m128i __a)
68		{
69		return (__m128i)__builtin_elementwise_abs((__v16qs)__a);
	64	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	65	_mm_abs_epi8(__m128i __a) {
	66	return (__m128i)__builtin_elementwise_abs((__v16qs)__a);
70	67	}
71	68
72	69	/// Computes the absolute value of each of the packed 16-bit signed
...	...	@@ -81,10 +78,8 @@ _mm_abs_epi8(__m128i __a)
81	78	/// A 64-bit vector of [4 x i16].
82	79	/// \returns A 64-bit integer vector containing the absolute values of the
83	80	/// elements in the operand.
84		static __inline__ __m64 __DEFAULT_FN_ATTRS
85		_mm_abs_pi16(__m64 __a)
86		{
87		return (__m64)__builtin_elementwise_abs((__v4hi)__a);
	81	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_abs_pi16(__m64 __a) {
	82	return (__m64)__builtin_elementwise_abs((__v4hi)__a);
88	83	}
89	84
90	85	/// Computes the absolute value of each of the packed 16-bit signed
...	...	@@ -99,10 +94,9 @@ _mm_abs_pi16(__m64 __a)
99	94	/// A 128-bit vector of [8 x i16].
100	95	/// \returns A 128-bit integer vector containing the absolute values of the
101	96	/// elements in the operand.
102		static __inline__ __m128i __DEFAULT_FN_ATTRS
103		_mm_abs_epi16(__m128i __a)
104		{
105		return (__m128i)__builtin_elementwise_abs((__v8hi)__a);
	97	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	98	_mm_abs_epi16(__m128i __a) {
	99	return (__m128i)__builtin_elementwise_abs((__v8hi)__a);
106	100	}
107	101
108	102	/// Computes the absolute value of each of the packed 32-bit signed
...	...	@@ -117,10 +111,8 @@ _mm_abs_epi16(__m128i __a)
117	111	/// A 64-bit vector of [2 x i32].
118	112	/// \returns A 64-bit integer vector containing the absolute values of the
119	113	/// elements in the operand.
120		static __inline__ __m64 __DEFAULT_FN_ATTRS
121		_mm_abs_pi32(__m64 __a)
122		{
123		return (__m64)__builtin_elementwise_abs((__v2si)__a);
	114	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_abs_pi32(__m64 __a) {
	115	return (__m64)__builtin_elementwise_abs((__v2si)__a);
124	116	}
125	117
126	118	/// Computes the absolute value of each of the packed 32-bit signed
...	...	@@ -135,10 +127,9 @@ _mm_abs_pi32(__m64 __a)
135	127	/// A 128-bit vector of [4 x i32].
136	128	/// \returns A 128-bit integer vector containing the absolute values of the
137	129	/// elements in the operand.
138		static __inline__ __m128i __DEFAULT_FN_ATTRS
139		_mm_abs_epi32(__m128i __a)
140		{
141		return (__m128i)__builtin_elementwise_abs((__v4si)__a);
	130	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	131	_mm_abs_epi32(__m128i __a) {
	132	return (__m128i)__builtin_elementwise_abs((__v4si)__a);
142	133	}
143	134
144	135	/// Concatenates the two 128-bit integer vector operands, and
...	...	@@ -184,11 +175,12 @@ _mm_abs_epi32(__m128i __a)
184	175	/// An immediate operand specifying how many bytes to right-shift the result.
185	176	/// \returns A 64-bit integer vector containing the concatenated right-shifted
186	177	/// value.
187		#define _mm_alignr_pi8(a, b, n) \
188		((__m64)__builtin_shufflevector( \
189		__builtin_ia32_psrldqi128_byteshift( \
190		__builtin_shufflevector((__v1di)(a), (__v1di)(b), 1, 0), \
191		(n)), __extension__ (__v2di){}, 0))
	178	#define _mm_alignr_pi8(a, b, n) \
	179	((__m64)__builtin_shufflevector( \
	180	(__v2di)__builtin_ia32_psrldqi128_byteshift( \
	181	(__v16qi)__builtin_shufflevector((__v1di)(a), (__v1di)(b), 1, 0), \
	182	(n)), \
	183	__extension__(__v2di){}, 0))
192	184
193	185	/// Horizontally adds the adjacent pairs of values contained in 2 packed
194	186	/// 128-bit vectors of [8 x i16].
...	...	@@ -207,10 +199,9 @@ _mm_abs_epi32(__m128i __a)
207	199	/// destination.
208	200	/// \returns A 128-bit vector of [8 x i16] containing the horizontal sums of
209	201	/// both operands.
210		static __inline__ __m128i __DEFAULT_FN_ATTRS
211		_mm_hadd_epi16(__m128i __a, __m128i __b)
212		{
213		return (__m128i)__builtin_ia32_phaddw128((__v8hi)__a, (__v8hi)__b);
	202	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	203	_mm_hadd_epi16(__m128i __a, __m128i __b) {
	204	return (__m128i)__builtin_ia32_phaddw128((__v8hi)__a, (__v8hi)__b);
214	205	}
215	206
216	207	/// Horizontally adds the adjacent pairs of values contained in 2 packed
...	...	@@ -230,10 +221,9 @@ _mm_hadd_epi16(__m128i __a, __m128i __b)
230	221	/// destination.
231	222	/// \returns A 128-bit vector of [4 x i32] containing the horizontal sums of
232	223	/// both operands.
233		static __inline__ __m128i __DEFAULT_FN_ATTRS
234		_mm_hadd_epi32(__m128i __a, __m128i __b)
235		{
236		return (__m128i)__builtin_ia32_phaddd128((__v4si)__a, (__v4si)__b);
	224	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	225	_mm_hadd_epi32(__m128i __a, __m128i __b) {
	226	return (__m128i)__builtin_ia32_phaddd128((__v4si)__a, (__v4si)__b);
237	227	}
238	228
239	229	/// Horizontally adds the adjacent pairs of values contained in 2 packed
...	...	@@ -253,11 +243,10 @@ _mm_hadd_epi32(__m128i __a, __m128i __b)
253	243	/// destination.
254	244	/// \returns A 64-bit vector of [4 x i16] containing the horizontal sums of both
255	245	/// operands.
256		static __inline__ __m64 __DEFAULT_FN_ATTRS
257		_mm_hadd_pi16(__m64 __a, __m64 __b)
258		{
259		return __trunc64(__builtin_ia32_phaddw128(
260		(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
	246	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hadd_pi16(__m64 __a,
	247	__m64 __b) {
	248	return __trunc64(__builtin_ia32_phaddw128(
	249	(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
261	250	}
262	251
263	252	/// Horizontally adds the adjacent pairs of values contained in 2 packed
...	...	@@ -277,11 +266,10 @@ _mm_hadd_pi16(__m64 __a, __m64 __b)
277	266	/// destination.
278	267	/// \returns A 64-bit vector of [2 x i32] containing the horizontal sums of both
279	268	/// operands.
280		static __inline__ __m64 __DEFAULT_FN_ATTRS
281		_mm_hadd_pi32(__m64 __a, __m64 __b)
282		{
283		return __trunc64(__builtin_ia32_phaddd128(
284		(__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
	269	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hadd_pi32(__m64 __a,
	270	__m64 __b) {
	271	return __trunc64(__builtin_ia32_phaddd128(
	272	(__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
285	273	}
286	274
287	275	/// Horizontally adds, with saturation, the adjacent pairs of values contained
...	...	@@ -304,10 +292,9 @@ _mm_hadd_pi32(__m64 __a, __m64 __b)
304	292	/// destination.
305	293	/// \returns A 128-bit vector of [8 x i16] containing the horizontal saturated
306	294	/// sums of both operands.
307		static __inline__ __m128i __DEFAULT_FN_ATTRS
308		_mm_hadds_epi16(__m128i __a, __m128i __b)
309		{
310		return (__m128i)__builtin_ia32_phaddsw128((__v8hi)__a, (__v8hi)__b);
	295	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	296	_mm_hadds_epi16(__m128i __a, __m128i __b) {
	297	return (__m128i)__builtin_ia32_phaddsw128((__v8hi)__a, (__v8hi)__b);
311	298	}
312	299
313	300	/// Horizontally adds, with saturation, the adjacent pairs of values contained
...	...	@@ -330,11 +317,10 @@ _mm_hadds_epi16(__m128i __a, __m128i __b)
330	317	/// destination.
331	318	/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
332	319	/// sums of both operands.
333		static __inline__ __m64 __DEFAULT_FN_ATTRS
334		_mm_hadds_pi16(__m64 __a, __m64 __b)
335		{
336		return __trunc64(__builtin_ia32_phaddsw128(
337		(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
	320	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hadds_pi16(__m64 __a,
	321	__m64 __b) {
	322	return __trunc64(__builtin_ia32_phaddsw128(
	323	(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
338	324	}
339	325
340	326	/// Horizontally subtracts the adjacent pairs of values contained in 2
...	...	@@ -354,10 +340,9 @@ _mm_hadds_pi16(__m64 __a, __m64 __b)
354	340	/// the destination.
355	341	/// \returns A 128-bit vector of [8 x i16] containing the horizontal differences
356	342	/// of both operands.
357		static __inline__ __m128i __DEFAULT_FN_ATTRS
358		_mm_hsub_epi16(__m128i __a, __m128i __b)
359		{
360		return (__m128i)__builtin_ia32_phsubw128((__v8hi)__a, (__v8hi)__b);
	343	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	344	_mm_hsub_epi16(__m128i __a, __m128i __b) {
	345	return (__m128i)__builtin_ia32_phsubw128((__v8hi)__a, (__v8hi)__b);
361	346	}
362	347
363	348	/// Horizontally subtracts the adjacent pairs of values contained in 2
...	...	@@ -377,10 +362,9 @@ _mm_hsub_epi16(__m128i __a, __m128i __b)
377	362	/// the destination.
378	363	/// \returns A 128-bit vector of [4 x i32] containing the horizontal differences
379	364	/// of both operands.
380		static __inline__ __m128i __DEFAULT_FN_ATTRS
381		_mm_hsub_epi32(__m128i __a, __m128i __b)
382		{
383		return (__m128i)__builtin_ia32_phsubd128((__v4si)__a, (__v4si)__b);
	365	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	366	_mm_hsub_epi32(__m128i __a, __m128i __b) {
	367	return (__m128i)__builtin_ia32_phsubd128((__v4si)__a, (__v4si)__b);
384	368	}
385	369
386	370	/// Horizontally subtracts the adjacent pairs of values contained in 2
...	...	@@ -400,11 +384,10 @@ _mm_hsub_epi32(__m128i __a, __m128i __b)
400	384	/// the destination.
401	385	/// \returns A 64-bit vector of [4 x i16] containing the horizontal differences
402	386	/// of both operands.
403		static __inline__ __m64 __DEFAULT_FN_ATTRS
404		_mm_hsub_pi16(__m64 __a, __m64 __b)
405		{
406		return __trunc64(__builtin_ia32_phsubw128(
407		(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
	387	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hsub_pi16(__m64 __a,
	388	__m64 __b) {
	389	return __trunc64(__builtin_ia32_phsubw128(
	390	(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
408	391	}
409	392
410	393	/// Horizontally subtracts the adjacent pairs of values contained in 2
...	...	@@ -424,11 +407,10 @@ _mm_hsub_pi16(__m64 __a, __m64 __b)
424	407	/// the destination.
425	408	/// \returns A 64-bit vector of [2 x i32] containing the horizontal differences
426	409	/// of both operands.
427		static __inline__ __m64 __DEFAULT_FN_ATTRS
428		_mm_hsub_pi32(__m64 __a, __m64 __b)
429		{
430		return __trunc64(__builtin_ia32_phsubd128(
431		(__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
	410	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hsub_pi32(__m64 __a,
	411	__m64 __b) {
	412	return __trunc64(__builtin_ia32_phsubd128(
	413	(__v4si)__builtin_shufflevector(__a, __b, 0, 1), (__v4si){}));
432	414	}
433	415
434	416	/// Horizontally subtracts, with saturation, the adjacent pairs of values
...	...	@@ -451,10 +433,9 @@ _mm_hsub_pi32(__m64 __a, __m64 __b)
451	433	/// the destination.
452	434	/// \returns A 128-bit vector of [8 x i16] containing the horizontal saturated
453	435	/// differences of both operands.
454		static __inline__ __m128i __DEFAULT_FN_ATTRS
455		_mm_hsubs_epi16(__m128i __a, __m128i __b)
456		{
457		return (__m128i)__builtin_ia32_phsubsw128((__v8hi)__a, (__v8hi)__b);
	436	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	437	_mm_hsubs_epi16(__m128i __a, __m128i __b) {
	438	return (__m128i)__builtin_ia32_phsubsw128((__v8hi)__a, (__v8hi)__b);
458	439	}
459	440
460	441	/// Horizontally subtracts, with saturation, the adjacent pairs of values
...	...	@@ -477,11 +458,10 @@ _mm_hsubs_epi16(__m128i __a, __m128i __b)
477	458	/// the destination.
478	459	/// \returns A 64-bit vector of [4 x i16] containing the horizontal saturated
479	460	/// differences of both operands.
480		static __inline__ __m64 __DEFAULT_FN_ATTRS
481		_mm_hsubs_pi16(__m64 __a, __m64 __b)
482		{
483		return __trunc64(__builtin_ia32_phsubsw128(
484		(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
	461	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_hsubs_pi16(__m64 __a,
	462	__m64 __b) {
	463	return __trunc64(__builtin_ia32_phsubsw128(
	464	(__v8hi)__builtin_shufflevector(__a, __b, 0, 1), (__v8hi){}));
485	465	}
486	466
487	467	/// Multiplies corresponding pairs of packed 8-bit unsigned integer
...	...	@@ -512,10 +492,9 @@ _mm_hsubs_pi16(__m64 __a, __m64 __b)
512	492	/// \a R5 := (\a __a10 * \a __b10) + (\a __a11 * \a __b11) \n
513	493	/// \a R6 := (\a __a12 * \a __b12) + (\a __a13 * \a __b13) \n
514	494	/// \a R7 := (\a __a14 * \a __b14) + (\a __a15 * \a __b15)
515		static __inline__ __m128i __DEFAULT_FN_ATTRS
516		_mm_maddubs_epi16(__m128i __a, __m128i __b)
517		{
518		return (__m128i)__builtin_ia32_pmaddubsw128((__v16qi)__a, (__v16qi)__b);
	495	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	496	_mm_maddubs_epi16(__m128i __a, __m128i __b) {
	497	return (__m128i)__builtin_ia32_pmaddubsw128((__v16qi)__a, (__v16qi)__b);
519	498	}
520	499
521	500	/// Multiplies corresponding pairs of packed 8-bit unsigned integer
...	...	@@ -542,11 +521,10 @@ _mm_maddubs_epi16(__m128i __a, __m128i __b)
542	521	/// \a R1 := (\a __a2 * \a __b2) + (\a __a3 * \a __b3) \n
543	522	/// \a R2 := (\a __a4 * \a __b4) + (\a __a5 * \a __b5) \n
544	523	/// \a R3 := (\a __a6 * \a __b6) + (\a __a7 * \a __b7)
545		static __inline__ __m64 __DEFAULT_FN_ATTRS
546		_mm_maddubs_pi16(__m64 __a, __m64 __b)
547		{
548		return __trunc64(__builtin_ia32_pmaddubsw128((__v16qi)__anyext128(__a),
549		(__v16qi)__anyext128(__b)));
	524	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR
	525	_mm_maddubs_pi16(__m64 __a, __m64 __b) {
	526	return __trunc64(__builtin_ia32_pmaddubsw128((__v16qi)__zext128(__a),
	527	(__v16qi)__zext128(__b)));
550	528	}
551	529
552	530	/// Multiplies packed 16-bit signed integer values, truncates the 32-bit
...	...	@@ -563,10 +541,9 @@ _mm_maddubs_pi16(__m64 __a, __m64 __b)
563	541	/// A 128-bit vector of [8 x i16] containing one of the source operands.
564	542	/// \returns A 128-bit vector of [8 x i16] containing the rounded and scaled
565	543	/// products of both operands.
566		static __inline__ __m128i __DEFAULT_FN_ATTRS
567		_mm_mulhrs_epi16(__m128i __a, __m128i __b)
568		{
569		return (__m128i)__builtin_ia32_pmulhrsw128((__v8hi)__a, (__v8hi)__b);
	544	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	545	_mm_mulhrs_epi16(__m128i __a, __m128i __b) {
	546	return (__m128i)__builtin_ia32_pmulhrsw128((__v8hi)__a, (__v8hi)__b);
570	547	}
571	548
572	549	/// Multiplies packed 16-bit signed integer values, truncates the 32-bit
...	...	@@ -583,11 +560,10 @@ _mm_mulhrs_epi16(__m128i __a, __m128i __b)
583	560	/// A 64-bit vector of [4 x i16] containing one of the source operands.
584	561	/// \returns A 64-bit vector of [4 x i16] containing the rounded and scaled
585	562	/// products of both operands.
586		static __inline__ __m64 __DEFAULT_FN_ATTRS
587		_mm_mulhrs_pi16(__m64 __a, __m64 __b)
588		{
589		return __trunc64(__builtin_ia32_pmulhrsw128((__v8hi)__anyext128(__a),
590		(__v8hi)__anyext128(__b)));
	563	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR
	564	_mm_mulhrs_pi16(__m64 __a, __m64 __b) {
	565	return __trunc64(__builtin_ia32_pmulhrsw128((__v8hi)__zext128(__a),
	566	(__v8hi)__zext128(__b)));
591	567	}
592	568
593	569	/// Copies the 8-bit integers from a 128-bit integer vector to the
...	...	@@ -610,10 +586,9 @@ _mm_mulhrs_pi16(__m64 __a, __m64 __b)
610	586	/// Bits [6:4] Reserved. \n
611	587	/// Bits [3:0] select the source byte to be copied.
612	588	/// \returns A 128-bit integer vector containing the copied or cleared values.
613		static __inline__ __m128i __DEFAULT_FN_ATTRS
614		_mm_shuffle_epi8(__m128i __a, __m128i __b)
615		{
616		return (__m128i)__builtin_ia32_pshufb128((__v16qi)__a, (__v16qi)__b);
	589	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	590	_mm_shuffle_epi8(__m128i __a, __m128i __b) {
	591	return (__m128i)__builtin_ia32_pshufb128((__v16qi)__a, (__v16qi)__b);
617	592	}
618	593
619	594	/// Copies the 8-bit integers from a 64-bit integer vector to the
...	...	@@ -635,13 +610,12 @@ _mm_shuffle_epi8(__m128i __a, __m128i __b)
635	610	/// destination. \n
636	611	/// Bits [2:0] select the source byte to be copied.
637	612	/// \returns A 64-bit integer vector containing the copied or cleared values.
638		static __inline__ __m64 __DEFAULT_FN_ATTRS
639		_mm_shuffle_pi8(__m64 __a, __m64 __b)
640		{
641		return __trunc64(__builtin_ia32_pshufb128(
642		(__v16qi)__builtin_shufflevector(
643		(__v2si)(__a), __extension__ (__v2si){}, 0, 1, 0, 1),
644		(__v16qi)__anyext128(__b)));
	613	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR
	614	_mm_shuffle_pi8(__m64 __a, __m64 __b) {
	615	return __trunc64(__builtin_ia32_pshufb128(
	616	(__v16qi)__builtin_shufflevector((__v2si)(__a), __extension__(__v2si){},
	617	0, 1, 0, 1),
	618	(__v16qi)__zext128(__b)));
645	619	}
646	620
647	621	/// For each 8-bit integer in the first source operand, perform one of
...	...	@@ -664,10 +638,9 @@ _mm_shuffle_pi8(__m64 __a, __m64 __b)
664	638	/// A 128-bit integer vector containing control bytes corresponding to
665	639	/// positions in the destination.
666	640	/// \returns A 128-bit integer vector containing the resultant values.
667		static __inline__ __m128i __DEFAULT_FN_ATTRS
668		_mm_sign_epi8(__m128i __a, __m128i __b)
669		{
670		return (__m128i)__builtin_ia32_psignb128((__v16qi)__a, (__v16qi)__b);
	641	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	642	_mm_sign_epi8(__m128i __a, __m128i __b) {
	643	return (__m128i)__builtin_ia32_psignb128((__v16qi)__a, (__v16qi)__b);
671	644	}
672	645
673	646	/// For each 16-bit integer in the first source operand, perform one of
...	...	@@ -690,10 +663,9 @@ _mm_sign_epi8(__m128i __a, __m128i __b)
690	663	/// A 128-bit integer vector containing control words corresponding to
691	664	/// positions in the destination.
692	665	/// \returns A 128-bit integer vector containing the resultant values.
693		static __inline__ __m128i __DEFAULT_FN_ATTRS
694		_mm_sign_epi16(__m128i __a, __m128i __b)
695		{
696		return (__m128i)__builtin_ia32_psignw128((__v8hi)__a, (__v8hi)__b);
	666	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	667	_mm_sign_epi16(__m128i __a, __m128i __b) {
	668	return (__m128i)__builtin_ia32_psignw128((__v8hi)__a, (__v8hi)__b);
697	669	}
698	670
699	671	/// For each 32-bit integer in the first source operand, perform one of
...	...	@@ -716,10 +688,9 @@ _mm_sign_epi16(__m128i __a, __m128i __b)
716	688	/// A 128-bit integer vector containing control doublewords corresponding to
717	689	/// positions in the destination.
718	690	/// \returns A 128-bit integer vector containing the resultant values.
719		static __inline__ __m128i __DEFAULT_FN_ATTRS
720		_mm_sign_epi32(__m128i __a, __m128i __b)
721		{
722		return (__m128i)__builtin_ia32_psignd128((__v4si)__a, (__v4si)__b);
	691	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
	692	_mm_sign_epi32(__m128i __a, __m128i __b) {
	693	return (__m128i)__builtin_ia32_psignd128((__v4si)__a, (__v4si)__b);
723	694	}
724	695
725	696	/// For each 8-bit integer in the first source operand, perform one of
...	...	@@ -742,11 +713,10 @@ _mm_sign_epi32(__m128i __a, __m128i __b)
742	713	/// A 64-bit integer vector containing control bytes corresponding to
743	714	/// positions in the destination.
744	715	/// \returns A 64-bit integer vector containing the resultant values.
745		static __inline__ __m64 __DEFAULT_FN_ATTRS
746		_mm_sign_pi8(__m64 __a, __m64 __b)
747		{
748		return __trunc64(__builtin_ia32_psignb128((__v16qi)__anyext128(__a),
749		(__v16qi)__anyext128(__b)));
	716	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sign_pi8(__m64 __a,
	717	__m64 __b) {
	718	return __trunc64(__builtin_ia32_psignb128((__v16qi)__zext128(__a),
	719	(__v16qi)__zext128(__b)));
750	720	}
751	721
752	722	/// For each 16-bit integer in the first source operand, perform one of
...	...	@@ -769,11 +739,10 @@ _mm_sign_pi8(__m64 __a, __m64 __b)
769	739	/// A 64-bit integer vector containing control words corresponding to
770	740	/// positions in the destination.
771	741	/// \returns A 64-bit integer vector containing the resultant values.
772		static __inline__ __m64 __DEFAULT_FN_ATTRS
773		_mm_sign_pi16(__m64 __a, __m64 __b)
774		{
775		return __trunc64(__builtin_ia32_psignw128((__v8hi)__anyext128(__a),
776		(__v8hi)__anyext128(__b)));
	742	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sign_pi16(__m64 __a,
	743	__m64 __b) {
	744	return __trunc64(
	745	__builtin_ia32_psignw128((__v8hi)__zext128(__a), (__v8hi)__zext128(__b)));
777	746	}
778	747
779	748	/// For each 32-bit integer in the first source operand, perform one of
...	...	@@ -796,15 +765,15 @@ _mm_sign_pi16(__m64 __a, __m64 __b)
796	765	/// A 64-bit integer vector containing two control doublewords corresponding
797	766	/// to positions in the destination.
798	767	/// \returns A 64-bit integer vector containing the resultant values.
799		static __inline__ __m64 __DEFAULT_FN_ATTRS
800		_mm_sign_pi32(__m64 __a, __m64 __b)
801		{
802		return __trunc64(__builtin_ia32_psignd128((__v4si)__anyext128(__a),
803		(__v4si)__anyext128(__b)));
	768	static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sign_pi32(__m64 __a,
	769	__m64 __b) {
	770	return __trunc64(
	771	__builtin_ia32_psignd128((__v4si)__zext128(__a), (__v4si)__zext128(__b)));
804	772	}
805	773
806		#undef __anyext128
	774	#undef __zext128
807	775	#undef __trunc64
808	776	#undef __DEFAULT_FN_ATTRS
	777	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
809	778
810	779	#endif /* __TMMINTRIN_H */

lib/include/vaesintrin.h+1-2

...	...	@@ -19,8 +19,7 @@
19	19
20	20	/* Default attributes for ZMM forms. */
21	21	#define __DEFAULT_FN_ATTRS_F \
22		__attribute__((__always_inline__, __nodebug__, \
23		__target__("avx512f,evex512,vaes"), \
	22	__attribute__((__always_inline__, __nodebug__, __target__("avx512f,vaes"), \
24	23	__min_vector_width__(512)))
25	24
26	25	static __inline__ __m256i __DEFAULT_FN_ATTRS

lib/include/xmmintrin.h+44-70

...	...	@@ -16,7 +16,6 @@
16	16
17	17	#include <mmintrin.h>
18	18
19		typedef int __v4si __attribute__((__vector_size__(16)));
20	19	typedef float __v4sf __attribute__((__vector_size__(16)));
21	20	typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
22	21
...	...	@@ -24,6 +23,8 @@ typedef float __m128_u __attribute__((__vector_size__(16), __aligned__(1)));
24	23
25	24	/* Unsigned types */
26	25	typedef unsigned int __v4su __attribute__((__vector_size__(16)));
	26	typedef unsigned short __v8hu __attribute__((__vector_size__(16)));
	27	typedef unsigned char __v16qu __attribute__((__vector_size__(16)));
27	28
28	29	/* This header should only be included in a hosted environment as it depends on
29	30	* a standard library to provide allocation routines. */
...	...	@@ -32,21 +33,12 @@ typedef unsigned int __v4su __attribute__((__vector_size__(16)));
32	33	#endif
33	34
34	35	/* Define the default attributes for the functions in this file. */
35		#if defined(__EVEX512__) && !defined(__AVX10_1_512__)
36		#define __DEFAULT_FN_ATTRS \
37		__attribute__((__always_inline__, __nodebug__, __target__("sse,no-evex512"), \
38		__min_vector_width__(128)))
39		#define __DEFAULT_FN_ATTRS_SSE2 \
40		__attribute__((__always_inline__, __nodebug__, \
41		__target__("sse2,no-evex512"), __min_vector_width__(128)))
42		#else
43	36	#define __DEFAULT_FN_ATTRS \
44	37	__attribute__((__always_inline__, __nodebug__, __target__("sse"), \
45	38	__min_vector_width__(128)))
46	39	#define __DEFAULT_FN_ATTRS_SSE2 \
47	40	__attribute__((__always_inline__, __nodebug__, __target__("sse2"), \
48	41	__min_vector_width__(128)))
49		#endif
50	42
51	43	#if defined(__cplusplus) && (__cplusplus >= 201103L)
52	44	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
...	...	@@ -239,10 +231,9 @@ _mm_div_ps(__m128 __a, __m128 __b) {
239	231	/// used in the calculation.
240	232	/// \returns A 128-bit vector of [4 x float] containing the square root of the
241	233	/// value in the low-order bits of the operand.
242		static __inline__ __m128 __DEFAULT_FN_ATTRS
243		_mm_sqrt_ss(__m128 __a)
244		{
245		return (__m128)__builtin_ia32_sqrtss((__v4sf)__a);
	234	static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_sqrt_ss(__m128 __a) {
	235	__a[0] = __builtin_elementwise_sqrt(__a[0]);
	236	return __a;
246	237	}
247	238
248	239	/// Calculates the square roots of the values stored in a 128-bit vector
...	...	@@ -256,10 +247,8 @@ _mm_sqrt_ss(__m128 __a)
256	247	/// A 128-bit vector of [4 x float].
257	248	/// \returns A 128-bit vector of [4 x float] containing the square roots of the
258	249	/// values in the operand.
259		static __inline__ __m128 __DEFAULT_FN_ATTRS
260		_mm_sqrt_ps(__m128 __a)
261		{
262		return __builtin_ia32_sqrtps((__v4sf)__a);
	250	static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_sqrt_ps(__m128 __a) {
	251	return __builtin_elementwise_sqrt(__a);
263	252	}
264	253
265	254	/// Calculates the approximate reciprocal of the value stored in the
...	...	@@ -352,9 +341,7 @@ _mm_rsqrt_ps(__m128 __a)
352	341	/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
353	342	/// minimum value between both operands. The upper 96 bits are copied from
354	343	/// the upper 96 bits of the first source operand.
355		static __inline__ __m128 __DEFAULT_FN_ATTRS
356		_mm_min_ss(__m128 __a, __m128 __b)
357		{
	344	static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_min_ss(__m128 __a, __m128 __b) {
358	345	return __builtin_ia32_minss((__v4sf)__a, (__v4sf)__b);
359	346	}
360	347
...	...	@@ -373,9 +360,8 @@ _mm_min_ss(__m128 __a, __m128 __b)
373	360	/// A 128-bit vector of [4 x float] containing one of the operands.
374	361	/// \returns A 128-bit vector of [4 x float] containing the minimum values
375	362	/// between both operands.
376		static __inline__ __m128 __DEFAULT_FN_ATTRS
377		_mm_min_ps(__m128 __a, __m128 __b)
378		{
	363	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_min_ps(__m128 __a,
	364	__m128 __b) {
379	365	return __builtin_ia32_minps((__v4sf)__a, (__v4sf)__b);
380	366	}
381	367
...	...	@@ -398,9 +384,7 @@ _mm_min_ps(__m128 __a, __m128 __b)
398	384	/// \returns A 128-bit vector of [4 x float] whose lower 32 bits contain the
399	385	/// maximum value between both operands. The upper 96 bits are copied from
400	386	/// the upper 96 bits of the first source operand.
401		static __inline__ __m128 __DEFAULT_FN_ATTRS
402		_mm_max_ss(__m128 __a, __m128 __b)
403		{
	387	static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_max_ss(__m128 __a, __m128 __b) {
404	388	return __builtin_ia32_maxss((__v4sf)__a, (__v4sf)__b);
405	389	}
406	390
...	...	@@ -419,9 +403,8 @@ _mm_max_ss(__m128 __a, __m128 __b)
419	403	/// A 128-bit vector of [4 x float] containing one of the operands.
420	404	/// \returns A 128-bit vector of [4 x float] containing the maximum values
421	405	/// between both operands.
422		static __inline__ __m128 __DEFAULT_FN_ATTRS
423		_mm_max_ps(__m128 __a, __m128 __b)
424		{
	406	static __inline__ __m128 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_max_ps(__m128 __a,
	407	__m128 __b) {
425	408	return __builtin_ia32_maxps((__v4sf)__a, (__v4sf)__b);
426	409	}
427	410
...	...	@@ -1688,7 +1671,7 @@ _mm_cvtsi64_ss(__m128 __a, long long __b) {
1688	1671	/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
1689	1672	/// converted value of the second operand. The upper 64 bits are copied from
1690	1673	/// the upper 64 bits of the first operand.
1691		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	1674	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1692	1675	_mm_cvtpi32_ps(__m128 __a, __m64 __b)
1693	1676	{
1694	1677	return (__m128)__builtin_shufflevector(
...	...	@@ -1714,7 +1697,7 @@ _mm_cvtpi32_ps(__m128 __a, __m64 __b)
1714	1697	/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
1715	1698	/// converted value from the second operand. The upper 64 bits are copied
1716	1699	/// from the upper 64 bits of the first operand.
1717		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	1700	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
1718	1701	_mm_cvt_pi2ps(__m128 __a, __m64 __b)
1719	1702	{
1720	1703	return _mm_cvtpi32_ps(__a, __b);
...	...	@@ -2353,9 +2336,8 @@ void _mm_sfence(void);
2353	2336	/// \param __b
2354	2337	/// A 64-bit integer vector containing one of the source operands.
2355	2338	/// \returns A 64-bit integer vector containing the comparison results.
2356		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2357		_mm_max_pi16(__m64 __a, __m64 __b)
2358		{
	2339	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2340	_mm_max_pi16(__m64 __a, __m64 __b) {
2359	2341	return (__m64)__builtin_elementwise_max((__v4hi)__a, (__v4hi)__b);
2360	2342	}
2361	2343
...	...	@@ -2372,9 +2354,8 @@ _mm_max_pi16(__m64 __a, __m64 __b)
2372	2354	/// \param __b
2373	2355	/// A 64-bit integer vector containing one of the source operands.
2374	2356	/// \returns A 64-bit integer vector containing the comparison results.
2375		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2376		_mm_max_pu8(__m64 __a, __m64 __b)
2377		{
	2357	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2358	_mm_max_pu8(__m64 __a, __m64 __b) {
2378	2359	return (__m64)__builtin_elementwise_max((__v8qu)__a, (__v8qu)__b);
2379	2360	}
2380	2361
...	...	@@ -2391,9 +2372,8 @@ _mm_max_pu8(__m64 __a, __m64 __b)
2391	2372	/// \param __b
2392	2373	/// A 64-bit integer vector containing one of the source operands.
2393	2374	/// \returns A 64-bit integer vector containing the comparison results.
2394		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2395		_mm_min_pi16(__m64 __a, __m64 __b)
2396		{
	2375	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2376	_mm_min_pi16(__m64 __a, __m64 __b) {
2397	2377	return (__m64)__builtin_elementwise_min((__v4hi)__a, (__v4hi)__b);
2398	2378	}
2399	2379
...	...	@@ -2410,9 +2390,8 @@ _mm_min_pi16(__m64 __a, __m64 __b)
2410	2390	/// \param __b
2411	2391	/// A 64-bit integer vector containing one of the source operands.
2412	2392	/// \returns A 64-bit integer vector containing the comparison results.
2413		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2414		_mm_min_pu8(__m64 __a, __m64 __b)
2415		{
	2393	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2394	_mm_min_pu8(__m64 __a, __m64 __b) {
2416	2395	return (__m64)__builtin_elementwise_min((__v8qu)__a, (__v8qu)__b);
2417	2396	}
2418	2397
...	...	@@ -2428,9 +2407,8 @@ _mm_min_pu8(__m64 __a, __m64 __b)
2428	2407	/// A 64-bit integer vector containing the values with bits to be extracted.
2429	2408	/// \returns The most significant bit from each 8-bit element in \a __a,
2430	2409	/// written to bits [7:0].
2431		static __inline__ int __DEFAULT_FN_ATTRS_SSE2
2432		_mm_movemask_pi8(__m64 __a)
2433		{
	2410	static __inline__ int __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2411	_mm_movemask_pi8(__m64 __a) {
2434	2412	return __builtin_ia32_pmovmskb128((__v16qi)__zext128(__a));
2435	2413	}
2436	2414
...	...	@@ -2447,11 +2425,11 @@ _mm_movemask_pi8(__m64 __a)
2447	2425	/// \param __b
2448	2426	/// A 64-bit integer vector containing one of the source operands.
2449	2427	/// \returns A 64-bit integer vector containing the products of both operands.
2450		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
	2428	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2451	2429	_mm_mulhi_pu16(__m64 __a, __m64 __b)
2452	2430	{
2453		return __trunc64(__builtin_ia32_pmulhuw128((__v8hi)__anyext128(__a),
2454		(__v8hi)__anyext128(__b)));
	2431	return __trunc64(__builtin_ia32_pmulhuw128((__v8hu)__zext128(__a),
	2432	(__v8hu)__zext128(__b)));
2455	2433	}
2456	2434
2457	2435	/// Shuffles the 4 16-bit integers from a 64-bit integer vector to the
...	...	@@ -2530,8 +2508,8 @@ _mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
2530	2508	// If there's a risk of spurious trap due to a 128-bit write, back up the
2531	2509	// pointer by 8 bytes and shift values in registers to match.
2532	2510	__p -= 8;
2533		__d128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__d128, 8);
2534		__n128 = __builtin_ia32_pslldqi128_byteshift((__v2di)__n128, 8);
	2511	__d128 = (__m128i)__builtin_ia32_pslldqi128_byteshift((__v16qi)__d128, 8);
	2512	__n128 = (__m128i)__builtin_ia32_pslldqi128_byteshift((__v16qi)__n128, 8);
2535	2513	}
2536	2514
2537	2515	__builtin_ia32_maskmovdqu((__v16qi)__d128, (__v16qi)__n128, __p);
...	...	@@ -2550,11 +2528,10 @@ _mm_maskmove_si64(__m64 __d, __m64 __n, char *__p)
2550	2528	/// \param __b
2551	2529	/// A 64-bit integer vector containing one of the source operands.
2552	2530	/// \returns A 64-bit integer vector containing the averages of both operands.
2553		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2554		_mm_avg_pu8(__m64 __a, __m64 __b)
2555		{
2556		return __trunc64(__builtin_ia32_pavgb128((__v16qi)__anyext128(__a),
2557		(__v16qi)__anyext128(__b)));
	2531	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2532	_mm_avg_pu8(__m64 __a, __m64 __b) {
	2533	return __trunc64(__builtin_ia32_pavgb128((__v16qu)__zext128(__a),
	2534	(__v16qu)__zext128(__b)));
2558	2535	}
2559	2536
2560	2537	/// Computes the rounded averages of the packed unsigned 16-bit integer
...	...	@@ -2570,11 +2547,10 @@ _mm_avg_pu8(__m64 __a, __m64 __b)
2570	2547	/// \param __b
2571	2548	/// A 64-bit integer vector containing one of the source operands.
2572	2549	/// \returns A 64-bit integer vector containing the averages of both operands.
2573		static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
2574		_mm_avg_pu16(__m64 __a, __m64 __b)
2575		{
2576		return __trunc64(__builtin_ia32_pavgw128((__v8hi)__anyext128(__a),
2577		(__v8hi)__anyext128(__b)));
	2550	static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
	2551	_mm_avg_pu16(__m64 __a, __m64 __b) {
	2552	return __trunc64(
	2553	__builtin_ia32_pavgw128((__v8hu)__zext128(__a), (__v8hu)__zext128(__b)));
2578	2554	}
2579	2555
2580	2556	/// Subtracts the corresponding 8-bit unsigned integer values of the two
...	...	@@ -2873,7 +2849,7 @@ _mm_movelh_ps(__m128 __a, __m128 __b) {
2873	2849	/// from the corresponding elements in this operand.
2874	2850	/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2875	2851	/// values from the operand.
2876		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	2852	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2877	2853	_mm_cvtpi16_ps(__m64 __a)
2878	2854	{
2879	2855	return __builtin_convertvector((__v4hi)__a, __v4sf);
...	...	@@ -2891,7 +2867,7 @@ _mm_cvtpi16_ps(__m64 __a)
2891	2867	/// destination are copied from the corresponding elements in this operand.
2892	2868	/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2893	2869	/// values from the operand.
2894		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	2870	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2895	2871	_mm_cvtpu16_ps(__m64 __a)
2896	2872	{
2897	2873	return __builtin_convertvector((__v4hu)__a, __v4sf);
...	...	@@ -2909,7 +2885,7 @@ _mm_cvtpu16_ps(__m64 __a)
2909	2885	/// from the corresponding lower 4 elements in this operand.
2910	2886	/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2911	2887	/// values from the operand.
2912		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	2888	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2913	2889	_mm_cvtpi8_ps(__m64 __a)
2914	2890	{
2915	2891	return __builtin_convertvector(
...	...	@@ -2930,7 +2906,7 @@ _mm_cvtpi8_ps(__m64 __a)
2930	2906	/// operand.
2931	2907	/// \returns A 128-bit vector of [4 x float] containing the copied and converted
2932	2908	/// values from the source operand.
2933		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	2909	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2934	2910	_mm_cvtpu8_ps(__m64 __a)
2935	2911	{
2936	2912	return __builtin_convertvector(
...	...	@@ -2954,7 +2930,7 @@ _mm_cvtpu8_ps(__m64 __a)
2954	2930	/// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
2955	2931	/// copied and converted values from the first operand. The upper 64 bits
2956	2932	/// contain the copied and converted values from the second operand.
2957		static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
	2933	static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
2958	2934	_mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
2959	2935	{
2960	2936	return __builtin_convertvector(
...	...	@@ -3029,9 +3005,7 @@ _mm_cvtps_pi8(__m128 __a)
3029	3005	/// \returns A 32-bit integer value. Bits [3:0] contain the sign bits from each
3030	3006	/// single-precision floating-point element of the parameter. Bits [31:4] are
3031	3007	/// set to zero.
3032		static __inline__ int __DEFAULT_FN_ATTRS
3033		_mm_movemask_ps(__m128 __a)
3034		{
	3008	static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR _mm_movemask_ps(__m128 __a) {
3035	3009	return __builtin_ia32_movmskps((__v4sf)__a);
3036	3010	}
3037	3011

lib/include/xopintrin.h+20-10

...	...	@@ -20,6 +20,14 @@
20	20	#define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xop"), __min_vector_width__(128)))
21	21	#define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("xop"), __min_vector_width__(256)))
22	22
	23	#if defined(__cplusplus) && (__cplusplus >= 201103L)
	24	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
	25	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
	26	#else
	27	#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
	28	#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
	29	#endif
	30
23	31	static __inline__ __m128i __DEFAULT_FN_ATTRS
24	32	_mm_maccs_epi16(__m128i __A, __m128i __B, __m128i __C)
25	33	{
...	...	@@ -182,13 +190,13 @@ _mm_hsubq_epi32(__m128i __A)
182	190	return (__m128i)__builtin_ia32_vphsubdq((__v4si)__A);
183	191	}
184	192
185		static __inline__ __m128i __DEFAULT_FN_ATTRS
	193	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
186	194	_mm_cmov_si128(__m128i __A, __m128i __B, __m128i __C)
187	195	{
188	196	return (__m128i)(((__v2du)__A & (__v2du)__C) \| ((__v2du)__B & ~(__v2du)__C));
189	197	}
190	198
191		static __inline__ __m256i __DEFAULT_FN_ATTRS256
	199	static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
192	200	_mm256_cmov_si256(__m256i __A, __m256i __B, __m256i __C)
193	201	{
194	202	return (__m256i)(((__v4du)__A & (__v4du)__C) \| ((__v4du)__B & ~(__v4du)__C));
...	...	@@ -200,28 +208,28 @@ _mm_perm_epi8(__m128i __A, __m128i __B, __m128i __C)
200	208	return (__m128i)__builtin_ia32_vpperm((__v16qi)__A, (__v16qi)__B, (__v16qi)__C);
201	209	}
202	210
203		static __inline__ __m128i __DEFAULT_FN_ATTRS
	211	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
204	212	_mm_rot_epi8(__m128i __A, __m128i __B)
205	213	{
206		return (__m128i)__builtin_ia32_vprotb((__v16qi)__A, (__v16qi)__B);
	214	return (__m128i)__builtin_elementwise_fshl((__v16qu)__A, (__v16qu)__A, (__v16qu)__B);
207	215	}
208	216
209		static __inline__ __m128i __DEFAULT_FN_ATTRS
	217	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
210	218	_mm_rot_epi16(__m128i __A, __m128i __B)
211	219	{
212		return (__m128i)__builtin_ia32_vprotw((__v8hi)__A, (__v8hi)__B);
	220	return (__m128i)__builtin_elementwise_fshl((__v8hu)__A, (__v8hu)__A, (__v8hu)__B);
213	221	}
214	222
215		static __inline__ __m128i __DEFAULT_FN_ATTRS
	223	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
216	224	_mm_rot_epi32(__m128i __A, __m128i __B)
217	225	{
218		return (__m128i)__builtin_ia32_vprotd((__v4si)__A, (__v4si)__B);
	226	return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__A, (__v4su)__B);
219	227	}
220	228
221		static __inline__ __m128i __DEFAULT_FN_ATTRS
	229	static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
222	230	_mm_rot_epi64(__m128i __A, __m128i __B)
223	231	{
224		return (__m128i)__builtin_ia32_vprotq((__v2di)__A, (__v2di)__B);
	232	return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__A, (__v2du)__B);
225	233	}
226	234
227	235	#define _mm_roti_epi8(A, N) \
...	...	@@ -766,5 +774,7 @@ _mm256_frcz_pd(__m256d __A)
766	774
767	775	#undef __DEFAULT_FN_ATTRS
768	776	#undef __DEFAULT_FN_ATTRS256
	777	#undef __DEFAULT_FN_ATTRS_CONSTEXPR
	778	#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
769	779
770	780	#endif /* __XOPINTRIN_H */

lib/libcxx/include/__algorithm/all_of.h+11-5

...	...	@@ -10,24 +10,28 @@
10	10	#ifndef _LIBCPP___ALGORITHM_ALL_OF_H
11	11	#define _LIBCPP___ALGORITHM_ALL_OF_H
12	12
	13	#include <__algorithm/any_of.h>
13	14	#include <__config>
14	15	#include <__functional/identity.h>
15	16	#include <__type_traits/invoke.h>
	17	#include <__utility/forward.h>
	18	#include <__utility/move.h>
16	19
17	20	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	21	# pragma GCC system_header
19	22	#endif
20	23
	24	_LIBCPP_PUSH_MACROS
	25	#include <__undef_macros>
	26
21	27	_LIBCPP_BEGIN_NAMESPACE_STD
22	28
23	29	template <class _Iter, class _Sent, class _Proj, class _Pred>
24	30	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
25	31	__all_of(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
26		for (; __first != __last; ++__first) {
27		if (!std::__invoke(__pred, std::__invoke(__proj, *__first)))
28		return false;
29		}
30		return true;
	32	using _Ref = decltype(std::__invoke(__proj, *__first));
	33	auto __negated_pred = [&__pred](_Ref __arg) -> bool { return !std::__invoke(__pred, std::forward<_Ref>(__arg)); };
	34	return !std::__any_of(std::move(__first), std::move(__last), __negated_pred, __proj);
31	35	}
32	36
33	37	template <class _InputIterator, class _Predicate>
...	...	@@ -39,4 +43,6 @@ all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
39	43
40	44	_LIBCPP_END_NAMESPACE_STD
41	45
	46	_LIBCPP_POP_MACROS
	47
42	48	#endif // _LIBCPP___ALGORITHM_ALL_OF_H

lib/libcxx/include/__algorithm/comp.h+4

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__config>
13	13	#include <__type_traits/desugars_to.h>
	14	#include <__type_traits/is_generic_transparent_comparator.h>
14	15	#include <__type_traits/is_integral.h>
15	16
16	17	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -48,6 +49,9 @@ inline const bool __desugars_to_v<__less_tag, __less<>, _Tp, _Tp> = true;
48	49	template <class _Tp>
49	50	inline const bool __desugars_to_v<__totally_ordered_less_tag, __less<>, _Tp, _Tp> = is_integral<_Tp>::value;
50	51
	52	template <>
	53	inline const bool __is_generic_transparent_comparator_v<__less<> > = true;
	54
51	55	_LIBCPP_END_NAMESPACE_STD
52	56
53	57	#endif // _LIBCPP___ALGORITHM_COMP_H

lib/libcxx/include/__algorithm/copy.h+28-147

...	...	@@ -12,11 +12,10 @@
12	12	#include <__algorithm/copy_move_common.h>
13	13	#include <__algorithm/for_each_segment.h>
14	14	#include <__algorithm/min.h>
	15	#include <__algorithm/specialized_algorithms.h>
15	16	#include <__config>
16		#include <__fwd/bit_reference.h>
17	17	#include <__iterator/iterator_traits.h>
18	18	#include <__iterator/segmented_iterator.h>
19		#include <__memory/pointer_traits.h>
20	19	#include <__type_traits/common_type.h>
21	20	#include <__type_traits/enable_if.h>
22	21	#include <__utility/move.h>
...	...	@@ -38,124 +37,14 @@ copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result);
38	37	template <class _InIter, class _Sent, class _OutIter>
39	38	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter> __copy(_InIter, _Sent, _OutIter);
40	39
41		template <class _Cp, bool _IsConst>
42		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_aligned(
43		__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
44		using _In = __bit_iterator<_Cp, _IsConst>;
45		using difference_type = typename _In::difference_type;
46		using __storage_type = typename _In::__storage_type;
47
48		const int __bits_per_word = _In::__bits_per_word;
49		difference_type __n = __last - __first;
50		if (__n > 0) {
51		// do first word
52		if (__first.__ctz_ != 0) {
53		unsigned __clz = __bits_per_word - __first.__ctz_;
54		difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
55		__n -= __dn;
56		__storage_type __m = std::__middle_mask<__storage_type>(__clz - __dn, __first.__ctz_);
57		__storage_type __b = *__first.__seg_ & __m;
58		*__result.__seg_ &= ~__m;
59		*__result.__seg_ \|= __b;
60		__result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
61		__result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
62		++__first.__seg_;
63		// __first.__ctz_ = 0;
64		}
65		// __first.__ctz_ == 0;
66		// do middle words
67		__storage_type __nw = __n / __bits_per_word;
68		std::copy(std::__to_address(__first.__seg_),
69		std::__to_address(__first.__seg_ + __nw),
70		std::__to_address(__result.__seg_));
71		__n -= __nw * __bits_per_word;
72		__result.__seg_ += __nw;
73		// do last word
74		if (__n > 0) {
75		__first.__seg_ += __nw;
76		__storage_type __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
77		__storage_type __b = *__first.__seg_ & __m;
78		*__result.__seg_ &= ~__m;
79		*__result.__seg_ \|= __b;
80		__result.__ctz_ = static_cast<unsigned>(__n);
81		}
82		}
83		return __result;
84		}
85
86		template <class _Cp, bool _IsConst>
87		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> __copy_unaligned(
88		__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
89		using _In = __bit_iterator<_Cp, _IsConst>;
90		using difference_type = typename _In::difference_type;
91		using __storage_type = typename _In::__storage_type;
92
93		const int __bits_per_word = _In::__bits_per_word;
94		difference_type __n = __last - __first;
95		if (__n > 0) {
96		// do first word
97		if (__first.__ctz_ != 0) {
98		unsigned __clz_f = __bits_per_word - __first.__ctz_;
99		difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
100		__n -= __dn;
101		__storage_type __m = std::__middle_mask<__storage_type>(__clz_f - __dn, __first.__ctz_);
102		__storage_type __b = *__first.__seg_ & __m;
103		unsigned __clz_r = __bits_per_word - __result.__ctz_;
104		__storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
105		__m = std::__middle_mask<__storage_type>(__clz_r - __ddn, __result.__ctz_);
106		*__result.__seg_ &= ~__m;
107		if (__result.__ctz_ > __first.__ctz_)
108		*__result.__seg_ \|= __b << (__result.__ctz_ - __first.__ctz_);
109		else
110		*__result.__seg_ \|= __b >> (__first.__ctz_ - __result.__ctz_);
111		__result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
112		__result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
113		__dn -= __ddn;
114		if (__dn > 0) {
115		__m = std::__trailing_mask<__storage_type>(__bits_per_word - __dn);
116		*__result.__seg_ &= ~__m;
117		*__result.__seg_ \|= __b >> (__first.__ctz_ + __ddn);
118		__result.__ctz_ = static_cast<unsigned>(__dn);
119		}
120		++__first.__seg_;
121		// __first.__ctz_ = 0;
122		}
123		// __first.__ctz_ == 0;
124		// do middle words
125		unsigned __clz_r = __bits_per_word - __result.__ctz_;
126		__storage_type __m = std::__leading_mask<__storage_type>(__result.__ctz_);
127		for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
128		__storage_type __b = *__first.__seg_;
129		*__result.__seg_ &= ~__m;
130		*__result.__seg_ \|= __b << __result.__ctz_;
131		++__result.__seg_;
132		*__result.__seg_ &= __m;
133		*__result.__seg_ \|= __b >> __clz_r;
134		}
135		// do last word
136		if (__n > 0) {
137		__m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
138		__storage_type __b = *__first.__seg_ & __m;
139		__storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
140		__m = std::__middle_mask<__storage_type>(__clz_r - __dn, __result.__ctz_);
141		*__result.__seg_ &= ~__m;
142		*__result.__seg_ \|= __b << __result.__ctz_;
143		__result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
144		__result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
145		__n -= __dn;
146		if (__n > 0) {
147		__m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
148		*__result.__seg_ &= ~__m;
149		*__result.__seg_ \|= __b >> __dn;
150		__result.__ctz_ = static_cast<unsigned>(__n);
151		}
152		}
153		}
154		return __result;
155		}
156
157	40	struct __copy_impl {
158		template <class _InIter, class _Sent, class _OutIter>
	41	template <class _InIter,
	42	class _Sent,
	43	class _OutIter,
	44	__enable_if_t<!__specialized_algorithm<_Algorithm::__copy,
	45	__iterator_pair<_InIter, _Sent>,
	46	__single_iterator<_OutIter> >::__has_algorithm,
	47	int> = 0>
159	48	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
160	49	operator()(_InIter __first, _Sent __last, _OutIter __result) const {
161	50	while (__first != __last) {
...	...	@@ -167,37 +56,39 @@ struct __copy_impl {
167	56	return std::make_pair(std::move(__first), std::move(__result));
168	57	}
169	58
170		template <class _InIter, class _OutIter>
171		struct _CopySegment {
172		using _Traits _LIBCPP_NODEBUG = __segmented_iterator_traits<_InIter>;
173
174		_OutIter& __result_;
175
176		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit _CopySegment(_OutIter& __result)
177		: __result_(__result) {}
178
179		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
180		operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
181		__result_ = std::__copy(__lfirst, __llast, std::move(__result_)).second;
182		}
183		};
	59	template <class _InIter,
	60	class _Sent,
	61	class _OutIter,
	62	__enable_if_t<__specialized_algorithm<_Algorithm::__copy,
	63	__iterator_pair<_InIter, _Sent>,
	64	__single_iterator<_OutIter> >::__has_algorithm,
	65	int> = 0>
	66	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 static pair<_InIter, _OutIter>
	67	operator()(_InIter __first, _Sent __last, _OutIter __result) {
	68	return __specialized_algorithm<_Algorithm::__copy, __iterator_pair<_InIter, _Sent>, __single_iterator<_OutIter> >()(
	69	std::move(__first), std::move(__last), std::move(__result));
	70	}
184	71
185		template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
	72	template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator_v<_InIter>, int> = 0>
186	73	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
187	74	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
188		std::__for_each_segment(__first, __last, _CopySegment<_InIter, _OutIter>(__result));
	75	using __local_iterator = typename __segmented_iterator_traits<_InIter>::__local_iterator;
	76	std::__for_each_segment(__first, __last, [&__result](__local_iterator __lfirst, __local_iterator __llast) {
	77	__result = std::__copy(std::move(__lfirst), std::move(__llast), std::move(__result)).second;
	78	});
189	79	return std::make_pair(__last, std::move(__result));
190	80	}
191	81
192	82	template <class _InIter,
193	83	class _OutIter,
194	84	__enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
195		!__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
	85	!__is_segmented_iterator_v<_InIter> && __is_segmented_iterator_v<_OutIter>,
196	86	int> = 0>
197	87	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
198	88	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
199	89	using _Traits = __segmented_iterator_traits<_OutIter>;
200		using _DiffT = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
	90	using _DiffT =
	91	typename common_type<__iterator_difference_type<_InIter>, __iterator_difference_type<_OutIter> >::type;
201	92
202	93	if (__first == __last)
203	94	return std::make_pair(std::move(__first), std::move(__result));
...	...	@@ -217,16 +108,6 @@ struct __copy_impl {
217	108	}
218	109	}
219	110
220		template <class _Cp, bool _IsConst>
221		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
222		operator()(__bit_iterator<_Cp, _IsConst> __first,
223		__bit_iterator<_Cp, _IsConst> __last,
224		__bit_iterator<_Cp, false> __result) const {
225		if (__first.__ctz_ == __result.__ctz_)
226		return std::make_pair(__last, std::__copy_aligned(__first, __last, __result));
227		return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result));
228		}
229
230	111	// At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
231	112	template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
232	113	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In, _Out>

lib/libcxx/include/__algorithm/copy_backward.h+9-24

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__algorithm/copy_move_common.h>
13	13	#include <__algorithm/copy_n.h>
	14	#include <__algorithm/for_each_segment.h>
14	15	#include <__algorithm/iterator_operations.h>
15	16	#include <__algorithm/min.h>
16	17	#include <__config>
...	...	@@ -170,37 +171,20 @@ struct __copy_backward_impl {
170	171	return std::make_pair(std::move(__original_last_iter), std::move(__result));
171	172	}
172	173
173		template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
	174	template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator_v<_InIter>, int> = 0>
174	175	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
175	176	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
176		using _Traits = __segmented_iterator_traits<_InIter>;
177		auto __sfirst = _Traits::__segment(__first);
178		auto __slast = _Traits::__segment(__last);
179		if (__sfirst == __slast) {
180		auto __iters =
181		std::__copy_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__local(__last), std::move(__result));
182		return std::make_pair(__last, __iters.second);
183		}
184
185		__result =
186		std::__copy_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__local(__last), std::move(__result))
187		.second;
188		--__slast;
189		while (__sfirst != __slast) {
190		__result =
191		std::__copy_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__end(__slast), std::move(__result))
192		.second;
193		--__slast;
194		}
195		__result = std::__copy_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__end(__slast), std::move(__result))
196		.second;
	177	using __local_iterator = typename __segmented_iterator_traits<_InIter>::__local_iterator;
	178	std::__for_each_segment_backward(__first, __last, [&__result](__local_iterator __lfirst, __local_iterator __llast) {
	179	__result = std::__copy_backward<_AlgPolicy>(std::move(__lfirst), std::move(__llast), std::move(__result)).second;
	180	});
197	181	return std::make_pair(__last, std::move(__result));
198	182	}
199	183
200	184	template <class _InIter,
201	185	class _OutIter,
202	186	__enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
203		!__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
	187	!__is_segmented_iterator_v<_InIter> && __is_segmented_iterator_v<_OutIter>,
204	188	int> = 0>
205	189	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
206	190	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
...	...	@@ -214,7 +198,8 @@ struct __copy_backward_impl {
214	198
215	199	auto __local_last = _Traits::__local(__result);
216	200	while (true) {
217		using _DiffT = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
	201	using _DiffT =
	202	typename common_type<__iterator_difference_type<_InIter>, __iterator_difference_type<_OutIter> >::type;
218	203
219	204	auto __local_first = _Traits::__begin(__segment_iterator);
220	205	auto __size = std::min<_DiffT>(__local_last - __local_first, __last - __first);

lib/libcxx/include/__algorithm/copy_n.h+50-16

...	...	@@ -10,31 +10,63 @@
10	10	#define _LIBCPP___ALGORITHM_COPY_N_H
11	11
12	12	#include <__algorithm/copy.h>
	13	#include <__algorithm/iterator_operations.h>
13	14	#include <__config>
14	15	#include <__iterator/iterator_traits.h>
15	16	#include <__type_traits/enable_if.h>
16	17	#include <__utility/convert_to_integral.h>
	18	#include <__utility/move.h>
	19	#include <__utility/pair.h>
17	20
18	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
19	22	# pragma GCC system_header
20	23	#endif
21	24
	25	_LIBCPP_PUSH_MACROS
	26	#include <__undef_macros>
	27
22	28	_LIBCPP_BEGIN_NAMESPACE_STD
23	29
	30	template <class _AlgPolicy,
	31	class _InIter,
	32	class _OutIter,
	33	__enable_if_t<__has_random_access_iterator_category<_InIter>::value, int> = 0>
	34	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InIter, _OutIter>
	35	__copy_n(_InIter __first, typename _IterOps<_AlgPolicy>::template __difference_type<_InIter> __n, _OutIter __result) {
	36	return std::__copy(__first, __first + __n, std::move(__result));
	37	}
	38
	39	template <class _AlgPolicy,
	40	class _InIter,
	41	class _OutIter,
	42	__enable_if_t<!__has_random_access_iterator_category<_InIter>::value, int> = 0>
	43	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InIter, _OutIter>
	44	__copy_n(_InIter __first, typename _IterOps<_AlgPolicy>::template __difference_type<_InIter> __n, _OutIter __result) {
	45	while (__n != 0) {
	46	__result = __first;
	47	++__first;
	48	++__result;
	49	--__n;
	50	}
	51	return std::make_pair(std::move(__first), std::move(__result));
	52	}
	53
	54	// The InputIterator case is handled specially here because it's been written in a way to avoid incrementing __first
	55	// if not absolutely required. This was done to allow its use with istream_iterator and we want to avoid breaking
	56	// people, at least currently.
	57	// See https://github.com/llvm/llvm-project/commit/99847d2bf132854fffa019bab19818768102ccad
24	58	template <class _InputIterator,
25	59	class _Size,
26	60	class _OutputIterator,
27		__enable_if_t<__has_input_iterator_category<_InputIterator>::value &&
28		!__has_random_access_iterator_category<_InputIterator>::value,
29		int> = 0>
30		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
31		copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
32		typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
33		_IntegralSize __n = __orig_n;
34		if (__n > 0) {
	61	__enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
	62	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
	63	copy_n(_InputIterator __first, _Size __n, _OutputIterator __result) {
	64	using _IntegralSize = decltype(std::__convert_to_integral(__n));
	65	_IntegralSize __converted = __n;
	66	if (__converted > 0) {
35	67	__result = __first;
36	68	++__result;
37		for (--__n; __n > 0; --__n) {
	69	for (--__converted; __converted > 0; --__converted) {
38	70	++__first;
39	71	__result = __first;
40	72	++__result;
...	...	@@ -46,15 +78,17 @@ copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
46	78	template <class _InputIterator,
47	79	class _Size,
48	80	class _OutputIterator,
49		__enable_if_t<__has_random_access_iterator_category<_InputIterator>::value, int> = 0>
50		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
51		copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result) {
52		typedef typename iterator_traits<_InputIterator>::difference_type difference_type;
53		typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
54		_IntegralSize __n = __orig_n;
55		return std::copy(__first, __first + difference_type(__n), __result);
	81	__enable_if_t<!__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
	82	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
	83	copy_n(_InputIterator __first, _Size __n, _OutputIterator __result) {
	84	using _IntegralSize = decltype(std::__convert_to_integral(__n));
	85	_IntegralSize __converted = __n;
	86	return std::__copy_n<_ClassicAlgPolicy>(__first, __iterator_difference_type<_InputIterator>(__converted), __result)
	87	.second;
56	88	}
57	89
58	90	_LIBCPP_END_NAMESPACE_STD
59	91
	92	_LIBCPP_POP_MACROS
	93
60	94	#endif // _LIBCPP___ALGORITHM_COPY_N_H

lib/libcxx/include/__algorithm/count.h+2-2

...	...	@@ -72,7 +72,7 @@ __count_bool(__bit_iterator<_Cp, _IsConst> __first, typename __size_difference_t
72	72	}
73	73
74	74	template <class, class _Cp, bool _IsConst, class _Tp, class _Proj, __enable_if_t<__is_identity<_Proj>::value, int> = 0>
75		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iter_diff_t<__bit_iterator<_Cp, _IsConst> >
	75	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iterator_difference_type<__bit_iterator<_Cp, _IsConst> >
76	76	__count(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, const _Tp& __value, _Proj&) {
77	77	if (__value)
78	78	return std::__count_bool<true>(
...	...	@@ -82,7 +82,7 @@ __count(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __l
82	82	}
83	83
84	84	template <class _InputIterator, class _Tp>
85		[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iter_diff_t<_InputIterator>
	85	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __iterator_difference_type<_InputIterator>
86	86	count(_InputIterator __first, _InputIterator __last, const _Tp& __value) {
87	87	__identity __proj;
88	88	return std::__count<_ClassicAlgPolicy>(__first, __last, __value, __proj);

lib/libcxx/include/__algorithm/equal.h+43-55

...	...	@@ -160,22 +160,28 @@ template <class _Cp,
160	160	bool _IsConst1,
161	161	bool _IsConst2,
162	162	class _BinaryPredicate,
163		__enable_if_t<__desugars_to_v<__equal_tag, _BinaryPredicate, bool, bool>, int> = 0>
	163	class _Proj1,
	164	class _Proj2,
	165	__enable_if_t<__is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
	166	__desugars_to_v<__equal_tag, _BinaryPredicate, bool, bool>,
	167	int> = 0>
164	168	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_iter_impl(
165	169	__bit_iterator<_Cp, _IsConst1> __first1,
166	170	__bit_iterator<_Cp, _IsConst1> __last1,
167	171	__bit_iterator<_Cp, _IsConst2> __first2,
168		_BinaryPredicate) {
	172	_BinaryPredicate,
	173	_Proj1&,
	174	_Proj2&) {
169	175	if (__first1.__ctz_ == __first2.__ctz_)
170	176	return std::__equal_aligned(__first1, __last1, __first2);
171	177	return std::__equal_unaligned(__first1, __last1, __first2);
172	178	}
173	179
174		template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
	180	template <class _InIter1, class _Sent1, class _InIter2, class _Pred, class _Proj1, class _Proj2>
175	181	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_iter_impl(
176		_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate& __pred) {
	182	_InIter1 __first1, _Sent1 __last1, _InIter2 __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
177	183	for (; __first1 != __last1; ++__first1, (void)++__first2)
178		if (!__pred(__first1, __first2))
	184	if (!std::__invoke(__pred, std::__invoke(__proj1, __first1), std::__invoke(__proj2, __first2)))
179	185	return false;
180	186	return true;
181	187	}
...	...	@@ -183,19 +189,23 @@ template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
183	189	template <class _Tp,
184	190	class _Up,
185	191	class _BinaryPredicate,
186		__enable_if_t<__desugars_to_v<__equal_tag, _BinaryPredicate, _Tp, _Up> && !is_volatile<_Tp>::value &&
187		!is_volatile<_Up>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
	192	class _Proj1,
	193	class _Proj2,
	194	__enable_if_t<__is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
	195	__desugars_to_v<__equal_tag, _BinaryPredicate, _Tp, _Up> && !is_volatile<_Tp>::value &&
	196	!is_volatile<_Up>::value && __is_trivially_equality_comparable_v<_Tp, _Up>,
188	197	int> = 0>
189	198	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
190		__equal_iter_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _BinaryPredicate&) {
	199	__equal_iter_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _BinaryPredicate&, _Proj1&, _Proj2&) {
191	200	return std::__constexpr_memcmp_equal(__first1, __first2, __element_count(__last1 - __first1));
192	201	}
193	202
194	203	template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
195	204	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
196	205	equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate __pred) {
	206	__identity __proj;
197	207	return std::__equal_iter_impl(
198		std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred);
	208	std::__unwrap_iter(__first1), std::__unwrap_iter(__last1), std::__unwrap_iter(__first2), __pred, __proj, __proj);
199	209	}
200	210
201	211	template <class _InputIterator1, class _InputIterator2>
...	...	@@ -206,52 +216,28 @@ equal(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first
206	216
207	217	#if _LIBCPP_STD_VER >= 14
208	218
209		template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
210		[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_impl(
211		_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __comp, _Proj1& __proj1, _Proj2& __proj2) {
212		while (__first1 != __last1 && __first2 != __last2) {
213		if (!std::__invoke(__comp, std::__invoke(__proj1, __first1), std::__invoke(__proj2, __first2)))
214		return false;
215		++__first1;
216		++__first2;
217		}
218		return __first1 == __last1 && __first2 == __last2;
219		}
220
221		template <class _Tp,
222		class _Up,
223		class _Pred,
224		class _Proj1,
225		class _Proj2,
226		__enable_if_t<__desugars_to_v<__equal_tag, _Pred, _Tp, _Up> && __is_identity<_Proj1>::value &&
227		__is_identity<_Proj2>::value && !is_volatile<_Tp>::value && !is_volatile<_Up>::value &&
228		__libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
229		int> = 0>
230		[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
231		__equal_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _Up*, _Pred&, _Proj1&, _Proj2&) {
232		return std::__constexpr_memcmp_equal(__first1, __first2, __element_count(__last1 - __first1));
233		}
234
235		template <class _Cp,
236		bool _IsConst1,
237		bool _IsConst2,
	219	template <bool __known_equal_length,
	220	class _Iter1,
	221	class _Sent1,
	222	class _Iter2,
	223	class _Sent2,
238	224	class _Pred,
239	225	class _Proj1,
240		class _Proj2,
241		__enable_if_t<__desugars_to_v<__equal_tag, _Pred, bool, bool> && __is_identity<_Proj1>::value &&
242		__is_identity<_Proj2>::value,
243		int> = 0>
	226	class _Proj2>
244	227	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_impl(
245		__bit_iterator<_Cp, _IsConst1> __first1,
246		__bit_iterator<_Cp, _IsConst1> __last1,
247		__bit_iterator<_Cp, _IsConst2> __first2,
248		__bit_iterator<_Cp, _IsConst2>,
249		_Pred&,
250		_Proj1&,
251		_Proj2&) {
252		if (__first1.__ctz_ == __first2.__ctz_)
253		return std::__equal_aligned(__first1, __last1, __first2);
254		return std::__equal_unaligned(__first1, __last1, __first2);
	228	_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2, _Pred& __comp, _Proj1& __proj1, _Proj2& __proj2) {
	229	if constexpr (__known_equal_length) {
	230	return std::__equal_iter_impl(
	231	std::move(__first1), std::move(__last1), std::move(__first2), __comp, __proj1, __proj2);
	232	} else {
	233	while (__first1 != __last1 && __first2 != __last2) {
	234	if (!std::__invoke(__comp, std::__invoke(__proj1, __first1), std::__invoke(__proj2, __first2)))
	235	return false;
	236	++__first1;
	237	++__first2;
	238	}
	239	return __first1 == __last1 && __first2 == __last2;
	240	}
255	241	}
256	242
257	243	template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
...	...	@@ -261,13 +247,15 @@ equal(_InputIterator1 __first1,
261	247	_InputIterator2 __first2,
262	248	_InputIterator2 __last2,
263	249	_BinaryPredicate __pred) {
264		if constexpr (__has_random_access_iterator_category<_InputIterator1>::value &&
265		__has_random_access_iterator_category<_InputIterator2>::value) {
	250	static constexpr bool __both_random_access =
	251	__has_random_access_iterator_category<_InputIterator1>::value &&
	252	__has_random_access_iterator_category<_InputIterator2>::value;
	253	if constexpr (__both_random_access) {
266	254	if (std::distance(__first1, __last1) != std::distance(__first2, __last2))
267	255	return false;
268	256	}
269	257	__identity __proj;
270		return std::__equal_impl(
	258	return std::__equal_impl<__both_random_access>(
271	259	std::__unwrap_iter(__first1),
272	260	std::__unwrap_iter(__last1),
273	261	std::__unwrap_iter(__first2),

lib/libcxx/include/__algorithm/fill.h+26-8

...	...	@@ -10,8 +10,12 @@
10	10	#define _LIBCPP___ALGORITHM_FILL_H
11	11
12	12	#include <__algorithm/fill_n.h>
	13	#include <__algorithm/for_each_segment.h>
13	14	#include <__config>
14	15	#include <__iterator/iterator_traits.h>
	16	#include <__iterator/segmented_iterator.h>
	17	#include <__type_traits/enable_if.h>
	18	#include <__type_traits/is_same.h>
15	19
16	20	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17	21	# pragma GCC system_header
...	...	@@ -21,23 +25,37 @@ _LIBCPP_BEGIN_NAMESPACE_STD
21	25
22	26	// fill isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
23	27
24		template <class _ForwardIterator, class _Tp>
25		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
26		__fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value, forward_iterator_tag) {
	28	template <class _ForwardIterator, class _Sentinel, class _Tp>
	29	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
	30	__fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __value) {
	31	#ifndef _LIBCPP_CXX03_LANG
	32	if constexpr (is_same<_ForwardIterator, _Sentinel>::value && __is_segmented_iterator_v<_ForwardIterator>) {
	33	using __local_iterator_t = typename __segmented_iterator_traits<_ForwardIterator>::__local_iterator;
	34	std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
	35	std::__fill(__lfirst, __llast, __value);
	36	});
	37	return __last;
	38	}
	39	#endif
27	40	for (; __first != __last; ++__first)
28	41	*__first = __value;
	42	return __first;
29	43	}
30	44
31		template <class _RandomAccessIterator, class _Tp>
32		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
33		__fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value, random_access_iterator_tag) {
34		std::fill_n(__first, __last - __first, __value);
	45	template <class _RandomAccessIterator,
	46	class _Tp,
	47	__enable_if_t<__has_random_access_iterator_category<_RandomAccessIterator>::value &&
	48	!__is_segmented_iterator_v<_RandomAccessIterator>,
	49	int> = 0>
	50	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator
	51	__fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value) {
	52	return std::__fill_n(__first, __last - __first, __value);
35	53	}
36	54
37	55	template <class _ForwardIterator, class _Tp>
38	56	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
39	57	fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
40		std::__fill(__first, __last, __value, typename iterator_traits<_ForwardIterator>::iterator_category());
	58	std::__fill(__first, __last, __value);
41	59	}
42	60
43	61	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__algorithm/fill_n.h+32-46

...	...	@@ -9,11 +9,14 @@
9	9	#ifndef _LIBCPP___ALGORITHM_FILL_N_H
10	10	#define _LIBCPP___ALGORITHM_FILL_N_H
11	11
12		#include <__algorithm/min.h>
	12	#include <__algorithm/for_each_n_segment.h>
	13	#include <__algorithm/specialized_algorithms.h>
13	14	#include <__config>
14		#include <__fwd/bit_reference.h>
15		#include <__memory/pointer_traits.h>
	15	#include <__iterator/iterator_traits.h>
	16	#include <__iterator/segmented_iterator.h>
	17	#include <__type_traits/enable_if.h>
16	18	#include <__utility/convert_to_integral.h>
	19	#include <__utility/move.h>
17	20
18	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
19	22	# pragma GCC system_header
...	...	@@ -26,56 +29,39 @@ _LIBCPP_BEGIN_NAMESPACE_STD
26	29
27	30	// fill_n isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
28	31
29		template <class _OutputIterator, class _Size, class _Tp>
30		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
31		__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value);
32
33		template <bool _FillVal, class _Cp>
34		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
35		__fill_n_bool(__bit_iterator<_Cp, false> __first, typename __size_difference_type_traits<_Cp>::size_type __n) {
36		using _It = __bit_iterator<_Cp, false>;
37		using __storage_type = typename _It::__storage_type;
38
39		const int __bits_per_word = _It::__bits_per_word;
40		// do first partial word
41		if (__first.__ctz_ != 0) {
42		__storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
43		__storage_type __dn = std::min(__clz_f, __n);
44		std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
45		__n -= __dn;
46		++__first.__seg_;
47		}
48		// do middle whole words
49		__storage_type __nw = __n / __bits_per_word;
50		std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
51		__n -= __nw * __bits_per_word;
52		// do last partial word
53		if (__n > 0) {
54		__first.__seg_ += __nw;
55		std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
56		}
57		}
58
59		template <class _Cp, class _Size>
60		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
61		__fill_n(__bit_iterator<_Cp, false> __first, _Size __n, const bool& __value) {
62		if (__n > 0) {
63		if (__value)
64		std::__fill_n_bool<true>(__first, __n);
65		else
66		std::__fill_n_bool<false>(__first, __n);
67		}
68		return __first + __n;
69		}
70
71		template <class _OutputIterator, class _Size, class _Tp>
	32	template <
	33	class _OutputIterator,
	34	class _Size,
	35	class _Tp,
	36	__enable_if_t<!__specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutputIterator> >::__has_algorithm,
	37	int> = 0>
72	38	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
73	39	__fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
	40	#ifndef _LIBCPP_CXX03_LANG
	41	if constexpr (__is_segmented_iterator_v<_OutputIterator>) {
	42	using __local_iterator = typename __segmented_iterator_traits<_OutputIterator>::__local_iterator;
	43	if constexpr (__has_random_access_iterator_category<__local_iterator>::value) {
	44	return std::__for_each_n_segment(__first, __n, [&](__local_iterator __lfirst, __local_iterator __llast) {
	45	std::__fill_n(__lfirst, __llast - __lfirst, __value);
	46	});
	47	}
	48	}
	49	#endif
74	50	for (; __n > 0; ++__first, (void)--__n)
75	51	*__first = __value;
76	52	return __first;
77	53	}
78	54
	55	template <class _OutIter,
	56	class _Size,
	57	class _Tp,
	58	__enable_if_t<__specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutIter> >::__has_algorithm,
	59	int> = 0>
	60	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutIter __fill_n(_OutIter __first, _Size __n, const _Tp& __value) {
	61	return __specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutIter> >()(
	62	std::move(__first), __n, __value);
	63	}
	64
79	65	template <class _OutputIterator, class _Size, class _Tp>
80	66	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
81	67	fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {

lib/libcxx/include/__algorithm/find.h+96-39

...	...	@@ -12,16 +12,19 @@
12	12
13	13	#include <__algorithm/find_segment_if.h>
14	14	#include <__algorithm/min.h>
	15	#include <__algorithm/simd_utils.h>
15	16	#include <__algorithm/unwrap_iter.h>
16	17	#include <__bit/countr.h>
17	18	#include <__bit/invert_if.h>
18	19	#include <__config>
	20	#include <__cstddef/size_t.h>
19	21	#include <__functional/identity.h>
20	22	#include <__fwd/bit_reference.h>
21	23	#include <__iterator/segmented_iterator.h>
22	24	#include <__string/constexpr_c_functions.h>
23	25	#include <__type_traits/enable_if.h>
24	26	#include <__type_traits/invoke.h>
	27	#include <__type_traits/is_constant_evaluated.h>
25	28	#include <__type_traits/is_equality_comparable.h>
26	29	#include <__type_traits/is_integral.h>
27	30	#include <__type_traits/is_signed.h>
...	...	@@ -44,46 +47,108 @@ _LIBCPP_BEGIN_NAMESPACE_STD
44	47	// generic implementation
45	48	template <class _Iter, class _Sent, class _Tp, class _Proj>
46	49	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
47		__find(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
	50	__find_loop(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
48	51	for (; __first != __last; ++__first)
49	52	if (std::__invoke(__proj, *__first) == __value)
50	53	break;
51	54	return __first;
52	55	}
53	56
54		// trivially equality comparable implementations
55		template <class _Tp,
56		class _Up,
57		class _Proj,
58		__enable_if_t<__is_identity<_Proj>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
59		sizeof(_Tp) == 1,
60		int> = 0>
61		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __find(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {
62		if (auto __ret = std::__constexpr_memchr(__first, __value, __last - __first))
63		return __ret;
64		return __last;
	57	template <class _Iter, class _Sent, class _Tp, class _Proj>
	58	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Iter
	59	__find(_Iter __first, _Sent __last, const _Tp& __value, _Proj& __proj) {
	60	return std::__find_loop(std::move(__first), std::move(__last), __value, __proj);
65	61	}
66	62
67		#if _LIBCPP_HAS_WIDE_CHARACTERS
	63	#if _LIBCPP_VECTORIZE_ALGORITHMS
	64	template <class _Tp, class _Up>
	65	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
	66	_LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __find_vectorized(_Tp* __first, _Tp* __last, _Up __value) {
	67	if (!__libcpp_is_constant_evaluated()) {
	68	constexpr size_t __unroll_count = 4;
	69	constexpr size_t __vec_size = __native_vector_size<_Tp>;
	70	using __vec = __simd_vector<_Tp, __vec_size>;
	71
	72	auto __orig_first = __first;
	73
	74	auto __values = static_cast<__simd_vector<_Tp, __vec_size>>(__value); // broadcast the value
	75	while (static_cast<size_t>(__last - __first) >= __unroll_count * __vec_size) [[__unlikely__]] {
	76	__vec __lhs[__unroll_count];
	77
	78	for (size_t __i = 0; __i != __unroll_count; ++__i)
	79	__lhs[__i] = std::__load_vector<__vec>(__first + __i * __vec_size);
	80
	81	for (size_t __i = 0; __i != __unroll_count; ++__i) {
	82	if (auto __cmp_res = __lhs[__i] == __values; std::__any_of(__cmp_res)) {
	83	auto __offset = __i * __vec_size + std::__find_first_set(__cmp_res);
	84	return __first + __offset;
	85	}
	86	}
	87
	88	__first += __unroll_count * __vec_size;
	89	}
	90
	91	// check the remaining 0-3 vectors
	92	while (static_cast<size_t>(__last - __first) >= __vec_size) {
	93	if (auto __cmp_res = std::__load_vector<__vec>(__first) == __values; std::__any_of(__cmp_res)) {
	94	return __first + std::__find_first_set(__cmp_res);
	95	}
	96	__first += __vec_size;
	97	}
	98
	99	if (__last - __first == 0)
	100	return __first;
	101
	102	// Check if we can load elements in front of the current pointer. If that's the case load a vector at
	103	// (last - vector_size) to check the remaining elements
	104	if (static_cast<size_t>(__first - __orig_first) >= __vec_size) {
	105	__first = __last - __vec_size;
	106	return __first + std::__find_first_set(std::__load_vector<__vec>(__first) == __values);
	107	}
	108	}
	109
	110	__identity __proj;
	111	return std::__find_loop(__first, __last, __value, __proj);
	112	}
	113	#endif
	114
	115	#ifndef _LIBCPP_CXX03_LANG
	116	// trivially equality comparable implementations
68	117	template <class _Tp,
69	118	class _Up,
70	119	class _Proj,
71		__enable_if_t<__is_identity<_Proj>::value && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
72		sizeof(_Tp) == sizeof(wchar_t) && _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t),
73		int> = 0>
	120	__enable_if_t<__is_identity<_Proj>::value && __is_trivially_equality_comparable_v<_Tp, _Up>, int> = 0>
74	121	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __find(_Tp* __first, _Tp* __last, const _Up& __value, _Proj&) {
75		if (auto __ret = std::__constexpr_wmemchr(__first, __value, __last - __first))
76		return __ret;
77		return __last;
	122	if constexpr (sizeof(_Tp) == 1) {
	123	if (auto __ret = std::__constexpr_memchr(__first, __value, __last - __first))
	124	return __ret;
	125	return __last;
	126	}
	127	# if _LIBCPP_HAS_WIDE_CHARACTERS
	128	else if constexpr (sizeof(_Tp) == sizeof(wchar_t) && _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t)) {
	129	if (auto __ret = std::__constexpr_wmemchr(__first, __value, __last - __first))
	130	return __ret;
	131	return __last;
	132	}
	133	# endif
	134	# if _LIBCPP_VECTORIZE_ALGORITHMS
	135	else if constexpr (is_integral<_Tp>::value) {
	136	return std::__find_vectorized(__first, __last, __value);
	137	}
	138	# endif
	139	else {
	140	__identity __proj;
	141	return std::__find_loop(__first, __last, __value, __proj);
	142	}
78	143	}
79		#endif // _LIBCPP_HAS_WIDE_CHARACTERS
	144	#endif
80	145
81	146	// TODO: This should also be possible to get right with different signedness
82	147	// cast integral types to allow vectorization
83	148	template <class _Tp,
84	149	class _Up,
85	150	class _Proj,
86		__enable_if_t<__is_identity<_Proj>::value && !__libcpp_is_trivially_equality_comparable<_Tp, _Up>::value &&
	151	__enable_if_t<__is_identity<_Proj>::value && !__is_trivially_equality_comparable_v<_Tp, _Up> &&
87	152	is_integral<_Tp>::value && is_integral<_Up>::value &&
88	153	is_signed<_Tp>::value == is_signed<_Up>::value,
89	154	int> = 0>
...	...	@@ -143,31 +208,23 @@ __find(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __la
143	208
144	209	// segmented iterator implementation
145	210
146		template <class>
147		struct __find_segment;
148
149	211	template <class _SegmentedIterator,
150	212	class _Tp,
151	213	class _Proj,
152		__enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
	214	__enable_if_t<__is_segmented_iterator_v<_SegmentedIterator>, int> = 0>
153	215	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _SegmentedIterator
154	216	__find(_SegmentedIterator __first, _SegmentedIterator __last, const _Tp& __value, _Proj& __proj) {
155		return std::__find_segment_if(std::move(__first), std::move(__last), __find_segment<_Tp>(__value), __proj);
	217	using __local_iterator = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
	218	return std::__find_segment_if(
	219	std::move(__first),
	220	std::move(__last),
	221	[&__value](__local_iterator __lfirst, __local_iterator __llast, _Proj& __lproj) {
	222	return std::__rewrap_iter(
	223	__lfirst, std::__find(std::__unwrap_iter(__lfirst), std::__unwrap_iter(__llast), __value, __lproj));
	224	},
	225	__proj);
156	226	}
157	227
158		template <class _Tp>
159		struct __find_segment {
160		const _Tp& __value_;
161
162		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __find_segment(const _Tp& __value) : __value_(__value) {}
163
164		template <class _InputIterator, class _Proj>
165		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _InputIterator
166		operator()(_InputIterator __first, _InputIterator __last, _Proj& __proj) const {
167		return std::__find(__first, __last, __value_, __proj);
168		}
169		};
170
171	228	// public API
172	229	template <class _InputIterator, class _Tp>
173	230	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator

lib/libcxx/include/__algorithm/find_end.h+105

...	...	@@ -76,6 +76,111 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1>
76	76	}
77	77	}
78	78
	79	template <class _AlgPolicy,
	80	class _Pred,
	81	class _Iter1,
	82	class _Sent1,
	83	class _Iter2,
	84	class _Sent2,
	85	class _Proj1,
	86	class _Proj2>
	87	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter1, _Iter1> __find_end_impl(
	88	_Iter1 __first1,
	89	_Sent1 __sent1,
	90	_Iter2 __first2,
	91	_Sent2 __sent2,
	92	_Pred& __pred,
	93	_Proj1& __proj1,
	94	_Proj2& __proj2,
	95	bidirectional_iterator_tag,
	96	bidirectional_iterator_tag) {
	97	auto __last1 = _IterOps<_AlgPolicy>::next(__first1, __sent1);
	98	auto __last2 = _IterOps<_AlgPolicy>::next(__first2, __sent2);
	99	// modeled after search algorithm (in reverse)
	100	if (__first2 == __last2)
	101	return std::make_pair(__last1, __last1); // Everything matches an empty sequence
	102	_Iter1 __l1 = __last1;
	103	_Iter2 __l2 = __last2;
	104	--__l2;
	105	while (true) {
	106	// Find last element in sequence 1 that matches *(__last2-1), with a mininum of loop checks
	107	while (true) {
	108	if (__first1 == __l1) // return __last1 if no element matches *__first2
	109	return std::make_pair(__last1, __last1);
	110	if (std::__invoke(__pred, std::__invoke(__proj1, --__l1), std::__invoke(__proj2, __l2)))
	111	break;
	112	}
	113	// __l1 matches __l2, now match elements before here
	114	_Iter1 __match_last = __l1;
	115	_Iter1 __m1 = __l1;
	116	_Iter2 __m2 = __l2;
	117	while (true) {
	118	if (__m2 == __first2) // If pattern exhausted, __m1 is the answer (works for 1 element pattern)
	119	return std::make_pair(__m1, ++__match_last);
	120	if (__m1 == __first1) // Otherwise if source exhaused, pattern not found
	121	return std::make_pair(__last1, __last1);
	122
	123	// if there is a mismatch, restart with a new __l1
	124	if (!std::__invoke(__pred, std::__invoke(__proj1, --__m1), std::__invoke(__proj2, --__m2))) {
	125	break;
	126	} // else there is a match, check next elements
	127	}
	128	}
	129	}
	130
	131	template <class _AlgPolicy,
	132	class _Pred,
	133	class _Iter1,
	134	class _Sent1,
	135	class _Iter2,
	136	class _Sent2,
	137	class _Proj1,
	138	class _Proj2>
	139	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter1, _Iter1> __find_end_impl(
	140	_Iter1 __first1,
	141	_Sent1 __sent1,
	142	_Iter2 __first2,
	143	_Sent2 __sent2,
	144	_Pred& __pred,
	145	_Proj1& __proj1,
	146	_Proj2& __proj2,
	147	random_access_iterator_tag,
	148	random_access_iterator_tag) {
	149	typedef typename iterator_traits<_Iter1>::difference_type _D1;
	150	auto __last1 = _IterOps<_AlgPolicy>::next(__first1, __sent1);
	151	auto __last2 = _IterOps<_AlgPolicy>::next(__first2, __sent2);
	152	// Take advantage of knowing source and pattern lengths. Stop short when source is smaller than pattern
	153	auto __len2 = __last2 - __first2;
	154	if (__len2 == 0)
	155	return std::make_pair(__last1, __last1);
	156	auto __len1 = __last1 - __first1;
	157	if (__len1 < __len2)
	158	return std::make_pair(__last1, __last1);
	159	const _Iter1 __s = __first1 + _D1(__len2 - 1); // End of pattern match can't go before here
	160	_Iter1 __l1 = __last1;
	161	_Iter2 __l2 = __last2;
	162	--__l2;
	163	while (true) {
	164	while (true) {
	165	if (__s == __l1)
	166	return std::make_pair(__last1, __last1);
	167	if (std::__invoke(__pred, std::__invoke(__proj1, --__l1), std::__invoke(__proj2, __l2)))
	168	break;
	169	}
	170	_Iter1 __last_match = __l1;
	171	_Iter1 __m1 = __l1;
	172	_Iter2 __m2 = __l2;
	173	while (true) {
	174	if (__m2 == __first2)
	175	return std::make_pair(__m1, ++__last_match);
	176	// no need to check range on __m1 because __s guarantees we have enough source
	177	if (!std::__invoke(__pred, std::__invoke(__proj1, --__m1), std::__invoke(__proj2, --__m2))) {
	178	break;
	179	}
	180	}
	181	}
	182	}
	183
79	184	template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
80	185	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _ForwardIterator1 __find_end_classic(
81	186	_ForwardIterator1 __first1,

lib/libcxx/include/__algorithm/for_each.h+18-24

...	...	@@ -11,45 +11,41 @@
11	11	#define _LIBCPP___ALGORITHM_FOR_EACH_H
12	12
13	13	#include <__algorithm/for_each_segment.h>
	14	#include <__algorithm/specialized_algorithms.h>
14	15	#include <__config>
15	16	#include <__functional/identity.h>
16	17	#include <__iterator/segmented_iterator.h>
17		#include <__type_traits/enable_if.h>
18	18	#include <__type_traits/invoke.h>
19		#include <__utility/move.h>
	19	#include <__type_traits/is_same.h>
20	20
21	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
22	22	# pragma GCC system_header
23	23	#endif
24	24
25		_LIBCPP_PUSH_MACROS
26		#include <__undef_macros>
27
28	25	_LIBCPP_BEGIN_NAMESPACE_STD
29	26
30	27	template <class _InputIterator, class _Sent, class _Func, class _Proj>
31	28	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
32		__for_each(_InputIterator __first, _Sent __last, _Func& __f, _Proj& __proj) {
	29	__for_each(_InputIterator __first, _Sent __last, _Func& __func, _Proj& __proj) {
	30	#ifndef _LIBCPP_CXX03_LANG
	31	if constexpr (using _SpecialAlg =
	32	__specialized_algorithm<_Algorithm::__for_each, __iterator_pair<_InputIterator, _Sent>>;
	33	_SpecialAlg::__has_algorithm) {
	34	_SpecialAlg()(__first, __last, __func, __proj);
	35	return __last;
	36	} else if constexpr (is_same<_InputIterator, _Sent>::value && __is_segmented_iterator_v<_InputIterator>) {
	37	using __local_iterator_t = typename __segmented_iterator_traits<_InputIterator>::__local_iterator;
	38	std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
	39	std::__for_each(__lfirst, __llast, __func, __proj);
	40	});
	41	return __last;
	42	}
	43	#endif
33	44	for (; __first != __last; ++__first)
34		std::__invoke(__f, std::__invoke(__proj, *__first));
	45	std::__invoke(__func, std::__invoke(__proj, *__first));
35	46	return __first;
36	47	}
37	48
38		#ifndef _LIBCPP_CXX03_LANG
39		template <class _SegmentedIterator,
40		class _Func,
41		class _Proj,
42		__enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
43		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _SegmentedIterator
44		__for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Func& __func, _Proj& __proj) {
45		using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
46		std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
47		std::__for_each(__lfirst, __llast, __func, __proj);
48		});
49		return __last;
50		}
51		#endif // !_LIBCPP_CXX03_LANG
52
53	49	template <class _InputIterator, class _Func>
54	50	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Func
55	51	for_each(_InputIterator __first, _InputIterator __last, _Func __f) {
...	...	@@ -60,6 +56,4 @@ for_each(_InputIterator __first, _InputIterator __last, _Func __f) {
60	56
61	57	_LIBCPP_END_NAMESPACE_STD
62	58
63		_LIBCPP_POP_MACROS
64
65	59	#endif // _LIBCPP___ALGORITHM_FOR_EACH_H

lib/libcxx/include/__algorithm/for_each_n.h+20-47

...	...	@@ -16,10 +16,7 @@
16	16	#include <__functional/identity.h>
17	17	#include <__iterator/iterator_traits.h>
18	18	#include <__iterator/segmented_iterator.h>
19		#include <__type_traits/disjunction.h>
20		#include <__type_traits/enable_if.h>
21	19	#include <__type_traits/invoke.h>
22		#include <__type_traits/negation.h>
23	20	#include <__utility/convert_to_integral.h>
24	21	#include <__utility/move.h>
25	22
...	...	@@ -32,57 +29,33 @@ _LIBCPP_PUSH_MACROS
32	29
33	30	_LIBCPP_BEGIN_NAMESPACE_STD
34	31
35		template <class _InputIterator,
36		class _Size,
37		class _Func,
38		class _Proj,
39		__enable_if_t<!__has_random_access_iterator_category<_InputIterator>::value &&
40		_Or< _Not<__is_segmented_iterator<_InputIterator> >,
41		_Not<__has_random_access_local_iterator<_InputIterator> > >::value,
42		int> = 0>
	32	template <class _InputIterator, class _Size, class _Func, class _Proj>
43	33	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
44	34	__for_each_n(_InputIterator __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
45	35	typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
46	36	_IntegralSize __n = __orig_n;
47		while (__n > 0) {
48		std::__invoke(__f, std::__invoke(__proj, *__first));
49		++__first;
50		--__n;
51		}
52		return std::move(__first);
53		}
54
55		template <class _RandIter,
56		class _Size,
57		class _Func,
58		class _Proj,
59		__enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
60		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandIter
61		__for_each_n(_RandIter __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
62		typename std::iterator_traits<_RandIter>::difference_type __n = __orig_n;
63		auto __last = __first + __n;
64		std::__for_each(__first, __last, __f, __proj);
65		return __last;
66		}
67	37
68	38	#ifndef _LIBCPP_CXX03_LANG
69		template <class _SegmentedIterator,
70		class _Size,
71		class _Func,
72		class _Proj,
73		__enable_if_t<!__has_random_access_iterator_category<_SegmentedIterator>::value &&
74		__is_segmented_iterator<_SegmentedIterator>::value &&
75		__has_random_access_iterator_category<
76		typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator>::value,
77		int> = 0>
78		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _SegmentedIterator
79		__for_each_n(_SegmentedIterator __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
80		using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
81		return std::__for_each_n_segment(__first, __orig_n, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
82		std::__for_each(__lfirst, __llast, __f, __proj);
83		});
	39	if constexpr (__is_segmented_iterator_v<_InputIterator>) {
	40	using __local_iterator = typename __segmented_iterator_traits<_InputIterator>::__local_iterator;
	41	if constexpr (__has_random_access_iterator_category<__local_iterator>::value) {
	42	return std::__for_each_n_segment(__first, __orig_n, [&](__local_iterator __lfirst, __local_iterator __llast) {
	43	std::__for_each(__lfirst, __llast, __f, __proj);
	44	});
	45	} else {
	46	return std::__for_each(__first, __first + __n, __f, __proj);
	47	}
	48	} else
	49	#endif
	50	{
	51	while (__n > 0) {
	52	std::__invoke(__f, std::__invoke(__proj, *__first));
	53	++__first;
	54	--__n;
	55	}
	56	return std::move(__first);
	57	}
84	58	}
85		#endif // !_LIBCPP_CXX03_LANG
86	59
87	60	#if _LIBCPP_STD_VER >= 17
88	61

lib/libcxx/include/__algorithm/for_each_n_segment.h+1-1

...	...	@@ -27,7 +27,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
27	27	template <class _SegmentedIterator, class _Size, class _Functor>
28	28	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _SegmentedIterator
29	29	__for_each_n_segment(_SegmentedIterator __first, _Size __orig_n, _Functor __func) {
30		static_assert(__is_segmented_iterator<_SegmentedIterator>::value &&
	30	static_assert(__is_segmented_iterator_v<_SegmentedIterator> &&
31	31	__has_random_access_iterator_category<
32	32	typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator>::value,
33	33	"__for_each_n_segment only works with segmented iterators with random-access local iterators");

lib/libcxx/include/__algorithm/for_each_segment.h+26

...	...	@@ -48,6 +48,32 @@ __for_each_segment(_SegmentedIterator __first, _SegmentedIterator __last, _Funct
48	48	__func(_Traits::__begin(__sfirst), _Traits::__local(__last));
49	49	}
50	50
	51	template <class _SegmentedIterator, class _Functor>
	52	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
	53	__for_each_segment_backward(_SegmentedIterator __first, _SegmentedIterator __last, _Functor __func) {
	54	using _Traits = __segmented_iterator_traits<_SegmentedIterator>;
	55
	56	auto __sfirst = _Traits::__segment(__first);
	57	auto __slast = _Traits::__segment(__last);
	58
	59	// We are in a single segment, so we might not be at the beginning or end
	60	if (__sfirst == __slast) {
	61	__func(_Traits::__local(__first), _Traits::__local(__last));
	62	return;
	63	}
	64
	65	// We have more than one segment. Iterate over the last segment, since we might not start at the end
	66	__func(_Traits::__begin(__slast), _Traits::__local(__last));
	67	--__slast;
	68	// iterate over the segments which are guaranteed to be completely in the range
	69	while (__sfirst != __slast) {
	70	__func(_Traits::__begin(__slast), _Traits::__end(__slast));
	71	--__slast;
	72	}
	73	// iterate over the first segment
	74	__func(_Traits::__local(__first), _Traits::__end(__slast));
	75	}
	76
51	77	_LIBCPP_END_NAMESPACE_STD
52	78
53	79	#endif // _LIBCPP___ALGORITHM_FOR_EACH_SEGMENT_H

lib/libcxx/include/__algorithm/generate.h+4-2

...	...	@@ -9,7 +9,9 @@
9	9	#ifndef _LIBCPP___ALGORITHM_GENERATE_H
10	10	#define _LIBCPP___ALGORITHM_GENERATE_H
11	11
	12	#include <__algorithm/for_each.h>
12	13	#include <__config>
	14	#include <__utility/forward.h>
13	15
14	16	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
15	17	# pragma GCC system_header
...	...	@@ -20,8 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
20	22	template <class _ForwardIterator, class _Generator>
21	23	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
22	24	generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __gen) {
23		for (; __first != __last; ++__first)
24		*__first = __gen();
	25	using __iter_ref = decltype(*__first);
	26	std::for_each(__first, __last, [&](__iter_ref __element) { std::forward<__iter_ref>(__element) = __gen(); });
25	27	}
26	28
27	29	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__algorithm/generate_n.h+19-6

...	...	@@ -9,25 +9,38 @@
9	9	#ifndef _LIBCPP___ALGORITHM_GENERATE_N_H
10	10	#define _LIBCPP___ALGORITHM_GENERATE_N_H
11	11
	12	#include <__algorithm/for_each_n.h>
12	13	#include <__config>
13		#include <__utility/convert_to_integral.h>
	14	#include <__functional/identity.h>
	15	#include <__utility/forward.h>
	16	#include <__utility/move.h>
14	17
15	18	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
16	19	# pragma GCC system_header
17	20	#endif
18	21
	22	_LIBCPP_PUSH_MACROS
	23	#include <__undef_macros>
	24
19	25	_LIBCPP_BEGIN_NAMESPACE_STD
20	26
	27	template <class _OutputIterator, class _Size, class _Generator>
	28	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
	29	__generate_n(_OutputIterator __first, _Size __orig_n, _Generator& __gen) {
	30	using __iter_ref = decltype(*__first);
	31	__identity __proj;
	32	auto __f = [&](__iter_ref __element) { std::forward<__iter_ref>(__element) = __gen(); };
	33	return std::__for_each_n(std::move(__first), __orig_n, __f, __proj);
	34	}
	35
21	36	template <class _OutputIterator, class _Size, class _Generator>
22	37	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
23	38	generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen) {
24		typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
25		_IntegralSize __n = __orig_n;
26		for (; __n > 0; ++__first, (void)--__n)
27		*__first = __gen();
28		return __first;
	39	return std::__generate_n(std::move(__first), __orig_n, __gen);
29	40	}
30	41
31	42	_LIBCPP_END_NAMESPACE_STD
32	43
	44	_LIBCPP_POP_MACROS
	45
33	46	#endif // _LIBCPP___ALGORITHM_GENERATE_N_H

lib/libcxx/include/__algorithm/is_permutation.h+4-4

...	...	@@ -78,7 +78,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation_impl(
78	78	_Pred&& __pred,
79	79	_Proj1&& __proj1,
80	80	_Proj2&& __proj2) {
81		using _D1 = __iter_diff_t<_Iter1>;
	81	using _D1 = __iterator_difference_type<_Iter1>;
82	82
83	83	for (auto __i = __first1; __i != __last1; ++__i) {
84	84	// Have we already counted the number of *__i in [f1, l1)?
...	...	@@ -126,7 +126,7 @@ template <class _AlgPolicy, class _ForwardIterator1, class _Sentinel1, class _Fo
126	126	return true;
127	127
128	128	// __first1 != __last1 && __first1 != __first2
129		using _D1 = __iter_diff_t<_ForwardIterator1>;
	129	using _D1 = __iterator_difference_type<_ForwardIterator1>;
130	130	_D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1);
131	131	if (__l1 == _D1(1))
132	132	return false;
...	...	@@ -173,10 +173,10 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __is_permutation(
173	173	if (__first2 == __last2) // Second range is shorter
174	174	return false;
175	175
176		using _D1 = __iter_diff_t<_Iter1>;
	176	using _D1 = __iterator_difference_type<_Iter1>;
177	177	_D1 __l1 = _IterOps<_AlgPolicy>::distance(__first1, __last1);
178	178
179		using _D2 = __iter_diff_t<_Iter2>;
	179	using _D2 = __iterator_difference_type<_Iter2>;
180	180	_D2 __l2 = _IterOps<_AlgPolicy>::distance(__first2, __last2);
181	181	if (__l1 != __l2)
182	182	return false;

lib/libcxx/include/__algorithm/iterator_operations.h+3

...	...	@@ -219,6 +219,9 @@ private:
219	219	template <class _AlgPolicy, class _Iter>
220	220	using __policy_iter_diff_t _LIBCPP_NODEBUG = typename _IterOps<_AlgPolicy>::template __difference_type<_Iter>;
221	221
	222	template <class _AlgPolicy, class _Iter>
	223	using __policy_value_type _LIBCPP_NODEBUG = typename _IterOps<_AlgPolicy>::template __value_type<_Iter>;
	224
222	225	_LIBCPP_END_NAMESPACE_STD
223	226
224	227	_LIBCPP_POP_MACROS

lib/libcxx/include/__algorithm/lexicographical_compare.h+2-2

...	...	@@ -66,8 +66,8 @@ template <class _Tp,
66	66	class _Proj2,
67	67	class _Comp,
68	68	__enable_if_t<__desugars_to_v<__totally_ordered_less_tag, _Comp, _Tp, _Tp> && !is_volatile<_Tp>::value &&
69		__libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value &&
70		__is_identity<_Proj1>::value && __is_identity<_Proj2>::value,
	69	__is_trivially_equality_comparable_v<_Tp, _Tp> && __is_identity<_Proj1>::value &&
	70	__is_identity<_Proj2>::value,
71	71	int> = 0>
72	72	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
73	73	__lexicographical_compare(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Tp* __last2, _Comp&, _Proj1&, _Proj2&) {

lib/libcxx/include/__algorithm/lexicographical_compare_three_way.h+6-6

...	...	@@ -37,13 +37,13 @@ template <class _InputIterator1, class _InputIterator2, class _Cmp>
37	37	_LIBCPP_HIDE_FROM_ABI constexpr auto __lexicographical_compare_three_way_fast_path(
38	38	_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _InputIterator2 __last2, _Cmp& __comp)
39	39	-> decltype(__comp(__first1, __first2)) {
40		static_assert(
41		signed_integral<__iter_diff_t<_InputIterator1>>, "Using a non-integral difference_type is undefined behavior.");
42		static_assert(
43		signed_integral<__iter_diff_t<_InputIterator2>>, "Using a non-integral difference_type is undefined behavior.");
	40	static_assert(signed_integral<__iterator_difference_type<_InputIterator1>>,
	41	"Using a non-integral difference_type is undefined behavior.");
	42	static_assert(signed_integral<__iterator_difference_type<_InputIterator2>>,
	43	"Using a non-integral difference_type is undefined behavior.");
44	44
45		using _Len1 = __iter_diff_t<_InputIterator1>;
46		using _Len2 = __iter_diff_t<_InputIterator2>;
	45	using _Len1 = __iterator_difference_type<_InputIterator1>;
	46	using _Len2 = __iterator_difference_type<_InputIterator2>;
47	47	using _Common = common_type_t<_Len1, _Len2>;
48	48
49	49	_Len1 __len1 = __last1 - __first1;

lib/libcxx/include/__algorithm/make_heap.h+16-4

...	...	@@ -12,9 +12,11 @@
12	12	#include <__algorithm/comp.h>
13	13	#include <__algorithm/comp_ref_type.h>
14	14	#include <__algorithm/iterator_operations.h>
	15	#include <__algorithm/push_heap.h>
15	16	#include <__algorithm/sift_down.h>
16	17	#include <__config>
17	18	#include <__iterator/iterator_traits.h>
	19	#include <__type_traits/is_arithmetic.h>
18	20	#include <__utility/move.h>
19	21
20	22	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -31,13 +33,23 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
31	33	__make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare&& __comp) {
32	34	__comp_ref_type<_Compare> __comp_ref = __comp;
33	35
34		using difference_type = typename iterator_traits<_RandomAccessIterator>::difference_type;
35		difference_type __n = __last - __first;
	36	using __diff_t = __iterator_difference_type<_RandomAccessIterator>;
	37	const __diff_t __n = __last - __first;
	38
	39	const bool __assume_both_children = is_arithmetic<__iterator_value_type<_RandomAccessIterator> >::value;
	40
	41	// While it would be correct to always assume we have both children, in practice we observed this to be a performance
	42	// improvement only for arithmetic types.
	43	const __diff_t __sift_down_n = __assume_both_children ? ((__n & 1) ? __n : __n - 1) : __n;
	44
36	45	if (__n > 1) {
37	46	// start from the first parent, there is no need to consider children
38		for (difference_type __start = (__n - 2) / 2; __start >= 0; --__start) {
39		std::__sift_down<_AlgPolicy>(__first, __comp_ref, __n, __first + __start);
	47
	48	for (__diff_t __start = (__sift_down_n - 2) / 2; __start >= 0; --__start) {
	49	std::__sift_down<_AlgPolicy, __assume_both_children>(__first, __comp_ref, __sift_down_n, __start);
40	50	}
	51	if _LIBCPP_CONSTEXPR (__assume_both_children)
	52	std::__sift_up<_AlgPolicy>(__first, __last, __comp, __n);
41	53	}
42	54	}
43	55

lib/libcxx/include/__algorithm/mismatch.h+2-2

...	...	@@ -60,7 +60,7 @@ __mismatch(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Pred& __pred, _Pro
60	60	template <class _Iter>
61	61	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Iter, _Iter>
62	62	__mismatch_vectorized(_Iter __first1, _Iter __last1, _Iter __first2) {
63		using __value_type = __iter_value_type<_Iter>;
	63	using __value_type = __iterator_value_type<_Iter>;
64	64	constexpr size_t __unroll_count = 4;
65	65	constexpr size_t __vec_size = __native_vector_size<__value_type>;
66	66	using __vec = __simd_vector<__value_type, __vec_size>;
...	...	@@ -136,7 +136,7 @@ template <class _Tp,
136	136	class _Proj2,
137	137	__enable_if_t<!is_integral<_Tp>::value && __desugars_to_v<__equal_tag, _Pred, _Tp, _Tp> &&
138	138	__is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
139		__can_map_to_integer_v<_Tp> && __libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
	139	__can_map_to_integer_v<_Tp> && __is_trivially_equality_comparable_v<_Tp, _Tp>,
140	140	int> = 0>
141	141	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_Tp, _Tp>
142	142	__mismatch(_Tp* __first1, _Tp* __last1, _Tp* __first2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {

lib/libcxx/include/__algorithm/move.h+8-19

...	...	@@ -50,37 +50,26 @@ struct __move_impl {
50	50	return std::make_pair(std::move(__first), std::move(__result));
51	51	}
52	52
53		template <class _InIter, class _OutIter>
54		struct _MoveSegment {
55		using _Traits _LIBCPP_NODEBUG = __segmented_iterator_traits<_InIter>;
56
57		_OutIter& __result_;
58
59		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit _MoveSegment(_OutIter& __result)
60		: __result_(__result) {}
61
62		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
63		operator()(typename _Traits::__local_iterator __lfirst, typename _Traits::__local_iterator __llast) {
64		__result_ = std::__move<_AlgPolicy>(__lfirst, __llast, std::move(__result_)).second;
65		}
66		};
67
68		template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
	53	template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator_v<_InIter>, int> = 0>
69	54	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
70	55	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
71		std::__for_each_segment(__first, __last, _MoveSegment<_InIter, _OutIter>(__result));
	56	using __local_iterator = typename __segmented_iterator_traits<_InIter>::__local_iterator;
	57	std::__for_each_segment(__first, __last, [&__result](__local_iterator __lfirst, __local_iterator __llast) {
	58	__result = std::__move<_AlgPolicy>(__lfirst, __llast, std::move(__result)).second;
	59	});
72	60	return std::make_pair(__last, std::move(__result));
73	61	}
74	62
75	63	template <class _InIter,
76	64	class _OutIter,
77	65	__enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
78		!__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
	66	!__is_segmented_iterator_v<_InIter> && __is_segmented_iterator_v<_OutIter>,
79	67	int> = 0>
80	68	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
81	69	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
82	70	using _Traits = __segmented_iterator_traits<_OutIter>;
83		using _DiffT = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
	71	using _DiffT =
	72	typename common_type<__iterator_difference_type<_InIter>, __iterator_difference_type<_OutIter> >::type;
84	73
85	74	if (__first == __last)
86	75	return std::make_pair(std::move(__first), std::move(__result));

lib/libcxx/include/__algorithm/move_backward.h+9-24

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__algorithm/copy_backward.h>
13	13	#include <__algorithm/copy_move_common.h>
	14	#include <__algorithm/for_each_segment.h>
14	15	#include <__algorithm/iterator_operations.h>
15	16	#include <__algorithm/min.h>
16	17	#include <__config>
...	...	@@ -51,42 +52,26 @@ struct __move_backward_impl {
51	52	return std::make_pair(std::move(__original_last_iter), std::move(__result));
52	53	}
53	54
54		template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator<_InIter>::value, int> = 0>
	55	template <class _InIter, class _OutIter, __enable_if_t<__is_segmented_iterator_v<_InIter>, int> = 0>
55	56	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
56	57	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
57		using _Traits = __segmented_iterator_traits<_InIter>;
58		auto __sfirst = _Traits::__segment(__first);
59		auto __slast = _Traits::__segment(__last);
60		if (__sfirst == __slast) {
61		auto __iters =
62		std::__move_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__local(__last), std::move(__result));
63		return std::make_pair(__last, __iters.second);
64		}
65
66		__result =
67		std::__move_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__local(__last), std::move(__result))
68		.second;
69		--__slast;
70		while (__sfirst != __slast) {
71		__result =
72		std::__move_backward<_AlgPolicy>(_Traits::__begin(__slast), _Traits::__end(__slast), std::move(__result))
73		.second;
74		--__slast;
75		}
76		__result = std::__move_backward<_AlgPolicy>(_Traits::__local(__first), _Traits::__end(__slast), std::move(__result))
77		.second;
	58	using __local_iterator = typename __segmented_iterator_traits<_InIter>::__local_iterator;
	59	std::__for_each_segment_backward(__first, __last, [&__result](__local_iterator __lfirst, __local_iterator __llast) {
	60	__result = std::__move_backward<_AlgPolicy>(std::move(__lfirst), std::move(__llast), std::move(__result)).second;
	61	});
78	62	return std::make_pair(__last, std::move(__result));
79	63	}
80	64
81	65	template <class _InIter,
82	66	class _OutIter,
83	67	__enable_if_t<__has_random_access_iterator_category<_InIter>::value &&
84		!__is_segmented_iterator<_InIter>::value && __is_segmented_iterator<_OutIter>::value,
	68	!__is_segmented_iterator_v<_InIter> && __is_segmented_iterator_v<_OutIter>,
85	69	int> = 0>
86	70	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
87	71	operator()(_InIter __first, _InIter __last, _OutIter __result) const {
88	72	using _Traits = __segmented_iterator_traits<_OutIter>;
89		using _DiffT = typename common_type<__iter_diff_t<_InIter>, __iter_diff_t<_OutIter> >::type;
	73	using _DiffT =
	74	typename common_type<__iterator_difference_type<_InIter>, __iterator_difference_type<_OutIter> >::type;
90	75
91	76	// When the range contains no elements, __result might not be a valid iterator
92	77	if (__first == __last)

lib/libcxx/include/__algorithm/none_of.h+4-4

...	...	@@ -10,7 +10,9 @@
10	10	#ifndef _LIBCPP___ALGORITHM_NONE_OF_H
11	11	#define _LIBCPP___ALGORITHM_NONE_OF_H
12	12
	13	#include <__algorithm/any_of.h>
13	14	#include <__config>
	15	#include <__functional/identity.h>
14	16
15	17	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
16	18	# pragma GCC system_header
...	...	@@ -21,10 +23,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
21	23	template <class _InputIterator, class _Predicate>
22	24	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
23	25	none_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
24		for (; __first != __last; ++__first)
25		if (__pred(*__first))
26		return false;
27		return true;
	26	__identity __proj;
	27	return !std::__any_of(__first, __last, __pred, __proj);
28	28	}
29	29
30	30	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__algorithm/partial_sort.h+1-1

...	...	@@ -45,7 +45,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandomAccessIterator __part
45	45	for (; __i != __last; ++__i) {
46	46	if (__comp(__i, __first)) {
47	47	_IterOps<_AlgPolicy>::iter_swap(__i, __first);
48		std::__sift_down<_AlgPolicy>(__first, __comp, __len, __first);
	48	std::__sift_down<_AlgPolicy, false>(__first, __comp, __len, 0);
49	49	}
50	50	}
51	51	std::__sort_heap<_AlgPolicy>(std::move(__first), std::move(__middle), __comp);

lib/libcxx/include/__algorithm/partial_sort_copy.h+1-1

...	...	@@ -60,7 +60,7 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<_InputIterator, _Random
60	60	for (; __first != __last; ++__first)
61	61	if (std::__invoke(__comp, std::__invoke(__proj1, __first), std::__invoke(__proj2, __result_first))) {
62	62	__result_first = __first;
63		std::__sift_down<_AlgPolicy>(__result_first, __projected_comp, __len, __result_first);
	63	std::__sift_down<_AlgPolicy, false>(__result_first, __projected_comp, __len, 0);
64	64	}
65	65	std::__sort_heap<_AlgPolicy>(__result_first, __r, __projected_comp);
66	66	}

lib/libcxx/include/__algorithm/pstl.h+9-9

...	...	@@ -115,7 +115,7 @@ template <class _ExecutionPolicy,
115	115	class _Predicate,
116	116	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
117	117	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
118		_LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator>
	118	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __iterator_difference_type<_ForwardIterator>
119	119	count_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
120	120	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
121	121	_ForwardIterator, "count_if(first, last, pred) requires [first, last) to be ForwardIterators");
...	...	@@ -129,7 +129,7 @@ template <class _ExecutionPolicy,
129	129	class _Tp,
130	130	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
131	131	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
132		_LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator>
	132	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __iterator_difference_type<_ForwardIterator>
133	133	count(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
134	134	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(
135	135	_ForwardIterator, "count(first, last, val) requires [first, last) to be ForwardIterators");
...	...	@@ -144,7 +144,7 @@ template <class _ExecutionPolicy,
144	144	class _Pred,
145	145	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
146	146	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
147		_LIBCPP_HIDE_FROM_ABI bool
	147	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
148	148	equal(_ExecutionPolicy&& __policy,
149	149	_ForwardIterator1 __first1,
150	150	_ForwardIterator1 __last1,
...	...	@@ -166,7 +166,7 @@ template <class _ExecutionPolicy,
166	166	class _ForwardIterator2,
167	167	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
168	168	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
169		_LIBCPP_HIDE_FROM_ABI bool
	169	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
170	170	equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) {
171	171	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators");
172	172	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators");
...	...	@@ -185,7 +185,7 @@ template <class _ExecutionPolicy,
185	185	class _Pred,
186	186	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
187	187	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
188		_LIBCPP_HIDE_FROM_ABI bool
	188	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
189	189	equal(_ExecutionPolicy&& __policy,
190	190	_ForwardIterator1 __first1,
191	191	_ForwardIterator1 __last1,
...	...	@@ -209,7 +209,7 @@ template <class _ExecutionPolicy,
209	209	class _ForwardIterator2,
210	210	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
211	211	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
212		_LIBCPP_HIDE_FROM_ABI bool
	212	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool
213	213	equal(_ExecutionPolicy&& __policy,
214	214	_ForwardIterator1 __first1,
215	215	_ForwardIterator1 __last1,
...	...	@@ -259,7 +259,7 @@ template <class _ExecutionPolicy,
259	259	class _Predicate,
260	260	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
261	261	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
262		_LIBCPP_HIDE_FROM_ABI _ForwardIterator
	262	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _ForwardIterator
263	263	find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
264	264	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators");
265	265	using _Implementation = __pstl::__dispatch<__pstl::__find_if, __pstl::__current_configuration, _RawPolicy>;
...	...	@@ -272,7 +272,7 @@ template <class _ExecutionPolicy,
272	272	class _Predicate,
273	273	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
274	274	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
275		_LIBCPP_HIDE_FROM_ABI _ForwardIterator
	275	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _ForwardIterator
276	276	find_if_not(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
277	277	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if_not requires ForwardIterators");
278	278	using _Implementation = __pstl::__dispatch<__pstl::__find_if_not, __pstl::__current_configuration, _RawPolicy>;
...	...	@@ -285,7 +285,7 @@ template <class _ExecutionPolicy,
285	285	class _Tp,
286	286	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
287	287	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
288		_LIBCPP_HIDE_FROM_ABI _ForwardIterator
	288	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _ForwardIterator
289	289	find(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
290	290	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find requires ForwardIterators");
291	291	using _Implementation = __pstl::__dispatch<__pstl::__find, __pstl::__current_configuration, _RawPolicy>;

lib/libcxx/include/__algorithm/radix_sort.h+27-25

...	...	@@ -72,14 +72,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD
72	72	#if _LIBCPP_STD_VER >= 14
73	73
74	74	template <class _InputIterator, class _OutputIterator>
75		_LIBCPP_HIDE_FROM_ABI constexpr pair<_OutputIterator, __iter_value_type<_InputIterator>>
	75	_LIBCPP_HIDE_FROM_ABI constexpr pair<_OutputIterator, __iterator_value_type<_InputIterator>>
76	76	__partial_sum_max(_InputIterator __first, _InputIterator __last, _OutputIterator __result) {
77	77	if (__first == __last)
78	78	return {__result, 0};
79	79
80		auto __max = *__first;
81		__iter_value_type<_InputIterator> __sum = *__first;
82		*__result = __sum;
	80	auto __max = *__first;
	81	__iterator_value_type<_InputIterator> __sum = *__first;
	82	*__result = __sum;
83	83
84	84	while (++__first != __last) {
85	85	if (__max < *__first) {
...	...	@@ -124,7 +124,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto __nth_radix(size_t __radix_number, _Radix _
124	124	template <class _ForwardIterator, class _Map, class _RandomAccessIterator>
125	125	_LIBCPP_HIDE_FROM_ABI constexpr void
126	126	__collect(_ForwardIterator __first, _ForwardIterator __last, _Map __map, _RandomAccessIterator __counters) {
127		using __value_type = __iter_value_type<_ForwardIterator>;
	127	using __value_type = __iterator_value_type<_ForwardIterator>;
128	128	using __traits = __counting_sort_traits<__value_type, _Map>;
129	129
130	130	std::for_each(__first, __last, [&__counters, &__map](const auto& __preimage) { ++__counters[__map(__preimage)]; });
...	...	@@ -160,7 +160,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __collect_impl(
160	160	_RandomAccessIterator1 __counters,
161	161	_RandomAccessIterator2 __maximums,
162	162	index_sequence<_Radices...>) {
163		using __value_type = __iter_value_type<_ForwardIterator>;
	163	using __value_type = __iterator_value_type<_ForwardIterator>;
164	164	constexpr auto __radix_value_range = __radix_sort_traits<__value_type, _Map, _Radix>::__radix_value_range;
165	165
166	166	auto __previous = numeric_limits<__invoke_result_t<_Map, __value_type>>::min();
...	...	@@ -189,7 +189,7 @@ __collect(_ForwardIterator __first,
189	189	_Radix __radix,
190	190	_RandomAccessIterator1 __counters,
191	191	_RandomAccessIterator2 __maximums) {
192		using __value_type = __iter_value_type<_ForwardIterator>;
	192	using __value_type = __iterator_value_type<_ForwardIterator>;
193	193	constexpr auto __radix_count = __radix_sort_traits<__value_type, _Map, _Radix>::__radix_count;
194	194	return std::__collect_impl(
195	195	__first, __last, __map, __radix, __counters, __maximums, make_index_sequence<__radix_count>());
...	...	@@ -213,10 +213,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __dispose_backward(
213	213	template <class _ForwardIterator, class _RandomAccessIterator, class _Map>
214	214	_LIBCPP_HIDE_FROM_ABI constexpr _RandomAccessIterator
215	215	__counting_sort_impl(_ForwardIterator __first, _ForwardIterator __last, _RandomAccessIterator __result, _Map __map) {
216		using __value_type = __iter_value_type<_ForwardIterator>;
	216	using __value_type = __iterator_value_type<_ForwardIterator>;
217	217	using __traits = __counting_sort_traits<__value_type, _Map>;
218	218
219		__iter_diff_t<_RandomAccessIterator> __counters[__traits::__value_range + 1] = {0};
	219	__iterator_difference_type<_RandomAccessIterator> __counters[__traits::__value_range + 1] = {0};
220	220
221	221	std::__collect(__first, __last, __map, std::next(std::begin(__counters)));
222	222	std::__dispose(__first, __last, __result, __map, std::begin(__counters));
...	...	@@ -224,12 +224,13 @@ __counting_sort_impl(_ForwardIterator __first, _ForwardIterator __last, _RandomA
224	224	return __result + __counters[__traits::__value_range];
225	225	}
226	226
227		template <class _RandomAccessIterator1,
228		class _RandomAccessIterator2,
229		class _Map,
230		class _Radix,
231		enable_if_t< __radix_sort_traits<__iter_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count == 1,
232		int> = 0>
	227	template <
	228	class _RandomAccessIterator1,
	229	class _RandomAccessIterator2,
	230	class _Map,
	231	class _Radix,
	232	enable_if_t<__radix_sort_traits<__iterator_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count == 1,
	233	int> = 0>
233	234	_LIBCPP_HIDE_FROM_ABI constexpr void __radix_sort_impl(
234	235	_RandomAccessIterator1 __first,
235	236	_RandomAccessIterator1 __last,
...	...	@@ -243,24 +244,25 @@ _LIBCPP_HIDE_FROM_ABI constexpr void __radix_sort_impl(
243	244	std::move(__buffer, __buffer_end, __first);
244	245	}
245	246
246		template <
247		class _RandomAccessIterator1,
248		class _RandomAccessIterator2,
249		class _Map,
250		class _Radix,
251		enable_if_t< __radix_sort_traits<__iter_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count % 2 == 0,
252		int> = 0 >
	247	template <class _RandomAccessIterator1,
	248	class _RandomAccessIterator2,
	249	class _Map,
	250	class _Radix,
	251	enable_if_t<
	252	__radix_sort_traits<__iterator_value_type<_RandomAccessIterator1>, _Map, _Radix>::__radix_count % 2 == 0,
	253	int> = 0>
253	254	_LIBCPP_HIDE_FROM_ABI constexpr void __radix_sort_impl(
254	255	_RandomAccessIterator1 __first,
255	256	_RandomAccessIterator1 __last,
256	257	_RandomAccessIterator2 __buffer_begin,
257	258	_Map __map,
258	259	_Radix __radix) {
259		using __value_type = __iter_value_type<_RandomAccessIterator1>;
	260	using __value_type = __iterator_value_type<_RandomAccessIterator1>;
260	261	using __traits = __radix_sort_traits<__value_type, _Map, _Radix>;
261	262
262		__iter_diff_t<_RandomAccessIterator1> __counters[__traits::__radix_count][__traits::__radix_value_range] = {{0}};
263		__iter_diff_t<_RandomAccessIterator1> __maximums[__traits::__radix_count] = {0};
	263	__iterator_difference_type<_RandomAccessIterator1>
	264	__counters[__traits::__radix_count][__traits::__radix_value_range] = {{0}};
	265	__iterator_difference_type<_RandomAccessIterator1> __maximums[__traits::__radix_count] = {0};
264	266	const auto __is_sorted = std::__collect(__first, __last, __map, __radix, __counters, __maximums);
265	267	if (!__is_sorted) {
266	268	const auto __range_size = std::distance(__first, __last);

lib/libcxx/include/__algorithm/ranges_copy_n.h+3-26

...	...	@@ -9,16 +9,12 @@
9	9	#ifndef _LIBCPP___ALGORITHM_RANGES_COPY_N_H
10	10	#define _LIBCPP___ALGORITHM_RANGES_COPY_N_H
11	11
12		#include <__algorithm/copy.h>
	12	#include <__algorithm/copy_n.h>
13	13	#include <__algorithm/in_out_result.h>
14	14	#include <__algorithm/iterator_operations.h>
15		#include <__algorithm/ranges_copy.h>
16	15	#include <__config>
17		#include <__functional/identity.h>
18	16	#include <__iterator/concepts.h>
19	17	#include <__iterator/incrementable_traits.h>
20		#include <__iterator/unreachable_sentinel.h>
21		#include <__iterator/wrap_iter.h>
22	18	#include <__utility/move.h>
23	19
24	20	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -37,32 +33,13 @@ namespace ranges {
37	33	template <class _Ip, class _Op>
38	34	using copy_n_result = in_out_result<_Ip, _Op>;
39	35
40		// TODO: Merge this with copy_n
41	36	struct __copy_n {
42		template <class _InIter, class _DiffType, class _OutIter>
43		_LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
44		__go(_InIter __first, _DiffType __n, _OutIter __result) {
45		while (__n != 0) {
46		__result = __first;
47		++__first;
48		++__result;
49		--__n;
50		}
51		return {std::move(__first), std::move(__result)};
52		}
53
54		template <random_access_iterator _InIter, class _DiffType, random_access_iterator _OutIter>
55		_LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
56		__go(_InIter __first, _DiffType __n, _OutIter __result) {
57		auto __ret = std::__copy(__first, __first + __n, __result);
58		return {__ret.first, __ret.second};
59		}
60
61	37	template <input_iterator _Ip, weakly_incrementable _Op>
62	38	requires indirectly_copyable<_Ip, _Op>
63	39	_LIBCPP_HIDE_FROM_ABI constexpr copy_n_result<_Ip, _Op>
64	40	operator()(_Ip __first, iter_difference_t<_Ip> __n, _Op __result) const {
65		return __go(std::move(__first), __n, std::move(__result));
	41	auto __res = std::__copy_n<_RangeAlgPolicy>(std::move(__first), __n, std::move(__result));
	42	return {std::move(__res.first), std::move(__res.second)};
66	43	}
67	44	};
68	45

lib/libcxx/include/__algorithm/ranges_equal.h+17-26

...	...	@@ -13,13 +13,12 @@
13	13	#include <__algorithm/unwrap_range.h>
14	14	#include <__config>
15	15	#include <__functional/identity.h>
16		#include <__functional/invoke.h>
17	16	#include <__functional/ranges_operations.h>
18	17	#include <__iterator/concepts.h>
19		#include <__iterator/distance.h>
20	18	#include <__iterator/indirectly_comparable.h>
21	19	#include <__ranges/access.h>
22	20	#include <__ranges/concepts.h>
	21	#include <__ranges/size.h>
23	22	#include <__utility/move.h>
24	23
25	24	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -51,20 +50,17 @@ struct __equal {
51	50	_Pred __pred = {},
52	51	_Proj1 __proj1 = {},
53	52	_Proj2 __proj2 = {}) const {
54		if constexpr (sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2>) {
	53	static constexpr bool __both_sized = sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2>;
	54	if constexpr (__both_sized) {
55	55	if (__last1 - __first1 != __last2 - __first2)
56	56	return false;
57	57	}
58		auto __unwrapped1 = std::__unwrap_range(std::move(__first1), std::move(__last1));
59		auto __unwrapped2 = std::__unwrap_range(std::move(__first2), std::move(__last2));
60		return std::__equal_impl(
61		std::move(__unwrapped1.first),
62		std::move(__unwrapped1.second),
63		std::move(__unwrapped2.first),
64		std::move(__unwrapped2.second),
65		__pred,
66		__proj1,
67		__proj2);
	58
	59	auto [__ufirst1, __ulast1] = std::__unwrap_range(std::move(__first1), std::move(__last1));
	60	auto [__ufirst2, __ulast2] = std::__unwrap_range(std::move(__first2), std::move(__last2));
	61
	62	return std::__equal_impl<__both_sized>(
	63	std::move(__ufirst1), std::move(__ulast1), std::move(__ufirst2), std::move(__ulast2), __pred, __proj1, __proj2);
68	64	}
69	65
70	66	template <input_range _Range1,
...	...	@@ -75,21 +71,16 @@ struct __equal {
75	71	requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
76	72	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
77	73	_Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
78		if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
79		if (ranges::distance(__range1) != ranges::distance(__range2))
	74	static constexpr bool __both_sized = sized_range<_Range1> && sized_range<_Range2>;
	75	if constexpr (__both_sized) {
	76	if (ranges::size(__range1) != ranges::size(__range2))
80	77	return false;
81	78	}
82		auto __unwrapped1 = std::__unwrap_range(ranges::begin(__range1), ranges::end(__range1));
83		auto __unwrapped2 = std::__unwrap_range(ranges::begin(__range2), ranges::end(__range2));
84		return std::__equal_impl(
85		std::move(__unwrapped1.first),
86		std::move(__unwrapped1.second),
87		std::move(__unwrapped2.first),
88		std::move(__unwrapped2.second),
89		__pred,
90		__proj1,
91		__proj2);
92		return false;
	79
	80	auto [__ufirst1, __ulast1] = std::__unwrap_range(ranges::begin(__range1), ranges::end(__range1));
	81	auto [__ufirst2, __ulast2] = std::__unwrap_range(ranges::begin(__range2), ranges::end(__range2));
	82	return std::__equal_impl<__both_sized>(
	83	std::move(__ufirst1), std::move(__ulast1), std::move(__ufirst2), std::move(__ulast2), __pred, __proj1, __proj2);
93	84	}
94	85	};
95	86

lib/libcxx/include/__algorithm/ranges_fill.h+7-6

...	...	@@ -9,12 +9,14 @@
9	9	#ifndef _LIBCPP___ALGORITHM_RANGES_FILL_H
10	10	#define _LIBCPP___ALGORITHM_RANGES_FILL_H
11	11
12		#include <__algorithm/ranges_fill_n.h>
	12	#include <__algorithm/fill.h>
	13	#include <__algorithm/fill_n.h>
13	14	#include <__config>
14	15	#include <__iterator/concepts.h>
15	16	#include <__ranges/access.h>
16	17	#include <__ranges/concepts.h>
17	18	#include <__ranges/dangling.h>
	19	#include <__utility/move.h>
18	20
19	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
20	22	# pragma GCC system_header
...	...	@@ -31,12 +33,11 @@ namespace ranges {
31	33	struct __fill {
32	34	template <class _Type, output_iterator<const _Type&> _Iter, sentinel_for<_Iter> _Sent>
33	35	_LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last, const _Type& __value) const {
34		if constexpr (random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
35		return ranges::fill_n(__first, __last - __first, __value);
	36	if constexpr (sized_sentinel_for<_Sent, _Iter>) {
	37	auto __n = __last - __first;
	38	return std::__fill_n(std::move(__first), __n, __value);
36	39	} else {
37		for (; __first != __last; ++__first)
38		*__first = __value;
39		return __first;
	40	return std::__fill(std::move(__first), std::move(__last), __value);
40	41	}
41	42	}
42	43

lib/libcxx/include/__algorithm/ranges_for_each.h+9-1

...	...	@@ -12,6 +12,7 @@
12	12	#include <__algorithm/for_each.h>
13	13	#include <__algorithm/for_each_n.h>
14	14	#include <__algorithm/in_fun_result.h>
	15	#include <__algorithm/specialized_algorithms.h>
15	16	#include <__concepts/assignable.h>
16	17	#include <__config>
17	18	#include <__functional/identity.h>
...	...	@@ -20,6 +21,7 @@
20	21	#include <__ranges/access.h>
21	22	#include <__ranges/concepts.h>
22	23	#include <__ranges/dangling.h>
	24	#include <__type_traits/remove_cvref.h>
23	25	#include <__utility/move.h>
24	26
25	27	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -71,7 +73,13 @@ public:
71	73	indirectly_unary_invocable<projected<iterator_t<_Range>, _Proj>> _Func>
72	74	_LIBCPP_HIDE_FROM_ABI constexpr for_each_result<borrowed_iterator_t<_Range>, _Func>
73	75	operator()(_Range&& __range, _Func __func, _Proj __proj = {}) const {
74		return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
	76	using _SpecialAlg = __specialized_algorithm<_Algorithm::__for_each, __single_range<remove_cvref_t<_Range>>>;
	77	if constexpr (_SpecialAlg::__has_algorithm) {
	78	auto [__iter, __func2] = _SpecialAlg()(__range, std::move(__func), std::move(__proj));
	79	return {std::move(__iter), std::move(__func)};
	80	} else {
	81	return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
	82	}
75	83	}
76	84	};
77	85

lib/libcxx/include/__algorithm/ranges_generate_n.h+2-6

...	...	@@ -9,6 +9,7 @@
9	9	#ifndef _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
10	10	#define _LIBCPP___ALGORITHM_RANGES_GENERATE_N_H
11	11
	12	#include <__algorithm/generate_n.h>
12	13	#include <__concepts/constructible.h>
13	14	#include <__concepts/invocable.h>
14	15	#include <__config>
...	...	@@ -38,12 +39,7 @@ struct __generate_n {
38	39	requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
39	40	_LIBCPP_HIDE_FROM_ABI constexpr _OutIter
40	41	operator()(_OutIter __first, iter_difference_t<_OutIter> __n, _Func __gen) const {
41		for (; __n > 0; --__n) {
42		*__first = __gen();
43		++__first;
44		}
45
46		return __first;
	42	return std::__generate_n(std::move(__first), __n, __gen);
47	43	}
48	44	};
49	45

lib/libcxx/include/__algorithm/ranges_search_n.h+2-2

...	...	@@ -54,8 +54,8 @@ struct __search_n {
54	54	}
55	55
56	56	if constexpr (random_access_iterator<_Iter1>) {
57		auto __ret = std::__search_n_random_access_impl<_RangeAlgPolicy>(
58		__first, __last, __count, __value, __pred, __proj, __size);
	57	auto __ret =
	58	std::__search_n_random_access_impl<_RangeAlgPolicy>(__first, __count, __value, __pred, __proj, __size);
59	59	return {std::move(__ret.first), std::move(__ret.second)};
60	60	}
61	61	}

lib/libcxx/include/__algorithm/rotate.h+27-44

...	...	@@ -12,16 +12,13 @@
12	12	#include <__algorithm/copy.h>
13	13	#include <__algorithm/copy_backward.h>
14	14	#include <__algorithm/iterator_operations.h>
	15	#include <__algorithm/min.h>
15	16	#include <__algorithm/move.h>
16	17	#include <__algorithm/move_backward.h>
17	18	#include <__algorithm/swap_ranges.h>
18	19	#include <__config>
19		#include <__cstddef/size_t.h>
20	20	#include <__fwd/bit_reference.h>
21	21	#include <__iterator/iterator_traits.h>
22		#include <__memory/construct_at.h>
23		#include <__memory/pointer_traits.h>
24		#include <__type_traits/is_constant_evaluated.h>
25	22	#include <__type_traits/is_trivially_assignable.h>
26	23	#include <__utility/move.h>
27	24	#include <__utility/pair.h>
...	...	@@ -89,46 +86,32 @@ __rotate_forward(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIt
89	86	return __r;
90	87	}
91	88
92		template <typename _Integral>
93		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Integral __algo_gcd(_Integral __x, _Integral __y) {
94		do {
95		_Integral __t = __x % __y;
96		__x = __y;
97		__y = __t;
98		} while (__y);
99		return __x;
100		}
101
102		template <class _AlgPolicy, typename _RandomAccessIterator>
103		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _RandomAccessIterator
104		__rotate_gcd(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) {
105		typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
106		typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
107		using _Ops = _IterOps<_AlgPolicy>;
108
109		const difference_type __m1 = __middle - __first;
110		const difference_type __m2 = _Ops::distance(__middle, __last);
111		if (__m1 == __m2) {
112		std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle, __last);
113		return __middle;
114		}
115		const difference_type __g = std::__algo_gcd(__m1, __m2);
116		for (_RandomAccessIterator __p = __first + __g; __p != __first;) {
117		value_type __t(_Ops::__iter_move(--__p));
118		_RandomAccessIterator __p1 = __p;
119		_RandomAccessIterator __p2 = __p1 + __m1;
120		do {
121		*__p1 = _Ops::__iter_move(__p2);
122		__p1 = __p2;
123		const difference_type __d = _Ops::distance(__p2, __last);
124		if (__m1 < __d)
125		__p2 += __m1;
126		else
127		__p2 = __first + (__m1 - __d);
128		} while (__p2 != __p);
129		*__p1 = std::move(__t);
	89	template <class _AlgPolicy, class _Iter, class _Sent>
	90	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter
	91	__rotate_random_access(_Iter __first, _Iter __middle, _Sent __sent) {
	92	auto __left = _IterOps<_AlgPolicy>::distance(__first, __middle);
	93	auto __right = _IterOps<_AlgPolicy>::distance(__middle, __sent);
	94	auto __last = __first + __right;
	95
	96	auto __min_len = std::min(__left, __right);
	97
	98	while (__min_len > 0) {
	99	if (__left <= __right) {
	100	do {
	101	std::__swap_ranges<_AlgPolicy>(__first, __first + __left, __first + __left);
	102	__first += __left;
	103	__right -= __left;
	104	} while (__left <= __right);
	105	__min_len = __right;
	106	} else {
	107	do {
	108	std::__swap_ranges<_AlgPolicy>(__first + (__left - __right), __first + __left, __first + __left);
	109	__left -= __right;
	110	} while (__left > __right);
	111	__min_len = __left;
	112	}
130	113	}
131		return __first + __m2;
	114	return __last;
132	115	}
133	116
134	117	template <class _AlgPolicy, class _ForwardIterator>
...	...	@@ -170,7 +153,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator
170	153	return std::__rotate_left<_AlgPolicy>(__first, __last);
171	154	if (_IterOps<_AlgPolicy>::next(__middle) == __last)
172	155	return std::__rotate_right<_AlgPolicy>(__first, __last);
173		return std::__rotate_gcd<_AlgPolicy>(__first, __middle, __last);
	156	return std::__rotate_random_access<_AlgPolicy>(__first, __middle, __last);
174	157	}
175	158	return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last);
176	159	}

lib/libcxx/include/__algorithm/search_n.h+49-37

...	...	@@ -14,11 +14,7 @@
14	14	#include <__algorithm/iterator_operations.h>
15	15	#include <__config>
16	16	#include <__functional/identity.h>
17		#include <__iterator/advance.h>
18		#include <__iterator/concepts.h>
19		#include <__iterator/distance.h>
20	17	#include <__iterator/iterator_traits.h>
21		#include <__ranges/concepts.h>
22	18	#include <__type_traits/enable_if.h>
23	19	#include <__type_traits/invoke.h>
24	20	#include <__type_traits/is_callable.h>
...	...	@@ -68,44 +64,60 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter> __search_
68	64	}
69	65	}
70	66
71		template <class _AlgPolicy, class _Pred, class _Iter, class _Sent, class _SizeT, class _Type, class _Proj, class _DiffT>
	67	// Finds the longest suffix in [__first, __last) where each element satisfies __pred.
	68	template <class _RAIter, class _Pred, class _Proj, class _ValueT>
	69	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RAIter
	70	__find_longest_suffix(_RAIter __first, _RAIter __last, const _ValueT& __value, _Pred& __pred, _Proj& __proj) {
	71	while (__first != __last) {
	72	if (!std::__invoke(__pred, std::__invoke(__proj, *--__last), __value)) {
	73	return ++__last;
	74	}
	75	}
	76	return __first;
	77	}
	78
	79	template <class _AlgPolicy, class _Pred, class _Iter, class _SizeT, class _Type, class _Proj, class _DiffT>
72	80	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 std::pair<_Iter, _Iter> __search_n_random_access_impl(
73		_Iter __first, _Sent __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj, _DiffT __size1) {
74		using difference_type = typename iterator_traits<_Iter>::difference_type;
	81	_Iter __first, _SizeT __count_in, const _Type& __value, _Pred& __pred, _Proj& __proj, _DiffT __size) {
	82	auto __last = __first + __size;
	83	auto __count = static_cast<_DiffT>(__count_in);
	84
75	85	if (__count == 0)
76	86	return std::make_pair(__first, __first);
77		if (__size1 < static_cast<_DiffT>(__count)) {
78		_IterOps<_AlgPolicy>::__advance_to(__first, __last);
79		return std::make_pair(__first, __first);
80		}
	87	if (__size < __count)
	88	return std::make_pair(__last, __last);
	89
	90	// [__match_start, __match_start + __count) is the subrange which we currently check whether it only contains matching
	91	// elements. This subrange is returned in case all the elements match.
	92	// [__match_start, __matched_until) is the longest subrange where all elements are known to match at any given point
	93	// in time.
	94	// [__matched_until, __match_start + __count) is the subrange where we don't know whether the elements match.
	95
	96	// This algorithm tries to expand the subrange [__match_start, __matched_until) into a range of sufficient length.
	97	// When we fail to do that because we find a mismatching element, we move it forward to the beginning of the next
	98	// consecutive sequence that is not known not to match.
	99
	100	const _Iter __try_match_until = __last - __count;
	101	_Iter __match_start = __first;
	102	_Iter __matched_until = __first;
81	103
82		const auto __s = __first + __size1 - difference_type(__count - 1); // Start of pattern match can't go beyond here
83	104	while (true) {
84		// Find first element in sequence that matchs __value, with a mininum of loop checks
85		while (true) {
86		if (__first >= __s) { // return __last if no element matches __value
87		_IterOps<_AlgPolicy>::__advance_to(__first, __last);
88		return std::make_pair(__first, __first);
89		}
90		if (std::__invoke(__pred, std::__invoke(__proj, *__first), __value))
91		break;
92		++__first;
93		}
94		// *__first matches __value_, now match elements after here
95		auto __m = __first;
96		_SizeT __c(0);
97		while (true) {
98		if (++__c == __count) // If pattern exhausted, __first is the answer (works for 1 element pattern)
99		return std::make_pair(__first, __first + _DiffT(__count));
100		++__m; // no need to check range on __m because __s guarantees we have enough source
	105	// There's no chance of expanding the subrange into a sequence of sufficient length, since we don't have enough
	106	// elements in the haystack anymore.
	107	if (__match_start > __try_match_until)
	108	return std::make_pair(__last, __last);
101	109
102		// if there is a mismatch, restart with a new __first
103		if (!std::__invoke(__pred, std::__invoke(__proj, *__m), __value)) {
104		__first = __m;
105		++__first;
106		break;
107		} // else there is a match, check next elements
108		}
	110	auto __mismatch = std::__find_longest_suffix(__matched_until, __match_start + __count, __value, __pred, __proj);
	111
	112	// If all elements in [__matched_until, __match_start + __count) match, we know that
	113	// [__match_start, __match_start + __count) is a full sequence of matching elements, so we're done.
	114	if (__mismatch == __matched_until)
	115	return std::make_pair(__match_start, __match_start + __count);
	116
	117	// Otherwise, we have to move the [__match_start, __matched_until) subrange forward past the point where we know for
	118	// sure a match is impossible.
	119	__matched_until = __match_start + __count;
	120	__match_start = __mismatch;
109	121	}
110	122	}
111	123
...	...	@@ -119,7 +131,7 @@ template <class _Iter,
119	131	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_Iter, _Iter>
120	132	__search_n_impl(_Iter __first, _Sent __last, _DiffT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
121	133	return std::__search_n_random_access_impl<_ClassicAlgPolicy>(
122		__first, __last, __count, __value, __pred, __proj, __last - __first);
	134	__first, __count, __value, __pred, __proj, __last - __first);
123	135	}
124	136
125	137	template <class _Iter1,

lib/libcxx/include/__algorithm/sift_down.h+19-18

...	...	@@ -24,59 +24,60 @@ _LIBCPP_PUSH_MACROS
24	24
25	25	_LIBCPP_BEGIN_NAMESPACE_STD
26	26
27		template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>
	27	template <class _AlgPolicy, bool __assume_both_children, class _Compare, class _RandomAccessIterator>
28	28	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
29	29	__sift_down(_RandomAccessIterator __first,
30	30	_Compare&& __comp,
31		typename iterator_traits<_RandomAccessIterator>::difference_type __len,
32		_RandomAccessIterator __start) {
	31	__iterator_difference_type<_RandomAccessIterator> __len,
	32	__iterator_difference_type<_RandomAccessIterator> __start) {
33	33	using _Ops = _IterOps<_AlgPolicy>;
34	34
35	35	typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
36	36	typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
37	37	// left-child of __start is at 2 * __start + 1
38	38	// right-child of __start is at 2 * __start + 2
39		difference_type __child = __start - __first;
	39	difference_type __child = __start;
40	40
41	41	if (__len < 2 \|\| (__len - 2) / 2 < __child)
42	42	return;
43	43
44		__child = 2 * __child + 1;
45		_RandomAccessIterator __child_i = __first + __child;
	44	__child = 2 * __child + 1;
46	45
47		if ((__child + 1) < __len && __comp(__child_i, (__child_i + difference_type(1)))) {
	46	if _LIBCPP_CONSTEXPR (__assume_both_children) {
	47	// right-child exists and is greater than left-child
	48	__child += __comp(__first[__child], __first[__child + 1]);
	49	} else if ((__child + 1) < __len && __comp(__first[__child], __first[__child + 1])) {
48	50	// right-child exists and is greater than left-child
49		++__child_i;
50	51	++__child;
51	52	}
52	53
53	54	// check if we are in heap-order
54		if (__comp(__child_i, __start))
	55	if (__comp(__first[__child], __first[__start]))
55	56	// we are, __start is larger than its largest child
56	57	return;
57	58
58		value_type __top(_Ops::__iter_move(__start));
	59	value_type __top(_Ops::__iter_move(__first + __start));
59	60	do {
60	61	// we are not in heap-order, swap the parent with its largest child
61		*__start = _Ops::__iter_move(__child_i);
62		__start = __child_i;
	62	__first[__start] = _Ops::__iter_move(__first + __child);
	63	__start = __child;
63	64
64	65	if ((__len - 2) / 2 < __child)
65	66	break;
66	67
67	68	// recompute the child based off of the updated parent
68		__child = 2 * __child + 1;
69		__child_i = __first + __child;
	69	__child = 2 * __child + 1;
70	70
71		if ((__child + 1) < __len && __comp(__child_i, (__child_i + difference_type(1)))) {
	71	if _LIBCPP_CONSTEXPR (__assume_both_children) {
	72	__child += __comp(__first[__child], __first[__child + 1]);
	73	} else if ((__child + 1) < __len && __comp(__first[__child], __first[__child + 1])) {
72	74	// right-child exists and is greater than left-child
73		++__child_i;
74	75	++__child;
75	76	}
76	77
77	78	// check if we are in heap-order
78		} while (!__comp(*__child_i, __top));
79		*__start = std::move(__top);
	79	} while (!__comp(__first[__child], __top));
	80	__first[__start] = std::move(__top);
80	81	}
81	82
82	83	template <class _AlgPolicy, class _Compare, class _RandomAccessIterator>

lib/libcxx/include/__algorithm/simd_utils.h+33-4

...	...	@@ -26,9 +26,7 @@ _LIBCPP_PUSH_MACROS
26	26	#include <__undef_macros>
27	27
28	28	// TODO: Find out how altivec changes things and allow vectorizations there too.
29		// TODO: Simplify this condition once we stop building with AppleClang 15 in the CI.
30		#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(__ALTIVEC__) && \
31		!(defined(_LIBCPP_APPLE_CLANG_VER) && _LIBCPP_APPLE_CLANG_VER < 1600)
	29	#if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(__ALTIVEC__)
32	30	# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1
33	31	#else
34	32	# define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
...	...	@@ -116,16 +114,47 @@ template <class _VecT, class _Iter>
116	114	}(make_index_sequence<__simd_vector_size_v<_VecT>>{});
117	115	}
118	116
	117	// Load the first _Np elements, zero the rest
	118	_LIBCPP_DIAGNOSTIC_PUSH
	119	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wpsabi")
	120	template <class _VecT, size_t _Np, class _Iter>
	121	[[__nodiscard__]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __partial_load(_Iter __iter) noexcept {
	122	return [=]<size_t... _LoadIndices, size_t... _ZeroIndices>(
	123	index_sequence<_LoadIndices...>, index_sequence<_ZeroIndices...>) _LIBCPP_ALWAYS_INLINE noexcept {
	124	return _VecT{__iter[_LoadIndices]..., ((void)_ZeroIndices, 0)...};
	125	}(make_index_sequence<_Np>{}, make_index_sequence<__simd_vector_size_v<_VecT> - _Np>{});
	126	}
	127
	128	// Create a vector where every elements is __val
	129	template <class _VecT>
	130	[[__nodiscard__]] _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT
	131	__broadcast(__simd_vector_underlying_type_t<_VecT> __val) {
	132	return [&]<std::size_t... _Indices>(index_sequence<_Indices...>) {
	133	return _VecT{((void)_Indices, __val)...};
	134	}(make_index_sequence<__simd_vector_size_v<_VecT>>());
	135	}
	136	_LIBCPP_DIAGNOSTIC_POP
	137
	138	template <class _Tp, size_t _Np>
	139	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool __any_of(__simd_vector<_Tp, _Np> __vec) noexcept {
	140	return __builtin_reduce_or(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
	141	}
	142
119	143	template <class _Tp, size_t _Np>
120	144	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
121	145	return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
122	146	}
123	147
	148	template <class _Tp, size_t _Np>
	149	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool __none_of(__simd_vector<_Tp, _Np> __vec) noexcept {
	150	return !__builtin_reduce_or(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
	151	}
	152
124	153	template <class _Tp, size_t _Np>
125	154	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
126	155	using __mask_vec = __simd_vector<bool, _Np>;
127	156
128		// This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
	157	// This has MSan disabled du to https://llvm.org/PR85876
129	158	auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
130	159	# if defined(_LIBCPP_BIG_ENDIAN)
131	160	return std::min<size_t>(

lib/libcxx/include/__algorithm/specialized_algorithms.h created+54

...	...	@@ -0,0 +1,54 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H
	10	#define _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H
	11
	12	#include <__config>
	13
	14	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	15	# pragma GCC system_header
	16	#endif
	17
	18	_LIBCPP_BEGIN_NAMESPACE_STD
	19
	20	namespace _Algorithm {
	21	struct __copy {};
	22	struct __fill_n {};
	23	struct __for_each {};
	24	} // namespace _Algorithm
	25
	26	template <class>
	27	struct __single_iterator;
	28
	29	template <class, class>
	30	struct __iterator_pair;
	31
	32	template <class>
	33	struct __single_range;
	34
	35	// This struct allows specializing algorithms for specific arguments. This is useful when we know a more efficient
	36	// algorithm implementation for e.g. library-defined iterators. _Alg is one of tags defined inside the _Algorithm
	37	// namespace above. _Ranges is an essentially arbitrary subset of the arguments to the algorithm that are used for
	38	// dispatching. This set is specific to the algorithm: look at each algorithm to see which arguments they use for
	39	// dispatching to specialized algorithms.
	40	//
	41	// A specialization of `__specialized_algorithm` has to define `__has_algorithm` to true for the specialized algorithm
	42	// to be used. This is intended for cases where iterators can do generic unwrapping and forward to a different
	43	// specialization of `__specialized_algorithm`.
	44	//
	45	// If __has_algorithm is true, there has to be an operator() which will get called with the actual arguments to the
	46	// algorithm.
	47	template <class _Alg, class... _Ranges>
	48	struct __specialized_algorithm {
	49	static const bool __has_algorithm = false;
	50	};
	51
	52	_LIBCPP_END_NAMESPACE_STD
	53
	54	#endif // _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H

lib/libcxx/include/__algorithm/stable_sort.h+1-1

...	...	@@ -247,7 +247,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void __stable_sort(
247	247	constexpr auto __default_comp = __desugars_to_v<__less_tag, _Compare, value_type, value_type >;
248	248	constexpr auto __radix_sortable =
249	249	__is_ordered_integer_representable_v<value_type> &&
250		is_same_v< value_type&, __iter_reference<_RandomAccessIterator>>;
	250	is_same_v< value_type&, __iterator_reference<_RandomAccessIterator>>;
251	251	if constexpr (__default_comp && __radix_sortable) {
252	252	if (__len <= __buff_size && __len >= static_cast<difference_type>(std::__radix_sort_min_bound<value_type>()) &&
253	253	__len <= static_cast<difference_type>(std::__radix_sort_max_bound<value_type>())) {

lib/libcxx/include/__assertion_handler+31-4

...	...	@@ -16,6 +16,7 @@
16	16	# include <__cxx03/__verbose_trap>
17	17	#else
18	18	# include <__config>
	19	# include <__log_hardening_failure>
19	20	# include <__verbose_abort>
20	21	# include <__verbose_trap>
21	22	#endif
...	...	@@ -24,14 +25,40 @@
24	25	# pragma GCC system_header
25	26	#endif
26	27
27		#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
	28	#if __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
28	29
29		# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message)
	30	// Keep the old implementation that doesn't support assertion semantics for backward compatibility with the frozen C++03
	31	// mode.
	32	# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
	33	# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message)
	34	# else
	35	# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message)
	36	# endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
30	37
31	38	#else
32	39
33		# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message)
	40	# if _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_IGNORE
	41	# define _LIBCPP_ASSERTION_HANDLER(message) ((void)0)
	42
	43	# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_OBSERVE
	44	# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_LOG_HARDENING_FAILURE(message)
	45
	46	# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE
	47	# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_TRAP(message)
	48
	49	# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	50	# define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message)
	51
	52	# else
	53
	54	# error _LIBCPP_ASSERTION_SEMANTIC must be set to one of the following values: \
	55	_LIBCPP_ASSERTION_SEMANTIC_IGNORE, \
	56	_LIBCPP_ASSERTION_SEMANTIC_OBSERVE, \
	57	_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE, \
	58	_LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	59
	60	# endif // _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_IGNORE
34	61
35		#endif // _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
	62	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
36	63
37	64	#endif // _LIBCPP___ASSERTION_HANDLER

lib/libcxx/include/__atomic/atomic.h+36-60

...	...	@@ -10,7 +10,9 @@
10	10	#define _LIBCPP___ATOMIC_ATOMIC_H
11	11
12	12	#include <__atomic/atomic_sync.h>
	13	#include <__atomic/atomic_waitable_traits.h>
13	14	#include <__atomic/check_memory_order.h>
	15	#include <__atomic/floating_point_helper.h>
14	16	#include <__atomic/is_always_lock_free.h>
15	17	#include <__atomic/memory_order.h>
16	18	#include <__atomic/support.h>
...	...	@@ -47,10 +49,10 @@ struct __atomic_base // false
47	49	static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;
48	50	#endif
49	51
50		_LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
	52	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {
51	53	return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));
52	54	}
53		_LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
	55	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {
54	56	return static_cast<__atomic_base const volatile*>(this)->is_lock_free();
55	57	}
56	58	_LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT
...	...	@@ -61,11 +63,11 @@ struct __atomic_base // false
61	63	_LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {
62	64	std::__cxx_atomic_store(std::addressof(__a_), __d, __m);
63	65	}
64		_LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
	66	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT
65	67	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
66	68	return std::__cxx_atomic_load(std::addressof(__a_), __m);
67	69	}
68		_LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
	70	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT
69	71	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
70	72	return std::__cxx_atomic_load(std::addressof(__a_), __m);
71	73	}
...	...	@@ -113,22 +115,16 @@ struct __atomic_base // false
113	115	}
114	116
115	117	#if _LIBCPP_STD_VER >= 20
116		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const
117		volatile _NOEXCEPT {
	118	_LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
118	119	std::__atomic_wait(*this, __v, __m);
119	120	}
120		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
121		wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
	121	_LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
122	122	std::__atomic_wait(*this, __v, __m);
123	123	}
124		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
125		std::__atomic_notify_one(*this);
126		}
127		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
128		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
129		std::__atomic_notify_all(*this);
130		}
131		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
	124	_LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); }
	125	_LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
	126	_LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); }
	127	_LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
132	128	#endif // _LIBCPP_STD_VER >= 20
133	129
134	130	#if _LIBCPP_STD_VER >= 20
...	...	@@ -205,12 +201,15 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {
205	201	_LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }
206	202	};
207	203
	204	#if _LIBCPP_STD_VER >= 20
208	205	// Here we need _IsIntegral because the default template argument is not enough
209	206	// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from
210	207	// __atomic_base<int, false> and the caller of the wait function is
211	208	// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work
212	209	template <class _Tp, bool _IsIntegral>
213	210	struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
	211	using __value_type _LIBCPP_NODEBUG = _Tp;
	212
214	213	static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {
215	214	return __a.load(__order);
216	215	}
...	...	@@ -231,6 +230,8 @@ struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {
231	230	}
232	231	};
233	232
	233	#endif // _LIBCPP_STD_VER >= 20
	234
234	235	template <typename _Tp>
235	236	struct __check_atomic_mandates {
236	237	using type _LIBCPP_NODEBUG = _Tp;
...	...	@@ -324,50 +325,26 @@ struct atomic<_Tp> : public __atomic_base<_Tp> {
324	325	atomic& operator=(const atomic&) volatile = delete;
325	326	};
326	327
	328	#if _LIBCPP_STD_VER >= 20
327	329	template <class _Tp>
328	330	struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};
329	331
330		#if _LIBCPP_STD_VER >= 20
331	332	template <class _Tp>
332	333	requires is_floating_point_v<_Tp>
333	334	struct atomic<_Tp> : __atomic_base<_Tp> {
334	335	private:
335		_LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
336		// Only x87-fp80 long double has 64-bit mantissa
337		return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
338		}
339
340		_LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
341		# ifndef _LIBCPP_COMPILER_CLANG_BASED
342		return false;
343		# else
344		// The builtin __cxx_atomic_fetch_add errors during compilation for
345		// long double on platforms with fp80 format.
346		// For more details, see
347		// lib/Sema/SemaChecking.cpp function IsAllowedValueType
348		// LLVM Parser does not allow atomicrmw with x86_fp80 type.
349		// if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
350		// &Context.getTargetInfo().getLongDoubleFormat() ==
351		// &llvm::APFloat::x87DoubleExtended())
352		// For more info
353		// https://github.com/llvm/llvm-project/issues/68602
354		// https://reviews.llvm.org/D53965
355		return !__is_fp80_long_double();
356		# endif
357		}
358
359	336	template <class _This, class _Operation, class _BuiltinOp>
360	337	_LIBCPP_HIDE_FROM_ABI static _Tp
361	338	__rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
362		if constexpr (__has_rmw_builtin()) {
	339	if constexpr (std::__has_rmw_builtin<_Tp>()) {
363	340	return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
364	341	} else {
365	342	_Tp __old = __self.load(memory_order_relaxed);
366	343	_Tp __new = __operation(__old, __operand);
367	344	while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
368	345	# ifdef _LIBCPP_COMPILER_CLANG_BASED
369		if constexpr (__is_fp80_long_double()) {
370		// https://github.com/llvm/llvm-project/issues/47978
	346	if constexpr (std::__is_fp80_long_double<_Tp>()) {
	347	// https://llvm.org/PR47978
371	348	// clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
372	349	// Note __old = __self.load(memory_order_relaxed) will not work
373	350	std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);
...	...	@@ -462,12 +439,12 @@ public:
462	439	// atomic_is_lock_free
463	440
464	441	template <class _Tp>
465		_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
	442	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {
466	443	return __o->is_lock_free();
467	444	}
468	445
469	446	template <class _Tp>
470		_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
	447	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {
471	448	return __o->is_lock_free();
472	449	}
473	450
...	...	@@ -516,25 +493,25 @@ atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, me
516	493	// atomic_load
517	494
518	495	template <class _Tp>
519		_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
	496	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {
520	497	return __o->load();
521	498	}
522	499
523	500	template <class _Tp>
524		_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
	501	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {
525	502	return __o->load();
526	503	}
527	504
528	505	// atomic_load_explicit
529	506
530	507	template <class _Tp>
531		_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
532		_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
	508	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp
	509	atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
533	510	return __o->load(__m);
534	511	}
535	512
536	513	template <class _Tp>
537		_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
	514	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT
538	515	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
539	516	return __o->load(__m);
540	517	}
...	...	@@ -642,28 +619,27 @@ _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
642	619	// atomic_wait
643	620
644	621	template <class _Tp>
645		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
	622	_LIBCPP_HIDE_FROM_ABI void
646	623	atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
647	624	return __o->wait(__v);
648	625	}
649	626
650	627	template <class _Tp>
651		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
652		atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
	628	_LIBCPP_HIDE_FROM_ABI void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {
653	629	return __o->wait(__v);
654	630	}
655	631
656	632	// atomic_wait_explicit
657	633
658	634	template <class _Tp>
659		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
	635	_LIBCPP_HIDE_FROM_ABI void
660	636	atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
661	637	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
662	638	return __o->wait(__v, __m);
663	639	}
664	640
665	641	template <class _Tp>
666		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
	642	_LIBCPP_HIDE_FROM_ABI void
667	643	atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT
668	644	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {
669	645	return __o->wait(__v, __m);
...	...	@@ -672,22 +648,22 @@ atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __
672	648	// atomic_notify_one
673	649
674	650	template <class _Tp>
675		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
	651	_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
676	652	__o->notify_one();
677	653	}
678	654	template <class _Tp>
679		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
	655	_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
680	656	__o->notify_one();
681	657	}
682	658
683	659	// atomic_notify_all
684	660
685	661	template <class _Tp>
686		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
	662	_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
687	663	__o->notify_all();
688	664	}
689	665	template <class _Tp>
690		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
	666	_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
691	667	__o->notify_all();
692	668	}
693	669

lib/libcxx/include/__atomic/atomic_flag.h+19-37

...	...	@@ -10,6 +10,7 @@
10	10	#define _LIBCPP___ATOMIC_ATOMIC_FLAG_H
11	11
12	12	#include <__atomic/atomic_sync.h>
	13	#include <__atomic/atomic_waitable_traits.h>
13	14	#include <__atomic/contention_t.h>
14	15	#include <__atomic/memory_order.h>
15	16	#include <__atomic/support.h>
...	...	@@ -49,22 +50,16 @@ struct atomic_flag {
49	50	}
50	51
51	52	#if _LIBCPP_STD_VER >= 20
52		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const
53		volatile _NOEXCEPT {
	53	_LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
54	54	std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
55	55	}
56		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
57		wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
	56	_LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
58	57	std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
59	58	}
60		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
61		std::__atomic_notify_one(*this);
62		}
63		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
64		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
65		std::__atomic_notify_all(*this);
66		}
67		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
	59	_LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); }
	60	_LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
	61	_LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); }
	62	_LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
68	63	#endif
69	64
70	65	#if _LIBCPP_STD_VER >= 20
...	...	@@ -80,8 +75,11 @@ struct atomic_flag {
80	75	atomic_flag& operator=(const atomic_flag&) volatile = delete;
81	76	};
82	77
	78	#if _LIBCPP_STD_VER >= 20
83	79	template <>
84	80	struct __atomic_waitable_traits<atomic_flag> {
	81	using __value_type _LIBCPP_NODEBUG = _LIBCPP_ATOMIC_FLAG_TYPE;
	82
85	83	static _LIBCPP_HIDE_FROM_ABI _LIBCPP_ATOMIC_FLAG_TYPE __atomic_load(const atomic_flag& __a, memory_order __order) {
86	84	return std::__cxx_atomic_load(&__a.__a_, __order);
87	85	}
...	...	@@ -101,6 +99,7 @@ struct __atomic_waitable_traits<atomic_flag> {
101	99	return std::addressof(__a.__a_);
102	100	}
103	101	};
	102	#endif // _LIBCPP_STD_VER >= 20
104	103
105	104	inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); }
106	105
...	...	@@ -143,43 +142,26 @@ inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, m
143	142	}
144	143
145	144	#if _LIBCPP_STD_VER >= 20
146		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
147		atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT {
	145	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT {
148	146	__o->wait(__v);
149	147	}
150	148
151		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
152		atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT {
153		__o->wait(__v);
154		}
	149	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT { __o->wait(__v); }
155	150
156		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
	151	inline _LIBCPP_HIDE_FROM_ABI void
157	152	atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
158	153	__o->wait(__v, __m);
159	154	}
160	155
161		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
	156	inline _LIBCPP_HIDE_FROM_ABI void
162	157	atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
163	158	__o->wait(__v, __m);
164	159	}
165	160
166		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
167		atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT {
168		__o->notify_one();
169		}
170
171		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT {
172		__o->notify_one();
173		}
174
175		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
176		atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT {
177		__o->notify_all();
178		}
179
180		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT {
181		__o->notify_all();
182		}
	161	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT { __o->notify_one(); }
	162	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT { __o->notify_one(); }
	163	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT { __o->notify_all(); }
	164	inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT { __o->notify_all(); }
183	165	#endif // _LIBCPP_STD_VER >= 20
184	166
185	167	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__atomic/atomic_ref.h+29-12

...	...	@@ -19,7 +19,9 @@
19	19
20	20	#include <__assert>
21	21	#include <__atomic/atomic_sync.h>
	22	#include <__atomic/atomic_waitable_traits.h>
22	23	#include <__atomic/check_memory_order.h>
	24	#include <__atomic/floating_point_helper.h>
23	25	#include <__atomic/memory_order.h>
24	26	#include <__atomic/to_gcc_order.h>
25	27	#include <__concepts/arithmetic.h>
...	...	@@ -121,7 +123,9 @@ public:
121	123	static constexpr bool is_always_lock_free =
122	124	__atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance));
123	125
124		_LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }
	126	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept {
	127	return __atomic_is_lock_free(sizeof(_Tp), __ptr_);
	128	}
125	129
126	130	_LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept
127	131	_LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {
...	...	@@ -136,7 +140,7 @@ public:
136	140	return __desired;
137	141	}
138	142
139		_LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
	143	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept
140	144	_LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {
141	145	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
142	146	__order == memory_order::relaxed \|\| __order == memory_order::consume \|\| __order == memory_order::acquire \|\|
...	...	@@ -219,6 +223,9 @@ public:
219	223	}
220	224	_LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }
221	225	_LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }
	226	# if _LIBCPP_STD_VER >= 26
	227	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* address() const noexcept { return __ptr_; }
	228	# endif
222	229
223	230	protected:
224	231	using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp;
...	...	@@ -229,6 +236,8 @@ protected:
229	236
230	237	template <class _Tp>
231	238	struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {
	239	using __value_type _LIBCPP_NODEBUG = _Tp;
	240
232	241	static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {
233	242	return __a.load(__order);
234	243	}
...	...	@@ -322,20 +331,28 @@ struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {
322	331	atomic_ref& operator=(const atomic_ref&) = delete;
323	332
324	333	_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
325		_Tp __old = this->load(memory_order_relaxed);
326		_Tp __new = __old + __arg;
327		while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
328		__new = __old + __arg;
	334	if constexpr (std::__has_rmw_builtin<_Tp>()) {
	335	return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));
	336	} else {
	337	_Tp __old = this->load(memory_order_relaxed);
	338	_Tp __new = __old + __arg;
	339	while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
	340	__new = __old + __arg;
	341	}
	342	return __old;
329	343	}
330		return __old;
331	344	}
332	345	_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {
333		_Tp __old = this->load(memory_order_relaxed);
334		_Tp __new = __old - __arg;
335		while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
336		__new = __old - __arg;
	346	if constexpr (std::__has_rmw_builtin<_Tp>()) {
	347	return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));
	348	} else {
	349	_Tp __old = this->load(memory_order_relaxed);
	350	_Tp __new = __old - __arg;
	351	while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {
	352	__new = __old - __arg;
	353	}
	354	return __old;
337	355	}
338		return __old;
339	356	}
340	357
341	358	_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }

lib/libcxx/include/__atomic/atomic_sync.h+127-55

...	...	@@ -9,6 +9,7 @@
9	9	#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
10	10	#define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
11	11
	12	#include <__atomic/atomic_waitable_traits.h>
12	13	#include <__atomic/contention_t.h>
13	14	#include <__atomic/memory_order.h>
14	15	#include <__atomic/to_gcc_order.h>
...	...	@@ -19,6 +20,7 @@
19	20	#include <__type_traits/conjunction.h>
20	21	#include <__type_traits/decay.h>
21	22	#include <__type_traits/invoke.h>
	23	#include <__type_traits/is_same.h>
22	24	#include <__type_traits/void_t.h>
23	25	#include <__utility/declval.h>
24	26	#include <cstring>
...	...	@@ -29,50 +31,89 @@
29	31
30	32	_LIBCPP_BEGIN_NAMESPACE_STD
31	33
32		// The customisation points to enable the following functions:
33		// - __atomic_wait
34		// - __atomic_wait_unless
35		// - __atomic_notify_one
36		// - __atomic_notify_all
37		// Note that std::atomic<T>::wait was back-ported to C++03
38		// The below implementations look ugly to support C++03
39		template <class _Tp, class = void>
40		struct __atomic_waitable_traits {
41		template <class _AtomicWaitable>
42		static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
43
44		template <class _AtomicWaitable>
45		static void __atomic_contention_address(_AtomicWaitable&&) = delete;
46		};
47
48		template <class _Tp, class = void>
49		struct __atomic_waitable : false_type {};
50
51		template <class _Tp>
52		struct __atomic_waitable< _Tp,
53		__void_t<decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(
54		std::declval<const _Tp&>(), std::declval<memory_order>())),
55		decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(
56		std::declval<const _Tp&>()))> > : true_type {};
57
58	34	#if _LIBCPP_STD_VER >= 20
59	35	# if _LIBCPP_HAS_THREADS
60	36
61		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
62		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
63		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
64		__libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
65		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
66		__libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT;
67
68		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
69		__cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
70		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
71		__cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
72		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
	37	# if !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	38
	39	// old dylib interface kept for backwards compatibility
	40	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile*) _NOEXCEPT;
	41	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile*) _NOEXCEPT;
	42	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile*) _NOEXCEPT;
	43	_LIBCPP_EXPORTED_FROM_ABI void __libcpp_atomic_wait(void const volatile*, __cxx_contention_t) _NOEXCEPT;
	44
	45	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
	46	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
	47	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
73	48	__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile*) _NOEXCEPT;
74		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
	49	_LIBCPP_EXPORTED_FROM_ABI void
75	50	__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
	51	# endif // !_LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	52
	53	// new dylib interface
	54
	55	// return the global contention state's current value for the address
	56	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
	57	__atomic_monitor_global(void const* __address) _NOEXCEPT;
	58
	59	// wait on the global contention state to be changed from the given value for the address
	60	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
	61	__atomic_wait_global_table(void const* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
	62
	63	// notify one waiter waiting on the global contention state for the address
	64	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const*) _NOEXCEPT;
	65
	66	// notify all waiters waiting on the global contention state for the address
	67	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const*) _NOEXCEPT;
	68
	69	// wait on the address directly with the native platform wait
	70	template <std::size_t _Size>
	71	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
	72	__atomic_wait_native(void const* __address, void const* __old_value) _NOEXCEPT;
	73
	74	// notify one waiter waiting on the address directly with the native platform wait
	75	template <std::size_t _Size>
	76	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(const void*) _NOEXCEPT;
	77
	78	// notify all waiters waiting on the address directly with the native platform wait
	79	template <std::size_t _Size>
	80	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT;
	81
	82	# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	83
	84	template <class _AtomicWaitable, class _Poll>
	85	struct __atomic_wait_backoff_impl {
	86	const _AtomicWaitable& __a_;
	87	_Poll __poll_;
	88	memory_order __order_;
	89
	90	using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
	91	using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
	92
	93	_LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
	94	if (__elapsed > chrono::microseconds(4)) {
	95	auto __contention_address = const_cast<const void*>(
	96	static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
	97
	98	if constexpr (__has_native_atomic_wait<__value_type>) {
	99	auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
	100	if (__poll_(__atomic_value))
	101	return __backoff_results::__poll_success;
	102	std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
	103	} else {
	104	__cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
	105	auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
	106	if (__poll_(__atomic_value))
	107	return __backoff_results::__poll_success;
	108	std::__atomic_wait_global_table(__contention_address, __monitor_val);
	109	}
	110	} else {
	111	} // poll
	112	return __backoff_results::__continue_poll;
	113	}
	114	};
	115
	116	# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
76	117
77	118	template <class _AtomicWaitable, class _Poll>
78	119	struct __atomic_wait_backoff_impl {
...	...	@@ -82,7 +123,6 @@ struct __atomic_wait_backoff_impl {
82	123
83	124	using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
84	125
85		_LIBCPP_AVAILABILITY_SYNC
86	126	_LIBCPP_HIDE_FROM_ABI bool
87	127	__update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const {
88	128	// In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>,
...	...	@@ -95,7 +135,6 @@ struct __atomic_wait_backoff_impl {
95	135	return __poll_(__monitor_val);
96	136	}
97	137
98		_LIBCPP_AVAILABILITY_SYNC
99	138	_LIBCPP_HIDE_FROM_ABI bool
100	139	__update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const {
101	140	// In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t
...	...	@@ -105,20 +144,21 @@ struct __atomic_wait_backoff_impl {
105	144	return __poll_(__current_val);
106	145	}
107	146
108		_LIBCPP_AVAILABILITY_SYNC
109		_LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
	147	_LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
110	148	if (__elapsed > chrono::microseconds(4)) {
111	149	auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
112	150	__cxx_contention_t __monitor_val;
113	151	if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
114		return true;
	152	return __backoff_results::__poll_success;
115	153	std::__libcpp_atomic_wait(__contention_address, __monitor_val);
116	154	} else {
117	155	} // poll
118		return false;
	156	return __backoff_results::__continue_poll;
119	157	}
120	158	};
121	159
	160	# endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	161
122	162	// The semantics of this function are similar to `atomic`'s
123	163	// `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
124	164	// predicate (is the loaded value unequal to `old`?), the predicate function is
...	...	@@ -128,9 +168,8 @@ struct __atomic_wait_backoff_impl {
128	168	// `false`, it must set the argument to its current understanding of the atomic
129	169	// value. The predicate function must not return `false` spuriously.
130	170	template <class _AtomicWaitable, class _Poll>
131		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
132		__atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
133		static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
	171	_LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) {
	172	static_assert(__atomic_waitable<_AtomicWaitable>);
134	173	__atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order};
135	174	std::__libcpp_thread_poll_with_backoff(
136	175	/* poll */
...	...	@@ -141,18 +180,52 @@ __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& _
141	180	/* backoff */ __backoff_fn);
142	181	}
143	182
	183	# if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	184
144	185	template <class _AtomicWaitable>
145		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
146		static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
	186	_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
	187	static_assert(__atomic_waitable<_AtomicWaitable>);
	188	using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
	189	using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
	190	auto __contention_address =
	191	const_cast<const void>(static_cast<const volatile void>(__waitable_traits::__atomic_contention_address(__a)));
	192	if constexpr (__has_native_atomic_wait<__value_type>) {
	193	std::__atomic_notify_one_native<sizeof(__value_type)>(__contention_address);
	194	} else {
	195	std::__atomic_notify_one_global_table(__contention_address);
	196	}
	197	}
	198
	199	template <class _AtomicWaitable>
	200	_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
	201	static_assert(__atomic_waitable<_AtomicWaitable>);
	202	using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
	203	using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
	204	auto __contention_address =
	205	const_cast<const void>(static_cast<const volatile void>(__waitable_traits::__atomic_contention_address(__a)));
	206	if constexpr (__has_native_atomic_wait<__value_type>) {
	207	std::__atomic_notify_all_native<sizeof(__value_type)>(__contention_address);
	208	} else {
	209	std::__atomic_notify_all_global_table(__contention_address);
	210	}
	211	}
	212
	213	# else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	214
	215	template <class _AtomicWaitable>
	216	_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
	217	static_assert(__atomic_waitable<_AtomicWaitable>);
147	218	std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
148	219	}
149	220
150	221	template <class _AtomicWaitable>
151		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
152		static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
	222	_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
	223	static_assert(__atomic_waitable<_AtomicWaitable>);
153	224	std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
154	225	}
155	226
	227	# endif
	228
156	229	# else // _LIBCPP_HAS_THREADS
157	230
158	231	template <class _AtomicWaitable, class _Poll>
...	...	@@ -180,9 +253,8 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp c
180	253	}
181	254
182	255	template <class _AtomicWaitable, class _Tp>
183		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
184		__atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) {
185		static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
	256	_LIBCPP_HIDE_FROM_ABI void __atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) {
	257	static_assert(__atomic_waitable<_AtomicWaitable>);
186	258	std::__atomic_wait_unless(__a, __order, [&](_Tp const& __current) {
187	259	return !std::__cxx_nonatomic_compare_equal(__current, __val);
188	260	});

lib/libcxx/include/__atomic/atomic_sync_timed.h created+144

...	...	@@ -0,0 +1,144 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
	10	#define _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H
	11
	12	#include <__atomic/atomic_waitable_traits.h>
	13	#include <__atomic/contention_t.h>
	14	#include <__atomic/memory_order.h>
	15	#include <__atomic/to_gcc_order.h>
	16	#include <__chrono/duration.h>
	17	#include <__config>
	18	#include <__memory/addressof.h>
	19	#include <__thread/poll_with_backoff.h>
	20	#include <__thread/timed_backoff_policy.h>
	21	#include <__type_traits/conjunction.h>
	22	#include <__type_traits/decay.h>
	23	#include <__type_traits/has_unique_object_representation.h>
	24	#include <__type_traits/invoke.h>
	25	#include <__type_traits/is_same.h>
	26	#include <__type_traits/is_trivially_copyable.h>
	27	#include <__type_traits/void_t.h>
	28	#include <__utility/declval.h>
	29	#include <cstdint>
	30	#include <cstring>
	31
	32	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	33	# pragma GCC system_header
	34	#endif
	35
	36	_LIBCPP_BEGIN_NAMESPACE_STD
	37
	38	#if _LIBCPP_STD_VER >= 20
	39	# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	40
	41	_LIBCPP_AVAILABILITY_NEW_SYNC
	42	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __address) _NOEXCEPT;
	43
	44	// wait on the global contention state to be changed from the given value for the address
	45	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
	46	void const* __address, __cxx_contention_t __monitor_value, uint64_t __timeout_ns) _NOEXCEPT;
	47
	48	// wait on the address directly with the native platform wait
	49	template <std::size_t _Size>
	50	_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
	51	__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) _NOEXCEPT;
	52
	53	template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
	54	struct __atomic_wait_timed_backoff_impl {
	55	const _AtomicWaitable& __a_;
	56	_Poll __poll_;
	57	memory_order __order_;
	58	chrono::duration<_Rep, _Period> __rel_time_;
	59
	60	using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
	61	using __value_type _LIBCPP_NODEBUG = typename __waitable_traits::__value_type;
	62
	63	_LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
	64	if (__elapsed > chrono::microseconds(4)) {
	65	auto __contention_address = const_cast<const void*>(
	66	static_cast<const volatile void*>(__waitable_traits::__atomic_contention_address(__a_)));
	67
	68	uint64_t __timeout_ns =
	69	static_cast<uint64_t>((chrono::duration_cast<chrono::nanoseconds>(__rel_time_) - __elapsed).count());
	70
	71	if constexpr (__has_native_atomic_wait<__value_type>) {
	72	auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
	73	if (__poll_(__atomic_value))
	74	return __backoff_results::__poll_success;
	75	std::__atomic_wait_native_with_timeout<sizeof(__value_type)>(
	76	__contention_address, std::addressof(__atomic_value), __timeout_ns);
	77	} else {
	78	__cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
	79	auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
	80	if (__poll_(__atomic_value))
	81	return __backoff_results::__poll_success;
	82	std::__atomic_wait_global_table_with_timeout(__contention_address, __monitor_val, __timeout_ns);
	83	}
	84	} else {
	85	} // poll
	86	return __backoff_results::__continue_poll;
	87	}
	88	};
	89
	90	// The semantics of this function are similar to `atomic`'s
	91	// `.wait(T old, std::memory_order order)` with a timeout, but instead of having a hardcoded
	92	// predicate (is the loaded value unequal to `old`?), the predicate function is
	93	// specified as an argument. The loaded value is given as an in-out argument to
	94	// the predicate. If the predicate function returns `true`,
	95	// `__atomic_wait_unless_with_timeout` will return. If the predicate function returns
	96	// `false`, it must set the argument to its current understanding of the atomic
	97	// value. The predicate function must not return `false` spuriously.
	98	template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
	99	_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
	100	const _AtomicWaitable& __a,
	101	memory_order __order,
	102	_Poll&& __poll,
	103	chrono::duration<_Rep, _Period> const& __rel_time) {
	104	static_assert(__atomic_waitable<_AtomicWaitable>, "");
	105	__atomic_wait_timed_backoff_impl<_AtomicWaitable, __decay_t<_Poll>, _Rep, _Period> __backoff_fn = {
	106	__a, __poll, __order, __rel_time};
	107	auto __poll_result = std::__libcpp_thread_poll_with_backoff(
	108	/* poll */
	109	[&]() {
	110	auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
	111	return __poll(__current_val);
	112	},
	113	/* backoff */ __backoff_fn,
	114	__rel_time);
	115
	116	return __poll_result == __poll_with_backoff_results::__poll_success;
	117	}
	118
	119	# elif _LIBCPP_HAS_THREADS // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	120
	121	template <class _AtomicWaitable, class _Poll, class _Rep, class _Period>
	122	_LIBCPP_HIDE_FROM_ABI bool __atomic_wait_unless_with_timeout(
	123	const _AtomicWaitable& __a,
	124	memory_order __order,
	125	_Poll&& __poll,
	126	chrono::duration<_Rep, _Period> const& __rel_time) {
	127	auto __res = std::__libcpp_thread_poll_with_backoff(
	128	/* poll */
	129	[&]() {
	130	auto __current_val = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_load(__a, __order);
	131	return __poll(__current_val);
	132	},
	133	/* backoff */ __libcpp_timed_backoff_policy(),
	134	__rel_time);
	135	return __res == __poll_with_backoff_results::__poll_success;
	136	}
	137
	138	# endif // _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
	139
	140	#endif // C++20
	141
	142	_LIBCPP_END_NAMESPACE_STD
	143
	144	#endif // _LIBCPP___ATOMIC_ATOMIC_SYNC_TIMED_H

lib/libcxx/include/__atomic/atomic_waitable_traits.h created+103

...	...	@@ -0,0 +1,103 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H
	10	#define _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H
	11
	12	#include <__atomic/contention_t.h>
	13	#include <__atomic/memory_order.h>
	14	#include <__config>
	15	#include <__type_traits/decay.h>
	16	#include <__type_traits/has_unique_object_representation.h>
	17	#include <__type_traits/is_same.h>
	18	#include <__type_traits/is_trivially_copyable.h>
	19	#include <__type_traits/void_t.h>
	20	#include <__utility/declval.h>
	21	#include <cstring>
	22
	23	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	24	# pragma GCC system_header
	25	#endif
	26
	27	_LIBCPP_BEGIN_NAMESPACE_STD
	28
	29	#if _LIBCPP_STD_VER >= 20
	30
	31	// The customisation points to enable the following functions:
	32	// - __atomic_wait
	33	// - __atomic_wait_unless
	34	// - __atomic_notify_one
	35	// - __atomic_notify_all
	36	template <class _Tp, class = void>
	37	struct __atomic_waitable_traits {
	38	using __value_type _LIBCPP_NODEBUG = void;
	39
	40	template <class _AtomicWaitable>
	41	static void __atomic_load(_AtomicWaitable&&, memory_order) = delete;
	42
	43	template <class _AtomicWaitable>
	44	static void __atomic_contention_address(_AtomicWaitable&&) = delete;
	45	};
	46
	47	template <class _Tp>
	48	concept __atomic_waitable = requires(const _Tp __t, memory_order __order) {
	49	typename __atomic_waitable_traits<__decay_t<_Tp> >::__value_type;
	50	{ __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(__t, __order) };
	51	{ __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(__t) };
	52	};
	53
	54	# ifdef __linux__
	55	# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
	56	# elif defined(__APPLE__)
	57	# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \
	58	_APPLY(4) \
	59	_APPLY(8)
	60	# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
	61	# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
	62	# elif defined(_WIN32)
	63	# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
	64	# else
	65	# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
	66	# endif // __linux__
	67
	68	// concepts defines the types are supported natively by the platform's wait
	69
	70	# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
	71
	72	template <class _Tp>
	73	_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl() {
	74	if (alignof(_Tp) % sizeof(_Tp) != 0)
	75	return false;
	76	switch (sizeof(_Tp)) {
	77	# define _LIBCPP_MAKE_CASE(n) \
	78	case n: \
	79	return true;
	80	_LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
	81	default:
	82	return false;
	83	# undef _LIBCPP_MAKE_CASE
	84	};
	85	}
	86
	87	template <class _Tp>
	88	concept __has_native_atomic_wait =
	89	has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> &&
	90	std::__has_native_atomic_wait_impl<_Tp>();
	91
	92	# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
	93
	94	template <class _Tp>
	95	concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
	96
	97	# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
	98
	99	#endif // C++20
	100
	101	_LIBCPP_END_NAMESPACE_STD
	102
	103	#endif // _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H

lib/libcxx/include/__atomic/contention_t.h+27-3

...	...	@@ -19,11 +19,35 @@
19	19
20	20	_LIBCPP_BEGIN_NAMESPACE_STD
21	21
22		#if defined(__linux__) \|\| (defined(_AIX) && !defined(__64BIT__))
	22	// The original definition of `__cxx_contention_t` seemed a bit arbitrary.
	23	// When we enable the _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE ABI,
	24	// use definitions that are based on what the underlying platform supports
	25	// instead.
	26	#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
	27
	28	# ifdef __linux__
	29	using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
	30	# elif defined(__APPLE__)
	31	using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
	32	# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
	33	using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
	34	# elif defined(_AIX) && !defined(__64BIT__)
	35	using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
	36	# elif defined(_WIN32)
	37	using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
	38	# else
	39	using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
	40	# endif // __linux__
	41
	42	#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
	43
	44	# if defined(__linux__) \|\| (defined(_AIX) && !defined(__64BIT__))
23	45	using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
24		#else
	46	# else
25	47	using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
26		#endif // __linux__ \|\| (_AIX && !__64BIT__)
	48	# endif // __linux__ \|\| (_AIX && !__64BIT__)
	49
	50	#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
27	51
28	52	using __cxx_atomic_contention_t _LIBCPP_NODEBUG = __cxx_atomic_impl<__cxx_contention_t>;
29	53

lib/libcxx/include/__atomic/floating_point_helper.h created+55

...	...	@@ -0,0 +1,55 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H
	10	#define _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H
	11
	12	#include <__config>
	13	#include <__type_traits/is_floating_point.h>
	14	#include <__type_traits/is_same.h>
	15
	16	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	17	# pragma GCC system_header
	18	#endif
	19
	20	_LIBCPP_BEGIN_NAMESPACE_STD
	21
	22	#if _LIBCPP_STD_VER >= 20
	23
	24	template <class _Tp>
	25	_LIBCPP_HIDE_FROM_ABI constexpr bool __is_fp80_long_double() {
	26	// Only x87-fp80 long double has 64-bit mantissa
	27	return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
	28	}
	29
	30	template <class _Tp>
	31	_LIBCPP_HIDE_FROM_ABI constexpr bool __has_rmw_builtin() {
	32	static_assert(std::is_floating_point_v<_Tp>);
	33	# ifndef _LIBCPP_COMPILER_CLANG_BASED
	34	return false;
	35	# else
	36	// The builtin __cxx_atomic_fetch_add errors during compilation for
	37	// long double on platforms with fp80 format.
	38	// For more details, see
	39	// lib/Sema/SemaChecking.cpp function IsAllowedValueType
	40	// LLVM Parser does not allow atomicrmw with x86_fp80 type.
	41	// if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
	42	// &Context.getTargetInfo().getLongDoubleFormat() ==
	43	// &llvm::APFloat::x87DoubleExtended())
	44	// For more info
	45	// https://llvm.org/PR68602
	46	// https://reviews.llvm.org/D53965
	47	return !std::__is_fp80_long_double<_Tp>();
	48	# endif
	49	}
	50
	51	#endif
	52
	53	_LIBCPP_END_NAMESPACE_STD
	54
	55	#endif // _LIBCPP___ATOMIC_FLOATING_POINT_HELPER_H

lib/libcxx/include/__bit/countl.h+1-2

...	...	@@ -24,7 +24,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
24	24
25	25	template <class _Tp>
26	26	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 int __countl_zero(_Tp __t) _NOEXCEPT {
27		static_assert(__is_unsigned_integer_v<_Tp>, "__countl_zero requires an unsigned integer type");
28	27	return __builtin_clzg(__t, numeric_limits<_Tp>::digits);
29	28	}
30	29
...	...	@@ -37,7 +36,7 @@ template <__unsigned_integer _Tp>
37	36
38	37	template <__unsigned_integer _Tp>
39	38	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countl_one(_Tp __t) noexcept {
40		return __t != numeric_limits<_Tp>::max() ? std::countl_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
	39	return std::countl_zero(static_cast<_Tp>(~__t));
41	40	}
42	41
43	42	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__bit/countr.h+1-2

...	...	@@ -24,7 +24,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
24	24
25	25	template <class _Tp>
26	26	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __countr_zero(_Tp __t) _NOEXCEPT {
27		static_assert(__is_unsigned_integer_v<_Tp>, "__countr_zero only works with unsigned types");
28	27	return __builtin_ctzg(__t, numeric_limits<_Tp>::digits);
29	28	}
30	29
...	...	@@ -37,7 +36,7 @@ template <__unsigned_integer _Tp>
37	36
38	37	template <__unsigned_integer _Tp>
39	38	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr int countr_one(_Tp __t) noexcept {
40		return __t != numeric_limits<_Tp>::max() ? std::countr_zero(static_cast<_Tp>(~__t)) : numeric_limits<_Tp>::digits;
	39	return std::countr_zero(static_cast<_Tp>(~__t));
41	40	}
42	41
43	42	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__bit/has_single_bit.h+1-1

...	...	@@ -25,7 +25,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
25	25
26	26	template <__unsigned_integer _Tp>
27	27	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_single_bit(_Tp __t) noexcept {
28		return __t != 0 && (((__t & (__t - 1)) == 0));
	28	return __builtin_popcountg(__t) == 1;
29	29	}
30	30
31	31	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__bit/popcount.h-1

...	...	@@ -23,7 +23,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
23	23
24	24	template <class _Tp>
25	25	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int __popcount(_Tp __t) _NOEXCEPT {
26		static_assert(__is_unsigned_integer_v<_Tp>, "__popcount only works with unsigned types");
27	26	return __builtin_popcountg(__t);
28	27	}
29	28

lib/libcxx/include/__bit/rotate.h+15-26

...	...	@@ -22,46 +22,35 @@ _LIBCPP_BEGIN_NAMESPACE_STD
22	22	// Writing two full functions for rotl and rotr makes it easier for the compiler
23	23	// to optimize the code. On x86 this function becomes the ROL instruction and
24	24	// the rotr function becomes the ROR instruction.
25		template <class _Tp>
26		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotl(_Tp __x, int __s) _NOEXCEPT {
27		static_assert(__is_unsigned_integer_v<_Tp>, "__rotl requires an unsigned integer type");
	25
	26	#if _LIBCPP_STD_VER >= 20
	27
	28	template <__unsigned_integer _Tp>
	29	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotl(_Tp __t, int __cnt) noexcept {
28	30	const int __n = numeric_limits<_Tp>::digits;
29		int __r = __s % __n;
	31	int __r = __cnt % __n;
30	32
31	33	if (__r == 0)
32		return __x;
	34	return __t;
33	35
34	36	if (__r > 0)
35		return (__x << __r) \| (__x >> (__n - __r));
	37	return (__t << __r) \| (__t >> (__n - __r));
36	38
37		return (__x >> -__r) \| (__x << (__n + __r));
	39	return (__t >> -__r) \| (__t << (__n + __r));
38	40	}
39	41
40		template <class _Tp>
41		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp __rotr(_Tp __x, int __s) _NOEXCEPT {
42		static_assert(__is_unsigned_integer_v<_Tp>, "__rotr requires an unsigned integer type");
	42	template <__unsigned_integer _Tp>
	43	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotr(_Tp __t, int __cnt) noexcept {
43	44	const int __n = numeric_limits<_Tp>::digits;
44		int __r = __s % __n;
	45	int __r = __cnt % __n;
45	46
46	47	if (__r == 0)
47		return __x;
	48	return __t;
48	49
49	50	if (__r > 0)
50		return (__x >> __r) \| (__x << (__n - __r));
51
52		return (__x << -__r) \| (__x >> (__n + __r));
53		}
	51	return (__t >> __r) \| (__t << (__n - __r));
54	52
55		#if _LIBCPP_STD_VER >= 20
56
57		template <__unsigned_integer _Tp>
58		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotl(_Tp __t, int __cnt) noexcept {
59		return std::__rotl(__t, __cnt);
60		}
61
62		template <__unsigned_integer _Tp>
63		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp rotr(_Tp __t, int __cnt) noexcept {
64		return std::__rotr(__t, __cnt);
	53	return (__t << -__r) \| (__t >> (__n + __r));
65	54	}
66	55
67	56	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__bit_reference+207-18

...	...	@@ -15,8 +15,10 @@
15	15	#include <__algorithm/copy_backward.h>
16	16	#include <__algorithm/copy_n.h>
17	17	#include <__algorithm/equal.h>
	18	#include <__algorithm/fill_n.h>
18	19	#include <__algorithm/min.h>
19	20	#include <__algorithm/rotate.h>
	21	#include <__algorithm/specialized_algorithms.h>
20	22	#include <__algorithm/swap_ranges.h>
21	23	#include <__assert>
22	24	#include <__bit/countr.h>
...	...	@@ -137,7 +139,7 @@ public:
137	139	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator bool() const _NOEXCEPT {
138	140	return static_cast<bool>(*__seg_ & __mask_);
139	141	}
140		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool operator~() const _NOEXCEPT {
	142	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool operator~() const _NOEXCEPT {
141	143	return !static_cast<bool>(*this);
142	144	}
143	145
...	...	@@ -307,6 +309,15 @@ public:
307	309	{
308	310	}
309	311
	312	#ifdef _LIBCPP_ABI_TRIVIALLY_COPYABLE_BIT_ITERATOR
	313	template <bool _IsConstDep = _IsConst, __enable_if_t<_IsConstDep, int> = 0>
	314	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator(const __bit_iterator<_Cp, false>& __it) _NOEXCEPT
	315	: __seg_(__it.__seg_),
	316	__ctz_(__it.__ctz_) {}
	317
	318	_LIBCPP_HIDE_FROM_ABI __bit_iterator(const __bit_iterator&) = default;
	319	_LIBCPP_HIDE_FROM_ABI __bit_iterator& operator=(const __bit_iterator&) = default;
	320	#else
310	321	// When _IsConst=false, this is the copy constructor.
311	322	// It is non-trivial. Making it trivial would break ABI.
312	323	// When _IsConst=true, this is a converting constructor;
...	...	@@ -327,6 +338,7 @@ public:
327	338	__ctz_ = __it.__ctz_;
328	339	return *this;
329	340	}
	341	#endif
330	342
331	343	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator*() const _NOEXCEPT {
332	344	_LIBCPP_ASSERT_INTERNAL(__ctz_ < __bits_per_word, "Dereferencing an invalid __bit_iterator.");
...	...	@@ -467,20 +479,6 @@ private:
467	479	template <class _Dp>
468	480	friend struct __bit_array;
469	481
470		template <bool _FillVal, class _Dp>
471		_LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
472		__fill_n_bool(__bit_iterator<_Dp, false> __first, typename __size_difference_type_traits<_Dp>::size_type __n);
473
474		template <class _Dp, bool _IC>
475		_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_aligned(
476		__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
477		template <class _Dp, bool _IC>
478		_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_unaligned(
479		__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
480		template <class _Dp, bool _IC>
481		_LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Dp, _IC>, __bit_iterator<_Dp, false> >
482		__copy_impl::operator()(
483		__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result) const;
484	482	template <class _Dp, bool _IC>
485	483	_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false> __copy_backward_aligned(
486	484	__bit_iterator<_Dp, _IC> __first, __bit_iterator<_Dp, _IC> __last, __bit_iterator<_Dp, false> __result);
...	...	@@ -511,10 +509,20 @@ private:
511	509	bool _IsConst1,
512	510	bool _IsConst2,
513	511	class _BinaryPredicate,
514		__enable_if_t<__desugars_to_v<__equal_tag, _BinaryPredicate, bool, bool>, int> >
	512	class _Proj1,
	513	class _Proj2,
	514	__enable_if_t<__is_identity<_Proj1>::value && __is_identity<_Proj2>::value &&
	515	__desugars_to_v<__equal_tag, _BinaryPredicate, bool, bool>,
	516	int> >
515	517	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool __equal_iter_impl(
516		__bit_iterator<_Dp, _IsConst1>, __bit_iterator<_Dp, _IsConst1>, __bit_iterator<_Dp, _IsConst2>, _BinaryPredicate);
517		template <class _Dp,
	518	__bit_iterator<_Dp, _IsConst1>,
	519	__bit_iterator<_Dp, _IsConst1>,
	520	__bit_iterator<_Dp, _IsConst2>,
	521	_BinaryPredicate,
	522	_Proj1&,
	523	_Proj2&);
	524	template <bool,
	525	class _Dp,
518	526	bool _IsConst1,
519	527	bool _IsConst2,
520	528	class _Pred,
...	...	@@ -537,6 +545,187 @@ private:
537	545	template <bool _ToCount, class _Dp, bool _IC>
538	546	friend typename __bit_iterator<_Dp, _IC>::difference_type _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
539	547	__count_bool(__bit_iterator<_Dp, _IC>, typename __size_difference_type_traits<_Dp>::size_type);
	548
	549	template <class, class...>
	550	friend struct __specialized_algorithm;
	551	};
	552
	553	template <class _Cp>
	554	struct __specialized_algorithm<_Algorithm::__fill_n, __single_iterator<__bit_iterator<_Cp, false> > > {
	555	static const bool __has_algorithm = true;
	556
	557	template <bool _FillVal>
	558	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void
	559	__impl(__bit_iterator<_Cp, false> __first, typename __size_difference_type_traits<_Cp>::size_type __n) {
	560	using _It = __bit_iterator<_Cp, false>;
	561	using __storage_type = typename _It::__storage_type;
	562
	563	const int __bits_per_word = _It::__bits_per_word;
	564	// do first partial word
	565	if (__first.__ctz_ != 0) {
	566	__storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
	567	__storage_type __dn = std::min(__clz_f, __n);
	568	std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
	569	__n -= __dn;
	570	++__first.__seg_;
	571	}
	572	// do middle whole words
	573	__storage_type __nw = __n / __bits_per_word;
	574	std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
	575	__n -= __nw * __bits_per_word;
	576	// do last partial word
	577	if (__n > 0) {
	578	__first.__seg_ += __nw;
	579	std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
	580	}
	581	}
	582
	583	template <class _Size, class _Tp>
	584	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
	585	operator()(__bit_iterator<_Cp, false> __first, _Size __n, const _Tp& __value) {
	586	if (__n > 0) {
	587	if (__value)
	588	__impl<true>(__first, __n);
	589	else
	590	__impl<false>(__first, __n);
	591	}
	592	return __first + __n;
	593	}
	594	};
	595
	596	template <class _Cp, bool _IsConst>
	597	struct __specialized_algorithm<_Algorithm::__copy,
	598	__iterator_pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, _IsConst> >,
	599	__single_iterator<__bit_iterator<_Cp, false> > > {
	600	static const bool __has_algorithm = true;
	601
	602	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
	603	__aligned_impl(__bit_iterator<_Cp, _IsConst> __first,
	604	__bit_iterator<_Cp, _IsConst> __last,
	605	__bit_iterator<_Cp, false> __result) {
	606	using _In = __bit_iterator<_Cp, _IsConst>;
	607	using difference_type = typename _In::difference_type;
	608	using __storage_type = typename _In::__storage_type;
	609
	610	const int __bits_per_word = _In::__bits_per_word;
	611	difference_type __n = __last - __first;
	612	if (__n > 0) {
	613	// do first word
	614	if (__first.__ctz_ != 0) {
	615	unsigned __clz = __bits_per_word - __first.__ctz_;
	616	difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
	617	__n -= __dn;
	618	__storage_type __m = std::__middle_mask<__storage_type>(__clz - __dn, __first.__ctz_);
	619	__storage_type __b = *__first.__seg_ & __m;
	620	*__result.__seg_ &= ~__m;
	621	*__result.__seg_ \|= __b;
	622	__result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
	623	__result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
	624	++__first.__seg_;
	625	// __first.__ctz_ = 0;
	626	}
	627	// __first.__ctz_ == 0;
	628	// do middle words
	629	__storage_type __nw = __n / __bits_per_word;
	630	std::copy(std::__to_address(__first.__seg_),
	631	std::__to_address(__first.__seg_ + __nw),
	632	std::__to_address(__result.__seg_));
	633	__n -= __nw * __bits_per_word;
	634	__result.__seg_ += __nw;
	635	// do last word
	636	if (__n > 0) {
	637	__first.__seg_ += __nw;
	638	__storage_type __m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
	639	__storage_type __b = *__first.__seg_ & __m;
	640	*__result.__seg_ &= ~__m;
	641	*__result.__seg_ \|= __b;
	642	__result.__ctz_ = static_cast<unsigned>(__n);
	643	}
	644	}
	645	return __result;
	646	}
	647
	648	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
	649	__unaligned_impl(__bit_iterator<_Cp, _IsConst> __first,
	650	__bit_iterator<_Cp, _IsConst> __last,
	651	__bit_iterator<_Cp, false> __result) {
	652	using _In = __bit_iterator<_Cp, _IsConst>;
	653	using difference_type = typename _In::difference_type;
	654	using __storage_type = typename _In::__storage_type;
	655
	656	const int __bits_per_word = _In::__bits_per_word;
	657	difference_type __n = __last - __first;
	658	if (__n > 0) {
	659	// do first word
	660	if (__first.__ctz_ != 0) {
	661	unsigned __clz_f = __bits_per_word - __first.__ctz_;
	662	difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
	663	__n -= __dn;
	664	__storage_type __m = std::__middle_mask<__storage_type>(__clz_f - __dn, __first.__ctz_);
	665	__storage_type __b = *__first.__seg_ & __m;
	666	unsigned __clz_r = __bits_per_word - __result.__ctz_;
	667	__storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
	668	__m = std::__middle_mask<__storage_type>(__clz_r - __ddn, __result.__ctz_);
	669	*__result.__seg_ &= ~__m;
	670	if (__result.__ctz_ > __first.__ctz_)
	671	*__result.__seg_ \|= __b << (__result.__ctz_ - __first.__ctz_);
	672	else
	673	*__result.__seg_ \|= __b >> (__first.__ctz_ - __result.__ctz_);
	674	__result.__seg_ += (__ddn + __result.__ctz_) / __bits_per_word;
	675	__result.__ctz_ = static_cast<unsigned>((__ddn + __result.__ctz_) % __bits_per_word);
	676	__dn -= __ddn;
	677	if (__dn > 0) {
	678	__m = std::__trailing_mask<__storage_type>(__bits_per_word - __dn);
	679	*__result.__seg_ &= ~__m;
	680	*__result.__seg_ \|= __b >> (__first.__ctz_ + __ddn);
	681	__result.__ctz_ = static_cast<unsigned>(__dn);
	682	}
	683	++__first.__seg_;
	684	// __first.__ctz_ = 0;
	685	}
	686	// __first.__ctz_ == 0;
	687	// do middle words
	688	unsigned __clz_r = __bits_per_word - __result.__ctz_;
	689	__storage_type __m = std::__leading_mask<__storage_type>(__result.__ctz_);
	690	for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first.__seg_) {
	691	__storage_type __b = *__first.__seg_;
	692	*__result.__seg_ &= ~__m;
	693	*__result.__seg_ \|= __b << __result.__ctz_;
	694	++__result.__seg_;
	695	*__result.__seg_ &= __m;
	696	*__result.__seg_ \|= __b >> __clz_r;
	697	}
	698	// do last word
	699	if (__n > 0) {
	700	__m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
	701	__storage_type __b = *__first.__seg_ & __m;
	702	__storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
	703	__m = std::__middle_mask<__storage_type>(__clz_r - __dn, __result.__ctz_);
	704	*__result.__seg_ &= ~__m;
	705	*__result.__seg_ \|= __b << __result.__ctz_;
	706	__result.__seg_ += (__dn + __result.__ctz_) / __bits_per_word;
	707	__result.__ctz_ = static_cast<unsigned>((__dn + __result.__ctz_) % __bits_per_word);
	708	__n -= __dn;
	709	if (__n > 0) {
	710	__m = std::__trailing_mask<__storage_type>(__bits_per_word - __n);
	711	*__result.__seg_ &= ~__m;
	712	*__result.__seg_ \|= __b >> __dn;
	713	__result.__ctz_ = static_cast<unsigned>(__n);
	714	}
	715	}
	716	}
	717	return __result;
	718	}
	719
	720	_LIBCPP_HIDE_FROM_ABI
	721	_LIBCPP_CONSTEXPR_SINCE_CXX20 static pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
	722	operator()(__bit_iterator<_Cp, _IsConst> __first,
	723	__bit_iterator<_Cp, _IsConst> __last,
	724	__bit_iterator<_Cp, false> __result) {
	725	if (__first.__ctz_ == __result.__ctz_)
	726	return std::make_pair(__last, __aligned_impl(__first, __last, __result));
	727	return std::make_pair(__last, __unaligned_impl(__first, __last, __result));
	728	}
540	729	};
541	730
542	731	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__charconv/from_chars_integral.h+1-1

...	...	@@ -18,8 +18,8 @@
18	18	#include <__memory/addressof.h>
19	19	#include <__system_error/errc.h>
20	20	#include <__type_traits/enable_if.h>
21		#include <__type_traits/integral_constant.h>
22	21	#include <__type_traits/is_integral.h>
	22	#include <__type_traits/is_signed.h>
23	23	#include <__type_traits/is_unsigned.h>
24	24	#include <__type_traits/make_unsigned.h>
25	25	#include <limits>

lib/libcxx/include/__charconv/from_chars_result.h+1-1

...	...	@@ -21,7 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
21	21
22	22	#if _LIBCPP_STD_VER >= 17
23	23
24		struct _LIBCPP_EXPORTED_FROM_ABI from_chars_result {
	24	struct from_chars_result {
25	25	const char* ptr;
26	26	errc ec;
27	27	# if _LIBCPP_STD_VER >= 20

lib/libcxx/include/__charconv/to_chars_integral.h+1

...	...	@@ -24,6 +24,7 @@
24	24	#include <__type_traits/integral_constant.h>
25	25	#include <__type_traits/is_integral.h>
26	26	#include <__type_traits/is_same.h>
	27	#include <__type_traits/is_signed.h>
27	28	#include <__type_traits/make_32_64_or_128_bit.h>
28	29	#include <__type_traits/make_unsigned.h>
29	30	#include <__utility/unreachable.h>

lib/libcxx/include/__charconv/to_chars_result.h+1-1

...	...	@@ -21,7 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
21	21
22	22	#if _LIBCPP_STD_VER >= 17
23	23
24		struct _LIBCPP_EXPORTED_FROM_ABI to_chars_result {
	24	struct to_chars_result {
25	25	char* ptr;
26	26	errc ec;
27	27	# if _LIBCPP_STD_VER >= 20

lib/libcxx/include/__charconv/traits.h+3-24

...	...	@@ -113,31 +113,10 @@ struct _LIBCPP_HIDDEN __traits_base<_Tp, __enable_if_t<sizeof(_Tp) == sizeof(__u
113	113	};
114	114	# endif
115	115
116		template <typename _Tp>
117		inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool
118		__mul_overflowed(unsigned char __a, _Tp __b, unsigned char& __r) {
119		auto __c = __a * __b;
120		__r = __c;
121		return __c > numeric_limits<unsigned char>::max();
122		}
123
124		template <typename _Tp>
125		inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool
126		__mul_overflowed(unsigned short __a, _Tp __b, unsigned short& __r) {
127		auto __c = __a * __b;
128		__r = __c;
129		return __c > numeric_limits<unsigned short>::max();
130		}
131
132		template <typename _Tp>
133		inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool __mul_overflowed(_Tp __a, _Tp __b, _Tp& __r) {
134		static_assert(is_unsigned<_Tp>::value, "");
135		return __builtin_mul_overflow(__a, __b, std::addressof(__r));
136		}
137
138	116	template <typename _Tp, typename _Up>
139		inline _LIBCPP_HIDE_FROM_ABI bool _LIBCPP_CONSTEXPR_SINCE_CXX23 __mul_overflowed(_Tp __a, _Up __b, _Tp& __r) {
140		return __itoa::__mul_overflowed(__a, static_cast<_Tp>(__b), __r);
	117	_LIBCPP_HIDE_FROM_ABI bool _LIBCPP_CONSTEXPR_SINCE_CXX23 __mul_overflowed(_Tp __a, _Up __b, _Tp& __r) {
	118	static_assert(is_unsigned<_Tp>::value);
	119	return __builtin_mul_overflow(__a, static_cast<_Tp>(__b), std::addressof(__r));
141	120	}
142	121
143	122	template <typename _Tp>

lib/libcxx/include/__chrono/day.h+11

...	...	@@ -13,6 +13,8 @@
13	13	#include <__chrono/duration.h>
14	14	#include <__compare/ordering.h>
15	15	#include <__config>
	16	#include <__cstddef/size_t.h>
	17	#include <__functional/hash.h>
16	18
17	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	20	# pragma GCC system_header
...	...	@@ -92,6 +94,15 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr day& day::operator-=(const days& __dd) no
92	94
93	95	} // namespace chrono
94	96
	97	# if _LIBCPP_STD_VER >= 26
	98
	99	template <>
	100	struct hash<chrono::day> {
	101	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::day& __d) noexcept { return static_cast<unsigned>(__d); }
	102	};
	103
	104	# endif // _LIBCPP_STD_VER >= 26
	105
95	106	_LIBCPP_END_NAMESPACE_STD
96	107
97	108	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/duration.h+58-33

...	...	@@ -13,6 +13,8 @@
13	13	#include <__compare/ordering.h>
14	14	#include <__compare/three_way_comparable.h>
15	15	#include <__config>
	16	#include <__cstddef/size_t.h>
	17	#include <__functional/hash.h>
16	18	#include <__type_traits/common_type.h>
17	19	#include <__type_traits/enable_if.h>
18	20	#include <__type_traits/is_convertible.h>
...	...	@@ -102,7 +104,8 @@ struct __duration_cast<_FromDuration, _ToDuration, _Period, false, false> {
102	104	};
103	105
104	106	template <class _ToDuration, class _Rep, class _Period, __enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
105		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration duration_cast(const duration<_Rep, _Period>& __fd) {
	107	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration
	108	duration_cast(const duration<_Rep, _Period>& __fd) {
106	109	return __duration_cast<duration<_Rep, _Period>, _ToDuration>()(__fd);
107	110	}
108	111
...	...	@@ -117,14 +120,18 @@ inline constexpr bool treat_as_floating_point_v = treat_as_floating_point<_Rep>:
117	120	template <class _Rep>
118	121	struct duration_values {
119	122	public:
120		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT { return _Rep(0); }
121		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT { return numeric_limits<_Rep>::max(); }
122		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT { return numeric_limits<_Rep>::lowest(); }
	123	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep zero() _NOEXCEPT { return _Rep(0); }
	124	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep max() _NOEXCEPT {
	125	return numeric_limits<_Rep>::max();
	126	}
	127	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR _Rep min() _NOEXCEPT {
	128	return numeric_limits<_Rep>::lowest();
	129	}
123	130	};
124	131
125	132	#if _LIBCPP_STD_VER >= 17
126	133	template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
127		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration floor(const duration<_Rep, _Period>& __d) {
	134	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration floor(const duration<_Rep, _Period>& __d) {
128	135	_ToDuration __t = chrono::duration_cast<_ToDuration>(__d);
129	136	if (__t > __d)
130	137	__t = __t - _ToDuration{1};
...	...	@@ -132,7 +139,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration floor(const duration<
132	139	}
133	140
134	141	template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
135		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration ceil(const duration<_Rep, _Period>& __d) {
	142	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration ceil(const duration<_Rep, _Period>& __d) {
136	143	_ToDuration __t = chrono::duration_cast<_ToDuration>(__d);
137	144	if (__t < __d)
138	145	__t = __t + _ToDuration{1};
...	...	@@ -140,7 +147,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration ceil(const duration<_
140	147	}
141	148
142	149	template <class _ToDuration, class _Rep, class _Period, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
143		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration round(const duration<_Rep, _Period>& __d) {
	150	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _ToDuration round(const duration<_Rep, _Period>& __d) {
144	151	_ToDuration __lower = chrono::floor<_ToDuration>(__d);
145	152	_ToDuration __upper = __lower + _ToDuration{1};
146	153	auto __lower_diff = __d - __lower;
...	...	@@ -220,14 +227,14 @@ public:
220	227
221	228	// observer
222	229
223		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR rep count() const { return __rep_; }
	230	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR rep count() const { return __rep_; }
224	231
225	232	// arithmetic
226	233
227		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator+() const {
	234	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator+() const {
228	235	return typename common_type<duration>::type(*this);
229	236	}
230		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator-() const {
	237	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<duration>::type operator-() const {
231	238	return typename common_type<duration>::type(-__rep_);
232	239	}
233	240	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 duration& operator++() {
...	...	@@ -269,13 +276,13 @@ public:
269	276
270	277	// special values
271	278
272		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT {
	279	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration zero() _NOEXCEPT {
273	280	return duration(duration_values<rep>::zero());
274	281	}
275		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT {
	282	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration min() _NOEXCEPT {
276	283	return duration(duration_values<rep>::min());
277	284	}
278		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT {
	285	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR duration max() _NOEXCEPT {
279	286	return duration(duration_values<rep>::max());
280	287	}
281	288	};
...	...	@@ -389,7 +396,7 @@ operator<=>(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Perio
389	396	// Duration +
390	397
391	398	template <class _Rep1, class _Period1, class _Rep2, class _Period2>
392		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
	399	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
393	400	typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
394	401	operator+(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
395	402	typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Cd;
...	...	@@ -399,7 +406,7 @@ operator+(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2
399	406	// Duration -
400	407
401	408	template <class _Rep1, class _Period1, class _Rep2, class _Period2>
402		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
	409	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
403	410	typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
404	411	operator-(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
405	412	typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Cd;
...	...	@@ -412,7 +419,8 @@ template <class _Rep1,
412	419	class _Period,
413	420	class _Rep2,
414	421	__enable_if_t<is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0>
415		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
	422	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	423	_LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
416	424	operator*(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
417	425	typedef typename common_type<_Rep1, _Rep2>::type _Cr;
418	426	typedef duration<_Cr, _Period> _Cd;
...	...	@@ -423,7 +431,8 @@ template <class _Rep1,
423	431	class _Period,
424	432	class _Rep2,
425	433	__enable_if_t<is_convertible<const _Rep1&, typename common_type<_Rep1, _Rep2>::type>::value, int> = 0>
426		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
	434	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	435	_LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
427	436	operator*(const _Rep1& __s, const duration<_Rep2, _Period>& __d) {
428	437	return __d * __s;
429	438	}
...	...	@@ -436,7 +445,8 @@ template <class _Rep1,
436	445	__enable_if_t<!__is_duration_v<_Rep2> &&
437	446	is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value,
438	447	int> = 0>
439		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
	448	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	449	_LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
440	450	operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
441	451	typedef typename common_type<_Rep1, _Rep2>::type _Cr;
442	452	typedef duration<_Cr, _Period> _Cd;
...	...	@@ -444,7 +454,7 @@ operator/(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
444	454	}
445	455
446	456	template <class _Rep1, class _Period1, class _Rep2, class _Period2>
447		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<_Rep1, _Rep2>::type
	457	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename common_type<_Rep1, _Rep2>::type
448	458	operator/(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
449	459	typedef typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type _Ct;
450	460	return _Ct(__lhs).count() / _Ct(__rhs).count();
...	...	@@ -458,7 +468,8 @@ template <class _Rep1,
458	468	__enable_if_t<!__is_duration_v<_Rep2> &&
459	469	is_convertible<const _Rep2&, typename common_type<_Rep1, _Rep2>::type>::value,
460	470	int> = 0>
461		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
	471	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	472	_LIBCPP_CONSTEXPR duration<typename common_type<_Rep1, _Rep2>::type, _Period>
462	473	operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
463	474	typedef typename common_type<_Rep1, _Rep2>::type _Cr;
464	475	typedef duration<_Cr, _Period> _Cd;
...	...	@@ -466,7 +477,7 @@ operator%(const duration<_Rep1, _Period>& __d, const _Rep2& __s) {
466	477	}
467	478
468	479	template <class _Rep1, class _Period1, class _Rep2, class _Period2>
469		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
	480	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
470	481	typename common_type<duration<_Rep1, _Period1>, duration<_Rep2, _Period2> >::type
471	482	operator%(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
472	483	typedef typename common_type<_Rep1, _Rep2>::type _Cr;
...	...	@@ -481,51 +492,53 @@ operator%(const duration<_Rep1, _Period1>& __lhs, const duration<_Rep2, _Period2
481	492	inline namespace literals {
482	493	inline namespace chrono_literals {
483	494
484		_LIBCPP_HIDE_FROM_ABI constexpr chrono::hours operator""h(unsigned long long __h) {
	495	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::hours operator""h(unsigned long long __h) {
485	496	return chrono::hours(static_cast<chrono::hours::rep>(__h));
486	497	}
487	498
488		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<3600, 1>> operator""h(long double __h) {
	499	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<3600, 1>>
	500	operator""h(long double __h) {
489	501	return chrono::duration<long double, ratio<3600, 1>>(__h);
490	502	}
491	503
492		_LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes operator""min(unsigned long long __m) {
	504	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::minutes operator""min(unsigned long long __m) {
493	505	return chrono::minutes(static_cast<chrono::minutes::rep>(__m));
494	506	}
495	507
496		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<60, 1>> operator""min(long double __m) {
	508	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, ratio<60, 1>>
	509	operator""min(long double __m) {
497	510	return chrono::duration<long double, ratio<60, 1>>(__m);
498	511	}
499	512
500		_LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds operator""s(unsigned long long __s) {
	513	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::seconds operator""s(unsigned long long __s) {
501	514	return chrono::seconds(static_cast<chrono::seconds::rep>(__s));
502	515	}
503	516
504		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double> operator""s(long double __s) {
	517	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double> operator""s(long double __s) {
505	518	return chrono::duration<long double>(__s);
506	519	}
507	520
508		_LIBCPP_HIDE_FROM_ABI constexpr chrono::milliseconds operator""ms(unsigned long long __ms) {
	521	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::milliseconds operator""ms(unsigned long long __ms) {
509	522	return chrono::milliseconds(static_cast<chrono::milliseconds::rep>(__ms));
510	523	}
511	524
512		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, milli> operator""ms(long double __ms) {
	525	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, milli> operator""ms(long double __ms) {
513	526	return chrono::duration<long double, milli>(__ms);
514	527	}
515	528
516		_LIBCPP_HIDE_FROM_ABI constexpr chrono::microseconds operator""us(unsigned long long __us) {
	529	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::microseconds operator""us(unsigned long long __us) {
517	530	return chrono::microseconds(static_cast<chrono::microseconds::rep>(__us));
518	531	}
519	532
520		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, micro> operator""us(long double __us) {
	533	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, micro> operator""us(long double __us) {
521	534	return chrono::duration<long double, micro>(__us);
522	535	}
523	536
524		_LIBCPP_HIDE_FROM_ABI constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) {
	537	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::nanoseconds operator""ns(unsigned long long __ns) {
525	538	return chrono::nanoseconds(static_cast<chrono::nanoseconds::rep>(__ns));
526	539	}
527	540
528		_LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, nano> operator""ns(long double __ns) {
	541	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr chrono::duration<long double, nano> operator""ns(long double __ns) {
529	542	return chrono::duration<long double, nano>(__ns);
530	543	}
531	544
...	...	@@ -538,6 +551,18 @@ using namespace literals::chrono_literals;
538	551
539	552	#endif // _LIBCPP_STD_VER >= 14
540	553
	554	#if _LIBCPP_STD_VER >= 26
	555
	556	template <class _Rep, class _Period>
	557	requires __has_enabled_hash<_Rep>::value
	558	struct hash<chrono::duration<_Rep, _Period>> {
	559	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::duration<_Rep, _Period>& __d) {
	560	return hash<_Rep>{}(__d.count());
	561	}
	562	};
	563
	564	#endif // _LIBCPP_STD_VER >= 26
	565
541	566	_LIBCPP_END_NAMESPACE_STD
542	567
543	568	_LIBCPP_POP_MACROS

lib/libcxx/include/__chrono/file_clock.h+4-2

...	...	@@ -60,16 +60,18 @@ struct _FilesystemClock {
60	60
61	61	_LIBCPP_EXPORTED_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false;
62	62
63		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_EXPORTED_FROM_ABI static time_point now() noexcept;
	63	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI static time_point now() noexcept;
64	64
65	65	# if _LIBCPP_STD_VER >= 20
66	66	template <class _Duration>
	67	[[nodiscard]]
67	68	_LIBCPP_HIDE_FROM_ABI static chrono::sys_time<_Duration> to_sys(const chrono::file_time<_Duration>& __t) {
68	69	return chrono::sys_time<_Duration>(__t.time_since_epoch());
69	70	}
70	71
71	72	template <class _Duration>
72		_LIBCPP_HIDE_FROM_ABI static chrono::file_time<_Duration> from_sys(const chrono::sys_time<_Duration>& __t) {
	73	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static chrono::file_time<_Duration>
	74	from_sys(const chrono::sys_time<_Duration>& __t) {
73	75	return chrono::file_time<_Duration>(__t.time_since_epoch());
74	76	}
75	77	# endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/is_clock.h created+72

...	...	@@ -0,0 +1,72 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___CHRONO_IS_CLOCK_H
	11	#define _LIBCPP___CHRONO_IS_CLOCK_H
	12
	13	#include <__config>
	14
	15	#include <__chrono/duration.h>
	16	#include <__chrono/time_point.h>
	17	#include <__concepts/same_as.h>
	18	#include <__type_traits/integral_constant.h>
	19	#include <__type_traits/is_arithmetic.h>
	20	#include <__type_traits/is_class.h>
	21	#include <__type_traits/is_union.h>
	22	#include <ratio>
	23
	24	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	25	# pragma GCC system_header
	26	#endif
	27
	28	#if _LIBCPP_STD_VER >= 20
	29
	30	_LIBCPP_BEGIN_NAMESPACE_STD
	31
	32	namespace chrono {
	33
	34	// Helper to check that _Tp::time_point has the form time_point<_, typename _Tp::duration>.
	35	template <class _TimePoint, class _ClockType>
	36	inline constexpr bool __is_valid_clock_time_point_v = false;
	37
	38	template <class _TimePointClock, class _ClockType>
	39	inline constexpr bool
	40	__is_valid_clock_time_point_v<time_point<_TimePointClock, typename _ClockType::duration>, _ClockType> = true;
	41
	42	// Check if a clock satisfies the Cpp17Clock requirements as defined in [time.clock.req]
	43	template <class _Tp>
	44	_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_clock_v = requires {
	45	typename _Tp::rep;
	46	requires is_arithmetic_v<typename _Tp::rep> \|\| is_class_v<typename _Tp::rep> \|\| is_union_v<typename _Tp::rep>;
	47
	48	typename _Tp::period;
	49	requires __is_ratio_v<typename _Tp::period>;
	50
	51	typename _Tp::duration;
	52	requires same_as<typename _Tp::duration, duration<typename _Tp::rep, typename _Tp::period>>;
	53
	54	typename _Tp::time_point;
	55	requires __is_valid_clock_time_point_v<typename _Tp::time_point, _Tp>;
	56
	57	_Tp::is_steady;
	58	requires same_as<decltype((_Tp::is_steady)), const bool&>;
	59
	60	_Tp::now();
	61	requires same_as<decltype(_Tp::now()), typename _Tp::time_point>;
	62	};
	63
	64	template <class _Tp>
	65	struct _LIBCPP_NO_SPECIALIZATIONS is_clock : bool_constant<is_clock_v<_Tp>> {};
	66
	67	} // namespace chrono
	68
	69	_LIBCPP_END_NAMESPACE_STD
	70
	71	#endif // _LIBCPP_STD_VER
	72	#endif // _LIBCPP___CHRONO_IS_CLOCK_H

lib/libcxx/include/__chrono/leap_second.h+13

...	...	@@ -22,6 +22,8 @@
22	22	# include <__compare/ordering.h>
23	23	# include <__compare/three_way_comparable.h>
24	24	# include <__config>
	25	# include <__cstddef/size_t.h>
	26	# include <__functional/hash.h>
25	27	# include <__utility/private_constructor_tag.h>
26	28
27	29	# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -122,6 +124,17 @@ private:
122	124
123	125	} // namespace chrono
124	126
	127	# if _LIBCPP_STD_VER >= 26
	128
	129	template <>
	130	struct hash<chrono::leap_second> {
	131	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::leap_second& __lp) noexcept {
	132	return std::__hash_combine(hash<chrono::sys_seconds>{}(__lp.date()), hash<chrono::seconds>{}(__lp.value()));
	133	}
	134	};
	135
	136	# endif // _LIBCPP_STD_VER >= 26
	137
125	138	# endif // _LIBCPP_STD_VER >= 20
126	139
127	140	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__chrono/month.h+13

...	...	@@ -13,6 +13,8 @@
13	13	#include <__chrono/duration.h>
14	14	#include <__compare/ordering.h>
15	15	#include <__config>
	16	#include <__cstddef/size_t.h>
	17	#include <__functional/hash.h>
16	18
17	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	20	# pragma GCC system_header
...	...	@@ -108,6 +110,17 @@ inline constexpr month December{12};
108	110
109	111	} // namespace chrono
110	112
	113	# if _LIBCPP_STD_VER >= 26
	114
	115	template <>
	116	struct hash<chrono::month> {
	117	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::month& __m) noexcept {
	118	return static_cast<unsigned>(__m);
	119	}
	120	};
	121
	122	# endif // _LIBCPP_STD_VER >= 26
	123
111	124	_LIBCPP_END_NAMESPACE_STD
112	125
113	126	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/month_weekday.h+22

...	...	@@ -13,6 +13,8 @@
13	13	#include <__chrono/month.h>
14	14	#include <__chrono/weekday.h>
15	15	#include <__config>
	16	#include <__cstddef/size_t.h>
	17	#include <__functional/hash.h>
16	18
17	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	20	# pragma GCC system_header
...	...	@@ -98,6 +100,26 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr month_weekday_last operator/(const weekda
98	100	}
99	101	} // namespace chrono
100	102
	103	# if _LIBCPP_STD_VER >= 26
	104
	105	template <>
	106	struct hash<chrono::month_weekday> {
	107	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::month_weekday& __mw) noexcept {
	108	return std::__hash_combine(
	109	hash<chrono::month>{}(__mw.month()), hash<chrono::weekday_indexed>{}(__mw.weekday_indexed()));
	110	}
	111	};
	112
	113	template <>
	114	struct hash<chrono::month_weekday_last> {
	115	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::month_weekday_last& __mwl) noexcept {
	116	return std::__hash_combine(
	117	hash<chrono::month>{}(__mwl.month()), hash<chrono::weekday_last>{}(__mwl.weekday_last()));
	118	}
	119	};
	120
	121	# endif // _LIBCPP_STD_VER >= 26
	122
101	123	_LIBCPP_END_NAMESPACE_STD
102	124
103	125	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/monthday.h+20

...	...	@@ -15,6 +15,8 @@
15	15	#include <__chrono/month.h>
16	16	#include <__compare/ordering.h>
17	17	#include <__config>
	18	#include <__cstddef/size_t.h>
	19	#include <__functional/hash.h>
18	20
19	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
20	22	# pragma GCC system_header
...	...	@@ -126,6 +128,24 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr month_day_last operator/(last_spec, int _
126	128
127	129	} // namespace chrono
128	130
	131	# if _LIBCPP_STD_VER >= 26
	132
	133	template <>
	134	struct hash<chrono::month_day> {
	135	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::month_day& __md) noexcept {
	136	return std::__hash_combine(hash<chrono::month>{}(__md.month()), hash<chrono::day>{}(__md.day()));
	137	}
	138	};
	139
	140	template <>
	141	struct hash<chrono::month_day_last> {
	142	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::month_day_last& __mdl) noexcept {
	143	return hash<chrono::month>{}(__mdl.month());
	144	}
	145	};
	146
	147	# endif // _LIBCPP_STD_VER >= 26
	148
129	149	_LIBCPP_END_NAMESPACE_STD
130	150
131	151	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/steady_clock.h+1-1

...	...	@@ -31,7 +31,7 @@ public:
31	31	typedef chrono::time_point<steady_clock, duration> time_point;
32	32	static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = true;
33	33
34		static time_point now() _NOEXCEPT;
	34	[[__nodiscard__]] static time_point now() _NOEXCEPT;
35	35	};
36	36	#endif
37	37

lib/libcxx/include/__chrono/system_clock.h+3-3

...	...	@@ -31,9 +31,9 @@ public:
31	31	typedef chrono::time_point<system_clock> time_point;
32	32	static _LIBCPP_CONSTEXPR_SINCE_CXX14 const bool is_steady = false;
33	33
34		static time_point now() _NOEXCEPT;
35		static time_t to_time_t(const time_point& __t) _NOEXCEPT;
36		static time_point from_time_t(time_t __t) _NOEXCEPT;
	34	[[__nodiscard__]] static time_point now() _NOEXCEPT;
	35	[[__nodiscard__]] static time_t to_time_t(const time_point& __t) _NOEXCEPT;
	36	[[__nodiscard__]] static time_point from_time_t(time_t __t) _NOEXCEPT;
37	37	};
38	38
39	39	#if _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/time_point.h+37-13

...	...	@@ -14,6 +14,8 @@
14	14	#include <__compare/ordering.h>
15	15	#include <__compare/three_way_comparable.h>
16	16	#include <__config>
	17	#include <__cstddef/size_t.h>
	18	#include <__functional/hash.h>
17	19	#include <__type_traits/common_type.h>
18	20	#include <__type_traits/enable_if.h>
19	21	#include <__type_traits/is_convertible.h>
...	...	@@ -54,7 +56,9 @@ public:
54	56
55	57	// observer
56	58
57		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 duration time_since_epoch() const { return __d_; }
	59	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 duration time_since_epoch() const {
	60	return __d_;
	61	}
58	62
59	63	// arithmetic
60	64
...	...	@@ -82,8 +86,12 @@ public:
82	86
83	87	// special values
84	88
85		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT { return time_point(duration::min()); }
86		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT { return time_point(duration::max()); }
	89	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point min() _NOEXCEPT {
	90	return time_point(duration::min());
	91	}
	92	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR time_point max() _NOEXCEPT {
	93	return time_point(duration::max());
	94	}
87	95	};
88	96
89	97	} // namespace chrono
...	...	@@ -95,30 +103,33 @@ struct common_type<chrono::time_point<_Clock, _Duration1>, chrono::time_point<_C
95	103
96	104	namespace chrono {
97	105
98		template <class _ToDuration, class _Clock, class _Duration>
99		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, _ToDuration>
	106	template <class _ToDuration, class _Clock, class _Duration, __enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
	107	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, _ToDuration>
100	108	time_point_cast(const time_point<_Clock, _Duration>& __t) {
101	109	return time_point<_Clock, _ToDuration>(chrono::duration_cast<_ToDuration>(__t.time_since_epoch()));
102	110	}
103	111
104	112	#if _LIBCPP_STD_VER >= 17
105	113	template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
106		inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> floor(const time_point<_Clock, _Duration>& __t) {
	114	[[nodiscard]] inline
	115	_LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> floor(const time_point<_Clock, _Duration>& __t) {
107	116	return time_point<_Clock, _ToDuration>{chrono::floor<_ToDuration>(__t.time_since_epoch())};
108	117	}
109	118
110	119	template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
111		inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> ceil(const time_point<_Clock, _Duration>& __t) {
	120	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration>
	121	ceil(const time_point<_Clock, _Duration>& __t) {
112	122	return time_point<_Clock, _ToDuration>{chrono::ceil<_ToDuration>(__t.time_since_epoch())};
113	123	}
114	124
115	125	template <class _ToDuration, class _Clock, class _Duration, enable_if_t<__is_duration_v<_ToDuration>, int> = 0>
116		inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration> round(const time_point<_Clock, _Duration>& __t) {
	126	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr time_point<_Clock, _ToDuration>
	127	round(const time_point<_Clock, _Duration>& __t) {
117	128	return time_point<_Clock, _ToDuration>{chrono::round<_ToDuration>(__t.time_since_epoch())};
118	129	}
119	130
120	131	template <class _Rep, class _Period, enable_if_t<numeric_limits<_Rep>::is_signed, int> = 0>
121		inline _LIBCPP_HIDE_FROM_ABI constexpr duration<_Rep, _Period> abs(duration<_Rep, _Period> __d) {
	132	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr duration<_Rep, _Period> abs(duration<_Rep, _Period> __d) {
122	133	return __d >= __d.zero() ? +__d : -__d;
123	134	}
124	135	#endif // _LIBCPP_STD_VER >= 17
...	...	@@ -188,7 +199,7 @@ operator<=>(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock
188	199	// time_point operator+(time_point x, duration y);
189	200
190	201	template <class _Clock, class _Duration1, class _Rep2, class _Period2>
191		inline _LIBCPP_HIDE_FROM_ABI
	202	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
192	203	_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type>
193	204	operator+(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
194	205	typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Tr;
...	...	@@ -198,7 +209,7 @@ operator+(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Pe
198	209	// time_point operator+(duration x, time_point y);
199	210
200	211	template <class _Rep1, class _Period1, class _Clock, class _Duration2>
201		inline _LIBCPP_HIDE_FROM_ABI
	212	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
202	213	_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<duration<_Rep1, _Period1>, _Duration2>::type>
203	214	operator+(const duration<_Rep1, _Period1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
204	215	return __rhs + __lhs;
...	...	@@ -207,7 +218,7 @@ operator+(const duration<_Rep1, _Period1>& __lhs, const time_point<_Clock, _Dura
207	218	// time_point operator-(time_point x, duration y);
208	219
209	220	template <class _Clock, class _Duration1, class _Rep2, class _Period2>
210		inline _LIBCPP_HIDE_FROM_ABI
	221	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
211	222	_LIBCPP_CONSTEXPR_SINCE_CXX14 time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type>
212	223	operator-(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Period2>& __rhs) {
213	224	typedef time_point<_Clock, typename common_type<_Duration1, duration<_Rep2, _Period2> >::type> _Ret;
...	...	@@ -217,13 +228,26 @@ operator-(const time_point<_Clock, _Duration1>& __lhs, const duration<_Rep2, _Pe
217	228	// duration operator-(time_point x, time_point y);
218	229
219	230	template <class _Clock, class _Duration1, class _Duration2>
220		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename common_type<_Duration1, _Duration2>::type
	231	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	232	typename common_type<_Duration1, _Duration2>::type
221	233	operator-(const time_point<_Clock, _Duration1>& __lhs, const time_point<_Clock, _Duration2>& __rhs) {
222	234	return __lhs.time_since_epoch() - __rhs.time_since_epoch();
223	235	}
224	236
225	237	} // namespace chrono
226	238
	239	#if _LIBCPP_STD_VER >= 26
	240
	241	template <class _Clock, class _Duration>
	242	requires __has_enabled_hash<_Duration>::value
	243	struct hash<chrono::time_point<_Clock, _Duration>> {
	244	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::time_point<_Clock, _Duration>& __tp) {
	245	return hash<_Duration>{}(__tp.time_since_epoch());
	246	}
	247	};
	248
	249	#endif // _LIBCPP_STD_VER >= 26
	250
227	251	_LIBCPP_END_NAMESPACE_STD
228	252
229	253	_LIBCPP_POP_MACROS

lib/libcxx/include/__chrono/weekday.h+25

...	...	@@ -15,6 +15,8 @@
15	15	#include <__chrono/system_clock.h>
16	16	#include <__chrono/time_point.h>
17	17	#include <__config>
	18	#include <__cstddef/size_t.h>
	19	#include <__functional/hash.h>
18	20
19	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
20	22	# pragma GCC system_header
...	...	@@ -160,6 +162,29 @@ inline constexpr weekday Saturday{6};
160	162
161	163	} // namespace chrono
162	164
	165	# if _LIBCPP_STD_VER >= 26
	166
	167	template <>
	168	struct hash<chrono::weekday> {
	169	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::weekday& __w) noexcept { return __w.c_encoding(); }
	170	};
	171
	172	template <>
	173	struct hash<chrono::weekday_indexed> {
	174	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::weekday_indexed& __wi) noexcept {
	175	return std::__hash_combine(hash<chrono::weekday>{}(__wi.weekday()), __wi.index());
	176	}
	177	};
	178
	179	template <>
	180	struct hash<chrono::weekday_last> {
	181	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::weekday_last& __wl) noexcept {
	182	return hash<chrono::weekday>{}(__wl.weekday());
	183	}
	184	};
	185
	186	# endif // _LIBCPP_STD_VER >= 26
	187
163	188	_LIBCPP_END_NAMESPACE_STD
164	189
165	190	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/year.h+11

...	...	@@ -13,6 +13,8 @@
13	13	#include <__chrono/duration.h>
14	14	#include <__compare/ordering.h>
15	15	#include <__config>
	16	#include <__cstddef/size_t.h>
	17	#include <__functional/hash.h>
16	18	#include <limits>
17	19
18	20	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -109,6 +111,15 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool year::ok() const noexcept {
109	111
110	112	} // namespace chrono
111	113
	114	# if _LIBCPP_STD_VER >= 26
	115
	116	template <>
	117	struct hash<chrono::year> {
	118	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year& __y) noexcept { return static_cast<int>(__y); }
	119	};
	120
	121	# endif // _LIBCPP_STD_VER >= 26
	122
112	123	_LIBCPP_END_NAMESPACE_STD
113	124
114	125	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/year_month.h+13

...	...	@@ -15,6 +15,8 @@
15	15	#include <__chrono/year.h>
16	16	#include <__compare/ordering.h>
17	17	#include <__config>
	18	#include <__cstddef/size_t.h>
	19	#include <__functional/hash.h>
18	20
19	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
20	22	# pragma GCC system_header
...	...	@@ -116,6 +118,17 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr year_month& year_month::operator-=(const
116	118
117	119	} // namespace chrono
118	120
	121	# if _LIBCPP_STD_VER >= 26
	122
	123	template <>
	124	struct hash<chrono::year_month> {
	125	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year_month& __ym) noexcept {
	126	return std::__hash_combine(hash<chrono::year>{}(__ym.year()), hash<chrono::month>{}(__ym.month()));
	127	}
	128	};
	129
	130	# endif // _LIBCPP_STD_VER >= 26
	131
119	132	_LIBCPP_END_NAMESPACE_STD
120	133
121	134	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/year_month_day.h+23

...	...	@@ -21,6 +21,8 @@
21	21	#include <__chrono/year_month.h>
22	22	#include <__compare/ordering.h>
23	23	#include <__config>
	24	#include <__cstddef/size_t.h>
	25	#include <__functional/hash.h>
24	26	#include <limits>
25	27
26	28	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -330,6 +332,27 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr bool year_month_day::ok() const noexcept
330	332
331	333	} // namespace chrono
332	334
	335	# if _LIBCPP_STD_VER >= 26
	336
	337	template <>
	338	struct hash<chrono::year_month_day> {
	339	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year_month_day& __ymd) noexcept {
	340	return std::__hash_combine(
	341	hash<chrono::year>{}(__ymd.year()),
	342	std::__hash_combine(hash<chrono::month>{}(__ymd.month()), hash<chrono::day>{}(__ymd.day())));
	343	}
	344	};
	345
	346	template <>
	347	struct hash<chrono::year_month_day_last> {
	348	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year_month_day_last& __ymdl) noexcept {
	349	return std::__hash_combine(
	350	hash<chrono::year>{}(__ymdl.year()), hash<chrono::month_day_last>{}(__ymdl.month_day_last()));
	351	}
	352	};
	353
	354	# endif // _LIBCPP_STD_VER >= 26
	355
333	356	_LIBCPP_END_NAMESPACE_STD
334	357
335	358	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/year_month_weekday.h+26

...	...	@@ -22,6 +22,8 @@
22	22	#include <__chrono/year_month.h>
23	23	#include <__chrono/year_month_day.h>
24	24	#include <__config>
	25	#include <__cstddef/size_t.h>
	26	#include <__functional/hash.h>
25	27
26	28	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
27	29	# pragma GCC system_header
...	...	@@ -280,6 +282,30 @@ year_month_weekday_last::operator-=(const years& __dy) noexcept {
280	282
281	283	} // namespace chrono
282	284
	285	# if _LIBCPP_STD_VER >= 26
	286
	287	template <>
	288	struct hash<chrono::year_month_weekday> {
	289	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year_month_weekday& __ymw) noexcept {
	290	return std::__hash_combine(
	291	hash<chrono::year>{}(__ymw.year()),
	292	std::__hash_combine(
	293	hash<chrono::month>{}(__ymw.month()), hash<chrono::weekday_indexed>{}(__ymw.weekday_indexed())));
	294	}
	295	};
	296
	297	template <>
	298	struct hash<chrono::year_month_weekday_last> {
	299	_LIBCPP_HIDE_FROM_ABI static size_t operator()(const chrono::year_month_weekday_last& __ymwl) noexcept {
	300	return std::__hash_combine(
	301	hash<chrono::year>{}(__ymwl.year()),
	302	std::__hash_combine(
	303	hash<chrono::month>{}(__ymwl.month()), hash<chrono::weekday_last>{}(__ymwl.weekday_last())));
	304	}
	305	};
	306
	307	# endif // _LIBCPP_STD_VER >= 26
	308
283	309	_LIBCPP_END_NAMESPACE_STD
284	310
285	311	#endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/__chrono/zoned_time.h+16

...	...	@@ -24,6 +24,8 @@
24	24	# include <__chrono/tzdb_list.h>
25	25	# include <__concepts/constructible.h>
26	26	# include <__config>
	27	# include <__cstddef/size_t.h>
	28	# include <__functional/hash.h>
27	29	# include <__type_traits/common_type.h>
28	30	# include <__type_traits/conditional.h>
29	31	# include <__type_traits/remove_cvref.h>
...	...	@@ -216,6 +218,20 @@ operator==(const zoned_time<_Duration1, _TimeZonePtr>& __lhs, const zoned_time<_
216	218
217	219	} // namespace chrono
218	220
	221	# if _LIBCPP_STD_VER >= 26
	222
	223	template <class _Duration, class _TimeZonePtr>
	224	requires __has_enabled_hash<_Duration>::value && __has_enabled_hash<_TimeZonePtr>::value
	225	struct hash<chrono::zoned_time<_Duration, _TimeZonePtr>> {
	226	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static size_t
	227	operator()(const chrono::zoned_time<_Duration, _TimeZonePtr>& __zt) {
	228	return std::__hash_combine(
	229	hash<chrono::sys_time<_Duration>>{}(__zt.get_sys_time()), hash<_TimeZonePtr>{}(__zt.get_time_zone()));
	230	}
	231	};
	232
	233	# endif // _LIBCPP_STD_VER >= 26
	234
219	235	# endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_TIME_ZONE_DATABASE && _LIBCPP_HAS_FILESYSTEM &&
220	236	// _LIBCPP_HAS_LOCALIZATION
221	237

lib/libcxx/include/__compare/is_eq.h+6-6

...	...	@@ -20,12 +20,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD
20	20
21	21	#if _LIBCPP_STD_VER >= 20
22	22
23		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_eq(partial_ordering __c) noexcept { return __c == 0; }
24		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_neq(partial_ordering __c) noexcept { return __c != 0; }
25		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lt(partial_ordering __c) noexcept { return __c < 0; }
26		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lteq(partial_ordering __c) noexcept { return __c <= 0; }
27		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gt(partial_ordering __c) noexcept { return __c > 0; }
28		_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gteq(partial_ordering __c) noexcept { return __c >= 0; }
	23	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_eq(partial_ordering __c) noexcept { return __c == 0; }
	24	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_neq(partial_ordering __c) noexcept { return __c != 0; }
	25	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lt(partial_ordering __c) noexcept { return __c < 0; }
	26	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_lteq(partial_ordering __c) noexcept { return __c <= 0; }
	27	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gt(partial_ordering __c) noexcept { return __c > 0; }
	28	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool is_gteq(partial_ordering __c) noexcept { return __c >= 0; }
29	29
30	30	#endif // _LIBCPP_STD_VER >= 20
31	31

lib/libcxx/include/__compare/strong_order.h+12-30

...	...	@@ -13,7 +13,6 @@
13	13	#include <__compare/compare_three_way.h>
14	14	#include <__compare/ordering.h>
15	15	#include <__config>
16		#include <__math/exponential_functions.h>
17	16	#include <__math/traits.h>
18	17	#include <__type_traits/conditional.h>
19	18	#include <__type_traits/decay.h>
...	...	@@ -53,38 +52,21 @@ struct __fn {
53	52	template <class _Tp, class _Up, class _Dp = decay_t<_Tp>>
54	53	requires is_same_v<_Dp, decay_t<_Up>> && is_floating_point_v<_Dp>
55	54	_LIBCPP_HIDE_FROM_ABI static constexpr strong_ordering __go(_Tp&& __t, _Up&& __u, __priority_tag<1>) noexcept {
56		if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int32_t)) {
57		int32_t __rx = std::bit_cast<int32_t>(__t);
58		int32_t __ry = std::bit_cast<int32_t>(__u);
59		__rx = (__rx < 0) ? (numeric_limits<int32_t>::min() - __rx - 1) : __rx;
60		__ry = (__ry < 0) ? (numeric_limits<int32_t>::min() - __ry - 1) : __ry;
61		return (__rx <=> __ry);
62		} else if constexpr (numeric_limits<_Dp>::is_iec559 && sizeof(_Dp) == sizeof(int64_t)) {
63		int64_t __rx = std::bit_cast<int64_t>(__t);
64		int64_t __ry = std::bit_cast<int64_t>(__u);
65		__rx = (__rx < 0) ? (numeric_limits<int64_t>::min() - __rx - 1) : __rx;
66		__ry = (__ry < 0) ? (numeric_limits<int64_t>::min() - __ry - 1) : __ry;
	55	if constexpr (numeric_limits<_Dp>::is_iec559 &&
	56	(sizeof(_Dp) == sizeof(int32_t) \|\| sizeof(_Dp) == sizeof(int64_t))) {
	57	using _IntT = conditional_t<sizeof(_Dp) == sizeof(int32_t), int32_t, int64_t>;
	58	_IntT __rx = std::bit_cast<_IntT>(__t);
	59	_IntT __ry = std::bit_cast<_IntT>(__u);
	60	__rx = (__rx < 0) ? (numeric_limits<_IntT>::min() - __rx - 1) : __rx;
	61	__ry = (__ry < 0) ? (numeric_limits<_IntT>::min() - __ry - 1) : __ry;
67	62	return (__rx <=> __ry);
68	63	} else if (__t < __u) {
69	64	return strong_ordering::less;
70	65	} else if (__t > __u) {
71	66	return strong_ordering::greater;
72	67	} else if (__t == __u) {
73		if constexpr (numeric_limits<_Dp>::radix == 2) {
74		return __math::signbit(__u) <=> __math::signbit(__t);
75		} else {
76		// This is bullet 3 of the IEEE754 algorithm, relevant
77		// only for decimal floating-point;
78		// see https://stackoverflow.com/questions/69068075/
79		if (__t == 0 \|\| __math::isinf(__t)) {
80		return __math::signbit(__u) <=> __math::signbit(__t);
81		} else {
82		int __texp, __uexp;
83		(void)__math::frexp(__t, &__texp);
84		(void)__math::frexp(__u, &__uexp);
85		return (__t < 0) ? (__texp <=> __uexp) : (__uexp <=> __texp);
86		}
87		}
	68	static_assert(numeric_limits<_Dp>::radix == 2, "floating point type with a radix other than 2?");
	69	return __math::signbit(__u) <=> __math::signbit(__t);
88	70	} else {
89	71	// They're unordered, so one of them must be a NAN.
90	72	// The order is -QNAN, -SNAN, numbers, +SNAN, +QNAN.
...	...	@@ -93,9 +75,9 @@ struct __fn {
93	75	bool __t_is_negative = __math::signbit(__t);
94	76	bool __u_is_negative = __math::signbit(__u);
95	77	using _IntType =
96		conditional_t< sizeof(__t) == sizeof(int32_t),
97		int32_t,
98		conditional_t< sizeof(__t) == sizeof(int64_t), int64_t, void> >;
	78	conditional_t<sizeof(__t) == sizeof(int32_t),
	79	int32_t,
	80	conditional_t<sizeof(__t) == sizeof(int64_t), int64_t, void>>;
99	81	if constexpr (is_same_v<_IntType, void>) {
100	82	static_assert(sizeof(_Dp) == 0, "std::strong_order is unimplemented for this floating-point type");
101	83	} else if (__t_is_nan && __u_is_nan) {

lib/libcxx/include/__compare/three_way_comparable.h+2-2

...	...	@@ -12,6 +12,7 @@
12	12	#include <__compare/common_comparison_category.h>
13	13	#include <__compare/ordering.h>
14	14	#include <__concepts/common_reference_with.h>
	15	#include <__concepts/comparison_common_type.h>
15	16	#include <__concepts/equality_comparable.h>
16	17	#include <__concepts/same_as.h>
17	18	#include <__concepts/totally_ordered.h>
...	...	@@ -39,8 +40,7 @@ concept three_way_comparable =
39	40
40	41	template <class _Tp, class _Up, class _Cat = partial_ordering>
41	42	concept three_way_comparable_with =
42		three_way_comparable<_Tp, _Cat> && three_way_comparable<_Up, _Cat> &&
43		common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
	43	three_way_comparable<_Tp, _Cat> && three_way_comparable<_Up, _Cat> && __comparison_common_type_with<_Tp, _Up> &&
44	44	three_way_comparable<common_reference_t<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>>, _Cat> &&
45	45	__weakly_equality_comparable_with<_Tp, _Up> && __partially_ordered_with<_Tp, _Up> &&
46	46	requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {

lib/libcxx/include/__concepts/comparison_common_type.h created+40

...	...	@@ -0,0 +1,40 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___CONCEPTS_COMPARISON_COMMON_TYPE_H
	10	#define _LIBCPP___CONCEPTS_COMPARISON_COMMON_TYPE_H
	11
	12	#include <__concepts/convertible_to.h>
	13	#include <__concepts/same_as.h>
	14	#include <__config>
	15	#include <__type_traits/common_reference.h>
	16	#include <__type_traits/remove_cvref.h>
	17
	18	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	19	# pragma GCC system_header
	20	#endif
	21
	22	_LIBCPP_BEGIN_NAMESPACE_STD
	23
	24	#if _LIBCPP_STD_VER >= 20
	25
	26	template <class _Tp, class _Up, class _CommonRef = common_reference_t<const _Tp&, const _Up&>>
	27	concept __comparison_common_type_with_impl =
	28	same_as<common_reference_t<const _Tp&, const _Up&>, common_reference_t<const _Up&, const _Tp&>> && requires {
	29	requires convertible_to<const _Tp&, const _CommonRef&> \|\| convertible_to<_Tp, const _CommonRef&>;
	30	requires convertible_to<const _Up&, const _CommonRef&> \|\| convertible_to<_Up, const _CommonRef&>;
	31	};
	32
	33	template <class _Tp, class _Up>
	34	concept __comparison_common_type_with = __comparison_common_type_with_impl<remove_cvref_t<_Tp>, remove_cvref_t<_Up>>;
	35
	36	#endif // _LIBCPP_STD_VER >= 20
	37
	38	_LIBCPP_END_NAMESPACE_STD
	39
	40	#endif // _LIBCPP___CONCEPTS_COMPARISON_COMMON_TYPE_H

lib/libcxx/include/__concepts/equality_comparable.h+2-1

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__concepts/boolean_testable.h>
13	13	#include <__concepts/common_reference_with.h>
	14	#include <__concepts/comparison_common_type.h>
14	15	#include <__config>
15	16	#include <__type_traits/common_reference.h>
16	17	#include <__type_traits/make_const_lvalue_ref.h>
...	...	@@ -41,7 +42,7 @@ concept equality_comparable = __weakly_equality_comparable_with<_Tp, _Tp>;
41	42	template <class _Tp, class _Up>
42	43	concept equality_comparable_with =
43	44	equality_comparable<_Tp> && equality_comparable<_Up> &&
44		common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
	45	__comparison_common_type_with<_Tp, _Up> &&
45	46	equality_comparable<
46	47	common_reference_t<
47	48	__make_const_lvalue_ref<_Tp>,

lib/libcxx/include/__condition_variable/condition_variable.h+1-1

...	...	@@ -170,7 +170,7 @@ public:
170	170	wait_for(unique_lock<mutex>& __lk, const chrono::duration<_Rep, _Period>& __d, _Predicate __pred);
171	171
172	172	typedef __libcpp_condvar_t* native_handle_type;
173		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; }
	173	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__cv_; }
174	174
175	175	private:
176	176	void

lib/libcxx/include/__config+63-280

...	...	@@ -14,6 +14,8 @@
14	14	#include <__configuration/abi.h>
15	15	#include <__configuration/availability.h>
16	16	#include <__configuration/compiler.h>
	17	#include <__configuration/experimental.h>
	18	#include <__configuration/hardening.h>
17	19	#include <__configuration/language.h>
18	20	#include <__configuration/platform.h>
19	21
...	...	@@ -28,7 +30,7 @@
28	30	// _LIBCPP_VERSION represents the version of libc++, which matches the version of LLVM.
29	31	// Given a LLVM release LLVM XX.YY.ZZ (e.g. LLVM 17.0.1 == 17.00.01), _LIBCPP_VERSION is
30	32	// defined to XXYYZZ.
31		# define _LIBCPP_VERSION 210100
	33	# define _LIBCPP_VERSION 220104
32	34
33	35	# define _LIBCPP_CONCAT_IMPL(_X, _Y) _X##_Y
34	36	# define _LIBCPP_CONCAT(_X, _Y) _LIBCPP_CONCAT_IMPL(_X, _Y)
...	...	@@ -38,195 +40,6 @@
38	40	# define _LIBCPP_FREESTANDING
39	41	# endif
40	42
41		// NOLINTNEXTLINE(libcpp-cpp-version-check)
42		# if __cplusplus < 201103L
43		# define _LIBCPP_CXX03_LANG
44		# endif
45
46		# if __has_feature(experimental_library)
47		# ifndef _LIBCPP_ENABLE_EXPERIMENTAL
48		# define _LIBCPP_ENABLE_EXPERIMENTAL
49		# endif
50		# endif
51
52		// Incomplete features get their own specific disabling flags. This makes it
53		// easier to grep for target specific flags once the feature is complete.
54		# if defined(_LIBCPP_ENABLE_EXPERIMENTAL) \|\| defined(_LIBCPP_BUILDING_LIBRARY)
55		# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 1
56		# else
57		# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 0
58		# endif
59
60		# define _LIBCPP_HAS_EXPERIMENTAL_PSTL _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
61		# define _LIBCPP_HAS_EXPERIMENTAL_TZDB _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
62		# define _LIBCPP_HAS_EXPERIMENTAL_SYNCSTREAM _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
63		# define _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
64
65		// HARDENING {
66
67		// TODO(LLVM 23): Remove this. We're making these an error to catch folks who might not have migrated.
68		// Since hardening went through several changes (many of which impacted user-facing macros),
69		// we're keeping these checks around for a bit longer than usual. Failure to properly configure
70		// hardening results in checks being dropped silently, which is a pretty big deal.
71		# if defined(_LIBCPP_ENABLE_ASSERTIONS)
72		# error "_LIBCPP_ENABLE_ASSERTIONS has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
73		# endif
74		# if defined(_LIBCPP_ENABLE_HARDENED_MODE)
75		# error "_LIBCPP_ENABLE_HARDENED_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
76		# endif
77		# if defined(_LIBCPP_ENABLE_SAFE_MODE)
78		# error "_LIBCPP_ENABLE_SAFE_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
79		# endif
80		# if defined(_LIBCPP_ENABLE_DEBUG_MODE)
81		# error "_LIBCPP_ENABLE_DEBUG_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
82		# endif
83
84		// The library provides the macro `_LIBCPP_HARDENING_MODE` which can be set to one of the following values:
85		//
86		// - `_LIBCPP_HARDENING_MODE_NONE`;
87		// - `_LIBCPP_HARDENING_MODE_FAST`;
88		// - `_LIBCPP_HARDENING_MODE_EXTENSIVE`;
89		// - `_LIBCPP_HARDENING_MODE_DEBUG`.
90		//
91		// These values have the following effects:
92		//
93		// - `_LIBCPP_HARDENING_MODE_NONE` -- sets the hardening mode to "none" which disables all runtime hardening checks;
94		//
95		// - `_LIBCPP_HARDENING_MODE_FAST` -- sets that hardening mode to "fast". The fast mode enables security-critical checks
96		// that can be done with relatively little runtime overhead in constant time;
97		//
98		// - `_LIBCPP_HARDENING_MODE_EXTENSIVE` -- sets the hardening mode to "extensive". The extensive mode is a superset of
99		// the fast mode that additionally enables checks that are relatively cheap and prevent common types of logic errors
100		// but are not necessarily security-critical;
101		//
102		// - `_LIBCPP_HARDENING_MODE_DEBUG` -- sets the hardening mode to "debug". The debug mode is a superset of the extensive
103		// mode and enables all checks available in the library, including internal assertions. Checks that are part of the
104		// debug mode can be very expensive and thus the debug mode is intended to be used for testing, not in production.
105
106		// Inside the library, assertions are categorized so they can be cherry-picked based on the chosen hardening mode. These
107		// macros are only for internal use -- users should only pick one of the high-level hardening modes described above.
108		//
109		// - `_LIBCPP_ASSERT_VALID_INPUT_RANGE` -- checks that ranges (whether expressed as an iterator pair, an iterator and
110		// a sentinel, an iterator and a count, or a `std::range`) given as input to library functions are valid:
111		// - the sentinel is reachable from the begin iterator;
112		// - TODO(hardening): both iterators refer to the same container.
113		//
114		// - `_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS` -- checks that any attempts to access a container element, whether through
115		// the container object or through an iterator, are valid and do not attempt to go out of bounds or otherwise access
116		// a non-existent element. For iterator checks to work, bounded iterators must be enabled in the ABI. Types like
117		// `optional` and `function` are considered one-element containers for the purposes of this check.
118		//
119		// - `_LIBCPP_ASSERT_NON_NULL` -- checks that the pointer being dereferenced is not null. On most modern platforms zero
120		// address does not refer to an actual location in memory, so a null pointer dereference would not compromize the
121		// memory security of a program (however, it is still undefined behavior that can result in strange errors due to
122		// compiler optimizations).
123		//
124		// - `_LIBCPP_ASSERT_NON_OVERLAPPING_RANGES` -- for functions that take several ranges as arguments, checks that the
125		// given ranges do not overlap.
126		//
127		// - `_LIBCPP_ASSERT_VALID_DEALLOCATION` -- checks that an attempt to deallocate memory is valid (e.g. the given object
128		// was allocated by the given allocator). Violating this category typically results in a memory leak.
129		//
130		// - `_LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL` -- checks that a call to an external API doesn't fail in
131		// an unexpected manner. This includes triggering documented cases of undefined behavior in an external library (like
132		// attempting to unlock an unlocked mutex in pthreads). Any API external to the library falls under this category
133		// (from system calls to compiler intrinsics). We generally don't expect these failures to compromize memory safety or
134		// otherwise create an immediate security issue.
135		//
136		// - `_LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR` -- checks any operations that exchange nodes between containers to make sure
137		// the containers have compatible allocators.
138		//
139		// - `_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN` -- checks that the given argument is within the domain of valid arguments
140		// for the function. Violating this typically produces an incorrect result (e.g. the clamp algorithm returns the
141		// original value without clamping it due to incorrect functors) or puts an object into an invalid state (e.g.
142		// a string view where only a subset of elements is possible to access). This category is for assertions violating
143		// which doesn't cause any immediate issues in the library -- whatever the consequences are, they will happen in the
144		// user code.
145		//
146		// - `_LIBCPP_ASSERT_PEDANTIC` -- checks prerequisites which are imposed by the Standard, but violating which happens to
147		// be benign in our implementation.
148		//
149		// - `_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT` -- checks that the given argument satisfies the semantic requirements imposed
150		// by the Standard. Typically, there is no simple way to completely prove that a semantic requirement is satisfied;
151		// thus, this would often be a heuristic check and it might be quite expensive.
152		//
153		// - `_LIBCPP_ASSERT_INTERNAL` -- checks that internal invariants of the library hold. These assertions don't depend on
154		// user input.
155		//
156		// - `_LIBCPP_ASSERT_UNCATEGORIZED` -- for assertions that haven't been properly classified yet.
157
158		// clang-format off
159		# define _LIBCPP_HARDENING_MODE_NONE (1 << 1)
160		# define _LIBCPP_HARDENING_MODE_FAST (1 << 2)
161		# define _LIBCPP_HARDENING_MODE_EXTENSIVE (1 << 4) // Deliberately not ordered.
162		# define _LIBCPP_HARDENING_MODE_DEBUG (1 << 3)
163		// clang-format on
164
165		# ifndef _LIBCPP_HARDENING_MODE
166
167		# ifndef _LIBCPP_HARDENING_MODE_DEFAULT
168		# error _LIBCPP_HARDENING_MODE_DEFAULT is not defined. This definition should be set at configuration time in the \
169		`__config_site` header, please make sure your installation of libc++ is not broken.
170		# endif
171
172		# define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_DEFAULT
173		# endif
174
175		# if _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_NONE && \
176		_LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_FAST && \
177		_LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_EXTENSIVE && \
178		_LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_DEBUG
179		# error _LIBCPP_HARDENING_MODE must be set to one of the following values: \
180		_LIBCPP_HARDENING_MODE_NONE, \
181		_LIBCPP_HARDENING_MODE_FAST, \
182		_LIBCPP_HARDENING_MODE_EXTENSIVE, \
183		_LIBCPP_HARDENING_MODE_DEBUG
184		# endif
185
186		// Hardening assertion semantics generally mirror the evaluation semantics of C++26 Contracts:
187		// - `ignore` evaluates the assertion but doesn't do anything if it fails (note that it differs from the Contracts
188		// `ignore` semantic which wouldn't evaluate the assertion at all);
189		// - `observe` logs an error (indicating, if possible, that the error is fatal) and continues execution;
190		// - `quick-enforce` terminates the program as fast as possible (via trapping);
191		// - `enforce` logs an error and then terminates the program.
192		//
193		// Notes:
194		// - Continuing execution after a hardening check fails results in undefined behavior; the `observe` semantic is meant
195		// to make adopting hardening easier but should not be used outside of this scenario;
196		// - C++26 wording for Library Hardening precludes a conforming Hardened implementation from using the Contracts
197		// `ignore` semantic when evaluating hardened preconditions in the Library. Libc++ allows using this semantic for
198		// hardened preconditions, however, be aware that using `ignore` does not produce a conforming "Hardened"
199		// implementation, unlike the other semantics above.
200		// clang-format off
201		# define _LIBCPP_ASSERTION_SEMANTIC_IGNORE (1 << 1)
202		# define _LIBCPP_ASSERTION_SEMANTIC_OBSERVE (1 << 2)
203		# define _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE (1 << 3)
204		# define _LIBCPP_ASSERTION_SEMANTIC_ENFORCE (1 << 4)
205		// clang-format on
206
207		// Allow users to define an arbitrary assertion semantic; otherwise, use the default mapping from modes to semantics.
208		// The default is for production-capable modes to use `quick-enforce` (i.e., trap) and for the `debug` mode to use
209		// `enforce` (i.e., log and abort).
210		# ifndef _LIBCPP_ASSERTION_SEMANTIC
211
212		# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
213		# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_ENFORCE
214		# else
215		# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE
216		# endif
217
218		# else
219		# if !_LIBCPP_HAS_EXPERIMENTAL_LIBRARY
220		# error "Assertion semantics are an experimental feature."
221		# endif
222		# if defined(_LIBCPP_CXX03_LANG)
223		# error "Assertion semantics are not available in the C++03 mode."
224		# endif
225
226		# endif // _LIBCPP_ASSERTION_SEMANTIC
227
228		// } HARDENING
229
230	43	# define _LIBCPP_TOSTRING2(x) #x
231	44	# define _LIBCPP_TOSTRING(x) _LIBCPP_TOSTRING2(x)
232	45
...	...	@@ -320,13 +133,6 @@ _LIBCPP_HARDENING_MODE_DEBUG
320	133	// When this option is used, the token passed to `std::random_device`'s
321	134	// constructor must be "/dev/urandom" -- anything else is an error.
322	135	//
323		// _LIBCPP_USING_NACL_RANDOM
324		// NaCl's sandbox (which PNaCl also runs in) doesn't allow filesystem access,
325		// including accesses to the special files under `/dev`. This implementation
326		// uses the NaCL syscall `nacl_secure_random_init()` to get entropy.
327		// When this option is used, the token passed to `std::random_device`'s
328		// constructor must be "/dev/urandom" -- anything else is an error.
329		//
330	136	// _LIBCPP_USING_WIN32_RANDOM
331	137	// Use rand_s(), for use on Windows.
332	138	// When this option is used, the token passed to `std::random_device`'s
...	...	@@ -338,8 +144,6 @@ _LIBCPP_HARDENING_MODE_DEBUG
338	144	# define _LIBCPP_USING_GETENTROPY
339	145	# elif defined(__Fuchsia__)
340	146	# define _LIBCPP_USING_FUCHSIA_CPRNG
341		# elif defined(__native_client__)
342		# define _LIBCPP_USING_NACL_RANDOM
343	147	# elif defined(_LIBCPP_WIN32API)
344	148	# define _LIBCPP_USING_WIN32_RANDOM
345	149	# else
...	...	@@ -348,7 +152,7 @@ _LIBCPP_HARDENING_MODE_DEBUG
348	152
349	153	# ifndef _LIBCPP_CXX03_LANG
350	154
351		# define _LIBCPP_ALIGNOF(_Tp) alignof(_Tp)
	155	# define _LIBCPP_ALIGNOF(...) alignof(__VA_ARGS__)
352	156	# define _ALIGNAS_TYPE(x) alignas(x)
353	157	# define _ALIGNAS(x) alignas(x)
354	158	# define _NOEXCEPT noexcept
...	...	@@ -357,7 +161,7 @@ _LIBCPP_HARDENING_MODE_DEBUG
357	161
358	162	# else
359	163
360		# define _LIBCPP_ALIGNOF(_Tp) _Alignof(_Tp)
	164	# define _LIBCPP_ALIGNOF(...) _Alignof(__VA_ARGS__)
361	165	# define _ALIGNAS_TYPE(x) __attribute__((__aligned__(_LIBCPP_ALIGNOF(x))))
362	166	# define _ALIGNAS(x) __attribute__((__aligned__(x)))
363	167	# define nullptr __nullptr
...	...	@@ -471,6 +275,12 @@ typedef __char32_t char32_t;
471	275	# define _LIBCPP_GCC_DIAGNOSTIC_IGNORED(str)
472	276	# endif
473	277
	278	// Macros to enter and leave a state where deprecation warnings are suppressed.
	279	# define _LIBCPP_SUPPRESS_DEPRECATED_PUSH \
	280	_LIBCPP_DIAGNOSTIC_PUSH _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wdeprecated") \
	281	_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wdeprecated-declarations")
	282	# define _LIBCPP_SUPPRESS_DEPRECATED_POP _LIBCPP_DIAGNOSTIC_POP
	283
474	284	# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_FAST
475	285	# define _LIBCPP_HARDENING_SIG f
476	286	# elif _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_EXTENSIVE
...	...	@@ -481,6 +291,16 @@ typedef __char32_t char32_t;
481	291	# define _LIBCPP_HARDENING_SIG n // "none"
482	292	# endif
483	293
	294	# if _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_OBSERVE
	295	# define _LIBCPP_ASSERTION_SEMANTIC_SIG o
	296	# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE
	297	# define _LIBCPP_ASSERTION_SEMANTIC_SIG q
	298	# elif _LIBCPP_ASSERTION_SEMANTIC == _LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	299	# define _LIBCPP_ASSERTION_SEMANTIC_SIG e
	300	# else
	301	# define _LIBCPP_ASSERTION_SEMANTIC_SIG i // `ignore`
	302	# endif
	303
484	304	# if !_LIBCPP_HAS_EXCEPTIONS
485	305	# define _LIBCPP_EXCEPTIONS_SIG n
486	306	# else
...	...	@@ -488,7 +308,9 @@ typedef __char32_t char32_t;
488	308	# endif
489	309
490	310	# define _LIBCPP_ODR_SIGNATURE \
491		_LIBCPP_CONCAT(_LIBCPP_CONCAT(_LIBCPP_HARDENING_SIG, _LIBCPP_EXCEPTIONS_SIG), _LIBCPP_VERSION)
	311	_LIBCPP_CONCAT( \
	312	_LIBCPP_CONCAT(_LIBCPP_CONCAT(_LIBCPP_HARDENING_SIG, _LIBCPP_ASSERTION_SEMANTIC_SIG), _LIBCPP_EXCEPTIONS_SIG), \
	313	_LIBCPP_VERSION)
492	314
493	315	// This macro marks a symbol as being hidden from libc++'s ABI. This is achieved
494	316	// on two levels:
...	...	@@ -550,16 +372,6 @@ typedef __char32_t char32_t;
550	372	# endif
551	373	# define _LIBCPP_HIDE_FROM_ABI_VIRTUAL _LIBCPP_HIDDEN _LIBCPP_EXCLUDE_FROM_EXPLICIT_INSTANTIATION
552	374
553		# ifdef _LIBCPP_BUILDING_LIBRARY
554		# if _LIBCPP_ABI_VERSION > 1
555		# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
556		# else
557		# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1
558		# endif
559		# else
560		# define _LIBCPP_HIDE_FROM_ABI_AFTER_V1 _LIBCPP_HIDE_FROM_ABI
561		# endif
562
563	375	// Clang modules take a significant compile time hit when pushing and popping diagnostics.
564	376	// Since all the headers are marked as system headers unless _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER is defined, we can
565	377	// simply disable this pushing and popping when _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER isn't defined.
...	...	@@ -676,27 +488,6 @@ typedef __char32_t char32_t;
676	488	# endif
677	489	# endif
678	490
679		// It is not yet possible to use aligned_alloc() on all Apple platforms since
680		// 10.15 was the first version to ship an implementation of aligned_alloc().
681		# if defined(__APPLE__)
682		# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \
683		__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500) \|\| \
684		(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \
685		__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000) \|\| \
686		(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \
687		__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 60000) \|\| \
688		(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 130000)
689		# define _LIBCPP_HAS_C11_ALIGNED_ALLOC 0
690		# else
691		# define _LIBCPP_HAS_C11_ALIGNED_ALLOC 1
692		# endif
693		# elif defined(__ANDROID__) && __ANDROID_API__ < 28
694		// Android only provides aligned_alloc when targeting API 28 or higher.
695		# define _LIBCPP_HAS_C11_ALIGNED_ALLOC 0
696		# else
697		# define _LIBCPP_HAS_C11_ALIGNED_ALLOC 1
698		# endif
699
700	491	# if defined(__APPLE__) \|\| defined(__FreeBSD__)
701	492	# define _LIBCPP_WCTYPE_IS_MASK
702	493	# endif
...	...	@@ -727,6 +518,15 @@ typedef __char32_t char32_t;
727	518	# define _LIBCPP_DEPRECATED_(m)
728	519	# endif
729	520
	521	// FIXME: using `#warning` causes diagnostics from system headers which include deprecated headers. This can only be
	522	// enabled again once https://github.com/llvm/llvm-project/pull/168041 (or a similar feature) has landed, since that
	523	// allows suppression in system headers.
	524	# if defined(__DEPRECATED) && __DEPRECATED && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) && 0
	525	# define _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS 1
	526	# else
	527	# define _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS 0
	528	# endif
	529
730	530	# if !defined(_LIBCPP_CXX03_LANG)
731	531	# define _LIBCPP_DEPRECATED_IN_CXX11 _LIBCPP_DEPRECATED
732	532	# else
...	...	@@ -771,17 +571,6 @@ typedef __char32_t char32_t;
771	571	# define _LIBCPP_DEPRECATED_WITH_CHAR8_T
772	572	# endif
773	573
774		// Macros to enter and leave a state where deprecation warnings are suppressed.
775		# if defined(_LIBCPP_COMPILER_CLANG_BASED) \|\| defined(_LIBCPP_COMPILER_GCC)
776		# define _LIBCPP_SUPPRESS_DEPRECATED_PUSH \
777		_Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wdeprecated\"") \
778		_Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"")
779		# define _LIBCPP_SUPPRESS_DEPRECATED_POP _Pragma("GCC diagnostic pop")
780		# else
781		# define _LIBCPP_SUPPRESS_DEPRECATED_PUSH
782		# define _LIBCPP_SUPPRESS_DEPRECATED_POP
783		# endif
784
785	574	# if _LIBCPP_STD_VER <= 11
786	575	# define _LIBCPP_EXPLICIT_SINCE_CXX14
787	576	# else
...	...	@@ -861,18 +650,10 @@ typedef __char32_t char32_t;
861	650	# endif // _LIBCPP_HAS_THREAD_API
862	651	# endif // _LIBCPP_HAS_THREADS
863	652
864		# if _LIBCPP_HAS_THREAD_API_PTHREAD
865		# if defined(__ANDROID__) && __ANDROID_API__ >= 30
866		# define _LIBCPP_HAS_COND_CLOCKWAIT 1
867		# elif defined(_LIBCPP_GLIBC_PREREQ)
868		# if _LIBCPP_GLIBC_PREREQ(2, 30)
869		# define _LIBCPP_HAS_COND_CLOCKWAIT 1
870		# else
871		# define _LIBCPP_HAS_COND_CLOCKWAIT 0
872		# endif
873		# else
874		# define _LIBCPP_HAS_COND_CLOCKWAIT 0
875		# endif
	653	# if !_LIBCPP_HAS_THREAD_API_PTHREAD
	654	# define _LIBCPP_HAS_COND_CLOCKWAIT 0
	655	# elif (defined(__ANDROID__) && __ANDROID_API__ >= 30) \|\| _LIBCPP_GLIBC_PREREQ(2, 30)
	656	# define _LIBCPP_HAS_COND_CLOCKWAIT 1
876	657	# else
877	658	# define _LIBCPP_HAS_COND_CLOCKWAIT 0
878	659	# endif
...	...	@@ -951,8 +732,8 @@ typedef __char32_t char32_t;
951	732	# endif
952	733	# endif
953	734
954		# if defined(__FreeBSD__) && defined(__clang__) && __has_attribute(__no_thread_safety_analysis__)
955		# define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS __attribute__((__no_thread_safety_analysis__))
	735	# if __has_cpp_attribute(_Clang::__no_thread_safety_analysis__)
	736	# define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS [[_Clang::__no_thread_safety_analysis__]]
956	737	# else
957	738	# define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
958	739	# endif
...	...	@@ -1038,12 +819,8 @@ typedef __char32_t char32_t;
1038	819	// the latter depends on internal GNU libc details that are not appropriate
1039	820	// to depend on here, so any declarations present when __cpp_char8_t is not
1040	821	// defined are ignored.
1041		# if defined(_LIBCPP_GLIBC_PREREQ)
1042		# if _LIBCPP_GLIBC_PREREQ(2, 36) && defined(__cpp_char8_t)
1043		# define _LIBCPP_HAS_C8RTOMB_MBRTOC8 1
1044		# else
1045		# define _LIBCPP_HAS_C8RTOMB_MBRTOC8 0
1046		# endif
	822	# if _LIBCPP_GLIBC_PREREQ(2, 36) && defined(__cpp_char8_t)
	823	# define _LIBCPP_HAS_C8RTOMB_MBRTOC8 1
1047	824	# else
1048	825	# define _LIBCPP_HAS_C8RTOMB_MBRTOC8 0
1049	826	# endif
...	...	@@ -1067,8 +844,7 @@ typedef __char32_t char32_t;
1067	844	# define _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(_ClassName) static_assert(true, "")
1068	845	# endif
1069	846
1070		// TODO(LLVM 22): Remove the workaround
1071		# if defined(__OBJC__) && (!defined(_LIBCPP_CLANG_VER) \|\| _LIBCPP_CLANG_VER < 2001)
	847	# if defined(__OBJC__) && defined(_LIBCPP_APPLE_CLANG_VER)
1072	848	# define _LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS
1073	849	# endif
1074	850
...	...	@@ -1153,12 +929,33 @@ typedef __char32_t char32_t;
1153	929	# define _LIBCPP_DIAGNOSE_WARNING(...)
1154	930	# endif
1155	931
	932	# if __has_attribute(__diagnose_if__) && !defined(_LIBCPP_APPLE_CLANG_VER) && \
	933	(!defined(_LIBCPP_CLANG_VER) \|\| _LIBCPP_CLANG_VER >= 2001)
	934	# define _LIBCPP_DIAGNOSE_IF(...) __attribute__((__diagnose_if__(__VA_ARGS__)))
	935	# else
	936	# define _LIBCPP_DIAGNOSE_IF(...)
	937	# endif
	938
	939	# define _LIBCPP_DIAGNOSE_NULLPTR_IF(condition, condition_description) \
	940	_LIBCPP_DIAGNOSE_IF( \
	941	condition, \
	942	"null passed to callee that requires a non-null argument" condition_description, \
	943	"warning", \
	944	"nonnull")
	945
1156	946	# if __has_cpp_attribute(_Clang::__lifetimebound__)
1157	947	# define _LIBCPP_LIFETIMEBOUND [[_Clang::__lifetimebound__]]
1158	948	# else
1159	949	# define _LIBCPP_LIFETIMEBOUND
1160	950	# endif
1161	951
	952	// This is to work around https://llvm.org/PR156809
	953	# ifndef _LIBCPP_CXX03_LANG
	954	# define _LIBCPP_CTOR_LIFETIMEBOUND _LIBCPP_LIFETIMEBOUND
	955	# else
	956	# define _LIBCPP_CTOR_LIFETIMEBOUND
	957	# endif
	958
1162	959	# if __has_cpp_attribute(_Clang::__noescape__)
1163	960	# define _LIBCPP_NOESCAPE [[_Clang::__noescape__]]
1164	961	# else
...	...	@@ -1172,12 +969,6 @@ typedef __char32_t char32_t;
1172	969	# define _LIBCPP_NO_SPECIALIZATIONS
1173	970	# endif
1174	971
1175		# if __has_cpp_attribute(_Clang::__standalone_debug__)
1176		# define _LIBCPP_STANDALONE_DEBUG [[_Clang::__standalone_debug__]]
1177		# else
1178		# define _LIBCPP_STANDALONE_DEBUG
1179		# endif
1180
1181	972	# if __has_cpp_attribute(_Clang::__preferred_name__)
1182	973	# define _LIBCPP_PREFERRED_NAME(x) [[_Clang::__preferred_name__(x)]]
1183	974	# else
...	...	@@ -1257,14 +1048,6 @@ typedef __char32_t char32_t;
1257	1048	# define _LIBCPP_DIAGNOSE_NULLPTR
1258	1049	# endif
1259	1050
1260		// TODO(LLVM 22): Remove this macro once LLVM19 support ends. __cpp_explicit_this_parameter has been set in LLVM20.
1261		// Clang-18 has support for deducing this, but it does not set the FTM.
1262		# if defined(__cpp_explicit_this_parameter) \|\| (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1800)
1263		# define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 1
1264		# else
1265		# define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 0
1266		# endif
1267
1268	1051	#endif // __cplusplus
1269	1052
1270	1053	#endif // _LIBCPP___CONFIG

lib/libcxx/include/__configuration/abi.h+14-24

...	...	@@ -61,19 +61,9 @@
61	61	// According to the Standard, `bitset::operator[] const` returns bool
62	62	# define _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
63	63
64		// In LLVM 20, we've changed to take these ABI breaks unconditionally. These flags only exist in case someone is running
65		// into the static_asserts we added to catch the ABI break and don't care that it is one.
66		// TODO(LLVM 22): Remove these flags
67		# define _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
68		# define _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
69		# define _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB
70		# define _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
71
72	64	// These flags are documented in ABIGuarantees.rst
73	65	# define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
74		# define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
75		# define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
76		# define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
	66	# define _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
77	67	# define _LIBCPP_ABI_ENABLE_SHARED_PTR_TRIVIAL_ABI
78	68	# define _LIBCPP_ABI_ENABLE_UNIQUE_PTR_TRIVIAL_ABI
79	69	# define _LIBCPP_ABI_FIX_CITYHASH_IMPLEMENTATION
...	...	@@ -84,25 +74,16 @@
84	74	# define _LIBCPP_ABI_NO_FILESYSTEM_INLINE_NAMESPACE
85	75	# define _LIBCPP_ABI_NO_ITERATOR_BASES
86	76	# define _LIBCPP_ABI_NO_RANDOM_DEVICE_COMPATIBILITY_LAYOUT
	77	# define _LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER
87	78	# define _LIBCPP_ABI_OPTIMIZED_FUNCTION
88	79	# define _LIBCPP_ABI_REGEX_CONSTANTS_NONZERO
89	80	# define _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
90	81	# define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_ARRAY
91	82	# define _LIBCPP_ABI_USE_WRAP_ITER_IN_STD_STRING_VIEW
92	83	# define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
	84	# define _LIBCPP_ABI_TRIVIALLY_COPYABLE_BIT_ITERATOR
93	85
94	86	#elif _LIBCPP_ABI_VERSION == 1
95		# if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) \|\| defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
96		// Enable compiling copies of now inline methods into the dylib to support
97		// applications compiled against older libraries. This is unnecessary with
98		// COFF dllexport semantics, since dllexport forces a non-inline definition
99		// of inline functions to be emitted anyway. Our own non-inline copy would
100		// conflict with the dllexport-emitted copy, so we disable it. For XCOFF,
101		// the linker will take issue with the symbols in the shared object if the
102		// weak inline methods get visibility (such as from -fvisibility-inlines-hidden),
103		// so disable it.
104		# define _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
105		# endif
106	87	// Feature macros for disabling pre ABI v1 features. All of these options
107	88	// are deprecated.
108	89	# if defined(__FreeBSD__)
...	...	@@ -110,11 +91,20 @@
110	91	# endif
111	92	#endif
112	93
	94	// TODO(LLVM 22): Remove this check
	95	#if defined(_LIBCPP_ABI_NO_ITERATOR_BASES) && !defined(_LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER)
	96	# ifndef _LIBCPP_ONLY_NO_ITERATOR_BASES
	97	# error "You probably want to define _LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER. This has been split out from" \
	98	" _LIBCPP_ABI_NO_ITERATOR_BASES to allow only removing the second iterator member, since they aren't really related." \
	99	"If you actually want this ABI configuration, please define _LIBCPP_ONLY_NO_ITERATOR_BASES instead."
	100	# endif
	101	#endif
	102
113	103	// We had some bugs where we use [[no_unique_address]] together with construct_at,
114	104	// which causes UB as the call on construct_at could write to overlapping subobjects
115	105	//
116		// https://github.com/llvm/llvm-project/issues/70506
117		// https://github.com/llvm/llvm-project/issues/70494
	106	// https://llvm.org/PR70506
	107	// https://llvm.org/PR70494
118	108	//
119	109	// To fix the bug we had to change the ABI of some classes to remove [[no_unique_address]] under certain conditions.
120	110	// The macro below is used for all classes whose ABI have changed as part of fixing these bugs.

lib/libcxx/include/__configuration/availability.h+65-118

...	...	@@ -17,62 +17,17 @@
17	17	# pragma GCC system_header
18	18	#endif
19	19
20		// Libc++ is shipped by various vendors. In particular, it is used as a system
21		// library on macOS, iOS and other Apple platforms. In order for users to be
22		// able to compile a binary that is intended to be deployed to an older version
23		// of a platform, Clang provides availability attributes [1]. These attributes
24		// can be placed on declarations and are used to describe the life cycle of a
25		// symbol in the library.
26		//
27		// The main goal is to ensure a compile-time error if a symbol that hasn't been
28		// introduced in a previously released library is used in a program that targets
29		// that previously released library. Normally, this would be a load-time error
30		// when one tries to launch the program against the older library.
31		//
32		// For example, the filesystem library was introduced in the dylib in LLVM 9.
33		// On Apple platforms, this corresponds to macOS 10.15. If a user compiles on
34		// a macOS 10.15 host but targets macOS 10.13 with their program, the compiler
35		// would normally not complain (because the required declarations are in the
36		// headers), but the dynamic loader would fail to find the symbols when actually
37		// trying to launch the program on macOS 10.13. To turn this into a compile-time
38		// issue instead, declarations are annotated with when they were introduced, and
39		// the compiler can produce a diagnostic if the program references something that
40		// isn't available on the deployment target.
41		//
42		// This mechanism is general in nature, and any vendor can add their markup to
43		// the library (see below). Whenever a new feature is added that requires support
44		// in the shared library, two macros are added below to allow marking the feature
45		// as unavailable:
46		// 1. A macro named `_LIBCPP_AVAILABILITY_HAS_<feature>` which must be defined
47		// to `_LIBCPP_INTRODUCED_IN_<version>` for the appropriate LLVM version.
48		// 2. A macro named `_LIBCPP_AVAILABILITY_<feature>`, which must be defined to
49		// `_LIBCPP_INTRODUCED_IN_<version>_MARKUP` for the appropriate LLVM version.
50		//
51		// When vendors decide to ship the feature as part of their shared library, they
52		// can update the `_LIBCPP_INTRODUCED_IN_<version>` macro (and the markup counterpart)
53		// based on the platform version they shipped that version of LLVM in. The library
54		// will then use this markup to provide an optimal user experience on these platforms.
55		//
56		// Furthermore, many features in the standard library have corresponding
57		// feature-test macros. The `_LIBCPP_AVAILABILITY_HAS_<feature>` macros
58		// are checked by the corresponding feature-test macros generated by
59		// generate_feature_test_macro_components.py to ensure that the library
60		// doesn't announce a feature as being implemented if it is unavailable on
61		// the deployment target.
62		//
63		// Note that this mechanism is disabled by default in the "upstream" libc++.
64		// Availability annotations are only meaningful when shipping libc++ inside
65		// a platform (i.e. as a system library), and so vendors that want them should
66		// turn those annotations on at CMake configuration time.
67		//
68		// [1]: https://clang.llvm.org/docs/AttributeReference.html#availability
	20	// This file defines a framework that can be used by vendors to encode the version of an operating system that various
	21	// features of libc++ has been shipped in. This is primarily intended to allow safely deploying an executable built with
	22	// a new version of the library on a platform containing an older version of the built library.
	23	// Detailed documentation for this can be found at https://libcxx.llvm.org/VendorDocumentation.html#availability-markup
69	24
70	25	// Availability markup is disabled when building the library, or when a non-Clang
71	26	// compiler is used because only Clang supports the necessary attributes.
72	27	//
73	28	// We also allow users to force-disable availability markup via the `_LIBCPP_DISABLE_AVAILABILITY`
74	29	// macro because that is the only way to work around a Clang bug related to availability
75		// attributes: https://github.com/llvm/llvm-project/issues/134151.
	30	// attributes: https://llvm.org/PR134151.
76	31	// Once that bug has been fixed, we should remove the macro.
77	32	#if defined(_LIBCPP_BUILDING_LIBRARY) \|\| defined(_LIBCXXABI_BUILDING_LIBRARY) \|\| \
78	33	!defined(_LIBCPP_COMPILER_CLANG_BASED) \|\| defined(_LIBCPP_DISABLE_AVAILABILITY)
...	...	@@ -84,6 +39,9 @@
84	39	// in all versions of the library are available.
85	40	#if !_LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS
86	41
	42	# define _LIBCPP_INTRODUCED_IN_LLVM_22 1
	43	# define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE /* nothing */
	44
87	45	# define _LIBCPP_INTRODUCED_IN_LLVM_21 1
88	46	# define _LIBCPP_INTRODUCED_IN_LLVM_21_ATTRIBUTE /* nothing */
89	47
...	...	@@ -108,32 +66,55 @@
108	66	# define _LIBCPP_INTRODUCED_IN_LLVM_12 1
109	67	# define _LIBCPP_INTRODUCED_IN_LLVM_12_ATTRIBUTE /* nothing */
110	68
111		# define _LIBCPP_INTRODUCED_IN_LLVM_11 1
112		# define _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE /* nothing */
113
114		# define _LIBCPP_INTRODUCED_IN_LLVM_9 1
115		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE /* nothing */
116		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH /* nothing */
117		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP /* nothing */
118
119	69	#elif defined(__APPLE__)
120	70
121	71	// clang-format off
122	72
	73	// LLVM 22
	74	// TODO: Fill this in
	75	# define _LIBCPP_INTRODUCED_IN_LLVM_22 0
	76	# define _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE __attribute__((unavailable))
	77
123	78	// LLVM 21
124	79	// TODO: Fill this in
125	80	# define _LIBCPP_INTRODUCED_IN_LLVM_21 0
126	81	# define _LIBCPP_INTRODUCED_IN_LLVM_21_ATTRIBUTE __attribute__((unavailable))
127	82
128	83	// LLVM 20
129		// TODO: Fill this in
130		# define _LIBCPP_INTRODUCED_IN_LLVM_20 0
131		# define _LIBCPP_INTRODUCED_IN_LLVM_20_ATTRIBUTE __attribute__((unavailable))
	84	//
	85	// Note that versions for most Apple OSes were bumped forward and aligned in that release.
	86	# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 260000) \|\| \
	87	(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 260000) \|\| \
	88	(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 260000) \|\| \
	89	(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 260000) \|\| \
	90	(defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 100000)
	91	# define _LIBCPP_INTRODUCED_IN_LLVM_20 0
	92	# else
	93	# define _LIBCPP_INTRODUCED_IN_LLVM_20 1
	94	# endif
	95	# define _LIBCPP_INTRODUCED_IN_LLVM_20_ATTRIBUTE \
	96	__attribute__((availability(macos, strict, introduced = 26.0))) \
	97	__attribute__((availability(ios, strict, introduced = 26.0))) \
	98	__attribute__((availability(tvos, strict, introduced = 26.0))) \
	99	__attribute__((availability(watchos, strict, introduced = 26.0))) \
	100	__attribute__((availability(bridgeos, strict, introduced = 10.0)))
132	101
133	102	// LLVM 19
134		// TODO: Fill this in
135		# define _LIBCPP_INTRODUCED_IN_LLVM_19 0
136		# define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE __attribute__((unavailable))
	103	# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 150400) \|\| \
	104	(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 180400) \|\| \
	105	(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 180400) \|\| \
	106	(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 110400) \|\| \
	107	(defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 90400)
	108	# define _LIBCPP_INTRODUCED_IN_LLVM_19 0
	109	# else
	110	# define _LIBCPP_INTRODUCED_IN_LLVM_19 1
	111	# endif
	112	# define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE \
	113	__attribute__((availability(macos, strict, introduced = 15.4))) \
	114	__attribute__((availability(ios, strict, introduced = 18.4))) \
	115	__attribute__((availability(tvos, strict, introduced = 18.4))) \
	116	__attribute__((availability(watchos, strict, introduced = 11.4))) \
	117	__attribute__((availability(bridgeos, strict, introduced = 9.4)))
137	118
138	119	// LLVM 18
139	120	# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 150000) \|\| \
...	...	@@ -215,47 +196,13 @@
215	196	__attribute__((availability(bridgeos, strict, introduced = 6.0))) \
216	197	__attribute__((availability(driverkit, strict, introduced = 21.3)))
217	198
218		// LLVM 11
219		# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000) \|\| \
220		(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 140000) \|\| \
221		(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 140000) \|\| \
222		(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 70000)
223		# define _LIBCPP_INTRODUCED_IN_LLVM_11 0
224		# else
225		# define _LIBCPP_INTRODUCED_IN_LLVM_11 1
	199	# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000) \|\| \
	200	(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 150000) \|\| \
	201	(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 150000) \|\| \
	202	(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 80000) \|\| \
	203	(defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 200000)
	204	# warning "The selected platform is no longer supported by libc++."
226	205	# endif
227		# define _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE \
228		__attribute__((availability(macos, strict, introduced = 11.0))) \
229		__attribute__((availability(ios, strict, introduced = 14.0))) \
230		__attribute__((availability(tvos, strict, introduced = 14.0))) \
231		__attribute__((availability(watchos, strict, introduced = 7.0)))
232
233		// LLVM 9
234		# if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500) \|\| \
235		(defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000) \|\| \
236		(defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 130000) \|\| \
237		(defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 60000)
238		# define _LIBCPP_INTRODUCED_IN_LLVM_9 0
239		# else
240		# define _LIBCPP_INTRODUCED_IN_LLVM_9 1
241		# endif
242		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE \
243		__attribute__((availability(macos, strict, introduced = 10.15))) \
244		__attribute__((availability(ios, strict, introduced = 13.0))) \
245		__attribute__((availability(tvos, strict, introduced = 13.0))) \
246		__attribute__((availability(watchos, strict, introduced = 6.0)))
247		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH \
248		_Pragma("clang attribute push(__attribute__((availability(macos,strict,introduced=10.15))), apply_to=any(function,record))") \
249		_Pragma("clang attribute push(__attribute__((availability(ios,strict,introduced=13.0))), apply_to=any(function,record))") \
250		_Pragma("clang attribute push(__attribute__((availability(tvos,strict,introduced=13.0))), apply_to=any(function,record))") \
251		_Pragma("clang attribute push(__attribute__((availability(watchos,strict,introduced=6.0))), apply_to=any(function,record))")
252		# define _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP \
253		_Pragma("clang attribute pop") \
254		_Pragma("clang attribute pop") \
255		_Pragma("clang attribute pop") \
256		_Pragma("clang attribute pop")
257
258		// clang-format on
259	206
260	207	#else
261	208
...	...	@@ -266,19 +213,12 @@
266	213
267	214	#endif
268	215
269		// These macros control the availability of all parts of <filesystem> that
270		// depend on something in the dylib.
271		#define _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9
272		#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE
273		#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_PUSH
274		#define _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP _LIBCPP_INTRODUCED_IN_LLVM_9_ATTRIBUTE_POP
275
276		// This controls the availability of the C++20 synchronization library,
277		// which requires shared library support for various operations
278		// (see libcxx/src/atomic.cpp). This includes <barier>, <latch>,
279		// <semaphore>, and notification functions on std::atomic.
280		#define _LIBCPP_AVAILABILITY_HAS_SYNC _LIBCPP_INTRODUCED_IN_LLVM_11
281		#define _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INTRODUCED_IN_LLVM_11_ATTRIBUTE
	216	// This controls the availability of new implementation of std::atomic's
	217	// wait, notify_one and notify_all. The new implementation uses
	218	// the native atomic wait/notify operations on platforms that support them
	219	// based on the size of the atomic type, instead of the type itself.
	220	#define _LIBCPP_AVAILABILITY_HAS_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22
	221	#define _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_INTRODUCED_IN_LLVM_22_ATTRIBUTE
282	222
283	223	// Enable additional explicit instantiations of iostreams components. This
284	224	// reduces the number of weak definitions generated in programs that use
...	...	@@ -307,7 +247,7 @@
307	247	// This controls the availability of the C++17 std::pmr library,
308	248	// which is implemented in large part in the built library.
309	249	//
310		// TODO: Enable std::pmr markup once https://github.com/llvm/llvm-project/issues/40340 has been fixed
	250	// TODO: Enable std::pmr markup once https://llvm.org/PR40340 has been fixed
311	251	// Until then, it is possible for folks to try to use `std::pmr` when back-deploying to targets that don't support
312	252	// it and it'll be a load-time error, but we don't have a good alternative because the library won't compile if we
313	253	// use availability annotations until that bug has been fixed.
...	...	@@ -364,4 +304,11 @@
364	304	# define _LIBCPP_AVAILABILITY_INIT_PRIMARY_EXCEPTION
365	305	#endif
366	306
	307	// Only define a bunch of symbols in the dylib if we need to be compatible with LLVM 7 headers or older
	308	# if defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 8
	309	# define _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8
	310	# else
	311	# define _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 _LIBCPP_HIDE_FROM_ABI
	312	# endif
	313
367	314	#endif // _LIBCPP___CONFIGURATION_AVAILABILITY_H

lib/libcxx/include/__configuration/compiler.h+6-6

...	...	@@ -33,16 +33,16 @@
33	33	// Warn if a compiler version is used that is not supported anymore
34	34	// LLVM RELEASE Update the minimum compiler versions
35	35	# if defined(_LIBCPP_CLANG_VER)
36		# if _LIBCPP_CLANG_VER < 1900
37		# warning "Libc++ only supports Clang 19 and later"
	36	# if _LIBCPP_CLANG_VER < 2001
	37	# warning "Libc++ only supports Clang 20 and later"
38	38	# endif
39	39	# elif defined(_LIBCPP_APPLE_CLANG_VER)
40		# if _LIBCPP_APPLE_CLANG_VER < 1500
41		# warning "Libc++ only supports AppleClang 15 and later"
	40	# if _LIBCPP_APPLE_CLANG_VER < 1700
	41	# warning "Libc++ only supports AppleClang 26 and later"
42	42	# endif
43	43	# elif defined(_LIBCPP_GCC_VER)
44		# if _LIBCPP_GCC_VER < 1400
45		# warning "Libc++ only supports GCC 14 and later"
	44	# if _LIBCPP_GCC_VER < 1500
	45	# warning "Libc++ only supports GCC 15 and later"
46	46	# endif
47	47	# endif
48	48

lib/libcxx/include/__configuration/experimental.h created+38

...	...	@@ -0,0 +1,38 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___CONFIGURATION_EXPERIMENTAL_H
	10	#define _LIBCPP___CONFIGURATION_EXPERIMENTAL_H
	11
	12	/* zig patch: instead of including __config_site, zig adds -D flags when compiling */
	13
	14	#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
	15	# pragma GCC system_header
	16	#endif
	17
	18	#if __has_feature(experimental_library)
	19	# ifndef _LIBCPP_ENABLE_EXPERIMENTAL
	20	# define _LIBCPP_ENABLE_EXPERIMENTAL
	21	# endif
	22	#endif
	23
	24	// Incomplete features get their own specific disabling flags. This makes it
	25	// easier to grep for target specific flags once the feature is complete.
	26	#if defined(_LIBCPP_ENABLE_EXPERIMENTAL) \|\| defined(_LIBCPP_BUILDING_LIBRARY)
	27	# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 1
	28	#else
	29	# define _LIBCPP_HAS_EXPERIMENTAL_LIBRARY 0
	30	#endif
	31
	32	#define _LIBCPP_HAS_EXPERIMENTAL_PSTL _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	33	#define _LIBCPP_HAS_EXPERIMENTAL_TZDB _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	34	#define _LIBCPP_HAS_EXPERIMENTAL_SYNCSTREAM _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	35	#define _LIBCPP_HAS_EXPERIMENTAL_HARDENING_OBSERVE_SEMANTIC _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	36	#define _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR _LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	37
	38	#endif // _LIBCPP___CONFIGURATION_EXPERIMENTAL_H

lib/libcxx/include/__configuration/hardening.h created+215

...	...	@@ -0,0 +1,215 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___CONFIGURATION_HARDENING_H
	10	#define _LIBCPP___CONFIGURATION_HARDENING_H
	11
	12	/* zig patch: instead of including __config_site, zig adds -D flags when compiling */
	13	#include <__configuration/experimental.h>
	14	#include <__configuration/language.h>
	15
	16	#ifndef _LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER
	17	# pragma GCC system_header
	18	#endif
	19
	20	// TODO(LLVM 23): Remove this. We're making these an error to catch folks who might not have migrated.
	21	// Since hardening went through several changes (many of which impacted user-facing macros),
	22	// we're keeping these checks around for a bit longer than usual. Failure to properly configure
	23	// hardening results in checks being dropped silently, which is a pretty big deal.
	24	#if defined(_LIBCPP_ENABLE_ASSERTIONS)
	25	# error "_LIBCPP_ENABLE_ASSERTIONS has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
	26	#endif
	27	#if defined(_LIBCPP_ENABLE_HARDENED_MODE)
	28	# error "_LIBCPP_ENABLE_HARDENED_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
	29	#endif
	30	#if defined(_LIBCPP_ENABLE_SAFE_MODE)
	31	# error "_LIBCPP_ENABLE_SAFE_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
	32	#endif
	33	#if defined(_LIBCPP_ENABLE_DEBUG_MODE)
	34	# error "_LIBCPP_ENABLE_DEBUG_MODE has been removed, please use _LIBCPP_HARDENING_MODE=<mode> instead (see docs)"
	35	#endif
	36
	37	// The library provides the macro `_LIBCPP_HARDENING_MODE` which can be set to one of the following values:
	38	//
	39	// - `_LIBCPP_HARDENING_MODE_NONE`;
	40	// - `_LIBCPP_HARDENING_MODE_FAST`;
	41	// - `_LIBCPP_HARDENING_MODE_EXTENSIVE`;
	42	// - `_LIBCPP_HARDENING_MODE_DEBUG`.
	43	//
	44	// These values have the following effects:
	45	//
	46	// - `_LIBCPP_HARDENING_MODE_NONE` -- sets the hardening mode to "none" which disables all runtime hardening checks;
	47	//
	48	// - `_LIBCPP_HARDENING_MODE_FAST` -- sets that hardening mode to "fast". The fast mode enables security-critical checks
	49	// that can be done with relatively little runtime overhead in constant time;
	50	//
	51	// - `_LIBCPP_HARDENING_MODE_EXTENSIVE` -- sets the hardening mode to "extensive". The extensive mode is a superset of
	52	// the fast mode that additionally enables checks that are relatively cheap and prevent common types of logic errors
	53	// but are not necessarily security-critical;
	54	//
	55	// - `_LIBCPP_HARDENING_MODE_DEBUG` -- sets the hardening mode to "debug". The debug mode is a superset of the extensive
	56	// mode and enables all checks available in the library, including internal assertions. Checks that are part of the
	57	// debug mode can be very expensive and thus the debug mode is intended to be used for testing, not in production.
	58
	59	// Inside the library, assertions are categorized so they can be cherry-picked based on the chosen hardening mode. These
	60	// macros are only for internal use -- users should only pick one of the high-level hardening modes described above.
	61	//
	62	// - `_LIBCPP_ASSERT_VALID_INPUT_RANGE` -- checks that ranges (whether expressed as an iterator pair, an iterator and
	63	// a sentinel, an iterator and a count, or a `std::range`) given as input to library functions are valid:
	64	// - the sentinel is reachable from the begin iterator;
	65	// - TODO(hardening): both iterators refer to the same container.
	66	//
	67	// - `_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS` -- checks that any attempts to access a container element, whether through
	68	// the container object or through an iterator, are valid and do not attempt to go out of bounds or otherwise access
	69	// a non-existent element. For iterator checks to work, bounded iterators must be enabled in the ABI. Types like
	70	// `optional` and `function` are considered one-element containers for the purposes of this check.
	71	//
	72	// - `_LIBCPP_ASSERT_NON_NULL` -- checks that the pointer being dereferenced is not null. On most modern platforms zero
	73	// address does not refer to an actual location in memory, so a null pointer dereference would not compromize the
	74	// memory security of a program (however, it is still undefined behavior that can result in strange errors due to
	75	// compiler optimizations).
	76	//
	77	// - `_LIBCPP_ASSERT_NON_OVERLAPPING_RANGES` -- for functions that take several ranges as arguments, checks that the
	78	// given ranges do not overlap.
	79	//
	80	// - `_LIBCPP_ASSERT_VALID_DEALLOCATION` -- checks that an attempt to deallocate memory is valid (e.g. the given object
	81	// was allocated by the given allocator). Violating this category typically results in a memory leak.
	82	//
	83	// - `_LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL` -- checks that a call to an external API doesn't fail in
	84	// an unexpected manner. This includes triggering documented cases of undefined behavior in an external library (like
	85	// attempting to unlock an unlocked mutex in pthreads). Any API external to the library falls under this category
	86	// (from system calls to compiler intrinsics). We generally don't expect these failures to compromize memory safety or
	87	// otherwise create an immediate security issue.
	88	//
	89	// - `_LIBCPP_ASSERT_COMPATIBLE_ALLOCATOR` -- checks any operations that exchange nodes between containers to make sure
	90	// the containers have compatible allocators.
	91	//
	92	// - `_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN` -- checks that the given argument is within the domain of valid arguments
	93	// for the function. Violating this typically produces an incorrect result (e.g. the clamp algorithm returns the
	94	// original value without clamping it due to incorrect functors) or puts an object into an invalid state (e.g.
	95	// a string view where only a subset of elements is possible to access). This category is for assertions violating
	96	// which doesn't cause any immediate issues in the library -- whatever the consequences are, they will happen in the
	97	// user code.
	98	//
	99	// - `_LIBCPP_ASSERT_PEDANTIC` -- checks prerequisites which are imposed by the Standard, but violating which happens to
	100	// be benign in our implementation.
	101	//
	102	// - `_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT` -- checks that the given argument satisfies the semantic requirements imposed
	103	// by the Standard. Typically, there is no simple way to completely prove that a semantic requirement is satisfied;
	104	// thus, this would often be a heuristic check and it might be quite expensive.
	105	//
	106	// - `_LIBCPP_ASSERT_INTERNAL` -- checks that internal invariants of the library hold. These assertions don't depend on
	107	// user input.
	108	//
	109	// - `_LIBCPP_ASSERT_UNCATEGORIZED` -- for assertions that haven't been properly classified yet.
	110
	111	// clang-format off
	112	# define _LIBCPP_HARDENING_MODE_NONE (1 << 1)
	113	# define _LIBCPP_HARDENING_MODE_FAST (1 << 2)
	114	# define _LIBCPP_HARDENING_MODE_EXTENSIVE (1 << 4) // Deliberately not ordered.
	115	# define _LIBCPP_HARDENING_MODE_DEBUG (1 << 3)
	116	// clang-format on
	117
	118	#ifndef _LIBCPP_HARDENING_MODE
	119
	120	# ifndef _LIBCPP_HARDENING_MODE_DEFAULT
	121	# error _LIBCPP_HARDENING_MODE_DEFAULT is not defined. This definition should be set at configuration time in the \
	122	`__config_site` header, please make sure your installation of libc++ is not broken.
	123	# endif
	124
	125	# define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_DEFAULT
	126	#endif
	127
	128	#if _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_NONE && _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_FAST && \
	129	_LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_EXTENSIVE && \
	130	_LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_DEBUG
	131	# error _LIBCPP_HARDENING_MODE must be set to one of the following values: \
	132	_LIBCPP_HARDENING_MODE_NONE, \
	133	_LIBCPP_HARDENING_MODE_FAST, \
	134	_LIBCPP_HARDENING_MODE_EXTENSIVE, \
	135	_LIBCPP_HARDENING_MODE_DEBUG
	136	#endif
	137
	138	// The library provides the macro `_LIBCPP_ASSERTION_SEMANTIC` for configuring the assertion semantic used by hardening;
	139	// it can be set to one of the following values:
	140	//
	141	// - `_LIBCPP_ASSERTION_SEMANTIC_IGNORE`;
	142	// - `_LIBCPP_ASSERTION_SEMANTIC_OBSERVE`;
	143	// - `_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE`;
	144	// - `_LIBCPP_ASSERTION_SEMANTIC_ENFORCE`.
	145	//
	146	// libc++ assertion semantics generally mirror the evaluation semantics of C++26 Contracts:
	147	// - `ignore` evaluates the assertion but doesn't do anything if it fails (note that it differs from the Contracts
	148	// `ignore` semantic which wouldn't evaluate the assertion at all);
	149	// - `observe` logs an error (indicating, if possible, that the error is fatal) and continues execution;
	150	// - `quick-enforce` terminates the program as fast as possible (via trapping);
	151	// - `enforce` logs an error and then terminates the program.
	152	//
	153	// Additionally, a special `hardening-dependent` value selects the assertion semantic based on the hardening mode in
	154	// effect: the production-capable modes (`fast` and `extensive`) map to `quick_enforce` and the `debug` mode maps to
	155	// `enforce`. The `hardening-dependent` semantic cannot be selected explicitly, it is only used when no assertion
	156	// semantic is provided by the user _and_ the library's default semantic is configured to be dependent on hardening.
	157	//
	158	// Notes:
	159	// - Continuing execution after a hardening check fails results in undefined behavior; the `observe` semantic is meant
	160	// to make adopting hardening easier but should not be used outside of this scenario;
	161	// - C++26 wording for Library Hardening precludes a conforming Hardened implementation from using the Contracts
	162	// `ignore` semantic when evaluating hardened preconditions in the Library. Libc++ allows using this semantic for
	163	// hardened preconditions, however, be aware that using `ignore` does not produce a conforming "Hardened"
	164	// implementation, unlike the other semantics above.
	165	// clang-format off
	166	# define _LIBCPP_ASSERTION_SEMANTIC_HARDENING_DEPENDENT (1 << 1)
	167	# define _LIBCPP_ASSERTION_SEMANTIC_IGNORE (1 << 2)
	168	# define _LIBCPP_ASSERTION_SEMANTIC_OBSERVE (1 << 3)
	169	# define _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE (1 << 4)
	170	# define _LIBCPP_ASSERTION_SEMANTIC_ENFORCE (1 << 5)
	171	// clang-format on
	172
	173	// If the user attempts to configure the assertion semantic, check that it is allowed in the current environment.
	174	#if defined(_LIBCPP_ASSERTION_SEMANTIC)
	175	# if !_LIBCPP_HAS_EXPERIMENTAL_LIBRARY
	176	# error "Assertion semantics are an experimental feature."
	177	# endif
	178	# if defined(_LIBCPP_CXX03_LANG)
	179	# error "Assertion semantics are not available in the C++03 mode."
	180	# endif
	181	#endif // defined(_LIBCPP_ASSERTION_SEMANTIC)
	182
	183	// User-provided semantic takes top priority -- don't override if set.
	184	#ifndef _LIBCPP_ASSERTION_SEMANTIC
	185
	186	# ifndef _LIBCPP_ASSERTION_SEMANTIC_DEFAULT
	187	# error _LIBCPP_ASSERTION_SEMANTIC_DEFAULT is not defined. This definition should be set at configuration time in \
	188	the `__config_site` header, please make sure your installation of libc++ is not broken.
	189	# endif
	190
	191	# if _LIBCPP_ASSERTION_SEMANTIC_DEFAULT != _LIBCPP_ASSERTION_SEMANTIC_HARDENING_DEPENDENT
	192	# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_DEFAULT
	193	# else
	194	# if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
	195	# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	196	# else
	197	# define _LIBCPP_ASSERTION_SEMANTIC _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE
	198	# endif
	199	# endif // _LIBCPP_ASSERTION_SEMANTIC_DEFAULT != _LIBCPP_ASSERTION_SEMANTIC_HARDENING_DEPENDENT
	200
	201	#endif // #ifndef _LIBCPP_ASSERTION_SEMANTIC
	202
	203	// Finally, validate the selected semantic (in case the user tries setting it to an incorrect value):
	204	#if _LIBCPP_ASSERTION_SEMANTIC != _LIBCPP_ASSERTION_SEMANTIC_IGNORE && \
	205	_LIBCPP_ASSERTION_SEMANTIC != _LIBCPP_ASSERTION_SEMANTIC_OBSERVE && \
	206	_LIBCPP_ASSERTION_SEMANTIC != _LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE && \
	207	_LIBCPP_ASSERTION_SEMANTIC != _LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	208	# error _LIBCPP_ASSERTION_SEMANTIC must be set to one of the following values: \
	209	_LIBCPP_ASSERTION_SEMANTIC_IGNORE, \
	210	_LIBCPP_ASSERTION_SEMANTIC_OBSERVE, \
	211	_LIBCPP_ASSERTION_SEMANTIC_QUICK_ENFORCE, \
	212	_LIBCPP_ASSERTION_SEMANTIC_ENFORCE
	213	#endif
	214
	215	#endif // _LIBCPP___CONFIGURATION_HARDENING_H

lib/libcxx/include/__configuration/language.h+3

...	...	@@ -18,6 +18,9 @@
18	18
19	19	// NOLINTBEGIN(libcpp-cpp-version-check)
20	20	#ifdef __cplusplus
	21	# if __cplusplus < 201103L
	22	# define _LIBCPP_CXX03_LANG
	23	# endif
21	24	# if __cplusplus <= 201103L
22	25	# define _LIBCPP_STD_VER 11
23	26	# elif __cplusplus <= 201402L

lib/libcxx/include/__configuration/platform.h+9-16

...	...	@@ -31,22 +31,15 @@
31	31	#endif
32	32
33	33	// Need to detect which libc we're using if we're on Linux.
34		#if defined(__linux__) \|\| defined(__AMDGPU__) \|\| defined(__NVPTX__)
35		# if __has_include(<features.h>)
36		# include <features.h>
37		# if defined(__GLIBC_PREREQ)
38		# define _LIBCPP_GLIBC_PREREQ(a, b) __GLIBC_PREREQ(a, b)
39		# else
40		# define _LIBCPP_GLIBC_PREREQ(a, b) 0
41		# endif // defined(__GLIBC_PREREQ)
42		# endif
43		#endif
44
45		// This is required in order for _NEWLIB_VERSION to be defined in places where we use it.
46		// TODO: We shouldn't be including arbitrarily-named headers from libc++ since this can break valid
47		// user code. Move code paths that need _NEWLIB_VERSION to another customization mechanism.
48		#if __has_include(<picolibc.h>)
49		# include <picolibc.h>
	34	#if (defined(__linux__) \|\| defined(__AMDGPU__) \|\| defined(__NVPTX__)) && __has_include(<features.h>)
	35	# include <features.h>
	36	# if defined(__GLIBC_PREREQ)
	37	# define _LIBCPP_GLIBC_PREREQ(a, b) __GLIBC_PREREQ(a, b)
	38	# else
	39	# define _LIBCPP_GLIBC_PREREQ(a, b) 0
	40	# endif // defined(__GLIBC_PREREQ)
	41	#else
	42	# define _LIBCPP_GLIBC_PREREQ(a, b) 0
50	43	#endif
51	44
52	45	#ifndef __BYTE_ORDER__

lib/libcxx/include/__coroutine/coroutine_handle.h+9-9

...	...	@@ -44,9 +44,9 @@ public:
44	44	}
45	45
46	46	// [coroutine.handle.export.import], export/import
47		_LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
	47	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
48	48
49		_LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
	49	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
50	50	coroutine_handle __tmp;
51	51	__tmp.__handle_ = __addr;
52	52	return __tmp;
...	...	@@ -55,7 +55,7 @@ public:
55	55	// [coroutine.handle.observers], observers
56	56	_LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; }
57	57
58		_LIBCPP_HIDE_FROM_ABI bool done() const {
	58	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool done() const {
59	59	_LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "done() can be called only on suspended coroutines");
60	60	return __builtin_coro_done(__handle_);
61	61	}
...	...	@@ -100,7 +100,7 @@ public:
100	100
101	101	_LIBCPP_HIDE_FROM_ABI constexpr coroutine_handle(nullptr_t) noexcept {}
102	102
103		_LIBCPP_HIDE_FROM_ABI static coroutine_handle from_promise(_Promise& __promise) {
	103	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static coroutine_handle from_promise(_Promise& __promise) {
104	104	using _RawPromise = __remove_cv_t<_Promise>;
105	105	coroutine_handle __tmp;
106	106	__tmp.__handle_ =
...	...	@@ -114,9 +114,9 @@ public:
114	114	}
115	115
116	116	// [coroutine.handle.export.import], export/import
117		_LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
	117	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
118	118
119		_LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
	119	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr coroutine_handle from_address(void* __addr) noexcept {
120	120	coroutine_handle __tmp;
121	121	__tmp.__handle_ = __addr;
122	122	return __tmp;
...	...	@@ -130,7 +130,7 @@ public:
130	130	// [coroutine.handle.observers], observers
131	131	_LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return __handle_ != nullptr; }
132	132
133		_LIBCPP_HIDE_FROM_ABI bool done() const {
	133	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool done() const {
134	134	_LIBCPP_ASSERT_VALID_EXTERNAL_API_CALL(__is_suspended(), "done() can be called only on suspended coroutines");
135	135	return __builtin_coro_done(__handle_);
136	136	}
...	...	@@ -150,7 +150,7 @@ public:
150	150	}
151	151
152	152	// [coroutine.handle.promise], promise access
153		_LIBCPP_HIDE_FROM_ABI _Promise& promise() const {
	153	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Promise& promise() const {
154	154	return static_cast<_Promise>(__builtin_coro_promise(this->__handle_, alignof(_Promise), false));
155	155	}
156	156
...	...	@@ -165,7 +165,7 @@ private:
165	165	// [coroutine.handle.hash]
166	166	template <class _Tp>
167	167	struct hash<coroutine_handle<_Tp>> {
168		_LIBCPP_HIDE_FROM_ABI size_t operator()(const coroutine_handle<_Tp>& __v) const noexcept {
	168	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const coroutine_handle<_Tp>& __v) const noexcept {
169	169	return hash<void*>()(__v.address());
170	170	}
171	171	};

lib/libcxx/include/__coroutine/noop_coroutine_handle.h+11-13

...	...	@@ -20,8 +20,6 @@
20	20
21	21	_LIBCPP_BEGIN_NAMESPACE_STD
22	22
23		# if __has_builtin(__builtin_coro_noop) \|\| defined(_LIBCPP_COMPILER_GCC)
24
25	23	// [coroutine.noop]
26	24	// [coroutine.promise.noop]
27	25	struct noop_coroutine_promise {};
...	...	@@ -37,7 +35,7 @@ public:
37	35
38	36	// [coroutine.handle.noop.observers], observers
39	37	_LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return true; }
40		_LIBCPP_HIDE_FROM_ABI constexpr bool done() const noexcept { return false; }
	38	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool done() const noexcept { return false; }
41	39
42	40	// [coroutine.handle.noop.resumption], resumption
43	41	_LIBCPP_HIDE_FROM_ABI constexpr void operator()() const noexcept {}
...	...	@@ -45,23 +43,23 @@ public:
45	43	_LIBCPP_HIDE_FROM_ABI constexpr void destroy() const noexcept {}
46	44
47	45	// [coroutine.handle.noop.promise], promise access
48		_LIBCPP_HIDE_FROM_ABI noop_coroutine_promise& promise() const noexcept {
	46	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI noop_coroutine_promise& promise() const noexcept {
49	47	return static_cast<noop_coroutine_promise>(
50	48	__builtin_coro_promise(this->__handle_, alignof(noop_coroutine_promise), false));
51	49	}
52	50
53	51	// [coroutine.handle.noop.address], address
54		_LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
	52	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr void* address() const noexcept { return __handle_; }
55	53
56	54	private:
57	55	_LIBCPP_HIDE_FROM_ABI friend coroutine_handle<noop_coroutine_promise> noop_coroutine() noexcept;
58	56
59		# if __has_builtin(__builtin_coro_noop)
	57	# if __has_builtin(__builtin_coro_noop)
60	58	_LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept { this->__handle_ = __builtin_coro_noop(); }
61	59
62	60	void* __handle_ = nullptr;
63	61
64		# elif defined(_LIBCPP_COMPILER_GCC)
	62	# elif defined(_LIBCPP_COMPILER_GCC)
65	63	// GCC doesn't implement __builtin_coro_noop().
66	64	// Construct the coroutine frame manually instead.
67	65	struct __noop_coroutine_frame_ty_ {
...	...	@@ -78,19 +76,19 @@ private:
78	76
79	77	_LIBCPP_HIDE_FROM_ABI coroutine_handle() noexcept = default;
80	78
81		# endif // __has_builtin(__builtin_coro_noop)
	79	# endif // __has_builtin(__builtin_coro_noop)
82	80	};
83	81
84	82	using noop_coroutine_handle = coroutine_handle<noop_coroutine_promise>;
85	83
86		# if defined(_LIBCPP_COMPILER_GCC)
	84	# if defined(_LIBCPP_COMPILER_GCC)
87	85	inline noop_coroutine_handle::__noop_coroutine_frame_ty_ noop_coroutine_handle::__noop_coroutine_frame_{};
88		# endif
	86	# endif
89	87
90	88	// [coroutine.noop.coroutine]
91		inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept { return noop_coroutine_handle(); }
92
93		# endif // __has_builtin(__builtin_coro_noop) \|\| defined(_LIBCPP_COMPILER_GCC)
	89	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept {
	90	return noop_coroutine_handle();
	91	}
94	92
95	93	_LIBCPP_END_NAMESPACE_STD
96	94

lib/libcxx/include/__debug_utils/strict_weak_ordering_check.h+1-1

...	...	@@ -27,7 +27,7 @@ template <class _RandomAccessIterator, class _Comp>
27	27	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
28	28	__check_strict_weak_ordering_sorted(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp& __comp) {
29	29	#if _LIBCPP_HARDENING_MODE == _LIBCPP_HARDENING_MODE_DEBUG
30		using __diff_t = __iter_diff_t<_RandomAccessIterator>;
	30	using __diff_t = __iterator_difference_type<_RandomAccessIterator>;
31	31	using _Comp_ref = __comp_ref_type<_Comp>;
32	32	if (!__libcpp_is_constant_evaluated()) {
33	33	// Check if the range is actually sorted.

lib/libcxx/include/__exception/exception.h+6-4

...	...	@@ -48,13 +48,15 @@ public:
48	48	__data_._DoFree = true;
49	49	}
50	50
51		exception(exception const&) _NOEXCEPT {}
	51	exception(exception const&) _NOEXCEPT : __data_() {}
52	52
53	53	exception& operator=(exception const&) _NOEXCEPT { return *this; }
54	54
55	55	virtual ~exception() _NOEXCEPT {}
56	56
57		virtual char const* what() const _NOEXCEPT { return __data_._What ? __data_._What : "Unknown exception"; }
	57	[[__nodiscard__]] virtual char const* what() const _NOEXCEPT {
	58	return __data_._What ? __data_._What : "Unknown exception";
	59	}
58	60
59	61	private:
60	62	__std_exception_data __data_;
...	...	@@ -76,7 +78,7 @@ public:
76	78	_LIBCPP_HIDE_FROM_ABI exception& operator=(const exception&) _NOEXCEPT = default;
77	79
78	80	virtual ~exception() _NOEXCEPT;
79		virtual const char* what() const _NOEXCEPT;
	81	[[__nodiscard__]] virtual const char* what() const _NOEXCEPT;
80	82	};
81	83
82	84	class _LIBCPP_EXPORTED_FROM_ABI bad_exception : public exception {
...	...	@@ -85,7 +87,7 @@ public:
85	87	_LIBCPP_HIDE_FROM_ABI bad_exception(const bad_exception&) _NOEXCEPT = default;
86	88	_LIBCPP_HIDE_FROM_ABI bad_exception& operator=(const bad_exception&) _NOEXCEPT = default;
87	89	~bad_exception() _NOEXCEPT override;
88		const char* what() const _NOEXCEPT override;
	90	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
89	91	};
90	92	#endif // !_LIBCPP_ABI_VCRUNTIME
91	93

lib/libcxx/include/__exception/exception_ptr.h+27-5

...	...	@@ -11,18 +11,24 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/nullptr_t.h>
	14	#include <__cstddef/size_t.h>
14	15	#include <__exception/operations.h>
15	16	#include <__memory/addressof.h>
16	17	#include <__memory/construct_at.h>
17	18	#include <__type_traits/decay.h>
18	19	#include <__type_traits/is_pointer.h>
19		#include <cstdlib>
	20	#include <__utility/move.h>
	21	#include <__utility/swap.h>
	22	#include <__verbose_abort>
20	23	#include <typeinfo>
21	24
22	25	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
23	26	# pragma GCC system_header
24	27	#endif
25	28
	29	_LIBCPP_PUSH_MACROS
	30	#include <__undef_macros>
	31
26	32	#ifndef _LIBCPP_ABI_MICROSOFT
27	33
28	34	# if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
...	...	@@ -30,7 +36,7 @@
30	36	namespace __cxxabiv1 {
31	37
32	38	extern "C" {
33		_LIBCPP_OVERRIDABLE_FUNC_VIS void* __cxa_allocate_exception(size_t) throw();
	39	_LIBCPP_OVERRIDABLE_FUNC_VIS void* __cxa_allocate_exception(std::size_t) throw();
34	40	_LIBCPP_OVERRIDABLE_FUNC_VIS void __cxa_free_exception(void*) throw();
35	41
36	42	struct __cxa_exception;
...	...	@@ -57,6 +63,8 @@ _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
57	63
58	64	#ifndef _LIBCPP_ABI_MICROSOFT
59	65
	66	inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT;
	67
60	68	class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
61	69	void* __ptr_;
62	70
...	...	@@ -67,15 +75,21 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr {
67	75
68	76	public:
69	77	// exception_ptr is basically a COW string so it is trivially relocatable.
70		// It is also replaceable because assignment has normal value semantics.
71	78	using __trivially_relocatable _LIBCPP_NODEBUG = exception_ptr;
72		using __replaceable _LIBCPP_NODEBUG = exception_ptr;
73	79
74	80	_LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {}
75	81	_LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {}
76	82
77	83	exception_ptr(const exception_ptr&) _NOEXCEPT;
	84	_LIBCPP_HIDE_FROM_ABI exception_ptr(exception_ptr&& __other) _NOEXCEPT : __ptr_(__other.__ptr_) {
	85	__other.__ptr_ = nullptr;
	86	}
78	87	exception_ptr& operator=(const exception_ptr&) _NOEXCEPT;
	88	_LIBCPP_HIDE_FROM_ABI exception_ptr& operator=(exception_ptr&& __other) _NOEXCEPT {
	89	exception_ptr __tmp(std::move(__other));
	90	std::swap(__tmp, *this);
	91	return *this;
	92	}
79	93	~exception_ptr() _NOEXCEPT;
80	94
81	95	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __ptr_ != nullptr; }
...	...	@@ -88,10 +102,16 @@ public:
88	102	return !(__x == __y);
89	103	}
90	104
	105	friend _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT;
	106
91	107	friend _LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
92	108	friend _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
93	109	};
94	110
	111	inline _LIBCPP_HIDE_FROM_ABI void swap(exception_ptr& __x, exception_ptr& __y) _NOEXCEPT {
	112	std::swap(__x.__ptr_, __y.__ptr_);
	113	}
	114
95	115	# if _LIBCPP_HAS_EXCEPTIONS
96	116	# if _LIBCPP_AVAILABILITY_HAS_INIT_PRIMARY_EXCEPTION
97	117	template <class _Ep>
...	...	@@ -153,7 +173,7 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
153	173	# else // !_LIBCPP_HAS_EXCEPTIONS
154	174	template <class _Ep>
155	175	_LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT {
156		std::abort();
	176	_LIBCPP_VERBOSE_ABORT("make_exception_ptr was called in -fno-exceptions mode");
157	177	}
158	178	# endif // _LIBCPP_HAS_EXCEPTIONS
159	179
...	...	@@ -201,4 +221,6 @@ _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep __e) _NOEXCEPT {
201	221	#endif // _LIBCPP_ABI_MICROSOFT
202	222	_LIBCPP_END_UNVERSIONED_NAMESPACE_STD
203	223
	224	_LIBCPP_POP_MACROS
	225
204	226	#endif // _LIBCPP___EXCEPTION_EXCEPTION_PTR_H

lib/libcxx/include/__exception/nested_exception.h+2-2

...	...	@@ -40,7 +40,7 @@ public:
40	40
41	41	// access functions
42	42	[[__noreturn__]] void rethrow_nested() const;
43		_LIBCPP_HIDE_FROM_ABI exception_ptr nested_ptr() const _NOEXCEPT { return __ptr_; }
	43	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI exception_ptr nested_ptr() const _NOEXCEPT { return __ptr_; }
44	44	};
45	45
46	46	template <class _Tp>
...	...	@@ -73,7 +73,7 @@ template <class _Tp>
73	73	__throw_with_nested<_Tp,
74	74	_Up,
75	75	is_class<_Up>::value && !is_base_of<nested_exception, _Up>::value &&
76		!__libcpp_is_final<_Up>::value>::__do_throw(std::forward<_Tp>(__t));
	76	!__is_final_v<_Up> >::__do_throw(std::forward<_Tp>(__t));
77	77	#else
78	78	((void)__t);
79	79	// FIXME: Make this abort

lib/libcxx/include/__exception/operations.h+5-5

...	...	@@ -20,22 +20,22 @@ _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
20	20	defined(_LIBCPP_BUILDING_LIBRARY)
21	21	using unexpected_handler = void (*)();
22	22	_LIBCPP_EXPORTED_FROM_ABI unexpected_handler set_unexpected(unexpected_handler) _NOEXCEPT;
23		_LIBCPP_EXPORTED_FROM_ABI unexpected_handler get_unexpected() _NOEXCEPT;
	23	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI unexpected_handler get_unexpected() _NOEXCEPT;
24	24	[[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void unexpected();
25	25	#endif
26	26
27	27	using terminate_handler = void (*)();
28	28	_LIBCPP_EXPORTED_FROM_ABI terminate_handler set_terminate(terminate_handler) _NOEXCEPT;
29		_LIBCPP_EXPORTED_FROM_ABI terminate_handler get_terminate() _NOEXCEPT;
	29	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI terminate_handler get_terminate() _NOEXCEPT;
30	30
31	31	#if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION)
32		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 bool uncaught_exception() _NOEXCEPT;
	32	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 bool uncaught_exception() _NOEXCEPT;
33	33	#endif // _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION)
34		_LIBCPP_EXPORTED_FROM_ABI int uncaught_exceptions() _NOEXCEPT;
	34	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI int uncaught_exceptions() _NOEXCEPT;
35	35
36	36	class _LIBCPP_EXPORTED_FROM_ABI exception_ptr;
37	37
38		_LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
	38	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI exception_ptr current_exception() _NOEXCEPT;
39	39	[[__noreturn__]] _LIBCPP_EXPORTED_FROM_ABI void rethrow_exception(exception_ptr);
40	40	_LIBCPP_END_UNVERSIONED_NAMESPACE_STD
41	41

lib/libcxx/include/__expected/bad_expected_access.h+8-6

...	...	@@ -43,9 +43,11 @@ protected:
43	43
44	44	public:
45	45	# if _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION
46		const char* what() const noexcept override;
	46	[[nodiscard]] const char* what() const noexcept override;
47	47	# else
48		_LIBCPP_HIDE_FROM_ABI_VIRTUAL const char* what() const noexcept override { return "bad access to std::expected"; }
	48	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI_VIRTUAL const char* what() const noexcept override {
	49	return "bad access to std::expected";
	50	}
49	51	# endif
50	52	};
51	53	_LIBCPP_DIAGNOSTIC_POP
...	...	@@ -55,10 +57,10 @@ class bad_expected_access : public bad_expected_access<void> {
55	57	public:
56	58	_LIBCPP_HIDE_FROM_ABI explicit bad_expected_access(_Err __e) : __unex_(std::move(__e)) {}
57	59
58		_LIBCPP_HIDE_FROM_ABI _Err& error() & noexcept { return __unex_; }
59		_LIBCPP_HIDE_FROM_ABI const _Err& error() const& noexcept { return __unex_; }
60		_LIBCPP_HIDE_FROM_ABI _Err&& error() && noexcept { return std::move(__unex_); }
61		_LIBCPP_HIDE_FROM_ABI const _Err&& error() const&& noexcept { return std::move(__unex_); }
	60	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Err& error() & noexcept { return __unex_; }
	61	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const _Err& error() const& noexcept { return __unex_; }
	62	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Err&& error() && noexcept { return std::move(__unex_); }
	63	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const _Err&& error() const&& noexcept { return std::move(__unex_); }
62	64
63	65	private:
64	66	_Err __unex_;

lib/libcxx/include/__expected/expected.h+62-64

...	...	@@ -30,7 +30,6 @@
30	30	#include <__type_traits/is_nothrow_assignable.h>
31	31	#include <__type_traits/is_nothrow_constructible.h>
32	32	#include <__type_traits/is_reference.h>
33		#include <__type_traits/is_replaceable.h>
34	33	#include <__type_traits/is_same.h>
35	34	#include <__type_traits/is_swappable.h>
36	35	#include <__type_traits/is_trivially_constructible.h>
...	...	@@ -472,8 +471,6 @@ public:
472	471	__conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value && __libcpp_is_trivially_relocatable<_Err>::value,
473	472	expected,
474	473	void>;
475		using __replaceable _LIBCPP_NODEBUG =
476		__conditional_t<__is_replaceable_v<_Tp> && __is_replaceable_v<_Err>, expected, void>;
477	474
478	475	template <class _Up>
479	476	using rebind = expected<_Up, error_type>;
...	...	@@ -555,9 +552,10 @@ public:
555	552	is_nothrow_constructible_v<_Tp, _Up> && is_nothrow_constructible_v<_Err, _OtherErr>) // strengthened
556	553	: __base(__other.__has_val(), std::move(__other.__union())) {}
557	554
558		template <class _Up = _Tp>
	555	template <class _Up = remove_cv_t<_Tp>>
559	556	requires(!is_same_v<remove_cvref_t<_Up>, in_place_t> && !is_same_v<expected, remove_cvref_t<_Up>> &&
560		is_constructible_v<_Tp, _Up> && !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
	557	!is_same_v<remove_cvref_t<_Up>, unexpect_t> && is_constructible_v<_Tp, _Up> &&
	558	!__is_std_unexpected<remove_cvref_t<_Up>>::value &&
561	559	(!is_same_v<remove_cv_t<_Tp>, bool> \|\| !__is_std_expected<remove_cvref_t<_Up>>::value))
562	560	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp>)
563	561	expected(_Up&& __u) noexcept(is_nothrow_constructible_v<_Tp, _Up>) // strengthened
...	...	@@ -668,7 +666,7 @@ public:
668	666	return *this;
669	667	}
670	668
671		template <class _Up = _Tp>
	669	template <class _Up = remove_cv_t<_Tp>>
672	670	_LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(_Up&& __v)
673	671	requires(!is_same_v<expected, remove_cvref_t<_Up>> && !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
674	672	is_constructible_v<_Tp, _Up> && is_assignable_v<_Tp&, _Up> &&
...	...	@@ -800,25 +798,25 @@ public:
800	798	return std::addressof(this->__val());
801	799	}
802	800
803		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator*() const& noexcept {
	801	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator*() const& noexcept {
804	802	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
805	803	this->__has_val(), "expected::operator* requires the expected to contain a value");
806	804	return this->__val();
807	805	}
808	806
809		_LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() & noexcept {
	807	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() & noexcept {
810	808	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
811	809	this->__has_val(), "expected::operator* requires the expected to contain a value");
812	810	return this->__val();
813	811	}
814	812
815		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& operator*() const&& noexcept {
	813	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& operator*() const&& noexcept {
816	814	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
817	815	this->__has_val(), "expected::operator* requires the expected to contain a value");
818	816	return std::move(this->__val());
819	817	}
820	818
821		_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator*() && noexcept {
	819	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator*() && noexcept {
822	820	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
823	821	this->__has_val(), "expected::operator* requires the expected to contain a value");
824	822	return std::move(this->__val());
...	...	@@ -826,9 +824,9 @@ public:
826	824
827	825	_LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return this->__has_val(); }
828	826
829		_LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
	827	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
830	828
831		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& value() const& {
	829	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& value() const& {
832	830	static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
833	831	if (!this->__has_val()) {
834	832	std::__throw_bad_expected_access<_Err>(std::as_const(error()));
...	...	@@ -836,7 +834,7 @@ public:
836	834	return this->__val();
837	835	}
838	836
839		_LIBCPP_HIDE_FROM_ABI constexpr _Tp& value() & {
	837	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& value() & {
840	838	static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
841	839	if (!this->__has_val()) {
842	840	std::__throw_bad_expected_access<_Err>(std::as_const(error()));
...	...	@@ -844,7 +842,7 @@ public:
844	842	return this->__val();
845	843	}
846	844
847		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& value() const&& {
	845	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& value() const&& {
848	846	static_assert(is_copy_constructible_v<_Err> && is_constructible_v<_Err, decltype(std::move(error()))>,
849	847	"error_type has to be both copy constructible and constructible from decltype(std::move(error()))");
850	848	if (!this->__has_val()) {
...	...	@@ -853,7 +851,7 @@ public:
853	851	return std::move(this->__val());
854	852	}
855	853
856		_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& value() && {
	854	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& value() && {
857	855	static_assert(is_copy_constructible_v<_Err> && is_constructible_v<_Err, decltype(std::move(error()))>,
858	856	"error_type has to be both copy constructible and constructible from decltype(std::move(error()))");
859	857	if (!this->__has_val()) {
...	...	@@ -862,46 +860,46 @@ public:
862	860	return std::move(this->__val());
863	861	}
864	862
865		_LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
	863	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
866	864	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
867	865	!this->__has_val(), "expected::error requires the expected to contain an error");
868	866	return this->__unex();
869	867	}
870	868
871		_LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
	869	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
872	870	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
873	871	!this->__has_val(), "expected::error requires the expected to contain an error");
874	872	return this->__unex();
875	873	}
876	874
877		_LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
	875	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
878	876	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
879	877	!this->__has_val(), "expected::error requires the expected to contain an error");
880	878	return std::move(this->__unex());
881	879	}
882	880
883		_LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
	881	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
884	882	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
885	883	!this->__has_val(), "expected::error requires the expected to contain an error");
886	884	return std::move(this->__unex());
887	885	}
888	886
889		template <class _Up>
890		_LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) const& {
	887	template <class _Up = remove_cv_t<_Tp>>
	888	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) const& {
891	889	static_assert(is_copy_constructible_v<_Tp>, "value_type has to be copy constructible");
892	890	static_assert(is_convertible_v<_Up, _Tp>, "argument has to be convertible to value_type");
893	891	return this->__has_val() ? this->__val() : static_cast<_Tp>(std::forward<_Up>(__v));
894	892	}
895	893
896		template <class _Up>
897		_LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) && {
	894	template <class _Up = remove_cv_t<_Tp>>
	895	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) && {
898	896	static_assert(is_move_constructible_v<_Tp>, "value_type has to be move constructible");
899	897	static_assert(is_convertible_v<_Up, _Tp>, "argument has to be convertible to value_type");
900	898	return this->__has_val() ? std::move(this->__val()) : static_cast<_Tp>(std::forward<_Up>(__v));
901	899	}
902	900
903	901	template <class _Up = _Err>
904		_LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
	902	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
905	903	static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
906	904	static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
907	905	if (has_value())
...	...	@@ -910,7 +908,7 @@ public:
910	908	}
911	909
912	910	template <class _Up = _Err>
913		_LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
	911	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
914	912	static_assert(is_move_constructible_v<_Err>, "error_type has to be move constructible");
915	913	static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
916	914	if (has_value())
...	...	@@ -921,7 +919,7 @@ public:
921	919	// [expected.void.monadic], monadic
922	920	template <class _Func>
923	921	requires is_constructible_v<_Err, _Err&>
924		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
	922	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
925	923	using _Up = remove_cvref_t<invoke_result_t<_Func, _Tp&>>;
926	924	static_assert(__is_std_expected<_Up>::value, "The result of f(value()) must be a specialization of std::expected");
927	925	static_assert(is_same_v<typename _Up::error_type, _Err>,
...	...	@@ -934,7 +932,7 @@ public:
934	932
935	933	template <class _Func>
936	934	requires is_constructible_v<_Err, const _Err&>
937		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
	935	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
938	936	using _Up = remove_cvref_t<invoke_result_t<_Func, const _Tp&>>;
939	937	static_assert(__is_std_expected<_Up>::value, "The result of f(value()) must be a specialization of std::expected");
940	938	static_assert(is_same_v<typename _Up::error_type, _Err>,
...	...	@@ -947,7 +945,7 @@ public:
947	945
948	946	template <class _Func>
949	947	requires is_constructible_v<_Err, _Err&&>
950		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
	948	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
951	949	using _Up = remove_cvref_t<invoke_result_t<_Func, _Tp&&>>;
952	950	static_assert(
953	951	__is_std_expected<_Up>::value, "The result of f(std::move(value())) must be a specialization of std::expected");
...	...	@@ -961,7 +959,7 @@ public:
961	959
962	960	template <class _Func>
963	961	requires is_constructible_v<_Err, const _Err&&>
964		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
	962	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
965	963	using _Up = remove_cvref_t<invoke_result_t<_Func, const _Tp&&>>;
966	964	static_assert(
967	965	__is_std_expected<_Up>::value, "The result of f(std::move(value())) must be a specialization of std::expected");
...	...	@@ -975,7 +973,7 @@ public:
975	973
976	974	template <class _Func>
977	975	requires is_constructible_v<_Tp, _Tp&>
978		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
	976	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
979	977	using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&>>;
980	978	static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
981	979	static_assert(is_same_v<typename _Gp::value_type, _Tp>,
...	...	@@ -988,7 +986,7 @@ public:
988	986
989	987	template <class _Func>
990	988	requires is_constructible_v<_Tp, const _Tp&>
991		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
	989	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
992	990	using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&>>;
993	991	static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
994	992	static_assert(is_same_v<typename _Gp::value_type, _Tp>,
...	...	@@ -1001,7 +999,7 @@ public:
1001	999
1002	1000	template <class _Func>
1003	1001	requires is_constructible_v<_Tp, _Tp&&>
1004		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
	1002	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
1005	1003	using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&&>>;
1006	1004	static_assert(
1007	1005	__is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
...	...	@@ -1015,7 +1013,7 @@ public:
1015	1013
1016	1014	template <class _Func>
1017	1015	requires is_constructible_v<_Tp, const _Tp&&>
1018		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
	1016	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
1019	1017	using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&&>>;
1020	1018	static_assert(
1021	1019	__is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
...	...	@@ -1029,7 +1027,7 @@ public:
1029	1027
1030	1028	template <class _Func>
1031	1029	requires is_constructible_v<_Err, _Err&>
1032		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
	1030	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
1033	1031	using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&>>;
1034	1032	if (!has_value()) {
1035	1033	return expected<_Up, _Err>(unexpect, error());
...	...	@@ -1045,7 +1043,7 @@ public:
1045	1043
1046	1044	template <class _Func>
1047	1045	requires is_constructible_v<_Err, const _Err&>
1048		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
	1046	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
1049	1047	using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&>>;
1050	1048	if (!has_value()) {
1051	1049	return expected<_Up, _Err>(unexpect, error());
...	...	@@ -1061,7 +1059,7 @@ public:
1061	1059
1062	1060	template <class _Func>
1063	1061	requires is_constructible_v<_Err, _Err&&>
1064		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
	1062	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
1065	1063	using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&&>>;
1066	1064	if (!has_value()) {
1067	1065	return expected<_Up, _Err>(unexpect, std::move(error()));
...	...	@@ -1077,7 +1075,7 @@ public:
1077	1075
1078	1076	template <class _Func>
1079	1077	requires is_constructible_v<_Err, const _Err&&>
1080		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
	1078	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
1081	1079	using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&&>>;
1082	1080	if (!has_value()) {
1083	1081	return expected<_Up, _Err>(unexpect, std::move(error()));
...	...	@@ -1093,7 +1091,7 @@ public:
1093	1091
1094	1092	template <class _Func>
1095	1093	requires is_constructible_v<_Tp, _Tp&>
1096		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
	1094	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
1097	1095	using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&>>;
1098	1096	static_assert(__valid_std_unexpected<_Gp>::value,
1099	1097	"The result of f(error()) must be a valid template argument for unexpected");
...	...	@@ -1105,7 +1103,7 @@ public:
1105	1103
1106	1104	template <class _Func>
1107	1105	requires is_constructible_v<_Tp, const _Tp&>
1108		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
	1106	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
1109	1107	using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&>>;
1110	1108	static_assert(__valid_std_unexpected<_Gp>::value,
1111	1109	"The result of f(error()) must be a valid template argument for unexpected");
...	...	@@ -1117,7 +1115,7 @@ public:
1117	1115
1118	1116	template <class _Func>
1119	1117	requires is_constructible_v<_Tp, _Tp&&>
1120		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
	1118	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
1121	1119	using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&&>>;
1122	1120	static_assert(__valid_std_unexpected<_Gp>::value,
1123	1121	"The result of f(std::move(error())) must be a valid template argument for unexpected");
...	...	@@ -1130,7 +1128,7 @@ public:
1130	1128
1131	1129	template <class _Func>
1132	1130	requires is_constructible_v<_Tp, const _Tp&&>
1133		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
	1131	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
1134	1132	using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&&>>;
1135	1133	static_assert(__valid_std_unexpected<_Gp>::value,
1136	1134	"The result of f(std::move(error())) must be a valid template argument for unexpected");
...	...	@@ -1597,7 +1595,7 @@ public:
1597	1595	// [expected.void.obs], observers
1598	1596	_LIBCPP_HIDE_FROM_ABI constexpr explicit operator bool() const noexcept { return this->__has_val(); }
1599	1597
1600		_LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
	1598	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__has_val(); }
1601	1599
1602	1600	_LIBCPP_HIDE_FROM_ABI constexpr void operator*() const noexcept {
1603	1601	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
...	...	@@ -1618,32 +1616,32 @@ public:
1618	1616	}
1619	1617	}
1620	1618
1621		_LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
	1619	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept {
1622	1620	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
1623	1621	!this->__has_val(), "expected::error requires the expected to contain an error");
1624	1622	return this->__unex();
1625	1623	}
1626	1624
1627		_LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
	1625	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept {
1628	1626	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
1629	1627	!this->__has_val(), "expected::error requires the expected to contain an error");
1630	1628	return this->__unex();
1631	1629	}
1632	1630
1633		_LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
	1631	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept {
1634	1632	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
1635	1633	!this->__has_val(), "expected::error requires the expected to contain an error");
1636	1634	return std::move(this->__unex());
1637	1635	}
1638	1636
1639		_LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
	1637	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept {
1640	1638	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
1641	1639	!this->__has_val(), "expected::error requires the expected to contain an error");
1642	1640	return std::move(this->__unex());
1643	1641	}
1644	1642
1645	1643	template <class _Up = _Err>
1646		_LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
	1644	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) const& {
1647	1645	static_assert(is_copy_constructible_v<_Err>, "error_type has to be copy constructible");
1648	1646	static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
1649	1647	if (has_value()) {
...	...	@@ -1653,7 +1651,7 @@ public:
1653	1651	}
1654	1652
1655	1653	template <class _Up = _Err>
1656		_LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
	1654	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err error_or(_Up&& __error) && {
1657	1655	static_assert(is_move_constructible_v<_Err>, "error_type has to be move constructible");
1658	1656	static_assert(is_convertible_v<_Up, _Err>, "argument has to be convertible to error_type");
1659	1657	if (has_value()) {
...	...	@@ -1665,7 +1663,7 @@ public:
1665	1663	// [expected.void.monadic], monadic
1666	1664	template <class _Func>
1667	1665	requires is_constructible_v<_Err, _Err&>
1668		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
	1666	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
1669	1667	using _Up = remove_cvref_t<invoke_result_t<_Func>>;
1670	1668	static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
1671	1669	static_assert(
...	...	@@ -1678,7 +1676,7 @@ public:
1678	1676
1679	1677	template <class _Func>
1680	1678	requires is_constructible_v<_Err, const _Err&>
1681		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
	1679	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
1682	1680	using _Up = remove_cvref_t<invoke_result_t<_Func>>;
1683	1681	static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
1684	1682	static_assert(
...	...	@@ -1691,7 +1689,7 @@ public:
1691	1689
1692	1690	template <class _Func>
1693	1691	requires is_constructible_v<_Err, _Err&&>
1694		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
	1692	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
1695	1693	using _Up = remove_cvref_t<invoke_result_t<_Func>>;
1696	1694	static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
1697	1695	static_assert(
...	...	@@ -1704,7 +1702,7 @@ public:
1704	1702
1705	1703	template <class _Func>
1706	1704	requires is_constructible_v<_Err, const _Err&&>
1707		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
	1705	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
1708	1706	using _Up = remove_cvref_t<invoke_result_t<_Func>>;
1709	1707	static_assert(__is_std_expected<_Up>::value, "The result of f() must be a specialization of std::expected");
1710	1708	static_assert(
...	...	@@ -1716,7 +1714,7 @@ public:
1716	1714	}
1717	1715
1718	1716	template <class _Func>
1719		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
	1717	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) & {
1720	1718	using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&>>;
1721	1719	static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
1722	1720	static_assert(is_same_v<typename _Gp::value_type, _Tp>,
...	...	@@ -1728,7 +1726,7 @@ public:
1728	1726	}
1729	1727
1730	1728	template <class _Func>
1731		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
	1729	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const& {
1732	1730	using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&>>;
1733	1731	static_assert(__is_std_expected<_Gp>::value, "The result of f(error()) must be a specialization of std::expected");
1734	1732	static_assert(is_same_v<typename _Gp::value_type, _Tp>,
...	...	@@ -1740,7 +1738,7 @@ public:
1740	1738	}
1741	1739
1742	1740	template <class _Func>
1743		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
	1741	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) && {
1744	1742	using _Gp = remove_cvref_t<invoke_result_t<_Func, _Err&&>>;
1745	1743	static_assert(
1746	1744	__is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
...	...	@@ -1753,7 +1751,7 @@ public:
1753	1751	}
1754	1752
1755	1753	template <class _Func>
1756		_LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
	1754	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto or_else(_Func&& __f) const&& {
1757	1755	using _Gp = remove_cvref_t<invoke_result_t<_Func, const _Err&&>>;
1758	1756	static_assert(
1759	1757	__is_std_expected<_Gp>::value, "The result of f(std::move(error())) must be a specialization of std::expected");
...	...	@@ -1767,7 +1765,7 @@ public:
1767	1765
1768	1766	template <class _Func>
1769	1767	requires is_constructible_v<_Err, _Err&>
1770		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
	1768	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
1771	1769	using _Up = remove_cv_t<invoke_result_t<_Func>>;
1772	1770	if (!has_value()) {
1773	1771	return expected<_Up, _Err>(unexpect, error());
...	...	@@ -1782,7 +1780,7 @@ public:
1782	1780
1783	1781	template <class _Func>
1784	1782	requires is_constructible_v<_Err, const _Err&>
1785		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
	1783	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
1786	1784	using _Up = remove_cv_t<invoke_result_t<_Func>>;
1787	1785	if (!has_value()) {
1788	1786	return expected<_Up, _Err>(unexpect, error());
...	...	@@ -1797,7 +1795,7 @@ public:
1797	1795
1798	1796	template <class _Func>
1799	1797	requires is_constructible_v<_Err, _Err&&>
1800		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
	1798	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
1801	1799	using _Up = remove_cv_t<invoke_result_t<_Func>>;
1802	1800	if (!has_value()) {
1803	1801	return expected<_Up, _Err>(unexpect, std::move(error()));
...	...	@@ -1812,7 +1810,7 @@ public:
1812	1810
1813	1811	template <class _Func>
1814	1812	requires is_constructible_v<_Err, const _Err&&>
1815		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
	1813	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
1816	1814	using _Up = remove_cv_t<invoke_result_t<_Func>>;
1817	1815	if (!has_value()) {
1818	1816	return expected<_Up, _Err>(unexpect, std::move(error()));
...	...	@@ -1826,7 +1824,7 @@ public:
1826	1824	}
1827	1825
1828	1826	template <class _Func>
1829		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
	1827	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) & {
1830	1828	using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&>>;
1831	1829	static_assert(__valid_std_unexpected<_Gp>::value,
1832	1830	"The result of f(error()) must be a valid template argument for unexpected");
...	...	@@ -1837,7 +1835,7 @@ public:
1837	1835	}
1838	1836
1839	1837	template <class _Func>
1840		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
	1838	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const& {
1841	1839	using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&>>;
1842	1840	static_assert(__valid_std_unexpected<_Gp>::value,
1843	1841	"The result of f(error()) must be a valid template argument for unexpected");
...	...	@@ -1848,7 +1846,7 @@ public:
1848	1846	}
1849	1847
1850	1848	template <class _Func>
1851		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
	1849	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) && {
1852	1850	using _Gp = remove_cv_t<invoke_result_t<_Func, _Err&&>>;
1853	1851	static_assert(__valid_std_unexpected<_Gp>::value,
1854	1852	"The result of f(std::move(error())) must be a valid template argument for unexpected");
...	...	@@ -1860,7 +1858,7 @@ public:
1860	1858	}
1861	1859
1862	1860	template <class _Func>
1863		_LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
	1861	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform_error(_Func&& __f) const&& {
1864	1862	using _Gp = remove_cv_t<invoke_result_t<_Func, const _Err&&>>;
1865	1863	static_assert(__valid_std_unexpected<_Gp>::value,
1866	1864	"The result of f(std::move(error())) must be a valid template argument for unexpected");

lib/libcxx/include/__expected/unexpected.h+4-4

...	...	@@ -89,10 +89,10 @@ public:
89	89	_LIBCPP_HIDE_FROM_ABI constexpr unexpected& operator=(const unexpected&) = default;
90	90	_LIBCPP_HIDE_FROM_ABI constexpr unexpected& operator=(unexpected&&) = default;
91	91
92		_LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept { return __unex_; }
93		_LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept { return __unex_; }
94		_LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept { return std::move(__unex_); }
95		_LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept { return std::move(__unex_); }
	92	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err& error() const& noexcept { return __unex_; }
	93	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err& error() & noexcept { return __unex_; }
	94	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Err&& error() const&& noexcept { return std::move(__unex_); }
	95	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Err&& error() && noexcept { return std::move(__unex_); }
96	96
97	97	_LIBCPP_HIDE_FROM_ABI constexpr void swap(unexpected& __other) noexcept(is_nothrow_swappable_v<_Err>) {
98	98	static_assert(is_swappable_v<_Err>, "unexpected::swap requires is_swappable_v<E> to be true");

lib/libcxx/include/__filesystem/copy_options.h+4-4

...	...	@@ -34,19 +34,19 @@ enum class copy_options : unsigned short {
34	34	__in_recursive_copy = 512,
35	35	};
36	36
37		_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator&(copy_options __lhs, copy_options __rhs) {
	37	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator&(copy_options __lhs, copy_options __rhs) {
38	38	return static_cast<copy_options>(static_cast<unsigned short>(__lhs) & static_cast<unsigned short>(__rhs));
39	39	}
40	40
41		_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator\|(copy_options __lhs, copy_options __rhs) {
	41	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator\|(copy_options __lhs, copy_options __rhs) {
42	42	return static_cast<copy_options>(static_cast<unsigned short>(__lhs) \| static_cast<unsigned short>(__rhs));
43	43	}
44	44
45		_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator^(copy_options __lhs, copy_options __rhs) {
	45	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator^(copy_options __lhs, copy_options __rhs) {
46	46	return static_cast<copy_options>(static_cast<unsigned short>(__lhs) ^ static_cast<unsigned short>(__rhs));
47	47	}
48	48
49		_LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator~(copy_options __lhs) {
	49	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr copy_options operator~(copy_options __lhs) {
50	50	return static_cast<copy_options>(~static_cast<unsigned short>(__lhs));
51	51	}
52	52

lib/libcxx/include/__filesystem/directory_entry.h+37-33

...	...	@@ -40,8 +40,6 @@ _LIBCPP_PUSH_MACROS
40	40
41	41	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
42	42
43		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
44
45	43	class directory_entry {
46	44	typedef filesystem::path _Path;
47	45
...	...	@@ -89,80 +87,88 @@ public:
89	87
90	88	_LIBCPP_HIDE_FROM_ABI void refresh(error_code& __ec) noexcept { __refresh(&__ec); }
91	89
92		_LIBCPP_HIDE_FROM_ABI _Path const& path() const noexcept { return __p_; }
	90	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _Path const& path() const noexcept { return __p_; }
93	91
94	92	_LIBCPP_HIDE_FROM_ABI operator const _Path&() const noexcept { return __p_; }
95	93
96		_LIBCPP_HIDE_FROM_ABI bool exists() const { return filesystem::exists(file_status{__get_ft()}); }
	94	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool exists() const { return filesystem::exists(file_status{__get_ft()}); }
97	95
98		_LIBCPP_HIDE_FROM_ABI bool exists(error_code& __ec) const noexcept {
	96	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool exists(error_code& __ec) const noexcept {
99	97	return filesystem::exists(file_status{__get_ft(&__ec)});
100	98	}
101	99
102		_LIBCPP_HIDE_FROM_ABI bool is_block_file() const { return __get_ft() == file_type::block; }
	100	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_block_file() const { return __get_ft() == file_type::block; }
103	101
104		_LIBCPP_HIDE_FROM_ABI bool is_block_file(error_code& __ec) const noexcept {
	102	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_block_file(error_code& __ec) const noexcept {
105	103	return __get_ft(&__ec) == file_type::block;
106	104	}
107	105
108		_LIBCPP_HIDE_FROM_ABI bool is_character_file() const { return __get_ft() == file_type::character; }
	106	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_character_file() const { return __get_ft() == file_type::character; }
109	107
110		_LIBCPP_HIDE_FROM_ABI bool is_character_file(error_code& __ec) const noexcept {
	108	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_character_file(error_code& __ec) const noexcept {
111	109	return __get_ft(&__ec) == file_type::character;
112	110	}
113	111
114		_LIBCPP_HIDE_FROM_ABI bool is_directory() const { return __get_ft() == file_type::directory; }
	112	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_directory() const { return __get_ft() == file_type::directory; }
115	113
116		_LIBCPP_HIDE_FROM_ABI bool is_directory(error_code& __ec) const noexcept {
	114	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_directory(error_code& __ec) const noexcept {
117	115	return __get_ft(&__ec) == file_type::directory;
118	116	}
119	117
120		_LIBCPP_HIDE_FROM_ABI bool is_fifo() const { return __get_ft() == file_type::fifo; }
	118	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_fifo() const { return __get_ft() == file_type::fifo; }
121	119
122		_LIBCPP_HIDE_FROM_ABI bool is_fifo(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::fifo; }
	120	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_fifo(error_code& __ec) const noexcept {
	121	return __get_ft(&__ec) == file_type::fifo;
	122	}
123	123
124		_LIBCPP_HIDE_FROM_ABI bool is_other() const { return filesystem::is_other(file_status{__get_ft()}); }
	124	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_other() const { return filesystem::is_other(file_status{__get_ft()}); }
125	125
126		_LIBCPP_HIDE_FROM_ABI bool is_other(error_code& __ec) const noexcept {
	126	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_other(error_code& __ec) const noexcept {
127	127	return filesystem::is_other(file_status{__get_ft(&__ec)});
128	128	}
129	129
130		_LIBCPP_HIDE_FROM_ABI bool is_regular_file() const { return __get_ft() == file_type::regular; }
	130	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_regular_file() const { return __get_ft() == file_type::regular; }
131	131
132		_LIBCPP_HIDE_FROM_ABI bool is_regular_file(error_code& __ec) const noexcept {
	132	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_regular_file(error_code& __ec) const noexcept {
133	133	return __get_ft(&__ec) == file_type::regular;
134	134	}
135	135
136		_LIBCPP_HIDE_FROM_ABI bool is_socket() const { return __get_ft() == file_type::socket; }
	136	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_socket() const { return __get_ft() == file_type::socket; }
137	137
138		_LIBCPP_HIDE_FROM_ABI bool is_socket(error_code& __ec) const noexcept { return __get_ft(&__ec) == file_type::socket; }
	138	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_socket(error_code& __ec) const noexcept {
	139	return __get_ft(&__ec) == file_type::socket;
	140	}
139	141
140		_LIBCPP_HIDE_FROM_ABI bool is_symlink() const { return __get_sym_ft() == file_type::symlink; }
	142	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_symlink() const { return __get_sym_ft() == file_type::symlink; }
141	143
142		_LIBCPP_HIDE_FROM_ABI bool is_symlink(error_code& __ec) const noexcept {
	144	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_symlink(error_code& __ec) const noexcept {
143	145	return __get_sym_ft(&__ec) == file_type::symlink;
144	146	}
145		_LIBCPP_HIDE_FROM_ABI uintmax_t file_size() const { return __get_size(); }
	147	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI uintmax_t file_size() const { return __get_size(); }
146	148
147		_LIBCPP_HIDE_FROM_ABI uintmax_t file_size(error_code& __ec) const noexcept { return __get_size(&__ec); }
	149	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(error_code& __ec) const noexcept { return __get_size(&__ec); }
148	150
149		_LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count() const { return __get_nlink(); }
	151	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count() const { return __get_nlink(); }
150	152
151		_LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(error_code& __ec) const noexcept { return __get_nlink(&__ec); }
	153	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(error_code& __ec) const noexcept {
	154	return __get_nlink(&__ec);
	155	}
152	156
153		_LIBCPP_HIDE_FROM_ABI file_time_type last_write_time() const { return __get_write_time(); }
	157	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time() const { return __get_write_time(); }
154	158
155		_LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(error_code& __ec) const noexcept {
	159	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(error_code& __ec) const noexcept {
156	160	return __get_write_time(&__ec);
157	161	}
158	162
159		_LIBCPP_HIDE_FROM_ABI file_status status() const { return __get_status(); }
	163	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_status status() const { return __get_status(); }
160	164
161		_LIBCPP_HIDE_FROM_ABI file_status status(error_code& __ec) const noexcept { return __get_status(&__ec); }
	165	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_status status(error_code& __ec) const noexcept {
	166	return __get_status(&__ec);
	167	}
162	168
163		_LIBCPP_HIDE_FROM_ABI file_status symlink_status() const { return __get_symlink_status(); }
	169	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_status symlink_status() const { return __get_symlink_status(); }
164	170
165		_LIBCPP_HIDE_FROM_ABI file_status symlink_status(error_code& __ec) const noexcept {
	171	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_status symlink_status(error_code& __ec) const noexcept {
166	172	return __get_symlink_status(&__ec);
167	173	}
168	174
...	...	@@ -459,8 +465,6 @@ private:
459	465	directory_entry __elem_;
460	466	};
461	467
462		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
463
464	468	_LIBCPP_END_NAMESPACE_FILESYSTEM
465	469
466	470	#endif // _LIBCPP_STD_VER >= 17 && _LIBCPP_HAS_FILESYSTEM

lib/libcxx/include/__filesystem/directory_iterator.h+9-11

...	...	@@ -34,8 +34,6 @@ _LIBCPP_PUSH_MACROS
34	34
35	35	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
36	36
37		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
38
39	37	class _LIBCPP_HIDDEN __dir_stream;
40	38	class directory_iterator {
41	39	public:
...	...	@@ -73,7 +71,7 @@ public:
73	71
74	72	_LIBCPP_HIDE_FROM_ABI ~directory_iterator() = default;
75	73
76		_LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const {
	74	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const {
77	75	// Note: this check duplicates a check in `__dereference()`.
78	76	_LIBCPP_ASSERT_NON_NULL(__imp_, "The end iterator cannot be dereferenced");
79	77	return __dereference();
...	...	@@ -123,23 +121,23 @@ operator!=(const directory_iterator& __lhs, const directory_iterator& __rhs) noe
123	121	}
124	122
125	123	// enable directory_iterator range-based for statements
126		inline _LIBCPP_HIDE_FROM_ABI directory_iterator begin(directory_iterator __iter) noexcept { return __iter; }
127
128		inline _LIBCPP_HIDE_FROM_ABI directory_iterator end(directory_iterator) noexcept { return directory_iterator(); }
	124	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI directory_iterator begin(directory_iterator __iter) noexcept {
	125	return __iter;
	126	}
129	127
130		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
	128	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI directory_iterator end(directory_iterator) noexcept {
	129	return directory_iterator();
	130	}
131	131
132	132	_LIBCPP_END_NAMESPACE_FILESYSTEM
133	133
134	134	# if _LIBCPP_STD_VER >= 20
135	135
136	136	template <>
137		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
138		std::ranges::enable_borrowed_range<std::filesystem::directory_iterator> = true;
	137	inline constexpr bool std::ranges::enable_borrowed_range<std::filesystem::directory_iterator> = true;
139	138
140	139	template <>
141		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
142		std::ranges::enable_view<std::filesystem::directory_iterator> = true;
	140	inline constexpr bool std::ranges::enable_view<std::filesystem::directory_iterator> = true;
143	141
144	142	# endif // _LIBCPP_STD_VER >= 20
145	143

lib/libcxx/include/__filesystem/directory_options.h+7-4

...	...	@@ -22,19 +22,22 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
22	22
23	23	enum class directory_options : unsigned char { none = 0, follow_directory_symlink = 1, skip_permission_denied = 2 };
24	24
25		_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator&(directory_options __lhs, directory_options __rhs) {
	25	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr directory_options
	26	operator&(directory_options __lhs, directory_options __rhs) {
26	27	return static_cast<directory_options>(static_cast<unsigned char>(__lhs) & static_cast<unsigned char>(__rhs));
27	28	}
28	29
29		_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator\|(directory_options __lhs, directory_options __rhs) {
	30	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr directory_options
	31	operator\|(directory_options __lhs, directory_options __rhs) {
30	32	return static_cast<directory_options>(static_cast<unsigned char>(__lhs) \| static_cast<unsigned char>(__rhs));
31	33	}
32	34
33		_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator^(directory_options __lhs, directory_options __rhs) {
	35	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr directory_options
	36	operator^(directory_options __lhs, directory_options __rhs) {
34	37	return static_cast<directory_options>(static_cast<unsigned char>(__lhs) ^ static_cast<unsigned char>(__rhs));
35	38	}
36	39
37		_LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator~(directory_options __lhs) {
	40	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr directory_options operator~(directory_options __lhs) {
38	41	return static_cast<directory_options>(~static_cast<unsigned char>(__lhs));
39	42	}
40	43

lib/libcxx/include/__filesystem/file_status.h+3-3

...	...	@@ -22,7 +22,7 @@
22	22
23	23	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
24	24
25		class _LIBCPP_EXPORTED_FROM_ABI file_status {
	25	class file_status {
26	26	public:
27	27	// constructors
28	28	_LIBCPP_HIDE_FROM_ABI file_status() noexcept : file_status(file_type::none) {}
...	...	@@ -38,9 +38,9 @@ public:
38	38	_LIBCPP_HIDE_FROM_ABI file_status& operator=(file_status&&) noexcept = default;
39	39
40	40	// observers
41		_LIBCPP_HIDE_FROM_ABI file_type type() const noexcept { return __ft_; }
	41	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI file_type type() const noexcept { return __ft_; }
42	42
43		_LIBCPP_HIDE_FROM_ABI perms permissions() const noexcept { return __prms_; }
	43	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI perms permissions() const noexcept { return __prms_; }
44	44
45	45	// modifiers
46	46	_LIBCPP_HIDE_FROM_ABI void type(file_type __ft) noexcept { __ft_ = __ft; }

lib/libcxx/include/__filesystem/filesystem_error.h+9-10

...	...	@@ -27,7 +27,7 @@
27	27
28	28	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
29	29
30		class _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_EXPORTED_FROM_ABI filesystem_error : public system_error {
	30	class _LIBCPP_EXPORTED_FROM_ABI filesystem_error : public system_error {
31	31	public:
32	32	_LIBCPP_HIDE_FROM_ABI filesystem_error(const string& __what, error_code __ec)
33	33	: system_error(__ec, __what), __storage_(make_shared<_Storage>(path(), path())) {
...	...	@@ -44,15 +44,16 @@ public:
44	44	__create_what(2);
45	45	}
46	46
47		_LIBCPP_HIDE_FROM_ABI const path& path1() const noexcept { return __storage_->__p1_; }
	47	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const path& path1() const noexcept { return __storage_->__p1_; }
48	48
49		_LIBCPP_HIDE_FROM_ABI const path& path2() const noexcept { return __storage_->__p2_; }
	49	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const path& path2() const noexcept { return __storage_->__p2_; }
50	50
51		_LIBCPP_HIDE_FROM_ABI filesystem_error(const filesystem_error&) = default;
	51	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI filesystem_error(const filesystem_error&) = default;
52	52	~filesystem_error() override; // key function
53	53
54		_LIBCPP_HIDE_FROM_ABI_VIRTUAL
55		const char* what() const noexcept override { return __storage_->__what_.c_str(); }
	54	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI_VIRTUAL const char* what() const noexcept override {
	55	return __storage_->__what_.c_str();
	56	}
56	57
57	58	void __create_what(int __num_paths);
58	59
...	...	@@ -69,14 +70,12 @@ private:
69	70
70	71	# if _LIBCPP_HAS_EXCEPTIONS
71	72	template <class... _Args>
72		[[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY void
73		__throw_filesystem_error(_Args&&... __args) {
	73	[[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_filesystem_error(_Args&&... __args) {
74	74	throw filesystem_error(std::forward<_Args>(__args)...);
75	75	}
76	76	# else
77	77	template <class... _Args>
78		[[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY void
79		__throw_filesystem_error(_Args&&...) {
	78	[[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_filesystem_error(_Args&&...) {
80	79	_LIBCPP_VERBOSE_ABORT("filesystem_error was thrown in -fno-exceptions mode");
81	80	}
82	81	# endif

lib/libcxx/include/__filesystem/operations.h+97-66

...	...	@@ -31,8 +31,6 @@
31	31
32	32	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
33	33
34		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
35
36	34	_LIBCPP_EXPORTED_FROM_ABI path __absolute(const path&, error_code* __ec = nullptr);
37	35	_LIBCPP_EXPORTED_FROM_ABI path __canonical(const path&, error_code* __ec = nullptr);
38	36	_LIBCPP_EXPORTED_FROM_ABI bool
...	...	@@ -70,10 +68,14 @@ _LIBCPP_EXPORTED_FROM_ABI bool __fs_is_empty(const path& __p, error_code* __ec =
70	68	_LIBCPP_EXPORTED_FROM_ABI void __permissions(const path&, perms, perm_options, error_code* = nullptr);
71	69	_LIBCPP_EXPORTED_FROM_ABI space_info __space(const path&, error_code* __ec = nullptr);
72	70
73		inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p) { return __absolute(__p); }
74		inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p, error_code& __ec) { return __absolute(__p, &__ec); }
75		inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p) { return __canonical(__p); }
76		inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p, error_code& __ec) { return __canonical(__p, &__ec); }
	71	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p) { return __absolute(__p); }
	72	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path absolute(const path& __p, error_code& __ec) {
	73	return __absolute(__p, &__ec);
	74	}
	75	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p) { return __canonical(__p); }
	76	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path canonical(const path& __p, error_code& __ec) {
	77	return __canonical(__p, &__ec);
	78	}
77	79	inline _LIBCPP_HIDE_FROM_ABI bool copy_file(const path& __from, const path& __to) {
78	80	return __copy_file(__from, __to, copy_options::none);
79	81	}
...	...	@@ -137,85 +139,112 @@ inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const pat
137	139	inline _LIBCPP_HIDE_FROM_ABI void create_symlink(const path& __target, const path& __link, error_code& __ec) noexcept {
138	140	return __create_symlink(__target, __link, &__ec);
139	141	}
140		inline _LIBCPP_HIDE_FROM_ABI path current_path() { return __current_path(); }
141		inline _LIBCPP_HIDE_FROM_ABI path current_path(error_code& __ec) { return __current_path(&__ec); }
	142	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path current_path() { return __current_path(); }
	143	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path current_path(error_code& __ec) { return __current_path(&__ec); }
142	144	inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p) { __current_path(__p); }
143	145	inline _LIBCPP_HIDE_FROM_ABI void current_path(const path& __p, error_code& __ec) noexcept {
144	146	__current_path(__p, &__ec);
145	147	}
146		inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2) { return __equivalent(__p1, __p2); }
147		inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept {
	148	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool equivalent(const path& __p1, const path& __p2) {
	149	return __equivalent(__p1, __p2);
	150	}
	151	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool
	152	equivalent(const path& __p1, const path& __p2, error_code& __ec) noexcept {
148	153	return __equivalent(__p1, __p2, &__ec);
149	154	}
150		inline _LIBCPP_HIDE_FROM_ABI bool status_known(file_status __s) noexcept { return __s.type() != file_type::none; }
151		inline _LIBCPP_HIDE_FROM_ABI bool exists(file_status __s) noexcept {
	155	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool status_known(file_status __s) noexcept {
	156	return __s.type() != file_type::none;
	157	}
	158	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool exists(file_status __s) noexcept {
152	159	return status_known(__s) && __s.type() != file_type::not_found;
153	160	}
154		inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p) { return exists(__status(__p)); }
	161	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p) { return exists(__status(__p)); }
155	162
156		inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p, error_code& __ec) noexcept {
	163	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool exists(const path& __p, error_code& __ec) noexcept {
157	164	auto __s = __status(__p, &__ec);
158	165	if (status_known(__s))
159	166	__ec.clear();
160	167	return exists(__s);
161	168	}
162	169
163		inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p) { return __file_size(__p); }
164		inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p, error_code& __ec) noexcept {
	170	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p) { return __file_size(__p); }
	171	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI uintmax_t file_size(const path& __p, error_code& __ec) noexcept {
165	172	return __file_size(__p, &__ec);
166	173	}
167		inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p) { return __hard_link_count(__p); }
168		inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept {
	174	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p) { return __hard_link_count(__p); }
	175	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI uintmax_t hard_link_count(const path& __p, error_code& __ec) noexcept {
169	176	return __hard_link_count(__p, &__ec);
170	177	}
171		inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(file_status __s) noexcept { return __s.type() == file_type::block; }
172		inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p) { return is_block_file(__status(__p)); }
173		inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p, error_code& __ec) noexcept {
	178	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(file_status __s) noexcept {
	179	return __s.type() == file_type::block;
	180	}
	181	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p) { return is_block_file(__status(__p)); }
	182	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_block_file(const path& __p, error_code& __ec) noexcept {
174	183	return is_block_file(__status(__p, &__ec));
175	184	}
176		inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(file_status __s) noexcept {
	185	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(file_status __s) noexcept {
177	186	return __s.type() == file_type::character;
178	187	}
179		inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p) { return is_character_file(__status(__p)); }
180		inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p, error_code& __ec) noexcept {
	188	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p) {
	189	return is_character_file(__status(__p));
	190	}
	191	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_character_file(const path& __p, error_code& __ec) noexcept {
181	192	return is_character_file(__status(__p, &__ec));
182	193	}
183		inline _LIBCPP_HIDE_FROM_ABI bool is_directory(file_status __s) noexcept { return __s.type() == file_type::directory; }
184		inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p) { return is_directory(__status(__p)); }
185		inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p, error_code& __ec) noexcept {
	194	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_directory(file_status __s) noexcept {
	195	return __s.type() == file_type::directory;
	196	}
	197	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p) { return is_directory(__status(__p)); }
	198	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_directory(const path& __p, error_code& __ec) noexcept {
186	199	return is_directory(__status(__p, &__ec));
187	200	}
188		inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p) { return __fs_is_empty(__p); }
189		inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p, error_code& __ec) { return __fs_is_empty(__p, &__ec); }
190		inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(file_status __s) noexcept { return __s.type() == file_type::fifo; }
191		inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p) { return is_fifo(__status(__p)); }
192		inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p, error_code& __ec) noexcept {
	201	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p) { return __fs_is_empty(__p); }
	202	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_empty(const path& __p, error_code& __ec) {
	203	return __fs_is_empty(__p, &__ec);
	204	}
	205	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(file_status __s) noexcept {
	206	return __s.type() == file_type::fifo;
	207	}
	208	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p) { return is_fifo(__status(__p)); }
	209	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_fifo(const path& __p, error_code& __ec) noexcept {
193	210	return is_fifo(__status(__p, &__ec));
194	211	}
195		inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(file_status __s) noexcept { return __s.type() == file_type::regular; }
196		inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p) { return is_regular_file(__status(__p)); }
197		inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p, error_code& __ec) noexcept {
	212	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(file_status __s) noexcept {
	213	return __s.type() == file_type::regular;
	214	}
	215	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p) {
	216	return is_regular_file(__status(__p));
	217	}
	218	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_regular_file(const path& __p, error_code& __ec) noexcept {
198	219	return is_regular_file(__status(__p, &__ec));
199	220	}
200		inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(file_status __s) noexcept { return __s.type() == file_type::symlink; }
201		inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p) { return is_symlink(__symlink_status(__p)); }
202		inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p, error_code& __ec) noexcept {
	221	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(file_status __s) noexcept {
	222	return __s.type() == file_type::symlink;
	223	}
	224	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p) {
	225	return is_symlink(__symlink_status(__p));
	226	}
	227	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_symlink(const path& __p, error_code& __ec) noexcept {
203	228	return is_symlink(__symlink_status(__p, &__ec));
204	229	}
205		inline _LIBCPP_HIDE_FROM_ABI bool is_other(file_status __s) noexcept {
	230	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_other(file_status __s) noexcept {
206	231	return exists(__s) && !is_regular_file(__s) && !is_directory(__s) && !is_symlink(__s);
207	232	}
208		inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p) { return is_other(__status(__p)); }
209		inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p, error_code& __ec) noexcept {
	233	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p) { return is_other(__status(__p)); }
	234	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_other(const path& __p, error_code& __ec) noexcept {
210	235	return is_other(__status(__p, &__ec));
211	236	}
212		inline _LIBCPP_HIDE_FROM_ABI bool is_socket(file_status __s) noexcept { return __s.type() == file_type::socket; }
213		inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p) { return is_socket(__status(__p)); }
214		inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p, error_code& __ec) noexcept {
	237	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_socket(file_status __s) noexcept {
	238	return __s.type() == file_type::socket;
	239	}
	240	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p) { return is_socket(__status(__p)); }
	241	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool is_socket(const path& __p, error_code& __ec) noexcept {
215	242	return is_socket(__status(__p, &__ec));
216	243	}
217		inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p) { return __last_write_time(__p); }
218		inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p, error_code& __ec) noexcept {
	244	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p) {
	245	return __last_write_time(__p);
	246	}
	247	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_time_type last_write_time(const path& __p, error_code& __ec) noexcept {
219	248	return __last_write_time(__p, &__ec);
220	249	}
221	250	inline _LIBCPP_HIDE_FROM_ABI void last_write_time(const path& __p, file_time_type __t) { __last_write_time(__p, __t); }
...	...	@@ -233,7 +262,7 @@ inline _LIBCPP_HIDE_FROM_ABI void permissions(const path& __p, perms __prms, per
233	262	__permissions(__p, __prms, __opts, &__ec);
234	263	}
235	264
236		inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base, error_code& __ec) {
	265	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base, error_code& __ec) {
237	266	path __tmp = __weakly_canonical(__p, &__ec);
238	267	if (__ec)
239	268	return {};
...	...	@@ -243,16 +272,18 @@ inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base,
243	272	return __tmp.lexically_proximate(__tmp_base);
244	273	}
245	274
246		inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, error_code& __ec) {
	275	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, error_code& __ec) {
247	276	return proximate(__p, current_path(), __ec);
248	277	}
249		inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base = current_path()) {
	278	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path proximate(const path& __p, const path& __base = current_path()) {
250	279	return __weakly_canonical(__p).lexically_proximate(__weakly_canonical(__base));
251	280	}
252		inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p) { return __read_symlink(__p); }
253		inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p, error_code& __ec) { return __read_symlink(__p, &__ec); }
	281	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p) { return __read_symlink(__p); }
	282	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path read_symlink(const path& __p, error_code& __ec) {
	283	return __read_symlink(__p, &__ec);
	284	}
254	285
255		inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base, error_code& __ec) {
	286	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base, error_code& __ec) {
256	287	path __tmp = __weakly_canonical(__p, &__ec);
257	288	if (__ec)
258	289	return path();
...	...	@@ -262,10 +293,10 @@ inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base,
262	293	return __tmp.lexically_relative(__tmpbase);
263	294	}
264	295
265		inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, error_code& __ec) {
	296	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, error_code& __ec) {
266	297	return relative(__p, current_path(), __ec);
267	298	}
268		inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base = current_path()) {
	299	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path relative(const path& __p, const path& __base = current_path()) {
269	300	return __weakly_canonical(__p).lexically_relative(__weakly_canonical(__base));
270	301	}
271	302	inline _LIBCPP_HIDE_FROM_ABI uintmax_t remove_all(const path& __p) { return __remove_all(__p); }
...	...	@@ -282,27 +313,27 @@ inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns) {
282	313	inline _LIBCPP_HIDE_FROM_ABI void resize_file(const path& __p, uintmax_t __ns, error_code& __ec) noexcept {
283	314	return __resize_file(__p, __ns, &__ec);
284	315	}
285		inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p) { return __space(__p); }
286		inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p, error_code& __ec) noexcept {
	316	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p) { return __space(__p); }
	317	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI space_info space(const path& __p, error_code& __ec) noexcept {
287	318	return __space(__p, &__ec);
288	319	}
289		inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p) { return __status(__p); }
290		inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p, error_code& __ec) noexcept {
	320	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p) { return __status(__p); }
	321	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_status status(const path& __p, error_code& __ec) noexcept {
291	322	return __status(__p, &__ec);
292	323	}
293		inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p) { return __symlink_status(__p); }
294		inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p, error_code& __ec) noexcept {
	324	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p) { return __symlink_status(__p); }
	325	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI file_status symlink_status(const path& __p, error_code& __ec) noexcept {
295	326	return __symlink_status(__p, &__ec);
296	327	}
297		inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path() { return __temp_directory_path(); }
298		inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path(error_code& __ec) { return __temp_directory_path(&__ec); }
299		inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p) { return __weakly_canonical(__p); }
300		inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p, error_code& __ec) {
	328	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path() { return __temp_directory_path(); }
	329	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path temp_directory_path(error_code& __ec) {
	330	return __temp_directory_path(&__ec);
	331	}
	332	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p) { return __weakly_canonical(__p); }
	333	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI path weakly_canonical(path const& __p, error_code& __ec) {
301	334	return __weakly_canonical(__p, &__ec);
302	335	}
303	336
304		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
305
306	337	_LIBCPP_END_NAMESPACE_FILESYSTEM
307	338
308	339	#endif // _LIBCPP_STD_VER >= 17 && _LIBCPP_HAS_FILESYSTEM

lib/libcxx/include/__filesystem/path.h+68-64

...	...	@@ -42,8 +42,6 @@ _LIBCPP_PUSH_MACROS
42	42
43	43	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
44	44
45		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
46
47	45	template <class _Tp>
48	46	struct __can_convert_char {
49	47	static const bool value = false;
...	...	@@ -326,6 +324,7 @@ struct _PathCVT<char> {
326	324	}
327	325	};
328	326
	327	# if _LIBCPP_HAS_LOCALIZATION
329	328	template <class _ECharT>
330	329	struct _PathExport {
331	330	typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
...	...	@@ -366,7 +365,7 @@ struct _PathExport<char16_t> {
366	365	}
367	366	};
368	367
369		# if _LIBCPP_HAS_CHAR8_T
	368	# if _LIBCPP_HAS_CHAR8_T
370	369	template <>
371	370	struct _PathExport<char8_t> {
372	371	typedef __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__> _Narrower;
...	...	@@ -376,8 +375,9 @@ struct _PathExport<char8_t> {
376	375	_Narrower()(back_inserter(__dest), __src.data(), __src.data() + __src.size());
377	376	}
378	377	};
379		# endif // _LIBCPP_HAS_CHAR8_T
380		# endif /* _LIBCPP_WIN32API */
	378	# endif // _LIBCPP_HAS_CHAR8_T
	379	# endif // _LIBCPP_HAS_LOCALIZATION
	380	# endif // _LIBCPP_WIN32API
381	381
382	382	class _LIBCPP_EXPORTED_FROM_ABI path {
383	383	template <class _SourceOrIter, class _Tp = path&>
...	...	@@ -667,16 +667,16 @@ public:
667	667	_LIBCPP_HIDE_FROM_ABI void __reserve(size_t __s) { __pn_.reserve(__s); }
668	668
669	669	// native format observers
670		_LIBCPP_HIDE_FROM_ABI const string_type& native() const noexcept { return __pn_; }
	670	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const string_type& native() const noexcept { return __pn_; }
671	671
672		_LIBCPP_HIDE_FROM_ABI const value_type* c_str() const noexcept { return __pn_.c_str(); }
	672	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const value_type* c_str() const noexcept { return __pn_.c_str(); }
673	673
674	674	_LIBCPP_HIDE_FROM_ABI operator string_type() const { return __pn_; }
675	675
676	676	# if defined(_LIBCPP_WIN32API)
677		_LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return __pn_; }
	677	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return __pn_; }
678	678
679		_LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const {
	679	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const {
680	680	std::wstring __s;
681	681	__s.resize(__pn_.size());
682	682	std::replace_copy(__pn_.begin(), __pn_.end(), __s.begin(), '\\', '/');
...	...	@@ -685,6 +685,7 @@ public:
685	685
686	686	# if _LIBCPP_HAS_LOCALIZATION
687	687	template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
	688	[[nodiscard]]
688	689	_LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const {
689	690	using _Str = basic_string<_ECharT, _Traits, _Allocator>;
690	691	_Str __s(__a);
...	...	@@ -693,8 +694,8 @@ public:
693	694	return __s;
694	695	}
695	696
696		_LIBCPP_HIDE_FROM_ABI std::string string() const { return string<char>(); }
697		_LIBCPP_HIDE_FROM_ABI __u8_string u8string() const {
	697	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string string() const { return string<char>(); }
	698	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __u8_string u8string() const {
698	699	using _CVT = __narrow_to_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
699	700	__u8_string __s;
700	701	__s.reserve(__pn_.size());
...	...	@@ -702,12 +703,12 @@ public:
702	703	return __s;
703	704	}
704	705
705		_LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
706		_LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
	706	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
	707	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
707	708
708	709	// generic format observers
709	710	template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
710		_LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
	711	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
711	712	generic_string(const _Allocator& __a = _Allocator()) const {
712	713	using _Str = basic_string<_ECharT, _Traits, _Allocator>;
713	714	_Str __s = string<_ECharT, _Traits, _Allocator>(__a);
...	...	@@ -718,10 +719,10 @@ public:
718	719	return __s;
719	720	}
720	721
721		_LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return generic_string<char>(); }
722		_LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return generic_string<char16_t>(); }
723		_LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return generic_string<char32_t>(); }
724		_LIBCPP_HIDE_FROM_ABI __u8_string generic_u8string() const {
	722	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return generic_string<char>(); }
	723	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return generic_string<char16_t>(); }
	724	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return generic_string<char32_t>(); }
	725	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __u8_string generic_u8string() const {
725	726	__u8_string __s = u8string();
726	727	std::replace(__s.begin(), __s.end(), '\\', '/');
727	728	return __s;
...	...	@@ -729,15 +730,18 @@ public:
729	730	# endif // _LIBCPP_HAS_LOCALIZATION
730	731	# else /* _LIBCPP_WIN32API */
731	732
732		_LIBCPP_HIDE_FROM_ABI std::string string() const { return __pn_; }
	733	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string string() const { return __pn_; }
733	734	# if _LIBCPP_HAS_CHAR8_T
734		_LIBCPP_HIDE_FROM_ABI std::u8string u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
	735	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u8string u8string() const {
	736	return std::u8string(__pn_.begin(), __pn_.end());
	737	}
735	738	# else
736		_LIBCPP_HIDE_FROM_ABI std::string u8string() const { return __pn_; }
	739	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string u8string() const { return __pn_; }
737	740	# endif
738	741
739	742	# if _LIBCPP_HAS_LOCALIZATION
740	743	template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
	744	[[nodiscard]]
741	745	_LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator> string(const _Allocator& __a = _Allocator()) const {
742	746	using _CVT = __widen_from_utf8<sizeof(_ECharT) * __CHAR_BIT__>;
743	747	using _Str = basic_string<_ECharT, _Traits, _Allocator>;
...	...	@@ -748,32 +752,34 @@ public:
748	752	}
749	753
750	754	# if _LIBCPP_HAS_WIDE_CHARACTERS
751		_LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return string<wchar_t>(); }
	755	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::wstring wstring() const { return string<wchar_t>(); }
752	756	# endif
753		_LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
754		_LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
	757	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u16string u16string() const { return string<char16_t>(); }
	758	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u32string u32string() const { return string<char32_t>(); }
755	759	# endif // _LIBCPP_HAS_LOCALIZATION
756	760
757	761	// generic format observers
758		_LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return __pn_; }
	762	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string generic_string() const { return __pn_; }
759	763	# if _LIBCPP_HAS_CHAR8_T
760		_LIBCPP_HIDE_FROM_ABI std::u8string generic_u8string() const { return std::u8string(__pn_.begin(), __pn_.end()); }
	764	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u8string generic_u8string() const {
	765	return std::u8string(__pn_.begin(), __pn_.end());
	766	}
761	767	# else
762		_LIBCPP_HIDE_FROM_ABI std::string generic_u8string() const { return __pn_; }
	768	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::string generic_u8string() const { return __pn_; }
763	769	# endif
764	770
765	771	# if _LIBCPP_HAS_LOCALIZATION
766	772	template <class _ECharT, class _Traits = char_traits<_ECharT>, class _Allocator = allocator<_ECharT> >
767		_LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
	773	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<_ECharT, _Traits, _Allocator>
768	774	generic_string(const _Allocator& __a = _Allocator()) const {
769	775	return string<_ECharT, _Traits, _Allocator>(__a);
770	776	}
771	777
772	778	# if _LIBCPP_HAS_WIDE_CHARACTERS
773		_LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const { return string<wchar_t>(); }
	779	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::wstring generic_wstring() const { return string<wchar_t>(); }
774	780	# endif
775		_LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return string<char16_t>(); }
776		_LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return string<char32_t>(); }
	781	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u16string generic_u16string() const { return string<char16_t>(); }
	782	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::u32string generic_u32string() const { return string<char32_t>(); }
777	783	# endif // _LIBCPP_HAS_LOCALIZATION
778	784	# endif /* !_LIBCPP_WIN32API */
779	785
...	...	@@ -790,40 +796,40 @@ private:
790	796
791	797	public:
792	798	// compare
793		_LIBCPP_HIDE_FROM_ABI int compare(const path& __p) const noexcept { return __compare(__p.__pn_); }
794		_LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return __compare(__s); }
795		_LIBCPP_HIDE_FROM_ABI int compare(__string_view __s) const { return __compare(__s); }
796		_LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return __compare(__s); }
	799	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI int compare(const path& __p) const noexcept { return __compare(__p.__pn_); }
	800	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return __compare(__s); }
	801	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI int compare(__string_view __s) const { return __compare(__s); }
	802	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return __compare(__s); }
797	803
798	804	// decomposition
799		_LIBCPP_HIDE_FROM_ABI path root_name() const { return string_type(__root_name()); }
800		_LIBCPP_HIDE_FROM_ABI path root_directory() const { return string_type(__root_directory()); }
801		_LIBCPP_HIDE_FROM_ABI path root_path() const {
	805	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path root_name() const { return string_type(__root_name()); }
	806	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path root_directory() const { return string_type(__root_directory()); }
	807	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path root_path() const {
802	808	# if defined(_LIBCPP_WIN32API)
803	809	return string_type(__root_path_raw());
804	810	# else
805	811	return root_name().append(string_type(__root_directory()));
806	812	# endif
807	813	}
808		_LIBCPP_HIDE_FROM_ABI path relative_path() const { return string_type(__relative_path()); }
809		_LIBCPP_HIDE_FROM_ABI path parent_path() const { return string_type(__parent_path()); }
810		_LIBCPP_HIDE_FROM_ABI path filename() const { return string_type(__filename()); }
811		_LIBCPP_HIDE_FROM_ABI path stem() const { return string_type(__stem()); }
812		_LIBCPP_HIDE_FROM_ABI path extension() const { return string_type(__extension()); }
	814	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path relative_path() const { return string_type(__relative_path()); }
	815	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path parent_path() const { return string_type(__parent_path()); }
	816	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path filename() const { return string_type(__filename()); }
	817	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path stem() const { return string_type(__stem()); }
	818	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path extension() const { return string_type(__extension()); }
813	819
814	820	// query
815	821	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __pn_.empty(); }
816	822
817		_LIBCPP_HIDE_FROM_ABI bool has_root_name() const { return !__root_name().empty(); }
818		_LIBCPP_HIDE_FROM_ABI bool has_root_directory() const { return !__root_directory().empty(); }
819		_LIBCPP_HIDE_FROM_ABI bool has_root_path() const { return !__root_path_raw().empty(); }
820		_LIBCPP_HIDE_FROM_ABI bool has_relative_path() const { return !__relative_path().empty(); }
821		_LIBCPP_HIDE_FROM_ABI bool has_parent_path() const { return !__parent_path().empty(); }
822		_LIBCPP_HIDE_FROM_ABI bool has_filename() const { return !__filename().empty(); }
823		_LIBCPP_HIDE_FROM_ABI bool has_stem() const { return !__stem().empty(); }
824		_LIBCPP_HIDE_FROM_ABI bool has_extension() const { return !__extension().empty(); }
	823	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_root_name() const { return !__root_name().empty(); }
	824	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_root_directory() const { return !__root_directory().empty(); }
	825	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_root_path() const { return !__root_path_raw().empty(); }
	826	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_relative_path() const { return !__relative_path().empty(); }
	827	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_parent_path() const { return !__parent_path().empty(); }
	828	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_filename() const { return !__filename().empty(); }
	829	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_stem() const { return !__stem().empty(); }
	830	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_extension() const { return !__extension().empty(); }
825	831
826		_LIBCPP_HIDE_FROM_ABI bool is_absolute() const {
	832	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_absolute() const {
827	833	# if defined(_LIBCPP_WIN32API)
828	834	__string_view __root_name_str = __root_name();
829	835	__string_view __root_dir = __root_directory();
...	...	@@ -847,13 +853,13 @@ public:
847	853	return has_root_directory();
848	854	# endif
849	855	}
850		_LIBCPP_HIDE_FROM_ABI bool is_relative() const { return !is_absolute(); }
	856	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_relative() const { return !is_absolute(); }
851	857
852	858	// relative paths
853		path lexically_normal() const;
854		path lexically_relative(const path& __base) const;
	859	[[nodiscard]] path lexically_normal() const;
	860	[[nodiscard]] path lexically_relative(const path& __base) const;
855	861
856		_LIBCPP_HIDE_FROM_ABI path lexically_proximate(const path& __base) const {
	862	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI path lexically_proximate(const path& __base) const {
857	863	path __result = this->lexically_relative(__base);
858	864	if (__result.native().empty())
859	865	return *this;
...	...	@@ -861,11 +867,11 @@ public:
861	867	}
862	868
863	869	// iterators
864		class _LIBCPP_EXPORTED_FROM_ABI iterator;
	870	class iterator;
865	871	typedef iterator const_iterator;
866	872
867		iterator begin() const;
868		iterator end() const;
	873	[[nodiscard]] iterator begin() const;
	874	[[nodiscard]] iterator end() const;
869	875
870	876	# if _LIBCPP_HAS_LOCALIZATION
871	877	template <
...	...	@@ -908,17 +914,15 @@ private:
908	914
909	915	inline _LIBCPP_HIDE_FROM_ABI void swap(path& __lhs, path& __rhs) noexcept { __lhs.swap(__rhs); }
910	916
911		_LIBCPP_EXPORTED_FROM_ABI size_t hash_value(const path& __p) noexcept;
912
913		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
	917	[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI size_t hash_value(const path& __p) noexcept;
914	918
915	919	_LIBCPP_END_NAMESPACE_FILESYSTEM
916	920
917	921	_LIBCPP_BEGIN_NAMESPACE_STD
918	922
919	923	template <>
920		struct _LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY hash<filesystem::path> : __unary_function<filesystem::path, size_t> {
921		_LIBCPP_HIDE_FROM_ABI size_t operator()(filesystem::path const& __p) const noexcept {
	924	struct hash<filesystem::path> : __unary_function<filesystem::path, size_t> {
	925	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t operator()(filesystem::path const& __p) const noexcept {
922	926	return filesystem::hash_value(__p);
923	927	}
924	928	};

lib/libcxx/include/__filesystem/path_iterator.h+1-3

...	...	@@ -52,7 +52,7 @@ public:
52	52
53	53	_LIBCPP_HIDE_FROM_ABI iterator& operator=(const iterator&) = default;
54	54
55		_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __stashed_elem_; }
	55	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __stashed_elem_; }
56	56
57	57	_LIBCPP_HIDE_FROM_ABI pointer operator->() const { return &__stashed_elem_; }
58	58
...	...	@@ -95,12 +95,10 @@ private:
95	95	_ParserState __state_;
96	96	};
97	97
98		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
99	98	inline _LIBCPP_HIDE_FROM_ABI bool operator==(const path::iterator& __lhs, const path::iterator& __rhs) {
100	99	return __lhs.__path_ptr_ == __rhs.__path_ptr_ && __lhs.__entry_.data() == __rhs.__entry_.data();
101	100	}
102	101
103		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
104	102	inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const path::iterator& __lhs, const path::iterator& __rhs) {
105	103	return !(__lhs == __rhs);
106	104	}

lib/libcxx/include/__filesystem/perm_options.h+4-4

...	...	@@ -22,19 +22,19 @@ _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
22	22
23	23	enum class perm_options : unsigned char { replace = 1, add = 2, remove = 4, nofollow = 8 };
24	24
25		_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator&(perm_options __lhs, perm_options __rhs) {
	25	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator&(perm_options __lhs, perm_options __rhs) {
26	26	return static_cast<perm_options>(static_cast<unsigned>(__lhs) & static_cast<unsigned>(__rhs));
27	27	}
28	28
29		_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator\|(perm_options __lhs, perm_options __rhs) {
	29	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator\|(perm_options __lhs, perm_options __rhs) {
30	30	return static_cast<perm_options>(static_cast<unsigned>(__lhs) \| static_cast<unsigned>(__rhs));
31	31	}
32	32
33		_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator^(perm_options __lhs, perm_options __rhs) {
	33	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator^(perm_options __lhs, perm_options __rhs) {
34	34	return static_cast<perm_options>(static_cast<unsigned>(__lhs) ^ static_cast<unsigned>(__rhs));
35	35	}
36	36
37		_LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator~(perm_options __lhs) {
	37	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perm_options operator~(perm_options __lhs) {
38	38	return static_cast<perm_options>(~static_cast<unsigned>(__lhs));
39	39	}
40	40

lib/libcxx/include/__filesystem/perms.h+4-4

...	...	@@ -51,19 +51,19 @@ enum class perms : unsigned {
51	51	unknown = 0xFFFF,
52	52	};
53	53
54		_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator&(perms __lhs, perms __rhs) {
	54	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perms operator&(perms __lhs, perms __rhs) {
55	55	return static_cast<perms>(static_cast<unsigned>(__lhs) & static_cast<unsigned>(__rhs));
56	56	}
57	57
58		_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator\|(perms __lhs, perms __rhs) {
	58	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perms operator\|(perms __lhs, perms __rhs) {
59	59	return static_cast<perms>(static_cast<unsigned>(__lhs) \| static_cast<unsigned>(__rhs));
60	60	}
61	61
62		_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator^(perms __lhs, perms __rhs) {
	62	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perms operator^(perms __lhs, perms __rhs) {
63	63	return static_cast<perms>(static_cast<unsigned>(__lhs) ^ static_cast<unsigned>(__rhs));
64	64	}
65	65
66		_LIBCPP_HIDE_FROM_ABI inline constexpr perms operator~(perms __lhs) {
	66	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr perms operator~(perms __lhs) {
67	67	return static_cast<perms>(~static_cast<unsigned>(__lhs));
68	68	}
69	69

lib/libcxx/include/__filesystem/recursive_directory_iterator.h+9-14

...	...	@@ -33,8 +33,6 @@ _LIBCPP_PUSH_MACROS
33	33
34	34	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
35	35
36		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
37
38	36	class recursive_directory_iterator {
39	37	public:
40	38	using value_type = directory_entry;
...	...	@@ -73,7 +71,7 @@ public:
73	71
74	72	_LIBCPP_HIDE_FROM_ABI ~recursive_directory_iterator() = default;
75	73
76		_LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { return __dereference(); }
	74	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const directory_entry& operator*() const { return __dereference(); }
77	75
78	76	_LIBCPP_HIDE_FROM_ABI const directory_entry* operator->() const { return &__dereference(); }
79	77
...	...	@@ -87,14 +85,14 @@ public:
87	85
88	86	_LIBCPP_HIDE_FROM_ABI recursive_directory_iterator& increment(error_code& __ec) { return __increment(&__ec); }
89	87
90		_LIBCPP_EXPORTED_FROM_ABI directory_options options() const;
91		_LIBCPP_EXPORTED_FROM_ABI int depth() const;
	88	[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI directory_options options() const;
	89	[[nodiscard]] _LIBCPP_EXPORTED_FROM_ABI int depth() const;
92	90
93	91	_LIBCPP_HIDE_FROM_ABI void pop() { __pop(); }
94	92
95	93	_LIBCPP_HIDE_FROM_ABI void pop(error_code& __ec) { __pop(&__ec); }
96	94
97		_LIBCPP_HIDE_FROM_ABI bool recursion_pending() const { return __rec_; }
	95	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool recursion_pending() const { return __rec_; }
98	96
99	97	_LIBCPP_HIDE_FROM_ABI void disable_recursion_pending() { __rec_ = false; }
100	98
...	...	@@ -132,27 +130,24 @@ operator!=(const recursive_directory_iterator& __lhs, const recursive_directory_
132	130	return !(__lhs == __rhs);
133	131	}
134	132	// enable recursive_directory_iterator range-based for statements
135		inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator begin(recursive_directory_iterator __iter) noexcept {
	133	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator
	134	begin(recursive_directory_iterator __iter) noexcept {
136	135	return __iter;
137	136	}
138	137
139		inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator end(recursive_directory_iterator) noexcept {
	138	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI recursive_directory_iterator end(recursive_directory_iterator) noexcept {
140	139	return recursive_directory_iterator();
141	140	}
142	141
143		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
144
145	142	_LIBCPP_END_NAMESPACE_FILESYSTEM
146	143
147	144	# if _LIBCPP_STD_VER >= 20
148	145
149	146	template <>
150		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
151		std::ranges::enable_borrowed_range<std::filesystem::recursive_directory_iterator> = true;
	147	inline constexpr bool std::ranges::enable_borrowed_range<std::filesystem::recursive_directory_iterator> = true;
152	148
153	149	template <>
154		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY inline constexpr bool
155		std::ranges::enable_view<std::filesystem::recursive_directory_iterator> = true;
	150	inline constexpr bool std::ranges::enable_view<std::filesystem::recursive_directory_iterator> = true;
156	151
157	152	# endif // _LIBCPP_STD_VER >= 20
158	153

lib/libcxx/include/__filesystem/space_info.h+1-1

...	...	@@ -21,7 +21,7 @@
21	21
22	22	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
23	23
24		struct _LIBCPP_EXPORTED_FROM_ABI space_info {
	24	struct space_info {
25	25	uintmax_t capacity;
26	26	uintmax_t free;
27	27	uintmax_t available;

lib/libcxx/include/__filesystem/u8path.h+12-14

...	...	@@ -24,32 +24,32 @@
24	24
25	25	_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
26	26
27		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_PUSH
28
	27	# if !defined(_LIBCPP_WIN32API) \|\| _LIBCPP_HAS_LOCALIZATION
29	28	template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
30		_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _InputIt __l) {
	29	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _InputIt __l) {
31	30	static_assert(
32		# if _LIBCPP_HAS_CHAR8_T
	31	# if _LIBCPP_HAS_CHAR8_T
33	32	is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value \|\|
34		# endif
	33	# endif
35	34	is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
36	35	"u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
37	36	" or 'char8_t'");
38		# if defined(_LIBCPP_WIN32API)
	37	# if defined(_LIBCPP_WIN32API)
39	38	string __tmp(__f, __l);
40	39	using _CVT = __widen_from_utf8<sizeof(wchar_t) * __CHAR_BIT__>;
41	40	std::wstring __w;
42	41	__w.reserve(__tmp.size());
43	42	_CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
44	43	return path(__w);
45		# else
	44	# else
46	45	return path(__f, __l);
47		# endif /* !_LIBCPP_WIN32API */
	46	# endif // defined(_LIBCPP_WIN32API)
48	47	}
	48	# endif // !defined(_LIBCPP_WIN32API) \|\| _LIBCPP_HAS_LOCALIZATION
49	49
50		# if defined(_LIBCPP_WIN32API)
	50	# if defined(_LIBCPP_WIN32API) && _LIBCPP_HAS_LOCALIZATION
51	51	template <class _InputIt, __enable_if_t<__is_pathable<_InputIt>::value, int> = 0>
52		_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _NullSentinel) {
	52	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f, _NullSentinel) {
53	53	static_assert(
54	54	# if _LIBCPP_HAS_CHAR8_T
55	55	is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value \|\|
...	...	@@ -67,10 +67,10 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(_InputIt __f,
67	67	_CVT()(back_inserter(__w), __tmp.data(), __tmp.data() + __tmp.size());
68	68	return path(__w);
69	69	}
70		# endif /* _LIBCPP_WIN32API */
	70	# endif // defined(_LIBCPP_WIN32API) && _LIBCPP_HAS_LOCALIZATION
71	71
72	72	template <class _Source, __enable_if_t<__is_pathable<_Source>::value, int> = 0>
73		_LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source& __s) {
	73	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source& __s) {
74	74	static_assert(
75	75	# if _LIBCPP_HAS_CHAR8_T
76	76	is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value \|\|
...	...	@@ -86,8 +86,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_WITH_CHAR8_T path u8path(const _Source&
86	86	# endif
87	87	}
88	88
89		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY_POP
90
91	89	_LIBCPP_END_NAMESPACE_FILESYSTEM
92	90
93	91	#endif // _LIBCPP_STD_VER >= 17

lib/libcxx/include/__flat_map/flat_map.h+88-71

...	...	@@ -29,7 +29,6 @@
29	29	#include <__flat_map/key_value_iterator.h>
30	30	#include <__flat_map/sorted_unique.h>
31	31	#include <__flat_map/utils.h>
32		#include <__functional/invoke.h>
33	32	#include <__functional/is_transparent.h>
34	33	#include <__functional/operations.h>
35	34	#include <__fwd/memory.h>
...	...	@@ -48,7 +47,7 @@
48	47	#include <__ranges/container_compatible_range.h>
49	48	#include <__ranges/drop_view.h>
50	49	#include <__ranges/from_range.h>
51		#include <__ranges/ref_view.h>
	50	#include <__ranges/range_adaptor.h>
52	51	#include <__ranges/size.h>
53	52	#include <__ranges/subrange.h>
54	53	#include <__ranges/zip_view.h>
...	...	@@ -410,41 +409,45 @@ public:
410	409	}
411	410
412	411	// iterators
413		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
	412	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
414	413	return iterator(__containers_.keys.begin(), __containers_.values.begin());
415	414	}
416	415
417		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
	416	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
418	417	return const_iterator(__containers_.keys.begin(), __containers_.values.begin());
419	418	}
420	419
421		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
	420	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
422	421	return iterator(__containers_.keys.end(), __containers_.values.end());
423	422	}
424	423
425		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
	424	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
426	425	return const_iterator(__containers_.keys.end(), __containers_.values.end());
427	426	}
428	427
429		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
	428	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
430	429	return reverse_iterator(end());
431	430	}
432		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
	431	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
433	432	return const_reverse_iterator(end());
434	433	}
435		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
	434	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
436	435	return reverse_iterator(begin());
437	436	}
438		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
	437	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
439	438	return const_reverse_iterator(begin());
440	439	}
441	440
442		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept { return begin(); }
443		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept { return end(); }
444		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
	441	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept {
	442	return begin();
	443	}
	444	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept {
	445	return end();
	446	}
	447	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
445	448	return const_reverse_iterator(end());
446	449	}
447		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
	450	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
448	451	return const_reverse_iterator(begin());
449	452	}
450	453
...	...	@@ -453,11 +456,11 @@ public:
453	456	return __containers_.keys.empty();
454	457	}
455	458
456		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept {
	459	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept {
457	460	return __containers_.keys.size();
458	461	}
459	462
460		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept {
	463	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept {
461	464	return std::min<size_type>(__containers_.keys.max_size(), __containers_.values.max_size());
462	465	}
463	466
...	...	@@ -481,7 +484,7 @@ public:
481	484	return try_emplace(std::forward<_Kp>(__x)).first->second;
482	485	}
483	486
484		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 mapped_type& at(const key_type& __x) {
	487	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 mapped_type& at(const key_type& __x) {
485	488	auto __it = find(__x);
486	489	if (__it == end()) {
487	490	std::__throw_out_of_range("flat_map::at(const key_type&): Key does not exist");
...	...	@@ -489,7 +492,7 @@ public:
489	492	return __it->second;
490	493	}
491	494
492		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_type& at(const key_type& __x) const {
	495	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_type& at(const key_type& __x) const {
493	496	auto __it = find(__x);
494	497	if (__it == end()) {
495	498	std::__throw_out_of_range("flat_map::at(const key_type&) const: Key does not exist");
...	...	@@ -499,7 +502,7 @@ public:
499	502
500	503	template <class _Kp>
501	504	requires __is_compare_transparent
502		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 mapped_type& at(const _Kp& __x) {
	505	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 mapped_type& at(const _Kp& __x) {
503	506	auto __it = find(__x);
504	507	if (__it == end()) {
505	508	std::__throw_out_of_range("flat_map::at(const K&): Key does not exist");
...	...	@@ -509,7 +512,7 @@ public:
509	512
510	513	template <class _Kp>
511	514	requires __is_compare_transparent
512		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_type& at(const _Kp& __x) const {
	515	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_type& at(const _Kp& __x) const {
513	516	auto __it = find(__x);
514	517	if (__it == end()) {
515	518	std::__throw_out_of_range("flat_map::at(const K&) const: Key does not exist");
...	...	@@ -589,6 +592,15 @@ public:
589	592	__append_sort_merge_unique</WasSorted = / false>(ranges::begin(__range), ranges::end(__range));
590	593	}
591	594
	595	template <_ContainerCompatibleRange<value_type> _Range>
	596	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(sorted_unique_t, _Range&& __range) {
	597	if constexpr (ranges::sized_range<_Range>) {
	598	__reserve(ranges::size(__range));
	599	}
	600
	601	__append_sort_merge_unique</WasSorted = / true>(ranges::begin(__range), ranges::end(__range));
	602	}
	603
592	604	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
593	605	insert(__il.begin(), __il.end());
594	606	}
...	...	@@ -597,7 +609,7 @@ public:
597	609	insert(sorted_unique, __il.begin(), __il.end());
598	610	}
599	611
600		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 containers extract() && {
	612	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 containers extract() && {
601	613	auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
602	614	auto __ret = std::move(__containers_);
603	615	return __ret;
...	...	@@ -738,14 +750,17 @@ public:
738	750	return iterator(std::move(__key_it), std::move(__mapped_it));
739	751	}
740	752
741		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_map& __y) noexcept {
742		// warning: The spec has unconditional noexcept, which means that
743		// if any of the following functions throw an exception,
744		// std::terminate will be called.
745		// This is discussed in P2767, which hasn't been voted on yet.
	753	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	754	swap(flat_map& __y) noexcept(is_nothrow_swappable_v<key_container_type> &&
	755	is_nothrow_swappable_v<mapped_container_type> && is_nothrow_swappable_v<key_compare>) {
	756	auto __on_failure = std::__make_exception_guard([&]() noexcept {
	757	clear() /* noexcept */;
	758	__y.clear() /* noexcept */;
	759	});
746	760	ranges::swap(__compare_, __y.__compare_);
747	761	ranges::swap(__containers_.keys, __y.__containers_.keys);
748	762	ranges::swap(__containers_.values, __y.__containers_.values);
	763	__on_failure.__complete();
749	764	}
750	765
751	766	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept {
...	...	@@ -754,116 +769,121 @@ public:
754	769	}
755	770
756	771	// observers
757		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
758		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const {
	772	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
	773	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const {
759	774	return value_compare(__compare_);
760	775	}
761	776
762		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const key_container_type& keys() const noexcept {
	777	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const key_container_type& keys() const noexcept {
763	778	return __containers_.keys;
764	779	}
765		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_container_type& values() const noexcept {
	780	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_container_type&
	781	values() const noexcept {
766	782	return __containers_.values;
767	783	}
768	784
769	785	// map operations
770		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
	786	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
771	787	return __find_impl(*this, __x);
772	788	}
773	789
774		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
	790	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
775	791	return __find_impl(*this, __x);
776	792	}
777	793
778	794	template <class _Kp>
779	795	requires __is_compare_transparent
780		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
	796	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
781	797	return __find_impl(*this, __x);
782	798	}
783	799
784	800	template <class _Kp>
785	801	requires __is_compare_transparent
786		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
	802	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
787	803	return __find_impl(*this, __x);
788	804	}
789	805
790		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
	806	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
791	807	return contains(__x) ? 1 : 0;
792	808	}
793	809
794	810	template <class _Kp>
795	811	requires __is_compare_transparent
796		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
	812	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
797	813	return contains(__x) ? 1 : 0;
798	814	}
799	815
800		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
	816	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
801	817	return find(__x) != end();
802	818	}
803	819
804	820	template <class _Kp>
805	821	requires __is_compare_transparent
806		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
	822	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
807	823	return find(__x) != end();
808	824	}
809	825
810		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
	826	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
811	827	return __lower_bound<iterator>(*this, __x);
812	828	}
813	829
814		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const key_type& __x) const {
	830	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	831	lower_bound(const key_type& __x) const {
815	832	return __lower_bound<const_iterator>(*this, __x);
816	833	}
817	834
818	835	template <class _Kp>
819	836	requires __is_compare_transparent
820		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
	837	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
821	838	return __lower_bound<iterator>(*this, __x);
822	839	}
823	840
824	841	template <class _Kp>
825	842	requires __is_compare_transparent
826		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
	843	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
827	844	return __lower_bound<const_iterator>(*this, __x);
828	845	}
829	846
830		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
	847	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
831	848	return __upper_bound<iterator>(*this, __x);
832	849	}
833	850
834		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const key_type& __x) const {
	851	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	852	upper_bound(const key_type& __x) const {
835	853	return __upper_bound<const_iterator>(*this, __x);
836	854	}
837	855
838	856	template <class _Kp>
839	857	requires __is_compare_transparent
840		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
	858	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
841	859	return __upper_bound<iterator>(*this, __x);
842	860	}
843	861
844	862	template <class _Kp>
845	863	requires __is_compare_transparent
846		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
	864	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
847	865	return __upper_bound<const_iterator>(*this, __x);
848	866	}
849	867
850		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const key_type& __x) {
	868	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	869	equal_range(const key_type& __x) {
851	870	return __equal_range_impl(*this, __x);
852	871	}
853	872
854		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	873	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
855	874	equal_range(const key_type& __x) const {
856	875	return __equal_range_impl(*this, __x);
857	876	}
858	877
859	878	template <class _Kp>
860	879	requires __is_compare_transparent
861		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const _Kp& __x) {
	880	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	881	equal_range(const _Kp& __x) {
862	882	return __equal_range_impl(*this, __x);
863	883	}
864	884	template <class _Kp>
865	885	requires __is_compare_transparent
866		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	886	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
867	887	equal_range(const _Kp& __x) const {
868	888	return __equal_range_impl(*this, __x);
869	889	}
...	...	@@ -878,7 +898,8 @@ public:
878	898	__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
879	899	}
880	900
881		friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_map& __x, flat_map& __y) noexcept {
	901	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	902	swap(flat_map& __x, flat_map& __y) noexcept(noexcept(__x.swap(__y))) {
882	903	__x.swap(__y);
883	904	}
884	905
...	...	@@ -913,7 +934,7 @@ private:
913	934	__compare_(std::forward<_CompArg>(__comp)...) {}
914	935
915	936	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_sorted_and_unique(auto&& __key_container) const {
916		auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) { return !__compare_(__x, __y); };
	937	auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) -> bool { return !__compare_(__x, __y); };
917	938	return ranges::adjacent_find(__key_container, __greater_or_equal_to) == ranges::end(__key_container);
918	939	}
919	940
...	...	@@ -946,7 +967,7 @@ private:
946	967	auto __zv = ranges::views::zip(__containers_.keys, __containers_.values);
947	968	auto __append_start_offset = __containers_.keys.size() - __num_of_appended;
948	969	auto __end = __zv.end();
949		auto __compare_key = [this](const auto& __p1, const auto& __p2) {
	970	auto __compare_key = [this](const auto& __p1, const auto& __p2) -> bool {
950	971	return __compare_(std::get<0>(__p1), std::get<0>(__p2));
951	972	};
952	973	if constexpr (!_WasSorted) {
...	...	@@ -1125,8 +1146,7 @@ private:
1125	1146	};
1126	1147
1127	1148	template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
1128		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
1129		!__is_allocator<_MappedContainer>::value &&
	1149	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
1130	1150	is_invocable_v<const _Compare&,
1131	1151	const typename _KeyContainer::value_type&,
1132	1152	const typename _KeyContainer::value_type&>)
...	...	@@ -1139,7 +1159,7 @@ flat_map(_KeyContainer, _MappedContainer, _Compare = _Compare())
1139	1159
1140	1160	template <class _KeyContainer, class _MappedContainer, class _Allocator>
1141	1161	requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
1142		!__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
	1162	!__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer>)
1143	1163	flat_map(_KeyContainer, _MappedContainer, _Allocator)
1144	1164	-> flat_map<typename _KeyContainer::value_type,
1145	1165	typename _MappedContainer::value_type,
...	...	@@ -1148,9 +1168,8 @@ flat_map(_KeyContainer, _MappedContainer, _Allocator)
1148	1168	_MappedContainer>;
1149	1169
1150	1170	template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
1151		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
1152		!__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
1153		uses_allocator_v<_MappedContainer, _Allocator> &&
	1171	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
	1172	uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
1154	1173	is_invocable_v<const _Compare&,
1155	1174	const typename _KeyContainer::value_type&,
1156	1175	const typename _KeyContainer::value_type&>)
...	...	@@ -1162,8 +1181,7 @@ flat_map(_KeyContainer, _MappedContainer, _Compare, _Allocator)
1162	1181	_MappedContainer>;
1163	1182
1164	1183	template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
1165		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
1166		!__is_allocator<_MappedContainer>::value &&
	1184	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
1167	1185	is_invocable_v<const _Compare&,
1168	1186	const typename _KeyContainer::value_type&,
1169	1187	const typename _KeyContainer::value_type&>)
...	...	@@ -1176,7 +1194,7 @@ flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Compare = _Compare()
1176	1194
1177	1195	template <class _KeyContainer, class _MappedContainer, class _Allocator>
1178	1196	requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
1179		!__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
	1197	!__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer>)
1180	1198	flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Allocator)
1181	1199	-> flat_map<typename _KeyContainer::value_type,
1182	1200	typename _MappedContainer::value_type,
...	...	@@ -1185,9 +1203,8 @@ flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Allocator)
1185	1203	_MappedContainer>;
1186	1204
1187	1205	template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
1188		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
1189		!__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
1190		uses_allocator_v<_MappedContainer, _Allocator> &&
	1206	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
	1207	uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
1191	1208	is_invocable_v<const _Compare&,
1192	1209	const typename _KeyContainer::value_type&,
1193	1210	const typename _KeyContainer::value_type&>)
...	...	@@ -1199,19 +1216,19 @@ flat_map(sorted_unique_t, _KeyContainer, _MappedContainer, _Compare, _Allocator)
1199	1216	_MappedContainer>;
1200	1217
1201	1218	template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
1202		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	1219	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
1203	1220	flat_map(_InputIterator, _InputIterator, _Compare = _Compare())
1204	1221	-> flat_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
1205	1222
1206	1223	template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
1207		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	1224	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
1208	1225	flat_map(sorted_unique_t, _InputIterator, _InputIterator, _Compare = _Compare())
1209	1226	-> flat_map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
1210	1227
1211	1228	template <ranges::input_range _Range,
1212	1229	class _Compare = less<__range_key_type<_Range>>,
1213	1230	class _Allocator = allocator<byte>,
1214		class = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
	1231	class = __enable_if_t<!__is_allocator_v<_Compare> && __is_allocator_v<_Allocator>>>
1215	1232	flat_map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_map<
1216	1233	__range_key_type<_Range>,
1217	1234	__range_mapped_type<_Range>,
...	...	@@ -1219,7 +1236,7 @@ flat_map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator(
1219	1236	vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
1220	1237	vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
1221	1238
1222		template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
	1239	template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator_v<_Allocator>>>
1223	1240	flat_map(from_range_t, _Range&&, _Allocator) -> flat_map<
1224	1241	__range_key_type<_Range>,
1225	1242	__range_mapped_type<_Range>,
...	...	@@ -1228,11 +1245,11 @@ flat_map(from_range_t, _Range&&, _Allocator) -> flat_map<
1228	1245	vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
1229	1246
1230	1247	template <class _Key, class _Tp, class _Compare = less<_Key>>
1231		requires(!__is_allocator<_Compare>::value)
	1248	requires(!__is_allocator_v<_Compare>)
1232	1249	flat_map(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_map<_Key, _Tp, _Compare>;
1233	1250
1234	1251	template <class _Key, class _Tp, class _Compare = less<_Key>>
1235		requires(!__is_allocator<_Compare>::value)
	1252	requires(!__is_allocator_v<_Compare>)
1236	1253	flat_map(sorted_unique_t, initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_map<_Key, _Tp, _Compare>;
1237	1254
1238	1255	template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Allocator>

lib/libcxx/include/__flat_map/flat_multimap.h+251-172

...	...	@@ -22,7 +22,6 @@
22	22	#include <__algorithm/upper_bound.h>
23	23	#include <__assert>
24	24	#include <__compare/synth_three_way.h>
25		#include <__concepts/convertible_to.h>
26	25	#include <__concepts/swappable.h>
27	26	#include <__config>
28	27	#include <__cstddef/byte.h>
...	...	@@ -30,7 +29,6 @@
30	29	#include <__flat_map/key_value_iterator.h>
31	30	#include <__flat_map/sorted_equivalent.h>
32	31	#include <__flat_map/utils.h>
33		#include <__functional/invoke.h>
34	32	#include <__functional/is_transparent.h>
35	33	#include <__functional/operations.h>
36	34	#include <__fwd/vector.h>
...	...	@@ -47,7 +45,7 @@
47	45	#include <__ranges/container_compatible_range.h>
48	46	#include <__ranges/drop_view.h>
49	47	#include <__ranges/from_range.h>
50		#include <__ranges/ref_view.h>
	48	#include <__ranges/range_adaptor.h>
51	49	#include <__ranges/size.h>
52	50	#include <__ranges/subrange.h>
53	51	#include <__ranges/zip_view.h>
...	...	@@ -57,14 +55,12 @@
57	55	#include <__type_traits/is_allocator.h>
58	56	#include <__type_traits/is_nothrow_constructible.h>
59	57	#include <__type_traits/is_same.h>
60		#include <__type_traits/maybe_const.h>
61	58	#include <__utility/exception_guard.h>
62	59	#include <__utility/move.h>
63	60	#include <__utility/pair.h>
64	61	#include <__utility/scope_guard.h>
65	62	#include <__vector/vector.h>
66	63	#include <initializer_list>
67		#include <stdexcept>
68	64
69	65	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
70	66	# pragma GCC system_header
...	...	@@ -114,11 +110,12 @@ public:
114	110	class value_compare {
115	111	private:
116	112	_LIBCPP_NO_UNIQUE_ADDRESS key_compare __comp_;
117		_LIBCPP_HIDE_FROM_ABI value_compare(key_compare __c) : __comp_(__c) {}
	113	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare(key_compare __c) : __comp_(__c) {}
118	114	friend flat_multimap;
119	115
120	116	public:
121		_LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
	117	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
	118	operator()(const_reference __x, const_reference __y) const {
122	119	return __comp_(__x.first, __y.first);
123	120	}
124	121	};
...	...	@@ -137,17 +134,17 @@ private:
137	134
138	135	public:
139	136	// [flat.map.cons], construct/copy/destroy
140		_LIBCPP_HIDE_FROM_ABI flat_multimap() noexcept(
	137	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap() noexcept(
141	138	is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_MappedContainer> &&
142	139	is_nothrow_default_constructible_v<_Compare>)
143	140	: __containers_(), __compare_() {}
144	141
145		_LIBCPP_HIDE_FROM_ABI flat_multimap(const flat_multimap&) = default;
	142	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(const flat_multimap&) = default;
146	143
147	144	// The copy/move constructors are not specified in the spec, which means they should be defaulted.
148	145	// However, the move constructor can potentially leave a moved-from object in an inconsistent
149	146	// state if an exception is thrown.
150		_LIBCPP_HIDE_FROM_ABI flat_multimap(flat_multimap&& __other) noexcept(
	147	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(flat_multimap&& __other) noexcept(
151	148	is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_MappedContainer> &&
152	149	is_nothrow_move_constructible_v<_Compare>)
153	150	# if _LIBCPP_HAS_EXCEPTIONS
...	...	@@ -168,7 +165,8 @@ public:
168	165
169	166	template <class _Allocator>
170	167	requires __allocator_ctor_constraint<_Allocator>
171		_LIBCPP_HIDE_FROM_ABI flat_multimap(const flat_multimap& __other, const _Allocator& __alloc)
	168	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	169	flat_multimap(const flat_multimap& __other, const _Allocator& __alloc)
172	170	: flat_multimap(__ctor_uses_allocator_tag{},
173	171	__alloc,
174	172	__other.__containers_.keys,
...	...	@@ -177,7 +175,7 @@ public:
177	175
178	176	template <class _Allocator>
179	177	requires __allocator_ctor_constraint<_Allocator>
180		_LIBCPP_HIDE_FROM_ABI flat_multimap(flat_multimap&& __other, const _Allocator& __alloc)
	178	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(flat_multimap&& __other, const _Allocator& __alloc)
181	179	# if _LIBCPP_HAS_EXCEPTIONS
182	180	try
183	181	# endif // _LIBCPP_HAS_EXCEPTIONS
...	...	@@ -194,7 +192,7 @@ public:
194	192	# endif // _LIBCPP_HAS_EXCEPTIONS
195	193	}
196	194
197		_LIBCPP_HIDE_FROM_ABI flat_multimap(
	195	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
198	196	key_container_type __key_cont, mapped_container_type __mapped_cont, const key_compare& __comp = key_compare())
199	197	: __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
200	198	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
...	...	@@ -204,7 +202,7 @@ public:
204	202
205	203	template <class _Allocator>
206	204	requires __allocator_ctor_constraint<_Allocator>
207		_LIBCPP_HIDE_FROM_ABI flat_multimap(
	205	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
208	206	const key_container_type& __key_cont, const mapped_container_type& __mapped_cont, const _Allocator& __alloc)
209	207	: flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
210	208	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
...	...	@@ -214,22 +212,22 @@ public:
214	212
215	213	template <class _Allocator>
216	214	requires __allocator_ctor_constraint<_Allocator>
217		_LIBCPP_HIDE_FROM_ABI
218		flat_multimap(const key_container_type& __key_cont,
219		const mapped_container_type& __mapped_cont,
220		const key_compare& __comp,
221		const _Allocator& __alloc)
	215	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	216	const key_container_type& __key_cont,
	217	const mapped_container_type& __mapped_cont,
	218	const key_compare& __comp,
	219	const _Allocator& __alloc)
222	220	: flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
223	221	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
224	222	"flat_multimap keys and mapped containers have different size");
225	223	__sort();
226	224	}
227	225
228		_LIBCPP_HIDE_FROM_ABI
229		flat_multimap(sorted_equivalent_t,
230		key_container_type __key_cont,
231		mapped_container_type __mapped_cont,
232		const key_compare& __comp = key_compare())
	226	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	227	sorted_equivalent_t,
	228	key_container_type __key_cont,
	229	mapped_container_type __mapped_cont,
	230	const key_compare& __comp = key_compare())
233	231	: __containers_{.keys = std::move(__key_cont), .values = std::move(__mapped_cont)}, __compare_(__comp) {
234	232	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
235	233	"flat_multimap keys and mapped containers have different size");
...	...	@@ -238,11 +236,11 @@ public:
238	236
239	237	template <class _Allocator>
240	238	requires __allocator_ctor_constraint<_Allocator>
241		_LIBCPP_HIDE_FROM_ABI
242		flat_multimap(sorted_equivalent_t,
243		const key_container_type& __key_cont,
244		const mapped_container_type& __mapped_cont,
245		const _Allocator& __alloc)
	239	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	240	sorted_equivalent_t,
	241	const key_container_type& __key_cont,
	242	const mapped_container_type& __mapped_cont,
	243	const _Allocator& __alloc)
246	244	: flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont) {
247	245	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
248	246	"flat_multimap keys and mapped containers have different size");
...	...	@@ -251,33 +249,35 @@ public:
251	249
252	250	template <class _Allocator>
253	251	requires __allocator_ctor_constraint<_Allocator>
254		_LIBCPP_HIDE_FROM_ABI
255		flat_multimap(sorted_equivalent_t,
256		const key_container_type& __key_cont,
257		const mapped_container_type& __mapped_cont,
258		const key_compare& __comp,
259		const _Allocator& __alloc)
	252	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	253	sorted_equivalent_t,
	254	const key_container_type& __key_cont,
	255	const mapped_container_type& __mapped_cont,
	256	const key_compare& __comp,
	257	const _Allocator& __alloc)
260	258	: flat_multimap(__ctor_uses_allocator_tag{}, __alloc, __key_cont, __mapped_cont, __comp) {
261	259	_LIBCPP_ASSERT_VALID_INPUT_RANGE(__containers_.keys.size() == __containers_.values.size(),
262	260	"flat_multimap keys and mapped containers have different size");
263	261	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__is_sorted(__containers_.keys), "Key container is not sorted");
264	262	}
265	263
266		_LIBCPP_HIDE_FROM_ABI explicit flat_multimap(const key_compare& __comp) : __containers_(), __compare_(__comp) {}
	264	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_multimap(const key_compare& __comp)
	265	: __containers_(), __compare_(__comp) {}
267	266
268	267	template <class _Allocator>
269	268	requires __allocator_ctor_constraint<_Allocator>
270		_LIBCPP_HIDE_FROM_ABI flat_multimap(const key_compare& __comp, const _Allocator& __alloc)
	269	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	270	flat_multimap(const key_compare& __comp, const _Allocator& __alloc)
271	271	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {}
272	272
273	273	template <class _Allocator>
274	274	requires __allocator_ctor_constraint<_Allocator>
275		_LIBCPP_HIDE_FROM_ABI explicit flat_multimap(const _Allocator& __alloc)
	275	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_multimap(const _Allocator& __alloc)
276	276	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {}
277	277
278	278	template <class _InputIterator>
279	279	requires __has_input_iterator_category<_InputIterator>::value
280		_LIBCPP_HIDE_FROM_ABI
	280	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
281	281	flat_multimap(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
282	282	: __containers_(), __compare_(__comp) {
283	283	insert(__first, __last);
...	...	@@ -285,7 +285,7 @@ public:
285	285
286	286	template <class _InputIterator, class _Allocator>
287	287	requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
288		_LIBCPP_HIDE_FROM_ABI
	288	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
289	289	flat_multimap(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
290	290	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
291	291	insert(__first, __last);
...	...	@@ -293,91 +293,99 @@ public:
293	293
294	294	template <class _InputIterator, class _Allocator>
295	295	requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
296		_LIBCPP_HIDE_FROM_ABI flat_multimap(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
	296	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	297	flat_multimap(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
297	298	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
298	299	insert(__first, __last);
299	300	}
300	301
301	302	template <_ContainerCompatibleRange<value_type> _Range>
302		_LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t __fr, _Range&& __rg)
	303	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(from_range_t __fr, _Range&& __rg)
303	304	: flat_multimap(__fr, std::forward<_Range>(__rg), key_compare()) {}
304	305
305	306	template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
306	307	requires __allocator_ctor_constraint<_Allocator>
307		_LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const _Allocator& __alloc)
	308	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	309	flat_multimap(from_range_t, _Range&& __rg, const _Allocator& __alloc)
308	310	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
309	311	insert_range(std::forward<_Range>(__rg));
310	312	}
311	313
312	314	template <_ContainerCompatibleRange<value_type> _Range>
313		_LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_multimap(__comp) {
	315	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	316	flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp)
	317	: flat_multimap(__comp) {
314	318	insert_range(std::forward<_Range>(__rg));
315	319	}
316	320
317	321	template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
318	322	requires __allocator_ctor_constraint<_Allocator>
319		_LIBCPP_HIDE_FROM_ABI flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
	323	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	324	flat_multimap(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
320	325	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
321	326	insert_range(std::forward<_Range>(__rg));
322	327	}
323	328
324	329	template <class _InputIterator>
325	330	requires __has_input_iterator_category<_InputIterator>::value
326		_LIBCPP_HIDE_FROM_ABI flat_multimap(
	331	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
327	332	sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
328	333	: __containers_(), __compare_(__comp) {
329	334	insert(sorted_equivalent, __first, __last);
330	335	}
331	336	template <class _InputIterator, class _Allocator>
332	337	requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
333		_LIBCPP_HIDE_FROM_ABI
334		flat_multimap(sorted_equivalent_t,
335		_InputIterator __first,
336		_InputIterator __last,
337		const key_compare& __comp,
338		const _Allocator& __alloc)
	338	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	339	sorted_equivalent_t,
	340	_InputIterator __first,
	341	_InputIterator __last,
	342	const key_compare& __comp,
	343	const _Allocator& __alloc)
339	344	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc, __comp) {
340	345	insert(sorted_equivalent, __first, __last);
341	346	}
342	347
343	348	template <class _InputIterator, class _Allocator>
344	349	requires(__has_input_iterator_category<_InputIterator>::value && __allocator_ctor_constraint<_Allocator>)
345		_LIBCPP_HIDE_FROM_ABI
	350	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
346	351	flat_multimap(sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
347	352	: flat_multimap(__ctor_uses_allocator_empty_tag{}, __alloc) {
348	353	insert(sorted_equivalent, __first, __last);
349	354	}
350	355
351		_LIBCPP_HIDE_FROM_ABI flat_multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
	356	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	357	flat_multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
352	358	: flat_multimap(__il.begin(), __il.end(), __comp) {}
353	359
354	360	template <class _Allocator>
355	361	requires __allocator_ctor_constraint<_Allocator>
356		_LIBCPP_HIDE_FROM_ABI
	362	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
357	363	flat_multimap(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
358	364	: flat_multimap(__il.begin(), __il.end(), __comp, __alloc) {}
359	365
360	366	template <class _Allocator>
361	367	requires __allocator_ctor_constraint<_Allocator>
362		_LIBCPP_HIDE_FROM_ABI flat_multimap(initializer_list<value_type> __il, const _Allocator& __alloc)
	368	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	369	flat_multimap(initializer_list<value_type> __il, const _Allocator& __alloc)
363	370	: flat_multimap(__il.begin(), __il.end(), __alloc) {}
364	371
365		_LIBCPP_HIDE_FROM_ABI
	372	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
366	373	flat_multimap(sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
367	374	: flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __comp) {}
368	375
369	376	template <class _Allocator>
370	377	requires __allocator_ctor_constraint<_Allocator>
371		_LIBCPP_HIDE_FROM_ABI flat_multimap(
	378	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
372	379	sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
373	380	: flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __comp, __alloc) {}
374	381
375	382	template <class _Allocator>
376	383	requires __allocator_ctor_constraint<_Allocator>
377		_LIBCPP_HIDE_FROM_ABI flat_multimap(sorted_equivalent_t, initializer_list<value_type> __il, const _Allocator& __alloc)
	384	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	385	flat_multimap(sorted_equivalent_t, initializer_list<value_type> __il, const _Allocator& __alloc)
378	386	: flat_multimap(sorted_equivalent, __il.begin(), __il.end(), __alloc) {}
379	387
380		_LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(initializer_list<value_type> __il) {
	388	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap& operator=(initializer_list<value_type> __il) {
381	389	clear();
382	390	insert(__il);
383	391	return *this;
...	...	@@ -386,9 +394,9 @@ public:
386	394	// copy/move assignment are not specified in the spec (defaulted)
387	395	// but move assignment can potentially leave moved from object in an inconsistent
388	396	// state if an exception is thrown
389		_LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(const flat_multimap&) = default;
	397	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap& operator=(const flat_multimap&) = default;
390	398
391		_LIBCPP_HIDE_FROM_ABI flat_multimap& operator=(flat_multimap&& __other) noexcept(
	399	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap& operator=(flat_multimap&& __other) noexcept(
392	400	is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_MappedContainer> &&
393	401	is_nothrow_move_assignable_v<_Compare>) {
394	402	auto __clear_other_guard = std::__make_scope_guard([&]() noexcept { __other.clear() /* noexcept */; });
...	...	@@ -400,38 +408,58 @@ public:
400	408	}
401	409
402	410	// iterators
403		_LIBCPP_HIDE_FROM_ABI iterator begin() noexcept {
	411	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
404	412	return iterator(__containers_.keys.begin(), __containers_.values.begin());
405	413	}
406	414
407		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept {
	415	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
408	416	return const_iterator(__containers_.keys.begin(), __containers_.values.begin());
409	417	}
410	418
411		_LIBCPP_HIDE_FROM_ABI iterator end() noexcept {
	419	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
412	420	return iterator(__containers_.keys.end(), __containers_.values.end());
413	421	}
414	422
415		_LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept {
	423	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
416	424	return const_iterator(__containers_.keys.end(), __containers_.values.end());
417	425	}
418	426
419		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
420		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
421		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
422		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
	427	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
	428	return reverse_iterator(end());
	429	}
	430	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
	431	return const_reverse_iterator(end());
	432	}
	433	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
	434	return reverse_iterator(begin());
	435	}
	436	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
	437	return const_reverse_iterator(begin());
	438	}
423	439
424		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
425		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
426		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
427		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
	440	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept {
	441	return begin();
	442	}
	443	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept {
	444	return end();
	445	}
	446	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
	447	return const_reverse_iterator(end());
	448	}
	449	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
	450	return const_reverse_iterator(begin());
	451	}
428	452
429	453	// [flat.map.capacity], capacity
430		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __containers_.keys.empty(); }
	454	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool empty() const noexcept {
	455	return __containers_.keys.empty();
	456	}
431	457
432		_LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __containers_.keys.size(); }
	458	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept {
	459	return __containers_.keys.size();
	460	}
433	461
434		_LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept {
	462	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept {
435	463	return std::min<size_type>(__containers_.keys.max_size(), __containers_.values.max_size());
436	464	}
437	465
...	...	@@ -439,7 +467,7 @@ public:
439	467	template <class... _Args>
440	468	requires is_constructible_v<pair<key_type, mapped_type>, _Args...> && is_move_constructible_v<key_type> &&
441	469	is_move_constructible_v<mapped_type>
442		_LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
	470	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace(_Args&&... __args) {
443	471	std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
444	472	auto __key_it = std::upper_bound(__containers_.keys.begin(), __containers_.keys.end(), __pair.first, __compare_);
445	473	auto __mapped_it = __corresponding_mapped_it(*this, __key_it);
...	...	@@ -450,7 +478,7 @@ public:
450	478
451	479	template <class... _Args>
452	480	requires is_constructible_v<pair<key_type, mapped_type>, _Args...>
453		_LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
	481	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
454	482	std::pair<key_type, mapped_type> __pair(std::forward<_Args>(__args)...);
455	483
456	484	auto __prev_larger = __hint != cbegin() && __compare_(__pair.first, (__hint - 1)->first);
...	...	@@ -490,33 +518,35 @@ public:
490	518	*this, __key_iter, __mapped_iter, std::move(__pair.first), std::move(__pair.second));
491	519	}
492	520
493		_LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return emplace(__x); }
	521	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const value_type& __x) { return emplace(__x); }
494	522
495		_LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return emplace(std::move(__x)); }
	523	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(value_type&& __x) {
	524	return emplace(std::move(__x));
	525	}
496	526
497		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
	527	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, const value_type& __x) {
498	528	return emplace_hint(__hint, __x);
499	529	}
500	530
501		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
	531	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, value_type&& __x) {
502	532	return emplace_hint(__hint, std::move(__x));
503	533	}
504	534
505	535	template <class _PairLike>
506	536	requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
507		_LIBCPP_HIDE_FROM_ABI iterator insert(_PairLike&& __x) {
	537	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(_PairLike&& __x) {
508	538	return emplace(std::forward<_PairLike>(__x));
509	539	}
510	540
511	541	template <class _PairLike>
512	542	requires is_constructible_v<pair<key_type, mapped_type>, _PairLike>
513		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _PairLike&& __x) {
	543	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, _PairLike&& __x) {
514	544	return emplace_hint(__hint, std::forward<_PairLike>(__x));
515	545	}
516	546
517	547	template <class _InputIterator>
518	548	requires __has_input_iterator_category<_InputIterator>::value
519		_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
	549	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(_InputIterator __first, _InputIterator __last) {
520	550	if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
521	551	__reserve(__last - __first);
522	552	}
...	...	@@ -525,7 +555,8 @@ public:
525	555
526	556	template <class _InputIterator>
527	557	requires __has_input_iterator_category<_InputIterator>::value
528		_LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, _InputIterator __first, _InputIterator __last) {
	558	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	559	insert(sorted_equivalent_t, _InputIterator __first, _InputIterator __last) {
529	560	if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
530	561	__reserve(__last - __first);
531	562	}
...	...	@@ -534,7 +565,7 @@ public:
534	565	}
535	566
536	567	template <_ContainerCompatibleRange<value_type> _Range>
537		_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
	568	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(_Range&& __range) {
538	569	if constexpr (ranges::sized_range<_Range>) {
539	570	__reserve(ranges::size(__range));
540	571	}
...	...	@@ -542,19 +573,32 @@ public:
542	573	__append_sort_merge</WasSorted = / false>(ranges::begin(__range), ranges::end(__range));
543	574	}
544	575
545		_LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
	576	template <_ContainerCompatibleRange<value_type> _Range>
	577	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(sorted_equivalent_t, _Range&& __range) {
	578	if constexpr (ranges::sized_range<_Range>) {
	579	__reserve(ranges::size(__range));
	580	}
	581
	582	__append_sort_merge</WasSorted = / true>(ranges::begin(__range), ranges::end(__range));
	583	}
	584
	585	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
	586	insert(__il.begin(), __il.end());
	587	}
546	588
547		_LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, initializer_list<value_type> __il) {
	589	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	590	insert(sorted_equivalent_t, initializer_list<value_type> __il) {
548	591	insert(sorted_equivalent, __il.begin(), __il.end());
549	592	}
550	593
551		_LIBCPP_HIDE_FROM_ABI containers extract() && {
	594	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 containers extract() && {
552	595	auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
553	596	auto __ret = std::move(__containers_);
554	597	return __ret;
555	598	}
556	599
557		_LIBCPP_HIDE_FROM_ABI void replace(key_container_type&& __key_cont, mapped_container_type&& __mapped_cont) {
	600	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	601	replace(key_container_type&& __key_cont, mapped_container_type&& __mapped_cont) {
558	602	_LIBCPP_ASSERT_VALID_INPUT_RANGE(
559	603	__key_cont.size() == __mapped_cont.size(), "flat_multimap keys and mapped containers have different size");
560	604
...	...	@@ -565,15 +609,15 @@ public:
565	609	__guard.__complete();
566	610	}
567	611
568		_LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
	612	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(iterator __position) {
569	613	return __erase(__position.__key_iter_, __position.__mapped_iter_);
570	614	}
571	615
572		_LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position) {
	616	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(const_iterator __position) {
573	617	return __erase(__position.__key_iter_, __position.__mapped_iter_);
574	618	}
575	619
576		_LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
	620	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(const key_type& __x) {
577	621	auto [__first, __last] = equal_range(__x);
578	622	auto __res = __last - __first;
579	623	erase(__first, __last);
...	...	@@ -583,14 +627,14 @@ public:
583	627	template <class _Kp>
584	628	requires(__is_compare_transparent && !is_convertible_v<_Kp &&, iterator> &&
585	629	!is_convertible_v<_Kp &&, const_iterator>)
586		_LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
	630	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(_Kp&& __x) {
587	631	auto [__first, __last] = equal_range(__x);
588	632	auto __res = __last - __first;
589	633	erase(__first, __last);
590	634	return __res;
591	635	}
592	636
593		_LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
	637	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(const_iterator __first, const_iterator __last) {
594	638	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
595	639	auto __key_it = __containers_.keys.erase(__first.__key_iter_, __last.__key_iter_);
596	640	auto __mapped_it = __containers_.values.erase(__first.__mapped_iter_, __last.__mapped_iter_);
...	...	@@ -598,146 +642,178 @@ public:
598	642	return iterator(std::move(__key_it), std::move(__mapped_it));
599	643	}
600	644
601		_LIBCPP_HIDE_FROM_ABI void swap(flat_multimap& __y) noexcept {
602		// warning: The spec has unconditional noexcept, which means that
603		// if any of the following functions throw an exception,
604		// std::terminate will be called
	645	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_multimap& __y) noexcept(
	646	is_nothrow_swappable_v<key_container_type> && is_nothrow_swappable_v<mapped_container_type> &&
	647	is_nothrow_swappable_v<key_compare>) {
	648	auto __on_failure = std::__make_exception_guard([&]() noexcept {
	649	clear() /* noexcept */;
	650	__y.clear() /* noexcept */;
	651	});
605	652	ranges::swap(__compare_, __y.__compare_);
606	653	ranges::swap(__containers_.keys, __y.__containers_.keys);
607	654	ranges::swap(__containers_.values, __y.__containers_.values);
	655	__on_failure.__complete();
608	656	}
609	657
610		_LIBCPP_HIDE_FROM_ABI void clear() noexcept {
	658	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept {
611	659	__containers_.keys.clear();
612	660	__containers_.values.clear();
613	661	}
614	662
615	663	// observers
616		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
617		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__compare_); }
	664	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
	665	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const {
	666	return value_compare(__compare_);
	667	}
618	668
619		_LIBCPP_HIDE_FROM_ABI const key_container_type& keys() const noexcept { return __containers_.keys; }
620		_LIBCPP_HIDE_FROM_ABI const mapped_container_type& values() const noexcept { return __containers_.values; }
	669	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const key_container_type& keys() const noexcept {
	670	return __containers_.keys;
	671	}
	672	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const mapped_container_type&
	673	values() const noexcept {
	674	return __containers_.values;
	675	}
621	676
622	677	// map operations
623		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
	678	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
	679	return __find_impl(*this, __x);
	680	}
624	681
625		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
	682	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
	683	return __find_impl(*this, __x);
	684	}
626	685
627	686	template <class _Kp>
628	687	requires __is_compare_transparent
629		_LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
	688	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
630	689	return __find_impl(*this, __x);
631	690	}
632	691
633	692	template <class _Kp>
634	693	requires __is_compare_transparent
635		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
	694	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
636	695	return __find_impl(*this, __x);
637	696	}
638	697
639		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const {
	698	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
640	699	auto [__first, __last] = equal_range(__x);
641	700	return __last - __first;
642	701	}
643	702
644	703	template <class _Kp>
645	704	requires __is_compare_transparent
646		_LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
	705	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
647	706	auto [__first, __last] = equal_range(__x);
648	707	return __last - __first;
649	708	}
650	709
651		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
	710	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
	711	return find(__x) != end();
	712	}
652	713
653	714	template <class _Kp>
654	715	requires __is_compare_transparent
655		_LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
	716	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
656	717	return find(__x) != end();
657	718	}
658	719
659		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) { return __lower_bound<iterator>(*this, __x); }
	720	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
	721	return __lower_bound<iterator>(*this, __x);
	722	}
660	723
661		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
	724	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	725	lower_bound(const key_type& __x) const {
662	726	return __lower_bound<const_iterator>(*this, __x);
663	727	}
664	728
665	729	template <class _Kp>
666	730	requires __is_compare_transparent
667		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
	731	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
668	732	return __lower_bound<iterator>(*this, __x);
669	733	}
670	734
671	735	template <class _Kp>
672	736	requires __is_compare_transparent
673		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
	737	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
674	738	return __lower_bound<const_iterator>(*this, __x);
675	739	}
676	740
677		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) { return __upper_bound<iterator>(*this, __x); }
	741	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
	742	return __upper_bound<iterator>(*this, __x);
	743	}
678	744
679		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
	745	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	746	upper_bound(const key_type& __x) const {
680	747	return __upper_bound<const_iterator>(*this, __x);
681	748	}
682	749
683	750	template <class _Kp>
684	751	requires __is_compare_transparent
685		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
	752	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
686	753	return __upper_bound<iterator>(*this, __x);
687	754	}
688	755
689	756	template <class _Kp>
690	757	requires __is_compare_transparent
691		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
	758	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
692	759	return __upper_bound<const_iterator>(*this, __x);
693	760	}
694	761
695		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
	762	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	763	equal_range(const key_type& __x) {
696	764	return __equal_range_impl(*this, __x);
697	765	}
698	766
699		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
	767	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	768	equal_range(const key_type& __x) const {
700	769	return __equal_range_impl(*this, __x);
701	770	}
702	771
703	772	template <class _Kp>
704	773	requires __is_compare_transparent
705		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
	774	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	775	equal_range(const _Kp& __x) {
706	776	return __equal_range_impl(*this, __x);
707	777	}
708	778	template <class _Kp>
709	779	requires __is_compare_transparent
710		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
	780	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	781	equal_range(const _Kp& __x) const {
711	782	return __equal_range_impl(*this, __x);
712	783	}
713	784
714		friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_multimap& __x, const flat_multimap& __y) {
	785	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
	786	operator==(const flat_multimap& __x, const flat_multimap& __y) {
715	787	return ranges::equal(__x, __y);
716	788	}
717	789
718		friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_multimap& __x, const flat_multimap& __y) {
	790	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 auto
	791	operator<=>(const flat_multimap& __x, const flat_multimap& __y) {
719	792	return std::lexicographical_compare_three_way(
720	793	__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
721	794	}
722	795
723		friend _LIBCPP_HIDE_FROM_ABI void swap(flat_multimap& __x, flat_multimap& __y) noexcept { __x.swap(__y); }
	796	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	797	swap(flat_multimap& __x, flat_multimap& __y) noexcept(noexcept(__x.swap(__y))) {
	798	__x.swap(__y);
	799	}
724	800
725	801	private:
726	802	struct __ctor_uses_allocator_tag {
727		explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_tag() = default;
	803	explicit _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __ctor_uses_allocator_tag() = default;
728	804	};
729	805	struct __ctor_uses_allocator_empty_tag {
730		explicit _LIBCPP_HIDE_FROM_ABI __ctor_uses_allocator_empty_tag() = default;
	806	explicit _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __ctor_uses_allocator_empty_tag() = default;
731	807	};
732	808
733	809	template <class _Allocator, class _KeyCont, class _MappedCont, class... _CompArg>
734	810	requires __allocator_ctor_constraint<_Allocator>
735		_LIBCPP_HIDE_FROM_ABI
736		flat_multimap(__ctor_uses_allocator_tag,
737		const _Allocator& __alloc,
738		_KeyCont&& __key_cont,
739		_MappedCont&& __mapped_cont,
740		_CompArg&&... __comp)
	811	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multimap(
	812	__ctor_uses_allocator_tag,
	813	const _Allocator& __alloc,
	814	_KeyCont&& __key_cont,
	815	_MappedCont&& __mapped_cont,
	816	_CompArg&&... __comp)
741	817	: __containers_{.keys = std::make_obj_using_allocator<key_container_type>(
742	818	__alloc, std::forward<_KeyCont>(__key_cont)),
743	819	.values = std::make_obj_using_allocator<mapped_container_type>(
...	...	@@ -746,36 +822,39 @@ private:
746	822
747	823	template <class _Allocator, class... _CompArg>
748	824	requires __allocator_ctor_constraint<_Allocator>
749		_LIBCPP_HIDE_FROM_ABI flat_multimap(__ctor_uses_allocator_empty_tag, const _Allocator& __alloc, _CompArg&&... __comp)
	825	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	826	flat_multimap(__ctor_uses_allocator_empty_tag, const _Allocator& __alloc, _CompArg&&... __comp)
750	827	: __containers_{.keys = std::make_obj_using_allocator<key_container_type>(__alloc),
751	828	.values = std::make_obj_using_allocator<mapped_container_type>(__alloc)},
752	829	__compare_(std::forward<_CompArg>(__comp)...) {}
753	830
754		_LIBCPP_HIDE_FROM_ABI bool __is_sorted(auto&& __key_container) const {
	831	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_sorted(auto&& __key_container) const {
755	832	return ranges::is_sorted(__key_container, __compare_);
756	833	}
757	834
758		_LIBCPP_HIDE_FROM_ABI void __sort() {
	835	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __sort() {
759	836	auto __zv = ranges::views::zip(__containers_.keys, __containers_.values);
760	837	ranges::sort(__zv, __compare_, [](const auto& __p) -> decltype(auto) { return std::get<0>(__p); });
761	838	}
762	839
763	840	template <class _Self, class _KeyIter>
764		_LIBCPP_HIDE_FROM_ABI static auto __corresponding_mapped_it(_Self&& __self, _KeyIter&& __key_iter) {
	841	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto
	842	__corresponding_mapped_it(_Self&& __self, _KeyIter&& __key_iter) {
765	843	return __self.__containers_.values.begin() +
766	844	static_cast<ranges::range_difference_t<mapped_container_type>>(
767	845	ranges::distance(__self.__containers_.keys.begin(), __key_iter));
768	846	}
769	847
770	848	template <bool _WasSorted, class _InputIterator, class _Sentinel>
771		_LIBCPP_HIDE_FROM_ABI void __append_sort_merge(_InputIterator __first, _Sentinel __last) {
	849	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	850	__append_sort_merge(_InputIterator __first, _Sentinel __last) {
772	851	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
773	852	size_t __num_appended = __flat_map_utils::__append(*this, std::move(__first), std::move(__last));
774	853	if (__num_appended != 0) {
775	854	auto __zv = ranges::views::zip(__containers_.keys, __containers_.values);
776	855	auto __append_start_offset = __containers_.keys.size() - __num_appended;
777	856	auto __end = __zv.end();
778		auto __compare_key = [this](const auto& __p1, const auto& __p2) {
	857	auto __compare_key = [this](const auto& __p1, const auto& __p2) -> bool {
779	858	return __compare_(std::get<0>(__p1), std::get<0>(__p2));
780	859	};
781	860	if constexpr (!_WasSorted) {
...	...	@@ -791,7 +870,7 @@ private:
791	870	}
792	871
793	872	template <class _Self, class _Kp>
794		_LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
	873	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __find_impl(_Self&& __self, const _Kp& __key) {
795	874	auto __it = __self.lower_bound(__key);
796	875	auto __last = __self.end();
797	876	if (__it == __last \|\| __self.__compare_(__key, __it->first)) {
...	...	@@ -801,7 +880,7 @@ private:
801	880	}
802	881
803	882	template <class _Self, class _Kp>
804		_LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
	883	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
805	884	auto [__key_first, __key_last] =
806	885	std::equal_range(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __key, __self.__compare_);
807	886
...	...	@@ -811,7 +890,7 @@ private:
811	890	}
812	891
813	892	template <class _Res, class _Self, class _Kp>
814		_LIBCPP_HIDE_FROM_ABI static _Res __lower_bound(_Self&& __self, _Kp& __x) {
	893	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static _Res __lower_bound(_Self&& __self, _Kp& __x) {
815	894	auto __key_iter =
816	895	std::lower_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
817	896	auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
...	...	@@ -819,14 +898,14 @@ private:
819	898	}
820	899
821	900	template <class _Res, class _Self, class _Kp>
822		_LIBCPP_HIDE_FROM_ABI static _Res __upper_bound(_Self&& __self, _Kp& __x) {
	901	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static _Res __upper_bound(_Self&& __self, _Kp& __x) {
823	902	auto __key_iter =
824	903	std::upper_bound(__self.__containers_.keys.begin(), __self.__containers_.keys.end(), __x, __self.__compare_);
825	904	auto __mapped_iter = __corresponding_mapped_it(__self, __key_iter);
826	905	return _Res(std::move(__key_iter), std::move(__mapped_iter));
827	906	}
828	907
829		_LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
	908	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __reserve(size_t __size) {
830	909	if constexpr (__container_traits<_KeyContainer>::__reservable) {
831	910	__containers_.keys.reserve(__size);
832	911	}
...	...	@@ -837,7 +916,8 @@ private:
837	916	}
838	917
839	918	template <class _KIter, class _MIter>
840		_LIBCPP_HIDE_FROM_ABI iterator __erase(_KIter __key_iter_to_remove, _MIter __mapped_iter_to_remove) {
	919	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator
	920	__erase(_KIter __key_iter_to_remove, _MIter __mapped_iter_to_remove) {
841	921	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
842	922	auto __key_iter = __containers_.keys.erase(__key_iter_to_remove);
843	923	auto __mapped_iter = __containers_.values.erase(__mapped_iter_to_remove);
...	...	@@ -847,7 +927,8 @@ private:
847	927
848	928	template <class _Key2, class _Tp2, class _Compare2, class _KeyContainer2, class _MappedContainer2, class _Predicate>
849	929	friend typename flat_multimap<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>::size_type
850		erase_if(flat_multimap<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>&, _Predicate);
	930	_LIBCPP_CONSTEXPR_SINCE_CXX26
	931	erase_if(flat_multimap<_Key2, _Tp2, _Compare2, _KeyContainer2, _MappedContainer2>&, _Predicate);
851	932
852	933	friend __flat_map_utils;
853	934
...	...	@@ -855,8 +936,9 @@ private:
855	936	_LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
856	937
857	938	struct __key_equiv {
858		_LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
859		_LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
	939	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __key_equiv(key_compare __c) : __comp_(__c) {}
	940	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
	941	operator()(const_reference __x, const_reference __y) const {
860	942	return !__comp_(std::get<0>(__x), std::get<0>(__y)) && !__comp_(std::get<0>(__y), std::get<0>(__x));
861	943	}
862	944	key_compare __comp_;
...	...	@@ -864,8 +946,7 @@ private:
864	946	};
865	947
866	948	template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
867		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
868		!__is_allocator<_MappedContainer>::value &&
	949	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
869	950	is_invocable_v<const _Compare&,
870	951	const typename _KeyContainer::value_type&,
871	952	const typename _KeyContainer::value_type&>)
...	...	@@ -878,7 +959,7 @@ flat_multimap(_KeyContainer, _MappedContainer, _Compare = _Compare())
878	959
879	960	template <class _KeyContainer, class _MappedContainer, class _Allocator>
880	961	requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
881		!__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
	962	!__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer>)
882	963	flat_multimap(_KeyContainer, _MappedContainer, _Allocator)
883	964	-> flat_multimap<typename _KeyContainer::value_type,
884	965	typename _MappedContainer::value_type,
...	...	@@ -887,9 +968,8 @@ flat_multimap(_KeyContainer, _MappedContainer, _Allocator)
887	968	_MappedContainer>;
888	969
889	970	template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
890		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
891		!__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
892		uses_allocator_v<_MappedContainer, _Allocator> &&
	971	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
	972	uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
893	973	is_invocable_v<const _Compare&,
894	974	const typename _KeyContainer::value_type&,
895	975	const typename _KeyContainer::value_type&>)
...	...	@@ -901,8 +981,7 @@ flat_multimap(_KeyContainer, _MappedContainer, _Compare, _Allocator)
901	981	_MappedContainer>;
902	982
903	983	template <class _KeyContainer, class _MappedContainer, class _Compare = less<typename _KeyContainer::value_type>>
904		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
905		!__is_allocator<_MappedContainer>::value &&
	984	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
906	985	is_invocable_v<const _Compare&,
907	986	const typename _KeyContainer::value_type&,
908	987	const typename _KeyContainer::value_type&>)
...	...	@@ -915,7 +994,7 @@ flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Compare = _
915	994
916	995	template <class _KeyContainer, class _MappedContainer, class _Allocator>
917	996	requires(uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
918		!__is_allocator<_KeyContainer>::value && !__is_allocator<_MappedContainer>::value)
	997	!__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer>)
919	998	flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Allocator)
920	999	-> flat_multimap<typename _KeyContainer::value_type,
921	1000	typename _MappedContainer::value_type,
...	...	@@ -924,9 +1003,8 @@ flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Allocator)
924	1003	_MappedContainer>;
925	1004
926	1005	template <class _KeyContainer, class _MappedContainer, class _Compare, class _Allocator>
927		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
928		!__is_allocator<_MappedContainer>::value && uses_allocator_v<_KeyContainer, _Allocator> &&
929		uses_allocator_v<_MappedContainer, _Allocator> &&
	1006	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> && !__is_allocator_v<_MappedContainer> &&
	1007	uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator> &&
930	1008	is_invocable_v<const _Compare&,
931	1009	const typename _KeyContainer::value_type&,
932	1010	const typename _KeyContainer::value_type&>)
...	...	@@ -938,19 +1016,19 @@ flat_multimap(sorted_equivalent_t, _KeyContainer, _MappedContainer, _Compare, _A
938	1016	_MappedContainer>;
939	1017
940	1018	template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
941		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	1019	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
942	1020	flat_multimap(_InputIterator, _InputIterator, _Compare = _Compare())
943	1021	-> flat_multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
944	1022
945	1023	template <class _InputIterator, class _Compare = less<__iter_key_type<_InputIterator>>>
946		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	1024	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
947	1025	flat_multimap(sorted_equivalent_t, _InputIterator, _InputIterator, _Compare = _Compare())
948	1026	-> flat_multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare>;
949	1027
950	1028	template <ranges::input_range _Range,
951	1029	class _Compare = less<__range_key_type<_Range>>,
952	1030	class _Allocator = allocator<byte>,
953		class = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
	1031	class = __enable_if_t<!__is_allocator_v<_Compare> && __is_allocator_v<_Allocator>>>
954	1032	flat_multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_multimap<
955	1033	__range_key_type<_Range>,
956	1034	__range_mapped_type<_Range>,
...	...	@@ -958,7 +1036,7 @@ flat_multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Alloc
958	1036	vector<__range_key_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_key_type<_Range>>>,
959	1037	vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
960	1038
961		template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
	1039	template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator_v<_Allocator>>>
962	1040	flat_multimap(from_range_t, _Range&&, _Allocator) -> flat_multimap<
963	1041	__range_key_type<_Range>,
964	1042	__range_mapped_type<_Range>,
...	...	@@ -967,11 +1045,11 @@ flat_multimap(from_range_t, _Range&&, _Allocator) -> flat_multimap<
967	1045	vector<__range_mapped_type<_Range>, __allocator_traits_rebind_t<_Allocator, __range_mapped_type<_Range>>>>;
968	1046
969	1047	template <class _Key, class _Tp, class _Compare = less<_Key>>
970		requires(!__is_allocator<_Compare>::value)
	1048	requires(!__is_allocator_v<_Compare>)
971	1049	flat_multimap(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare()) -> flat_multimap<_Key, _Tp, _Compare>;
972	1050
973	1051	template <class _Key, class _Tp, class _Compare = less<_Key>>
974		requires(!__is_allocator<_Compare>::value)
	1052	requires(!__is_allocator_v<_Compare>)
975	1053	flat_multimap(sorted_equivalent_t, initializer_list<pair<_Key, _Tp>>, _Compare = _Compare())
976	1054	-> flat_multimap<_Key, _Tp, _Compare>;
977	1055
...	...	@@ -980,8 +1058,9 @@ struct uses_allocator<flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedC
980	1058	: bool_constant<uses_allocator_v<_KeyContainer, _Allocator> && uses_allocator_v<_MappedContainer, _Allocator>> {};
981	1059
982	1060	template <class _Key, class _Tp, class _Compare, class _KeyContainer, class _MappedContainer, class _Predicate>
983		_LIBCPP_HIDE_FROM_ABI typename flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::size_type
984		erase_if(flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>& __flat_multimap, _Predicate __pred) {
	1061	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	1062	typename flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>::size_type
	1063	erase_if(flat_multimap<_Key, _Tp, _Compare, _KeyContainer, _MappedContainer>& __flat_multimap, _Predicate __pred) {
985	1064	auto __zv = ranges::views::zip(__flat_multimap.__containers_.keys, __flat_multimap.__containers_.values);
986	1065	auto __first = __zv.begin();
987	1066	auto __last = __zv.end();

lib/libcxx/include/__flat_map/key_value_iterator.h-1

...	...	@@ -20,7 +20,6 @@
20	20	#include <__type_traits/conditional.h>
21	21	#include <__utility/forward.h>
22	22	#include <__utility/move.h>
23		#include <__utility/pair.h>
24	23
25	24	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
26	25	# pragma GCC system_header

lib/libcxx/include/__flat_map/utils.h+1

...	...	@@ -16,6 +16,7 @@
16	16	#include <__utility/exception_guard.h>
17	17	#include <__utility/forward.h>
18	18	#include <__utility/move.h>
	19	#include <__vector/container_traits.h>
19	20
20	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
21	22	# pragma GCC system_header

lib/libcxx/include/__flat_set/flat_multiset.h+211-143

...	...	@@ -13,54 +13,41 @@
13	13	#include <__algorithm/equal_range.h>
14	14	#include <__algorithm/lexicographical_compare_three_way.h>
15	15	#include <__algorithm/lower_bound.h>
16		#include <__algorithm/min.h>
17	16	#include <__algorithm/ranges_equal.h>
18	17	#include <__algorithm/ranges_inplace_merge.h>
19	18	#include <__algorithm/ranges_is_sorted.h>
20	19	#include <__algorithm/ranges_sort.h>
21		#include <__algorithm/ranges_unique.h>
22	20	#include <__algorithm/remove_if.h>
23	21	#include <__algorithm/upper_bound.h>
24	22	#include <__assert>
25	23	#include <__compare/synth_three_way.h>
26		#include <__concepts/convertible_to.h>
27	24	#include <__concepts/swappable.h>
28	25	#include <__config>
29		#include <__cstddef/byte.h>
30		#include <__cstddef/ptrdiff_t.h>
31		#include <__flat_map/key_value_iterator.h>
32	26	#include <__flat_map/sorted_equivalent.h>
33	27	#include <__flat_set/ra_iterator.h>
34	28	#include <__flat_set/utils.h>
35		#include <__functional/invoke.h>
36	29	#include <__functional/is_transparent.h>
37	30	#include <__functional/operations.h>
38	31	#include <__fwd/vector.h>
39	32	#include <__iterator/concepts.h>
40		#include <__iterator/distance.h>
41	33	#include <__iterator/iterator_traits.h>
42	34	#include <__iterator/prev.h>
43		#include <__iterator/ranges_iterator_traits.h>
44	35	#include <__iterator/reverse_iterator.h>
45	36	#include <__memory/allocator_traits.h>
46	37	#include <__memory/uses_allocator.h>
47	38	#include <__memory/uses_allocator_construction.h>
48		#include <__ranges/access.h>
49	39	#include <__ranges/concepts.h>
50	40	#include <__ranges/container_compatible_range.h>
51	41	#include <__ranges/drop_view.h>
52	42	#include <__ranges/from_range.h>
53		#include <__ranges/ref_view.h>
	43	#include <__ranges/range_adaptor.h>
54	44	#include <__ranges/size.h>
55	45	#include <__ranges/subrange.h>
56		#include <__ranges/zip_view.h>
57		#include <__type_traits/conjunction.h>
58	46	#include <__type_traits/container_traits.h>
59	47	#include <__type_traits/invoke.h>
60	48	#include <__type_traits/is_allocator.h>
61	49	#include <__type_traits/is_nothrow_constructible.h>
62	50	#include <__type_traits/is_same.h>
63		#include <__type_traits/maybe_const.h>
64	51	#include <__utility/as_const.h>
65	52	#include <__utility/exception_guard.h>
66	53	#include <__utility/move.h>
...	...	@@ -108,16 +95,16 @@ public:
108	95
109	96	public:
110	97	// [flat.multiset.cons], constructors
111		_LIBCPP_HIDE_FROM_ABI flat_multiset() noexcept(is_nothrow_default_constructible_v<_KeyContainer> &&
112		is_nothrow_default_constructible_v<_Compare>)
	98	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset() noexcept(
	99	is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_Compare>)
113	100	: __keys_(), __compare_() {}
114	101
115		_LIBCPP_HIDE_FROM_ABI flat_multiset(const flat_multiset&) = default;
	102	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(const flat_multiset&) = default;
116	103
117	104	// The copy/move constructors are not specified in the spec, which means they should be defaulted.
118	105	// However, the move constructor can potentially leave a moved-from object in an inconsistent
119	106	// state if an exception is thrown.
120		_LIBCPP_HIDE_FROM_ABI flat_multiset(flat_multiset&& __other) noexcept(
	107	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(flat_multiset&& __other) noexcept(
121	108	is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_Compare>)
122	109	# if _LIBCPP_HAS_EXCEPTIONS
123	110	try
...	...	@@ -134,14 +121,16 @@ public:
134	121	# endif // _LIBCPP_HAS_EXCEPTIONS
135	122	}
136	123
137		_LIBCPP_HIDE_FROM_ABI explicit flat_multiset(const key_compare& __comp) : __keys_(), __compare_(__comp) {}
	124	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_multiset(const key_compare& __comp)
	125	: __keys_(), __compare_(__comp) {}
138	126
139		_LIBCPP_HIDE_FROM_ABI explicit flat_multiset(container_type __keys, const key_compare& __comp = key_compare())
	127	_LIBCPP_HIDE_FROM_ABI
	128	_LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_multiset(container_type __keys, const key_compare& __comp = key_compare())
140	129	: __keys_(std::move(__keys)), __compare_(__comp) {
141	130	ranges::sort(__keys_, __compare_);
142	131	}
143	132
144		_LIBCPP_HIDE_FROM_ABI
	133	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
145	134	flat_multiset(sorted_equivalent_t, container_type __keys, const key_compare& __comp = key_compare())
146	135	: __keys_(std::move(__keys)), __compare_(__comp) {
147	136	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
...	...	@@ -149,7 +138,7 @@ public:
149	138
150	139	template <class _InputIterator>
151	140	requires __has_input_iterator_category<_InputIterator>::value
152		_LIBCPP_HIDE_FROM_ABI
	141	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
153	142	flat_multiset(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
154	143	: __keys_(), __compare_(__comp) {
155	144	insert(__first, __last);
...	...	@@ -157,48 +146,53 @@ public:
157	146
158	147	template <class _InputIterator>
159	148	requires __has_input_iterator_category<_InputIterator>::value
160		_LIBCPP_HIDE_FROM_ABI flat_multiset(
	149	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(
161	150	sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
162	151	: __keys_(__first, __last), __compare_(__comp) {
163	152	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
164	153	}
165	154
166	155	template <_ContainerCompatibleRange<value_type> _Range>
167		_LIBCPP_HIDE_FROM_ABI flat_multiset(from_range_t __fr, _Range&& __rg)
	156	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(from_range_t __fr, _Range&& __rg)
168	157	: flat_multiset(__fr, std::forward<_Range>(__rg), key_compare()) {}
169	158
170	159	template <_ContainerCompatibleRange<value_type> _Range>
171		_LIBCPP_HIDE_FROM_ABI flat_multiset(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_multiset(__comp) {
	160	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	161	flat_multiset(from_range_t, _Range&& __rg, const key_compare& __comp)
	162	: flat_multiset(__comp) {
172	163	insert_range(std::forward<_Range>(__rg));
173	164	}
174	165
175		_LIBCPP_HIDE_FROM_ABI flat_multiset(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
	166	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	167	flat_multiset(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
176	168	: flat_multiset(__il.begin(), __il.end(), __comp) {}
177	169
178		_LIBCPP_HIDE_FROM_ABI
	170	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
179	171	flat_multiset(sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
180	172	: flat_multiset(sorted_equivalent, __il.begin(), __il.end(), __comp) {}
181	173
182	174	template <class _Allocator>
183	175	requires uses_allocator<container_type, _Allocator>::value
184		_LIBCPP_HIDE_FROM_ABI explicit flat_multiset(const _Allocator& __alloc)
	176	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_multiset(const _Allocator& __alloc)
185	177	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {}
186	178
187	179	template <class _Allocator>
188	180	requires uses_allocator<container_type, _Allocator>::value
189		_LIBCPP_HIDE_FROM_ABI flat_multiset(const key_compare& __comp, const _Allocator& __alloc)
	181	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	182	flat_multiset(const key_compare& __comp, const _Allocator& __alloc)
190	183	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {}
191	184
192	185	template <class _Allocator>
193	186	requires uses_allocator<container_type, _Allocator>::value
194		_LIBCPP_HIDE_FROM_ABI flat_multiset(const container_type& __keys, const _Allocator& __alloc)
	187	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	188	flat_multiset(const container_type& __keys, const _Allocator& __alloc)
195	189	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
196	190	ranges::sort(__keys_, __compare_);
197	191	}
198	192
199	193	template <class _Allocator>
200	194	requires uses_allocator<container_type, _Allocator>::value
201		_LIBCPP_HIDE_FROM_ABI
	195	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
202	196	flat_multiset(const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
203	197	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
204	198	ranges::sort(__keys_, __compare_);
...	...	@@ -206,14 +200,15 @@ public:
206	200
207	201	template <class _Allocator>
208	202	requires uses_allocator<container_type, _Allocator>::value
209		_LIBCPP_HIDE_FROM_ABI flat_multiset(sorted_equivalent_t, const container_type& __keys, const _Allocator& __alloc)
	203	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	204	flat_multiset(sorted_equivalent_t, const container_type& __keys, const _Allocator& __alloc)
210	205	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
211	206	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
212	207	}
213	208
214	209	template <class _Allocator>
215	210	requires uses_allocator<container_type, _Allocator>::value
216		_LIBCPP_HIDE_FROM_ABI
	211	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
217	212	flat_multiset(sorted_equivalent_t, const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
218	213	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
219	214	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
...	...	@@ -221,13 +216,14 @@ public:
221	216
222	217	template <class _Allocator>
223	218	requires uses_allocator<container_type, _Allocator>::value
224		_LIBCPP_HIDE_FROM_ABI flat_multiset(const flat_multiset& __other, const _Allocator& __alloc)
	219	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	220	flat_multiset(const flat_multiset& __other, const _Allocator& __alloc)
225	221	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __other.__keys_)),
226	222	__compare_(__other.__compare_) {}
227	223
228	224	template <class _Allocator>
229	225	requires uses_allocator<container_type, _Allocator>::value
230		_LIBCPP_HIDE_FROM_ABI flat_multiset(flat_multiset&& __other, const _Allocator& __alloc)
	226	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(flat_multiset&& __other, const _Allocator& __alloc)
231	227	# if _LIBCPP_HAS_EXCEPTIONS
232	228	try
233	229	# endif // _LIBCPP_HAS_EXCEPTIONS
...	...	@@ -243,14 +239,15 @@ public:
243	239
244	240	template <class _InputIterator, class _Allocator>
245	241	requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
246		_LIBCPP_HIDE_FROM_ABI flat_multiset(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
	242	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	243	flat_multiset(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
247	244	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
248	245	insert(__first, __last);
249	246	}
250	247
251	248	template <class _InputIterator, class _Allocator>
252	249	requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
253		_LIBCPP_HIDE_FROM_ABI
	250	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
254	251	flat_multiset(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
255	252	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
256	253	insert(__first, __last);
...	...	@@ -258,7 +255,7 @@ public:
258	255
259	256	template <class _InputIterator, class _Allocator>
260	257	requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
261		_LIBCPP_HIDE_FROM_ABI
	258	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
262	259	flat_multiset(sorted_equivalent_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
263	260	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_() {
264	261	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
...	...	@@ -266,53 +263,57 @@ public:
266	263
267	264	template <class _InputIterator, class _Allocator>
268	265	requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
269		_LIBCPP_HIDE_FROM_ABI
270		flat_multiset(sorted_equivalent_t,
271		_InputIterator __first,
272		_InputIterator __last,
273		const key_compare& __comp,
274		const _Allocator& __alloc)
	266	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(
	267	sorted_equivalent_t,
	268	_InputIterator __first,
	269	_InputIterator __last,
	270	const key_compare& __comp,
	271	const _Allocator& __alloc)
275	272	: __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_(__comp) {
276	273	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys_, __compare_), "Key container is not sorted");
277	274	}
278	275
279	276	template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
280	277	requires uses_allocator<container_type, _Allocator>::value
281		_LIBCPP_HIDE_FROM_ABI flat_multiset(from_range_t, _Range&& __rg, const _Allocator& __alloc)
	278	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	279	flat_multiset(from_range_t, _Range&& __rg, const _Allocator& __alloc)
282	280	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
283	281	insert_range(std::forward<_Range>(__rg));
284	282	}
285	283
286	284	template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
287	285	requires uses_allocator<container_type, _Allocator>::value
288		_LIBCPP_HIDE_FROM_ABI flat_multiset(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
	286	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	287	flat_multiset(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
289	288	: __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
290	289	insert_range(std::forward<_Range>(__rg));
291	290	}
292	291
293	292	template <class _Allocator>
294	293	requires uses_allocator<container_type, _Allocator>::value
295		_LIBCPP_HIDE_FROM_ABI flat_multiset(initializer_list<value_type> __il, const _Allocator& __alloc)
	294	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	295	flat_multiset(initializer_list<value_type> __il, const _Allocator& __alloc)
296	296	: flat_multiset(__il.begin(), __il.end(), __alloc) {}
297	297
298	298	template <class _Allocator>
299	299	requires uses_allocator<container_type, _Allocator>::value
300		_LIBCPP_HIDE_FROM_ABI
	300	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
301	301	flat_multiset(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
302	302	: flat_multiset(__il.begin(), __il.end(), __comp, __alloc) {}
303	303
304	304	template <class _Allocator>
305	305	requires uses_allocator<container_type, _Allocator>::value
306		_LIBCPP_HIDE_FROM_ABI flat_multiset(sorted_equivalent_t, initializer_list<value_type> __il, const _Allocator& __alloc)
	306	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
	307	flat_multiset(sorted_equivalent_t, initializer_list<value_type> __il, const _Allocator& __alloc)
307	308	: flat_multiset(sorted_equivalent, __il.begin(), __il.end(), __alloc) {}
308	309
309	310	template <class _Allocator>
310	311	requires uses_allocator<container_type, _Allocator>::value
311		_LIBCPP_HIDE_FROM_ABI flat_multiset(
	312	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset(
312	313	sorted_equivalent_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
313	314	: flat_multiset(sorted_equivalent, __il.begin(), __il.end(), __comp, __alloc) {}
314	315
315		_LIBCPP_HIDE_FROM_ABI flat_multiset& operator=(initializer_list<value_type> __il) {
	316	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset& operator=(initializer_list<value_type> __il) {
316	317	clear();
317	318	insert(__il);
318	319	return *this;
...	...	@@ -321,9 +322,9 @@ public:
321	322	// copy/move assignment are not specified in the spec (defaulted)
322	323	// but move assignment can potentially leave moved from object in an inconsistent
323	324	// state if an exception is thrown
324		_LIBCPP_HIDE_FROM_ABI flat_multiset& operator=(const flat_multiset&) = default;
	325	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset& operator=(const flat_multiset&) = default;
325	326
326		_LIBCPP_HIDE_FROM_ABI flat_multiset& operator=(flat_multiset&& __other) noexcept(
	327	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_multiset& operator=(flat_multiset&& __other) noexcept(
327	328	is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_Compare>) {
328	329	auto __clear_other_guard = std::__make_scope_guard([&]() noexcept { __other.clear() /* noexcept */; });
329	330	auto __clear_self_guard = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
...	...	@@ -334,30 +335,60 @@ public:
334	335	}
335	336
336	337	// iterators
337		_LIBCPP_HIDE_FROM_ABI iterator begin() noexcept { return iterator(std::as_const(__keys_).begin()); }
338		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept { return const_iterator(__keys_.begin()); }
339		_LIBCPP_HIDE_FROM_ABI iterator end() noexcept { return iterator(std::as_const(__keys_).end()); }
340		_LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept { return const_iterator(__keys_.end()); }
	338	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
	339	return iterator(std::as_const(__keys_).begin());
	340	}
	341	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
	342	return const_iterator(__keys_.begin());
	343	}
	344	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
	345	return iterator(std::as_const(__keys_).end());
	346	}
	347	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
	348	return const_iterator(__keys_.end());
	349	}
341	350
342		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
343		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
344		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
345		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
	351	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
	352	return reverse_iterator(end());
	353	}
	354	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
	355	return const_reverse_iterator(end());
	356	}
	357	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
	358	return reverse_iterator(begin());
	359	}
	360	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
	361	return const_reverse_iterator(begin());
	362	}
346	363
347		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
348		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
349		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
350		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
	364	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept {
	365	return begin();
	366	}
	367	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept {
	368	return end();
	369	}
	370	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
	371	return const_reverse_iterator(end());
	372	}
	373	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
	374	return const_reverse_iterator(begin());
	375	}
351	376
352	377	// capacity
353		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __keys_.empty(); }
354		_LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __keys_.size(); }
355		_LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept { return __keys_.max_size(); }
	378	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool empty() const noexcept {
	379	return __keys_.empty();
	380	}
	381	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept {
	382	return __keys_.size();
	383	}
	384	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept {
	385	return __keys_.max_size();
	386	}
356	387
357	388	// [flat.multiset.modifiers], modifiers
358	389	template <class... _Args>
359	390	requires is_constructible_v<value_type, _Args...>
360		_LIBCPP_HIDE_FROM_ABI iterator emplace(_Args&&... __args) {
	391	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace(_Args&&... __args) {
361	392	if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
362	393	return __emplace(std::forward<_Args>(__args)...);
363	394	} else {
...	...	@@ -367,7 +398,7 @@ public:
367	398
368	399	template <class... _Args>
369	400	requires is_constructible_v<value_type, _Args...>
370		_LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
	401	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
371	402	if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
372	403	return __emplace_hint(std::move(__hint), std::forward<_Args>(__args)...);
373	404	} else {
...	...	@@ -375,21 +406,23 @@ public:
375	406	}
376	407	}
377	408
378		_LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return emplace(__x); }
	409	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const value_type& __x) { return emplace(__x); }
379	410
380		_LIBCPP_HIDE_FROM_ABI iterator insert(value_type&& __x) { return emplace(std::move(__x)); }
	411	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(value_type&& __x) {
	412	return emplace(std::move(__x));
	413	}
381	414
382		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
	415	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, const value_type& __x) {
383	416	return emplace_hint(__hint, __x);
384	417	}
385	418
386		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
	419	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, value_type&& __x) {
387	420	return emplace_hint(__hint, std::move(__x));
388	421	}
389	422
390	423	template <class _InputIterator>
391	424	requires __has_input_iterator_category<_InputIterator>::value
392		_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
	425	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(_InputIterator __first, _InputIterator __last) {
393	426	if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
394	427	__reserve(__last - __first);
395	428	}
...	...	@@ -398,7 +431,8 @@ public:
398	431
399	432	template <class _InputIterator>
400	433	requires __has_input_iterator_category<_InputIterator>::value
401		_LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, _InputIterator __first, _InputIterator __last) {
	434	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	435	insert(sorted_equivalent_t, _InputIterator __first, _InputIterator __last) {
402	436	if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
403	437	__reserve(__last - __first);
404	438	}
...	...	@@ -407,7 +441,7 @@ public:
407	441	}
408	442
409	443	template <_ContainerCompatibleRange<value_type> _Range>
410		_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
	444	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(_Range&& __range) {
411	445	if constexpr (ranges::sized_range<_Range>) {
412	446	__reserve(ranges::size(__range));
413	447	}
...	...	@@ -415,26 +449,38 @@ public:
415	449	__append_sort_merge</WasSorted = / false>(std::forward<_Range>(__range));
416	450	}
417	451
418		_LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
	452	template <_ContainerCompatibleRange<value_type> _Range>
	453	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(sorted_equivalent_t, _Range&& __range) {
	454	if constexpr (ranges::sized_range<_Range>) {
	455	__reserve(ranges::size(__range));
	456	}
419	457
420		_LIBCPP_HIDE_FROM_ABI void insert(sorted_equivalent_t, initializer_list<value_type> __il) {
	458	__append_sort_merge</WasSorted = / true>(std::forward<_Range>(__range));
	459	}
	460
	461	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
	462	insert(__il.begin(), __il.end());
	463	}
	464
	465	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	466	insert(sorted_equivalent_t, initializer_list<value_type> __il) {
421	467	insert(sorted_equivalent, __il.begin(), __il.end());
422	468	}
423	469
424		_LIBCPP_HIDE_FROM_ABI container_type extract() && {
	470	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 container_type extract() && {
425	471	auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
426	472	auto __ret = std::move(__keys_);
427	473	return __ret;
428	474	}
429	475
430		_LIBCPP_HIDE_FROM_ABI void replace(container_type&& __keys) {
	476	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void replace(container_type&& __keys) {
431	477	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(ranges::is_sorted(__keys, __compare_), "Key container is not sorted");
432	478	auto __guard = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
433	479	__keys_ = std::move(__keys);
434	480	__guard.__complete();
435	481	}
436	482
437		_LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
	483	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(iterator __position) {
438	484	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
439	485	auto __key_iter = __keys_.erase(__position.__base());
440	486	__on_failure.__complete();
...	...	@@ -444,7 +490,7 @@ public:
444	490	// The following overload is the same as the iterator overload
445	491	// iterator erase(const_iterator __position);
446	492
447		_LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
	493	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(const key_type& __x) {
448	494	auto [__first, __last] = equal_range(__x);
449	495	auto __res = __last - __first;
450	496	erase(__first, __last);
...	...	@@ -454,149 +500,170 @@ public:
454	500	template <class _Kp>
455	501	requires(__is_transparent_v<_Compare> && !is_convertible_v<_Kp &&, iterator> &&
456	502	!is_convertible_v<_Kp &&, const_iterator>)
457		_LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
	503	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(_Kp&& __x) {
458	504	auto [__first, __last] = equal_range(__x);
459	505	auto __res = __last - __first;
460	506	erase(__first, __last);
461	507	return __res;
462	508	}
463	509
464		_LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
	510	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(const_iterator __first, const_iterator __last) {
465	511	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
466	512	auto __key_it = __keys_.erase(__first.__base(), __last.__base());
467	513	__on_failure.__complete();
468	514	return iterator(std::move(__key_it));
469	515	}
470	516
471		_LIBCPP_HIDE_FROM_ABI void swap(flat_multiset& __y) noexcept {
472		// warning: The spec has unconditional noexcept, which means that
473		// if any of the following functions throw an exception,
474		// std::terminate will be called
475		// This is discussed in P3567, which hasn't been voted on yet.
	517	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	518	swap(flat_multiset& __y) noexcept(is_nothrow_swappable_v<container_type> && is_nothrow_swappable_v<key_compare>) {
	519	auto __on_failure = std::__make_exception_guard([&]() noexcept {
	520	clear() /* noexcept */;
	521	__y.clear() /* noexcept */;
	522	});
476	523	ranges::swap(__compare_, __y.__compare_);
477	524	ranges::swap(__keys_, __y.__keys_);
	525	__on_failure.__complete();
478	526	}
479	527
480		_LIBCPP_HIDE_FROM_ABI void clear() noexcept { __keys_.clear(); }
	528	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept { __keys_.clear(); }
481	529
482	530	// observers
483		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
484		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __compare_; }
	531	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
	532	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const {
	533	return __compare_;
	534	}
485	535
486	536	// map operations
487		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
	537	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
	538	return __find_impl(*this, __x);
	539	}
488	540
489		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
	541	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
	542	return __find_impl(*this, __x);
	543	}
490	544
491	545	template <class _Kp>
492	546	requires __is_transparent_v<_Compare>
493		_LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
	547	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
494	548	return __find_impl(*this, __x);
495	549	}
496	550
497	551	template <class _Kp>
498	552	requires __is_transparent_v<_Compare>
499		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
	553	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
500	554	return __find_impl(*this, __x);
501	555	}
502	556
503		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const {
	557	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
504	558	auto [__first, __last] = equal_range(__x);
505	559	return __last - __first;
506	560	}
507	561
508	562	template <class _Kp>
509	563	requires __is_transparent_v<_Compare>
510		_LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
	564	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
511	565	auto [__first, __last] = equal_range(__x);
512	566	return __last - __first;
513	567	}
514	568
515		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
	569	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
	570	return find(__x) != end();
	571	}
516	572
517	573	template <class _Kp>
518	574	requires __is_transparent_v<_Compare>
519		_LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
	575	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
520	576	return find(__x) != end();
521	577	}
522	578
523		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) {
	579	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
524	580	const auto& __keys = __keys_;
525	581	return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
526	582	}
527	583
528		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
	584	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	585	lower_bound(const key_type& __x) const {
529	586	return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
530	587	}
531	588
532	589	template <class _Kp>
533	590	requires __is_transparent_v<_Compare>
534		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
	591	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
535	592	const auto& __keys = __keys_;
536	593	return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
537	594	}
538	595
539	596	template <class _Kp>
540	597	requires __is_transparent_v<_Compare>
541		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
	598	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
542	599	return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
543	600	}
544	601
545		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) {
	602	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
546	603	const auto& __keys = __keys_;
547	604	return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
548	605	}
549	606
550		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
	607	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	608	upper_bound(const key_type& __x) const {
551	609	return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
552	610	}
553	611
554	612	template <class _Kp>
555	613	requires __is_transparent_v<_Compare>
556		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
	614	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
557	615	const auto& __keys = __keys_;
558	616	return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
559	617	}
560	618
561	619	template <class _Kp>
562	620	requires __is_transparent_v<_Compare>
563		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
	621	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
564	622	return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
565	623	}
566	624
567		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
	625	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	626	equal_range(const key_type& __x) {
568	627	return __equal_range_impl(*this, __x);
569	628	}
570	629
571		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
	630	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	631	equal_range(const key_type& __x) const {
572	632	return __equal_range_impl(*this, __x);
573	633	}
574	634
575	635	template <class _Kp>
576	636	requires __is_transparent_v<_Compare>
577		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
	637	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	638	equal_range(const _Kp& __x) {
578	639	return __equal_range_impl(*this, __x);
579	640	}
580	641	template <class _Kp>
581	642	requires __is_transparent_v<_Compare>
582		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
	643	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	644	equal_range(const _Kp& __x) const {
583	645	return __equal_range_impl(*this, __x);
584	646	}
585	647
586		friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_multiset& __x, const flat_multiset& __y) {
	648	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
	649	operator==(const flat_multiset& __x, const flat_multiset& __y) {
587	650	return ranges::equal(__x, __y);
588	651	}
589	652
590		friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_multiset& __x, const flat_multiset& __y) {
	653	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 auto
	654	operator<=>(const flat_multiset& __x, const flat_multiset& __y) {
591	655	return std::lexicographical_compare_three_way(
592	656	__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
593	657	}
594	658
595		friend _LIBCPP_HIDE_FROM_ABI void swap(flat_multiset& __x, flat_multiset& __y) noexcept { __x.swap(__y); }
	659	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	660	swap(flat_multiset& __x, flat_multiset& __y) noexcept(noexcept(__x.swap(__y))) {
	661	__x.swap(__y);
	662	}
596	663
597	664	private:
598	665	template <bool _WasSorted, class... _Args>
599		_LIBCPP_HIDE_FROM_ABI void __append_sort_merge(_Args&&... __args) {
	666	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __append_sort_merge(_Args&&... __args) {
600	667	auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
601	668	size_type __old_size = size();
602	669	__flat_set_utils::__append(*this, std::forward<_Args>(__args)...);
...	...	@@ -604,20 +671,20 @@ private:
604	671	ranges::sort(__keys_.begin() + __old_size, __keys_.end(), __compare_);
605	672	} else {
606	673	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
607		ranges::is_sorted(__keys_ \| ranges::views::drop(__old_size)), "Key container is not sorted");
	674	ranges::is_sorted(__keys_ \| ranges::views::drop(__old_size), __compare_), "Key container is not sorted");
608	675	}
609	676	ranges::inplace_merge(__keys_.begin(), __keys_.begin() + __old_size, __keys_.end(), __compare_);
610	677	__on_failure.__complete();
611	678	}
612	679
613	680	template <class _Kp>
614		_LIBCPP_HIDE_FROM_ABI iterator __emplace(_Kp&& __key) {
	681	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator __emplace(_Kp&& __key) {
615	682	auto __it = upper_bound(__key);
616	683	return __flat_set_utils::__emplace_exact_pos(*this, __it, std::forward<_Kp>(__key));
617	684	}
618	685
619	686	template <class _Kp>
620		_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
	687	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
621	688	auto __prev_larger = __hint != cbegin() && __compare_(__key, *std::prev(__hint));
622	689	auto __next_smaller = __hint != cend() && __compare_(*__hint, __key);
623	690
...	...	@@ -649,7 +716,7 @@ private:
649	716	}
650	717
651	718	template <class _Self, class _Kp>
652		_LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
	719	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __find_impl(_Self&& __self, const _Kp& __key) {
653	720	auto __it = __self.lower_bound(__key);
654	721	auto __last = __self.end();
655	722	if (__it == __last \|\| __self.__compare_(__key, *__it)) {
...	...	@@ -659,29 +726,30 @@ private:
659	726	}
660	727
661	728	template <class _Self, class _Kp>
662		_LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
	729	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
663	730	using __iter = _If<is_const_v<__libcpp_remove_reference_t<_Self>>, const_iterator, iterator>;
664	731	auto [__key_first, __key_last] =
665	732	std::equal_range(__self.__keys_.begin(), __self.__keys_.end(), __key, __self.__compare_);
666	733	return std::make_pair(__iter(__key_first), __iter(__key_last));
667	734	}
668	735
669		_LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
	736	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __reserve(size_t __size) {
670	737	if constexpr (__container_traits<_KeyContainer>::__reservable) {
671	738	__keys_.reserve(__size);
672	739	}
673	740	}
674	741
675	742	template <class _Key2, class _Compare2, class _KeyContainer2, class _Predicate>
676		friend typename flat_multiset<_Key2, _Compare2, _KeyContainer2>::size_type
	743	friend typename flat_multiset<_Key2, _Compare2, _KeyContainer2>::size_type _LIBCPP_CONSTEXPR_SINCE_CXX26
677	744	erase_if(flat_multiset<_Key2, _Compare2, _KeyContainer2>&, _Predicate);
678	745
679	746	_KeyContainer __keys_;
680	747	_LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
681	748
682	749	struct __key_equiv {
683		_LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
684		_LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
	750	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __key_equiv(key_compare __c) : __comp_(__c) {}
	751	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
	752	operator()(const_reference __x, const_reference __y) const {
685	753	return !__comp_(std::get<0>(__x), std::get<0>(__y)) && !__comp_(std::get<0>(__y), std::get<0>(__x));
686	754	}
687	755	key_compare __comp_;
...	...	@@ -689,7 +757,7 @@ private:
689	757	};
690	758
691	759	template <class _KeyContainer, class _Compare = less<typename _KeyContainer::value_type>>
692		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	760	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
693	761	is_invocable_v<const _Compare&,
694	762	const typename _KeyContainer::value_type&,
695	763	const typename _KeyContainer::value_type&>)
...	...	@@ -697,12 +765,12 @@ flat_multiset(_KeyContainer, _Compare = _Compare())
697	765	-> flat_multiset<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
698	766
699	767	template <class _KeyContainer, class _Allocator>
700		requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator<_KeyContainer>::value)
	768	requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator_v<_KeyContainer>)
701	769	flat_multiset(_KeyContainer, _Allocator)
702	770	-> flat_multiset<typename _KeyContainer::value_type, less<typename _KeyContainer::value_type>, _KeyContainer>;
703	771
704	772	template <class _KeyContainer, class _Compare, class _Allocator>
705		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	773	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
706	774	uses_allocator_v<_KeyContainer, _Allocator> &&
707	775	is_invocable_v<const _Compare&,
708	776	const typename _KeyContainer::value_type&,
...	...	@@ -711,7 +779,7 @@ flat_multiset(_KeyContainer, _Compare, _Allocator)
711	779	-> flat_multiset<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
712	780
713	781	template <class _KeyContainer, class _Compare = less<typename _KeyContainer::value_type>>
714		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	782	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
715	783	is_invocable_v<const _Compare&,
716	784	const typename _KeyContainer::value_type&,
717	785	const typename _KeyContainer::value_type&>)
...	...	@@ -719,12 +787,12 @@ flat_multiset(sorted_equivalent_t, _KeyContainer, _Compare = _Compare())
719	787	-> flat_multiset<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
720	788
721	789	template <class _KeyContainer, class _Allocator>
722		requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator<_KeyContainer>::value)
	790	requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator_v<_KeyContainer>)
723	791	flat_multiset(sorted_equivalent_t, _KeyContainer, _Allocator)
724	792	-> flat_multiset<typename _KeyContainer::value_type, less<typename _KeyContainer::value_type>, _KeyContainer>;
725	793
726	794	template <class _KeyContainer, class _Compare, class _Allocator>
727		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	795	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
728	796	uses_allocator_v<_KeyContainer, _Allocator> &&
729	797	is_invocable_v<const _Compare&,
730	798	const typename _KeyContainer::value_type&,
...	...	@@ -732,37 +800,37 @@ template <class _KeyContainer, class _Compare, class _Allocator>
732	800	flat_multiset(sorted_equivalent_t, _KeyContainer, _Compare, _Allocator)
733	801	-> flat_multiset<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
734	802
735		template <class _InputIterator, class _Compare = less<__iter_value_type<_InputIterator>>>
736		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	803	template <class _InputIterator, class _Compare = less<__iterator_value_type<_InputIterator>>>
	804	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
737	805	flat_multiset(_InputIterator, _InputIterator, _Compare = _Compare())
738		-> flat_multiset<__iter_value_type<_InputIterator>, _Compare>;
	806	-> flat_multiset<__iterator_value_type<_InputIterator>, _Compare>;
739	807
740		template <class _InputIterator, class _Compare = less<__iter_value_type<_InputIterator>>>
741		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	808	template <class _InputIterator, class _Compare = less<__iterator_value_type<_InputIterator>>>
	809	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
742	810	flat_multiset(sorted_equivalent_t, _InputIterator, _InputIterator, _Compare = _Compare())
743		-> flat_multiset<__iter_value_type<_InputIterator>, _Compare>;
	811	-> flat_multiset<__iterator_value_type<_InputIterator>, _Compare>;
744	812
745	813	template <ranges::input_range _Range,
746	814	class _Compare = less<ranges::range_value_t<_Range>>,
747	815	class _Allocator = allocator<ranges::range_value_t<_Range>>,
748		class = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
	816	class = __enable_if_t<!__is_allocator_v<_Compare> && __is_allocator_v<_Allocator>>>
749	817	flat_multiset(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_multiset<
750	818	ranges::range_value_t<_Range>,
751	819	_Compare,
752	820	vector<ranges::range_value_t<_Range>, __allocator_traits_rebind_t<_Allocator, ranges::range_value_t<_Range>>>>;
753	821
754		template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
	822	template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator_v<_Allocator>>>
755	823	flat_multiset(from_range_t, _Range&&, _Allocator) -> flat_multiset<
756	824	ranges::range_value_t<_Range>,
757	825	less<ranges::range_value_t<_Range>>,
758	826	vector<ranges::range_value_t<_Range>, __allocator_traits_rebind_t<_Allocator, ranges::range_value_t<_Range>>>>;
759	827
760	828	template <class _Key, class _Compare = less<_Key>>
761		requires(!__is_allocator<_Compare>::value)
	829	requires(!__is_allocator_v<_Compare>)
762	830	flat_multiset(initializer_list<_Key>, _Compare = _Compare()) -> flat_multiset<_Key, _Compare>;
763	831
764	832	template <class _Key, class _Compare = less<_Key>>
765		requires(!__is_allocator<_Compare>::value)
	833	requires(!__is_allocator_v<_Compare>)
766	834	flat_multiset(sorted_equivalent_t, initializer_list<_Key>, _Compare = _Compare()) -> flat_multiset<_Key, _Compare>;
767	835
768	836	template <class _Key, class _Compare, class _KeyContainer, class _Allocator>
...	...	@@ -770,7 +838,7 @@ struct uses_allocator<flat_multiset<_Key, _Compare, _KeyContainer>, _Allocator>
770	838	: bool_constant<uses_allocator_v<_KeyContainer, _Allocator> > {};
771	839
772	840	template <class _Key, class _Compare, class _KeyContainer, class _Predicate>
773		_LIBCPP_HIDE_FROM_ABI typename flat_multiset<_Key, _Compare, _KeyContainer>::size_type
	841	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 typename flat_multiset<_Key, _Compare, _KeyContainer>::size_type
774	842	erase_if(flat_multiset<_Key, _Compare, _KeyContainer>& __flat_multiset, _Predicate __pred) {
775	843	auto __guard = std::__make_exception_guard([&] { __flat_multiset.clear(); });
776	844	auto __it =

lib/libcxx/include/__flat_set/flat_set.h+86-68

...	...	@@ -12,7 +12,6 @@
12	12
13	13	#include <__algorithm/lexicographical_compare_three_way.h>
14	14	#include <__algorithm/lower_bound.h>
15		#include <__algorithm/min.h>
16	15	#include <__algorithm/ranges_adjacent_find.h>
17	16	#include <__algorithm/ranges_equal.h>
18	17	#include <__algorithm/ranges_inplace_merge.h>
...	...	@@ -24,20 +23,16 @@
24	23	#include <__compare/synth_three_way.h>
25	24	#include <__concepts/swappable.h>
26	25	#include <__config>
27		#include <__cstddef/ptrdiff_t.h>
28	26	#include <__flat_map/sorted_unique.h>
29	27	#include <__flat_set/ra_iterator.h>
30	28	#include <__flat_set/utils.h>
31		#include <__functional/invoke.h>
32	29	#include <__functional/is_transparent.h>
33	30	#include <__functional/operations.h>
34	31	#include <__fwd/vector.h>
35	32	#include <__iterator/concepts.h>
36		#include <__iterator/distance.h>
37	33	#include <__iterator/iterator_traits.h>
38	34	#include <__iterator/next.h>
39	35	#include <__iterator/prev.h>
40		#include <__iterator/ranges_iterator_traits.h>
41	36	#include <__iterator/reverse_iterator.h>
42	37	#include <__memory/allocator_traits.h>
43	38	#include <__memory/uses_allocator.h>
...	...	@@ -47,10 +42,7 @@
47	42	#include <__ranges/container_compatible_range.h>
48	43	#include <__ranges/drop_view.h>
49	44	#include <__ranges/from_range.h>
50		#include <__ranges/ref_view.h>
51	45	#include <__ranges/size.h>
52		#include <__ranges/subrange.h>
53		#include <__type_traits/conjunction.h>
54	46	#include <__type_traits/container_traits.h>
55	47	#include <__type_traits/invoke.h>
56	48	#include <__type_traits/is_allocator.h>
...	...	@@ -347,38 +339,42 @@ public:
347	339	}
348	340
349	341	// iterators
350		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
	342	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
351	343	return iterator(std::as_const(__keys_).begin());
352	344	}
353		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
	345	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
354	346	return const_iterator(__keys_.begin());
355	347	}
356		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
	348	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
357	349	return iterator(std::as_const(__keys_).end());
358	350	}
359		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
	351	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
360	352	return const_iterator(__keys_.end());
361	353	}
362	354
363		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
	355	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
364	356	return reverse_iterator(end());
365	357	}
366		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
	358	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
367	359	return const_reverse_iterator(end());
368	360	}
369		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
	361	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
370	362	return reverse_iterator(begin());
371	363	}
372		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
	364	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
373	365	return const_reverse_iterator(begin());
374	366	}
375	367
376		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept { return begin(); }
377		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept { return end(); }
378		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
	368	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept {
	369	return begin();
	370	}
	371	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept {
	372	return end();
	373	}
	374	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
379	375	return const_reverse_iterator(end());
380	376	}
381		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
	377	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
382	378	return const_reverse_iterator(begin());
383	379	}
384	380
...	...	@@ -387,9 +383,13 @@ public:
387	383	return __keys_.empty();
388	384	}
389	385
390		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept { return __keys_.size(); }
	386	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept {
	387	return __keys_.size();
	388	}
391	389
392		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept { return __keys_.max_size(); }
	390	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept {
	391	return __keys_.max_size();
	392	}
393	393
394	394	// [flat.set.modifiers], modifiers
395	395	template <class... _Args>
...	...	@@ -466,6 +466,15 @@ public:
466	466	__append_sort_merge_unique</WasSorted = / false>(std::forward<_Range>(__range));
467	467	}
468	468
	469	template <_ContainerCompatibleRange<value_type> _Range>
	470	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(std::sorted_unique_t, _Range&& __range) {
	471	if constexpr (ranges::sized_range<_Range>) {
	472	__reserve(ranges::size(__range));
	473	}
	474
	475	__append_sort_merge_unique</WasSorted = / true>(std::forward<_Range>(__range));
	476	}
	477
469	478	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
470	479	insert(__il.begin(), __il.end());
471	480	}
...	...	@@ -474,7 +483,7 @@ public:
474	483	insert(sorted_unique, __il.begin(), __il.end());
475	484	}
476	485
477		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 container_type extract() && {
	486	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 container_type extract() && {
478	487	auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
479	488	auto __ret = std::move(__keys_);
480	489	return __ret;
...	...	@@ -524,123 +533,131 @@ public:
524	533	return iterator(std::move(__key_it));
525	534	}
526	535
527		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __y) noexcept {
528		// warning: The spec has unconditional noexcept, which means that
529		// if any of the following functions throw an exception,
530		// std::terminate will be called.
531		// This is discussed in P2767, which hasn't been voted on yet.
	536	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	537	swap(flat_set& __y) noexcept(is_nothrow_swappable_v<container_type> && is_nothrow_swappable_v<key_compare>) {
	538	auto __on_failure = std::__make_exception_guard([&]() noexcept {
	539	clear() /* noexcept */;
	540	__y.clear() /* noexcept */;
	541	});
532	542	ranges::swap(__compare_, __y.__compare_);
533	543	ranges::swap(__keys_, __y.__keys_);
	544	__on_failure.__complete();
534	545	}
535	546
536	547	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept { __keys_.clear(); }
537	548
538	549	// observers
539		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
540		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const { return __compare_; }
	550	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
	551	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const {
	552	return __compare_;
	553	}
541	554
542	555	// set operations
543		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
	556	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
544	557	return __find_impl(*this, __x);
545	558	}
546	559
547		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
	560	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
548	561	return __find_impl(*this, __x);
549	562	}
550	563
551	564	template <class _Kp>
552	565	requires __is_transparent_v<_Compare>
553		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
	566	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
554	567	return __find_impl(*this, __x);
555	568	}
556	569
557	570	template <class _Kp>
558	571	requires __is_transparent_v<_Compare>
559		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
	572	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
560	573	return __find_impl(*this, __x);
561	574	}
562	575
563		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
	576	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
564	577	return contains(__x) ? 1 : 0;
565	578	}
566	579
567	580	template <class _Kp>
568	581	requires __is_transparent_v<_Compare>
569		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
	582	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
570	583	return contains(__x) ? 1 : 0;
571	584	}
572	585
573		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
	586	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
574	587	return find(__x) != end();
575	588	}
576	589
577	590	template <class _Kp>
578	591	requires __is_transparent_v<_Compare>
579		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
	592	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
580	593	return find(__x) != end();
581	594	}
582	595
583		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
	596	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
584	597	const auto& __keys = __keys_;
585	598	return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
586	599	}
587	600
588		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const key_type& __x) const {
	601	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	602	lower_bound(const key_type& __x) const {
589	603	return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
590	604	}
591	605
592	606	template <class _Kp>
593	607	requires __is_transparent_v<_Compare>
594		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
	608	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
595	609	const auto& __keys = __keys_;
596	610	return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
597	611	}
598	612
599	613	template <class _Kp>
600	614	requires __is_transparent_v<_Compare>
601		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
	615	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
602	616	return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
603	617	}
604	618
605		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
	619	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
606	620	const auto& __keys = __keys_;
607	621	return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
608	622	}
609	623
610		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const key_type& __x) const {
	624	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator
	625	upper_bound(const key_type& __x) const {
611	626	return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
612	627	}
613	628
614	629	template <class _Kp>
615	630	requires __is_transparent_v<_Compare>
616		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
	631	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
617	632	const auto& __keys = __keys_;
618	633	return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
619	634	}
620	635
621	636	template <class _Kp>
622	637	requires __is_transparent_v<_Compare>
623		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
	638	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
624	639	return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
625	640	}
626	641
627		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const key_type& __x) {
	642	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	643	equal_range(const key_type& __x) {
628	644	return __equal_range_impl(*this, __x);
629	645	}
630	646
631		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	647	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
632	648	equal_range(const key_type& __x) const {
633	649	return __equal_range_impl(*this, __x);
634	650	}
635	651
636	652	template <class _Kp>
637	653	requires __is_transparent_v<_Compare>
638		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const _Kp& __x) {
	654	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator>
	655	equal_range(const _Kp& __x) {
639	656	return __equal_range_impl(*this, __x);
640	657	}
641	658	template <class _Kp>
642	659	requires __is_transparent_v<_Compare>
643		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
	660	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
644	661	equal_range(const _Kp& __x) const {
645	662	return __equal_range_impl(*this, __x);
646	663	}
...	...	@@ -655,13 +672,14 @@ public:
655	672	__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
656	673	}
657	674
658		friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __x, flat_set& __y) noexcept {
	675	friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
	676	swap(flat_set& __x, flat_set& __y) noexcept(noexcept(__x.swap(__y))) {
659	677	__x.swap(__y);
660	678	}
661	679
662	680	private:
663	681	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_sorted_and_unique(auto&& __key_container) const {
664		auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) { return !__compare_(__x, __y); };
	682	auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) -> bool { return !__compare_(__x, __y); };
665	683	return ranges::adjacent_find(__key_container, __greater_or_equal_to) == ranges::end(__key_container);
666	684	}
667	685
...	...	@@ -774,19 +792,19 @@ private:
774	792	};
775	793
776	794	template <class _KeyContainer, class _Compare = less<typename _KeyContainer::value_type>>
777		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	795	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
778	796	is_invocable_v<const _Compare&,
779	797	const typename _KeyContainer::value_type&,
780	798	const typename _KeyContainer::value_type&>)
781	799	flat_set(_KeyContainer, _Compare = _Compare()) -> flat_set<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
782	800
783	801	template <class _KeyContainer, class _Allocator>
784		requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator<_KeyContainer>::value)
	802	requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator_v<_KeyContainer>)
785	803	flat_set(_KeyContainer, _Allocator)
786	804	-> flat_set<typename _KeyContainer::value_type, less<typename _KeyContainer::value_type>, _KeyContainer>;
787	805
788	806	template <class _KeyContainer, class _Compare, class _Allocator>
789		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	807	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
790	808	uses_allocator_v<_KeyContainer, _Allocator> &&
791	809	is_invocable_v<const _Compare&,
792	810	const typename _KeyContainer::value_type&,
...	...	@@ -794,7 +812,7 @@ template <class _KeyContainer, class _Compare, class _Allocator>
794	812	flat_set(_KeyContainer, _Compare, _Allocator) -> flat_set<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
795	813
796	814	template <class _KeyContainer, class _Compare = less<typename _KeyContainer::value_type>>
797		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	815	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
798	816	is_invocable_v<const _Compare&,
799	817	const typename _KeyContainer::value_type&,
800	818	const typename _KeyContainer::value_type&>)
...	...	@@ -802,12 +820,12 @@ flat_set(sorted_unique_t, _KeyContainer, _Compare = _Compare())
802	820	-> flat_set<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
803	821
804	822	template <class _KeyContainer, class _Allocator>
805		requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator<_KeyContainer>::value)
	823	requires(uses_allocator_v<_KeyContainer, _Allocator> && !__is_allocator_v<_KeyContainer>)
806	824	flat_set(sorted_unique_t, _KeyContainer, _Allocator)
807	825	-> flat_set<typename _KeyContainer::value_type, less<typename _KeyContainer::value_type>, _KeyContainer>;
808	826
809	827	template <class _KeyContainer, class _Compare, class _Allocator>
810		requires(!__is_allocator<_Compare>::value && !__is_allocator<_KeyContainer>::value &&
	828	requires(!__is_allocator_v<_Compare> && !__is_allocator_v<_KeyContainer> &&
811	829	uses_allocator_v<_KeyContainer, _Allocator> &&
812	830	is_invocable_v<const _Compare&,
813	831	const typename _KeyContainer::value_type&,
...	...	@@ -815,37 +833,37 @@ template <class _KeyContainer, class _Compare, class _Allocator>
815	833	flat_set(sorted_unique_t, _KeyContainer, _Compare, _Allocator)
816	834	-> flat_set<typename _KeyContainer::value_type, _Compare, _KeyContainer>;
817	835
818		template <class _InputIterator, class _Compare = less<__iter_value_type<_InputIterator>>>
819		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	836	template <class _InputIterator, class _Compare = less<__iterator_value_type<_InputIterator>>>
	837	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
820	838	flat_set(_InputIterator, _InputIterator, _Compare = _Compare())
821		-> flat_set<__iter_value_type<_InputIterator>, _Compare>;
	839	-> flat_set<__iterator_value_type<_InputIterator>, _Compare>;
822	840
823		template <class _InputIterator, class _Compare = less<__iter_value_type<_InputIterator>>>
824		requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator<_Compare>::value)
	841	template <class _InputIterator, class _Compare = less<__iterator_value_type<_InputIterator>>>
	842	requires(__has_input_iterator_category<_InputIterator>::value && !__is_allocator_v<_Compare>)
825	843	flat_set(sorted_unique_t, _InputIterator, _InputIterator, _Compare = _Compare())
826		-> flat_set<__iter_value_type<_InputIterator>, _Compare>;
	844	-> flat_set<__iterator_value_type<_InputIterator>, _Compare>;
827	845
828	846	template <ranges::input_range _Range,
829	847	class _Compare = less<ranges::range_value_t<_Range>>,
830	848	class _Allocator = allocator<ranges::range_value_t<_Range>>,
831		class = __enable_if_t<!__is_allocator<_Compare>::value && __is_allocator<_Allocator>::value>>
	849	class = __enable_if_t<!__is_allocator_v<_Compare> && __is_allocator_v<_Allocator>>>
832	850	flat_set(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator()) -> flat_set<
833	851	ranges::range_value_t<_Range>,
834	852	_Compare,
835	853	vector<ranges::range_value_t<_Range>, __allocator_traits_rebind_t<_Allocator, ranges::range_value_t<_Range>>>>;
836	854
837		template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator<_Allocator>::value>>
	855	template <ranges::input_range _Range, class _Allocator, class = __enable_if_t<__is_allocator_v<_Allocator>>>
838	856	flat_set(from_range_t, _Range&&, _Allocator) -> flat_set<
839	857	ranges::range_value_t<_Range>,
840	858	less<ranges::range_value_t<_Range>>,
841	859	vector<ranges::range_value_t<_Range>, __allocator_traits_rebind_t<_Allocator, ranges::range_value_t<_Range>>>>;
842	860
843	861	template <class _Key, class _Compare = less<_Key>>
844		requires(!__is_allocator<_Compare>::value)
	862	requires(!__is_allocator_v<_Compare>)
845	863	flat_set(initializer_list<_Key>, _Compare = _Compare()) -> flat_set<_Key, _Compare>;
846	864
847	865	template <class _Key, class _Compare = less<_Key>>
848		requires(!__is_allocator<_Compare>::value)
	866	requires(!__is_allocator_v<_Compare>)
849	867	flat_set(sorted_unique_t, initializer_list<_Key>, _Compare = _Compare()) -> flat_set<_Key, _Compare>;
850	868
851	869	template <class _Key, class _Compare, class _KeyContainer, class _Allocator>

lib/libcxx/include/__format/concepts.h-14

...	...	@@ -15,12 +15,8 @@
15	15	#include <__config>
16	16	#include <__format/format_parse_context.h>
17	17	#include <__fwd/format.h>
18		#include <__fwd/tuple.h>
19		#include <__tuple/tuple_size.h>
20		#include <__type_traits/is_specialization.h>
21	18	#include <__type_traits/remove_const.h>
22	19	#include <__type_traits/remove_reference.h>
23		#include <__utility/pair.h>
24	20
25	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
26	22	# pragma GCC system_header
...	...	@@ -65,16 +61,6 @@ concept __formattable =
65	61	# if _LIBCPP_STD_VER >= 23
66	62	template <class _Tp, class _CharT>
67	63	concept formattable = __formattable<_Tp, _CharT>;
68
69		// [tuple.like] defines a tuple-like exposition only concept. This concept is
70		// not related to that. Therefore it uses a different name for the concept.
71		//
72		// TODO FMT Add a test to validate we fail when using that concept after P2165
73		// has been implemented.
74		template <class _Tp>
75		concept __fmt_pair_like =
76		__is_specialization_v<_Tp, pair> \|\| (__is_specialization_v<_Tp, tuple> && tuple_size_v<_Tp> == 2);
77
78	64	# endif // _LIBCPP_STD_VER >= 23
79	65	#endif // _LIBCPP_STD_VER >= 20
80	66

lib/libcxx/include/__format/extended_grapheme_cluster_table.h+3-2

...	...	@@ -61,7 +61,7 @@
61	61	#ifndef _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
62	62	#define _LIBCPP___FORMAT_EXTENDED_GRAPHEME_CLUSTER_TABLE_H
63	63
64		#include <__algorithm/ranges_upper_bound.h>
	64	#include <__algorithm/upper_bound.h>
65	65	#include <__config>
66	66	#include <__cstddef/ptrdiff_t.h>
67	67	#include <__iterator/access.h>
...	...	@@ -1647,7 +1647,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[1501] = {
1647	1647	// size. Then the upper bound for code point 3 will return the entry after
1648	1648	// 0x1810. After moving to the previous entry the algorithm arrives at the
1649	1649	// correct entry.
1650		ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) \| 0x7ffu) - __entries;
	1650	ptrdiff_t __i =
	1651	std::upper_bound(std::begin(__entries), std::end(__entries), (__code_point << 11) \| 0x7ffu) - __entries;
1651	1652	if (__i == 0)
1652	1653	return __property::__none;
1653	1654

lib/libcxx/include/__format/fmt_pair_like.h created+42

...	...	@@ -0,0 +1,42 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___FORMAT_FMT_PAIR_LIKE_H
	11	#define _LIBCPP___FORMAT_FMT_PAIR_LIKE_H
	12
	13	#include <__config>
	14	#include <__fwd/pair.h>
	15	#include <__fwd/tuple.h>
	16	#include <__tuple/tuple_size.h>
	17	#include <__type_traits/is_specialization.h>
	18
	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	20	# pragma GCC system_header
	21	#endif
	22
	23	_LIBCPP_BEGIN_NAMESPACE_STD
	24
	25	#if _LIBCPP_STD_VER >= 23
	26
	27	// [tuple.like] defines a tuple-like exposition only concept. This concept is not related to that. Therefore it uses a
	28	// different name for the concept.
	29	//
	30	// TODO FMT Add a test to validate we fail when using that concept after P2165 has been implemented.
	31
	32	// [format.range.fmtkind]/2.2.1 and [tab:formatter.range.type]:
	33	// "U is either a specialization of pair or a specialization of tuple such that tuple_size_v<U> is 2."
	34	template <class _Tp>
	35	concept __fmt_pair_like =
	36	__is_specialization_v<_Tp, pair> \|\| (__is_specialization_v<_Tp, tuple> && tuple_size_v<_Tp> == 2);
	37
	38	#endif // _LIBCPP_STD_VER >= 23
	39
	40	_LIBCPP_END_NAMESPACE_STD
	41
	42	#endif // _LIBCPP___FORMAT_FMT_PAIR_LIKE_H

lib/libcxx/include/__format/format_arg.h+7-10

...	...	@@ -149,7 +149,7 @@ _LIBCPP_HIDE_FROM_ABI decltype(auto) __visit_format_arg(_Visitor&& __vis, basic_
149	149	__libcpp_unreachable();
150	150	}
151	151
152		# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	152	# if _LIBCPP_STD_VER >= 26
153	153
154	154	template <class _Rp, class _Visitor, class _Context>
155	155	_LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
...	...	@@ -200,7 +200,7 @@ _LIBCPP_HIDE_FROM_ABI _Rp __visit_format_arg(_Visitor&& __vis, basic_format_arg<
200	200	__libcpp_unreachable();
201	201	}
202	202
203		# endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	203	# endif // _LIBCPP_STD_VER >= 26
204	204
205	205	/// Contains the values used in basic_format_arg.
206	206	///
...	...	@@ -285,7 +285,7 @@ public:
285	285
286	286	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const noexcept { return __type_ != __format::__arg_t::__none; }
287	287
288		# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	288	# if _LIBCPP_STD_VER >= 26
289	289
290	290	// This function is user facing, so it must wrap the non-standard types of
291	291	// the "variant" in a handle to stay conforming. See __arg_t for more details.
...	...	@@ -329,7 +329,7 @@ public:
329	329	}
330	330	}
331	331
332		# endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	332	# endif // _LIBCPP_STD_VER >= 26
333	333
334	334	private:
335	335	using char_type = typename _Context::char_type;
...	...	@@ -371,11 +371,8 @@ private:
371	371	// This function is user facing, so it must wrap the non-standard types of
372	372	// the "variant" in a handle to stay conforming. See __arg_t for more details.
373	373	template <class _Visitor, class _Context>
374		# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
375		_LIBCPP_DEPRECATED_IN_CXX26
376		# endif
377		_LIBCPP_HIDE_FROM_ABI decltype(auto)
378		visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
	374	_LIBCPP_DEPRECATED_IN_CXX26 _LIBCPP_HIDE_FROM_ABI decltype(auto)
	375	visit_format_arg(_Visitor&& __vis, basic_format_arg<_Context> __arg) {
379	376	switch (__arg.__type_) {
380	377	# if _LIBCPP_HAS_INT128
381	378	case __format::__arg_t::__i128: {
...	...	@@ -387,7 +384,7 @@ _LIBCPP_DEPRECATED_IN_CXX26
387	384	typename __basic_format_arg_value<_Context>::__handle __h{__arg.__value_.__u128_};
388	385	return std::invoke(std::forward<_Visitor>(__vis), typename basic_format_arg<_Context>::handle{__h});
389	386	}
390		# endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	387	# endif // _LIBCPP_HAS_INT128
391	388	default:
392	389	return std::__visit_format_arg(std::forward<_Visitor>(__vis), __arg);
393	390	}

lib/libcxx/include/__format/format_args.h+1-1

...	...	@@ -40,7 +40,7 @@ public:
40	40	}
41	41	}
42	42
43		_LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> get(size_t __id) const noexcept {
	43	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> get(size_t __id) const noexcept {
44	44	if (__id >= __size_)
45	45	return basic_format_arg<_Context>{};
46	46

lib/libcxx/include/__format/format_context.h+5-5

...	...	@@ -80,17 +80,17 @@ public:
80	80	template <class _Tp>
81	81	using formatter_type = formatter<_Tp, _CharT>;
82	82
83		_LIBCPP_HIDE_FROM_ABI basic_format_arg<basic_format_context> arg(size_t __id) const noexcept {
	83	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_format_arg<basic_format_context> arg(size_t __id) const noexcept {
84	84	return __args_.get(__id);
85	85	}
86	86	# if _LIBCPP_HAS_LOCALIZATION
87		_LIBCPP_HIDE_FROM_ABI std::locale locale() {
	87	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI std::locale locale() {
88	88	if (!__loc_)
89	89	__loc_ = std::locale{};
90	90	return *__loc_;
91	91	}
92	92	# endif
93		_LIBCPP_HIDE_FROM_ABI iterator out() { return std::move(__out_it_); }
	93	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI iterator out() { return std::move(__out_it_); }
94	94	_LIBCPP_HIDE_FROM_ABI void advance_to(iterator __it) { __out_it_ = std::move(__it); }
95	95
96	96	private:
...	...	@@ -175,13 +175,13 @@ public:
175	175	__format::__determine_arg_t<basic_format_context, decltype(__arg)>(),
176	176	__basic_format_arg_value<basic_format_context>(__arg)};
177	177	};
178		# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	178	# if _LIBCPP_STD_VER >= 26
179	179	return static_cast<_Context*>(__c)->arg(__id).visit(std::move(__visitor));
180	180	# else
181	181	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
182	182	return std::visit_format_arg(std::move(__visitor), static_cast<_Context*>(__c)->arg(__id));
183	183	_LIBCPP_SUPPRESS_DEPRECATED_POP
184		# endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	184	# endif // _LIBCPP_STD_VER >= 26
185	185	}) {
186	186	}
187	187

lib/libcxx/include/__format/format_parse_context.h+2-2

...	...	@@ -41,8 +41,8 @@ public:
41	41	basic_format_parse_context(const basic_format_parse_context&) = delete;
42	42	basic_format_parse_context& operator=(const basic_format_parse_context&) = delete;
43	43
44		_LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept { return __begin_; }
45		_LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { return __end_; }
	44	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept { return __begin_; }
	45	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { return __end_; }
46	46	_LIBCPP_HIDE_FROM_ABI constexpr void advance_to(const_iterator __it) { __begin_ = __it; }
47	47
48	48	_LIBCPP_HIDE_FROM_ABI constexpr size_t next_arg_id() {

lib/libcxx/include/__format/formatter_output.h+30-34

...	...	@@ -45,7 +45,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
45	45
46	46	namespace __formatter {
47	47
48		struct _LIBCPP_EXPORTED_FROM_ABI __padding_size_result {
	48	struct __padding_size_result {
49	49	size_t __before_;
50	50	size_t __after_;
51	51	};
...	...	@@ -151,45 +151,41 @@ _LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, _CharT __value)
151	151	}
152	152	}
153	153
154		# if _LIBCPP_HAS_UNICODE
155	154	template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
156		requires(same_as<_CharT, char>)
157	155	_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
158		std::size_t __bytes = std::countl_one(static_cast<unsigned char>(__value.__data[0]));
159		if (__bytes == 0)
160		return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
161
162		for (size_t __i = 0; __i < __n; ++__i)
163		__out_it = __formatter::__copy(
164		std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + __bytes, std::move(__out_it));
165		return __out_it;
166		}
167
	156	# if _LIBCPP_HAS_UNICODE
	157	if constexpr (same_as<_CharT, char>) {
	158	std::size_t __bytes = std::countl_one(static_cast<unsigned char>(__value.__data[0]));
	159	if (__bytes == 0)
	160	return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
	161
	162	for (size_t __i = 0; __i < __n; ++__i)
	163	__out_it = __formatter::__copy(
	164	std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + __bytes, std::move(__out_it));
	165	return __out_it;
168	166	# if _LIBCPP_HAS_WIDE_CHARACTERS
169		template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
170		requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 2)
171		_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
172		if (!__unicode::__is_high_surrogate(__value.__data[0]))
173		return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
174
175		for (size_t __i = 0; __i < __n; ++__i)
176		__out_it = __formatter::__copy(
177		std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + 2, std::move(__out_it));
178		return __out_it;
179		}
180
181		template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
182		requires(same_as<_CharT, wchar_t> && sizeof(wchar_t) == 4)
183		_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
184		return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
185		}
	167	} else if constexpr (same_as<_CharT, wchar_t>) {
	168	if constexpr (sizeof(wchar_t) == 2) {
	169	if (!__unicode::__is_high_surrogate(__value.__data[0]))
	170	return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
	171
	172	for (size_t __i = 0; __i < __n; ++__i)
	173	__out_it = __formatter::__copy(
	174	std::addressof(__value.__data[0]), std::addressof(__value.__data[0]) + 2, std::move(__out_it));
	175	return __out_it;
	176	} else if constexpr (sizeof(wchar_t) == 4) {
	177	return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
	178	} else {
	179	static_assert(false, "expected sizeof(wchar_t) to be 2 or 4");
	180	}
186	181	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
187		# else // _LIBCPP_HAS_UNICODE
188		template <__fmt_char_type _CharT, output_iterator<const _CharT&> _OutIt>
189		_LIBCPP_HIDE_FROM_ABI _OutIt __fill(_OutIt __out_it, size_t __n, __format_spec::__code_point<_CharT> __value) {
	182	} else {
	183	static_assert(false, "Unexpected CharT");
	184	}
	185	# else // _LIBCPP_HAS_UNICODE
190	186	return __formatter::__fill(std::move(__out_it), __n, __value.__data[0]);
	187	# endif // _LIBCPP_HAS_UNICODE
191	188	}
192		# endif // _LIBCPP_HAS_UNICODE
193	189
194	190	/// Writes the input to the output with the required padding.
195	191	///

lib/libcxx/include/__format/indic_conjunct_break_table.h+3-2

...	...	@@ -61,7 +61,7 @@
61	61	#ifndef _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
62	62	#define _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
63	63
64		#include <__algorithm/ranges_upper_bound.h>
	64	#include <__algorithm/upper_bound.h>
65	65	#include <__config>
66	66	#include <__cstddef/ptrdiff_t.h>
67	67	#include <__iterator/access.h>
...	...	@@ -531,7 +531,8 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[403] = {
531	531	// size. Then the upper bound for code point 3 will return the entry after
532	532	// 0x1810. After moving to the previous entry the algorithm arrives at the
533	533	// correct entry.
534		ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) \| 0x7ffu) - __entries;
	534	ptrdiff_t __i =
	535	std::upper_bound(std::begin(__entries), std::end(__entries), (__code_point << 11) \| 0x7ffu) - __entries;
535	536	if (__i == 0)
536	537	return __property::__none;
537	538

lib/libcxx/include/__format/range_default_formatter.h+2-41

...	...	@@ -16,10 +16,11 @@
16	16
17	17	#include <__algorithm/ranges_copy.h>
18	18	#include <__chrono/statically_widen.h>
19		#include <__concepts/same_as.h>
20	19	#include <__config>
21	20	#include <__format/concepts.h>
	21	#include <__format/fmt_pair_like.h>
22	22	#include <__format/formatter.h>
	23	#include <__format/range_format.h>
23	24	#include <__format/range_formatter.h>
24	25	#include <__iterator/back_insert_iterator.h>
25	26	#include <__ranges/concepts.h>
...	...	@@ -42,51 +43,11 @@ concept __const_formattable_range =
42	43	template <class _Rp, class _CharT>
43	44	using __fmt_maybe_const _LIBCPP_NODEBUG = conditional_t<__const_formattable_range<_Rp, _CharT>, const _Rp, _Rp>;
44	45
45		_LIBCPP_DIAGNOSTIC_PUSH
46		_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wshadow")
47		_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wshadow")
48		// This shadows map, set, and string.
49		enum class range_format { disabled, map, set, sequence, string, debug_string };
50		_LIBCPP_DIAGNOSTIC_POP
51
52	46	// There is no definition of this struct, it's purely intended to be used to
53	47	// generate diagnostics.
54	48	template <class _Rp>
55	49	struct __instantiated_the_primary_template_of_format_kind;
56	50
57		template <class _Rp>
58		constexpr range_format format_kind = [] {
59		// [format.range.fmtkind]/1
60		// A program that instantiates the primary template of format_kind is ill-formed.
61		static_assert(sizeof(_Rp) != sizeof(_Rp), "create a template specialization of format_kind for your type");
62		return range_format::disabled;
63		}();
64
65		template <ranges::input_range _Rp>
66		requires same_as<_Rp, remove_cvref_t<_Rp>>
67		inline constexpr range_format format_kind<_Rp> = [] {
68		// [format.range.fmtkind]/2
69
70		// 2.1 If same_as<remove_cvref_t<ranges::range_reference_t<R>>, R> is true,
71		// Otherwise format_kind<R> is range_format::disabled.
72		if constexpr (same_as<remove_cvref_t<ranges::range_reference_t<_Rp>>, _Rp>)
73		return range_format::disabled;
74		// 2.2 Otherwise, if the qualified-id R::key_type is valid and denotes a type:
75		else if constexpr (requires { typename _Rp::key_type; }) {
76		// 2.2.1 If the qualified-id R::mapped_type is valid and denotes a type ...
77		if constexpr (requires { typename _Rp::mapped_type; } &&
78		// 2.2.1 ... If either U is a specialization of pair or U is a specialization
79		// of tuple and tuple_size_v<U> == 2
80		__fmt_pair_like<remove_cvref_t<ranges::range_reference_t<_Rp>>>)
81		return range_format::map;
82		else
83		// 2.2.2 Otherwise format_kind<R> is range_format::set.
84		return range_format::set;
85		} else
86		// 2.3 Otherwise, format_kind<R> is range_format::sequence.
87		return range_format::sequence;
88		}();
89
90	51	template <range_format _Kp, ranges::input_range _Rp, class _CharT>
91	52	struct __range_default_formatter;
92	53

lib/libcxx/include/__format/range_format.h created+71

...	...	@@ -0,0 +1,71 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___FORMAT_RANGE_FORMAT_H
	11	#define _LIBCPP___FORMAT_RANGE_FORMAT_H
	12
	13	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	14	# pragma GCC system_header
	15	#endif
	16
	17	#include <__concepts/same_as.h>
	18	#include <__config>
	19	#include <__format/fmt_pair_like.h>
	20	#include <__ranges/concepts.h>
	21	#include <__type_traits/remove_cvref.h>
	22
	23	_LIBCPP_BEGIN_NAMESPACE_STD
	24
	25	#if _LIBCPP_STD_VER >= 23
	26
	27	_LIBCPP_DIAGNOSTIC_PUSH
	28	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wshadow")
	29	_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wshadow")
	30	// This shadows map, set, and string.
	31	enum class range_format { disabled, map, set, sequence, string, debug_string };
	32	_LIBCPP_DIAGNOSTIC_POP
	33
	34	template <class _Rp>
	35	constexpr range_format format_kind = [] {
	36	// [format.range.fmtkind]/1
	37	// A program that instantiates the primary template of format_kind is ill-formed.
	38	static_assert(sizeof(_Rp) != sizeof(_Rp), "create a template specialization of format_kind for your type");
	39	return range_format::disabled;
	40	}();
	41
	42	template <ranges::input_range _Rp>
	43	requires same_as<_Rp, remove_cvref_t<_Rp>>
	44	inline constexpr range_format format_kind<_Rp> = [] {
	45	// [format.range.fmtkind]/2
	46
	47	// 2.1 If same_as<remove_cvref_t<ranges::range_reference_t<R>>, R> is true,
	48	// Otherwise format_kind<R> is range_format::disabled.
	49	if constexpr (same_as<remove_cvref_t<ranges::range_reference_t<_Rp>>, _Rp>)
	50	return range_format::disabled;
	51	// 2.2 Otherwise, if the qualified-id R::key_type is valid and denotes a type:
	52	else if constexpr (requires { typename _Rp::key_type; }) {
	53	// 2.2.1 If the qualified-id R::mapped_type is valid and denotes a type ...
	54	if constexpr (requires { typename _Rp::mapped_type; } &&
	55	// 2.2.1 ... If either U is a specialization of pair or U is a specialization
	56	// of tuple and tuple_size_v<U> == 2
	57	__fmt_pair_like<remove_cvref_t<ranges::range_reference_t<_Rp>>>)
	58	return range_format::map;
	59	else
	60	// 2.2.2 Otherwise format_kind<R> is range_format::set.
	61	return range_format::set;
	62	} else
	63	// 2.3 Otherwise, format_kind<R> is range_format::sequence.
	64	return range_format::sequence;
	65	}();
	66
	67	#endif // _LIBCPP_STD_VER >= 23
	68
	69	_LIBCPP_END_NAMESPACE_STD
	70
	71	#endif

lib/libcxx/include/__format/range_formatter.h+1

...	...	@@ -20,6 +20,7 @@
20	20	#include <__config>
21	21	#include <__format/buffer.h>
22	22	#include <__format/concepts.h>
	23	#include <__format/fmt_pair_like.h>
23	24	#include <__format/format_context.h>
24	25	#include <__format/format_error.h>
25	26	#include <__format/formatter.h>

lib/libcxx/include/__format/width_estimation_table.h+4-2

...	...	@@ -61,9 +61,10 @@
61	61	#ifndef _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
62	62	#define _LIBCPP___FORMAT_WIDTH_ESTIMATION_TABLE_H
63	63
64		#include <__algorithm/ranges_upper_bound.h>
	64	#include <__algorithm/upper_bound.h>
65	65	#include <__config>
66	66	#include <__cstddef/ptrdiff_t.h>
	67	#include <__iterator/access.h>
67	68	#include <cstdint>
68	69
69	70	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -255,7 +256,8 @@ inline constexpr uint32_t __table_upper_bound = 0x0003fffd;
255	256	if (__code_point < (__entries[0] >> 14))
256	257	return 1;
257	258
258		ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 14) \| 0x3fffu) - __entries;
	259	ptrdiff_t __i =
	260	std::upper_bound(std::begin(__entries), std::end(__entries), (__code_point << 14) \| 0x3fffu) - __entries;
259	261	if (__i == 0)
260	262	return 1;
261	263

lib/libcxx/include/__functional/bind.h+10-15

...	...	@@ -81,17 +81,12 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& __mu(reference_w
81	81	return __t.get();
82	82	}
83	83
84		template <class _Ti, class... _Uj, size_t... _Indx>
85		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __invoke_result_t<_Ti&, _Uj...>
86		__mu_expand(_Ti& __ti, tuple<_Uj...>& __uj, __tuple_indices<_Indx...>) {
87		return __ti(std::forward<_Uj>(std::get<_Indx>(__uj))...);
88		}
89
90	84	template <class _Ti, class... _Uj, __enable_if_t<is_bind_expression<_Ti>::value, int> = 0>
91	85	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __invoke_result_t<_Ti&, _Uj...>
92	86	__mu(_Ti& __ti, tuple<_Uj...>& __uj) {
93		typedef typename __make_tuple_indices<sizeof...(_Uj)>::type __indices;
94		return std::__mu_expand(__ti, __uj, __indices());
	87	return [&]<size_t... _Indices>(__index_sequence<_Indices...>) -> __invoke_result_t<_Ti&, _Uj...> {
	88	return __ti(std::forward<_Uj>(std::get<_Indices>(__uj))...);
	89	}(__index_sequence_for<_Uj...>{});
95	90	}
96	91
97	92	template <bool _IsPh, class _Ti, class _Uj>
...	...	@@ -191,7 +186,7 @@ struct __bind_return<_Fp, const tuple<_BoundArgs...>, _TupleUj, true> {
191	186
192	187	template <class _Fp, class _BoundArgs, size_t... _Indx, class _Args>
193	188	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fp, _BoundArgs, _Args>::type
194		__apply_functor(_Fp& __f, _BoundArgs& __bound_args, __tuple_indices<_Indx...>, _Args&& __args) {
	189	__apply_functor(_Fp& __f, _BoundArgs& __bound_args, __index_sequence<_Indx...>, _Args&& __args) {
195	190	return std::__invoke(__f, std::__mu(std::get<_Indx>(__bound_args), __args)...);
196	191	}
197	192
...	...	@@ -205,8 +200,6 @@ private:
205	200	_Fd __f_;
206	201	_Td __bound_args_;
207	202
208		typedef typename __make_tuple_indices<sizeof...(_BoundArgs)>::type __indices;
209
210	203	public:
211	204	template <
212	205	class _Gp,
...	...	@@ -219,14 +212,16 @@ public:
219	212	template <class... _Args>
220	213	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __bind_return<_Fd, _Td, tuple<_Args&&...> >::type
221	214	operator()(_Args&&... __args) {
222		return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
	215	return std::__apply_functor(
	216	__f_, __bound_args_, __index_sequence_for<_BoundArgs...>(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
223	217	}
224	218
225	219	template <class... _Args>
226	220	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
227	221	typename __bind_return<const _Fd, const _Td, tuple<_Args&&...> >::type
228	222	operator()(_Args&&... __args) const {
229		return std::__apply_functor(__f_, __bound_args_, __indices(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
	223	return std::__apply_functor(
	224	__f_, __bound_args_, __index_sequence_for<_BoundArgs...>(), tuple<_Args&&...>(std::forward<_Args>(__args)...));
230	225	}
231	226	};
232	227
...	...	@@ -273,14 +268,14 @@ template <class _Rp, class _Fp, class... _BoundArgs>
273	268	struct is_bind_expression<__bind_r<_Rp, _Fp, _BoundArgs...> > : public true_type {};
274	269
275	270	template <class _Fp, class... _BoundArgs>
276		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind<_Fp, _BoundArgs...>
	271	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind<_Fp, _BoundArgs...>
277	272	bind(_Fp&& __f, _BoundArgs&&... __bound_args) {
278	273	typedef __bind<_Fp, _BoundArgs...> type;
279	274	return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...);
280	275	}
281	276
282	277	template <class _Rp, class _Fp, class... _BoundArgs>
283		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind_r<_Rp, _Fp, _BoundArgs...>
	278	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bind_r<_Rp, _Fp, _BoundArgs...>
284	279	bind(_Fp&& __f, _BoundArgs&&... __bound_args) {
285	280	typedef __bind_r<_Rp, _Fp, _BoundArgs...> type;
286	281	return type(std::forward<_Fp>(__f), std::forward<_BoundArgs>(__bound_args)...);

lib/libcxx/include/__functional/bind_back.h+1-1

...	...	@@ -64,7 +64,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto __bind_back(_Fn&& __f, _Args&&... __args) n
64	64
65	65	# if _LIBCPP_STD_VER >= 23
66	66	template <class _Fn, class... _Args>
67		_LIBCPP_HIDE_FROM_ABI constexpr auto bind_back(_Fn&& __f, _Args&&... __args) {
	67	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto bind_back(_Fn&& __f, _Args&&... __args) {
68	68	static_assert(is_constructible_v<decay_t<_Fn>, _Fn>, "bind_back requires decay_t<F> to be constructible from F");
69	69	static_assert(is_move_constructible_v<decay_t<_Fn>>, "bind_back requires decay_t<F> to be move constructible");
70	70	static_assert((is_constructible_v<decay_t<_Args>, _Args> && ...),

lib/libcxx/include/__functional/bind_front.h+1-1

...	...	@@ -43,7 +43,7 @@ struct __bind_front_t : __perfect_forward<__bind_front_op, _Fn, _BoundArgs...> {
43	43	template <class _Fn, class... _Args>
44	44	requires is_constructible_v<decay_t<_Fn>, _Fn> && is_move_constructible_v<decay_t<_Fn>> &&
45	45	(is_constructible_v<decay_t<_Args>, _Args> && ...) && (is_move_constructible_v<decay_t<_Args>> && ...)
46		_LIBCPP_HIDE_FROM_ABI constexpr auto bind_front(_Fn&& __f, _Args&&... __args) {
	46	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto bind_front(_Fn&& __f, _Args&&... __args) {
47	47	return __bind_front_t<decay_t<_Fn>, decay_t<_Args>...>(std::forward<_Fn>(__f), std::forward<_Args>(__args)...);
48	48	}
49	49

lib/libcxx/include/__functional/function.h+57-75

...	...	@@ -15,16 +15,14 @@
15	15	#include <__cstddef/nullptr_t.h>
16	16	#include <__exception/exception.h>
17	17	#include <__functional/binary_function.h>
18		#include <__functional/invoke.h>
19	18	#include <__functional/unary_function.h>
20	19	#include <__memory/addressof.h>
21	20	#include <__type_traits/aligned_storage.h>
22	21	#include <__type_traits/decay.h>
23		#include <__type_traits/is_core_convertible.h>
	22	#include <__type_traits/invoke.h>
24	23	#include <__type_traits/is_scalar.h>
25	24	#include <__type_traits/is_trivially_constructible.h>
26	25	#include <__type_traits/is_trivially_destructible.h>
27		#include <__type_traits/is_void.h>
28	26	#include <__type_traits/strip_signature.h>
29	27	#include <__utility/forward.h>
30	28	#include <__utility/move.h>
...	...	@@ -95,29 +93,29 @@ template <class _Rp, class _A1, class _A2>
95	93	struct __maybe_derive_from_binary_function<_Rp(_A1, _A2)> : public __binary_function<_A1, _A2, _Rp> {};
96	94
97	95	template <class _Fp>
98		_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp const&) {
99		return true;
	96	_LIBCPP_HIDE_FROM_ABI bool __is_null(_Fp const&) {
	97	return false;
100	98	}
101	99
102	100	template <class _Fp>
103		_LIBCPP_HIDE_FROM_ABI bool __not_null(_Fp* __ptr) {
104		return __ptr;
	101	_LIBCPP_HIDE_FROM_ABI bool __is_null(_Fp* __ptr) {
	102	return !__ptr;
105	103	}
106	104
107	105	template <class _Ret, class _Class>
108		_LIBCPP_HIDE_FROM_ABI bool __not_null(_Ret _Class::*__ptr) {
109		return __ptr;
	106	_LIBCPP_HIDE_FROM_ABI bool __is_null(_Ret _Class::* __ptr) {
	107	return !__ptr;
110	108	}
111	109
112	110	template <class _Fp>
113		_LIBCPP_HIDE_FROM_ABI bool __not_null(function<_Fp> const& __f) {
114		return !!__f;
	111	_LIBCPP_HIDE_FROM_ABI bool __is_null(function<_Fp> const& __f) {
	112	return !__f;
115	113	}
116	114
117	115	# if __has_extension(blocks)
118	116	template <class _Rp, class... _Args>
119		_LIBCPP_HIDE_FROM_ABI bool __not_null(_Rp (^__p)(_Args...)) {
120		return __p;
	117	_LIBCPP_HIDE_FROM_ABI bool __is_null(_Rp (^__p)(_Args...)) {
	118	return !__p;
121	119	}
122	120	# endif
123	121
...	...	@@ -206,12 +204,13 @@ public:
206	204	_LIBCPP_HIDE_FROM_ABI explicit __value_func(_Fp&& __f) : __f_(nullptr) {
207	205	typedef __function::__func<_Fp, _Rp(_ArgTypes...)> _Fun;
208	206
209		if (__function::__not_null(__f)) {
210		if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) {
211		__f_ = ::new (std::addressof(__buf_)) _Fun(std::move(__f));
212		} else {
213		__f_ = new _Fun(std::move(__f));
214		}
	207	if (__function::__is_null(__f))
	208	return;
	209
	210	if (sizeof(_Fun) <= sizeof(__buf_) && is_nothrow_copy_constructible<_Fp>::value) {
	211	__f_ = ::new (std::addressof(__buf_)) _Fun(std::move(__f));
	212	} else {
	213	__f_ = new _Fun(std::move(__f));
215	214	}
216	215	}
217	216
...	...	@@ -356,7 +355,31 @@ struct __policy {
356	355	// type.
357	356	template <typename _Fun>
358	357	_LIBCPP_HIDE_FROM_ABI static const __policy* __create() {
359		return __choose_policy<_Fun>(__use_small_storage<_Fun>());
	358	if constexpr (__use_small_storage<_Fun>::value) {
	359	static constexpr __policy __policy = {
	360	nullptr,
	361	nullptr,
	362	false,
	363	# if _LIBCPP_HAS_RTTI
	364	&typeid(_Fun)
	365	# else
	366	nullptr
	367	# endif
	368	};
	369	return &__policy;
	370	} else {
	371	static constexpr __policy __policy = {
	372	std::addressof(__large_clone<_Fun>),
	373	std::addressof(__large_destroy<_Fun>),
	374	false,
	375	# if _LIBCPP_HAS_RTTI
	376	&typeid(_Fun)
	377	# else
	378	nullptr
	379	# endif
	380	};
	381	return &__policy;
	382	}
360	383	}
361	384
362	385	_LIBCPP_HIDE_FROM_ABI static const __policy* __create_empty() {
...	...	@@ -384,36 +407,6 @@ private:
384	407	_LIBCPP_HIDE_FROM_ABI static void __large_destroy(void* __s) {
385	408	delete static_cast<_Fun*>(__s);
386	409	}
387
388		template <typename _Fun>
389		_LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ false_type) {
390		static constexpr __policy __policy = {
391		std::addressof(__large_clone<_Fun>),
392		std::addressof(__large_destroy<_Fun>),
393		false,
394		# if _LIBCPP_HAS_RTTI
395		&typeid(_Fun)
396		# else
397		nullptr
398		# endif
399		};
400		return &__policy;
401		}
402
403		template <typename _Fun>
404		_LIBCPP_HIDE_FROM_ABI static const __policy* __choose_policy(/* is_small = */ true_type) {
405		static constexpr __policy __policy = {
406		nullptr,
407		nullptr,
408		false,
409		# if _LIBCPP_HAS_RTTI
410		&typeid(_Fun)
411		# else
412		nullptr
413		# endif
414		};
415		return &__policy;
416		}
417	410	};
418	411
419	412	// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is
...	...	@@ -455,14 +448,15 @@ public:
455	448
456	449	template <class _Fp, __enable_if_t<!is_same<__decay_t<_Fp>, __policy_func>::value, int> = 0>
457	450	_LIBCPP_HIDE_FROM_ABI explicit __policy_func(_Fp&& __f) : __policy_(__policy::__create_empty()) {
458		if (__function::__not_null(__f)) {
459		__func_ = __call_func<_Fp>;
460		__policy_ = __policy::__create<_Fp>();
461		if (__use_small_storage<_Fp>()) {
462		::new ((void*)&__buf_.__small) _Fp(std::move(__f));
463		} else {
464		__buf_.__large = ::new _Fp(std::move(__f));
465		}
	451	if (__function::__is_null(__f))
	452	return;
	453
	454	__func_ = __call_func<_Fp>;
	455	__policy_ = __policy::__create<_Fp>();
	456	if (__use_small_storage<_Fp>()) {
	457	::new ((void*)&__buf_.__small) _Fp(std::move(__f));
	458	} else {
	459	__buf_.__large = ::new _Fp(std::move(__f));
466	460	}
467	461	}
468	462
...	...	@@ -615,21 +609,9 @@ class function<_Rp(_ArgTypes...)>
615	609
616	610	__func __f_;
617	611
618		template <class _Fp,
619		bool = _And<_IsNotSame<__remove_cvref_t<_Fp>, function>, __is_invocable<_Fp, _ArgTypes...> >::value>
620		struct __callable;
621		template <class _Fp>
622		struct __callable<_Fp, true> {
623		static const bool value =
624		is_void<_Rp>::value \|\| __is_core_convertible<__invoke_result_t<_Fp, _ArgTypes...>, _Rp>::value;
625		};
626		template <class _Fp>
627		struct __callable<_Fp, false> {
628		static const bool value = false;
629		};
630
631	612	template <class _Fp>
632		using _EnableIfLValueCallable _LIBCPP_NODEBUG = __enable_if_t<__callable<_Fp&>::value>;
	613	using _EnableIfLValueCallable _LIBCPP_NODEBUG = __enable_if_t<
	614	_And<_IsNotSame<__remove_cvref_t<_Fp>, function>, __is_invocable_r<_Rp, _Fp&, _ArgTypes...>>::value>;
633	615
634	616	public:
635	617	typedef _Rp result_type;
...	...	@@ -690,11 +672,11 @@ public:
690	672
691	673	# if _LIBCPP_HAS_RTTI
692	674	// function target access:
693		_LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT;
	675	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const std::type_info& target_type() const _NOEXCEPT;
694	676	template <typename _Tp>
695		_LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT;
	677	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Tp* target() _NOEXCEPT;
696	678	template <typename _Tp>
697		_LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT;
	679	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const _Tp* target() const _NOEXCEPT;
698	680	# endif // _LIBCPP_HAS_RTTI
699	681	};
700	682

lib/libcxx/include/__functional/hash.h+1-10

...	...	@@ -433,13 +433,10 @@ struct __hash_impl<long double> : __scalar_hash<long double> {
433	433	template <class _Tp>
434	434	struct hash : public __hash_impl<_Tp> {};
435	435
436		#if _LIBCPP_STD_VER >= 17
437
438	436	template <>
439	437	struct hash<nullptr_t> : public __unary_function<nullptr_t, size_t> {
440		_LIBCPP_HIDE_FROM_ABI size_t operator()(nullptr_t) const _NOEXCEPT { return 662607004ull; }
	438	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t operator()(nullptr_t) const _NOEXCEPT { return 662607004ull; }
441	439	};
442		#endif
443	440
444	441	#ifndef _LIBCPP_CXX03_LANG
445	442	template <class _Key, class _Hash>
...	...	@@ -452,18 +449,12 @@ template <class _Key, class _Hash = hash<_Key> >
452	449	using __has_enabled_hash _LIBCPP_NODEBUG =
453	450	integral_constant<bool, __check_hash_requirements<_Key, _Hash>::value && is_default_constructible<_Hash>::value >;
454	451
455		# if _LIBCPP_STD_VER >= 17
456	452	template <class _Type, class>
457	453	using __enable_hash_helper_imp _LIBCPP_NODEBUG = _Type;
458	454
459	455	template <class _Type, class... _Keys>
460	456	using __enable_hash_helper _LIBCPP_NODEBUG =
461	457	__enable_hash_helper_imp<_Type, __enable_if_t<__all<__has_enabled_hash<_Keys>::value...>::value> >;
462		# else
463		template <class _Type, class...>
464		using __enable_hash_helper _LIBCPP_NODEBUG = _Type;
465		# endif
466
467	458	#endif // !_LIBCPP_CXX03_LANG
468	459
469	460	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__functional/identity.h+1-1

...	...	@@ -44,7 +44,7 @@ struct __is_identity<reference_wrapper<const __identity> > : true_type {};
44	44
45	45	struct identity {
46	46	template <class _Tp>
47		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator()(_Tp&& __t) const noexcept {
	47	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator()(_LIBCPP_LIFETIMEBOUND _Tp&& __t) const noexcept {
48	48	return std::forward<_Tp>(__t);
49	49	}
50	50

lib/libcxx/include/__functional/is_transparent.h+8

...	...	@@ -29,6 +29,14 @@ inline const bool __is_transparent_v<_Tp, _Key, __void_t<typename _Tp::is_transp
29	29
30	30	#endif
31	31
	32	// Two types are considered transparently comparable if `comparator(key, arg)` is equivalent to `comparator(key,
	33	// <implicit cast to KeyT>(arg))`.
	34	//
	35	// This is different from `__is_transparent_v`, which is only a property of the comparator and doesn't provide
	36	// additional semantic guarantees.
	37	template <class _Comparator, class _KeyT, class _Arg, class = void>
	38	inline const bool __is_transparently_comparable_v = false;
	39
32	40	_LIBCPP_END_NAMESPACE_STD
33	41
34	42	#endif // _LIBCPP___FUNCTIONAL_IS_TRANSPARENT

lib/libcxx/include/__functional/mem_fn.h+2-1

...	...	@@ -43,7 +43,8 @@ public:
43	43	};
44	44
45	45	template <class _Rp, class _Tp>
46		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn<_Rp _Tp::> mem_fn(_Rp _Tp::__pm) _NOEXCEPT {
	46	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __mem_fn<_Rp _Tp::*>
	47	mem_fn(_Rp _Tp::* __pm) _NOEXCEPT {
47	48	return __mem_fn<_Rp _Tp::*>(__pm);
48	49	}
49	50

lib/libcxx/include/__functional/operations.h+18

...	...	@@ -15,7 +15,9 @@
15	15	#include <__functional/unary_function.h>
16	16	#include <__fwd/functional.h>
17	17	#include <__type_traits/desugars_to.h>
	18	#include <__type_traits/is_generic_transparent_comparator.h>
18	19	#include <__type_traits/is_integral.h>
	20	#include <__type_traits/make_transparent.h>
19	21	#include <__utility/forward.h>
20	22
21	23	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -377,6 +379,14 @@ struct less<void> {
377	379	typedef void is_transparent;
378	380	};
379	381
	382	template <class _Tp>
	383	struct __make_transparent<_Tp, less<_Tp> > {
	384	using type _LIBCPP_NODEBUG = less<>;
	385	};
	386
	387	template <>
	388	inline const bool __is_generic_transparent_comparator_v<less<>> = true;
	389
380	390	template <class _Tp, class _Up>
381	391	inline const bool __desugars_to_v<__less_tag, less<>, _Tp, _Up> = true;
382	392
...	...	@@ -466,6 +476,14 @@ struct greater<void> {
466	476
467	477	template <class _Tp, class _Up>
468	478	inline const bool __desugars_to_v<__greater_tag, greater<>, _Tp, _Up> = true;
	479
	480	template <class _Tp>
	481	struct __make_transparent<_Tp, greater<_Tp>> {
	482	using type _LIBCPP_NODEBUG = greater<>;
	483	};
	484
	485	template <>
	486	inline const bool __is_generic_transparent_comparator_v<greater<>> = true;
469	487	#endif
470	488
471	489	// Logical operations

lib/libcxx/include/__functional/ranges_operations.h+7

...	...	@@ -14,6 +14,7 @@
14	14	#include <__concepts/totally_ordered.h>
15	15	#include <__config>
16	16	#include <__type_traits/desugars_to.h>
	17	#include <__type_traits/is_generic_transparent_comparator.h>
17	18	#include <__utility/forward.h>
18	19
19	20	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -108,6 +109,12 @@ inline const bool __desugars_to_v<__less_tag, ranges::less, _Tp, _Up> = true;
108	109	template <class _Tp, class _Up>
109	110	inline const bool __desugars_to_v<__greater_tag, ranges::greater, _Tp, _Up> = true;
110	111
	112	template <>
	113	inline const bool __is_generic_transparent_comparator_v<ranges::less> = true;
	114
	115	template <>
	116	inline const bool __is_generic_transparent_comparator_v<ranges::greater> = true;
	117
111	118	#endif // _LIBCPP_STD_VER >= 20
112	119
113	120	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__functional/reference_wrapper.h+7-5

...	...	@@ -58,7 +58,7 @@ public:
58	58
59	59	// access
60	60	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator type&() const _NOEXCEPT { return *__f_; }
61		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 type& get() const _NOEXCEPT { return *__f_; }
	61	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 type& get() const _NOEXCEPT { return *__f_; }
62	62
63	63	// invoke
64	64	template <class... _ArgTypes>
...	...	@@ -128,23 +128,25 @@ reference_wrapper(_Tp&) -> reference_wrapper<_Tp>;
128	128	#endif
129	129
130	130	template <class _Tp>
131		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {
	131	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	132	_LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp> ref(_Tp& __t) _NOEXCEPT {
132	133	return reference_wrapper<_Tp>(__t);
133	134	}
134	135
135	136	template <class _Tp>
136		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp>
	137	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<_Tp>
137	138	ref(reference_wrapper<_Tp> __t) _NOEXCEPT {
138	139	return __t;
139	140	}
140	141
141	142	template <class _Tp>
142		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp> cref(const _Tp& __t) _NOEXCEPT {
	143	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp>
	144	cref(const _Tp& __t) _NOEXCEPT {
143	145	return reference_wrapper<const _Tp>(__t);
144	146	}
145	147
146	148	template <class _Tp>
147		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp>
	149	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper<const _Tp>
148	150	cref(reference_wrapper<_Tp> __t) _NOEXCEPT {
149	151	return __t;
150	152	}

lib/libcxx/include/__functional/weak_result_type.h+14-28

...	...	@@ -13,9 +13,9 @@
13	13	#include <__config>
14	14	#include <__functional/binary_function.h>
15	15	#include <__functional/unary_function.h>
16		#include <__type_traits/integral_constant.h>
17	16	#include <__type_traits/invoke.h>
18	17	#include <__type_traits/is_same.h>
	18	#include <__type_traits/void_t.h>
19	19	#include <__utility/declval.h>
20	20
21	21	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -24,50 +24,36 @@
24	24
25	25	_LIBCPP_BEGIN_NAMESPACE_STD
26	26
27		template <class _Tp>
28		struct __has_result_type {
29		private:
30		template <class _Up>
31		static false_type __test(...);
32		template <class _Up>
33		static true_type __test(typename _Up::result_type* = 0);
	27	template <class _Tp, class = void>
	28	inline const bool __has_result_type_v = false;
34	29
35		public:
36		static const bool value = decltype(__test<_Tp>(0))::value;
37		};
	30	template <class _Tp>
	31	inline const bool __has_result_type_v<_Tp, __void_t<typename _Tp::result_type*> > = true;
38	32
39	33	// __weak_result_type
40	34
41	35	template <class _Tp>
42	36	struct __derives_from_unary_function {
43	37	private:
44		struct __two {
45		char __lx;
46		char __lxx;
47		};
48		static __two __test(...);
	38	static void __find_base(...);
49	39	template <class _Ap, class _Rp>
50		static __unary_function<_Ap, _Rp> __test(const volatile __unary_function<_Ap, _Rp>*);
	40	static __unary_function<_Ap, _Rp> __find_base(const volatile __unary_function<_Ap, _Rp>*);
51	41
52	42	public:
53		static const bool value = !is_same<decltype(__test((_Tp*)0)), __two>::value;
54		typedef decltype(__test((_Tp*)0)) type;
	43	using type = decltype(__find_base(static_cast<_Tp*>(nullptr)));
	44	static const bool value = !is_same<type, void>::value;
55	45	};
56	46
57	47	template <class _Tp>
58	48	struct __derives_from_binary_function {
59	49	private:
60		struct __two {
61		char __lx;
62		char __lxx;
63		};
64		static __two __test(...);
	50	static void __find_base(...);
65	51	template <class _A1, class _A2, class _Rp>
66		static __binary_function<_A1, _A2, _Rp> __test(const volatile __binary_function<_A1, _A2, _Rp>*);
	52	static __binary_function<_A1, _A2, _Rp> __find_base(const volatile __binary_function<_A1, _A2, _Rp>*);
67	53
68	54	public:
69		static const bool value = !is_same<decltype(__test((_Tp*)0)), __two>::value;
70		typedef decltype(__test((_Tp*)0)) type;
	55	using type = decltype(__find_base(static_cast<_Tp*>(nullptr)));
	56	static const bool value = !is_same<type, void>::value;
71	57	};
72	58
73	59	template <class _Tp, bool = __derives_from_unary_function<_Tp>::value>
...	...	@@ -85,7 +71,7 @@ struct __maybe_derive_from_binary_function // bool is true
85	71	template <class _Tp>
86	72	struct __maybe_derive_from_binary_function<_Tp, false> {};
87	73
88		template <class _Tp, bool = __has_result_type<_Tp>::value>
	74	template <class _Tp, bool = __has_result_type_v<_Tp> >
89	75	struct __weak_result_type_imp // bool is true
90	76	: public __maybe_derive_from_unary_function<_Tp>,
91	77	public __maybe_derive_from_binary_function<_Tp> {

lib/libcxx/include/__fwd/ios.h+1-1

...	...	@@ -31,7 +31,7 @@ using wios = basic_ios<wchar_t>;
31	31	template <class _CharT, class _Traits>
32	32	class _LIBCPP_PREFERRED_NAME(ios) _LIBCPP_IF_WIDE_CHARACTERS(_LIBCPP_PREFERRED_NAME(wios)) basic_ios;
33	33
34		#if defined(_NEWLIB_VERSION)
	34	#if _LIBCPP_LIBC_NEWLIB
35	35	// On newlib, off_t is 'long int'
36	36	using streamoff = long int; // for char_traits in <string>
37	37	#else

lib/libcxx/include/__fwd/tuple.h+14

...	...	@@ -21,11 +21,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD
21	21	template <size_t, class>
22	22	struct tuple_element;
23	23
	24	template <size_t _Np, class _Tp>
	25	using __tuple_element_t _LIBCPP_NODEBUG = typename tuple_element<_Np, _Tp>::type;
	26
24	27	#ifndef _LIBCPP_CXX03_LANG
25	28
26	29	template <class...>
27	30	class tuple;
28	31
	32	template <class>
	33	inline const bool __is_tuple_v = false;
	34
	35	template <class... _Tp>
	36	inline const bool __is_tuple_v<tuple<_Tp...>> = true;
	37
	38	template <size_t _Ip, class... _Tp>
	39	struct tuple_element<_Ip, tuple<_Tp...> > {
	40	using type _LIBCPP_NODEBUG = __type_pack_element<_Ip, _Tp...>;
	41	};
	42
29	43	template <class>
30	44	struct tuple_size;
31	45

lib/libcxx/include/__hash_table+371-357

...	...	@@ -10,6 +10,7 @@
10	10	#ifndef _LIBCPP___HASH_TABLE
11	11	#define _LIBCPP___HASH_TABLE
12	12
	13	#include <__algorithm/fill_n.h>
13	14	#include <__algorithm/max.h>
14	15	#include <__algorithm/min.h>
15	16	#include <__assert>
...	...	@@ -28,7 +29,6 @@
28	29	#include <__memory/swap_allocator.h>
29	30	#include <__memory/unique_ptr.h>
30	31	#include <__new/launder.h>
31		#include <__type_traits/can_extract_key.h>
32	32	#include <__type_traits/copy_cvref.h>
33	33	#include <__type_traits/enable_if.h>
34	34	#include <__type_traits/invoke.h>
...	...	@@ -44,7 +44,9 @@
44	44	#include <__utility/forward.h>
45	45	#include <__utility/move.h>
46	46	#include <__utility/pair.h>
	47	#include <__utility/scope_guard.h>
47	48	#include <__utility/swap.h>
	49	#include <__utility/try_key_extraction.h>
48	50	#include <limits>
49	51
50	52	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -81,18 +83,6 @@ struct __hash_node_base {
81	83	typedef _NodePtr __node_pointer;
82	84	typedef __node_base_pointer __next_pointer;
83	85
84		// TODO(LLVM 22): Remove this check
85		#ifndef _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB
86		static_assert(sizeof(__node_base_pointer) == sizeof(__node_pointer) && _LIBCPP_ALIGNOF(__node_base_pointer) ==
87		_LIBCPP_ALIGNOF(__node_pointer),
88		"It looks like you are using std::__hash_table (an implementation detail for the unordered containers) "
89		"with a fancy pointer type that thas a different representation depending on whether it points to a "
90		"__hash_table base pointer or a __hash_table node pointer (both of which are implementation details of "
91		"the standard library). This means that your ABI is being broken between LLVM 19 and LLVM 20. If you "
92		"don't care about your ABI being broken, define the _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB macro to "
93		"silence this diagnostic.");
94		#endif
95
96	86	__next_pointer __next_;
97	87
98	88	_LIBCPP_HIDE_FROM_ABI __next_pointer __ptr() _NOEXCEPT {
...	...	@@ -122,6 +112,19 @@ struct __get_hash_node_value_type<__hash_value_type<_Key, _Tp> > {
122	112	template <class _Tp>
123	113	using __get_hash_node_value_type_t _LIBCPP_NODEBUG = typename __get_hash_node_value_type<_Tp>::type;
124	114
	115	template <class _Tp>
	116	struct __get_hash_node_key_type {
	117	using type _LIBCPP_NODEBUG = _Tp;
	118	};
	119
	120	template <class _Key, class _Tp>
	121	struct __get_hash_node_key_type<__hash_value_type<_Key, _Tp> > {
	122	using type _LIBCPP_NODEBUG = _Key;
	123	};
	124
	125	template <class _Tp>
	126	using __get_hash_node_key_type_t _LIBCPP_NODEBUG = typename __get_hash_node_key_type<_Tp>::type;
	127
125	128	template <class _Tp, class _VoidPtr>
126	129	struct __hash_node : public __hash_node_base< __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > > {
127	130	using __node_value_type _LIBCPP_NODEBUG = __get_hash_node_value_type_t<_Tp>;
...	...	@@ -152,7 +155,12 @@ public:
152	155	}
153	156	#endif
154	157
155		_LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {}
	158	template <class _Alloc, class... _Args>
	159	_LIBCPP_HIDE_FROM_ABI explicit __hash_node(size_t __hash, _Alloc& __na, _Args&&... __args)
	160	: _Base(nullptr), __hash_(__hash) {
	161	allocator_traits<_Alloc>::construct(__na, std::addressof(__get_value()), std::forward<_Args>(__args)...);
	162	}
	163
156	164	_LIBCPP_HIDE_FROM_ABI ~__hash_node() {}
157	165	};
158	166
...	...	@@ -182,85 +190,16 @@ class __hash_map_iterator;
182	190	template <class _HashIterator>
183	191	class __hash_map_const_iterator;
184	192
185		template <class _Tp>
186		struct __hash_key_value_types {
187		static_assert(!is_reference<_Tp>::value && !is_const<_Tp>::value, "");
188		typedef _Tp key_type;
189		typedef _Tp __node_value_type;
190		typedef _Tp __container_value_type;
191		static const bool __is_map = false;
192
193		_LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(_Tp const& __v) { return __v; }
194		_LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(__node_value_type const& __v) { return __v; }
195		_LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) { return std::addressof(__n); }
196		_LIBCPP_HIDE_FROM_ABI static __container_value_type&& __move(__node_value_type& __v) { return std::move(__v); }
197		};
198
199		template <class _Key, class _Tp>
200		struct __hash_key_value_types<__hash_value_type<_Key, _Tp> > {
201		typedef _Key key_type;
202		typedef _Tp mapped_type;
203		typedef __hash_value_type<_Key, _Tp> __node_value_type;
204		typedef pair<const _Key, _Tp> __container_value_type;
205		typedef __container_value_type __map_value_type;
206		static const bool __is_map = true;
207
208		_LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__container_value_type const& __v) { return __v.first; }
209
210		template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __node_value_type>::value, int> = 0>
211		_LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
212		return __t.__get_value();
213		}
214
215		template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __container_value_type>::value, int> = 0>
216		_LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
217		return __t;
218		}
219
220		_LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__container_value_type& __n) {
221		return std::addressof(__n);
222		}
223		_LIBCPP_HIDE_FROM_ABI static pair<key_type&&, mapped_type&&> __move(__node_value_type& __v) { return __v.__move(); }
224		};
225
226		template <class _Tp, class _AllocPtr, class _KVTypes = __hash_key_value_types<_Tp>, bool = _KVTypes::__is_map>
227		struct __hash_map_pointer_types {};
228
229		template <class _Tp, class _AllocPtr, class _KVTypes>
230		struct __hash_map_pointer_types<_Tp, _AllocPtr, _KVTypes, true> {
231		typedef typename _KVTypes::__map_value_type _Mv;
232		typedef __rebind_pointer_t<_AllocPtr, _Mv> __map_value_type_pointer;
233		typedef __rebind_pointer_t<_AllocPtr, const _Mv> __const_map_value_type_pointer;
234		};
235
236	193	template <class _NodePtr, class _NodeT = typename pointer_traits<_NodePtr>::element_type>
237	194	struct __hash_node_types;
238	195
239	196	template <class _NodePtr, class _Tp, class _VoidPtr>
240		struct __hash_node_types<_NodePtr, __hash_node<_Tp, _VoidPtr> >
241		: public __hash_key_value_types<_Tp>,
242		__hash_map_pointer_types<_Tp, _VoidPtr>
243
244		{
245		typedef __hash_key_value_types<_Tp> __base;
246
247		public:
248		typedef ptrdiff_t difference_type;
249		typedef size_t size_type;
250
251		typedef __rebind_pointer_t<_NodePtr, void> __void_pointer;
252
	197	struct __hash_node_types<_NodePtr, __hash_node<_Tp, _VoidPtr> > {
253	198	typedef typename pointer_traits<_NodePtr>::element_type __node_type;
254		typedef _NodePtr __node_pointer;
255	199
256		typedef __hash_node_base<__node_pointer> __node_base_type;
257		typedef __rebind_pointer_t<_NodePtr, __node_base_type> __node_base_pointer;
258
259		typedef typename __node_base_type::__next_pointer __next_pointer;
	200	typedef typename __hash_node_base<_NodePtr>::__next_pointer __next_pointer;
260	201
261	202	using __node_value_type _LIBCPP_NODEBUG = __get_hash_node_value_type_t<_Tp>;
262		typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer;
263		typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> __const_node_value_type_pointer;
264	203
265	204	private:
266	205	static_assert(!is_const<__node_type>::value, "_NodePtr should never be a pointer to const");
...	...	@@ -281,13 +220,6 @@ struct __hash_node_types_from_iterator<__hash_local_iterator<_NodePtr> > : __has
281	220	template <class _NodePtr>
282	221	struct __hash_node_types_from_iterator<__hash_const_local_iterator<_NodePtr> > : __hash_node_types<_NodePtr> {};
283	222
284		template <class _NodeValueTp, class _VoidPtr>
285		struct __make_hash_node_types {
286		typedef __hash_node<_NodeValueTp, _VoidPtr> _NodeTp;
287		typedef __rebind_pointer_t<_VoidPtr, _NodeTp> _NodePtr;
288		typedef __hash_node_types<_NodePtr> type;
289		};
290
291	223	template <class _NodePtr>
292	224	class __hash_iterator {
293	225	typedef __hash_node_types<_NodePtr> _NodeTypes;
...	...	@@ -299,9 +231,9 @@ class __hash_iterator {
299	231	public:
300	232	typedef forward_iterator_tag iterator_category;
301	233	typedef typename _NodeTypes::__node_value_type value_type;
302		typedef typename _NodeTypes::difference_type difference_type;
	234	using difference_type = ptrdiff_t;
303	235	typedef value_type& reference;
304		typedef typename _NodeTypes::__node_value_type_pointer pointer;
	236	using pointer = __rebind_pointer_t<_NodePtr, value_type>;
305	237
306	238	_LIBCPP_HIDE_FROM_ABI __hash_iterator() _NOEXCEPT : __node_(nullptr) {}
307	239
...	...	@@ -366,9 +298,9 @@ public:
366	298
367	299	typedef forward_iterator_tag iterator_category;
368	300	typedef typename _NodeTypes::__node_value_type value_type;
369		typedef typename _NodeTypes::difference_type difference_type;
	301	using difference_type = ptrdiff_t;
370	302	typedef const value_type& reference;
371		typedef typename _NodeTypes::__const_node_value_type_pointer pointer;
	303	using pointer = __rebind_pointer_t<_NodePtr, const value_type>;
372	304
373	305	_LIBCPP_HIDE_FROM_ABI __hash_const_iterator() _NOEXCEPT : __node_(nullptr) {}
374	306
...	...	@@ -431,9 +363,9 @@ class __hash_local_iterator {
431	363	public:
432	364	typedef forward_iterator_tag iterator_category;
433	365	typedef typename _NodeTypes::__node_value_type value_type;
434		typedef typename _NodeTypes::difference_type difference_type;
	366	using difference_type = ptrdiff_t;
435	367	typedef value_type& reference;
436		typedef typename _NodeTypes::__node_value_type_pointer pointer;
	368	using pointer = __rebind_pointer_t<_NodePtr, value_type>;
437	369
438	370	_LIBCPP_HIDE_FROM_ABI __hash_local_iterator() _NOEXCEPT : __node_(nullptr) {}
439	371
...	...	@@ -509,9 +441,9 @@ public:
509	441
510	442	typedef forward_iterator_tag iterator_category;
511	443	typedef typename _NodeTypes::__node_value_type value_type;
512		typedef typename _NodeTypes::difference_type difference_type;
	444	using difference_type = ptrdiff_t;
513	445	typedef const value_type& reference;
514		typedef typename _NodeTypes::__const_node_value_type_pointer pointer;
	446	using pointer = __rebind_pointer_t<_ConstNodePtr, const value_type>;
515	447
516	448	_LIBCPP_HIDE_FROM_ABI __hash_const_local_iterator() _NOEXCEPT : __node_(nullptr) {}
517	449
...	...	@@ -617,8 +549,6 @@ public:
617	549	typedef typename __alloc_traits::pointer pointer;
618	550
619	551	private:
620		typedef __hash_node_types<pointer> _NodeTypes;
621
622	552	allocator_type& __na_;
623	553
624	554	public:
...	...	@@ -633,7 +563,7 @@ public:
633	563
634	564	_LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
635	565	if (__value_constructed) {
636		__alloc_traits::destroy(__na_, _NodeTypes::__get_ptr(__p->__get_value()));
	566	__alloc_traits::destroy(__na_, std::addressof(__p->__get_value()));
637	567	std::__destroy_at(std::addressof(*__p));
638	568	}
639	569	if (__p)
...	...	@@ -684,18 +614,16 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
684	614	class __hash_table {
685	615	public:
686	616	using value_type = __get_hash_node_value_type_t<_Tp>;
	617	using key_type = __get_hash_node_key_type_t<_Tp>;
	618
687	619	typedef _Hash hasher;
688	620	typedef _Equal key_equal;
689	621	typedef _Alloc allocator_type;
690	622
691	623	private:
692	624	typedef allocator_traits<allocator_type> __alloc_traits;
693		typedef typename __make_hash_node_types<_Tp, typename __alloc_traits::void_pointer>::type _NodeTypes;
694	625
695	626	public:
696		typedef typename _NodeTypes::__node_value_type __node_value_type;
697		typedef typename _NodeTypes::__container_value_type __container_value_type;
698		typedef typename _NodeTypes::key_type key_type;
699	627	typedef value_type& reference;
700	628	typedef const value_type& const_reference;
701	629	typedef typename __alloc_traits::pointer pointer;
...	...	@@ -703,22 +631,23 @@ public:
703	631	#ifndef _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE
704	632	typedef typename __alloc_traits::size_type size_type;
705	633	#else
706		typedef typename _NodeTypes::size_type size_type;
	634	using size_type = size_t;
707	635	#endif
708		typedef typename _NodeTypes::difference_type difference_type;
	636	using difference_type = ptrdiff_t;
709	637
710	638	public:
711	639	// Create __node
712	640
713		typedef typename _NodeTypes::__node_type __node;
714		typedef __rebind_alloc<__alloc_traits, __node> __node_allocator;
715		typedef allocator_traits<__node_allocator> __node_traits;
716		typedef typename _NodeTypes::__void_pointer __void_pointer;
717		typedef typename _NodeTypes::__node_pointer __node_pointer;
718		typedef typename _NodeTypes::__node_pointer __node_const_pointer;
719		typedef typename _NodeTypes::__node_base_type __first_node;
720		typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
721		typedef typename _NodeTypes::__next_pointer __next_pointer;
	641	using __void_pointer _LIBCPP_NODEBUG = typename __alloc_traits::void_pointer;
	642
	643	using __node _LIBCPP_NODEBUG = __hash_node<_Tp, __void_pointer>;
	644	using __node_allocator _LIBCPP_NODEBUG = __rebind_alloc<__alloc_traits, __node>;
	645	using __node_traits _LIBCPP_NODEBUG = allocator_traits<__node_allocator>;
	646	using __node_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<__void_pointer, __node>;
	647
	648	using __first_node _LIBCPP_NODEBUG = __hash_node_base<__node_pointer>;
	649	using __node_base_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<__void_pointer, __first_node>;
	650	using __next_pointer _LIBCPP_NODEBUG = __node_base_pointer;
722	651
723	652	private:
724	653	// check for sane allocator pointer rebinding semantics. Rebinding the
...	...	@@ -747,6 +676,38 @@ private:
747	676
748	677	_LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __size_; }
749	678
	679	_LIBCPP_HIDE_FROM_ABI void
	680	__copy_construct(__next_pointer __other_iter, __next_pointer __own_iter, size_t __current_chash) {
	681	auto __bucket_count = bucket_count();
	682
	683	for (; __other_iter; __other_iter = __other_iter->__next_) {
	684	__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
	685
	686	size_t __new_chash = std::__constrain_hash(__new_node->__hash(), __bucket_count);
	687	if (__new_chash != __current_chash) {
	688	__bucket_list_[__new_chash] = __own_iter;
	689	__current_chash = __new_chash;
	690	}
	691
	692	__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
	693	__own_iter = __own_iter->__next_;
	694	}
	695	}
	696
	697	_LIBCPP_HIDE_FROM_ABI void __copy_construct(__next_pointer __other_iter) {
	698	__next_pointer __own_iter = __first_node_.__ptr();
	699	{
	700	__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
	701	__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
	702	}
	703
	704	size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), bucket_count());
	705	__bucket_list_[__current_chash] = __own_iter;
	706	__other_iter = __other_iter->__next_;
	707	__own_iter = __own_iter->__next_;
	708	__copy_construct(__other_iter, __own_iter, __current_chash);
	709	}
	710
750	711	public:
751	712	_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
752	713
...	...	@@ -811,40 +772,66 @@ public:
811	772	_LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
812	773	_LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(const_iterator __p, __node_pointer __nd);
813	774
814		template <class _Key, class... _Args>
815		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_key_args(_Key const& __k, _Args&&... __args);
816
817		template <class... _Args>
818		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_impl(_Args&&... __args);
819
820		template <class _Pp>
821		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Pp&& __x) {
822		return __emplace_unique_extract_key(std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
823		}
824
825		template <class _First,
826		class _Second,
827		__enable_if_t<__can_extract_map_key<_First, key_type, __container_value_type>::value, int> = 0>
828		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_First&& __f, _Second&& __s) {
829		return __emplace_unique_key_args(__f, std::forward<_First>(__f), std::forward<_Second>(__s));
830		}
831
832	775	template <class... _Args>
833	776	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Args&&... __args) {
834		return __emplace_unique_impl(std::forward<_Args>(__args)...);
835		}
836
837		template <class _Pp>
838		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) {
839		return __emplace_unique_impl(std::forward<_Pp>(__x));
840		}
841		template <class _Pp>
842		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) {
843		return __emplace_unique_key_args(__x, std::forward<_Pp>(__x));
844		}
845		template <class _Pp>
846		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) {
847		return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x));
	777	return std::__try_key_extraction<key_type>(
	778	[this](const key_type& __key, _Args&&... __args2) {
	779	size_t __hash = hash_function()(__key);
	780	size_type __bc = bucket_count();
	781	bool __inserted = false;
	782	__next_pointer __nd;
	783	size_t __chash;
	784	if (__bc != 0) {
	785	__chash = std::__constrain_hash(__hash, __bc);
	786	__nd = __bucket_list_[__chash];
	787	if (__nd != nullptr) {
	788	for (__nd = __nd->__next_;
	789	__nd != nullptr &&
	790	(__nd->__hash() == __hash \|\| std::__constrain_hash(__nd->__hash(), __bc) == __chash);
	791	__nd = __nd->__next_) {
	792	if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __key))
	793	goto __done;
	794	}
	795	}
	796	}
	797	{
	798	__node_holder __h = __construct_node_hash(__hash, std::forward<_Args>(__args2)...);
	799	if (size() + 1 > __bc * max_load_factor()) {
	800	__rehash_unique(std::max<size_type>(2 * __bc + !std::__is_hash_power2(__bc),
	801	size_type(__math::ceil(float(size() + 1) / max_load_factor()))));
	802	__bc = bucket_count();
	803	__chash = std::__constrain_hash(__hash, __bc);
	804	}
	805	// insert_after __bucket_list_[__chash], or __first_node if bucket is null
	806	__next_pointer __pn = __bucket_list_[__chash];
	807	if (__pn == nullptr) {
	808	__pn = __first_node_.__ptr();
	809	__h->__next_ = __pn->__next_;
	810	__pn->__next_ = __h.get()->__ptr();
	811	// fix up __bucket_list_
	812	__bucket_list_[__chash] = __pn;
	813	if (__h->__next_ != nullptr)
	814	__bucket_list_[std::__constrain_hash(__h->__next_->__hash(), __bc)] = __h.get()->__ptr();
	815	} else {
	816	__h->__next_ = __pn->__next_;
	817	__pn->__next_ = static_cast<__next_pointer>(__h.get());
	818	}
	819	__nd = static_cast<__next_pointer>(__h.release());
	820	// increment size
	821	++size();
	822	__inserted = true;
	823	}
	824	__done:
	825	return pair<iterator, bool>(iterator(__nd), __inserted);
	826	},
	827	[this](_Args&&... __args2) {
	828	__node_holder __h = __construct_node(std::forward<_Args>(__args2)...);
	829	pair<iterator, bool> __r = __node_insert_unique(__h.get());
	830	if (__r.second)
	831	__h.release();
	832	return __r;
	833	},
	834	std::forward<_Args>(__args)...);
848	835	}
849	836
850	837	template <class... _Args>
...	...	@@ -854,9 +841,7 @@ public:
854	841
855	842	template <class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
856	843	_LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(value_type&& __value) {
857		using __key_type = typename _NodeTypes::key_type;
858
859		__node_holder __h = __construct_node(const_cast<__key_type&&>(__value.first), std::move(__value.second));
	844	__node_holder __h = __construct_node(const_cast<key_type&&>(__value.first), std::move(__value.second));
860	845	__node_insert_unique(__h.get());
861	846	__h.release();
862	847	}
...	...	@@ -870,9 +855,7 @@ public:
870	855
871	856	template <class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
872	857	_LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(value_type&& __value) {
873		using __key_type = typename _NodeTypes::key_type;
874
875		__node_holder __h = __construct_node(const_cast<__key_type&&>(__value.first), std::move(__value.second));
	858	__node_holder __h = __construct_node(const_cast<key_type&&>(__value.first), std::move(__value.second));
876	859	__node_insert_multi(__h.get());
877	860	__h.release();
878	861	}
...	...	@@ -1017,8 +1000,8 @@ private:
1017	1000	template <class... _Args>
1018	1001	_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args);
1019	1002
1020		template <class _First, class... _Rest>
1021		_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest);
	1003	template <class... _Args>
	1004	_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_hash(size_t __hash, _Args&&... __args);
1022	1005
1023	1006	_LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __hash_table& __u) {
1024	1007	__copy_assign_alloc(__u, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
...	...	@@ -1042,17 +1025,29 @@ private:
1042	1025	}
1043	1026	_LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__hash_table&, false_type) _NOEXCEPT {}
1044	1027
1045		_LIBCPP_HIDE_FROM_ABI void __deallocate_node(__next_pointer __np) _NOEXCEPT;
	1028	_LIBCPP_HIDE_FROM_ABI void __deallocate_node(__node_pointer __nd) _NOEXCEPT {
	1029	auto& __alloc = __node_alloc();
	1030	__node_traits::destroy(__alloc, std::addressof(__nd->__get_value()));
	1031	std::__destroy_at(std::__to_address(__nd));
	1032	__node_traits::deallocate(__alloc, __nd, 1);
	1033	}
	1034
	1035	_LIBCPP_HIDE_FROM_ABI void __deallocate_node_list(__next_pointer __np) _NOEXCEPT {
	1036	while (__np != nullptr) {
	1037	__next_pointer __next = __np->__next_;
	1038	__deallocate_node(__np->__upcast());
	1039	__np = __next;
	1040	}
	1041	}
	1042
1046	1043	_LIBCPP_HIDE_FROM_ABI __next_pointer __detach() _NOEXCEPT;
1047	1044
1048	1045	template <class _From, class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
1049	1046	_LIBCPP_HIDE_FROM_ABI void __assign_value(__get_hash_node_value_type_t<_Tp>& __lhs, _From&& __rhs) {
1050		using __key_type = typename _NodeTypes::key_type;
1051
1052	1047	// This is technically UB, since the object was constructed as `const`.
1053	1048	// Clang doesn't optimize on this currently though.
1054		const_cast<__key_type&>(__lhs.first) = const_cast<__copy_cvref_t<_From, __key_type>&&>(__rhs.first);
1055		__lhs.second = std::forward<_From>(__rhs).second;
	1049	const_cast<key_type&>(__lhs.first) = const_cast<__copy_cvref_t<_From, key_type>&&>(__rhs.first);
	1050	__lhs.second = std::forward<_From>(__rhs).second;
1056	1051	}
1057	1052
1058	1053	template <class _From, class _ValueT = _Tp, __enable_if_t<!__is_hash_value_type<_ValueT>::value, int> = 0>
...	...	@@ -1101,16 +1096,29 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a
1101	1096	__max_load_factor_(1.0f) {}
1102	1097
1103	1098	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1104		__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u)
	1099	__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __other)
1105	1100	: __bucket_list_(nullptr,
1106		__bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction(
1107		__u.__bucket_list_.get_deleter().__alloc()),
	1101	__bucket_list_deleter(__pointer_alloc_traits::select_on_container_copy_construction(
	1102	__other.__bucket_list_.get_deleter().__alloc()),
1108	1103	0)),
1109		__node_alloc_(allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())),
	1104	__node_alloc_(__node_traits::select_on_container_copy_construction(__other.__node_alloc())),
1110	1105	__size_(0),
1111		__hasher_(__u.hash_function()),
1112		__max_load_factor_(__u.__max_load_factor_),
1113		__key_eq_(__u.__key_eq_) {}
	1106	__hasher_(__other.hash_function()),
	1107	__max_load_factor_(__other.__max_load_factor_),
	1108	__key_eq_(__other.__key_eq_) {
	1109	if (__other.size() == 0)
	1110	return;
	1111
	1112	auto& __bucket_list_del = __bucket_list_.get_deleter();
	1113	auto __bucket_count = __other.bucket_count();
	1114	__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
	1115	__bucket_list_del.size() = __bucket_count;
	1116
	1117	std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
	1118
	1119	__copy_construct(__other.__first_node_.__next_);
	1120	__size_ = __other.size();
	1121	}
1114	1122
1115	1123	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1116	1124	__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a)
...	...	@@ -1169,7 +1177,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::~__hash_table() {
1169	1177	static_assert(is_copy_constructible<hasher>::value, "Hasher must be copy-constructible.");
1170	1178	#endif
1171	1179
1172		__deallocate_node(__first_node_.__next_);
	1180	__deallocate_node_list(__first_node_.__next_);
1173	1181	}
1174	1182
1175	1183	template <class _Tp, class _Hash, class _Equal, class _Alloc>
...	...	@@ -1184,28 +1192,76 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_
1184	1192	}
1185	1193
1186	1194	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1187		__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) {
1188		if (this != std::addressof(__u)) {
1189		__copy_assign_alloc(__u);
1190		hash_function() = __u.hash_function();
1191		key_eq() = __u.key_eq();
1192		max_load_factor() = __u.max_load_factor();
1193		__assign_multi(__u.begin(), __u.end());
	1195	__hash_table<_Tp, _Hash, _Equal, _Alloc>&
	1196	__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __other) {
	1197	if (this == std::addressof(__other))
	1198	return *this;
	1199
	1200	__copy_assign_alloc(__other);
	1201	hash_function() = __other.hash_function();
	1202	key_eq() = __other.key_eq();
	1203	max_load_factor() = __other.max_load_factor();
	1204
	1205	if (__other.size() == 0) {
	1206	clear();
	1207	return *this;
1194	1208	}
1195		return *this;
1196		}
1197	1209
1198		template <class _Tp, class _Hash, class _Equal, class _Alloc>
1199		void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np) _NOEXCEPT {
1200		__node_allocator& __na = __node_alloc();
1201		while (__np != nullptr) {
1202		__next_pointer __next = __np->__next_;
1203		__node_pointer __real_np = __np->__upcast();
1204		__node_traits::destroy(__na, _NodeTypes::__get_ptr(__real_np->__get_value()));
1205		std::__destroy_at(std::addressof(*__real_np));
1206		__node_traits::deallocate(__na, __real_np, 1);
1207		__np = __next;
	1210	auto __bucket_count = __other.bucket_count();
	1211	if (__bucket_count != bucket_count()) {
	1212	auto& __bucket_list_del = __bucket_list_.get_deleter();
	1213	__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
	1214	__bucket_list_del.size() = __bucket_count;
	1215	}
	1216	std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
	1217
	1218	if (!__first_node_.__next_) {
	1219	__copy_construct(__other.__first_node_.__next_);
	1220	__size_ = __other.size();
	1221	return *this;
	1222	}
	1223
	1224	__next_pointer __other_iter = __other.__first_node_.__next_;
	1225	__next_pointer __own_iter = __first_node_.__ptr();
	1226	{
	1227	__node_pointer __next = __own_iter->__next_->__upcast();
	1228	__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
	1229	__next->__hash_ = __other_iter->__hash();
	1230	}
	1231	size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), __bucket_count);
	1232	__bucket_list_[__current_chash] = __own_iter;
	1233	__other_iter = __other_iter->__next_;
	1234	__own_iter = __own_iter->__next_;
	1235
	1236	// Go through the nodes of the incoming hash table and copy then into the destination hash table, reusing as many
	1237	// existing nodes as posssible in the destination.
	1238	while (__other_iter && __own_iter->__next_) {
	1239	__node_pointer __next = __own_iter->__next_->__upcast();
	1240	__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
	1241	__next->__hash_ = __other_iter->__hash();
	1242
	1243	size_t __new_chash = std::__constrain_hash(__next->__hash_, __bucket_count);
	1244	if (__new_chash != __current_chash) {
	1245	__bucket_list_[__new_chash] = __own_iter;
	1246	__current_chash = __new_chash;
	1247	}
	1248
	1249	__other_iter = __other_iter->__next_;
	1250	__own_iter = __own_iter->__next_;
1208	1251	}
	1252
	1253	// At this point we either have consumed the whole incoming hash table, or we don't have any more nodes to reuse in
	1254	// the destination. Either continue with constructing new nodes, or deallocate the left over nodes.
	1255	if (__own_iter->__next_) {
	1256	__deallocate_node_list(__own_iter->__next_);
	1257	__own_iter->__next_ = nullptr;
	1258	} else {
	1259	__copy_construct(__other_iter, __own_iter, __current_chash);
	1260	}
	1261
	1262	__size_ = __other.size();
	1263
	1264	return *this;
1209	1265	}
1210	1266
1211	1267	template <class _Tp, class _Hash, class _Equal, class _Alloc>
...	...	@@ -1251,23 +1307,14 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
1251	1307	max_load_factor() = __u.max_load_factor();
1252	1308	if (bucket_count() != 0) {
1253	1309	__next_pointer __cache = __detach();
1254		#if _LIBCPP_HAS_EXCEPTIONS
1255		try {
1256		#endif // _LIBCPP_HAS_EXCEPTIONS
1257		const_iterator __i = __u.begin();
1258		while (__cache != nullptr && __u.size() != 0) {
1259		__assign_value(__cache->__upcast()->__get_value(), std::move(__u.remove(__i++)->__get_value()));
1260		__next_pointer __next = __cache->__next_;
1261		__node_insert_multi(__cache->__upcast());
1262		__cache = __next;
1263		}
1264		#if _LIBCPP_HAS_EXCEPTIONS
1265		} catch (...) {
1266		__deallocate_node(__cache);
1267		throw;
	1310	auto __guard = std::__make_scope_guard([&] { __deallocate_node_list(__cache); });
	1311	const_iterator __i = __u.begin();
	1312	while (__cache != nullptr && __u.size() != 0) {
	1313	__assign_value(__cache->__upcast()->__get_value(), std::move(__u.remove(__i++)->__get_value()));
	1314	__next_pointer __next = __cache->__next_;
	1315	__node_insert_multi(__cache->__upcast());
	1316	__cache = __next;
1268	1317	}
1269		#endif // _LIBCPP_HAS_EXCEPTIONS
1270		__deallocate_node(__cache);
1271	1318	}
1272	1319	const_iterator __i = __u.begin();
1273	1320	while (__u.size() != 0)
...	...	@@ -1290,27 +1337,18 @@ template <class _InputIterator>
1290	1337	void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __first, _InputIterator __last) {
1291	1338	typedef iterator_traits<_InputIterator> _ITraits;
1292	1339	typedef typename _ITraits::value_type _ItValueType;
1293		static_assert(is_same<_ItValueType, __container_value_type>::value,
1294		"__assign_unique may only be called with the containers value type");
	1340	static_assert(
	1341	is_same<_ItValueType, value_type>::value, "__assign_unique may only be called with the containers value type");
1295	1342
1296	1343	if (bucket_count() != 0) {
1297	1344	__next_pointer __cache = __detach();
1298		#if _LIBCPP_HAS_EXCEPTIONS
1299		try {
1300		#endif // _LIBCPP_HAS_EXCEPTIONS
1301		for (; __cache != nullptr && __first != __last; ++__first) {
1302		__assign_value(__cache->__upcast()->__get_value(), *__first);
1303		__next_pointer __next = __cache->__next_;
1304		__node_insert_unique(__cache->__upcast());
1305		__cache = __next;
1306		}
1307		#if _LIBCPP_HAS_EXCEPTIONS
1308		} catch (...) {
1309		__deallocate_node(__cache);
1310		throw;
	1345	auto __guard = std::__make_scope_guard([&] { __deallocate_node_list(__cache); });
	1346	for (; __cache != nullptr && __first != __last; ++__first) {
	1347	__assign_value(__cache->__upcast()->__get_value(), *__first);
	1348	__next_pointer __next = __cache->__next_;
	1349	__node_insert_unique(__cache->__upcast());
	1350	__cache = __next;
1311	1351	}
1312		#endif // _LIBCPP_HAS_EXCEPTIONS
1313		__deallocate_node(__cache);
1314	1352	}
1315	1353	for (; __first != __last; ++__first)
1316	1354	__emplace_unique(*__first);
...	...	@@ -1321,31 +1359,20 @@ template <class _InputIterator>
1321	1359	void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __first, _InputIterator __last) {
1322	1360	typedef iterator_traits<_InputIterator> _ITraits;
1323	1361	typedef typename _ITraits::value_type _ItValueType;
1324		static_assert(
1325		(is_same<_ItValueType, __container_value_type>::value \|\| is_same<_ItValueType, __node_value_type>::value),
1326		"__assign_multi may only be called with the containers value type"
1327		" or the nodes value type");
	1362	static_assert(is_same<_ItValueType, value_type>::value,
	1363	"__assign_multi may only be called with the containers value type or the nodes value type");
1328	1364	if (bucket_count() != 0) {
1329	1365	__next_pointer __cache = __detach();
1330		#if _LIBCPP_HAS_EXCEPTIONS
1331		try {
1332		#endif // _LIBCPP_HAS_EXCEPTIONS
1333		for (; __cache != nullptr && __first != __last; ++__first) {
1334		__assign_value(__cache->__upcast()->__get_value(), *__first);
1335		__next_pointer __next = __cache->__next_;
1336		__node_insert_multi(__cache->__upcast());
1337		__cache = __next;
1338		}
1339		#if _LIBCPP_HAS_EXCEPTIONS
1340		} catch (...) {
1341		__deallocate_node(__cache);
1342		throw;
	1366	auto __guard = std::__make_scope_guard([&] { __deallocate_node_list(__cache); });
	1367	for (; __cache != nullptr && __first != __last; ++__first) {
	1368	__assign_value(__cache->__upcast()->__get_value(), *__first);
	1369	__next_pointer __next = __cache->__next_;
	1370	__node_insert_multi(__cache->__upcast());
	1371	__cache = __next;
1343	1372	}
1344		#endif // _LIBCPP_HAS_EXCEPTIONS
1345		__deallocate_node(__cache);
1346	1373	}
1347	1374	for (; __first != __last; ++__first)
1348		__emplace_multi(_NodeTypes::__get_value(*__first));
	1375	__emplace_multi(*__first);
1349	1376	}
1350	1377
1351	1378	template <class _Tp, class _Hash, class _Equal, class _Alloc>
...	...	@@ -1375,7 +1402,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::end() const _NOEXCEPT {
1375	1402	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1376	1403	void __hash_table<_Tp, _Hash, _Equal, _Alloc>::clear() _NOEXCEPT {
1377	1404	if (size() > 0) {
1378		__deallocate_node(__first_node_.__next_);
	1405	__deallocate_node_list(__first_node_.__next_);
1379	1406	__first_node_.__next_ = nullptr;
1380	1407	size_type __bc = bucket_count();
1381	1408	for (size_type __i = 0; __i < __bc; ++__i)
...	...	@@ -1561,69 +1588,6 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_insert_multi(const_iterator __p
1561	1588	return __node_insert_multi(__cp);
1562	1589	}
1563	1590
1564		template <class _Tp, class _Hash, class _Equal, class _Alloc>
1565		template <class _Key, class... _Args>
1566		pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
1567		__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) {
1568		size_t __hash = hash_function()(__k);
1569		size_type __bc = bucket_count();
1570		bool __inserted = false;
1571		__next_pointer __nd;
1572		size_t __chash;
1573		if (__bc != 0) {
1574		__chash = std::__constrain_hash(__hash, __bc);
1575		__nd = __bucket_list_[__chash];
1576		if (__nd != nullptr) {
1577		for (__nd = __nd->__next_;
1578		__nd != nullptr && (__nd->__hash() == __hash \|\| std::__constrain_hash(__nd->__hash(), __bc) == __chash);
1579		__nd = __nd->__next_) {
1580		if ((__nd->__hash() == __hash) && key_eq()(__nd->__upcast()->__get_value(), __k))
1581		goto __done;
1582		}
1583		}
1584		}
1585		{
1586		__node_holder __h = __construct_node_hash(__hash, std::forward<_Args>(__args)...);
1587		if (size() + 1 > __bc * max_load_factor() \|\| __bc == 0) {
1588		__rehash_unique(std::max<size_type>(
1589		2 * __bc + !std::__is_hash_power2(__bc), size_type(__math::ceil(float(size() + 1) / max_load_factor()))));
1590		__bc = bucket_count();
1591		__chash = std::__constrain_hash(__hash, __bc);
1592		}
1593		// insert_after __bucket_list_[__chash], or __first_node if bucket is null
1594		__next_pointer __pn = __bucket_list_[__chash];
1595		if (__pn == nullptr) {
1596		__pn = __first_node_.__ptr();
1597		__h->__next_ = __pn->__next_;
1598		__pn->__next_ = __h.get()->__ptr();
1599		// fix up __bucket_list_
1600		__bucket_list_[__chash] = __pn;
1601		if (__h->__next_ != nullptr)
1602		__bucket_list_[std::__constrain_hash(__h->__next_->__hash(), __bc)] = __h.get()->__ptr();
1603		} else {
1604		__h->__next_ = __pn->__next_;
1605		__pn->__next_ = static_cast<__next_pointer>(__h.get());
1606		}
1607		__nd = static_cast<__next_pointer>(__h.release());
1608		// increment size
1609		++size();
1610		__inserted = true;
1611		}
1612		__done:
1613		return pair<iterator, bool>(iterator(__nd), __inserted);
1614		}
1615
1616		template <class _Tp, class _Hash, class _Equal, class _Alloc>
1617		template <class... _Args>
1618		pair<typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator, bool>
1619		__hash_table<_Tp, _Hash, _Equal, _Alloc>::__emplace_unique_impl(_Args&&... __args) {
1620		__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1621		pair<iterator, bool> __r = __node_insert_unique(__h.get());
1622		if (__r.second)
1623		__h.release();
1624		return __r;
1625		}
1626
1627	1591	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1628	1592	template <class... _Args>
1629	1593	typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
...	...	@@ -1764,41 +1728,45 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__rehash(size_type __n) _LIBCPP_D
1764	1728
1765	1729	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1766	1730	template <bool _UniqueKeys>
1767		void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __nbc) {
1768		__pointer_allocator& __npa = __bucket_list_.get_deleter().__alloc();
1769		__bucket_list_.reset(__nbc > 0 ? __pointer_alloc_traits::allocate(__npa, __nbc) : nullptr);
1770		__bucket_list_.get_deleter().size() = __nbc;
1771		if (__nbc > 0) {
1772		for (size_type __i = 0; __i < __nbc; ++__i)
1773		__bucket_list_[__i] = nullptr;
1774		__next_pointer __pp = __first_node_.__ptr();
1775		__next_pointer __cp = __pp->__next_;
1776		if (__cp != nullptr) {
1777		size_type __chash = std::__constrain_hash(__cp->__hash(), __nbc);
1778		__bucket_list_[__chash] = __pp;
1779		size_type __phash = __chash;
1780		for (__pp = __cp, void(), __cp = __cp->__next_; __cp != nullptr; __cp = __pp->__next_) {
1781		__chash = std::__constrain_hash(__cp->__hash(), __nbc);
1782		if (__chash == __phash)
1783		__pp = __cp;
1784		else {
1785		if (__bucket_list_[__chash] == nullptr) {
1786		__bucket_list_[__chash] = __pp;
1787		__pp = __cp;
1788		__phash = __chash;
1789		} else {
1790		__next_pointer __np = __cp;
1791		if _LIBCPP_CONSTEXPR_SINCE_CXX17 (!_UniqueKeys) {
1792		for (; __np->__next_ != nullptr &&
1793		key_eq()(__cp->__upcast()->__get_value(), __np->__next_->__upcast()->__get_value());
1794		__np = __np->__next_)
1795		;
1796		}
1797		__pp->__next_ = __np->__next_;
1798		__np->__next_ = __bucket_list_[__chash]->__next_;
1799		__bucket_list_[__chash]->__next_ = __cp;
1800		}
	1731	void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__do_rehash(size_type __bucket_count) {
	1732	__pointer_allocator& __ptr_alloc = __bucket_list_.get_deleter().__alloc();
	1733	__bucket_list_.reset(__bucket_count > 0 ? __pointer_alloc_traits::allocate(__ptr_alloc, __bucket_count) : nullptr);
	1734	__bucket_list_.get_deleter().size() = __bucket_count;
	1735
	1736	if (__bucket_count == 0)
	1737	return;
	1738
	1739	for (size_type __i = 0; __i < __bucket_count; ++__i)
	1740	__bucket_list_[__i] = nullptr;
	1741	__next_pointer __pp = __first_node_.__ptr();
	1742	__next_pointer __cp = __pp->__next_;
	1743
	1744	if (!__cp)
	1745	return;
	1746
	1747	size_type __chash = std::__constrain_hash(__cp->__hash(), __bucket_count);
	1748	__bucket_list_[__chash] = __pp;
	1749	size_type __phash = __chash;
	1750	for (__pp = __cp, void(), __cp = __cp->__next_; __cp != nullptr; __cp = __pp->__next_) {
	1751	__chash = std::__constrain_hash(__cp->__hash(), __bucket_count);
	1752	if (__chash == __phash)
	1753	__pp = __cp;
	1754	else {
	1755	if (__bucket_list_[__chash] == nullptr) {
	1756	__bucket_list_[__chash] = __pp;
	1757	__pp = __cp;
	1758	__phash = __chash;
	1759	} else {
	1760	__next_pointer __np = __cp;
	1761	if _LIBCPP_CONSTEXPR (!_UniqueKeys) {
	1762	for (; __np->__next_ != nullptr &&
	1763	key_eq()(__cp->__upcast()->__get_value(), __np->__next_->__upcast()->__get_value());
	1764	__np = __np->__next_)
	1765	;
1801	1766	}
	1767	__pp->__next_ = __np->__next_;
	1768	__np->__next_ = __bucket_list_[__chash]->__next_;
	1769	__bucket_list_[__chash]->__next_ = __cp;
1802	1770	}
1803	1771	}
1804	1772	}
...	...	@@ -1854,16 +1822,13 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&&... __args) {
1854	1822	__node_allocator& __na = __node_alloc();
1855	1823	__node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
1856	1824
1857		// Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
1858		// held inside the node, since we need to use the allocator's construct() method for that.
	1825	// Begin the lifetime of the node itself and the value_type contained within.
1859	1826	//
1860	1827	// We don't use the allocator's construct() method to construct the node itself since the
1861	1828	// Cpp17FooInsertable named requirements don't require the allocator's construct() method
1862	1829	// to work on anything other than the value_type.
1863		std::__construct_at(std::addressof(__h), / next = / nullptr, / hash = */ 0);
	1830	std::__construct_at(std::addressof(__h), / hash = */ 0, __na, std::forward<_Args>(__args)...);
1864	1831
1865		// Now construct the value_type using the allocator's construct() method.
1866		__node_traits::construct(__na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_Args>(__args)...);
1867	1832	__h.get_deleter().__value_constructed = true;
1868	1833
1869	1834	__h->__hash_ = hash_function()(__h->__get_value());
...	...	@@ -1871,15 +1836,13 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node(_Args&&... __args) {
1871	1836	}
1872	1837
1873	1838	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1874		template <class _First, class... _Rest>
	1839	template <class... _Args>
1875	1840	typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::__node_holder
1876		__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash(size_t __hash, _First&& __f, _Rest&&... __rest) {
1877		static_assert(!__is_hash_value_type<_First, _Rest...>::value, "Construct cannot be called with a hash value type");
	1841	__hash_table<_Tp, _Hash, _Equal, _Alloc>::__construct_node_hash(size_t __hash, _Args&&... __args) {
	1842	static_assert(!__is_hash_value_type<_Args...>::value, "Construct cannot be called with a hash value type");
1878	1843	__node_allocator& __na = __node_alloc();
1879	1844	__node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
1880		std::__construct_at(std::addressof(__h), / next = / nullptr, / hash = */ __hash);
1881		__node_traits::construct(
1882		__na, _NodeTypes::__get_ptr(__h->__get_value()), std::forward<_First>(__f), std::forward<_Rest>(__rest)...);
	1845	std::__construct_at(std::addressof(__h), / hash = */ __hash, __na, std::forward<_Args>(__args)...);
1883	1846	__h.get_deleter().__value_constructed = true;
1884	1847	return __h;
1885	1848	}
...	...	@@ -1899,12 +1862,63 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __p) {
1899	1862	template <class _Tp, class _Hash, class _Equal, class _Alloc>
1900	1863	typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
1901	1864	__hash_table<_Tp, _Hash, _Equal, _Alloc>::erase(const_iterator __first, const_iterator __last) {
1902		for (const_iterator __p = __first; __first != __last; __p = __first) {
1903		++__first;
1904		erase(__p);
	1865	if (__first == __last)
	1866	return iterator(__last.__node_);
	1867
	1868	// current node
	1869	__next_pointer __current = __first.__node_;
	1870	size_type __bucket_count = bucket_count();
	1871	size_t __chash = std::__constrain_hash(__current->__hash(), __bucket_count);
	1872	// find previous node
	1873	__next_pointer __before_first = __bucket_list_[__chash];
	1874	for (; __before_first->__next_ != __current; __before_first = __before_first->__next_)
	1875	;
	1876
	1877	__next_pointer __last_node = __last.__node_;
	1878
	1879	// If __before_first is in the same bucket (i.e. the first element we erase is not the first in the bucket), clear
	1880	// this bucket first without re-linking it
	1881	if (__before_first != __first_node_.__ptr() &&
	1882	std::__constrain_hash(__before_first->__hash(), __bucket_count) == __chash) {
	1883	while (__current != __last_node) {
	1884	auto __next = __current->__next_;
	1885	__deallocate_node(__current->__upcast());
	1886	__current = __next;
	1887	--__size_;
	1888
	1889	if (__next) {
	1890	if (auto __next_chash = std::__constrain_hash(__next->__hash(), __bucket_count); __next_chash != __chash) {
	1891	__bucket_list_[__next_chash] = __before_first;
	1892	__chash = __next_chash;
	1893	break;
	1894	}
	1895	}
	1896	}
1905	1897	}
1906		__next_pointer __np = __last.__node_;
1907		return iterator(__np);
	1898
	1899	while (__current != __last_node) {
	1900	auto __next = __current->__next_;
	1901	__deallocate_node(__current->__upcast());
	1902	__current = __next;
	1903	--__size_;
	1904
	1905	// When switching buckets, set the old bucket to be empty and update the next bucket to have __before_first as its
	1906	// before-first element
	1907	if (__next) {
	1908	if (auto __next_chash = std::__constrain_hash(__next->__hash(), __bucket_count); __next_chash != __chash) {
	1909	__bucket_list_[__chash] = nullptr;
	1910	__bucket_list_[__next_chash] = __before_first;
	1911	__chash = __next_chash;
	1912	}
	1913	} else { // When __next is a nullptr we've fully erased the last bucket. Update the bucket list accordingly.
	1914	__bucket_list_[__chash] = nullptr;
	1915	}
	1916	}
	1917
	1918	// re-link __before_first with __last
	1919	__before_first->__next_ = __current;
	1920
	1921	return iterator(__last.__node_);
1908	1922	}
1909	1923
1910	1924	template <class _Tp, class _Hash, class _Equal, class _Alloc>

lib/libcxx/include/__ios/fpos.h+4-4

...	...	@@ -30,7 +30,7 @@ public:
30	30
31	31	_LIBCPP_HIDE_FROM_ABI operator streamoff() const { return __off_; }
32	32
33		_LIBCPP_HIDE_FROM_ABI _StateT state() const { return __st_; }
	33	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _StateT state() const { return __st_; }
34	34	_LIBCPP_HIDE_FROM_ABI void state(_StateT __st) { __st_ = __st; }
35	35
36	36	_LIBCPP_HIDE_FROM_ABI fpos& operator+=(streamoff __off) {
...	...	@@ -38,7 +38,7 @@ public:
38	38	return *this;
39	39	}
40	40
41		_LIBCPP_HIDE_FROM_ABI fpos operator+(streamoff __off) const {
	41	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI fpos operator+(streamoff __off) const {
42	42	fpos __t(*this);
43	43	__t += __off;
44	44	return __t;
...	...	@@ -49,7 +49,7 @@ public:
49	49	return *this;
50	50	}
51	51
52		_LIBCPP_HIDE_FROM_ABI fpos operator-(streamoff __off) const {
	52	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI fpos operator-(streamoff __off) const {
53	53	fpos __t(*this);
54	54	__t -= __off;
55	55	return __t;
...	...	@@ -57,7 +57,7 @@ public:
57	57	};
58	58
59	59	template <class _StateT>
60		inline _LIBCPP_HIDE_FROM_ABI streamoff operator-(const fpos<_StateT>& __x, const fpos<_StateT>& __y) {
	60	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI streamoff operator-(const fpos<_StateT>& __x, const fpos<_StateT>& __y) {
61	61	return streamoff(__x) - streamoff(__y);
62	62	}
63	63

lib/libcxx/include/__iterator/back_insert_iterator.h+1-7

...	...	@@ -26,15 +26,9 @@ _LIBCPP_PUSH_MACROS
26	26
27	27	_LIBCPP_BEGIN_NAMESPACE_STD
28	28
29		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
30	29	template <class _Container>
31	30	class back_insert_iterator
32		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
33		: public iterator<output_iterator_tag, void, void, void, void>
34		#endif
35		{
36		_LIBCPP_SUPPRESS_DEPRECATED_POP
37
	31	: public __iterator_base<back_insert_iterator<_Container>, output_iterator_tag, void, void, void, void> {
38	32	protected:
39	33	_Container* container;
40	34

lib/libcxx/include/__iterator/bounded_iter.h+7-8

...	...	@@ -74,12 +74,12 @@ struct __bounded_iter {
74	74	_LIBCPP_HIDE_FROM_ABI __bounded_iter(__bounded_iter const&) = default;
75	75	_LIBCPP_HIDE_FROM_ABI __bounded_iter(__bounded_iter&&) = default;
76	76
77		template < class _OtherIterator,
78		__enable_if_t<
79		_And< is_convertible<const _OtherIterator&, _Iterator>,
80		_Or<is_same<reference, __iter_reference<_OtherIterator> >,
81		is_same<reference, __make_const_lvalue_ref<__iter_reference<_OtherIterator> > > > >::value,
82		int> = 0>
	77	template <class _OtherIterator,
	78	__enable_if_t<
	79	_And<is_convertible<const _OtherIterator&, _Iterator>,
	80	_Or<is_same<reference, __iterator_reference<_OtherIterator> >,
	81	is_same<reference, __make_const_lvalue_ref<__iterator_reference<_OtherIterator> > > > >::value,
	82	int> = 0>
83	83	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __bounded_iter(__bounded_iter<_OtherIterator> const& __other) _NOEXCEPT
84	84	: __current_(__other.__current_),
85	85	__begin_(__other.__begin_),
...	...	@@ -116,8 +116,7 @@ public:
116	116	// These operations check that the iterator is dereferenceable. Since the class invariant is
117	117	// that the iterator is always within `[begin, end]`, we only need to check it's not pointing to
118	118	// `end`. This is easier for the optimizer because it aligns with the `iter != container.end()`
119		// checks that typical callers already use (see
120		// https://github.com/llvm/llvm-project/issues/78829).
	119	// checks that typical callers already use (see https://llvm.org/PR78829).
121	120	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator*() const _NOEXCEPT {
122	121	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
123	122	__current_ != __end_, "__bounded_iter::operator*: Attempt to dereference an iterator at the end");

lib/libcxx/include/__iterator/concepts.h+6-9

...	...	@@ -117,15 +117,12 @@ template <class _Tp>
117	117	concept __signed_integer_like = signed_integral<_Tp>;
118	118
119	119	template <class _Ip>
120		concept weakly_incrementable =
121		// TODO: remove this once the clang bug is fixed (bugs.llvm.org/PR48173).
122		!same_as<_Ip, bool> && // Currently, clang does not handle bool correctly.
123		movable<_Ip> && requires(_Ip __i) {
124		typename iter_difference_t<_Ip>;
125		requires __signed_integer_like<iter_difference_t<_Ip>>;
126		{ ++__i } -> same_as<_Ip&>; // not required to be equality-preserving
127		__i++; // not required to be equality-preserving
128		};
	120	concept weakly_incrementable = movable<_Ip> && requires(_Ip __i) {
	121	typename iter_difference_t<_Ip>;
	122	requires __signed_integer_like<iter_difference_t<_Ip>>;
	123	{ ++__i } -> same_as<_Ip&>; // not required to be equality-preserving
	124	__i++; // not required to be equality-preserving
	125	};
129	126
130	127	// [iterator.concept.inc]
131	128	template <class _Ip>

lib/libcxx/include/__iterator/cpp17_iterator_concepts.h+13-12

...	...	@@ -68,7 +68,8 @@ concept __cpp17_default_constructible = is_default_constructible_v<_Tp>;
68	68	template <class _Iter>
69	69	concept __cpp17_iterator =
70	70	__cpp17_copy_constructible<_Iter> && __cpp17_copy_assignable<_Iter> && __cpp17_destructible<_Iter> &&
71		(is_signed_v<__iter_diff_t<_Iter>> \|\| is_void_v<__iter_diff_t<_Iter>>) && requires(_Iter __iter) {
	71	(is_signed_v<__iterator_difference_type<_Iter>> \|\| is_void_v<__iterator_difference_type<_Iter>>) &&
	72	requires(_Iter __iter) {
72	73	{ *__iter };
73	74	{ ++__iter } -> same_as<_Iter&>;
74	75	};
...	...	@@ -81,8 +82,8 @@ concept __cpp17_input_iterator =
81	82	{ __lhs != std::as_const(__rhs) } -> __boolean_testable;
82	83	{ std::as_const(__lhs) != std::as_const(__rhs) } -> __boolean_testable;
83	84
84		{ *__lhs } -> same_as<__iter_reference<_Iter>>;
85		{ *std::as_const(__lhs) } -> same_as<__iter_reference<_Iter>>;
	85	{ *__lhs } -> same_as<__iterator_reference<_Iter>>;
	86	{ *std::as_const(__lhs) } -> same_as<__iterator_reference<_Iter>>;
86	87
87	88	{ ++__lhs } -> same_as<_Iter&>;
88	89	{ (void)__lhs++ };
...	...	@@ -101,19 +102,19 @@ template <class _Iter>
101	102	concept __cpp17_forward_iterator =
102	103	__cpp17_input_iterator<_Iter> && __cpp17_default_constructible<_Iter> && requires(_Iter __iter) {
103	104	{ __iter++ } -> convertible_to<const _Iter&>;
104		{ *__iter++ } -> same_as<__iter_reference<_Iter>>;
	105	{ *__iter++ } -> same_as<__iterator_reference<_Iter>>;
105	106	};
106	107
107	108	template <class _Iter>
108	109	concept __cpp17_bidirectional_iterator = __cpp17_forward_iterator<_Iter> && requires(_Iter __iter) {
109	110	{ --__iter } -> same_as<_Iter&>;
110	111	{ __iter-- } -> convertible_to<const _Iter&>;
111		{ *__iter-- } -> same_as<__iter_reference<_Iter>>;
	112	{ *__iter-- } -> same_as<__iterator_reference<_Iter>>;
112	113	};
113	114
114	115	template <class _Iter>
115	116	concept __cpp17_random_access_iterator =
116		__cpp17_bidirectional_iterator<_Iter> && requires(_Iter __iter, __iter_diff_t<_Iter> __n) {
	117	__cpp17_bidirectional_iterator<_Iter> && requires(_Iter __iter, __iterator_difference_type<_Iter> __n) {
117	118	{ __iter += __n } -> same_as<_Iter&>;
118	119
119	120	{ __iter + __n } -> same_as<_Iter>;
...	...	@@ -125,13 +126,13 @@ concept __cpp17_random_access_iterator =
125	126	{ __iter - __n } -> same_as<_Iter>;
126	127	{ std::as_const(__iter) - __n } -> same_as<_Iter>;
127	128
128		{ __iter - __iter } -> same_as<__iter_diff_t<_Iter>>;
129		{ std::as_const(__iter) - __iter } -> same_as<__iter_diff_t<_Iter>>;
130		{ __iter - std::as_const(__iter) } -> same_as<__iter_diff_t<_Iter>>;
131		{ std::as_const(__iter) - std::as_const(__iter) } -> same_as<__iter_diff_t<_Iter>>;
	129	{ __iter - __iter } -> same_as<__iterator_difference_type<_Iter>>;
	130	{ std::as_const(__iter) - __iter } -> same_as<__iterator_difference_type<_Iter>>;
	131	{ __iter - std::as_const(__iter) } -> same_as<__iterator_difference_type<_Iter>>;
	132	{ std::as_const(__iter) - std::as_const(__iter) } -> same_as<__iterator_difference_type<_Iter>>;
132	133
133		{ __iter[__n] } -> convertible_to<__iter_reference<_Iter>>;
134		{ std::as_const(__iter)[__n] } -> convertible_to<__iter_reference<_Iter>>;
	134	{ __iter[__n] } -> convertible_to<__iterator_reference<_Iter>>;
	135	{ std::as_const(__iter)[__n] } -> convertible_to<__iterator_reference<_Iter>>;
135	136
136	137	{ __iter < __iter } -> __boolean_testable;
137	138	{ std::as_const(__iter) < __iter } -> __boolean_testable;

lib/libcxx/include/__iterator/distance.h+40-18

...	...	@@ -10,41 +10,66 @@
10	10	#ifndef _LIBCPP___ITERATOR_DISTANCE_H
11	11	#define _LIBCPP___ITERATOR_DISTANCE_H
12	12
	13	#include <__algorithm/for_each_segment.h>
	14	#include <__concepts/same_as.h>
13	15	#include <__config>
14	16	#include <__iterator/concepts.h>
15	17	#include <__iterator/incrementable_traits.h>
16	18	#include <__iterator/iterator_traits.h>
	19	#include <__iterator/segmented_iterator.h>
17	20	#include <__ranges/access.h>
18	21	#include <__ranges/concepts.h>
19	22	#include <__ranges/size.h>
20	23	#include <__type_traits/decay.h>
	24	#include <__type_traits/enable_if.h>
21	25	#include <__type_traits/remove_cvref.h>
	26	#include <__utility/move.h>
22	27
23	28	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
24	29	# pragma GCC system_header
25	30	#endif
26	31
	32	_LIBCPP_PUSH_MACROS
	33	#include <__undef_macros>
	34
27	35	_LIBCPP_BEGIN_NAMESPACE_STD
28	36
29		template <class _InputIter>
30		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
31		__distance(_InputIter __first, _InputIter __last, input_iterator_tag) {
32		typename iterator_traits<_InputIter>::difference_type __r(0);
33		for (; __first != __last; ++__first)
34		++__r;
35		return __r;
36		}
	37	#if _LIBCPP_STD_VER >= 20
	38	template <class _Iter>
	39	using __iter_distance_t _LIBCPP_NODEBUG = std::iter_difference_t<_Iter>;
	40	#else
	41	template <class _Iter>
	42	using __iter_distance_t _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::difference_type;
	43	#endif
37	44
38		template <class _RandIter>
39		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_RandIter>::difference_type
40		__distance(_RandIter __first, _RandIter __last, random_access_iterator_tag) {
	45	template <class _RandIter, __enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
	46	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_RandIter>
	47	__distance(_RandIter __first, _RandIter __last) {
41	48	return __last - __first;
42	49	}
43	50
	51	template <class _InputIter, class _Sent>
	52	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 __iter_distance_t<_InputIter>
	53	__distance(_InputIter __first, _Sent __last) {
	54	__iter_distance_t<_InputIter> __r(0);
	55	#if _LIBCPP_STD_VER >= 20
	56	if constexpr (same_as<_InputIter, _Sent> && __is_segmented_iterator_v<_InputIter>) {
	57	std::__for_each_segment(__first, __last, [&__r](auto __lfirst, auto __llast) {
	58	__r += std::__distance(__lfirst, __llast);
	59	});
	60	} else
	61	#endif
	62	{
	63	for (; __first != __last; ++__first)
	64	++__r;
	65	}
	66	return __r;
	67	}
	68
44	69	template <class _InputIter>
45	70	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InputIter>::difference_type
46	71	distance(_InputIter __first, _InputIter __last) {
47		return std::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
	72	return std::__distance(__first, __last);
48	73	}
49	74
50	75	#if _LIBCPP_STD_VER >= 20
...	...	@@ -56,12 +81,7 @@ struct __distance {
56	81	template <class _Ip, sentinel_for<_Ip> _Sp>
57	82	requires(!sized_sentinel_for<_Sp, _Ip>)
58	83	_LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Ip> operator()(_Ip __first, _Sp __last) const {
59		iter_difference_t<_Ip> __n = 0;
60		while (__first != __last) {
61		++__first;
62		++__n;
63		}
64		return __n;
	84	return std::__distance(std::move(__first), std::move(__last));
65	85	}
66	86
67	87	template <class _Ip, sized_sentinel_for<decay_t<_Ip>> _Sp>
...	...	@@ -92,4 +112,6 @@ inline constexpr auto distance = __distance{};
92	112
93	113	_LIBCPP_END_NAMESPACE_STD
94	114
	115	_LIBCPP_POP_MACROS
	116
95	117	#endif // _LIBCPP___ITERATOR_DISTANCE_H

lib/libcxx/include/__iterator/front_insert_iterator.h+1-7

...	...	@@ -26,15 +26,9 @@ _LIBCPP_PUSH_MACROS
26	26
27	27	_LIBCPP_BEGIN_NAMESPACE_STD
28	28
29		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
30	29	template <class _Container>
31	30	class front_insert_iterator
32		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
33		: public iterator<output_iterator_tag, void, void, void, void>
34		#endif
35		{
36		_LIBCPP_SUPPRESS_DEPRECATED_POP
37
	31	: public __iterator_base<front_insert_iterator<_Container>, output_iterator_tag, void, void, void, void> {
38	32	protected:
39	33	_Container* container;
40	34

lib/libcxx/include/__iterator/insert_iterator.h+1-7

...	...	@@ -35,15 +35,9 @@ template <class _Container>
35	35	using __insert_iterator_iter_t _LIBCPP_NODEBUG = typename _Container::iterator;
36	36	#endif
37	37
38		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
39	38	template <class _Container>
40	39	class insert_iterator
41		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
42		: public iterator<output_iterator_tag, void, void, void, void>
43		#endif
44		{
45		_LIBCPP_SUPPRESS_DEPRECATED_POP
46
	40	: public __iterator_base<insert_iterator<_Container>, output_iterator_tag, void, void, void, void> {
47	41	protected:
48	42	_Container* container;
49	43	__insert_iterator_iter_t<_Container> iter;

lib/libcxx/include/__iterator/istream_iterator.h+6-7

...	...	@@ -25,15 +25,14 @@
25	25
26	26	_LIBCPP_BEGIN_NAMESPACE_STD
27	27
28		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
29	28	template <class _Tp, class _CharT = char, class _Traits = char_traits<_CharT>, class _Distance = ptrdiff_t>
30	29	class istream_iterator
31		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
32		: public iterator<input_iterator_tag, _Tp, _Distance, const _Tp*, const _Tp&>
33		#endif
34		{
35		_LIBCPP_SUPPRESS_DEPRECATED_POP
36
	30	: public __iterator_base<istream_iterator<_Tp, _CharT, _Traits, _Distance>,
	31	input_iterator_tag,
	32	_Tp,
	33	_Distance,
	34	const _Tp*,
	35	const _Tp&> {
37	36	public:
38	37	typedef input_iterator_tag iterator_category;
39	38	typedef _Tp value_type;

lib/libcxx/include/__iterator/istreambuf_iterator.h+6-7

...	...	@@ -25,15 +25,14 @@
25	25
26	26	_LIBCPP_BEGIN_NAMESPACE_STD
27	27
28		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
29	28	template <class _CharT, class _Traits>
30	29	class istreambuf_iterator
31		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
32		: public iterator<input_iterator_tag, _CharT, typename _Traits::off_type, _CharT*, _CharT>
33		#endif
34		{
35		_LIBCPP_SUPPRESS_DEPRECATED_POP
36
	30	: public __iterator_base<istreambuf_iterator<_CharT, _Traits>,
	31	input_iterator_tag,
	32	_CharT,
	33	typename _Traits::off_type,
	34	_CharT*,
	35	_CharT> {
37	36	public:
38	37	typedef input_iterator_tag iterator_category;
39	38	typedef _CharT value_type;

lib/libcxx/include/__iterator/iter_move.h+1-1

...	...	@@ -40,7 +40,7 @@ void iter_move() = delete;
40	40
41	41	template <class _Tp>
42	42	concept __unqualified_iter_move = __class_or_enum<remove_cvref_t<_Tp>> && requires(_Tp&& __t) {
43		// NOLINTNEXTLINE(libcpp-robust-against-adl) iter_swap ADL calls should only be made through ranges::iter_swap
	43	// NOLINTNEXTLINE(libcpp-robust-against-adl) iter_move ADL calls should only be made through ranges::iter_move
44	44	iter_move(std::forward<_Tp>(__t));
45	45	};
46	46

lib/libcxx/include/__iterator/iterator.h+13

...	...	@@ -28,6 +28,19 @@ struct _LIBCPP_DEPRECATED_IN_CXX17 iterator {
28	28	typedef _Category iterator_category;
29	29	};
30	30
	31	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
	32	#ifdef _LIBCPP_ABI_NO_ITERATOR_BASES
	33	template <class _Derived>
	34	struct __no_iterator_base {};
	35
	36	template <class _Derived, class _Category, class _Tp, class _Distance, class _Pointer, class _Reference>
	37	using __iterator_base _LIBCPP_NODEBUG = __no_iterator_base<_Derived>;
	38	#else
	39	template <class _Derived, class _Category, class _Tp, class _Distance, class _Pointer, class _Reference>
	40	using __iterator_base _LIBCPP_NODEBUG = iterator<_Category, _Tp, _Distance, _Pointer, _Reference>;
	41	#endif
	42	_LIBCPP_SUPPRESS_DEPRECATED_POP
	43
31	44	_LIBCPP_END_NAMESPACE_STD
32	45
33	46	#endif // _LIBCPP___ITERATOR_ITERATOR_H

lib/libcxx/include/__iterator/iterator_traits.h+13-14

...	...	@@ -420,44 +420,43 @@ using __has_exactly_bidirectional_iterator_category _LIBCPP_NODEBUG =
420	420	!__has_iterator_category_convertible_to<_Tp, random_access_iterator_tag>::value>;
421	421
422	422	template <class _InputIterator>
423		using __iter_value_type _LIBCPP_NODEBUG = typename iterator_traits<_InputIterator>::value_type;
	423	using __iterator_value_type _LIBCPP_NODEBUG = typename iterator_traits<_InputIterator>::value_type;
424	424
425	425	#if _LIBCPP_STD_VER >= 23
426	426	template <class _InputIterator>
427		using __iter_key_type _LIBCPP_NODEBUG = remove_const_t<tuple_element_t<0, __iter_value_type<_InputIterator>>>;
	427	using __iter_key_type _LIBCPP_NODEBUG = remove_const_t<tuple_element_t<0, __iterator_value_type<_InputIterator>>>;
428	428
429	429	template <class _InputIterator>
430		using __iter_mapped_type _LIBCPP_NODEBUG = tuple_element_t<1, __iter_value_type<_InputIterator>>;
	430	using __iter_mapped_type _LIBCPP_NODEBUG = tuple_element_t<1, __iterator_value_type<_InputIterator>>;
431	431
432	432	template <class _InputIterator>
433	433	using __iter_to_alloc_type _LIBCPP_NODEBUG =
434		pair<const tuple_element_t<0, __iter_value_type<_InputIterator>>,
435		tuple_element_t<1, __iter_value_type<_InputIterator>>>;
	434	pair<const tuple_element_t<0, __iterator_value_type<_InputIterator>>,
	435	tuple_element_t<1, __iterator_value_type<_InputIterator>>>;
436	436	#else
437	437	template <class _InputIterator>
438		using __iter_key_type _LIBCPP_NODEBUG =
439		__remove_const_t<typename iterator_traits<_InputIterator>::value_type::first_type>;
	438	using __iter_key_type _LIBCPP_NODEBUG = __remove_const_t<typename __iterator_value_type<_InputIterator>::first_type>;
440	439
441	440	template <class _InputIterator>
442		using __iter_mapped_type _LIBCPP_NODEBUG = typename iterator_traits<_InputIterator>::value_type::second_type;
	441	using __iter_mapped_type _LIBCPP_NODEBUG = typename __iterator_value_type<_InputIterator>::second_type;
443	442
444	443	template <class _InputIterator>
445	444	using __iter_to_alloc_type _LIBCPP_NODEBUG =
446		pair<const typename iterator_traits<_InputIterator>::value_type::first_type,
447		typename iterator_traits<_InputIterator>::value_type::second_type>;
	445	pair<const typename __iterator_value_type<_InputIterator>::first_type,
	446	typename __iterator_value_type<_InputIterator>::second_type>;
448	447	#endif // _LIBCPP_STD_VER >= 23
449	448
450	449	template <class _Iter>
451		using __iterator_category_type _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::iterator_category;
	450	using __iterator_iterator_category _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::iterator_category;
452	451
453	452	template <class _Iter>
454		using __iterator_pointer_type _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::pointer;
	453	using __iterator_pointer _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::pointer;
455	454
456	455	template <class _Iter>
457		using __iter_diff_t _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::difference_type;
	456	using __iterator_difference_type _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::difference_type;
458	457
459	458	template <class _Iter>
460		using __iter_reference _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::reference;
	459	using __iterator_reference _LIBCPP_NODEBUG = typename iterator_traits<_Iter>::reference;
461	460
462	461	#if _LIBCPP_STD_VER >= 20
463	462

lib/libcxx/include/__iterator/ostream_iterator.h+1-7

...	...	@@ -24,15 +24,9 @@
24	24
25	25	_LIBCPP_BEGIN_NAMESPACE_STD
26	26
27		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
28	27	template <class _Tp, class _CharT = char, class _Traits = char_traits<_CharT> >
29	28	class ostream_iterator
30		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
31		: public iterator<output_iterator_tag, void, void, void, void>
32		#endif
33		{
34		_LIBCPP_SUPPRESS_DEPRECATED_POP
35
	29	: public __iterator_base<ostream_iterator<_Tp, _CharT, _Traits>, output_iterator_tag, void, void, void, void> {
36	30	public:
37	31	typedef output_iterator_tag iterator_category;
38	32	typedef void value_type;

lib/libcxx/include/__iterator/ostreambuf_iterator.h+1-7

...	...	@@ -25,15 +25,9 @@
25	25
26	26	_LIBCPP_BEGIN_NAMESPACE_STD
27	27
28		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
29	28	template <class _CharT, class _Traits>
30	29	class ostreambuf_iterator
31		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
32		: public iterator<output_iterator_tag, void, void, void, void>
33		#endif
34		{
35		_LIBCPP_SUPPRESS_DEPRECATED_POP
36
	30	: public __iterator_base<ostreambuf_iterator<_CharT, _Traits>, output_iterator_tag, void, void, void, void> {
37	31	public:
38	32	typedef output_iterator_tag iterator_category;
39	33	typedef void value_type;

lib/libcxx/include/__iterator/reverse_iterator.h+8-13

...	...	@@ -46,21 +46,16 @@
46	46
47	47	_LIBCPP_BEGIN_NAMESPACE_STD
48	48
49		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
50	49	template <class _Iter>
51	50	class reverse_iterator
52		#if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
53		: public iterator<typename iterator_traits<_Iter>::iterator_category,
54		typename iterator_traits<_Iter>::value_type,
55		typename iterator_traits<_Iter>::difference_type,
56		typename iterator_traits<_Iter>::pointer,
57		typename iterator_traits<_Iter>::reference>
58		#endif
59		{
60		_LIBCPP_SUPPRESS_DEPRECATED_POP
61
	51	: public __iterator_base<reverse_iterator<_Iter>,
	52	typename iterator_traits<_Iter>::iterator_category,
	53	typename iterator_traits<_Iter>::value_type,
	54	typename iterator_traits<_Iter>::difference_type,
	55	typename iterator_traits<_Iter>::pointer,
	56	typename iterator_traits<_Iter>::reference> {
62	57	private:
63		#ifndef _LIBCPP_ABI_NO_ITERATOR_BASES
	58	#ifndef _LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER
64	59	_Iter __t_; // no longer used as of LWG #2360, not removed due to ABI break
65	60	#endif
66	61
...	...	@@ -91,7 +86,7 @@ public:
91	86	using reference = typename iterator_traits<_Iter>::reference;
92	87	#endif
93	88
94		#ifndef _LIBCPP_ABI_NO_ITERATOR_BASES
	89	#ifndef _LIBCPP_ABI_NO_REVERSE_ITERATOR_SECOND_MEMBER
95	90	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator() : __t_(), current() {}
96	91
97	92	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit reverse_iterator(_Iter __x) : __t_(__x), current(__x) {}

lib/libcxx/include/__iterator/segmented_iterator.h+3-8

...	...	@@ -67,18 +67,13 @@ struct __segmented_iterator_traits;
67	67	*/
68	68
69	69	template <class _Tp, size_t = 0>
70		struct __has_specialization : false_type {};
	70	inline const bool __has_specialization_v = false;
71	71
72	72	template <class _Tp>
73		struct __has_specialization<_Tp, sizeof(_Tp) * 0> : true_type {};
	73	inline const bool __has_specialization_v<_Tp, sizeof(_Tp) * 0> = true;
74	74
75	75	template <class _Iterator>
76		using __is_segmented_iterator _LIBCPP_NODEBUG = __has_specialization<__segmented_iterator_traits<_Iterator> >;
77
78		template <class _SegmentedIterator>
79		struct __has_random_access_local_iterator
80		: __has_random_access_iterator_category<
81		typename __segmented_iterator_traits< _SegmentedIterator >::__local_iterator > {};
	76	inline const bool __is_segmented_iterator_v = __has_specialization_v<__segmented_iterator_traits<_Iterator> >;
82	77
83	78	_LIBCPP_END_NAMESPACE_STD
84	79

lib/libcxx/include/__iterator/static_bounded_iter.h+3-3

...	...	@@ -99,9 +99,9 @@ struct __static_bounded_iter {
99	99
100	100	template <class _OtherIterator,
101	101	__enable_if_t<
102		_And< is_convertible<const _OtherIterator&, _Iterator>,
103		_Or<is_same<reference, __iter_reference<_OtherIterator> >,
104		is_same<reference, __make_const_lvalue_ref<__iter_reference<_OtherIterator> > > > >::value,
	102	_And<is_convertible<const _OtherIterator&, _Iterator>,
	103	_Or<is_same<reference, __iterator_reference<_OtherIterator> >,
	104	is_same<reference, __make_const_lvalue_ref<__iterator_reference<_OtherIterator> > > > >::value,
105	105	int> = 0>
106	106	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR
107	107	__static_bounded_iter(__static_bounded_iter<_OtherIterator, _Size> const& __other) _NOEXCEPT

lib/libcxx/include/__iterator/wrap_iter.h+8-6

...	...	@@ -49,12 +49,12 @@ private:
49	49
50	50	public:
51	51	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter() _NOEXCEPT : __i_() {}
52		template <
53		class _OtherIter,
54		__enable_if_t< _And< is_convertible<const _OtherIter&, _Iter>,
55		_Or<is_same<reference, __iter_reference<_OtherIter> >,
56		is_same<reference, __make_const_lvalue_ref<__iter_reference<_OtherIter> > > > >::value,
57		int> = 0>
	52	template <class _OtherIter,
	53	__enable_if_t<
	54	_And<is_convertible<const _OtherIter&, _Iter>,
	55	_Or<is_same<reference, __iterator_reference<_OtherIter> >,
	56	is_same<reference, __make_const_lvalue_ref<__iterator_reference<_OtherIter> > > > >::value,
	57	int> = 0>
58	58	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __wrap_iter(const __wrap_iter<_OtherIter>& __u) _NOEXCEPT
59	59	: __i_(__u.__i_) {}
60	60	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference operator() const _NOEXCEPT { return __i_; }
...	...	@@ -117,6 +117,8 @@ private:
117	117	friend class span;
118	118	template <class _Tp, size_t _Size>
119	119	friend struct array;
	120	template <class _Tp, class>
	121	friend struct __optional_iterator;
120	122	};
121	123
122	124	template <class _Iter1>

lib/libcxx/include/__locale+3-10

...	...	@@ -57,9 +57,8 @@ _LIBCPP_HIDE_FROM_ABI const _Facet& use_facet(const locale&);
57	57	class _LIBCPP_EXPORTED_FROM_ABI locale {
58	58	public:
59	59	// locale is essentially a shared_ptr that doesn't support weak_ptrs and never got a move constructor,
60		// so it is trivially relocatable. Like shared_ptr, it is also replaceable.
	60	// so it is trivially relocatable.
61	61	using __trivially_relocatable _LIBCPP_NODEBUG = locale;
62		using __replaceable _LIBCPP_NODEBUG = locale;
63	62
64	63	// types:
65	64	class _LIBCPP_EXPORTED_FROM_ABI facet;
...	...	@@ -389,7 +388,7 @@ public:
389	388	static const mask xdigit = _ISXDIGIT;
390	389	static const mask blank = _ISBLANK;
391	390	static const mask __regex_word = 0x8000;
392		# elif defined(_NEWLIB_VERSION)
	391	# elif _LIBCPP_LIBC_NEWLIB
393	392	// Same type as Newlib's _ctype_ array in newlib/libc/include/ctype.h.
394	393	typedef char mask;
395	394	// In case char is signed, static_cast is needed to avoid warning on
...	...	@@ -585,7 +584,7 @@ public:
585	584	# ifdef _CACHED_RUNES
586	585	static const size_t table_size = _CACHED_RUNES;
587	586	# else
588		static const size_t table_size = 256; // FIXME: Don't hardcode this.
	587	static const size_t table_size = 256;
589	588	# endif
590	589	_LIBCPP_HIDE_FROM_ABI const mask* table() const _NOEXCEPT { return __tab_; }
591	590	static const mask* classic_table() _NOEXCEPT;
...	...	@@ -1478,9 +1477,6 @@ public:
1478	1477
1479	1478	protected:
1480	1479	~numpunct_byname() override;
1481
1482		private:
1483		void __init(const char*);
1484	1480	};
1485	1481
1486	1482	# if _LIBCPP_HAS_WIDE_CHARACTERS
...	...	@@ -1495,9 +1491,6 @@ public:
1495	1491
1496	1492	protected:
1497	1493	~numpunct_byname() override;
1498
1499		private:
1500		void __init(const char*);
1501	1494	};
1502	1495	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
1503	1496

lib/libcxx/include/__locale_dir/locale_base_api.h+5-28

...	...	@@ -57,15 +57,11 @@
57	57	// float __strtof(const char, char*, __locale_t);
58	58	// double __strtod(const char, char*, __locale_t);
59	59	// long double __strtold(const char, char*, __locale_t);
60		// long long __strtoll(const char, char*, __locale_t);
61		// unsigned long long __strtoull(const char, char*, __locale_t);
62	60	// }
63	61	//
64	62	// Character manipulation functions
65	63	// --------------------------------
66	64	// namespace __locale {
67		// int __isdigit(int, __locale_t); // required by the headers
68		// int __isxdigit(int, __locale_t); // required by the headers
69	65	// int __toupper(int, __locale_t);
70	66	// int __tolower(int, __locale_t);
71	67	// int __strcoll(const char, const char, __locale_t);
...	...	@@ -106,7 +102,6 @@
106	102	//
107	103	// int __snprintf(char, size_t, __locale_t, const char, ...); // required by the headers
108	104	// int __asprintf(char*, __locale_t, const char, ...); // required by the headers
109		// int __sscanf(const char, __locale_t, const char, ...); // required by the headers
110	105	// }
111	106
112	107	#if _LIBCPP_HAS_LOCALIZATION
...	...	@@ -115,7 +110,6 @@
115	110	# include <__locale_dir/support/apple.h>
116	111	# elif defined(__FreeBSD__)
117	112	# include <__locale_dir/support/freebsd.h>
118		/* zig patch: https://github.com/llvm/llvm-project/pull/143055 */
119	113	# elif defined(__NetBSD__)
120	114	# include <__locale_dir/support/netbsd.h>
121	115	# elif defined(_LIBCPP_MSVCRT_LIKE)
...	...	@@ -124,20 +118,20 @@
124	118	# include <__locale_dir/support/fuchsia.h>
125	119	# elif defined(__linux__)
126	120	# include <__locale_dir/support/linux.h>
	121	# elif _LIBCPP_LIBC_NEWLIB
	122	# include <__locale_dir/support/newlib.h>
	123	# elif defined(_AIX)
	124	# include <__locale_dir/support/aix.h>
127	125	# else
128	126
129	127	// TODO: This is a temporary definition to bridge between the old way we defined the locale base API
130	128	// (by providing global non-reserved names) and the new API. As we move individual platforms
131	129	// towards the new way of defining the locale base API, this should disappear since each platform
132	130	// will define those directly.
133		# if defined(_AIX) \|\| defined(__MVS__)
	131	# if defined(__MVS__)
134	132	# include <__locale_dir/locale_base_api/ibm.h>
135		# elif defined(__ANDROID__)
136		# include <__locale_dir/locale_base_api/android.h>
137	133	# elif defined(__OpenBSD__)
138	134	# include <__locale_dir/locale_base_api/openbsd.h>
139		# elif defined(__wasi__) \|\| _LIBCPP_HAS_MUSL_LIBC
140		# include <__locale_dir/locale_base_api/musl.h>
141	135	# endif
142	136
143	137	# include <__locale_dir/locale_base_api/bsd_locale_fallbacks.h>
...	...	@@ -197,21 +191,9 @@ inline _LIBCPP_HIDE_FROM_ABI long double __strtold(const char* __nptr, char** __
197	191	return strtold_l(__nptr, __endptr, __loc);
198	192	}
199	193
200		inline _LIBCPP_HIDE_FROM_ABI long long __strtoll(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
201		return strtoll_l(__nptr, __endptr, __base, __loc);
202		}
203
204		inline _LIBCPP_HIDE_FROM_ABI unsigned long long
205		__strtoull(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
206		return strtoull_l(__nptr, __endptr, __base, __loc);
207		}
208
209	194	//
210	195	// Character manipulation functions
211	196	//
212		inline _LIBCPP_HIDE_FROM_ABI int __isdigit(int __ch, __locale_t __loc) { return isdigit_l(__ch, __loc); }
213		inline _LIBCPP_HIDE_FROM_ABI int __isxdigit(int __ch, __locale_t __loc) { return isxdigit_l(__ch, __loc); }
214
215	197	# if defined(_LIBCPP_BUILDING_LIBRARY)
216	198	inline _LIBCPP_HIDE_FROM_ABI int __strcoll(const char* __s1, const char* __s2, __locale_t __loc) {
217	199	return strcoll_l(__s1, __s2, __loc);
...	...	@@ -307,11 +289,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __
307	289	char** __s, __locale_t __loc, const char* __format, _Args&&... __args) {
308	290	return std::__libcpp_asprintf_l(__s, __loc, __format, std::forward<_Args>(__args)...);
309	291	}
310		template <class... _Args>
311		_LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __sscanf(
312		const char* __s, __locale_t __loc, const char* __format, _Args&&... __args) {
313		return std::__libcpp_sscanf_l(__s, __loc, __format, std::forward<_Args>(__args)...);
314		}
315	292	_LIBCPP_DIAGNOSTIC_POP
316	293	# undef _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT
317	294

lib/libcxx/include/__locale_dir/locale_base_api/bsd_locale_fallbacks.h-10

...	...	@@ -125,16 +125,6 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __libcpp_asprintf_l(
125	125	return __res;
126	126	}
127	127
128		inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __libcpp_sscanf_l(
129		const char* __s, locale_t __l, const char* __format, ...) {
130		va_list __va;
131		va_start(__va, __format);
132		__locale_guard __current(__l);
133		int __res = vsscanf(__s, __format, __va);
134		va_end(__va);
135		return __res;
136		}
137
138	128	_LIBCPP_END_NAMESPACE_STD
139	129
140	130	#endif // _LIBCPP___LOCALE_DIR_LOCALE_BASE_API_BSD_LOCALE_FALLBACKS_H

lib/libcxx/include/__locale_dir/messages.h+1-1

...	...	@@ -22,7 +22,7 @@
22	22
23	23	# if defined(__unix__) \|\| (defined(__APPLE__) && defined(__MACH__))
24	24	// Most unix variants have catopen. These are the specific ones that don't.
25		# if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
	25	# if !defined(__BIONIC__) && !_LIBCPP_LIBC_NEWLIB && !defined(__EMSCRIPTEN__)
26	26	# define _LIBCPP_HAS_CATOPEN 1
27	27	# include <nl_types.h>
28	28	# else

lib/libcxx/include/__locale_dir/money.h+2-2

...	...	@@ -433,7 +433,7 @@ bool money_get<_CharT, _InputIterator>::__do_get(
433	433	__err \|= ios_base::failbit;
434	434	return false;
435	435	}
436		for (++__b; __fd > 0; --__fd, ++__b) {
	436	for (++__b; __fd > 0; --__fd, (void)++__b) {
437	437	if (__b == __e \|\| !__ct.is(ctype_base::digit, *__b)) {
438	438	__err \|= ios_base::failbit;
439	439	return false;
...	...	@@ -451,7 +451,7 @@ bool money_get<_CharT, _InputIterator>::__do_get(
451	451	}
452	452	}
453	453	if (__trailing_sign) {
454		for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
	454	for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, (void)++__b) {
455	455	if (__b == __e \|\| __b != (__trailing_sign)[__i]) {
456	456	__err \|= ios_base::failbit;
457	457	return false;

lib/libcxx/include/__locale_dir/num.h+190-243

...	...	@@ -9,8 +9,10 @@
9	9	#ifndef _LIBCPP___LOCALE_DIR_NUM_H
10	10	#define _LIBCPP___LOCALE_DIR_NUM_H
11	11
	12	#include <__algorithm/copy.h>
12	13	#include <__algorithm/find.h>
13	14	#include <__algorithm/reverse.h>
	15	#include <__algorithm/simd_utils.h>
14	16	#include <__charconv/to_chars_integral.h>
15	17	#include <__charconv/traits.h>
16	18	#include <__config>
...	...	@@ -22,6 +24,7 @@
22	24	#include <__locale_dir/scan_keyword.h>
23	25	#include <__memory/unique_ptr.h>
24	26	#include <__system_error/errc.h>
	27	#include <__type_traits/is_signed.h>
25	28	#include <cerrno>
26	29	#include <ios>
27	30	#include <streambuf>
...	...	@@ -46,9 +49,9 @@ struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
46	49	static int __get_base(ios_base&);
47	50	static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
48	51	// count of leading characters in __src used for parsing integers ("012..X+-")
49		static const size_t __int_chr_cnt = 26;
	52	static inline const size_t __int_chr_cnt = 26;
50	53	// count of leading characters in __src used for parsing floating-point values ("012..-pP")
51		static const size_t __fp_chr_cnt = 28;
	54	static inline const size_t __fp_chr_cnt = 28;
52	55	};
53	56
54	57	template <class _CharT>
...	...	@@ -71,7 +74,8 @@ struct __num_get : protected __num_get_base {
71	74
72	75	[[__deprecated__("This exists only for ABI compatibility")]] static string
73	76	__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
74		static int __stage2_int_loop(
	77
	78	[[__deprecated__("This exists only for ABI compatibility")]] static int __stage2_int_loop(
75	79	_CharT __ct,
76	80	int __base,
77	81	char* __a,
...	...	@@ -83,11 +87,24 @@ struct __num_get : protected __num_get_base {
83	87	unsigned*& __g_end,
84	88	_CharT* __atoms);
85	89
86		_LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
87		locale __loc = __iob.getloc();
88		const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
89		__thousands_sep = __np.thousands_sep();
90		return __np.grouping();
	90	_LIBCPP_HIDE_FROM_ABI static ptrdiff_t __atoms_offset(const _CharT* __atoms, _CharT __val) {
	91	// TODO: Remove the manual vectorization once https://llvm.org/PR168551 is resolved
	92	# if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
	93	if constexpr (is_same<_CharT, char>::value) {
	94	// TODO(LLVM 24): This can be removed, since -Wpsabi doesn't warn on [[gnu::always_inline]] functions anymore.
	95	_LIBCPP_DIAGNOSTIC_PUSH
	96	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wpsabi")
	97	using __vec = __simd_vector<char, 32>;
	98	__vec __chars = std::__broadcast<__vec>(__val);
	99	__vec __cmp = std::__partial_load<__vec, __int_chr_cnt>(__atoms);
	100	auto __res = __chars == __cmp;
	101	if (std::__none_of(__res))
	102	return __int_chr_cnt;
	103	return std::min(__int_chr_cnt, std::__find_first_set(__res));
	104	_LIBCPP_DIAGNOSTIC_POP
	105	}
	106	# endif
	107	return std::find(__atoms, __atoms + __int_chr_cnt, __val) - __atoms;
91	108	}
92	109
93	110	_LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
...	...	@@ -120,54 +137,6 @@ string __num_get<_CharT>::__stage2_float_prep(
120	137	return __np.grouping();
121	138	}
122	139
123		template <class _CharT>
124		int __num_get<_CharT>::__stage2_int_loop(
125		_CharT __ct,
126		int __base,
127		char* __a,
128		char*& __a_end,
129		unsigned& __dc,
130		_CharT __thousands_sep,
131		const string& __grouping,
132		unsigned* __g,
133		unsigned*& __g_end,
134		_CharT* __atoms) {
135		if (__a_end == __a && (__ct == __atoms[24] \|\| __ct == __atoms[25])) {
136		*__a_end++ = __ct == __atoms[24] ? '+' : '-';
137		__dc = 0;
138		return 0;
139		}
140		if (__grouping.size() != 0 && __ct == __thousands_sep) {
141		if (__g_end - __g < __num_get_buf_sz) {
142		*__g_end++ = __dc;
143		__dc = 0;
144		}
145		return 0;
146		}
147		ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
148		if (__f >= 24)
149		return -1;
150		switch (__base) {
151		case 8:
152		case 10:
153		if (__f >= __base)
154		return -1;
155		break;
156		case 16:
157		if (__f < 22)
158		break;
159		if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
160		__dc = 0;
161		*__a_end++ = __src[__f];
162		return 0;
163		}
164		return -1;
165		}
166		*__a_end++ = __src[__f];
167		++__dc;
168		return 0;
169		}
170
171	140	template <class _CharT>
172	141	int __num_get<_CharT>::__stage2_float_loop(
173	142	_CharT __ct,
...	...	@@ -272,65 +241,6 @@ _LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end,
272	241	return 0;
273	242	}
274	243
275		template <class _Tp>
276		_LIBCPP_HIDE_FROM_ABI _Tp
277		__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
278		if (__a != __a_end) {
279		__libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
280		errno = 0;
281		char* __p2;
282		long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
283		__libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
284		if (__current_errno == 0)
285		errno = __save_errno;
286		if (__p2 != __a_end) {
287		__err = ios_base::failbit;
288		return 0;
289		} else if (__current_errno == ERANGE \|\| __ll < numeric_limits<_Tp>::min() \|\| numeric_limits<_Tp>::max() < __ll) {
290		__err = ios_base::failbit;
291		if (__ll > 0)
292		return numeric_limits<_Tp>::max();
293		else
294		return numeric_limits<_Tp>::min();
295		}
296		return static_cast<_Tp>(__ll);
297		}
298		__err = ios_base::failbit;
299		return 0;
300		}
301
302		template <class _Tp>
303		_LIBCPP_HIDE_FROM_ABI _Tp
304		__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
305		if (__a != __a_end) {
306		const bool __negate = *__a == '-';
307		if (__negate && ++__a == __a_end) {
308		__err = ios_base::failbit;
309		return 0;
310		}
311		__libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
312		errno = 0;
313		char* __p2;
314		unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
315		__libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
316		if (__current_errno == 0)
317		errno = __save_errno;
318		if (__p2 != __a_end) {
319		__err = ios_base::failbit;
320		return 0;
321		} else if (__current_errno == ERANGE \|\| numeric_limits<_Tp>::max() < __ll) {
322		__err = ios_base::failbit;
323		return numeric_limits<_Tp>::max();
324		}
325		_Tp __res = static_cast<_Tp>(__ll);
326		if (__negate)
327		__res = -__res;
328		return __res;
329		}
330		__err = ios_base::failbit;
331		return 0;
332		}
333
334	244	template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
335	245	class num_get : public locale::facet, private __num_get<_CharT> {
336	246	public:
...	...	@@ -468,137 +378,196 @@ protected:
468	378	return __b;
469	379	}
470	380
471		template <class _Signed>
472		_LIBCPP_HIDE_FROM_ABI iter_type
473		__do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
	381	template <class _MaybeSigned>
	382	iter_type __do_get_integral(
	383	iter_type __first, iter_type __last, ios_base& __iob, ios_base::iostate& __err, _MaybeSigned& __v) const {
	384	using _Unsigned = __make_unsigned_t<_MaybeSigned>;
	385
474	386	// Stage 1
475	387	int __base = this->__get_base(__iob);
476		// Stage 2
477		char_type __thousands_sep;
478		const int __atoms_size = __num_get_base::__int_chr_cnt;
479		char_type __atoms1[__atoms_size];
480		const char_type* __atoms = this->__do_widen(__iob, __atoms1);
481		string __grouping = this->__stage2_int_prep(__iob, __thousands_sep);
482		string __buf;
483		__buf.resize(__buf.capacity());
484		char* __a = &__buf[0];
485		char* __a_end = __a;
	388
	389	// Stages 2 & 3
	390	// These are combined into a single step where we parse the characters and calculate the value in one go instead of
	391	// storing the relevant characters first (in an allocated buffer) and parse the characters after we extracted them.
	392	// This makes the whole process significantly faster, since we avoid potential allocations and copies.
	393
	394	const auto& __numpunct = use_facet<numpunct<_CharT> >(__iob.getloc());
	395	char_type __thousands_sep = __numpunct.thousands_sep();
	396	string __grouping = __numpunct.grouping();
	397
	398	char_type __atoms_buffer[__num_get_base::__int_chr_cnt];
	399	const char_type* __atoms = this->__do_widen(__iob, __atoms_buffer);
486	400	unsigned __g[__num_get_base::__num_get_buf_sz];
487	401	unsigned* __g_end = __g;
488	402	unsigned __dc = 0;
489		for (; __b != __e; ++__b) {
490		if (__a_end == __a + __buf.size()) {
491		size_t __tmp = __buf.size();
492		__buf.resize(2 * __buf.size());
493		__buf.resize(__buf.capacity());
494		__a = &__buf[0];
495		__a_end = __a + __tmp;
	403
	404	if (__first == __last) {
	405	__err \|= ios_base::eofbit \| ios_base::failbit;
	406	__v = 0;
	407	return __first;
	408	}
	409
	410	while (!__grouping.empty() && *__first == __thousands_sep) {
	411	++__first;
	412	if (__g_end - __g < this->__num_get_buf_sz)
	413	*__g_end++ = 0;
	414	}
	415
	416	bool __negate = false;
	417	// __c == '+' \|\| __c == '-'
	418	if (auto __c = *__first; __c == __atoms[24] \|\| __c == __atoms[25]) {
	419	__negate = __c == __atoms[25];
	420	++__first;
	421	}
	422
	423	if (__first == __last) {
	424	__err \|= ios_base::eofbit \| ios_base::failbit;
	425	__v = 0;
	426	return __first;
	427	}
	428
	429	bool __parsed_num = false;
	430
	431	// If we don't have a pre-set base, figure it out and swallow any prefix
	432	if (__base == 0) {
	433	auto __c = *__first;
	434	// __c == '0'
	435	if (__c == __atoms[0]) {
	436	++__first;
	437	if (__first == __last) {
	438	__err \|= ios_base::eofbit;
	439	__v = 0;
	440	return __first;
	441	}
	442	// __c2 == 'x' \|\| __c2 == 'X'
	443	if (auto __c2 = *__first; __c2 == __atoms[22] \|\| __c2 == __atoms[23]) {
	444	__base = 16;
	445	++__first;
	446	} else {
	447	__base = 8;
	448	__parsed_num = true; // We only swallowed '0', so we've started to parse a number
	449	}
	450	} else {
	451	__base = 10;
	452	}
	453
	454	// If the base has been specified explicitly, try to swallow the appropriate prefix. We only need to do something
	455	// special for hex, since decimal has no prefix and octal's prefix is '0', which doesn't change the value that
	456	// we'll parse if we don't swallow it.
	457	} else if (__base == 16) {
	458	// Try to swallow '0x'
	459
	460	// *__first == '0'
	461	if (*__first == __atoms[0]) {
	462	++__first;
	463	if (__first == __last) {
	464	__err \|= ios_base::eofbit;
	465	__v = 0;
	466	return __first;
	467	}
	468	// __c == 'x' \|\| __c == 'X'
	469	if (auto __c = *__first; __c == __atoms[22] \|\| __c == __atoms[23])
	470	++__first;
	471	else
	472	__parsed_num = true; // We only swallowed '0', so we've started to parse a number
496	473	}
497		if (this->__stage2_int_loop(
498		*__b,
499		__base,
500		__a,
501		__a_end,
502		__dc,
503		__thousands_sep,
504		__grouping,
505		__g,
506		__g_end,
507		const_cast<char_type*>(__atoms)))
508		break;
509	474	}
510		if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
511		*__g_end++ = __dc;
512		// Stage 3
513		__v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
514		// Digit grouping checked
515		__check_grouping(__grouping, __g, __g_end, __err);
516		// EOF checked
517		if (__b == __e)
518		__err \|= ios_base::eofbit;
519		return __b;
520		}
521	475
522		template <class _Unsigned>
523		_LIBCPP_HIDE_FROM_ABI iter_type
524		__do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
525		// Stage 1
526		int __base = this->__get_base(__iob);
527		// Stage 2
528		char_type __thousands_sep;
529		const int __atoms_size = __num_get_base::__int_chr_cnt;
530		char_type __atoms1[__atoms_size];
531		const char_type* __atoms = this->__do_widen(__iob, __atoms1);
532		string __grouping = this->__stage2_int_prep(__iob, __thousands_sep);
533		string __buf;
534		__buf.resize(__buf.capacity());
535		char* __a = &__buf[0];
536		char* __a_end = __a;
537		unsigned __g[__num_get_base::__num_get_buf_sz];
538		unsigned* __g_end = __g;
539		unsigned __dc = 0;
540		for (; __b != __e; ++__b) {
541		if (__a_end == __a + __buf.size()) {
542		size_t __tmp = __buf.size();
543		__buf.resize(2 * __buf.size());
544		__buf.resize(__buf.capacity());
545		__a = &__buf[0];
546		__a_end = __a + __tmp;
	476	// Calculate the actual number
	477	_Unsigned __val = 0;
	478	bool __overflowed = false;
	479	for (; __first != __last; ++__first) {
	480	auto __c = *__first;
	481	if (!__grouping.empty() && __c == __thousands_sep) {
	482	if (__g_end - __g < this->__num_get_buf_sz) {
	483	*__g_end++ = __dc;
	484	__dc = 0;
	485	}
	486	continue;
547	487	}
548		if (this->__stage2_int_loop(
549		*__b,
550		__base,
551		__a,
552		__a_end,
553		__dc,
554		__thousands_sep,
555		__grouping,
556		__g,
557		__g_end,
558		const_cast<char_type*>(__atoms)))
	488	auto __offset = this->__atoms_offset(__atoms, __c);
	489	if (__offset >= 22) // Not a valid integer character
	490	break;
	491
	492	if (__base == 16 && __offset >= 16)
	493	__offset -= 6;
	494	if (__offset >= __base)
559	495	break;
	496	// __val = (__val * __base) + __offset
	497	__overflowed \|= __builtin_mul_overflow(__val, __base, std::addressof(__val)) \|\|
	498	__builtin_add_overflow(__val, __offset, std::addressof(__val));
	499	__parsed_num = true;
	500	++__dc;
	501	}
	502
	503	if (!__parsed_num) {
	504	__err \|= ios_base::failbit;
	505	__v = 0;
	506	} else if (__overflowed) {
	507	__err \|= ios_base::failbit;
	508	__v = is_signed<_MaybeSigned>::value && __negate
	509	? numeric_limits<_MaybeSigned>::min()
	510	: numeric_limits<_MaybeSigned>::max();
	511	} else if (!__negate) {
	512	if (__val > static_cast<_Unsigned>(numeric_limits<_MaybeSigned>::max())) {
	513	__err \|= ios_base::failbit;
	514	__v = numeric_limits<_MaybeSigned>::max();
	515	} else {
	516	__v = __val;
	517	}
	518	} else if (is_signed<_MaybeSigned>::value) {
	519	if (__val > static_cast<_Unsigned>(numeric_limits<_MaybeSigned>::max()) + 1) {
	520	__err \|= ios_base::failbit;
	521	__v = numeric_limits<_MaybeSigned>::min();
	522	} else if (__val == static_cast<_Unsigned>(numeric_limits<_MaybeSigned>::max()) + 1) {
	523	__v = numeric_limits<_MaybeSigned>::min();
	524	} else {
	525	__v = -__val;
	526	}
	527	} else {
	528	__v = -__val;
560	529	}
	530
561	531	if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
562	532	*__g_end++ = __dc;
563		// Stage 3
564		__v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
	533
565	534	// Digit grouping checked
566	535	__check_grouping(__grouping, __g, __g_end, __err);
567	536	// EOF checked
568		if (__b == __e)
	537	if (__first == __last)
569	538	__err \|= ios_base::eofbit;
570		return __b;
	539	return __first;
571	540	}
572	541
573	542	virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
574	543
575	544	virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
576		return this->__do_get_signed(__b, __e, __iob, __err, __v);
	545	return this->__do_get_integral(__b, __e, __iob, __err, __v);
577	546	}
578	547
579	548	virtual iter_type
580	549	do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
581		return this->__do_get_signed(__b, __e, __iob, __err, __v);
	550	return this->__do_get_integral(__b, __e, __iob, __err, __v);
582	551	}
583	552
584	553	virtual iter_type
585	554	do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
586		return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
	555	return this->__do_get_integral(__b, __e, __iob, __err, __v);
587	556	}
588	557
589	558	virtual iter_type
590	559	do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
591		return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
	560	return this->__do_get_integral(__b, __e, __iob, __err, __v);
592	561	}
593	562
594	563	virtual iter_type
595	564	do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
596		return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
	565	return this->__do_get_integral(__b, __e, __iob, __err, __v);
597	566	}
598	567
599	568	virtual iter_type
600	569	do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
601		return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
	570	return this->__do_get_integral(__b, __e, __iob, __err, __v);
602	571	}
603	572
604	573	virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
...	...	@@ -652,40 +621,13 @@ _InputIterator num_get<_CharT, _InputIterator>::do_get(
652	621	template <class _CharT, class _InputIterator>
653	622	_InputIterator num_get<_CharT, _InputIterator>::do_get(
654	623	iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
655		// Stage 1
656		int __base = 16;
657		// Stage 2
658		char_type __atoms[__num_get_base::__int_chr_cnt];
659		char_type __thousands_sep = char_type();
660		string __grouping;
661		std::use_facet<ctype<_CharT> >(__iob.getloc())
662		.widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
663		string __buf;
664		__buf.resize(__buf.capacity());
665		char* __a = &__buf[0];
666		char* __a_end = __a;
667		unsigned __g[__num_get_base::__num_get_buf_sz];
668		unsigned* __g_end = __g;
669		unsigned __dc = 0;
670		for (; __b != __e; ++__b) {
671		if (__a_end == __a + __buf.size()) {
672		size_t __tmp = __buf.size();
673		__buf.resize(2 * __buf.size());
674		__buf.resize(__buf.capacity());
675		__a = &__buf[0];
676		__a_end = __a + __tmp;
677		}
678		if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
679		break;
680		}
681		// Stage 3
682		__buf.resize(__a_end - __a);
683		if (__locale::__sscanf(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
684		__err = ios_base::failbit;
685		// EOF checked
686		if (__b == __e)
687		__err \|= ios_base::eofbit;
688		return __b;
	624	auto __flags = __iob.flags();
	625	__iob.flags((__flags & ~ios_base::basefield & ~ios_base::uppercase) \| ios_base::hex);
	626	uintptr_t __ptr;
	627	auto __res = __do_get_integral(__b, __e, __iob, __err, __ptr);
	628	__iob.flags(__flags);
	629	__v = reinterpret_cast<void*>(__ptr);
	630	return __res;
689	631	}
690	632
691	633	extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
...	...	@@ -748,6 +690,13 @@ void __num_put<_CharT>::__widen_and_group_int(
748	690	__op = __ob + (__np - __nb);
749	691	}
750	692
	693	_LIBCPP_HIDE_FROM_ABI inline bool __isdigit(char __c) { return __c >= '0' && __c <= '9'; }
	694
	695	_LIBCPP_HIDE_FROM_ABI inline bool __isxdigit(char __c) {
	696	auto __lower = __c \| 0x20;
	697	return std::__isdigit(__c) \|\| (__lower >= 'a' && __lower <= 'f');
	698	}
	699
751	700	template <class _CharT>
752	701	void __num_put<_CharT>::__widen_and_group_float(
753	702	char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT& __op, _CharT& __oe, const locale& __loc) {
...	...	@@ -763,11 +712,11 @@ void __num_put<_CharT>::__widen_and_group_float(
763	712	__oe++ = __ct.widen(__nf++);
764	713	__oe++ = __ct.widen(__nf++);
765	714	for (__ns = __nf; __ns < __ne; ++__ns)
766		if (!__locale::__isxdigit(*__ns, _LIBCPP_GET_C_LOCALE))
	715	if (!std::__isxdigit(*__ns))
767	716	break;
768	717	} else {
769	718	for (__ns = __nf; __ns < __ne; ++__ns)
770		if (!__locale::__isdigit(*__ns, _LIBCPP_GET_C_LOCALE))
	719	if (!std::__isdigit(*__ns))
771	720	break;
772	721	}
773	722	if (__grouping.empty()) {
...	...	@@ -885,9 +834,7 @@ num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_ty
885	834	const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
886	835	typedef typename numpunct<char_type>::string_type string_type;
887	836	string_type __nm = __v ? __np.truename() : __np.falsename();
888		for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
889		__s = __i;
890		return __s;
	837	return std::copy(__nm.begin(), __nm.end(), __s);
891	838	}
892	839
893	840	template <class _CharT, class _OutputIterator>

lib/libcxx/include/__locale_dir/pad_and_output.h+5-6

...	...	@@ -13,6 +13,8 @@
13	13
14	14	#if _LIBCPP_HAS_LOCALIZATION
15	15
	16	# include <__algorithm/copy.h>
	17	# include <__algorithm/fill_n.h>
16	18	# include <ios>
17	19
18	20	# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -30,12 +32,9 @@ _LIBCPP_HIDE_FROM_ABI _OutputIterator __pad_and_output(
30	32	__ns -= __sz;
31	33	else
32	34	__ns = 0;
33		for (; __ob < __op; ++__ob, ++__s)
34		__s = __ob;
35		for (; __ns; --__ns, ++__s)
36		*__s = __fl;
37		for (; __ob < __oe; ++__ob, ++__s)
38		__s = __ob;
	35	__s = std::copy(__ob, __op, __s);
	36	__s = std::fill_n(__s, __ns, __fl);
	37	__s = std::copy(__op, __oe, __s);
39	38	__iob.width(0);
40	39	return __s;
41	40	}

lib/libcxx/include/__locale_dir/support/bsd_like.h-20

...	...	@@ -24,7 +24,6 @@
24	24	# include <wctype.h>
25	25	#endif
26	26
27		/* zig patch: https://github.com/llvm/llvm-project/pull/143055 */
28	27	#if __has_include(<xlocale.h>)
29	28	# include <xlocale.h>
30	29	#endif
...	...	@@ -80,22 +79,9 @@ inline _LIBCPP_HIDE_FROM_ABI long double __strtold(const char* __nptr, char** __
80	79	return ::strtold_l(__nptr, __endptr, __loc);
81	80	}
82	81
83		inline _LIBCPP_HIDE_FROM_ABI long long __strtoll(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
84		return ::strtoll_l(__nptr, __endptr, __base, __loc);
85		}
86
87		inline _LIBCPP_HIDE_FROM_ABI unsigned long long
88		__strtoull(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
89		return ::strtoull_l(__nptr, __endptr, __base, __loc);
90		}
91
92	82	//
93	83	// Character manipulation functions
94	84	//
95		inline _LIBCPP_HIDE_FROM_ABI int __isdigit(int __c, __locale_t __loc) { return ::isdigit_l(__c, __loc); }
96
97		inline _LIBCPP_HIDE_FROM_ABI int __isxdigit(int __c, __locale_t __loc) { return ::isxdigit_l(__c, __loc); }
98
99	85	#if defined(_LIBCPP_BUILDING_LIBRARY)
100	86	inline _LIBCPP_HIDE_FROM_ABI int __toupper(int __c, __locale_t __loc) { return ::toupper_l(__c, __loc); }
101	87
...	...	@@ -216,12 +202,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __
216	202	char** __s, __locale_t __loc, const char* __format, _Args&&... __args) {
217	203	return ::asprintf_l(__s, __loc, __format, std::forward<_Args>(__args)...); // non-standard
218	204	}
219
220		template <class... _Args>
221		_LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __sscanf(
222		const char* __s, __locale_t __loc, const char* __format, _Args&&... __args) {
223		return ::sscanf_l(__s, __loc, __format, std::forward<_Args>(__args)...);
224		}
225	205	_LIBCPP_DIAGNOSTIC_POP
226	206	#undef _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT
227	207

lib/libcxx/include/__locale_dir/support/fuchsia.h-7

...	...	@@ -141,13 +141,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __
141	141	__locale_guard __current(__loc);
142	142	return ::asprintf(__s, __format, std::forward<_Args>(__args)...); // non-standard
143	143	}
144		template <class... _Args>
145		_LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __sscanf(
146		const char* __s, __locale_t __loc, const char* __format, _Args&&... __args) {
147		__locale_guard __current(__loc);
148		return std::sscanf(__s, __format, std::forward<_Args>(__args)...);
149		}
150
151	144	_LIBCPP_DIAGNOSTIC_POP
152	145	#undef _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT
153	146

lib/libcxx/include/__locale_dir/support/linux.h-37

...	...	@@ -94,32 +94,9 @@ inline _LIBCPP_HIDE_FROM_ABI long double __strtold(const char* __nptr, char** __
94	94	return ::strtold_l(__nptr, __endptr, __loc);
95	95	}
96	96
97		inline _LIBCPP_HIDE_FROM_ABI long long __strtoll(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
98		#if !_LIBCPP_HAS_MUSL_LIBC
99		return ::strtoll_l(__nptr, __endptr, __base, __loc);
100		#else
101		(void)__loc;
102		return ::strtoll(__nptr, __endptr, __base);
103		#endif
104		}
105
106		inline _LIBCPP_HIDE_FROM_ABI unsigned long long
107		__strtoull(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
108		#if !_LIBCPP_HAS_MUSL_LIBC
109		return ::strtoull_l(__nptr, __endptr, __base, __loc);
110		#else
111		(void)__loc;
112		return ::strtoull(__nptr, __endptr, __base);
113		#endif
114		}
115
116	97	//
117	98	// Character manipulation functions
118	99	//
119		inline _LIBCPP_HIDE_FROM_ABI int __isdigit(int __c, __locale_t __loc) { return isdigit_l(__c, __loc); }
120
121		inline _LIBCPP_HIDE_FROM_ABI int __isxdigit(int __c, __locale_t __loc) { return isxdigit_l(__c, __loc); }
122
123	100	#if defined(_LIBCPP_BUILDING_LIBRARY)
124	101	inline _LIBCPP_HIDE_FROM_ABI int __toupper(int __c, __locale_t __loc) { return toupper_l(__c, __loc); }
125	102
...	...	@@ -261,20 +238,6 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __asprintf(
261	238	va_end(__va);
262	239	return __res;
263	240	}
264
265		#ifndef _LIBCPP_COMPILER_GCC // GCC complains that this can't be always_inline due to C-style varargs
266		_LIBCPP_HIDE_FROM_ABI
267		#endif
268		inline _LIBCPP_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __sscanf(
269		const char* __s, __locale_t __loc, const char* __format, ...) {
270		va_list __va;
271		va_start(__va, __format);
272		__locale_guard __current(__loc);
273		int __res = std::vsscanf(__s, __format, __va);
274		va_end(__va);
275		return __res;
276		}
277
278	241	} // namespace __locale
279	242	_LIBCPP_END_NAMESPACE_STD
280	243

lib/libcxx/include/__locale_dir/support/netbsd.h-2

...	...	@@ -6,8 +6,6 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8
9		/* zig patch: https://github.com/llvm/llvm-project/pull/143055 */
10
11	9	#ifndef _LIBCPP___LOCALE_DIR_SUPPORT_NETBSD_H
12	10	#define _LIBCPP___LOCALE_DIR_SUPPORT_NETBSD_H
13	11

lib/libcxx/include/__locale_dir/support/newlib.h created+243

...	...	@@ -0,0 +1,243 @@
	1	//===-----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___LOCALE_DIR_SUPPORT_NEWLIB_H
	10	#define _LIBCPP___LOCALE_DIR_SUPPORT_NEWLIB_H
	11
	12	#include <__config>
	13	#include <__cstddef/size_t.h>
	14	#include <__std_mbstate_t.h>
	15	#include <clocale> // std::lconv
	16	#include <cstdio>
	17	#include <cstdlib>
	18	#include <ctype.h>
	19	#include <stdarg.h>
	20	#include <string.h>
	21	#include <time.h>
	22	#if _LIBCPP_HAS_WIDE_CHARACTERS
	23	# include <cwchar>
	24	# include <wctype.h>
	25	#endif
	26
	27	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	28	# pragma GCC system_header
	29	#endif
	30
	31	_LIBCPP_BEGIN_NAMESPACE_STD
	32	namespace __locale {
	33
	34	struct __locale_guard {
	35	_LIBCPP_HIDE_FROM_ABI __locale_guard(locale_t& __loc) : __old_loc_(::uselocale(__loc)) {}
	36
	37	_LIBCPP_HIDE_FROM_ABI ~__locale_guard() {
	38	if (__old_loc_)
	39	::uselocale(__old_loc_);
	40	}
	41
	42	locale_t __old_loc_;
	43
	44	__locale_guard(__locale_guard const&) = delete;
	45	__locale_guard& operator=(__locale_guard const&) = delete;
	46	};
	47
	48	//
	49	// Locale management
	50	//
	51	#define _LIBCPP_COLLATE_MASK LC_COLLATE_MASK
	52	#define _LIBCPP_CTYPE_MASK LC_CTYPE_MASK
	53	#define _LIBCPP_MONETARY_MASK LC_MONETARY_MASK
	54	#define _LIBCPP_NUMERIC_MASK LC_NUMERIC_MASK
	55	#define _LIBCPP_TIME_MASK LC_TIME_MASK
	56	#define _LIBCPP_MESSAGES_MASK LC_MESSAGES_MASK
	57	#define _LIBCPP_ALL_MASK LC_ALL_MASK
	58	#define _LIBCPP_LC_ALL LC_ALL
	59
	60	using __locale_t _LIBCPP_NODEBUG = ::locale_t;
	61
	62	#if defined(_LIBCPP_BUILDING_LIBRARY)
	63	using __lconv_t _LIBCPP_NODEBUG = std::lconv;
	64
	65	inline _LIBCPP_HIDE_FROM_ABI __locale_t __newlocale(int __category_mask, const char* __locale, __locale_t __base) {
	66	return ::newlocale(__category_mask, __locale, __base);
	67	}
	68
	69	inline _LIBCPP_HIDE_FROM_ABI void __freelocale(__locale_t __loc) { ::freelocale(__loc); }
	70
	71	inline _LIBCPP_HIDE_FROM_ABI char* __setlocale(int __category, char const* __locale) {
	72	return ::setlocale(__category, __locale);
	73	}
	74
	75	inline _LIBCPP_HIDE_FROM_ABI __lconv_t* __localeconv(__locale_t& __loc) {
	76	__locale_guard __current(__loc);
	77	return std::localeconv();
	78	}
	79	#endif // _LIBCPP_BUILDING_LIBRARY
	80
	81	//
	82	// Strtonum functions
	83	//
	84	inline _LIBCPP_HIDE_FROM_ABI float __strtof(const char* __nptr, char** __endptr, __locale_t __loc) {
	85	return ::strtof_l(__nptr, __endptr, __loc);
	86	}
	87
	88	inline _LIBCPP_HIDE_FROM_ABI double __strtod(const char* __nptr, char** __endptr, __locale_t __loc) {
	89	return ::strtod_l(__nptr, __endptr, __loc);
	90	}
	91
	92	inline _LIBCPP_HIDE_FROM_ABI long double __strtold(const char* __nptr, char** __endptr, __locale_t __loc) {
	93	return ::strtold_l(__nptr, __endptr, __loc);
	94	}
	95
	96	//
	97	// Character manipulation functions
	98	//
	99	#if defined(_LIBCPP_BUILDING_LIBRARY)
	100	inline _LIBCPP_HIDE_FROM_ABI int __toupper(int __c, __locale_t __loc) { return toupper_l(__c, __loc); }
	101
	102	inline _LIBCPP_HIDE_FROM_ABI int __tolower(int __c, __locale_t __loc) { return tolower_l(__c, __loc); }
	103
	104	inline _LIBCPP_HIDE_FROM_ABI int __strcoll(const char* __s1, const char* __s2, __locale_t __loc) {
	105	return strcoll_l(__s1, __s2, __loc);
	106	}
	107
	108	inline _LIBCPP_HIDE_FROM_ABI size_t __strxfrm(char* __dest, const char* __src, size_t __n, __locale_t __loc) {
	109	return strxfrm_l(__dest, __src, __n, __loc);
	110	}
	111
	112	# if _LIBCPP_HAS_WIDE_CHARACTERS
	113	inline _LIBCPP_HIDE_FROM_ABI int __iswctype(wint_t __c, wctype_t __type, __locale_t __loc) {
	114	return iswctype_l(__c, __type, __loc);
	115	}
	116
	117	inline _LIBCPP_HIDE_FROM_ABI int __iswspace(wint_t __c, __locale_t __loc) { return iswspace_l(__c, __loc); }
	118
	119	inline _LIBCPP_HIDE_FROM_ABI int __iswprint(wint_t __c, __locale_t __loc) { return iswprint_l(__c, __loc); }
	120
	121	inline _LIBCPP_HIDE_FROM_ABI int __iswcntrl(wint_t __c, __locale_t __loc) { return iswcntrl_l(__c, __loc); }
	122
	123	inline _LIBCPP_HIDE_FROM_ABI int __iswupper(wint_t __c, __locale_t __loc) { return iswupper_l(__c, __loc); }
	124
	125	inline _LIBCPP_HIDE_FROM_ABI int __iswlower(wint_t __c, __locale_t __loc) { return iswlower_l(__c, __loc); }
	126
	127	inline _LIBCPP_HIDE_FROM_ABI int __iswalpha(wint_t __c, __locale_t __loc) { return iswalpha_l(__c, __loc); }
	128
	129	inline _LIBCPP_HIDE_FROM_ABI int __iswblank(wint_t __c, __locale_t __loc) { return iswblank_l(__c, __loc); }
	130
	131	inline _LIBCPP_HIDE_FROM_ABI int __iswdigit(wint_t __c, __locale_t __loc) { return iswdigit_l(__c, __loc); }
	132
	133	inline _LIBCPP_HIDE_FROM_ABI int __iswpunct(wint_t __c, __locale_t __loc) { return iswpunct_l(__c, __loc); }
	134
	135	inline _LIBCPP_HIDE_FROM_ABI int __iswxdigit(wint_t __c, __locale_t __loc) { return iswxdigit_l(__c, __loc); }
	136
	137	inline _LIBCPP_HIDE_FROM_ABI wint_t __towupper(wint_t __c, __locale_t __loc) { return towupper_l(__c, __loc); }
	138
	139	inline _LIBCPP_HIDE_FROM_ABI wint_t __towlower(wint_t __c, __locale_t __loc) { return towlower_l(__c, __loc); }
	140
	141	inline _LIBCPP_HIDE_FROM_ABI int __wcscoll(const wchar_t* __ws1, const wchar_t* __ws2, __locale_t __loc) {
	142	return wcscoll_l(__ws1, __ws2, __loc);
	143	}
	144
	145	inline _LIBCPP_HIDE_FROM_ABI size_t __wcsxfrm(wchar_t* __dest, const wchar_t* __src, size_t __n, __locale_t __loc) {
	146	return wcsxfrm_l(__dest, __src, __n, __loc);
	147	}
	148	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
	149
	150	inline _LIBCPP_HIDE_FROM_ABI
	151	size_t __strftime(char* __s, size_t __max, const char* __format, const struct tm* __tm, __locale_t __loc) {
	152	return strftime_l(__s, __max, __format, __tm, __loc);
	153	}
	154
	155	//
	156	// Other functions
	157	//
	158	inline _LIBCPP_HIDE_FROM_ABI decltype(MB_CUR_MAX) __mb_len_max(__locale_t __loc) {
	159	__locale_guard __current(__loc);
	160	return MB_CUR_MAX;
	161	}
	162
	163	# if _LIBCPP_HAS_WIDE_CHARACTERS
	164	inline _LIBCPP_HIDE_FROM_ABI wint_t __btowc(int __c, __locale_t __loc) {
	165	__locale_guard __current(__loc);
	166	return std::btowc(__c);
	167	}
	168
	169	inline _LIBCPP_HIDE_FROM_ABI int __wctob(wint_t __c, __locale_t __loc) {
	170	__locale_guard __current(__loc);
	171	return std::wctob(__c);
	172	}
	173
	174	inline _LIBCPP_HIDE_FROM_ABI size_t
	175	__wcsnrtombs(char* __dest, const wchar_t** __src, size_t __nwc, size_t __len, mbstate_t* __ps, __locale_t __loc) {
	176	__locale_guard __current(__loc);
	177	return ::wcsnrtombs(__dest, __src, __nwc, __len, __ps); // non-standard
	178	}
	179
	180	inline _LIBCPP_HIDE_FROM_ABI size_t __wcrtomb(char* __s, wchar_t __wc, mbstate_t* __ps, __locale_t __loc) {
	181	__locale_guard __current(__loc);
	182	return std::wcrtomb(__s, __wc, __ps);
	183	}
	184
	185	inline _LIBCPP_HIDE_FROM_ABI size_t
	186	__mbsnrtowcs(wchar_t* __dest, const char** __src, size_t __nms, size_t __len, mbstate_t* __ps, __locale_t __loc) {
	187	__locale_guard __current(__loc);
	188	return ::mbsnrtowcs(__dest, __src, __nms, __len, __ps); // non-standard
	189	}
	190
	191	inline _LIBCPP_HIDE_FROM_ABI size_t
	192	__mbrtowc(wchar_t* __pwc, const char* __s, size_t __n, mbstate_t* __ps, __locale_t __loc) {
	193	__locale_guard __current(__loc);
	194	return std::mbrtowc(__pwc, __s, __n, __ps);
	195	}
	196
	197	inline _LIBCPP_HIDE_FROM_ABI int __mbtowc(wchar_t* __pwc, const char* __pmb, size_t __max, __locale_t __loc) {
	198	__locale_guard __current(__loc);
	199	return std::mbtowc(__pwc, __pmb, __max);
	200	}
	201
	202	inline _LIBCPP_HIDE_FROM_ABI size_t __mbrlen(const char* __s, size_t __n, mbstate_t* __ps, __locale_t __loc) {
	203	__locale_guard __current(__loc);
	204	return std::mbrlen(__s, __n, __ps);
	205	}
	206
	207	inline _LIBCPP_HIDE_FROM_ABI size_t
	208	__mbsrtowcs(wchar_t* __dest, const char** __src, size_t __len, mbstate_t* __ps, __locale_t __loc) {
	209	__locale_guard __current(__loc);
	210	return std::mbsrtowcs(__dest, __src, __len, __ps);
	211	}
	212	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
	213	#endif // _LIBCPP_BUILDING_LIBRARY
	214
	215	#ifndef _LIBCPP_COMPILER_GCC // GCC complains that this can't be always_inline due to C-style varargs
	216	_LIBCPP_HIDE_FROM_ABI
	217	#endif
	218	inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 4, 5) int __snprintf(
	219	char* __s, size_t __n, __locale_t __loc, const char* __format, ...) {
	220	va_list __va;
	221	va_start(__va, __format);
	222	__locale_guard __current(__loc);
	223	int __res = std::vsnprintf(__s, __n, __format, __va);
	224	va_end(__va);
	225	return __res;
	226	}
	227
	228	#ifndef _LIBCPP_COMPILER_GCC // GCC complains that this can't be always_inline due to C-style varargs
	229	_LIBCPP_HIDE_FROM_ABI
	230	#endif
	231	inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __asprintf(
	232	char** __s, __locale_t __loc, const char* __format, ...) {
	233	va_list __va;
	234	va_start(__va, __format);
	235	__locale_guard __current(__loc);
	236	int __res = ::vasprintf(__s, __format, __va); // non-standard
	237	va_end(__va);
	238	return __res;
	239	}
	240	} // namespace __locale
	241	_LIBCPP_END_NAMESPACE_STD
	242
	243	#endif // _LIBCPP___LOCALE_DIR_SUPPORT_NEWLIB_H

lib/libcxx/include/__locale_dir/support/no_locale/characters.h-4

...	...	@@ -29,10 +29,6 @@ namespace __locale {
29	29	//
30	30	// Character manipulation functions
31	31	//
32		inline _LIBCPP_HIDE_FROM_ABI int __isdigit(int __c, __locale_t) { return std::isdigit(__c); }
33
34		inline _LIBCPP_HIDE_FROM_ABI int __isxdigit(int __c, __locale_t) { return std::isxdigit(__c); }
35
36	32	#if defined(_LIBCPP_BUILDING_LIBRARY)
37	33	inline _LIBCPP_HIDE_FROM_ABI int __toupper(int __c, __locale_t) { return std::toupper(__c); }
38	34

lib/libcxx/include/__locale_dir/support/no_locale/strtonum.h-9

...	...	@@ -34,15 +34,6 @@ inline _LIBCPP_HIDE_FROM_ABI long double __strtold(const char* __nptr, char** __
34	34	return std::strtold(__nptr, __endptr);
35	35	}
36	36
37		inline _LIBCPP_HIDE_FROM_ABI long long __strtoll(const char* __nptr, char** __endptr, int __base, __locale_t) {
38		return std::strtoll(__nptr, __endptr, __base);
39		}
40
41		inline _LIBCPP_HIDE_FROM_ABI unsigned long long
42		__strtoull(const char* __nptr, char** __endptr, int __base, __locale_t) {
43		return std::strtoull(__nptr, __endptr, __base);
44		}
45
46	37	} // namespace __locale
47	38	_LIBCPP_END_NAMESPACE_STD
48	39

lib/libcxx/include/__locale_dir/support/windows.h-29

...	...	@@ -186,21 +186,9 @@ inline _LIBCPP_HIDE_FROM_ABI double __strtod(const char* __nptr, char** __endptr
186	186	return ::_strtod_l(__nptr, __endptr, __loc);
187	187	}
188	188
189		inline _LIBCPP_HIDE_FROM_ABI long long __strtoll(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
190		return ::_strtoi64_l(__nptr, __endptr, __base, __loc);
191		}
192		inline _LIBCPP_HIDE_FROM_ABI unsigned long long
193		__strtoull(const char* __nptr, char** __endptr, int __base, __locale_t __loc) {
194		return ::_strtoui64_l(__nptr, __endptr, __base, __loc);
195		}
196
197	189	//
198	190	// Character manipulation functions
199	191	//
200		inline _LIBCPP_HIDE_FROM_ABI int __isdigit(int __c, __locale_t __loc) { return _isdigit_l(__c, __loc); }
201
202		inline _LIBCPP_HIDE_FROM_ABI int __isxdigit(int __c, __locale_t __loc) { return _isxdigit_l(__c, __loc); }
203
204	192	#if defined(_LIBCPP_BUILDING_LIBRARY)
205	193	inline _LIBCPP_HIDE_FROM_ABI int __toupper(int __c, __locale_t __loc) { return ::_toupper_l(__c, __loc); }
206	194
...	...	@@ -280,23 +268,6 @@ _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 4, 5) int __snpri
280	268	_LIBCPP_EXPORTED_FROM_ABI
281	269	_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4) int __asprintf(char** __ret, __locale_t __loc, const char* __format, ...);
282	270
283		_LIBCPP_DIAGNOSTIC_PUSH
284		_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wgcc-compat")
285		_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral") // GCC doesn't support [[gnu::format]] on variadic templates
286		#ifdef _LIBCPP_COMPILER_CLANG_BASED
287		# define _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(...) _LIBCPP_ATTRIBUTE_FORMAT(__VA_ARGS__)
288		#else
289		# define _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(...) /* nothing */
290		#endif
291
292		template <class... _Args>
293		_LIBCPP_HIDE_FROM_ABI _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT(__scanf__, 3, 4) int __sscanf(
294		const char* __dest, __locale_t __loc, const char* __format, _Args&&... __args) {
295		return ::_sscanf_l(__dest, __format, __loc, std::forward<_Args>(__args)...);
296		}
297		_LIBCPP_DIAGNOSTIC_POP
298		#undef _LIBCPP_VARIADIC_ATTRIBUTE_FORMAT
299
300	271	#if defined(_LIBCPP_BUILDING_LIBRARY)
301	272	struct __locale_guard {
302	273	_LIBCPP_HIDE_FROM_ABI __locale_guard(__locale_t __l) : __status(_configthreadlocale(_ENABLE_PER_THREAD_LOCALE)) {

lib/libcxx/include/__locale_dir/time.h+3-7

...	...	@@ -601,17 +601,13 @@ private:
601	601	template <> \
602	602	_LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&); \
603	603	template <> \
604		_LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&); \
	604	void __time_get_storage<_CharT>::init(const ctype<_CharT>&); \
605	605	template <> \
606		_LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze( \
607		char, const ctype<_CharT>&); \
	606	__time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&); \
608	607	extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() \
609	608	const; \
610	609	extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*); \
611		extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&); \
612		extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&); \
613		extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type \
614		__time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);
	610	extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);
615	611
616	612	_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
617	613	# if _LIBCPP_HAS_WIDE_CHARACTERS

lib/libcxx/include/__math/hypot.h+1-1

...	...	@@ -53,7 +53,7 @@ inline _LIBCPP_HIDE_FROM_ABI __promote_t<_A1, _A2> hypot(_A1 __x, _A2 __y) _NOEX
53	53	// Computes the three-dimensional hypotenuse: `std::hypot(x,y,z)`.
54	54	// The naive implementation might over-/underflow which is why this implementation is more involved:
55	55	// If the square of an argument might run into issues, we scale the arguments appropriately.
56		// See https://github.com/llvm/llvm-project/issues/92782 for a detailed discussion and summary.
	56	// See https://llvm.org/PR92782 for a detailed discussion and summary.
57	57	template <class _Real>
58	58	_LIBCPP_HIDE_FROM_ABI _Real __hypot(_Real __x, _Real __y, _Real __z) {
59	59	// Factors needed to determine if over-/underflow might happen

lib/libcxx/include/__math/logarithms.h+1-1

...	...	@@ -58,7 +58,7 @@ inline _LIBCPP_HIDE_FROM_ABI double log10(_A1 __x) _NOEXCEPT {
58	58	inline _LIBCPP_HIDE_FROM_ABI int ilogb(float __x) _NOEXCEPT { return __builtin_ilogbf(__x); }
59	59
60	60	template <class = int>
61		_LIBCPP_HIDE_FROM_ABI double ilogb(double __x) _NOEXCEPT {
	61	_LIBCPP_HIDE_FROM_ABI int ilogb(double __x) _NOEXCEPT {
62	62	return __builtin_ilogb(__x);
63	63	}
64	64

lib/libcxx/include/__math/traits.h+80-11

...	...	@@ -25,33 +25,26 @@ namespace __math {
25	25
26	26	// signbit
27	27
28		// TODO(LLVM 22): Remove conditional once support for Clang 19 is dropped.
29		#if defined(_LIBCPP_COMPILER_GCC) \|\| __has_constexpr_builtin(__builtin_signbit)
30		# define _LIBCPP_SIGNBIT_CONSTEXPR _LIBCPP_CONSTEXPR_SINCE_CXX23
31		#else
32		# define _LIBCPP_SIGNBIT_CONSTEXPR
33		#endif
34
35	28	// The universal C runtime (UCRT) in the WinSDK provides floating point overloads
36	29	// for std::signbit(). By defining our overloads as templates, we can work around
37	30	// this issue as templates are less preferred than non-template functions.
38	31	template <class = void>
39		[[__nodiscard__]] inline _LIBCPP_SIGNBIT_CONSTEXPR _LIBCPP_HIDE_FROM_ABI bool signbit(float __x) _NOEXCEPT {
	32	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool signbit(float __x) _NOEXCEPT {
40	33	return __builtin_signbit(__x);
41	34	}
42	35
43	36	template <class = void>
44		[[__nodiscard__]] inline _LIBCPP_SIGNBIT_CONSTEXPR _LIBCPP_HIDE_FROM_ABI bool signbit(double __x) _NOEXCEPT {
	37	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool signbit(double __x) _NOEXCEPT {
45	38	return __builtin_signbit(__x);
46	39	}
47	40
48	41	template <class = void>
49		[[__nodiscard__]] inline _LIBCPP_SIGNBIT_CONSTEXPR _LIBCPP_HIDE_FROM_ABI bool signbit(long double __x) _NOEXCEPT {
	42	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool signbit(long double __x) _NOEXCEPT {
50	43	return __builtin_signbit(__x);
51	44	}
52	45
53	46	template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
54		[[__nodiscard__]] inline _LIBCPP_SIGNBIT_CONSTEXPR _LIBCPP_HIDE_FROM_ABI bool signbit(_A1 __x) _NOEXCEPT {
	47	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool signbit(_A1 __x) _NOEXCEPT {
55	48	return __x < 0;
56	49	}
57	50
...	...	@@ -189,6 +182,82 @@ template <class _A1, class _A2, __enable_if_t<is_arithmetic<_A1>::value && is_ar
189	182	return __builtin_isunordered((type)__x, (type)__y);
190	183	}
191	184
	185	// MS UCRT incorrectly defines some functions in a way not working with integer types. Until C++20, this was worked
	186	// around by -fdelayed-template-parsing. Since C++20, we can use standard feature "requires" instead.
	187
	188	// TODO: Remove the workaround once UCRT fixes these functions. Note that this doesn't seem planned as of 2025-07 per
	189	// https://developercommunity.visualstudio.com/t/10294165.
	190
	191	#if defined(_LIBCPP_MSVCRT) && _LIBCPP_STD_VER >= 20
	192	namespace __ucrt {
	193	template <class _A1>
	194	requires is_integral_v<_A1>
	195	[[nodiscard]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isfinite(_A1) noexcept {
	196	return true;
	197	}
	198
	199	template <class _A1>
	200	requires is_integral_v<_A1>
	201	[[nodiscard]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(_A1) noexcept {
	202	return false;
	203	}
	204
	205	template <class _A1>
	206	requires is_integral_v<_A1>
	207	[[nodiscard]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(_A1) noexcept {
	208	return false;
	209	}
	210
	211	template <class _A1>
	212	requires is_integral_v<_A1>
	213	[[nodiscard]] inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnormal(_A1 __x) noexcept {
	214	return __x != 0;
	215	}
	216
	217	template <class _A1, class _A2>
	218	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	219	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool isgreater(_A1 __x, _A2 __y) noexcept {
	220	using type = __promote_t<_A1, _A2>;
	221	return __builtin_isgreater((type)__x, (type)__y);
	222	}
	223
	224	template <class _A1, class _A2>
	225	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	226	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool isgreaterequal(_A1 __x, _A2 __y) noexcept {
	227	using type = __promote_t<_A1, _A2>;
	228	return __builtin_isgreaterequal((type)__x, (type)__y);
	229	}
	230
	231	template <class _A1, class _A2>
	232	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	233	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool isless(_A1 __x, _A2 __y) noexcept {
	234	using type = __promote_t<_A1, _A2>;
	235	return __builtin_isless((type)__x, (type)__y);
	236	}
	237
	238	template <class _A1, class _A2>
	239	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	240	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool islessequal(_A1 __x, _A2 __y) noexcept {
	241	using type = __promote_t<_A1, _A2>;
	242	return __builtin_islessequal((type)__x, (type)__y);
	243	}
	244
	245	template <class _A1, class _A2>
	246	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	247	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool islessgreater(_A1 __x, _A2 __y) noexcept {
	248	using type = __promote_t<_A1, _A2>;
	249	return __builtin_islessgreater((type)__x, (type)__y);
	250	}
	251
	252	template <class _A1, class _A2>
	253	requires is_arithmetic_v<_A1> && is_arithmetic_v<_A2>
	254	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool isunordered(_A1 __x, _A2 __y) noexcept {
	255	using type = __promote_t<_A1, _A2>;
	256	return __builtin_isunordered((type)__x, (type)__y);
	257	}
	258	} // namespace __ucrt
	259	#endif
	260
192	261	} // namespace __math
193	262
194	263	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__mdspan/extents.h+7-4

...	...	@@ -25,6 +25,7 @@
25	25	#include <__type_traits/integer_traits.h>
26	26	#include <__type_traits/is_convertible.h>
27	27	#include <__type_traits/is_nothrow_constructible.h>
	28	#include <__type_traits/is_signed.h>
28	29	#include <__type_traits/make_unsigned.h>
29	30	#include <__utility/integer_sequence.h>
30	31	#include <__utility/unreachable.h>
...	...	@@ -298,11 +299,13 @@ private:
298	299
299	300	public:
300	301	// [mdspan.extents.obs], observers of multidimensional index space
301		_LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return __rank_; }
302		_LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept { return __rank_dynamic_; }
	302	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return __rank_; }
	303	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept { return __rank_dynamic_; }
303	304
304		_LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept { return __vals_.__value(__r); }
305		_LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
	305	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept {
	306	return __vals_.__value(__r);
	307	}
	308	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
306	309	return _Values::__static_value(__r);
307	310	}
308	311

lib/libcxx/include/__mdspan/layout_stride.h+4-5

...	...	@@ -272,11 +272,10 @@ public:
272	272	return [&]<size_t... _Pos>(index_sequence<_Pos...>) {
273	273	if ((__extents_.extent(_Pos) * ... * 1) == 0)
274	274	return static_cast<index_type>(0);
275		else
276		return static_cast<index_type>(
277		static_cast<index_type>(1) +
278		(((__extents_.extent(_Pos) - static_cast<index_type>(1)) * __strides_[_Pos]) + ... +
279		static_cast<index_type>(0)));
	275
	276	return static_cast<index_type>(
	277	static_cast<index_type>(1) + (((__extents_.extent(_Pos) - static_cast<index_type>(1)) * __strides_[_Pos]) +
	278	... + static_cast<index_type>(0)));
280	279	}(make_index_sequence<__rank_>());
281	280	}
282	281	}

lib/libcxx/include/__mdspan/mdspan.h+29-23

...	...	@@ -87,16 +87,17 @@ public:
87	87	using data_handle_type = typename accessor_type::data_handle_type;
88	88	using reference = typename accessor_type::reference;
89	89
90		_LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return extents_type::rank(); }
91		_LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept { return extents_type::rank_dynamic(); }
92		_LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
	90	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank() noexcept { return extents_type::rank(); }
	91	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr rank_type rank_dynamic() noexcept {
	92	return extents_type::rank_dynamic();
	93	}
	94	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr size_t static_extent(rank_type __r) noexcept {
93	95	return extents_type::static_extent(__r);
94	96	}
95		_LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept {
	97	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr index_type extent(rank_type __r) const noexcept {
96	98	return __map_.extents().extent(__r);
97		};
	99	}
98	100
99		public:
100	101	//--------------------------------------------------------------------------------
101	102	// [mdspan.mdspan.cons], mdspan constructors, assignment, and destructor
102	103
...	...	@@ -185,7 +186,7 @@ public:
185	186	requires((is_convertible_v<_OtherIndexTypes, index_type> && ...) &&
186	187	(is_nothrow_constructible_v<index_type, _OtherIndexTypes> && ...) &&
187	188	(sizeof...(_OtherIndexTypes) == rank()))
188		_LIBCPP_HIDE_FROM_ABI constexpr reference operator[](_OtherIndexTypes... __indices) const {
	189	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](_OtherIndexTypes... __indices) const {
189	190	// Note the standard layouts would also check this, but user provided ones may not, so we
190	191	// check the precondition here
191	192	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__mdspan_detail::__is_multidimensional_index_in(extents(), __indices...),
...	...	@@ -196,7 +197,8 @@ public:
196	197	template <class _OtherIndexType>
197	198	requires(is_convertible_v<const _OtherIndexType&, index_type> &&
198	199	is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
199		_LIBCPP_HIDE_FROM_ABI constexpr reference operator[](const array< _OtherIndexType, rank()>& __indices) const {
	200	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference
	201	operator[](const array< _OtherIndexType, rank()>& __indices) const {
200	202	return __acc_.access(__ptr_, [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
201	203	return __map_(__indices[_Idxs]...);
202	204	}(make_index_sequence<rank()>()));
...	...	@@ -205,13 +207,13 @@ public:
205	207	template <class _OtherIndexType>
206	208	requires(is_convertible_v<const _OtherIndexType&, index_type> &&
207	209	is_nothrow_constructible_v<index_type, const _OtherIndexType&>)
208		_LIBCPP_HIDE_FROM_ABI constexpr reference operator[](span<_OtherIndexType, rank()> __indices) const {
	210	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](span<_OtherIndexType, rank()> __indices) const {
209	211	return __acc_.access(__ptr_, [&]<size_t... _Idxs>(index_sequence<_Idxs...>) {
210	212	return __map_(__indices[_Idxs]...);
211	213	}(make_index_sequence<rank()>()));
212	214	}
213	215
214		_LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept {
	216	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept {
215	217	// Could leave this as only checked in debug mode: semantically size() is never
216	218	// guaranteed to be related to any accessible range
217	219	_LIBCPP_ASSERT_UNCATEGORIZED(
...	...	@@ -237,24 +239,28 @@ public:
237	239	swap(__x.__acc_, __y.__acc_);
238	240	}
239	241
240		_LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept { return __map_.extents(); };
241		_LIBCPP_HIDE_FROM_ABI constexpr const data_handle_type& data_handle() const noexcept { return __ptr_; };
242		_LIBCPP_HIDE_FROM_ABI constexpr const mapping_type& mapping() const noexcept { return __map_; };
243		_LIBCPP_HIDE_FROM_ABI constexpr const accessor_type& accessor() const noexcept { return __acc_; };
	242	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const extents_type& extents() const noexcept {
	243	return __map_.extents();
	244	}
	245	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const data_handle_type& data_handle() const noexcept { return __ptr_; }
	246	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const mapping_type& mapping() const noexcept { return __map_; }
	247	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const accessor_type& accessor() const noexcept { return __acc_; }
244	248
245	249	// per LWG-4021 "mdspan::is_always_meow() should be noexcept"
246		_LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); };
247		_LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept {
	250	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept {
	251	return mapping_type::is_always_unique();
	252	}
	253	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept {
248	254	return mapping_type::is_always_exhaustive();
249		};
250		_LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept {
	255	}
	256	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept {
251	257	return mapping_type::is_always_strided();
252		};
	258	}
253	259
254		_LIBCPP_HIDE_FROM_ABI constexpr bool is_unique() const { return __map_.is_unique(); };
255		_LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const { return __map_.is_exhaustive(); };
256		_LIBCPP_HIDE_FROM_ABI constexpr bool is_strided() const { return __map_.is_strided(); };
257		_LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const { return __map_.stride(__r); };
	260	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool is_unique() const { return __map_.is_unique(); }
	261	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const { return __map_.is_exhaustive(); }
	262	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool is_strided() const { return __map_.is_strided(); }
	263	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr index_type stride(rank_type __r) const { return __map_.stride(__r); }
258	264
259	265	private:
260	266	_LIBCPP_NO_UNIQUE_ADDRESS data_handle_type __ptr_{};

lib/libcxx/include/__memory/addressof.h+7-5

...	...	@@ -19,7 +19,8 @@
19	19	_LIBCPP_BEGIN_NAMESPACE_STD
20	20
21	21	template <class _Tp>
22		inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp* addressof(_Tp& __x) _NOEXCEPT {
	22	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp*
	23	addressof(_Tp& __x) _NOEXCEPT {
23	24	return __builtin_addressof(__x);
24	25	}
25	26
...	...	@@ -27,24 +28,25 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_NO_CFI _LIBCPP_HIDE_FROM_ABI _Tp* a
27	28	// Objective-C++ Automatic Reference Counting uses qualified pointers
28	29	// that require special addressof() signatures.
29	30	template <class _Tp>
30		inline _LIBCPP_HIDE_FROM_ABI __strong _Tp* addressof(__strong _Tp& __x) _NOEXCEPT {
	31	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __strong _Tp* addressof(__strong _Tp& __x) _NOEXCEPT {
31	32	return &__x;
32	33	}
33	34
34	35	# if __has_feature(objc_arc_weak)
35	36	template <class _Tp>
36		inline _LIBCPP_HIDE_FROM_ABI __weak _Tp* addressof(__weak _Tp& __x) _NOEXCEPT {
	37	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __weak _Tp* addressof(__weak _Tp& __x) _NOEXCEPT {
37	38	return &__x;
38	39	}
39	40	# endif
40	41
41	42	template <class _Tp>
42		inline _LIBCPP_HIDE_FROM_ABI __autoreleasing _Tp* addressof(__autoreleasing _Tp& __x) _NOEXCEPT {
	43	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __autoreleasing _Tp* addressof(__autoreleasing _Tp& __x) _NOEXCEPT {
43	44	return &__x;
44	45	}
45	46
46	47	template <class _Tp>
47		inline _LIBCPP_HIDE_FROM_ABI __unsafe_unretained _Tp* addressof(__unsafe_unretained _Tp& __x) _NOEXCEPT {
	48	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __unsafe_unretained _Tp*
	49	addressof(__unsafe_unretained _Tp& __x) _NOEXCEPT {
48	50	return &__x;
49	51	}
50	52	#endif

lib/libcxx/include/__memory/align.h+18-1

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
	14	#include <cstdint>
14	15
15	16	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
16	17	# pragma GCC system_header
...	...	@@ -18,7 +19,23 @@
18	19
19	20	_LIBCPP_BEGIN_NAMESPACE_STD
20	21
21		_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space);
	22	inline namespace __align_inline {
	23	_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) {
	24	void* __r = nullptr;
	25	if (__sz <= __space) {
	26	char* __p1 = static_cast<char*>(__ptr);
	27	char* __p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(__p1 + (__align - 1)) & -__align);
	28	size_t __d = static_cast<size_t>(__p2 - __p1);
	29	if (__d <= __space - __sz) {
	30	__r = __p2;
	31	__ptr = __r;
	32	__space -= __d;
	33	}
	34	}
	35	return __r;
	36	}
	37
	38	} // namespace __align_inline
22	39
23	40	_LIBCPP_END_NAMESPACE_STD
24	41

lib/libcxx/include/__memory/allocate_at_least.h+15-10

...	...	@@ -19,26 +19,31 @@
19	19
20	20	_LIBCPP_BEGIN_NAMESPACE_STD
21	21
	22	template <class _Pointer, class _SizeT = size_t>
	23	struct __allocation_result {
	24	_Pointer ptr;
	25	_SizeT count;
	26
	27	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __allocation_result(_Pointer __ptr, _SizeT __count)
	28	: ptr(__ptr), count(__count) {}
	29	};
	30	_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__allocation_result);
	31
22	32	#if _LIBCPP_STD_VER >= 23
23	33
24	34	template <class _Alloc>
25	35	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto __allocate_at_least(_Alloc& __alloc, size_t __n) {
26		return std::allocator_traits<_Alloc>::allocate_at_least(__alloc, __n);
	36	auto __res = std::allocator_traits<_Alloc>::allocate_at_least(__alloc, __n);
	37	return __allocation_result{__res.ptr, __res.count};
27	38	}
28	39
29	40	#else
30	41
31		template <class _Pointer>
32		struct __allocation_result {
33		_Pointer ptr;
34		size_t count;
35		};
36
37		template <class _Alloc>
	42	template <class _Alloc, class _Traits = allocator_traits<_Alloc> >
38	43	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
39		_LIBCPP_CONSTEXPR __allocation_result<typename allocator_traits<_Alloc>::pointer>
	44	_LIBCPP_CONSTEXPR __allocation_result<typename _Traits::pointer, typename _Traits::size_type>
40	45	__allocate_at_least(_Alloc& __alloc, size_t __n) {
41		return {__alloc.allocate(__n), __n};
	46	return __allocation_result<typename _Traits::pointer, typename _Traits::size_type>(__alloc.allocate(__n), __n);
42	47	}
43	48
44	49	#endif // _LIBCPP_STD_VER >= 23

lib/libcxx/include/__memory/allocator.h+14-26

...	...	@@ -14,7 +14,6 @@
14	14	#include <__cstddef/ptrdiff_t.h>
15	15	#include <__cstddef/size_t.h>
16	16	#include <__memory/addressof.h>
17		#include <__memory/allocate_at_least.h>
18	17	#include <__memory/allocator_traits.h>
19	18	#include <__new/allocate.h>
20	19	#include <__new/exceptions.h>
...	...	@@ -51,33 +50,21 @@ public:
51	50	};
52	51	#endif // _LIBCPP_STD_VER <= 17
53	52
54		// This class provides a non-trivial default constructor to the class that derives from it
55		// if the condition is satisfied.
56		//
57		// The second template parameter exists to allow giving a unique type to __non_trivial_if,
58		// which makes it possible to avoid breaking the ABI when making this a base class of an
59		// existing class. Without that, imagine we have classes D1 and D2, both of which used to
60		// have no base classes, but which now derive from __non_trivial_if. The layout of a class
61		// that inherits from both D1 and D2 will change because the two __non_trivial_if base
62		// classes are not allowed to share the same address.
63		//
64		// By making those __non_trivial_if base classes unique, we work around this problem and
65		// it is safe to start deriving from __non_trivial_if in existing classes.
66		template <bool _Cond, class _Unique>
67		struct __non_trivial_if {};
	53	template <bool, class _Unique>
	54	struct __non_trivially_default_constructible_if {};
68	55
69	56	template <class _Unique>
70		struct __non_trivial_if<true, _Unique> {
71		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __non_trivial_if() _NOEXCEPT {}
	57	struct __non_trivially_default_constructible_if<true, _Unique> {
	58	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __non_trivially_default_constructible_if() {}
72	59	};
73	60
74		// allocator
75		//
76		// Note: For ABI compatibility between C++20 and previous standards, we make
77		// allocator<void> trivial in C++20.
78
79	61	template <class _Tp>
80		class allocator : private __non_trivial_if<!is_void<_Tp>::value, allocator<_Tp> > {
	62	class allocator
	63	// TODO(LLVM 24): Remove the opt-out
	64	#ifdef _LIBCPP_DEPRECATED_ABI_NON_TRIVIAL_ALLOCATOR
	65	: __non_trivially_default_constructible_if<!is_void<_Tp>::value, allocator<_Tp> >
	66	#endif
	67	{
81	68	static_assert(!is_const<_Tp>::value, "std::allocator does not support const types");
82	69	static_assert(!is_volatile<_Tp>::value, "std::allocator does not support volatile types");
83	70
...	...	@@ -133,10 +120,11 @@ public:
133	120	typedef allocator<_Up> other;
134	121	};
135	122
136		_LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI pointer address(reference __x) const _NOEXCEPT {
	123	[[__nodiscard__]] _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI pointer address(reference __x) const _NOEXCEPT {
137	124	return std::addressof(__x);
138	125	}
139		_LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer address(const_reference __x) const _NOEXCEPT {
	126	[[__nodiscard__]] _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer
	127	address(const_reference __x) const _NOEXCEPT {
140	128	return std::addressof(__x);
141	129	}
142	130
...	...	@@ -144,7 +132,7 @@ public:
144	132	return allocate(__n);
145	133	}
146	134
147		_LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	135	[[__nodiscard__]] _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
148	136	return size_type(~0) / sizeof(_Tp);
149	137	}
150	138

lib/libcxx/include/__memory/allocator_traits.h+6-4

...	...	@@ -314,23 +314,25 @@ struct allocator_traits {
314	314	}
315	315
316	316	template <class _Ap = _Alloc, __enable_if_t<__has_max_size_v<const _Ap>, int> = 0>
317		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type max_size(const allocator_type& __a) _NOEXCEPT {
	317	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
	318	max_size(const allocator_type& __a) _NOEXCEPT {
318	319	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
319	320	return __a.max_size();
320	321	_LIBCPP_SUPPRESS_DEPRECATED_POP
321	322	}
322	323	template <class _Ap = _Alloc, __enable_if_t<!__has_max_size_v<const _Ap>, int> = 0>
323		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type max_size(const allocator_type&) _NOEXCEPT {
	324	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static size_type
	325	max_size(const allocator_type&) _NOEXCEPT {
324	326	return numeric_limits<size_type>::max() / sizeof(value_type);
325	327	}
326	328
327	329	template <class _Ap = _Alloc, __enable_if_t<__has_select_on_container_copy_construction_v<const _Ap>, int> = 0>
328		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
	330	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
329	331	select_on_container_copy_construction(const allocator_type& __a) {
330	332	return __a.select_on_container_copy_construction();
331	333	}
332	334	template <class _Ap = _Alloc, __enable_if_t<!__has_select_on_container_copy_construction_v<const _Ap>, int> = 0>
333		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
	335	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static allocator_type
334	336	select_on_container_copy_construction(const allocator_type& __a) {
335	337	return __a;
336	338	}

lib/libcxx/include/__memory/compressed_pair.h+15-9

...	...	@@ -28,8 +28,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
28	28	// understand how it works). //
29	29	// ================================================================================================================== //
30	30
31		// The first member is aligned to the alignment of the second member to force padding in front of the compressed pair
32		// in case there are members before it.
	31	// On GCC, the first member is aligned to the alignment of the second member to force padding in front of the compressed
	32	// pair in case there are members before it.
33	33	//
34	34	// For example:
35	35	// (assuming x86-64 linux)
...	...	@@ -52,7 +52,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
52	52	//
53	53	// Furthermore, that alignment must be the same as what was used in the old __compressed_pair layout, so we must
54	54	// handle reference types specially since alignof(T&) == alignof(T).
55		// See https://github.com/llvm/llvm-project/issues/118559.
	55	// See https://llvm.org/PR118559.
	56	//
	57	// On Clang, this is unnecessary, since we use anonymous structs instead, which automatically handle the alignment
	58	// correctly.
56	59
57	60	#ifndef _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
58	61
...	...	@@ -64,7 +67,7 @@ inline const size_t __compressed_pair_alignment<_Tp&> = _LIBCPP_ALIGNOF(void*);
64	67
65	68	template <class _ToPad>
66	69	inline const bool __is_reference_or_unpadded_object =
67		(is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value) \|\| sizeof(_ToPad) == __datasizeof_v<_ToPad>;
	70	(is_empty<_ToPad>::value && !__is_final_v<_ToPad>) \|\| sizeof(_ToPad) == __datasizeof_v<_ToPad>;
68	71
69	72	template <class _Tp>
70	73	inline const bool __is_reference_or_unpadded_object<_Tp&> = true;
...	...	@@ -80,6 +83,10 @@ class __compressed_pair_padding {
80	83	template <class _ToPad>
81	84	class __compressed_pair_padding<_ToPad, true> {};
82	85
	86	# define _LIBCPP_COMPRESSED_ELEMENT(T1, Initializer1) \
	87	_LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1; \
	88	_LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding_, __LINE__, _)
	89
83	90	// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
84	91	# ifdef _LIBCPP_COMPILER_GCC
85	92	# define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2) \
...	...	@@ -100,8 +107,7 @@ class __compressed_pair_padding<_ToPad, true> {};
100	107	# else
101	108	# define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2) \
102	109	struct { \
103		_LIBCPP_NO_UNIQUE_ADDRESS \
104		__attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1; \
	110	_LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1; \
105	111	_LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \
106	112	_LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \
107	113	_LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \
...	...	@@ -109,9 +115,7 @@ class __compressed_pair_padding<_ToPad, true> {};
109	115
110	116	# define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3) \
111	117	struct { \
112		_LIBCPP_NO_UNIQUE_ADDRESS \
113		__attribute__((__aligned__(::std::__compressed_pair_alignment<T2>), \
114		__aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1; \
	118	_LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1; \
115	119	_LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _); \
116	120	_LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2; \
117	121	_LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _); \
...	...	@@ -121,6 +125,8 @@ class __compressed_pair_padding<_ToPad, true> {};
121	125	# endif
122	126
123	127	#else
	128	# define _LIBCPP_COMPRESSED_ELEMENT(T1, Initializer1) _LIBCPP_NO_UNIQUE_ADDRESS T1 Initializer1
	129
124	130	# define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2) \
125	131	_LIBCPP_NO_UNIQUE_ADDRESS T1 Name1; \
126	132	_LIBCPP_NO_UNIQUE_ADDRESS T2 Name2

lib/libcxx/include/__memory/construct_at.h+12-23

...	...	@@ -14,7 +14,6 @@
14	14	#include <__config>
15	15	#include <__memory/addressof.h>
16	16	#include <__new/placement_new_delete.h>
17		#include <__type_traits/enable_if.h>
18	17	#include <__type_traits/is_array.h>
19	18	#include <__utility/declval.h>
20	19	#include <__utility/forward.h>
...	...	@@ -33,7 +32,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
33	32	#if _LIBCPP_STD_VER >= 20
34	33
35	34	template <class _Tp, class... _Args, class = decltype(::new(std::declval<void*>()) _Tp(std::declval<_Args>()...))>
36		_LIBCPP_HIDE_FROM_ABI constexpr _Tp* construct_at(_Tp* __location, _Args&&... __args) {
	35	_LIBCPP_HIDE_FROM_ABI constexpr _Tp* construct_at(_Tp* _LIBCPP_DIAGNOSE_NULLPTR __location, _Args&&... __args) {
37	36	_LIBCPP_ASSERT_NON_NULL(__location != nullptr, "null pointer given to construct_at");
38	37	return ::new (static_cast<void*>(__location)) _Tp(std::forward<_Args>(__args)...);
39	38	}
...	...	@@ -55,35 +54,25 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __construct_at(_Tp* __l
55	54	// The internal functions are available regardless of the language version (with the exception of the `__destroy_at`
56	55	// taking an array).
57	56
58		template <class _Tp, __enable_if_t<!is_array<_Tp>::value, int> = 0>
	57	template <class _Tp>
59	58	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __destroy_at(_Tp* __loc) {
60	59	_LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
61		__loc->~_Tp();
62		}
63
64	60	#if _LIBCPP_STD_VER >= 20
65		template <class _Tp, __enable_if_t<is_array<_Tp>::value, int> = 0>
66		_LIBCPP_HIDE_FROM_ABI constexpr void __destroy_at(_Tp* __loc) {
67		_LIBCPP_ASSERT_NON_NULL(__loc != nullptr, "null pointer given to destroy_at");
68		for (auto&& __val : *__loc)
69		std::__destroy_at(std::addressof(__val));
70		}
	61	if constexpr (is_array_v<_Tp>) {
	62	for (auto&& __val : *__loc)
	63	std::__destroy_at(std::addressof(__val));
	64	} else
71	65	#endif
72
73		#if _LIBCPP_STD_VER >= 17
74
75		template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
76		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy_at(_Tp* __loc) {
77		std::__destroy_at(__loc);
	66	{
	67	__loc->~_Tp();
	68	}
78	69	}
79	70
80		# if _LIBCPP_STD_VER >= 20
81		template <class _Tp, enable_if_t<is_array_v<_Tp>, int> = 0>
82		_LIBCPP_HIDE_FROM_ABI constexpr void destroy_at(_Tp* __loc) {
	71	#if _LIBCPP_STD_VER >= 17
	72	template <class _Tp>
	73	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void destroy_at(_Tp* _LIBCPP_DIAGNOSE_NULLPTR __loc) {
83	74	std::__destroy_at(__loc);
84	75	}
85		# endif
86
87	76	#endif // _LIBCPP_STD_VER >= 17
88	77
89	78	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__memory/inout_ptr.h+1-1

...	...	@@ -96,7 +96,7 @@ private:
96	96	};
97	97
98	98	template <class _Pointer = void, class _Smart, class... _Args>
99		_LIBCPP_HIDE_FROM_ABI auto inout_ptr(_Smart& __s, _Args&&... __args) {
	99	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto inout_ptr(_Smart& __s, _Args&&... __args) {
100	100	using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
101	101	return std::inout_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
102	102	}

lib/libcxx/include/__memory/is_sufficiently_aligned.h+1-1

...	...	@@ -23,7 +23,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
23	23	#if _LIBCPP_STD_VER >= 26
24	24
25	25	template <size_t _Alignment, class _Tp>
26		_LIBCPP_HIDE_FROM_ABI bool is_sufficiently_aligned(_Tp* __ptr) {
	26	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_sufficiently_aligned(_Tp* __ptr) {
27	27	return reinterpret_cast<uintptr_t>(__ptr) % _Alignment == 0;
28	28	}
29	29

lib/libcxx/include/__memory/out_ptr.h+1-1

...	...	@@ -88,7 +88,7 @@ private:
88	88	};
89	89
90	90	template <class _Pointer = void, class _Smart, class... _Args>
91		_LIBCPP_HIDE_FROM_ABI auto out_ptr(_Smart& __s, _Args&&... __args) {
	91	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto out_ptr(_Smart& __s, _Args&&... __args) {
92	92	using _Ptr = conditional_t<is_void_v<_Pointer>, __pointer_of_t<_Smart>, _Pointer>;
93	93	return std::out_ptr_t<_Smart, _Ptr, _Args&&...>(__s, std::forward<_Args>(__args)...);
94	94	}

lib/libcxx/include/__memory/pointer_traits.h+1-1

...	...	@@ -255,7 +255,7 @@ concept __resettable_smart_pointer_with_args = requires(_Smart __s, _Pointer __p
255	255	// This function ensures safe conversions between fancy pointers at compile-time, where we avoid casts from/to
256	256	// `__void_pointer` by obtaining the underlying raw pointer from the fancy pointer using `std::to_address`,
257	257	// then dereferencing it to retrieve the pointed-to object, and finally constructing the target fancy pointer
258		// to that object using the `std::pointer_traits<>::pinter_to` function.
	258	// to that object using the `std::pointer_traits<>::pointer_to` function.
259	259	template <class _PtrTo, class _PtrFrom>
260	260	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI _PtrTo __static_fancy_pointer_cast(const _PtrFrom& __p) {
261	261	using __ptr_traits = pointer_traits<_PtrTo>;

lib/libcxx/include/__memory/raw_storage_iterator.h+3-9

...	...	@@ -28,15 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
28	28
29	29	#if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR)
30	30
31		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
32	31	template <class _OutputIterator, class _Tp>
33	32	class _LIBCPP_DEPRECATED_IN_CXX17 raw_storage_iterator
34		# if _LIBCPP_STD_VER <= 14 \|\| !defined(_LIBCPP_ABI_NO_ITERATOR_BASES)
35		: public iterator<output_iterator_tag, void, void, void, void>
36		# endif
37		{
38		_LIBCPP_SUPPRESS_DEPRECATED_POP
39
	33	: public __iterator_base<raw_storage_iterator<_OutputIterator, _Tp>, output_iterator_tag, void, void, void, void> {
40	34	private:
41	35	_OutputIterator __x_;
42	36
...	...	@@ -52,7 +46,7 @@ public:
52	46	typedef void reference;
53	47
54	48	_LIBCPP_HIDE_FROM_ABI explicit raw_storage_iterator(_OutputIterator __x) : __x_(__x) {}
55		_LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator() { return this; }
	49	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator() { return this; }
56	50	_LIBCPP_HIDE_FROM_ABI raw_storage_iterator& operator=(const _Tp& __element) {
57	51	::new ((void)std::addressof(__x_)) _Tp(__element);
58	52	return *this;
...	...	@@ -73,7 +67,7 @@ public:
73	67	return __t;
74	68	}
75	69	# if _LIBCPP_STD_VER >= 14
76		_LIBCPP_HIDE_FROM_ABI _OutputIterator base() const { return __x_; }
	70	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _OutputIterator base() const { return __x_; }
77	71	# endif
78	72	};
79	73

lib/libcxx/include/__memory/shared_count.h+9-30

...	...	@@ -22,37 +22,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
22	22	// NOTE: Relaxed and acq/rel atomics (for increment and decrement respectively)
23	23	// should be sufficient for thread safety.
24	24	// See https://llvm.org/PR22803
25		#if (defined(__clang__) && __has_builtin(__atomic_add_fetch) && defined(__ATOMIC_RELAXED) && \
26		defined(__ATOMIC_ACQ_REL)) \|\| \
27		defined(_LIBCPP_COMPILER_GCC)
28		# define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT 1
29		#else
30		# define _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT 0
31		#endif
32
33		template <class _ValueType>
34		inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_relaxed_load(_ValueType const* __value) {
35		#if _LIBCPP_HAS_THREADS && defined(__ATOMIC_RELAXED) && \
36		(__has_builtin(__atomic_load_n) \|\| defined(_LIBCPP_COMPILER_GCC))
37		return __atomic_load_n(__value, __ATOMIC_RELAXED);
38		#else
39		return *__value;
40		#endif
41		}
42
43		template <class _ValueType>
44		inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_acquire_load(_ValueType const* __value) {
45		#if _LIBCPP_HAS_THREADS && defined(__ATOMIC_ACQUIRE) && \
46		(__has_builtin(__atomic_load_n) \|\| defined(_LIBCPP_COMPILER_GCC))
47		return __atomic_load_n(__value, __ATOMIC_ACQUIRE);
48		#else
49		return *__value;
50		#endif
51		}
52	25
53	26	template <class _Tp>
54	27	inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_increment(_Tp& __t) _NOEXCEPT {
55		#if _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT && _LIBCPP_HAS_THREADS
	28	#if _LIBCPP_HAS_THREADS
56	29	return __atomic_add_fetch(std::addressof(__t), 1, __ATOMIC_RELAXED);
57	30	#else
58	31	return __t += 1;
...	...	@@ -61,7 +34,7 @@ inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_increment(_Tp& __t) _N
61	34
62	35	template <class _Tp>
63	36	inline _LIBCPP_HIDE_FROM_ABI _Tp __libcpp_atomic_refcount_decrement(_Tp& __t) _NOEXCEPT {
64		#if _LIBCPP_HAS_BUILTIN_ATOMIC_SUPPORT && _LIBCPP_HAS_THREADS
	37	#if _LIBCPP_HAS_THREADS
65	38	return __atomic_add_fetch(std::addressof(__t), -1, __ATOMIC_ACQ_REL);
66	39	#else
67	40	return __t -= 1;
...	...	@@ -95,7 +68,13 @@ public:
95	68	return false;
96	69	}
97	70	#endif
98		_LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __libcpp_relaxed_load(&__shared_owners_) + 1; }
	71	_LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT {
	72	#if _LIBCPP_HAS_THREADS
	73	return __atomic_load_n(&__shared_owners_, __ATOMIC_RELAXED) + 1;
	74	#else
	75	return __shared_owners_ + 1;
	76	#endif
	77	}
99	78	};
100	79
101	80	class _LIBCPP_EXPORTED_FROM_ABI __shared_weak_count : private __shared_count {

lib/libcxx/include/__memory/shared_ptr.h+100-145

...	...	@@ -41,19 +41,18 @@
41	41	#include <__type_traits/enable_if.h>
42	42	#include <__type_traits/integral_constant.h>
43	43	#include <__type_traits/is_array.h>
44		#include <__type_traits/is_bounded_array.h>
45	44	#include <__type_traits/is_constructible.h>
46	45	#include <__type_traits/is_convertible.h>
47	46	#include <__type_traits/is_function.h>
48	47	#include <__type_traits/is_reference.h>
49	48	#include <__type_traits/is_same.h>
50		#include <__type_traits/is_unbounded_array.h>
51	49	#include <__type_traits/nat.h>
52	50	#include <__type_traits/negation.h>
53	51	#include <__type_traits/remove_cv.h>
54	52	#include <__type_traits/remove_extent.h>
55	53	#include <__type_traits/remove_reference.h>
56	54	#include <__utility/declval.h>
	55	#include <__utility/exception_guard.h>
57	56	#include <__utility/forward.h>
58	57	#include <__utility/move.h>
59	58	#include <__utility/swap.h>
...	...	@@ -78,7 +77,7 @@ public:
78	77	_LIBCPP_HIDE_FROM_ABI bad_weak_ptr(const bad_weak_ptr&) _NOEXCEPT = default;
79	78	_LIBCPP_HIDE_FROM_ABI bad_weak_ptr& operator=(const bad_weak_ptr&) _NOEXCEPT = default;
80	79	~bad_weak_ptr() _NOEXCEPT override;
81		const char* what() const _NOEXCEPT override;
	80	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
82	81	};
83	82
84	83	[[__noreturn__]] inline _LIBCPP_HIDE_FROM_ABI void __throw_bad_weak_ptr() {
...	...	@@ -316,10 +315,8 @@ public:
316	315	#endif
317	316
318	317	// A shared_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
319		// any bookkeeping, so it's always trivially relocatable. It is also replaceable because assignment just rebinds the
320		// shared_ptr to manage a different object.
	318	// any bookkeeping, so it's always trivially relocatable.
321	319	using __trivially_relocatable _LIBCPP_NODEBUG = shared_ptr;
322		using __replaceable _LIBCPP_NODEBUG = shared_ptr;
323	320
324	321	private:
325	322	element_type* __ptr_;
...	...	@@ -352,23 +349,16 @@ public:
352	349
353	350	template <class _Yp, class _Dp, __enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
354	351	_LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d) : __ptr_(__p) {
355		#if _LIBCPP_HAS_EXCEPTIONS
356		try {
357		#endif // _LIBCPP_HAS_EXCEPTIONS
358		typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
359		typedef __shared_ptr_pointer<_Yp*, _Dp, _AllocT> _CntrlBlk;
	352	auto __guard = std::__make_exception_guard([&] { __d(__p); });
	353	typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
	354	typedef __shared_ptr_pointer<_Yp*, _Dp, _AllocT> _CntrlBlk;
360	355	#ifndef _LIBCPP_CXX03_LANG
361		__cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
	356	__cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
362	357	#else
363	358	__cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
364	359	#endif // not _LIBCPP_CXX03_LANG
365		__enable_weak_this(__p, __p);
366		#if _LIBCPP_HAS_EXCEPTIONS
367		} catch (...) {
368		__d(__p);
369		throw;
370		}
371		#endif // _LIBCPP_HAS_EXCEPTIONS
	360	__enable_weak_this(__p, __p);
	361	__guard.__complete();
372	362	}
373	363
374	364	template <class _Yp,
...	...	@@ -376,28 +366,21 @@ public:
376	366	class _Alloc,
377	367	__enable_if_t<__shared_ptr_deleter_ctor_reqs<_Dp, _Yp, _Tp>::value, int> = 0>
378	368	_LIBCPP_HIDE_FROM_ABI shared_ptr(_Yp* __p, _Dp __d, _Alloc __a) : __ptr_(__p) {
379		#if _LIBCPP_HAS_EXCEPTIONS
380		try {
381		#endif // _LIBCPP_HAS_EXCEPTIONS
382		typedef __shared_ptr_pointer<_Yp*, _Dp, _Alloc> _CntrlBlk;
383		typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
384		typedef __allocator_destructor<_A2> _D2;
385		_A2 __a2(__a);
386		unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
387		::new ((void)std::addressof(__hold2.get()))
	369	auto __guard = std::__make_exception_guard([&] { __d(__p); });
	370	typedef __shared_ptr_pointer<_Yp*, _Dp, _Alloc> _CntrlBlk;
	371	typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
	372	typedef __allocator_destructor<_A2> _D2;
	373	_A2 __a2(__a);
	374	unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
	375	::new ((void)std::addressof(__hold2.get()))
388	376	#ifndef _LIBCPP_CXX03_LANG
389		_CntrlBlk(__p, std::move(__d), __a);
	377	_CntrlBlk(__p, std::move(__d), __a);
390	378	#else
391	379	_CntrlBlk(__p, __d, __a);
392	380	#endif // not _LIBCPP_CXX03_LANG
393		__cntrl_ = std::addressof(*__hold2.release());
394		__enable_weak_this(__p, __p);
395		#if _LIBCPP_HAS_EXCEPTIONS
396		} catch (...) {
397		__d(__p);
398		throw;
399		}
400		#endif // _LIBCPP_HAS_EXCEPTIONS
	381	__cntrl_ = std::addressof(*__hold2.release());
	382	__enable_weak_this(__p, __p);
	383	__guard.__complete();
401	384	}
402	385
403	386	template <class _Dp>
...	...	@@ -406,22 +389,15 @@ public:
406	389	_Dp __d,
407	390	__enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
408	391	: __ptr_(nullptr) {
409		#if _LIBCPP_HAS_EXCEPTIONS
410		try {
411		#endif // _LIBCPP_HAS_EXCEPTIONS
412		typedef typename __shared_ptr_default_allocator<_Tp>::type _AllocT;
413		typedef __shared_ptr_pointer<nullptr_t, _Dp, _AllocT> _CntrlBlk;
	392	auto __guard = std::__make_exception_guard([&] { __d(__p); });
	393	typedef typename __shared_ptr_default_allocator<_Tp>::type _AllocT;
	394	typedef __shared_ptr_pointer<nullptr_t, _Dp, _AllocT> _CntrlBlk;
414	395	#ifndef _LIBCPP_CXX03_LANG
415		__cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
	396	__cntrl_ = new _CntrlBlk(__p, std::move(__d), _AllocT());
416	397	#else
417	398	__cntrl_ = new _CntrlBlk(__p, __d, _AllocT());
418	399	#endif // not _LIBCPP_CXX03_LANG
419		#if _LIBCPP_HAS_EXCEPTIONS
420		} catch (...) {
421		__d(__p);
422		throw;
423		}
424		#endif // _LIBCPP_HAS_EXCEPTIONS
	400	__guard.__complete();
425	401	}
426	402
427	403	template <class _Dp, class _Alloc>
...	...	@@ -431,27 +407,20 @@ public:
431	407	_Alloc __a,
432	408	__enable_if_t<__shared_ptr_nullptr_deleter_ctor_reqs<_Dp>::value, __nullptr_sfinae_tag> = __nullptr_sfinae_tag())
433	409	: __ptr_(nullptr) {
434		#if _LIBCPP_HAS_EXCEPTIONS
435		try {
436		#endif // _LIBCPP_HAS_EXCEPTIONS
437		typedef __shared_ptr_pointer<nullptr_t, _Dp, _Alloc> _CntrlBlk;
438		typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
439		typedef __allocator_destructor<_A2> _D2;
440		_A2 __a2(__a);
441		unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
442		::new ((void)std::addressof(__hold2.get()))
	410	auto __guard = std::__make_exception_guard([&] { __d(__p); });
	411	typedef __shared_ptr_pointer<nullptr_t, _Dp, _Alloc> _CntrlBlk;
	412	typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
	413	typedef __allocator_destructor<_A2> _D2;
	414	_A2 __a2(__a);
	415	unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
	416	::new ((void)std::addressof(__hold2.get()))
443	417	#ifndef _LIBCPP_CXX03_LANG
444		_CntrlBlk(__p, std::move(__d), __a);
	418	_CntrlBlk(__p, std::move(__d), __a);
445	419	#else
446	420	_CntrlBlk(__p, __d, __a);
447	421	#endif // not _LIBCPP_CXX03_LANG
448		__cntrl_ = std::addressof(*__hold2.release());
449		#if _LIBCPP_HAS_EXCEPTIONS
450		} catch (...) {
451		__d(__p);
452		throw;
453		}
454		#endif // _LIBCPP_HAS_EXCEPTIONS
	422	__cntrl_ = std::addressof(*__hold2.release());
	423	__guard.__complete();
455	424	}
456	425
457	426	template <class _Yp>
...	...	@@ -514,45 +483,16 @@ public:
514	483
515	484	template <class _Yp,
516	485	class _Dp,
517		__enable_if_t<!is_lvalue_reference<_Dp>::value && __compatible_with<_Yp, _Tp>::value &&
	486	__enable_if_t<__compatible_with<_Yp, _Tp>::value &&
518	487	is_convertible<typename unique_ptr<_Yp, _Dp>::pointer, element_type*>::value,
519	488	int> = 0>
520	489	_LIBCPP_HIDE_FROM_ABI shared_ptr(unique_ptr<_Yp, _Dp>&& __r) : __ptr_(__r.get()) {
521		#if _LIBCPP_STD_VER >= 14
522		if (__ptr_ == nullptr)
523		__cntrl_ = nullptr;
524		else
525		#endif
526		{
527		typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
528		typedef __shared_ptr_pointer<typename unique_ptr<_Yp, _Dp>::pointer, _Dp, _AllocT> _CntrlBlk;
529		__cntrl_ = new _CntrlBlk(__r.get(), std::move(__r.get_deleter()), _AllocT());
530		__enable_weak_this(__r.get(), __r.get());
531		}
532		__r.release();
533		}
	490	using _AllocT = typename __shared_ptr_default_allocator<_Yp>::type;
	491	using _Deleter = _If<is_lvalue_reference<_Dp>::value, reference_wrapper<__libcpp_remove_reference_t<_Dp> >, _Dp>;
	492	using _CntrlBlk = __shared_ptr_pointer<typename unique_ptr<_Yp, _Dp>::pointer, _Deleter, _AllocT>;
534	493
535		template <class _Yp,
536		class _Dp,
537		class = void,
538		__enable_if_t<is_lvalue_reference<_Dp>::value && __compatible_with<_Yp, _Tp>::value &&
539		is_convertible<typename unique_ptr<_Yp, _Dp>::pointer, element_type*>::value,
540		int> = 0>
541		_LIBCPP_HIDE_FROM_ABI shared_ptr(unique_ptr<_Yp, _Dp>&& __r) : __ptr_(__r.get()) {
542		#if _LIBCPP_STD_VER >= 14
543		if (__ptr_ == nullptr)
544		__cntrl_ = nullptr;
545		else
546		#endif
547		{
548		typedef typename __shared_ptr_default_allocator<_Yp>::type _AllocT;
549		typedef __shared_ptr_pointer<typename unique_ptr<_Yp, _Dp>::pointer,
550		reference_wrapper<__libcpp_remove_reference_t<_Dp> >,
551		_AllocT>
552		_CntrlBlk;
553		__cntrl_ = new _CntrlBlk(__r.get(), std::ref(__r.get_deleter()), _AllocT());
554		__enable_weak_this(__r.get(), __r.get());
555		}
	494	__cntrl_ = __ptr_ ? new _CntrlBlk(__r.get(), std::forward<_Dp>(__r.get_deleter()), _AllocT()) : nullptr;
	495	__enable_weak_this(__r.get(), __r.get());
556	496	__r.release();
557	497	}
558	498
...	...	@@ -628,37 +568,43 @@ public:
628	568	shared_ptr(__p, __d, __a).swap(*this);
629	569	}
630	570
631		_LIBCPP_HIDE_FROM_ABI element_type* get() const _NOEXCEPT { return __ptr_; }
	571	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI element_type* get() const _NOEXCEPT { return __ptr_; }
632	572
633		_LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator() const _NOEXCEPT { return __ptr_; }
	573	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator*() const _NOEXCEPT {
	574	return *__ptr_;
	575	}
634	576
635	577	_LIBCPP_HIDE_FROM_ABI element_type* operator->() const _NOEXCEPT {
636	578	static_assert(!is_array<_Tp>::value, "std::shared_ptr<T>::operator-> is only valid when T is not an array type.");
637	579	return __ptr_;
638	580	}
639	581
640		_LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __cntrl_ ? __cntrl_->use_count() : 0; }
	582	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT {
	583	return __cntrl_ ? __cntrl_->use_count() : 0;
	584	}
641	585
642	586	#if _LIBCPP_STD_VER < 20 \|\| defined(_LIBCPP_ENABLE_CXX20_REMOVED_SHARED_PTR_UNIQUE)
643		_LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI bool unique() const _NOEXCEPT { return use_count() == 1; }
	587	[[__nodiscard__]] _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI bool unique() const _NOEXCEPT {
	588	return use_count() == 1;
	589	}
644	590	#endif
645	591
646	592	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return get() != nullptr; }
647	593
648	594	template <class _Up>
649		_LIBCPP_HIDE_FROM_ABI bool owner_before(shared_ptr<_Up> const& __p) const _NOEXCEPT {
	595	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool owner_before(shared_ptr<_Up> const& __p) const _NOEXCEPT {
650	596	return __cntrl_ < __p.__cntrl_;
651	597	}
652	598
653	599	template <class _Up>
654		_LIBCPP_HIDE_FROM_ABI bool owner_before(weak_ptr<_Up> const& __p) const _NOEXCEPT {
	600	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool owner_before(weak_ptr<_Up> const& __p) const _NOEXCEPT {
655	601	return __cntrl_ < __p.__cntrl_;
656	602	}
657	603
658	604	_LIBCPP_HIDE_FROM_ABI bool __owner_equivalent(const shared_ptr& __p) const { return __cntrl_ == __p.__cntrl_; }
659	605
660	606	#if _LIBCPP_STD_VER >= 17
661		_LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator[](ptrdiff_t __i) const {
	607	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI __add_lvalue_reference_t<element_type> operator[](ptrdiff_t __i) const {
662	608	static_assert(is_array<_Tp>::value, "std::shared_ptr<T>::operator[] is only valid when T is an array type.");
663	609	return __ptr_[__i];
664	610	}
...	...	@@ -729,7 +675,7 @@ shared_ptr(unique_ptr<_Tp, _Dp>) -> shared_ptr<_Tp>;
729	675	// std::allocate_shared and std::make_shared
730	676	//
731	677	template <class _Tp, class _Alloc, class... _Args, __enable_if_t<!is_array<_Tp>::value, int> = 0>
732		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, _Args&&... __args) {
	678	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, _Args&&... __args) {
733	679	using _ControlBlock = __shared_ptr_emplace<_Tp, _Alloc>;
734	680	using _ControlBlockAllocator = typename __allocator_traits_rebind<_Alloc, _ControlBlock>::type;
735	681	__allocation_guard<_ControlBlockAllocator> __guard(__a, 1);
...	...	@@ -740,21 +686,21 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, _Args&&
740	686	}
741	687
742	688	template <class _Tp, class... _Args, __enable_if_t<!is_array<_Tp>::value, int> = 0>
743		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(_Args&&... __args) {
	689	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(_Args&&... __args) {
744	690	return std::allocate_shared<_Tp>(allocator<__remove_cv_t<_Tp> >(), std::forward<_Args>(__args)...);
745	691	}
746	692
747	693	#if _LIBCPP_STD_VER >= 20
748	694
749	695	template <class _Tp, class _Alloc, __enable_if_t<!is_array<_Tp>::value, int> = 0>
750		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
	696	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
751	697	using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
752	698	_ForOverwriteAllocator __alloc(__a);
753	699	return std::allocate_shared<_Tp>(__alloc);
754	700	}
755	701
756	702	template <class _Tp, __enable_if_t<!is_array<_Tp>::value, int> = 0>
757		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
	703	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
758	704	return std::allocate_shared_for_overwrite<_Tp>(allocator<__remove_cv_t<_Tp>>());
759	705	}
760	706
...	...	@@ -946,67 +892,69 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Array> __allocate_shared_bounded_array(const _
946	892
947	893	// bounded array variants
948	894	template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
949		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a) {
	895	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a) {
950	896	return std::__allocate_shared_bounded_array<_Tp>(__a);
951	897	}
952	898
953	899	template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
954		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, const remove_extent_t<_Tp>& __u) {
	900	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
	901	allocate_shared(const _Alloc& __a, const remove_extent_t<_Tp>& __u) {
955	902	return std::__allocate_shared_bounded_array<_Tp>(__a, __u);
956	903	}
957	904
958	905	template <class _Tp, class _Alloc, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
959		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
	906	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a) {
960	907	using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
961	908	_ForOverwriteAllocator __alloc(__a);
962	909	return std::__allocate_shared_bounded_array<_Tp>(__alloc);
963	910	}
964	911
965	912	template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
966		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared() {
	913	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared() {
967	914	return std::__allocate_shared_bounded_array<_Tp>(allocator<_Tp>());
968	915	}
969	916
970	917	template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
971		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(const remove_extent_t<_Tp>& __u) {
	918	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(const remove_extent_t<_Tp>& __u) {
972	919	return std::__allocate_shared_bounded_array<_Tp>(allocator<_Tp>(), __u);
973	920	}
974	921
975	922	template <class _Tp, __enable_if_t<is_bounded_array<_Tp>::value, int> = 0>
976		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
	923	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite() {
977	924	return std::__allocate_shared_bounded_array<_Tp>(allocator<__for_overwrite_tag>());
978	925	}
979	926
980	927	// unbounded array variants
981	928	template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
982		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, size_t __n) {
	929	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, size_t __n) {
983	930	return std::__allocate_shared_unbounded_array<_Tp>(__a, __n);
984	931	}
985	932
986	933	template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
987		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared(const _Alloc& __a, size_t __n, const remove_extent_t<_Tp>& __u) {
	934	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
	935	allocate_shared(const _Alloc& __a, size_t __n, const remove_extent_t<_Tp>& __u) {
988	936	return std::__allocate_shared_unbounded_array<_Tp>(__a, __n, __u);
989	937	}
990	938
991	939	template <class _Tp, class _Alloc, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
992		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a, size_t __n) {
	940	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> allocate_shared_for_overwrite(const _Alloc& __a, size_t __n) {
993	941	using _ForOverwriteAllocator = __allocator_traits_rebind_t<_Alloc, __for_overwrite_tag>;
994	942	_ForOverwriteAllocator __alloc(__a);
995	943	return std::__allocate_shared_unbounded_array<_Tp>(__alloc, __n);
996	944	}
997	945
998	946	template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
999		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n) {
	947	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n) {
1000	948	return std::__allocate_shared_unbounded_array<_Tp>(allocator<_Tp>(), __n);
1001	949	}
1002	950
1003	951	template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
1004		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n, const remove_extent_t<_Tp>& __u) {
	952	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared(size_t __n, const remove_extent_t<_Tp>& __u) {
1005	953	return std::__allocate_shared_unbounded_array<_Tp>(allocator<_Tp>(), __n, __u);
1006	954	}
1007	955
1008	956	template <class _Tp, __enable_if_t<is_unbounded_array<_Tp>::value, int> = 0>
1009		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite(size_t __n) {
	957	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> make_shared_for_overwrite(size_t __n) {
1010	958	return std::__allocate_shared_unbounded_array<_Tp>(allocator<__for_overwrite_tag>(), __n);
1011	959	}
1012	960
...	...	@@ -1135,7 +1083,8 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(shared_ptr<_Tp>& __x, shared_ptr<_Tp>& __
1135	1083	}
1136	1084
1137	1085	template <class _Tp, class _Up>
1138		inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
	1086	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
	1087	static_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
1139	1088	return shared_ptr<_Tp>(__r, static_cast< typename shared_ptr<_Tp>::element_type*>(__r.get()));
1140	1089	}
1141	1090
...	...	@@ -1143,13 +1092,14 @@ inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(const shared_pt
1143	1092	// We don't backport because it is an evolutionary change.
1144	1093	#if _LIBCPP_STD_VER >= 20
1145	1094	template <class _Tp, class _Up>
1146		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
	1095	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> static_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
1147	1096	return shared_ptr<_Tp>(std::move(__r), static_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
1148	1097	}
1149	1098	#endif
1150	1099
1151	1100	template <class _Tp, class _Up>
1152		inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
	1101	[[__nodiscard__]] inline
	1102	_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
1153	1103	typedef typename shared_ptr<_Tp>::element_type _ET;
1154	1104	_ET* __p = dynamic_cast<_ET*>(__r.get());
1155	1105	return __p ? shared_ptr<_Tp>(__r, __p) : shared_ptr<_Tp>();
...	...	@@ -1159,14 +1109,14 @@ inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(const shared_p
1159	1109	// We don't backport because it is an evolutionary change.
1160	1110	#if _LIBCPP_STD_VER >= 20
1161	1111	template <class _Tp, class _Up>
1162		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
	1112	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> dynamic_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
1163	1113	auto* __p = dynamic_cast<typename shared_ptr<_Tp>::element_type*>(__r.get());
1164	1114	return __p ? shared_ptr<_Tp>(std::move(__r), __p) : shared_ptr<_Tp>();
1165	1115	}
1166	1116	#endif
1167	1117
1168	1118	template <class _Tp, class _Up>
1169		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
	1119	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
1170	1120	typedef typename shared_ptr<_Tp>::element_type _RTp;
1171	1121	return shared_ptr<_Tp>(__r, const_cast<_RTp*>(__r.get()));
1172	1122	}
...	...	@@ -1175,13 +1125,13 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(const shared_ptr<_Up>&
1175	1125	// We don't backport because it is an evolutionary change.
1176	1126	#if _LIBCPP_STD_VER >= 20
1177	1127	template <class _Tp, class _Up>
1178		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
	1128	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> const_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
1179	1129	return shared_ptr<_Tp>(std::move(__r), const_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
1180	1130	}
1181	1131	#endif
1182	1132
1183	1133	template <class _Tp, class _Up>
1184		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
	1134	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(const shared_ptr<_Up>& __r) _NOEXCEPT {
1185	1135	return shared_ptr<_Tp>(__r, reinterpret_cast< typename shared_ptr<_Tp>::element_type*>(__r.get()));
1186	1136	}
1187	1137
...	...	@@ -1189,7 +1139,7 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(const shared_ptr<
1189	1139	// We don't backport because it is an evolutionary change.
1190	1140	#if _LIBCPP_STD_VER >= 20
1191	1141	template <class _Tp, class _Up>
1192		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
	1142	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(shared_ptr<_Up>&& __r) noexcept {
1193	1143	return shared_ptr<_Tp>(std::move(__r), reinterpret_cast<typename shared_ptr<_Tp>::element_type*>(__r.get()));
1194	1144	}
1195	1145	#endif
...	...	@@ -1197,7 +1147,7 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> reinterpret_pointer_cast(shared_ptr<_Up>&&
1197	1147	#if _LIBCPP_HAS_RTTI
1198	1148
1199	1149	template <class _Dp, class _Tp>
1200		inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXCEPT {
	1150	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXCEPT {
1201	1151	return __p.template __get_deleter<_Dp>();
1202	1152	}
1203	1153
...	...	@@ -1213,9 +1163,8 @@ public:
1213	1163	#endif
1214	1164
1215	1165	// A weak_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require
1216		// any bookkeeping, so it's always trivially relocatable. It's also replaceable for the same reason.
	1166	// any bookkeeping, so it's always trivially relocatable.
1217	1167	using __trivially_relocatable _LIBCPP_NODEBUG = weak_ptr;
1218		using __replaceable _LIBCPP_NODEBUG = weak_ptr;
1219	1168
1220	1169	private:
1221	1170	element_type* __ptr_;
...	...	@@ -1253,15 +1202,19 @@ public:
1253	1202	_LIBCPP_HIDE_FROM_ABI void swap(weak_ptr& __r) _NOEXCEPT;
1254	1203	_LIBCPP_HIDE_FROM_ABI void reset() _NOEXCEPT;
1255	1204
1256		_LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT { return __cntrl_ ? __cntrl_->use_count() : 0; }
1257		_LIBCPP_HIDE_FROM_ABI bool expired() const _NOEXCEPT { return __cntrl_ == nullptr \|\| __cntrl_->use_count() == 0; }
1258		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> lock() const _NOEXCEPT;
	1205	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI long use_count() const _NOEXCEPT {
	1206	return __cntrl_ ? __cntrl_->use_count() : 0;
	1207	}
	1208	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool expired() const _NOEXCEPT {
	1209	return __cntrl_ == nullptr \|\| __cntrl_->use_count() == 0;
	1210	}
	1211	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> lock() const _NOEXCEPT;
1259	1212	template <class _Up>
1260		_LIBCPP_HIDE_FROM_ABI bool owner_before(const shared_ptr<_Up>& __r) const _NOEXCEPT {
	1213	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool owner_before(const shared_ptr<_Up>& __r) const _NOEXCEPT {
1261	1214	return __cntrl_ < __r.__cntrl_;
1262	1215	}
1263	1216	template <class _Up>
1264		_LIBCPP_HIDE_FROM_ABI bool owner_before(const weak_ptr<_Up>& __r) const _NOEXCEPT {
	1217	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool owner_before(const weak_ptr<_Up>& __r) const _NOEXCEPT {
1265	1218	return __cntrl_ < __r.__cntrl_;
1266	1219	}
1267	1220
...	...	@@ -1445,13 +1398,15 @@ protected:
1445	1398	_LIBCPP_HIDE_FROM_ABI ~enable_shared_from_this() {}
1446	1399
1447	1400	public:
1448		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> shared_from_this() { return shared_ptr<_Tp>(__weak_this_); }
1449		_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp const> shared_from_this() const { return shared_ptr<const _Tp>(__weak_this_); }
	1401	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> shared_from_this() { return shared_ptr<_Tp>(__weak_this_); }
	1402	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp const> shared_from_this() const {
	1403	return shared_ptr<const _Tp>(__weak_this_);
	1404	}
1450	1405
1451	1406	#if _LIBCPP_STD_VER >= 17
1452		_LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> weak_from_this() _NOEXCEPT { return __weak_this_; }
	1407	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI weak_ptr<_Tp> weak_from_this() _NOEXCEPT { return __weak_this_; }
1453	1408
1454		_LIBCPP_HIDE_FROM_ABI weak_ptr<const _Tp> weak_from_this() const _NOEXCEPT { return __weak_this_; }
	1409	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI weak_ptr<const _Tp> weak_from_this() const _NOEXCEPT { return __weak_this_; }
1455	1410	#endif // _LIBCPP_STD_VER >= 17
1456	1411
1457	1412	template <class _Up>
...	...	@@ -1468,7 +1423,7 @@ struct hash<shared_ptr<_Tp> > {
1468	1423	_LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
1469	1424	#endif
1470	1425
1471		_LIBCPP_HIDE_FROM_ABI size_t operator()(const shared_ptr<_Tp>& __ptr) const _NOEXCEPT {
	1426	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const shared_ptr<_Tp>& __ptr) const _NOEXCEPT {
1472	1427	return hash<typename shared_ptr<_Tp>::element_type*>()(__ptr.get());
1473	1428	}
1474	1429	};

lib/libcxx/include/__memory/temp_value.h+1-2

...	...	@@ -12,7 +12,6 @@
12	12	#include <__config>
13	13	#include <__memory/addressof.h>
14	14	#include <__memory/allocator_traits.h>
15		#include <__type_traits/aligned_storage.h>
16	15	#include <__utility/forward.h>
17	16
18	17	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -26,7 +25,7 @@ struct __temp_value {
26	25	typedef allocator_traits<_Alloc> _Traits;
27	26
28	27	#ifdef _LIBCPP_CXX03_LANG
29		typename aligned_storage<sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)>::type __v;
	28	_ALIGNAS_TYPE(_Tp) char __v[sizeof(_Tp)];
30	29	#else
31	30	union {
32	31	_Tp __v;

lib/libcxx/include/__memory/uninitialized_algorithms.h+44-115

...	...	@@ -32,7 +32,6 @@
32	32	#include <__type_traits/is_trivially_assignable.h>
33	33	#include <__type_traits/is_trivially_constructible.h>
34	34	#include <__type_traits/is_trivially_relocatable.h>
35		#include <__type_traits/is_unbounded_array.h>
36	35	#include <__type_traits/remove_const.h>
37	36	#include <__type_traits/remove_extent.h>
38	37	#include <__utility/exception_guard.h>
...	...	@@ -61,17 +60,10 @@ template <class _ValueType, class _InputIterator, class _Sentinel1, class _Forwa
61	60	inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_copy(
62	61	_InputIterator __ifirst, _Sentinel1 __ilast, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
63	62	_ForwardIterator __idx = __ofirst;
64		#if _LIBCPP_HAS_EXCEPTIONS
65		try {
66		#endif
67		for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
68		::new (static_cast<void>(std::addressof(__idx))) _ValueType(*__ifirst);
69		#if _LIBCPP_HAS_EXCEPTIONS
70		} catch (...) {
71		std::__destroy(__ofirst, __idx);
72		throw;
73		}
74		#endif
	63	auto __guard = std::__make_exception_guard([&] { std::__destroy(__ofirst, __idx); });
	64	for (; __ifirst != __ilast && !__stop_copying(__idx); ++__ifirst, (void)++__idx)
	65	::new (static_cast<void>(std::addressof(__idx))) _ValueType(*__ifirst);
	66	__guard.__complete();
75	67
76	68	return pair<_InputIterator, _ForwardIterator>(std::move(__ifirst), std::move(__idx));
77	69	}
...	...	@@ -91,17 +83,10 @@ template <class _ValueType, class _InputIterator, class _Size, class _ForwardIte
91	83	inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator>
92	84	__uninitialized_copy_n(_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_copying) {
93	85	_ForwardIterator __idx = __ofirst;
94		#if _LIBCPP_HAS_EXCEPTIONS
95		try {
96		#endif
97		for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
98		::new (static_cast<void>(std::addressof(__idx))) _ValueType(*__ifirst);
99		#if _LIBCPP_HAS_EXCEPTIONS
100		} catch (...) {
101		std::__destroy(__ofirst, __idx);
102		throw;
103		}
104		#endif
	86	auto __guard = std::__make_exception_guard([&] { std::__destroy(__ofirst, __idx); });
	87	for (; __n > 0 && !__stop_copying(__idx); ++__ifirst, (void)++__idx, (void)--__n)
	88	::new (static_cast<void>(std::addressof(__idx))) _ValueType(*__ifirst);
	89	__guard.__complete();
105	90
106	91	return pair<_InputIterator, _ForwardIterator>(std::move(__ifirst), std::move(__idx));
107	92	}
...	...	@@ -121,17 +106,10 @@ template <class _ValueType, class _ForwardIterator, class _Sentinel, class _Tp>
121	106	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
122	107	__uninitialized_fill(_ForwardIterator __first, _Sentinel __last, const _Tp& __x) {
123	108	_ForwardIterator __idx = __first;
124		#if _LIBCPP_HAS_EXCEPTIONS
125		try {
126		#endif
127		for (; __idx != __last; ++__idx)
128		::new (static_cast<void>(std::addressof(__idx))) _ValueType(__x);
129		#if _LIBCPP_HAS_EXCEPTIONS
130		} catch (...) {
131		std::__destroy(__first, __idx);
132		throw;
133		}
134		#endif
	109	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	110	for (; __idx != __last; ++__idx)
	111	::new (static_cast<void>(std::addressof(__idx))) _ValueType(__x);
	112	__guard.__complete();
135	113
136	114	return __idx;
137	115	}
...	...	@@ -149,17 +127,10 @@ template <class _ValueType, class _ForwardIterator, class _Size, class _Tp>
149	127	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
150	128	__uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
151	129	_ForwardIterator __idx = __first;
152		#if _LIBCPP_HAS_EXCEPTIONS
153		try {
154		#endif
155		for (; __n > 0; ++__idx, (void)--__n)
156		::new (static_cast<void>(std::addressof(__idx))) _ValueType(__x);
157		#if _LIBCPP_HAS_EXCEPTIONS
158		} catch (...) {
159		std::__destroy(__first, __idx);
160		throw;
161		}
162		#endif
	130	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	131	for (; __n > 0; ++__idx, (void)--__n)
	132	::new (static_cast<void>(std::addressof(__idx))) _ValueType(__x);
	133	__guard.__complete();
163	134
164	135	return __idx;
165	136	}
...	...	@@ -178,18 +149,11 @@ uninitialized_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x) {
178	149	template <class _ValueType, class _ForwardIterator, class _Sentinel>
179	150	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
180	151	__uninitialized_default_construct(_ForwardIterator __first, _Sentinel __last) {
181		auto __idx = __first;
182		# if _LIBCPP_HAS_EXCEPTIONS
183		try {
184		# endif
185		for (; __idx != __last; ++__idx)
186		::new (static_cast<void>(std::addressof(__idx))) _ValueType;
187		# if _LIBCPP_HAS_EXCEPTIONS
188		} catch (...) {
189		std::__destroy(__first, __idx);
190		throw;
191		}
192		# endif
	152	auto __idx = __first;
	153	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	154	for (; __idx != __last; ++__idx)
	155	::new (static_cast<void>(std::addressof(__idx))) _ValueType;
	156	__guard.__complete();
193	157
194	158	return __idx;
195	159	}
...	...	@@ -205,17 +169,10 @@ inline _LIBCPP_HIDE_FROM_ABI void uninitialized_default_construct(_ForwardIterat
205	169	template <class _ValueType, class _ForwardIterator, class _Size>
206	170	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_default_construct_n(_ForwardIterator __first, _Size __n) {
207	171	auto __idx = __first;
208		# if _LIBCPP_HAS_EXCEPTIONS
209		try {
210		# endif
211		for (; __n > 0; ++__idx, (void)--__n)
212		::new (static_cast<void>(std::addressof(__idx))) _ValueType;
213		# if _LIBCPP_HAS_EXCEPTIONS
214		} catch (...) {
215		std::__destroy(__first, __idx);
216		throw;
217		}
218		# endif
	172	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	173	for (; __n > 0; ++__idx, (void)--__n)
	174	::new (static_cast<void>(std::addressof(__idx))) _ValueType;
	175	__guard.__complete();
219	176
220	177	return __idx;
221	178	}
...	...	@@ -231,18 +188,11 @@ inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator uninitialized_default_construct_n(
231	188	template <class _ValueType, class _ForwardIterator, class _Sentinel>
232	189	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator
233	190	__uninitialized_value_construct(_ForwardIterator __first, _Sentinel __last) {
234		auto __idx = __first;
235		# if _LIBCPP_HAS_EXCEPTIONS
236		try {
237		# endif
238		for (; __idx != __last; ++__idx)
239		::new (static_cast<void>(std::addressof(__idx))) _ValueType();
240		# if _LIBCPP_HAS_EXCEPTIONS
241		} catch (...) {
242		std::__destroy(__first, __idx);
243		throw;
244		}
245		# endif
	191	auto __idx = __first;
	192	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	193	for (; __idx != __last; ++__idx)
	194	::new (static_cast<void>(std::addressof(__idx))) _ValueType();
	195	__guard.__complete();
246	196
247	197	return __idx;
248	198	}
...	...	@@ -258,17 +208,10 @@ inline _LIBCPP_HIDE_FROM_ABI void uninitialized_value_construct(_ForwardIterator
258	208	template <class _ValueType, class _ForwardIterator, class _Size>
259	209	inline _LIBCPP_HIDE_FROM_ABI _ForwardIterator __uninitialized_value_construct_n(_ForwardIterator __first, _Size __n) {
260	210	auto __idx = __first;
261		# if _LIBCPP_HAS_EXCEPTIONS
262		try {
263		# endif
264		for (; __n > 0; ++__idx, (void)--__n)
265		::new (static_cast<void>(std::addressof(__idx))) _ValueType();
266		# if _LIBCPP_HAS_EXCEPTIONS
267		} catch (...) {
268		std::__destroy(__first, __idx);
269		throw;
270		}
271		# endif
	211	auto __guard = std::__make_exception_guard([&] { std::__destroy(__first, __idx); });
	212	for (; __n > 0; ++__idx, (void)--__n)
	213	::new (static_cast<void>(std::addressof(__idx))) _ValueType();
	214	__guard.__complete();
272	215
273	216	return __idx;
274	217	}
...	...	@@ -293,19 +236,12 @@ inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitiali
293	236	_ForwardIterator __ofirst,
294	237	_EndPredicate __stop_moving,
295	238	_IterMove __iter_move) {
296		auto __idx = __ofirst;
297		# if _LIBCPP_HAS_EXCEPTIONS
298		try {
299		# endif
300		for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
301		::new (static_cast<void>(std::addressof(__idx))) _ValueType(__iter_move(__ifirst));
302		}
303		# if _LIBCPP_HAS_EXCEPTIONS
304		} catch (...) {
305		std::__destroy(__ofirst, __idx);
306		throw;
	239	auto __idx = __ofirst;
	240	auto __guard = std::__make_exception_guard([&] { std::__destroy(__ofirst, __idx); });
	241	for (; __ifirst != __ilast && !__stop_moving(__idx); ++__idx, (void)++__ifirst) {
	242	::new (static_cast<void>(std::addressof(__idx))) _ValueType(__iter_move(__ifirst));
307	243	}
308		# endif
	244	__guard.__complete();
309	245
310	246	return {std::move(__ifirst), std::move(__idx)};
311	247	}
...	...	@@ -331,18 +267,11 @@ template <class _ValueType,
331	267	class _IterMove>
332	268	inline _LIBCPP_HIDE_FROM_ABI pair<_InputIterator, _ForwardIterator> __uninitialized_move_n(
333	269	_InputIterator __ifirst, _Size __n, _ForwardIterator __ofirst, _EndPredicate __stop_moving, _IterMove __iter_move) {
334		auto __idx = __ofirst;
335		# if _LIBCPP_HAS_EXCEPTIONS
336		try {
337		# endif
338		for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
339		::new (static_cast<void>(std::addressof(__idx))) _ValueType(__iter_move(__ifirst));
340		# if _LIBCPP_HAS_EXCEPTIONS
341		} catch (...) {
342		std::__destroy(__ofirst, __idx);
343		throw;
344		}
345		# endif
	270	auto __idx = __ofirst;
	271	auto __guard = std::__make_exception_guard([&] { std::__destroy(__ofirst, __idx); });
	272	for (; __n > 0 && !__stop_moving(__idx); ++__idx, (void)++__ifirst, --__n)
	273	::new (static_cast<void>(std::addressof(__idx))) _ValueType(__iter_move(__ifirst));
	274	__guard.__complete();
346	275
347	276	return {std::move(__ifirst), std::move(__idx)};
348	277	}

lib/libcxx/include/__memory/unique_ptr.h+14-16

...	...	@@ -32,18 +32,15 @@
32	32	#include <__type_traits/integral_constant.h>
33	33	#include <__type_traits/is_array.h>
34	34	#include <__type_traits/is_assignable.h>
35		#include <__type_traits/is_bounded_array.h>
36	35	#include <__type_traits/is_constant_evaluated.h>
37	36	#include <__type_traits/is_constructible.h>
38	37	#include <__type_traits/is_convertible.h>
39	38	#include <__type_traits/is_function.h>
40	39	#include <__type_traits/is_pointer.h>
41	40	#include <__type_traits/is_reference.h>
42		#include <__type_traits/is_replaceable.h>
43	41	#include <__type_traits/is_same.h>
44	42	#include <__type_traits/is_swappable.h>
45	43	#include <__type_traits/is_trivially_relocatable.h>
46		#include <__type_traits/is_unbounded_array.h>
47	44	#include <__type_traits/is_void.h>
48	45	#include <__type_traits/remove_extent.h>
49	46	#include <__type_traits/type_identity.h>
...	...	@@ -145,8 +142,6 @@ public:
145	142	__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
146	143	unique_ptr,
147	144	void>;
148		using __replaceable _LIBCPP_NODEBUG =
149		__conditional_t<__is_replaceable_v<pointer> && __is_replaceable_v<deleter_type>, unique_ptr, void>;
150	145
151	146	private:
152	147	_LIBCPP_COMPRESSED_PAIR(pointer, __ptr_, deleter_type, __deleter_);
...	...	@@ -263,14 +258,17 @@ public:
263	258	return *this;
264	259	}
265	260
266		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator*() const
	261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator*() const
267	262	_NOEXCEPT_(_NOEXCEPT_(*std::declval<pointer>())) {
268	263	return *__ptr_;
269	264	}
270	265	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer operator->() const _NOEXCEPT { return __ptr_; }
271		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer get() const _NOEXCEPT { return __ptr_; }
272		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 deleter_type& get_deleter() _NOEXCEPT { return __deleter_; }
273		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const deleter_type& get_deleter() const _NOEXCEPT {
	266	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer get() const _NOEXCEPT { return __ptr_; }
	267	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 deleter_type& get_deleter() _NOEXCEPT {
	268	return __deleter_;
	269	}
	270	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 const deleter_type&
	271	get_deleter() const _NOEXCEPT {
274	272	return __deleter_;
275	273	}
276	274	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit operator bool() const _NOEXCEPT {
...	...	@@ -413,8 +411,6 @@ public:
413	411	__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<deleter_type>::value,
414	412	unique_ptr,
415	413	void>;
416		using __replaceable _LIBCPP_NODEBUG =
417		__conditional_t<__is_replaceable_v<pointer> && __is_replaceable_v<deleter_type>, unique_ptr, void>;
418	414
419	415	private:
420	416	template <class _Up, class _OtherDeleter>
...	...	@@ -755,12 +751,13 @@ operator<=>(const unique_ptr<_T1, _D1>& __x, nullptr_t) {
755	751	#if _LIBCPP_STD_VER >= 14
756	752
757	753	template <class _Tp, class... _Args, enable_if_t<!is_array<_Tp>::value, int> = 0>
758		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique(_Args&&... __args) {
	754	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	755	_LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique(_Args&&... __args) {
759	756	return unique_ptr<_Tp>(new _Tp(std::forward<_Args>(__args)...));
760	757	}
761	758
762	759	template <class _Tp, enable_if_t<__is_unbounded_array_v<_Tp>, int> = 0>
763		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique(size_t __n) {
	760	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique(size_t __n) {
764	761	typedef __remove_extent_t<_Tp> _Up;
765	762	return unique_ptr<_Tp>(__private_constructor_tag(), new _Up[__n](), __n);
766	763	}
...	...	@@ -773,12 +770,13 @@ void make_unique(_Args&&...) = delete;
773	770	#if _LIBCPP_STD_VER >= 20
774	771
775	772	template <class _Tp, enable_if_t<!is_array_v<_Tp>, int> = 0>
776		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique_for_overwrite() {
	773	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique_for_overwrite() {
777	774	return unique_ptr<_Tp>(new _Tp);
778	775	}
779	776
780	777	template <class _Tp, enable_if_t<is_unbounded_array_v<_Tp>, int> = 0>
781		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp> make_unique_for_overwrite(size_t __n) {
	778	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr<_Tp>
	779	make_unique_for_overwrite(size_t __n) {
782	780	return unique_ptr<_Tp>(__private_constructor_tag(), new __remove_extent_t<_Tp>[__n], __n);
783	781	}
784	782
...	...	@@ -802,7 +800,7 @@ struct hash<__enable_hash_helper< unique_ptr<_Tp, _Dp>, typename unique_ptr<_Tp,
802	800	_LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
803	801	#endif
804	802
805		_LIBCPP_HIDE_FROM_ABI size_t operator()(const unique_ptr<_Tp, _Dp>& __ptr) const {
	803	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const unique_ptr<_Tp, _Dp>& __ptr) const {
806	804	typedef typename unique_ptr<_Tp, _Dp>::pointer pointer;
807	805	return hash<pointer>()(__ptr.get());
808	806	}

lib/libcxx/include/__memory/uses_allocator_construction.h+1

...	...	@@ -17,6 +17,7 @@
17	17	#include <__type_traits/remove_cv.h>
18	18	#include <__utility/declval.h>
19	19	#include <__utility/pair.h>
	20	#include <__utility/piecewise_construct.h>
20	21	#include <tuple>
21	22
22	23	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)

lib/libcxx/include/__memory_resource/memory_resource.h+4-2

...	...	@@ -42,7 +42,9 @@ public:
42	42	do_deallocate(__p, __bytes, __align);
43	43	}
44	44
45		_LIBCPP_HIDE_FROM_ABI bool is_equal(const memory_resource& __other) const noexcept { return do_is_equal(__other); }
	45	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool is_equal(const memory_resource& __other) const noexcept {
	46	return do_is_equal(__other);
	47	}
46	48
47	49	private:
48	50	virtual void* do_allocate(size_t, size_t) = 0;
...	...	@@ -68,7 +70,7 @@ operator!=(const memory_resource& __lhs, const memory_resource& __rhs) noexcept
68	70
69	71	// [mem.res.global]
70	72
71		[[__gnu__::__returns_nonnull__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
	73	[[nodiscard, __gnu__::__returns_nonnull__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*
72	74	get_default_resource() noexcept;
73	75
74	76	[[__gnu__::__returns_nonnull__]] _LIBCPP_AVAILABILITY_PMR _LIBCPP_EXPORTED_FROM_ABI memory_resource*

lib/libcxx/include/__memory_resource/monotonic_buffer_resource.h+1-1

...	...	@@ -93,7 +93,7 @@ public:
93	93	}
94	94	}
95	95
96		_LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
	96	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
97	97
98	98	protected:
99	99	void* do_allocate(size_t __bytes, size_t __alignment) override; // key function

lib/libcxx/include/__memory_resource/polymorphic_allocator.h+13-8

...	...	@@ -18,6 +18,7 @@
18	18	#include <__new/exceptions.h>
19	19	#include <__new/placement_new_delete.h>
20	20	#include <__utility/exception_guard.h>
	21	#include <__utility/piecewise_construct.h>
21	22	#include <limits>
22	23	#include <tuple>
23	24
...	...	@@ -50,7 +51,9 @@ public:
50	51
51	52	_LIBCPP_HIDE_FROM_ABI polymorphic_allocator() noexcept : __res_(std::pmr::get_default_resource()) {}
52	53
53		_LIBCPP_HIDE_FROM_ABI polymorphic_allocator(memory_resource* __r) noexcept : __res_(__r) {}
	54	_LIBCPP_HIDE_FROM_ABI polymorphic_allocator(memory_resource* _LIBCPP_DIAGNOSE_NULLPTR __r) noexcept : __res_(__r) {
	55	_LIBCPP_ASSERT_NON_NULL(__r, "Attempted to pass a nullptr resource to polymorphic_alloator");
	56	}
54	57
55	58	_LIBCPP_HIDE_FROM_ABI polymorphic_allocator(const polymorphic_allocator&) = default;
56	59
...	...	@@ -133,10 +136,10 @@ public:
133	136	piecewise_construct,
134	137	__transform_tuple(typename __uses_alloc_ctor< _T1, polymorphic_allocator&, _Args1... >::type(),
135	138	std::move(__x),
136		typename __make_tuple_indices<sizeof...(_Args1)>::type{}),
	139	make_index_sequence<sizeof...(_Args1)>()),
137	140	__transform_tuple(typename __uses_alloc_ctor< _T2, polymorphic_allocator&, _Args2... >::type(),
138	141	std::move(__y),
139		typename __make_tuple_indices<sizeof...(_Args2)>::type{}));
	142	make_index_sequence<sizeof...(_Args2)>()));
140	143	}
141	144
142	145	template <class _T1, class _T2>
...	...	@@ -170,11 +173,13 @@ public:
170	173	__p->~_Tp();
171	174	}
172	175
173		_LIBCPP_HIDE_FROM_ABI polymorphic_allocator select_on_container_copy_construction() const noexcept {
	176	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI polymorphic_allocator select_on_container_copy_construction() const noexcept {
174	177	return polymorphic_allocator();
175	178	}
176	179
177		_LIBCPP_HIDE_FROM_ABI memory_resource* resource() const noexcept { return __res_; }
	180	[[nodiscard, __gnu__::__returns_nonnull__]] _LIBCPP_HIDE_FROM_ABI memory_resource* resource() const noexcept {
	181	return __res_;
	182	}
178	183
179	184	_LIBCPP_HIDE_FROM_ABI friend bool
180	185	operator==(const polymorphic_allocator& __lhs, const polymorphic_allocator& __rhs) noexcept {
...	...	@@ -192,20 +197,20 @@ public:
192	197	private:
193	198	template <class... _Args, size_t... _Is>
194	199	_LIBCPP_HIDE_FROM_ABI tuple<_Args&&...>
195		__transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
	200	__transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, index_sequence<_Is...>) {
196	201	return std::forward_as_tuple(std::get<_Is>(std::move(__t))...);
197	202	}
198	203
199	204	template <class... _Args, size_t... _Is>
200	205	_LIBCPP_HIDE_FROM_ABI tuple<allocator_arg_t const&, polymorphic_allocator&, _Args&&...>
201		__transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
	206	__transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, index_sequence<_Is...>) {
202	207	using _Tup = tuple<allocator_arg_t const&, polymorphic_allocator&, _Args&&...>;
203	208	return _Tup(allocator_arg, *this, std::get<_Is>(std::move(__t))...);
204	209	}
205	210
206	211	template <class... _Args, size_t... _Is>
207	212	_LIBCPP_HIDE_FROM_ABI tuple<_Args&&..., polymorphic_allocator&>
208		__transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, __tuple_indices<_Is...>) {
	213	__transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, index_sequence<_Is...>) {
209	214	using _Tup = tuple<_Args&&..., polymorphic_allocator&>;
210	215	return _Tup(std::get<_Is>(std::move(__t))..., *this);
211	216	}

lib/libcxx/include/__memory_resource/pool_options.h+1-1

...	...	@@ -24,7 +24,7 @@ namespace pmr {
24	24
25	25	// [mem.res.pool.options]
26	26
27		struct _LIBCPP_EXPORTED_FROM_ABI pool_options {
	27	struct pool_options {
28	28	size_t max_blocks_per_chunk = 0;
29	29	size_t largest_required_pool_block = 0;
30	30	};

lib/libcxx/include/__memory_resource/synchronized_pool_resource.h+4-2

...	...	@@ -56,9 +56,11 @@ public:
56	56	__unsync_.release();
57	57	}
58	58
59		_LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __unsync_.upstream_resource(); }
	59	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const {
	60	return __unsync_.upstream_resource();
	61	}
60	62
61		_LIBCPP_HIDE_FROM_ABI pool_options options() const { return __unsync_.options(); }
	63	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI pool_options options() const { return __unsync_.options(); }
62	64
63	65	protected:
64	66	_LIBCPP_HIDE_FROM_ABI_VIRTUAL void* do_allocate(size_t __bytes, size_t __align) override {

lib/libcxx/include/__memory_resource/unsynchronized_pool_resource.h+1-1

...	...	@@ -76,7 +76,7 @@ public:
76	76
77	77	void release();
78	78
79		_LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
	79	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI memory_resource* upstream_resource() const { return __res_; }
80	80
81	81	[[__gnu__::__pure__]] pool_options options() const;
82	82

lib/libcxx/include/__mutex/mutex.h+2-2

...	...	@@ -37,11 +37,11 @@ public:
37	37	# endif
38	38
39	39	_LIBCPP_ACQUIRE_CAPABILITY() void lock();
40		_LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT;
	40	[[__nodiscard__]] _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT;
41	41	_LIBCPP_RELEASE_CAPABILITY void unlock() _NOEXCEPT;
42	42
43	43	typedef __libcpp_mutex_t* native_handle_type;
44		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
	44	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
45	45	};
46	46
47	47	static_assert(is_nothrow_default_constructible<mutex>::value, "the default constructor for std::mutex must be nothrow");

lib/libcxx/include/__mutex/once_flag.h+14-11

...	...	@@ -10,12 +10,11 @@
10	10	#define _LIBCPP___MUTEX_ONCE_FLAG_H
11	11
12	12	#include <__config>
13		#include <__functional/invoke.h>
14	13	#include <__memory/addressof.h>
15		#include <__memory/shared_count.h> // __libcpp_acquire_load
16		#include <__tuple/tuple_indices.h>
17	14	#include <__tuple/tuple_size.h>
	15	#include <__type_traits/invoke.h>
18	16	#include <__utility/forward.h>
	17	#include <__utility/integer_sequence.h>
19	18	#include <__utility/move.h>
20	19	#include <cstdint>
21	20	#ifndef _LIBCPP_CXX03_LANG
...	...	@@ -88,14 +87,9 @@ public:
88	87	_LIBCPP_HIDE_FROM_ABI explicit __call_once_param(_Fp& __f) : __f_(__f) {}
89	88
90	89	_LIBCPP_HIDE_FROM_ABI void operator()() {
91		typedef typename __make_tuple_indices<tuple_size<_Fp>::value, 1>::type _Index;
92		__execute(_Index());
93		}
94
95		private:
96		template <size_t... _Indices>
97		_LIBCPP_HIDE_FROM_ABI void __execute(__tuple_indices<_Indices...>) {
98		std::__invoke(std::get<0>(std::move(__f_)), std::get<_Indices>(std::move(__f_))...);
	90	[&]<size_t... _Indices>(__index_sequence<_Indices...>) -> void {
	91	std::__invoke(std::get<_Indices>(std::move(__f_))...);
	92	}(__make_index_sequence<tuple_size<_Fp>::value>());
99	93	}
100	94	};
101	95
...	...	@@ -121,6 +115,15 @@ void _LIBCPP_HIDE_FROM_ABI __call_once_proxy(void* __vp) {
121	115
122	116	_LIBCPP_EXPORTED_FROM_ABI void __call_once(volatile once_flag::_State_type&, void, void ()(void*));
123	117
	118	template <class _ValueType>
	119	inline _LIBCPP_HIDE_FROM_ABI _ValueType __libcpp_acquire_load(_ValueType const* __value) {
	120	#if _LIBCPP_HAS_THREADS
	121	return __atomic_load_n(__value, __ATOMIC_ACQUIRE);
	122	#else
	123	return *__value;
	124	#endif
	125	}
	126
124	127	#ifndef _LIBCPP_CXX03_LANG
125	128
126	129	template <class _Callable, class... _Args>

lib/libcxx/include/__mutex/tag_types.h+3-3

...	...	@@ -17,15 +17,15 @@
17	17
18	18	_LIBCPP_BEGIN_NAMESPACE_STD
19	19
20		struct _LIBCPP_EXPORTED_FROM_ABI defer_lock_t {
	20	struct defer_lock_t {
21	21	explicit defer_lock_t() = default;
22	22	};
23	23
24		struct _LIBCPP_EXPORTED_FROM_ABI try_to_lock_t {
	24	struct try_to_lock_t {
25	25	explicit try_to_lock_t() = default;
26	26	};
27	27
28		struct _LIBCPP_EXPORTED_FROM_ABI adopt_lock_t {
	28	struct adopt_lock_t {
29	29	explicit adopt_lock_t() = default;
30	30	};
31	31

lib/libcxx/include/__mutex/unique_lock.h+8-7

...	...	@@ -15,6 +15,7 @@
15	15	#include <__memory/addressof.h>
16	16	#include <__mutex/tag_types.h>
17	17	#include <__system_error/throw_system_error.h>
	18	#include <__utility/move.h>
18	19	#include <__utility/swap.h>
19	20	#include <cerrno>
20	21
...	...	@@ -22,6 +23,9 @@
22	23	# pragma GCC system_header
23	24	#endif
24	25
	26	_LIBCPP_PUSH_MACROS
	27	#include <__undef_macros>
	28
25	29	_LIBCPP_BEGIN_NAMESPACE_STD
26	30
27	31	template <class _Mutex>
...	...	@@ -74,13 +78,8 @@ public:
74	78	}
75	79
76	80	_LIBCPP_HIDE_FROM_ABI unique_lock& operator=(unique_lock&& __u) _NOEXCEPT {
77		if (__owns_)
78		__m_->unlock();
79
80		__m_ = __u.__m_;
81		__owns_ = __u.__owns_;
82		__u.__m_ = nullptr;
83		__u.__owns_ = false;
	81	if (this != std::addressof(__u))
	82	unique_lock(std::move(__u)).swap(*this);
84	83	return *this;
85	84	}
86	85
...	...	@@ -170,4 +169,6 @@ inline _LIBCPP_HIDE_FROM_ABI void swap(unique_lock<_Mutex>& __x, unique_lock<_Mu
170	169
171	170	_LIBCPP_END_NAMESPACE_STD
172	171
	172	_LIBCPP_POP_MACROS
	173
173	174	#endif // _LIBCPP___MUTEX_UNIQUE_LOCK_H

lib/libcxx/include/__new/align_val_t.h+6

...	...	@@ -16,6 +16,12 @@
16	16	# pragma GCC system_header
17	17	#endif
18	18
	19	// <vcruntime_exception.h> defines its own std::align_val_t type,
	20	// which we use in order to be ABI-compatible with other STLs on Windows.
	21	#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION && defined(_LIBCPP_ABI_VCRUNTIME)
	22	# include <vcruntime_new.h>
	23	#endif
	24
19	25	_LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
20	26	#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION && !defined(_LIBCPP_ABI_VCRUNTIME)
21	27	# ifndef _LIBCPP_CXX03_LANG

lib/libcxx/include/__new/allocate.h+1-2

...	...	@@ -13,7 +13,6 @@
13	13	#include <__cstddef/max_align_t.h>
14	14	#include <__cstddef/size_t.h>
15	15	#include <__new/align_val_t.h>
16		#include <__new/global_new_delete.h> // for _LIBCPP_HAS_SIZED_DEALLOCATION
17	16	#include <__type_traits/type_identity.h>
18	17	#include <__utility/element_count.h>
19	18
...	...	@@ -43,7 +42,7 @@ __libcpp_allocate(__element_count __n, [[__maybe_unused__]] size_t __align = _LI
43	42	return static_cast<_Tp*>(__builtin_operator_new(__size));
44	43	}
45	44
46		#if _LIBCPP_HAS_SIZED_DEALLOCATION
	45	#if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
47	46	# define _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(...) __VA_ARGS__
48	47	#else
49	48	# define _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(...) /* nothing */

lib/libcxx/include/__new/exceptions.h+8-2

...	...	@@ -17,6 +17,12 @@
17	17	# pragma GCC system_header
18	18	#endif
19	19
	20	// <vcruntime_exception.h> defines its own std::bad_alloc type,
	21	// which we use in order to be ABI-compatible with other STLs on Windows.
	22	#if defined(_LIBCPP_ABI_VCRUNTIME)
	23	# include <vcruntime_exception.h>
	24	#endif
	25
20	26	_LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
21	27	#if !defined(_LIBCPP_ABI_VCRUNTIME)
22	28
...	...	@@ -26,7 +32,7 @@ public:
26	32	_LIBCPP_HIDE_FROM_ABI bad_alloc(const bad_alloc&) _NOEXCEPT = default;
27	33	_LIBCPP_HIDE_FROM_ABI bad_alloc& operator=(const bad_alloc&) _NOEXCEPT = default;
28	34	~bad_alloc() _NOEXCEPT override;
29		const char* what() const _NOEXCEPT override;
	35	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
30	36	};
31	37
32	38	class _LIBCPP_EXPORTED_FROM_ABI bad_array_new_length : public bad_alloc {
...	...	@@ -35,7 +41,7 @@ public:
35	41	_LIBCPP_HIDE_FROM_ABI bad_array_new_length(const bad_array_new_length&) _NOEXCEPT = default;
36	42	_LIBCPP_HIDE_FROM_ABI bad_array_new_length& operator=(const bad_array_new_length&) _NOEXCEPT = default;
37	43	~bad_array_new_length() _NOEXCEPT override;
38		const char* what() const _NOEXCEPT override;
	44	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
39	45	};
40	46
41	47	#elif defined(_HAS_EXCEPTIONS) && _HAS_EXCEPTIONS == 0 // !_LIBCPP_ABI_VCRUNTIME

lib/libcxx/include/__new/global_new_delete.h+4-11

...	...	@@ -12,7 +12,6 @@
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
14	14	#include <__new/align_val_t.h>
15		#include <__new/exceptions.h>
16	15	#include <__new/nothrow_t.h>
17	16
18	17	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -25,12 +24,6 @@
25	24	# define _THROW_BAD_ALLOC
26	25	#endif
27	26
28		#if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
29		# define _LIBCPP_HAS_SIZED_DEALLOCATION 1
30		#else
31		# define _LIBCPP_HAS_SIZED_DEALLOCATION 0
32		#endif
33
34	27	#if defined(_LIBCPP_ABI_VCRUNTIME)
35	28	# include <new.h>
36	29	#else
...	...	@@ -39,7 +32,7 @@
39	32	_LIBCPP_NOALIAS;
40	33	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p) _NOEXCEPT;
41	34	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, const std::nothrow_t&) _NOEXCEPT;
42		# if _LIBCPP_HAS_SIZED_DEALLOCATION
	35	# if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
43	36	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _NOEXCEPT;
44	37	# endif
45	38
...	...	@@ -48,7 +41,7 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz) _
48	41	_LIBCPP_NOALIAS;
49	42	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p) _NOEXCEPT;
50	43	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, const std::nothrow_t&) _NOEXCEPT;
51		# if _LIBCPP_HAS_SIZED_DEALLOCATION
	44	# if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
52	45	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz) _NOEXCEPT;
53	46	# endif
54	47
...	...	@@ -58,7 +51,7 @@ _LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz)
58	51	operator new(std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
59	52	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t) _NOEXCEPT;
60	53	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
61		# if _LIBCPP_HAS_SIZED_DEALLOCATION
	54	# if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
62	55	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete(void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
63	56	# endif
64	57
...	...	@@ -68,7 +61,7 @@ operator new[](std::size_t __sz, std::align_val_t) _THROW_BAD_ALLOC;
68	61	operator new[](std::size_t __sz, std::align_val_t, const std::nothrow_t&) _NOEXCEPT _LIBCPP_NOALIAS;
69	62	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t) _NOEXCEPT;
70	63	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::align_val_t, const std::nothrow_t&) _NOEXCEPT;
71		# if _LIBCPP_HAS_SIZED_DEALLOCATION
	64	# if defined(__cpp_sized_deallocation) && __cpp_sized_deallocation >= 201309L
72	65	_LIBCPP_OVERRIDABLE_FUNC_VIS void operator delete[](void* __p, std::size_t __sz, std::align_val_t) _NOEXCEPT;
73	66	# endif
74	67	# endif

lib/libcxx/include/__new/interference_size.h-4

...	...	@@ -20,13 +20,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
20	20
21	21	#if _LIBCPP_STD_VER >= 17
22	22
23		# if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
24
25	23	inline constexpr size_t hardware_destructive_interference_size = __GCC_DESTRUCTIVE_SIZE;
26	24	inline constexpr size_t hardware_constructive_interference_size = __GCC_CONSTRUCTIVE_SIZE;
27	25
28		# endif // defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
29
30	26	#endif // _LIBCPP_STD_VER >= 17
31	27
32	28	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__new/launder.h+3-5

...	...	@@ -10,8 +10,6 @@
10	10	#define _LIBCPP___NEW_LAUNDER_H
11	11
12	12	#include <__config>
13		#include <__type_traits/is_function.h>
14		#include <__type_traits/is_void.h>
15	13
16	14	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17	15	# pragma GCC system_header
...	...	@@ -20,15 +18,15 @@
20	18	_LIBCPP_BEGIN_NAMESPACE_STD
21	19	template <class _Tp>
22	20	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp* __launder(_Tp* __p) _NOEXCEPT {
23		static_assert(!(is_function<_Tp>::value), "can't launder functions");
24		static_assert(!is_void<_Tp>::value, "can't launder cv-void");
	21	// The compiler diagnoses misuses of __builtin_launder, so we don't need to add any static_asserts
	22	// to implement the Mandates.
25	23	return __builtin_launder(__p);
26	24	}
27	25
28	26	#if _LIBCPP_STD_VER >= 17
29	27	template <class _Tp>
30	28	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp* launder(_Tp* __p) noexcept {
31		return std::__launder(__p);
	29	return __builtin_launder(__p);
32	30	}
33	31	#endif
34	32	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__new/nothrow_t.h+1-1

...	...	@@ -19,7 +19,7 @@
19	19	# include <new.h>
20	20	#else
21	21	_LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
22		struct _LIBCPP_EXPORTED_FROM_ABI nothrow_t {
	22	struct nothrow_t {
23	23	explicit nothrow_t() = default;
24	24	};
25	25	extern _LIBCPP_EXPORTED_FROM_ABI const nothrow_t nothrow;

lib/libcxx/include/__numeric/gcd_lcm.h+24-34

...	...	@@ -33,28 +33,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD
33	33
34	34	#if _LIBCPP_STD_VER >= 17
35	35
36		template <typename _Result, typename _Source, bool _IsSigned = is_signed<_Source>::value>
37		struct __ct_abs;
38
39		template <typename _Result, typename _Source>
40		struct __ct_abs<_Result, _Source, true> {
41		constexpr _LIBCPP_HIDE_FROM_ABI _Result operator()(_Source __t) const noexcept {
	36	template <class _Result, class _Source>
	37	constexpr _LIBCPP_HIDE_FROM_ABI _Result __abs_in_type(_Source __t) noexcept {
	38	if constexpr (is_signed_v<_Source>) {
42	39	if (__t >= 0)
43	40	return __t;
44	41	if (__t == numeric_limits<_Source>::min())
45	42	return -static_cast<_Result>(__t);
46	43	return -__t;
	44	} else {
	45	return __t;
47	46	}
48		};
49
50		template <typename _Result, typename _Source>
51		struct __ct_abs<_Result, _Source, false> {
52		constexpr _LIBCPP_HIDE_FROM_ABI _Result operator()(_Source __t) const noexcept { return __t; }
53		};
	47	}
54	48
55		template <class _Tp>
56		constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
57		static_assert(!is_signed<_Tp>::value, "");
	49	template <class _Tp, class _Up>
	50	constexpr _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> gcd(_Tp __m, _Up __n) {
	51	static_assert(is_integral<_Tp>::value && is_integral<_Up>::value, "Arguments to gcd must be integer types");
	52	static_assert(!is_same<__remove_cv_t<_Tp>, bool>::value, "First argument to gcd cannot be bool");
	53	static_assert(!is_same<__remove_cv_t<_Up>, bool>::value, "Second argument to gcd cannot be bool");
	54	using _Rp = common_type_t<_Tp, _Up>;
	55	using _Wp = make_unsigned_t<_Rp>;
58	56
59	57	// Using Binary GCD algorithm https://en.wikipedia.org/wiki/Binary_GCD_algorithm, based on an implementation
60	58	// from https://lemire.me/blog/2024/04/13/greatest-common-divisor-the-extended-euclidean-algorithm-and-speed/
...	...	@@ -67,22 +65,25 @@ constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
67	65	//
68	66	// And standard gcd algorithm where instead of modulo, minus is used.
69	67
	68	auto __a = static_cast<_Wp>(std::__abs_in_type<_Rp>(__m));
	69	auto __b = static_cast<_Wp>(std::__abs_in_type<_Rp>(__n));
	70
70	71	if (__a < __b) {
71		_Tp __tmp = __b;
	72	_Wp __tmp = __b;
72	73	__b = __a;
73	74	__a = __tmp;
74	75	}
75	76	if (__b == 0)
76		return __a;
	77	return static_cast<_Rp>(__a);
77	78	__a %= __b; // Make both argument of the same size, and early result in the easy case.
78	79	if (__a == 0)
79		return __b;
	80	return static_cast<_Rp>(__b);
80	81
81		_Tp __c = __a \| __b;
	82	_Wp __c = __a \| __b;
82	83	int __shift = std::__countr_zero(__c);
83	84	__a >>= std::__countr_zero(__a);
84	85	do {
85		_Tp __t = __b >> std::__countr_zero(__b);
	86	_Wp __t = __b >> std::__countr_zero(__b);
86	87	if (__a > __t) {
87	88	__b = __a - __t;
88	89	__a = __t;
...	...	@@ -90,18 +91,7 @@ constexpr _LIBCPP_HIDDEN _Tp __gcd(_Tp __a, _Tp __b) {
90	91	__b = __t - __a;
91	92	}
92	93	} while (__b != 0);
93		return __a << __shift;
94		}
95
96		template <class _Tp, class _Up>
97		constexpr _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> gcd(_Tp __m, _Up __n) {
98		static_assert(is_integral<_Tp>::value && is_integral<_Up>::value, "Arguments to gcd must be integer types");
99		static_assert(!is_same<__remove_cv_t<_Tp>, bool>::value, "First argument to gcd cannot be bool");
100		static_assert(!is_same<__remove_cv_t<_Up>, bool>::value, "Second argument to gcd cannot be bool");
101		using _Rp = common_type_t<_Tp, _Up>;
102		using _Wp = make_unsigned_t<_Rp>;
103		return static_cast<_Rp>(
104		std::__gcd(static_cast<_Wp>(__ct_abs<_Rp, _Tp>()(__m)), static_cast<_Wp>(__ct_abs<_Rp, _Up>()(__n))));
	94	return static_cast<_Rp>(__a << __shift);
105	95	}
106	96
107	97	template <class _Tp, class _Up>
...	...	@@ -113,8 +103,8 @@ constexpr _LIBCPP_HIDE_FROM_ABI common_type_t<_Tp, _Up> lcm(_Tp __m, _Up __n) {
113	103	return 0;
114	104
115	105	using _Rp = common_type_t<_Tp, _Up>;
116		_Rp __val1 = __ct_abs<_Rp, _Tp>()(__m) / std::gcd(__m, __n);
117		_Rp __val2 = __ct_abs<_Rp, _Up>()(__n);
	106	_Rp __val1 = std::__abs_in_type<_Rp>(__m) / std::gcd(__m, __n);
	107	_Rp __val2 = std::__abs_in_type<_Rp>(__n);
118	108	_Rp __res;
119	109	[[maybe_unused]] bool __overflow = __builtin_mul_overflow(__val1, __val2, std::addressof(__res));
120	110	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(!__overflow, "Overflow in lcm");

lib/libcxx/include/__numeric/midpoint.h+9-14

...	...	@@ -12,16 +12,13 @@
12	12
13	13	#include <__config>
14	14	#include <__cstddef/ptrdiff_t.h>
15		#include <__type_traits/enable_if.h>
16	15	#include <__type_traits/is_floating_point.h>
17	16	#include <__type_traits/is_integral.h>
18		#include <__type_traits/is_null_pointer.h>
19	17	#include <__type_traits/is_object.h>
20		#include <__type_traits/is_pointer.h>
21	18	#include <__type_traits/is_same.h>
22	19	#include <__type_traits/is_void.h>
23	20	#include <__type_traits/make_unsigned.h>
24		#include <__type_traits/remove_pointer.h>
	21	#include <__type_traits/remove_cv.h>
25	22	#include <limits>
26	23
27	24	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -35,8 +32,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
35	32
36	33	#if _LIBCPP_STD_VER >= 20
37	34	template <class _Tp>
38		_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<is_integral_v<_Tp> && !is_same_v<bool, _Tp> && !is_null_pointer_v<_Tp>, _Tp>
39		midpoint(_Tp __a, _Tp __b) noexcept _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
	35	requires(is_integral_v<_Tp> && !is_same_v<remove_cv_t<_Tp>, bool>)
	36	[[nodiscard]]
	37	_LIBCPP_HIDE_FROM_ABI constexpr _Tp midpoint(_Tp __a, _Tp __b) noexcept _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
40	38	using _Up = make_unsigned_t<_Tp>;
41	39	constexpr _Up __bitshift = numeric_limits<_Up>::digits - 1;
42	40
...	...	@@ -48,23 +46,20 @@ midpoint(_Tp __a, _Tp __b) noexcept _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
48	46	return __a + __half_diff;
49	47	}
50	48
51		template <class _Tp, enable_if_t<is_object_v<_Tp> && !is_void_v<_Tp> && (sizeof(_Tp) > 0), int> = 0>
52		_LIBCPP_HIDE_FROM_ABI constexpr _Tp* midpoint(_Tp* __a, _Tp* __b) noexcept {
	49	template <class _Tp>
	50	requires(is_object_v<_Tp> && (sizeof(_Tp) > 0))
	51	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* midpoint(_Tp* __a, _Tp* __b) noexcept {
53	52	return __a + std::midpoint(ptrdiff_t(0), __b - __a);
54	53	}
55	54
56		template <typename _Tp>
57		_LIBCPP_HIDE_FROM_ABI constexpr int __sign(_Tp __val) {
58		return (_Tp(0) < __val) - (__val < _Tp(0));
59		}
60
61	55	template <typename _Fp>
62	56	_LIBCPP_HIDE_FROM_ABI constexpr _Fp __fp_abs(_Fp __f) {
63	57	return __f >= 0 ? __f : -__f;
64	58	}
65	59
66	60	template <class _Fp>
67		_LIBCPP_HIDE_FROM_ABI constexpr enable_if_t<is_floating_point_v<_Fp>, _Fp> midpoint(_Fp __a, _Fp __b) noexcept {
	61	requires(is_floating_point_v<_Fp>)
	62	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Fp midpoint(_Fp __a, _Fp __b) noexcept {
68	63	constexpr _Fp __lo = numeric_limits<_Fp>::min() * 2;
69	64	constexpr _Fp __hi = numeric_limits<_Fp>::max() / 2;
70	65

lib/libcxx/include/__numeric/pstl.h+2-2

...	...	@@ -70,7 +70,7 @@ template <class _ExecutionPolicy,
70	70	class _ForwardIterator,
71	71	class _RawPolicy = __remove_cvref_t<_ExecutionPolicy>,
72	72	enable_if_t<is_execution_policy_v<_RawPolicy>, int> = 0>
73		_LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator>
	73	_LIBCPP_HIDE_FROM_ABI __iterator_value_type<_ForwardIterator>
74	74	reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) {
75	75	_LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators");
76	76	using _Implementation = __pstl::__dispatch<__pstl::__reduce, __pstl::__current_configuration, _RawPolicy>;
...	...	@@ -78,7 +78,7 @@ reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator _
78	78	std::forward<_ExecutionPolicy>(__policy),
79	79	std::move(__first),
80	80	std::move(__last),
81		__iter_value_type<_ForwardIterator>(),
	81	__iterator_value_type<_ForwardIterator>(),
82	82	plus{});
83	83	}
84	84

lib/libcxx/include/__numeric/saturation_arithmetic.h+13-5

...	...	@@ -30,6 +30,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
30	30
31	31	template <__signed_or_unsigned_integer _Tp>
32	32	_LIBCPP_HIDE_FROM_ABI constexpr _Tp __add_sat(_Tp __x, _Tp __y) noexcept {
	33	# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 2101
	34	return __builtin_elementwise_add_sat(__x, __y);
	35	# else
33	36	if (_Tp __sum; !__builtin_add_overflow(__x, __y, std::addressof(__sum)))
34	37	return __sum;
35	38	// Handle overflow
...	...	@@ -44,10 +47,14 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __add_sat(_Tp __x, _Tp __y) noexcept {
44	47	// Overflows if (x < 0 && y < 0)
45	48	return std::numeric_limits<_Tp>::min();
46	49	}
	50	# endif
47	51	}
48	52
49	53	template <__signed_or_unsigned_integer _Tp>
50	54	_LIBCPP_HIDE_FROM_ABI constexpr _Tp __sub_sat(_Tp __x, _Tp __y) noexcept {
	55	# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 2101
	56	return __builtin_elementwise_sub_sat(__x, __y);
	57	# else
51	58	if (_Tp __sub; !__builtin_sub_overflow(__x, __y, std::addressof(__sub)))
52	59	return __sub;
53	60	// Handle overflow
...	...	@@ -63,6 +70,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Tp __sub_sat(_Tp __x, _Tp __y) noexcept {
63	70	// Overflows if (x < 0 && y > 0)
64	71	return std::numeric_limits<_Tp>::min();
65	72	}
	73	# endif
66	74	}
67	75
68	76	template <__signed_or_unsigned_integer _Tp>
...	...	@@ -113,27 +121,27 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Rp __saturate_cast(_Tp __x) noexcept {
113	121	#if _LIBCPP_STD_VER >= 26
114	122
115	123	template <__signed_or_unsigned_integer _Tp>
116		_LIBCPP_HIDE_FROM_ABI constexpr _Tp add_sat(_Tp __x, _Tp __y) noexcept {
	124	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp add_sat(_Tp __x, _Tp __y) noexcept {
117	125	return std::__add_sat(__x, __y);
118	126	}
119	127
120	128	template <__signed_or_unsigned_integer _Tp>
121		_LIBCPP_HIDE_FROM_ABI constexpr _Tp sub_sat(_Tp __x, _Tp __y) noexcept {
	129	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp sub_sat(_Tp __x, _Tp __y) noexcept {
122	130	return std::__sub_sat(__x, __y);
123	131	}
124	132
125	133	template <__signed_or_unsigned_integer _Tp>
126		_LIBCPP_HIDE_FROM_ABI constexpr _Tp mul_sat(_Tp __x, _Tp __y) noexcept {
	134	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp mul_sat(_Tp __x, _Tp __y) noexcept {
127	135	return std::__mul_sat(__x, __y);
128	136	}
129	137
130	138	template <__signed_or_unsigned_integer _Tp>
131		_LIBCPP_HIDE_FROM_ABI constexpr _Tp div_sat(_Tp __x, _Tp __y) noexcept {
	139	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp div_sat(_Tp __x, _Tp __y) noexcept {
132	140	return std::__div_sat(__x, __y);
133	141	}
134	142
135	143	template <__signed_or_unsigned_integer _Rp, __signed_or_unsigned_integer _Tp>
136		_LIBCPP_HIDE_FROM_ABI constexpr _Rp saturate_cast(_Tp __x) noexcept {
	144	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Rp saturate_cast(_Tp __x) noexcept {
137	145	return std::__saturate_cast<_Rp>(__x);
138	146	}
139	147

lib/libcxx/include/__ostream/basic_ostream.h+8-8

...	...	@@ -53,7 +53,7 @@ public:
53	53	typedef typename traits_type::off_type off_type;
54	54
55	55	// 27.7.2.2 Constructor/destructor:
56		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_ostream(basic_streambuf<char_type, traits_type>* __sb) {
	56	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 explicit basic_ostream(basic_streambuf<char_type, traits_type>* __sb) {
57	57	this->init(__sb);
58	58	}
59	59	~basic_ostream() override;
...	...	@@ -67,7 +67,7 @@ protected:
67	67	// 27.7.2.3 Assign/swap
68	68	inline _LIBCPP_HIDE_FROM_ABI basic_ostream& operator=(basic_ostream&& __rhs);
69	69
70		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_ostream& __rhs) {
	70	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void swap(basic_ostream& __rhs) {
71	71	basic_ios<char_type, traits_type>::swap(__rhs);
72	72	}
73	73
...	...	@@ -76,17 +76,17 @@ public:
76	76	class sentry;
77	77
78	78	// 27.7.2.6 Formatted output:
79		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(basic_ostream& (*__pf)(basic_ostream&)) {
	79	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_ostream& operator<<(basic_ostream& (*__pf)(basic_ostream&)) {
80	80	return __pf(*this);
81	81	}
82	82
83		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream&
	83	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_ostream&
84	84	operator<<(basic_ios<char_type, traits_type>& (*__pf)(basic_ios<char_type, traits_type>&)) {
85	85	__pf(*this);
86	86	return *this;
87	87	}
88	88
89		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& operator<<(ios_base& (*__pf)(ios_base&)) {
	89	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_ostream& operator<<(ios_base& (*__pf)(ios_base&)) {
90	90	__pf(*this);
91	91	return *this;
92	92	}
...	...	@@ -174,9 +174,9 @@ public:
174	174	basic_ostream& flush();
175	175
176	176	// 27.7.2.5 seeks:
177		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type tellp();
178		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(pos_type __pos);
179		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_ostream& seekp(off_type __off, ios_base::seekdir __dir);
	177	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 pos_type tellp();
	178	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_ostream& seekp(pos_type __pos);
	179	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_ostream& seekp(off_type __off, ios_base::seekdir __dir);
180	180
181	181	protected:
182	182	_LIBCPP_HIDE_FROM_ABI basic_ostream() {} // extension, intentially does not initialize

lib/libcxx/include/__pstl/backends/default.h+14-14

...	...	@@ -102,7 +102,7 @@ struct __find<__default_backend_tag, _ExecutionPolicy> {
102	102	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) const noexcept {
103	103	using _FindIf = __dispatch<__find_if, __current_configuration, _ExecutionPolicy>;
104	104	return _FindIf()(
105		__policy, std::move(__first), std::move(__last), [&](__iter_reference<_ForwardIterator> __element) {
	105	__policy, std::move(__first), std::move(__last), [&](__iterator_reference<_ForwardIterator> __element) {
106	106	return __element == __value;
107	107	});
108	108	}
...	...	@@ -137,7 +137,7 @@ struct __all_of<__default_backend_tag, _ExecutionPolicy> {
137	137	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<bool>
138	138	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred) const noexcept {
139	139	using _AnyOf = __dispatch<__any_of, __current_configuration, _ExecutionPolicy>;
140		auto __res = _AnyOf()(__policy, __first, __last, [&](__iter_reference<_ForwardIterator> __value) {
	140	auto __res = _AnyOf()(__policy, __first, __last, [&](__iterator_reference<_ForwardIterator> __value) {
141	141	return !__pred(__value);
142	142	});
143	143	if (!__res)
...	...	@@ -204,7 +204,7 @@ struct __fill<__default_backend_tag, _ExecutionPolicy> {
204	204	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
205	205	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept {
206	206	using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
207		using _Ref = __iter_reference<_ForwardIterator>;
	207	using _Ref = __iterator_reference<_ForwardIterator>;
208	208	return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __value; });
209	209	}
210	210	};
...	...	@@ -233,7 +233,7 @@ struct __replace<__default_backend_tag, _ExecutionPolicy> {
233	233	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __old, _Tp const& __new)
234	234	const noexcept {
235	235	using _ReplaceIf = __dispatch<__replace_if, __current_configuration, _ExecutionPolicy>;
236		using _Ref = __iter_reference<_ForwardIterator>;
	236	using _Ref = __iterator_reference<_ForwardIterator>;
237	237	return _ReplaceIf()(
238	238	__policy, std::move(__first), std::move(__last), [&](_Ref __element) { return __element == __old; }, __new);
239	239	}
...	...	@@ -246,7 +246,7 @@ struct __replace_if<__default_backend_tag, _ExecutionPolicy> {
246	246	_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred&& __pred, _Tp const& __new_value)
247	247	const noexcept {
248	248	using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
249		using _Ref = __iter_reference<_ForwardIterator>;
	249	using _Ref = __iterator_reference<_ForwardIterator>;
250	250	return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) {
251	251	if (__pred(__element))
252	252	__element = __new_value;
...	...	@@ -260,7 +260,7 @@ struct __generate<__default_backend_tag, _ExecutionPolicy> {
260	260	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
261	261	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator&& __gen) const noexcept {
262	262	using _ForEach = __dispatch<__for_each, __current_configuration, _ExecutionPolicy>;
263		using _Ref = __iter_reference<_ForwardIterator>;
	263	using _Ref = __iterator_reference<_ForwardIterator>;
264	264	return _ForEach()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) { __element = __gen(); });
265	265	}
266	266	};
...	...	@@ -271,7 +271,7 @@ struct __generate_n<__default_backend_tag, _ExecutionPolicy> {
271	271	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty>
272	272	operator()(_Policy&& __policy, _ForwardIterator __first, _Size __n, _Generator&& __gen) const noexcept {
273	273	using _ForEachN = __dispatch<__for_each_n, __current_configuration, _ExecutionPolicy>;
274		using _Ref = __iter_reference<_ForwardIterator>;
	274	using _Ref = __iterator_reference<_ForwardIterator>;
275	275	return _ForEachN()(__policy, std::move(__first), __n, [&](_Ref __element) { __element = __gen(); });
276	276	}
277	277	};
...	...	@@ -295,11 +295,11 @@ struct __sort<__default_backend_tag, _ExecutionPolicy> {
295	295	template <class _ExecutionPolicy>
296	296	struct __count_if<__default_backend_tag, _ExecutionPolicy> {
297	297	template <class _Policy, class _ForwardIterator, class _Predicate>
298		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>> operator()(
	298	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iterator_difference_type<_ForwardIterator>> operator()(
299	299	_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate&& __pred) const noexcept {
300	300	using _TransformReduce = __dispatch<__transform_reduce, __current_configuration, _ExecutionPolicy>;
301		using _DiffT = __iter_diff_t<_ForwardIterator>;
302		using _Ref = __iter_reference<_ForwardIterator>;
	301	using _DiffT = __iterator_difference_type<_ForwardIterator>;
	302	using _Ref = __iterator_reference<_ForwardIterator>;
303	303	return _TransformReduce()(
304	304	__policy, std::move(__first), std::move(__last), _DiffT{}, std::plus{}, [&](_Ref __element) -> _DiffT {
305	305	return __pred(__element) ? _DiffT(1) : _DiffT(0);
...	...	@@ -310,10 +310,10 @@ struct __count_if<__default_backend_tag, _ExecutionPolicy> {
310	310	template <class _ExecutionPolicy>
311	311	struct __count<__default_backend_tag, _ExecutionPolicy> {
312	312	template <class _Policy, class _ForwardIterator, class _Tp>
313		[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iter_diff_t<_ForwardIterator>>
	313	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__iterator_difference_type<_ForwardIterator>>
314	314	operator()(_Policy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Tp const& __value) const noexcept {
315	315	using _CountIf = __dispatch<__count_if, __current_configuration, _ExecutionPolicy>;
316		using _Ref = __iter_reference<_ForwardIterator>;
	316	using _Ref = __iterator_reference<_ForwardIterator>;
317	317	return _CountIf()(__policy, std::move(__first), std::move(__last), [&](_Ref __element) -> bool {
318	318	return __element == __value;
319	319	});
...	...	@@ -402,7 +402,7 @@ struct __replace_copy_if<__default_backend_tag, _ExecutionPolicy> {
402	402	_Pred&& __pred,
403	403	_Tp const& __new_value) const noexcept {
404	404	using _Transform = __dispatch<__transform, __current_configuration, _ExecutionPolicy>;
405		using _Ref = __iter_reference<_ForwardIterator>;
	405	using _Ref = __iterator_reference<_ForwardIterator>;
406	406	auto __res =
407	407	_Transform()(__policy, std::move(__first), std::move(__last), std::move(__out_it), [&](_Ref __element) {
408	408	return __pred(__element) ? __new_value : __element;
...	...	@@ -424,7 +424,7 @@ struct __replace_copy<__default_backend_tag, _ExecutionPolicy> {
424	424	_Tp const& __old_value,
425	425	_Tp const& __new_value) const noexcept {
426	426	using _ReplaceCopyIf = __dispatch<__replace_copy_if, __current_configuration, _ExecutionPolicy>;
427		using _Ref = __iter_reference<_ForwardIterator>;
	427	using _Ref = __iterator_reference<_ForwardIterator>;
428	428	return _ReplaceCopyIf()(
429	429	__policy,
430	430	std::move(__first),

lib/libcxx/include/__pstl/backends/libdispatch.h+2-2

...	...	@@ -269,7 +269,7 @@ struct __cpu_traits<__libdispatch_backend_tag> {
269	269	return __empty{};
270	270	}
271	271
272		using _Value = __iter_value_type<_RandomAccessIterator>;
	272	using _Value = __iterator_value_type<_RandomAccessIterator>;
273	273
274	274	auto __destroy = [__size](_Value* __ptr) {
275	275	std::destroy_n(__ptr, __size);
...	...	@@ -282,7 +282,7 @@ struct __cpu_traits<__libdispatch_backend_tag> {
282	282	// Initialize all elements to a moved-from state
283	283	// TODO: Don't do this - this can be done in the first merge - see https://llvm.org/PR63928
284	284	std::__construct_at(__values.get(), std::move(*__first));
285		for (__iter_diff_t<_RandomAccessIterator> __i = 1; __i != __size; ++__i) {
	285	for (__iterator_difference_type<_RandomAccessIterator> __i = 1; __i != __size; ++__i) {
286	286	std::__construct_at(__values.get() + __i, std::move(__values.get()[__i - 1]));
287	287	}
288	288	*__first = std::move(__values.get()[__size - 1]);

lib/libcxx/include/__pstl/cpu_algos/find_if.h+1-1

...	...	@@ -119,7 +119,7 @@ struct __cpu_parallel_find_if {
119	119	true);
120	120	} else if constexpr (__is_unsequenced_execution_policy_v<_RawExecutionPolicy> &&
121	121	__has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
122		using __diff_t = __iter_diff_t<_ForwardIterator>;
	122	using __diff_t = __iterator_difference_type<_ForwardIterator>;
123	123	return __pstl::__simd_first<_Backend>(
124	124	__first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
125	125	return __pred(__iter[__i]);

lib/libcxx/include/__pstl/cpu_algos/transform.h+5-6

...	...	@@ -84,9 +84,8 @@ struct __cpu_parallel_transform {
84	84	__first,
85	85	__last - __first,
86	86	__result,
87		[&](__iter_reference<_ForwardIterator> __in_value, __iter_reference<_ForwardOutIterator> __out_value) {
88		__out_value = __op(__in_value);
89		});
	87	[&](__iterator_reference<_ForwardIterator> __in_value,
	88	__iterator_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in_value); });
90	89	} else {
91	90	return std::transform(__first, __last, __result, __op);
92	91	}
...	...	@@ -138,9 +137,9 @@ struct __cpu_parallel_transform_binary {
138	137	__last1 - __first1,
139	138	__first2,
140	139	__result,
141		[&](__iter_reference<_ForwardIterator1> __in1,
142		__iter_reference<_ForwardIterator2> __in2,
143		__iter_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); });
	140	[&](__iterator_reference<_ForwardIterator1> __in1,
	141	__iterator_reference<_ForwardIterator2> __in2,
	142	__iterator_reference<_ForwardOutIterator> __out_value) { __out_value = __op(__in1, __in2); });
144	143	} else {
145	144	return std::transform(__first1, __last1, __first2, __result, __op);
146	145	}

lib/libcxx/include/__pstl/cpu_algos/transform_reduce.h+5-4

...	...	@@ -148,9 +148,10 @@ struct __cpu_parallel_transform_reduce_binary {
148	148	__has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
149	149	__has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
150	150	return __pstl::__simd_transform_reduce<_Backend>(
151		__last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) {
152		return __transform(__first1[__i], __first2[__i]);
153		});
	151	__last1 - __first1,
	152	std::move(__init),
	153	std::move(__reduce),
	154	[&](__iterator_difference_type<_ForwardIterator1> __i) { return __transform(__first1[__i], __first2[__i]); });
154	155	} else {
155	156	return std::transform_reduce(
156	157	std::move(__first1),
...	...	@@ -200,7 +201,7 @@ struct __cpu_parallel_transform_reduce {
200	201	__last - __first,
201	202	std::move(__init),
202	203	std::move(__reduce),
203		[=, &__transform](__iter_diff_t<_ForwardIterator> __i) { return __transform(__first[__i]); });
	204	[=, &__transform](__iterator_difference_type<_ForwardIterator> __i) { return __transform(__first[__i]); });
204	205	} else {
205	206	return std::transform_reduce(
206	207	std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform));

lib/libcxx/include/__random/binomial_distribution.h+10-4

...	...	@@ -97,13 +97,19 @@ public:
97	97	}
98	98	};
99	99
100		// The LLVM C library provides this with conflicting `noexcept` attributes.
101		#if !defined(_LIBCPP_MSVCRT_LIKE) && !defined(__LLVM_LIBC__)
102		extern "C" double lgamma_r(double, int*);
	100	// Some libc declares the math functions to be `noexcept`.
	101	#if _LIBCPP_GLIBC_PREREQ(2, 8) \|\| defined(__LLVM_LIBC__)
	102	# define _LIBCPP_LGAMMA_R_NOEXCEPT _NOEXCEPT
	103	#else
	104	# define _LIBCPP_LGAMMA_R_NOEXCEPT
	105	#endif
	106
	107	#if !defined(_LIBCPP_MSVCRT_LIKE)
	108	extern "C" double lgamma_r(double, int*) _LIBCPP_LGAMMA_R_NOEXCEPT;
103	109	#endif
104	110
105	111	inline _LIBCPP_HIDE_FROM_ABI double __libcpp_lgamma(double __d) {
106		#if defined(_LIBCPP_MSVCRT_LIKE) \|\| defined(__LLVM_LIBC__)
	112	#if defined(_LIBCPP_MSVCRT_LIKE)
107	113	return lgamma(__d);
108	114	#else
109	115	int __sign;

lib/libcxx/include/__random/mersenne_twister_engine.h+50-154

...	...	@@ -62,24 +62,6 @@ _LIBCPP_HIDE_FROM_ABI bool
62	62	operator==(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
63	63	const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
64	64
65		template <class _UInt,
66		size_t _Wp,
67		size_t _Np,
68		size_t _Mp,
69		size_t _Rp,
70		_UInt _Ap,
71		size_t _Up,
72		_UInt _Dp,
73		size_t _Sp,
74		_UInt _Bp,
75		size_t _Tp,
76		_UInt _Cp,
77		size_t _Lp,
78		_UInt _Fp>
79		_LIBCPP_HIDE_FROM_ABI bool
80		operator!=(const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
81		const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
82
83	65	template <class _CharT,
84	66	class _Traits,
85	67	class _UInt,
...	...	@@ -194,14 +176,31 @@ public:
194	176	_LIBCPP_HIDE_FROM_ABI explicit mersenne_twister_engine(_Sseq& __q) {
195	177	seed(__q);
196	178	}
197		_LIBCPP_HIDE_FROM_ABI void seed(result_type __sd = default_seed);
	179	_LIBCPP_HIDE_FROM_ABI void seed(result_type __sd = default_seed) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK {
	180	__x_[0] = __sd & _Max;
	181	for (size_t __i = 1; __i < __n; ++__i)
	182	__x_[__i] = (__f * (__x_[__i - 1] ^ __rshift<__w - 2>(__x_[__i - 1])) + __i) & _Max;
	183	__i_ = 0;
	184	}
198	185	template <class _Sseq, __enable_if_t<__is_seed_sequence<_Sseq, mersenne_twister_engine>::value, int> = 0>
199	186	_LIBCPP_HIDE_FROM_ABI void seed(_Sseq& __q) {
200	187	__seed(__q, integral_constant<unsigned, 1 + (__w - 1) / 32>());
201	188	}
202	189
203	190	// generating functions
204		_LIBCPP_HIDE_FROM_ABI result_type operator()();
	191	_LIBCPP_HIDE_FROM_ABI result_type operator()() {
	192	const size_t __j = (__i_ + 1) % __n;
	193	const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
	194	const result_type __yp = (__x_[__i_] & ~__mask) \| (__x_[__j] & __mask);
	195	const size_t __k = (__i_ + __m) % __n;
	196	__x_[__i_] = __x_[__k] ^ __rshift<1>(__yp) ^ (__a * (__yp & 1));
	197	result_type __z = __x_[__i_] ^ (__rshift<__u>(__x_[__i_]) & __d);
	198	__i_ = __j;
	199	__z ^= __lshift<__s>(__z) & __b;
	200	__z ^= __lshift<__t>(__z) & __c;
	201	return __z ^ __rshift<__l>(__z);
	202	}
	203
205	204	_LIBCPP_HIDE_FROM_ABI void discard(unsigned long long __z) {
206	205	for (; __z; --__z)
207	206	operator()();
...	...	@@ -225,24 +224,6 @@ public:
225	224	const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
226	225	const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
227	226
228		template <class _UInt,
229		size_t _Wp,
230		size_t _Np,
231		size_t _Mp,
232		size_t _Rp,
233		_UInt _Ap,
234		size_t _Up,
235		_UInt _Dp,
236		size_t _Sp,
237		_UInt _Bp,
238		size_t _Tp,
239		_UInt _Cp,
240		size_t _Lp,
241		_UInt _Fp>
242		friend bool operator!=(
243		const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __x,
244		const mersenne_twister_engine<_UInt, _Wp, _Np, _Mp, _Rp, _Ap, _Up, _Dp, _Sp, _Bp, _Tp, _Cp, _Lp, _Fp>& __y);
245
246	227	template <class _CharT,
247	228	class _Traits,
248	229	class _UInt,
...	...	@@ -285,9 +266,38 @@ public:
285	266
286	267	private:
287	268	template <class _Sseq>
288		_LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 1>);
	269	_LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 1>) {
	270	const unsigned __k = 1;
	271	uint32_t __ar[__n * __k];
	272	__q.generate(__ar, __ar + __n * __k);
	273	for (size_t __i = 0; __i < __n; ++__i)
	274	__x_[__i] = static_cast<result_type>(__ar[__i] & _Max);
	275	const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
	276	__i_ = 0;
	277	if ((__x_[0] & ~__mask) == 0) {
	278	for (size_t __i = 1; __i < __n; ++__i)
	279	if (__x_[__i] != 0)
	280	return;
	281	__x_[0] = result_type(1) << (__w - 1);
	282	}
	283	}
	284
289	285	template <class _Sseq>
290		_LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 2>);
	286	_LIBCPP_HIDE_FROM_ABI void __seed(_Sseq& __q, integral_constant<unsigned, 2>) {
	287	const unsigned __k = 2;
	288	uint32_t __ar[__n * __k];
	289	__q.generate(__ar, __ar + __n * __k);
	290	for (size_t __i = 0; __i < __n; ++__i)
	291	__x_[__i] = static_cast<result_type>((__ar[2 * __i] + ((uint64_t)__ar[2 * __i + 1] << 32)) & _Max);
	292	const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
	293	__i_ = 0;
	294	if ((__x_[0] & ~__mask) == 0) {
	295	for (size_t __i = 1; __i < __n; ++__i)
	296	if (__x_[__i] != 0)
	297	return;
	298	__x_[0] = result_type(1) << (__w - 1);
	299	}
	300	}
291	301
292	302	template <size_t __count,
293	303	__enable_if_t<__count< __w, int> = 0> _LIBCPP_HIDE_FROM_ABI static result_type __lshift(result_type __x) {
...	...	@@ -310,120 +320,6 @@ private:
310	320	}
311	321	};
312	322
313		template <class _UIntType,
314		size_t __w,
315		size_t __n,
316		size_t __m,
317		size_t __r,
318		_UIntType __a,
319		size_t __u,
320		_UIntType __d,
321		size_t __s,
322		_UIntType __b,
323		size_t __t,
324		_UIntType __c,
325		size_t __l,
326		_UIntType __f>
327		void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::seed(
328		result_type __sd) _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK { // __w >= 2
329		__x_[0] = __sd & _Max;
330		for (size_t __i = 1; __i < __n; ++__i)
331		__x_[__i] = (__f * (__x_[__i - 1] ^ __rshift<__w - 2>(__x_[__i - 1])) + __i) & _Max;
332		__i_ = 0;
333		}
334
335		template <class _UIntType,
336		size_t __w,
337		size_t __n,
338		size_t __m,
339		size_t __r,
340		_UIntType __a,
341		size_t __u,
342		_UIntType __d,
343		size_t __s,
344		_UIntType __b,
345		size_t __t,
346		_UIntType __c,
347		size_t __l,
348		_UIntType __f>
349		template <class _Sseq>
350		void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::__seed(
351		_Sseq& __q, integral_constant<unsigned, 1>) {
352		const unsigned __k = 1;
353		uint32_t __ar[__n * __k];
354		__q.generate(__ar, __ar + __n * __k);
355		for (size_t __i = 0; __i < __n; ++__i)
356		__x_[__i] = static_cast<result_type>(__ar[__i] & _Max);
357		const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
358		__i_ = 0;
359		if ((__x_[0] & ~__mask) == 0) {
360		for (size_t __i = 1; __i < __n; ++__i)
361		if (__x_[__i] != 0)
362		return;
363		__x_[0] = result_type(1) << (__w - 1);
364		}
365		}
366
367		template <class _UIntType,
368		size_t __w,
369		size_t __n,
370		size_t __m,
371		size_t __r,
372		_UIntType __a,
373		size_t __u,
374		_UIntType __d,
375		size_t __s,
376		_UIntType __b,
377		size_t __t,
378		_UIntType __c,
379		size_t __l,
380		_UIntType __f>
381		template <class _Sseq>
382		void mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::__seed(
383		_Sseq& __q, integral_constant<unsigned, 2>) {
384		const unsigned __k = 2;
385		uint32_t __ar[__n * __k];
386		__q.generate(__ar, __ar + __n * __k);
387		for (size_t __i = 0; __i < __n; ++__i)
388		__x_[__i] = static_cast<result_type>((__ar[2 * __i] + ((uint64_t)__ar[2 * __i + 1] << 32)) & _Max);
389		const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
390		__i_ = 0;
391		if ((__x_[0] & ~__mask) == 0) {
392		for (size_t __i = 1; __i < __n; ++__i)
393		if (__x_[__i] != 0)
394		return;
395		__x_[0] = result_type(1) << (__w - 1);
396		}
397		}
398
399		template <class _UIntType,
400		size_t __w,
401		size_t __n,
402		size_t __m,
403		size_t __r,
404		_UIntType __a,
405		size_t __u,
406		_UIntType __d,
407		size_t __s,
408		_UIntType __b,
409		size_t __t,
410		_UIntType __c,
411		size_t __l,
412		_UIntType __f>
413		_UIntType
414		mersenne_twister_engine<_UIntType, __w, __n, __m, __r, __a, __u, __d, __s, __b, __t, __c, __l, __f>::operator()() {
415		const size_t __j = (__i_ + 1) % __n;
416		const result_type __mask = __r == _Dt ? result_type(~0) : (result_type(1) << __r) - result_type(1);
417		const result_type __yp = (__x_[__i_] & ~__mask) \| (__x_[__j] & __mask);
418		const size_t __k = (__i_ + __m) % __n;
419		__x_[__i_] = __x_[__k] ^ __rshift<1>(__yp) ^ (__a * (__yp & 1));
420		result_type __z = __x_[__i_] ^ (__rshift<__u>(__x_[__i_]) & __d);
421		__i_ = __j;
422		__z ^= __lshift<__s>(__z) & __b;
423		__z ^= __lshift<__t>(__z) & __c;
424		return __z ^ __rshift<__l>(__z);
425		}
426
427	323	template <class _UInt,
428	324	size_t _Wp,
429	325	size_t _Np,

lib/libcxx/include/__random/piecewise_constant_distribution.h+3-2

...	...	@@ -9,9 +9,11 @@
9	9	#ifndef _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
10	10	#define _LIBCPP___RANDOM_PIECEWISE_CONSTANT_DISTRIBUTION_H
11	11
	12	#include <__algorithm/copy_n.h>
12	13	#include <__algorithm/upper_bound.h>
13	14	#include <__config>
14	15	#include <__cstddef/ptrdiff_t.h>
	16	#include <__iterator/back_insert_iterator.h>
15	17	#include <__random/is_valid.h>
16	18	#include <__random/uniform_real_distribution.h>
17	19	#include <__vector/vector.h>
...	...	@@ -190,8 +192,7 @@ piecewise_constant_distribution<_RealType>::param_type::param_type(
190	192	__areas_.assign(1, 0.0);
191	193	} else {
192	194	__densities_.reserve(__b_.size() - 1);
193		for (size_t __i = 0; __i < __b_.size() - 1; ++__i, ++__f_w)
194		__densities_.push_back(*__f_w);
	195	std::copy_n(__f_w, __b_.size() - 1, std::back_inserter(__densities_));
195	196	__init();
196	197	}
197	198	}

lib/libcxx/include/__random/piecewise_linear_distribution.h+3-2

...	...	@@ -9,9 +9,11 @@
9	9	#ifndef _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
10	10	#define _LIBCPP___RANDOM_PIECEWISE_LINEAR_DISTRIBUTION_H
11	11
	12	#include <__algorithm/copy_n.h>
12	13	#include <__algorithm/upper_bound.h>
13	14	#include <__config>
14	15	#include <__cstddef/ptrdiff_t.h>
	16	#include <__iterator/back_insert_iterator.h>
15	17	#include <__random/is_valid.h>
16	18	#include <__random/uniform_real_distribution.h>
17	19	#include <__vector/comparison.h>
...	...	@@ -194,8 +196,7 @@ piecewise_linear_distribution<_RealType>::param_type::param_type(
194	196	__areas_.assign(1, 0.0);
195	197	} else {
196	198	__densities_.reserve(__b_.size());
197		for (size_t __i = 0; __i < __b_.size(); ++__i, ++__f_w)
198		__densities_.push_back(*__f_w);
	199	std::copy_n(__f_w, __b_.size(), std::back_inserter(__densities_));
199	200	__init();
200	201	}
201	202	}

lib/libcxx/include/__ranges/adjacent_transform_view.h created+406

...	...	@@ -0,0 +1,406 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___RANGES_ADJACENT_TRANSFORM_VIEW_H
	11	#define _LIBCPP___RANGES_ADJACENT_TRANSFORM_VIEW_H
	12
	13	#include <__config>
	14
	15	#include <__algorithm/min.h>
	16	#include <__compare/three_way_comparable.h>
	17	#include <__concepts/constructible.h>
	18	#include <__concepts/convertible_to.h>
	19	#include <__concepts/derived_from.h>
	20	#include <__concepts/equality_comparable.h>
	21	#include <__concepts/invocable.h>
	22	#include <__cstddef/size_t.h>
	23	#include <__functional/bind_back.h>
	24	#include <__functional/invoke.h>
	25	#include <__functional/operations.h>
	26	#include <__iterator/concepts.h>
	27	#include <__iterator/incrementable_traits.h>
	28	#include <__iterator/iter_move.h>
	29	#include <__iterator/iter_swap.h>
	30	#include <__iterator/iterator_traits.h>
	31	#include <__iterator/next.h>
	32	#include <__iterator/prev.h>
	33	#include <__memory/addressof.h>
	34	#include <__ranges/access.h>
	35	#include <__ranges/adjacent_view.h>
	36	#include <__ranges/all.h>
	37	#include <__ranges/concepts.h>
	38	#include <__ranges/empty_view.h>
	39	#include <__ranges/movable_box.h>
	40	#include <__ranges/range_adaptor.h>
	41	#include <__ranges/size.h>
	42	#include <__ranges/view_interface.h>
	43	#include <__ranges/zip_transform_view.h>
	44	#include <__type_traits/common_type.h>
	45	#include <__type_traits/decay.h>
	46	#include <__type_traits/is_nothrow_constructible.h>
	47	#include <__type_traits/is_object.h>
	48	#include <__type_traits/is_referenceable.h>
	49	#include <__type_traits/make_unsigned.h>
	50	#include <__type_traits/maybe_const.h>
	51	#include <__utility/declval.h>
	52	#include <__utility/forward.h>
	53	#include <__utility/in_place.h>
	54	#include <__utility/integer_sequence.h>
	55	#include <__utility/move.h>
	56
	57	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	58	# pragma GCC system_header
	59	#endif
	60
	61	_LIBCPP_PUSH_MACROS
	62	#include <__undef_macros>
	63
	64	_LIBCPP_BEGIN_NAMESPACE_STD
	65
	66	#if _LIBCPP_STD_VER >= 23
	67
	68	namespace ranges {
	69
	70	template <class _Fn, size_t _Np>
	71	struct __apply_n {
	72	template <class _Tp, size_t... _Is>
	73	static auto __apply(index_sequence<_Is...>) -> invoke_result_t<_Fn, decltype((void)_Is, std::declval<_Tp>())...>;
	74
	75	template <class _Tp>
	76	static auto operator()(_Tp&&) -> decltype(__apply<_Tp>(make_index_sequence<_Np>{}));
	77	};
	78
	79	template <forward_range _View, move_constructible _Fn, size_t _Np>
	80	requires view<_View> && (_Np > 0) && is_object_v<_Fn> &&
	81	regular_invocable<__apply_n<_Fn&, _Np>, range_reference_t<_View>> &&
	82	__referenceable<invoke_result_t<__apply_n<_Fn&, _Np>, range_reference_t<_View>>>
	83	class adjacent_transform_view : public view_interface<adjacent_transform_view<_View, _Fn, _Np>> {
	84	private:
	85	_LIBCPP_NO_UNIQUE_ADDRESS adjacent_view<_View, _Np> __inner_;
	86	_LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Fn> __fun_;
	87
	88	using _InnerView _LIBCPP_NODEBUG = adjacent_view<_View, _Np>;
	89
	90	template <bool _Const>
	91	using __inner_iterator _LIBCPP_NODEBUG = iterator_t<__maybe_const<_Const, _InnerView>>;
	92
	93	template <bool _Const>
	94	using __inner_sentinel _LIBCPP_NODEBUG = sentinel_t<__maybe_const<_Const, _InnerView>>;
	95
	96	template <bool>
	97	class __iterator;
	98
	99	template <bool>
	100	class __sentinel;
	101
	102	public:
	103	_LIBCPP_HIDE_FROM_ABI adjacent_transform_view() = default;
	104
	105	_LIBCPP_HIDE_FROM_ABI constexpr explicit adjacent_transform_view(_View __base, _Fn __fun)
	106	: __inner_(std::move(__base)), __fun_(std::in_place, std::move(__fun)) {}
	107
	108	_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	109	requires copy_constructible<_View>
	110	{
	111	return __inner_.base();
	112	}
	113	_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__inner_).base(); }
	114
	115	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() { return __iterator<false>(*this, __inner_.begin()); }
	116
	117	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	118	requires range<const _InnerView> && regular_invocable<__apply_n<const _Fn&, _Np>, range_reference_t<const _View>>
	119	{
	120	return __iterator<true>(*this, __inner_.begin());
	121	}
	122
	123	_LIBCPP_HIDE_FROM_ABI constexpr auto end() {
	124	if constexpr (common_range<_InnerView>) {
	125	return __iterator<false>(*this, __inner_.end());
	126	} else {
	127	return __sentinel<false>(__inner_.end());
	128	}
	129	}
	130
	131	_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	132	requires range<const _InnerView> && regular_invocable<__apply_n<const _Fn&, _Np>, range_reference_t<const _View>>
	133	{
	134	if constexpr (common_range<const _InnerView>) {
	135	return __iterator<true>(*this, __inner_.end());
	136	} else {
	137	return __sentinel<true>(__inner_.end());
	138	}
	139	}
	140
	141	_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	142	requires sized_range<_InnerView>
	143	{
	144	return __inner_.size();
	145	}
	146
	147	_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	148	requires sized_range<const _InnerView>
	149	{
	150	return __inner_.size();
	151	}
	152	};
	153
	154	template <forward_range _View, move_constructible _Fn, size_t _Np>
	155	requires view<_View> && (_Np > 0) && is_object_v<_Fn> &&
	156	regular_invocable<__apply_n<_Fn&, _Np>, range_reference_t<_View>> &&
	157	__referenceable<invoke_result_t<__apply_n<_Fn&, _Np>, range_reference_t<_View>>>
	158	template <bool _Const>
	159	class adjacent_transform_view<_View, _Fn, _Np>::__iterator {
	160	friend adjacent_transform_view;
	161
	162	using _Parent _LIBCPP_NODEBUG = __maybe_const<_Const, adjacent_transform_view>;
	163	using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _View>;
	164
	165	_Parent* __parent_ = nullptr;
	166	__inner_iterator<_Const> __inner_;
	167
	168	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(_Parent& __parent, __inner_iterator<_Const> __inner)
	169	: __parent_(std::addressof(__parent)), __inner_(std::move(__inner)) {}
	170
	171	static consteval auto __get_iterator_category() {
	172	using _Cat = iterator_traits<iterator_t<_Base>>::iterator_category;
	173	if constexpr (!is_reference_v<
	174	invoke_result_t<__apply_n<__maybe_const<_Const, _Fn>&, _Np>, range_reference_t<_Base>>>)
	175	return input_iterator_tag{};
	176	else if constexpr (derived_from<_Cat, random_access_iterator_tag>)
	177	return random_access_iterator_tag{};
	178	else if constexpr (derived_from<_Cat, bidirectional_iterator_tag>)
	179	return bidirectional_iterator_tag{};
	180	else if constexpr (derived_from<_Cat, forward_iterator_tag>)
	181	return forward_iterator_tag{};
	182	else
	183	return input_iterator_tag{};
	184	}
	185
	186	template <size_t... _Is>
	187	static consteval bool __noexcept_dereference(index_sequence<_Is...>) {
	188	return noexcept(std::invoke(
	189	std::declval<__maybe_const<_Const, _Fn>&>(), ((void)_Is, *std::declval<iterator_t<_Base> const&>())...));
	190	}
	191
	192	public:
	193	using iterator_category = decltype(__get_iterator_category());
	194	using iterator_concept = typename __inner_iterator<_Const>::iterator_concept;
	195	using value_type =
	196	remove_cvref_t<invoke_result_t<__apply_n<__maybe_const<_Const, _Fn>&, _Np>, range_reference_t<_Base>>>;
	197	using difference_type = range_difference_t<_Base>;
	198
	199	_LIBCPP_HIDE_FROM_ABI __iterator() = default;
	200
	201	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
	202	requires _Const && convertible_to<__inner_iterator<false>, __inner_iterator<true>>
	203	: __parent_(__i.__parent_), __inner_(std::move(__i.__inner_)) {}
	204
	205	_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const
	206	noexcept(__noexcept_dereference(make_index_sequence<_Np>{})) {
	207	return std::apply(
	208	[&](const auto&... __iters) -> decltype(auto) { return std::invoke(__parent_->__fun_, __iters...); },
	209	__adjacent_view_iter_access::__get_current(__inner_));
	210	}
	211
	212	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
	213	++__inner_;
	214	return *this;
	215	}
	216
	217	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) {
	218	auto __tmp = *this;
	219	++*this;
	220	return __tmp;
	221	}
	222
	223	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
	224	requires bidirectional_range<_Base>
	225	{
	226	--__inner_;
	227	return *this;
	228	}
	229
	230	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
	231	requires bidirectional_range<_Base>
	232	{
	233	auto __tmp = *this;
	234	--*this;
	235	return __tmp;
	236	}
	237
	238	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __x)
	239	requires random_access_range<_Base>
	240	{
	241	__inner_ += __x;
	242	return *this;
	243	}
	244
	245	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __x)
	246	requires random_access_range<_Base>
	247	{
	248	__inner_ -= __x;
	249	return *this;
	250	}
	251
	252	_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
	253	requires random_access_range<_Base>
	254	{
	255	return std::apply(
	256	[&](const auto&... __iters) -> decltype(auto) { return std::invoke(*__parent_->__fun_, __iters[__n]...); },
	257	__adjacent_view_iter_access::__get_current(__inner_));
	258	}
	259
	260	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
	261	return __x.__inner_ == __y.__inner_;
	262	}
	263
	264	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
	265	requires random_access_range<_Base>
	266	{
	267	return __x.__inner_ < __y.__inner_;
	268	}
	269
	270	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
	271	requires random_access_range<_Base>
	272	{
	273	return __x.__inner_ > __y.__inner_;
	274	}
	275
	276	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
	277	requires random_access_range<_Base>
	278	{
	279	return __x.__inner_ <= __y.__inner_;
	280	}
	281
	282	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
	283	requires random_access_range<_Base>
	284	{
	285	return __x.__inner_ >= __y.__inner_;
	286	}
	287
	288	_LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
	289	requires random_access_range<_Base> && three_way_comparable<__inner_iterator<_Const>>
	290	{
	291	return __x.__inner_ <=> __y.__inner_;
	292	}
	293
	294	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __i, difference_type __n)
	295	requires random_access_range<_Base>
	296	{
	297	return __iterator(*__i.__parent_, __i.__inner_ + __n);
	298	}
	299
	300	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, const __iterator& __i)
	301	requires random_access_range<_Base>
	302	{
	303	return __iterator(*__i.__parent_, __i.__inner_ + __n);
	304	}
	305
	306	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __i, difference_type __n)
	307	requires random_access_range<_Base>
	308	{
	309	return __iterator(*__i.__parent_, __i.__inner_ - __n);
	310	}
	311
	312	_LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
	313	requires sized_sentinel_for<__inner_iterator<_Const>, __inner_iterator<_Const>>
	314	{
	315	return __x.__inner_ - __y.__inner_;
	316	}
	317	};
	318
	319	template <forward_range _View, move_constructible _Fn, size_t _Np>
	320	requires view<_View> && (_Np > 0) && is_object_v<_Fn> &&
	321	regular_invocable<__apply_n<_Fn&, _Np>, range_reference_t<_View>> &&
	322	__referenceable<invoke_result_t<__apply_n<_Fn&, _Np>, range_reference_t<_View>>>
	323	template <bool _Const>
	324	class adjacent_transform_view<_View, _Fn, _Np>::__sentinel {
	325	friend adjacent_transform_view;
	326
	327	__inner_sentinel<_Const> __inner_;
	328
	329	_LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(__inner_sentinel<_Const> __inner)
	330	: __inner_(std::move(__inner)) {}
	331
	332	public:
	333	_LIBCPP_HIDE_FROM_ABI __sentinel() = default;
	334
	335	_LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __i)
	336	requires _Const && convertible_to<__inner_sentinel<false>, __inner_sentinel<_Const>>
	337	: __inner_(std::move(__i.__inner_)) {}
	338
	339	template <bool _OtherConst>
	340	requires sentinel_for<__inner_sentinel<_Const>, __inner_iterator<_OtherConst>>
	341	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	342	return __x.__inner_ == __y.__inner_;
	343	}
	344
	345	template <bool _OtherConst>
	346	requires sized_sentinel_for<__inner_sentinel<_Const>, __inner_iterator<_OtherConst>>
	347	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _InnerView>>
	348	operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	349	return __x.__inner_ - __y.__inner_;
	350	}
	351
	352	template <bool _OtherConst>
	353	requires sized_sentinel_for<__inner_sentinel<_Const>, __inner_iterator<_OtherConst>>
	354	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _InnerView>>
	355	operator-(const __sentinel& __x, const __iterator<_OtherConst>& __y) {
	356	return __x.__inner_ - __y.__inner_;
	357	}
	358	};
	359
	360	namespace views {
	361	namespace __adjacent_transform {
	362
	363	template <size_t _Np>
	364	struct __fn : __range_adaptor_closure<__fn<_Np>> {
	365	template <class _Range, class _Fn>
	366	requires(_Np == 0 && forward_range<_Range &&>)
	367	_LIBCPP_HIDE_FROM_ABI static constexpr auto
	368	operator()(_Range&&, _Fn&& __fn) noexcept(noexcept(views::zip_transform(std::forward<_Fn>(__fn))))
	369	-> decltype(views::zip_transform(std::forward<_Fn>(__fn))) {
	370	return views::zip_transform(std::forward<_Fn>(__fn));
	371	}
	372
	373	template <class _Range, class _Fn>
	374	_LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Range&& __range, _Fn&& __fn) noexcept(
	375	noexcept(adjacent_transform_view<views::all_t<_Range&&>, decay_t<_Fn>, _Np>(
	376	std::forward<_Range>(__range), std::forward<_Fn>(__fn))))
	377	-> decltype(adjacent_transform_view<views::all_t<_Range&&>, decay_t<_Fn>, _Np>(
	378	std::forward<_Range>(__range), std::forward<_Fn>(__fn))) {
	379	return adjacent_transform_view<views::all_t<_Range&&>, decay_t<_Fn>, _Np>(
	380	std::forward<_Range>(__range), std::forward<_Fn>(__fn));
	381	}
	382
	383	template <class _Fn>
	384	requires constructible_from<decay_t<_Fn>, _Fn>
	385	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __f) const
	386	noexcept(is_nothrow_constructible_v<decay_t<_Fn>, _Fn>) {
	387	return __pipeable(std::__bind_back(*this, std::forward<_Fn>(__f)));
	388	}
	389	};
	390
	391	} // namespace __adjacent_transform
	392	inline namespace __cpo {
	393	template <size_t _Np>
	394	inline constexpr auto adjacent_transform = __adjacent_transform::__fn<_Np>{};
	395	inline constexpr auto pairwise_transform = adjacent_transform<2>;
	396	} // namespace __cpo
	397	} // namespace views
	398	} // namespace ranges
	399
	400	#endif // _LIBCPP_STD_VER >= 23
	401
	402	_LIBCPP_END_NAMESPACE_STD
	403
	404	_LIBCPP_POP_MACROS
	405
	406	#endif // _LIBCPP___RANGES_ADJACENT_TRANSFORM_VIEW_H

lib/libcxx/include/__ranges/adjacent_view.h created+419

...	...	@@ -0,0 +1,419 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___RANGES_ADJACENT_VIEW_H
	11	#define _LIBCPP___RANGES_ADJACENT_VIEW_H
	12
	13	#include <__config>
	14
	15	#include <__algorithm/min.h>
	16	#include <__compare/three_way_comparable.h>
	17	#include <__concepts/constructible.h>
	18	#include <__concepts/convertible_to.h>
	19	#include <__concepts/equality_comparable.h>
	20	#include <__cstddef/size_t.h>
	21	#include <__functional/invoke.h>
	22	#include <__functional/operations.h>
	23	#include <__iterator/concepts.h>
	24	#include <__iterator/incrementable_traits.h>
	25	#include <__iterator/iter_move.h>
	26	#include <__iterator/iter_swap.h>
	27	#include <__iterator/iterator_traits.h>
	28	#include <__iterator/next.h>
	29	#include <__iterator/prev.h>
	30	#include <__ranges/access.h>
	31	#include <__ranges/all.h>
	32	#include <__ranges/concepts.h>
	33	#include <__ranges/empty_view.h>
	34	#include <__ranges/enable_borrowed_range.h>
	35	#include <__ranges/range_adaptor.h>
	36	#include <__ranges/size.h>
	37	#include <__ranges/view_interface.h>
	38	#include <__tuple/tuple_transform.h>
	39	#include <__type_traits/common_type.h>
	40	#include <__type_traits/is_nothrow_constructible.h>
	41	#include <__type_traits/make_unsigned.h>
	42	#include <__type_traits/maybe_const.h>
	43	#include <__utility/declval.h>
	44	#include <__utility/forward.h>
	45	#include <__utility/integer_sequence.h>
	46	#include <__utility/move.h>
	47	#include <array>
	48	#include <tuple>
	49
	50	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	51	# pragma GCC system_header
	52	#endif
	53
	54	_LIBCPP_PUSH_MACROS
	55	#include <__undef_macros>
	56
	57	_LIBCPP_BEGIN_NAMESPACE_STD
	58
	59	#if _LIBCPP_STD_VER >= 23
	60
	61	namespace ranges {
	62
	63	template <forward_range _View, size_t _Np>
	64	requires view<_View> && (_Np > 0)
	65	class adjacent_view : public view_interface<adjacent_view<_View, _Np>> {
	66	private:
	67	_LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();
	68
	69	template <bool>
	70	class __iterator;
	71
	72	template <bool>
	73	class __sentinel;
	74
	75	struct __as_sentinel {};
	76
	77	public:
	78	_LIBCPP_HIDE_FROM_ABI adjacent_view()
	79	requires default_initializable<_View>
	80	= default;
	81
	82	_LIBCPP_HIDE_FROM_ABI constexpr explicit adjacent_view(_View __base) : __base_(std::move(__base)) {}
	83
	84	_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	85	requires copy_constructible<_View>
	86	{
	87	return __base_;
	88	}
	89	_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	90
	91	_LIBCPP_HIDE_FROM_ABI constexpr auto begin()
	92	requires(!__simple_view<_View>)
	93	{
	94	return __iterator<false>(ranges::begin(__base_), ranges::end(__base_));
	95	}
	96
	97	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	98	requires range<const _View> // LWG4482 This is under-constrained.
	99	{
	100	return __iterator<true>(ranges::begin(__base_), ranges::end(__base_));
	101	}
	102
	103	_LIBCPP_HIDE_FROM_ABI constexpr auto end()
	104	requires(!__simple_view<_View>)
	105	{
	106	if constexpr (common_range<_View>) {
	107	return __iterator<false>(__as_sentinel{}, ranges::begin(__base_), ranges::end(__base_));
	108	} else {
	109	return __sentinel<false>(ranges::end(__base_));
	110	}
	111	}
	112
	113	_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	114	requires range<const _View> // LWG4482 This is under-constrained.
	115	{
	116	if constexpr (common_range<const _View>) {
	117	return __iterator<true>(__as_sentinel{}, ranges::begin(__base_), ranges::end(__base_));
	118	} else {
	119	return __sentinel<true>(ranges::end(__base_));
	120	}
	121	}
	122
	123	_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	124	requires sized_range<_View>
	125	{
	126	using _ST = decltype(ranges::size(__base_));
	127	using _CT = common_type_t<_ST, size_t>;
	128	auto __sz = static_cast<_CT>(ranges::size(__base_));
	129	__sz -= std::min<_CT>(__sz, _Np - 1);
	130	return static_cast<_ST>(__sz);
	131	}
	132
	133	_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	134	requires sized_range<const _View>
	135	{
	136	using _ST = decltype(ranges::size(__base_));
	137	using _CT = common_type_t<_ST, size_t>;
	138	auto __sz = static_cast<_CT>(ranges::size(__base_));
	139	__sz -= std::min<_CT>(__sz, _Np - 1);
	140	return static_cast<_ST>(__sz);
	141	}
	142	};
	143
	144	struct __adjacent_view_iter_access {
	145	template <class _Iter>
	146	_LIBCPP_HIDE_FROM_ABI constexpr static auto& __get_current(_Iter& __it) noexcept {
	147	return __it.__current_;
	148	}
	149	};
	150
	151	template <forward_range _View, size_t _Np>
	152	requires view<_View> && (_Np > 0)
	153	template <bool _Const>
	154	class adjacent_view<_View, _Np>::__iterator {
	155	friend __adjacent_view_iter_access;
	156	friend adjacent_view;
	157	using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _View>;
	158	array<iterator_t<_Base>, _Np> __current_ = array<iterator_t<_Base>, _Np>();
	159
	160	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(iterator_t<_Base> __first, sentinel_t<_Base> __last) {
	161	__current_[0] = __first;
	162	for (size_t __i = 1; __i < _Np; ++__i) {
	163	__current_[__i] = ranges::next(__current_[__i - 1], 1, __last);
	164	}
	165	}
	166
	167	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(__as_sentinel, iterator_t<_Base> __first, iterator_t<_Base> __last) {
	168	if constexpr (!bidirectional_range<_Base>) {
	169	__current_.fill(__last);
	170	} else {
	171	__current_[_Np - 1] = __last;
	172	for (int __i = static_cast<int>(_Np) - 2; __i >= 0; --__i) {
	173	__current_[__i] = ranges::prev(__current_[__i + 1], 1, __first);
	174	}
	175	}
	176	}
	177
	178	template <class _Iter, size_t... _Is>
	179	_LIBCPP_HIDE_FROM_ABI explicit constexpr __iterator(_Iter&& __i, index_sequence<_Is...>)
	180	: __current_{std::move(__i.__current_[_Is])...} {}
	181
	182	static consteval auto __get_iterator_concept() {
	183	if constexpr (random_access_range<_Base>)
	184	return random_access_iterator_tag{};
	185	else if constexpr (bidirectional_range<_Base>)
	186	return bidirectional_iterator_tag{};
	187	else
	188	return forward_iterator_tag{};
	189	}
	190
	191	template <class _Tp, size_t _Index>
	192	using __always _LIBCPP_NODEBUG = _Tp;
	193
	194	template <class _Tp, size_t... _Is>
	195	static auto __repeat_tuple_helper(index_sequence<_Is...>) -> tuple<__always<_Tp, _Is>...>;
	196
	197	public:
	198	using iterator_category = input_iterator_tag;
	199	using iterator_concept = decltype(__get_iterator_concept());
	200	using value_type = decltype(__repeat_tuple_helper<range_value_t<_Base>>(make_index_sequence<_Np>{}));
	201	using difference_type = range_difference_t<_Base>;
	202
	203	_LIBCPP_HIDE_FROM_ABI __iterator() = default;
	204	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
	205	requires _Const && convertible_to<iterator_t<_View>, iterator_t<const _View>>
	206	: __iterator(std::move(__i), make_index_sequence<_Np>{}) {}
	207
	208	_LIBCPP_HIDE_FROM_ABI constexpr auto operator*() const {
	209	return std::__tuple_transform([](auto& __i) -> decltype(auto) { return *__i; }, __current_);
	210	}
	211
	212	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
	213	for (auto& __i : __current_) {
	214	++__i;
	215	}
	216	return *this;
	217	}
	218
	219	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int) {
	220	auto __tmp = *this;
	221	++*this;
	222	return __tmp;
	223	}
	224
	225	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
	226	requires bidirectional_range<_Base>
	227	{
	228	for (auto& __i : __current_) {
	229	--__i;
	230	}
	231	return *this;
	232	}
	233
	234	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
	235	requires bidirectional_range<_Base>
	236	{
	237	auto __tmp = *this;
	238	--*this;
	239	return __tmp;
	240	}
	241
	242	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __x)
	243	requires random_access_range<_Base>
	244	{
	245	for (auto& __i : __current_) {
	246	__i += __x;
	247	}
	248	return *this;
	249	}
	250
	251	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __x)
	252	requires random_access_range<_Base>
	253	{
	254	for (auto& __i : __current_) {
	255	__i -= __x;
	256	}
	257	return *this;
	258	}
	259
	260	_LIBCPP_HIDE_FROM_ABI constexpr auto operator[](difference_type __n) const
	261	requires random_access_range<_Base>
	262	{
	263	return std::__tuple_transform([&](auto& __i) -> decltype(auto) { return __i[__n]; }, __current_);
	264	}
	265
	266	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
	267	return __x.__current_.back() == __y.__current_.back();
	268	}
	269
	270	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<(const __iterator& __x, const __iterator& __y)
	271	requires random_access_range<_Base>
	272	{
	273	return __x.__current_.back() < __y.__current_.back();
	274	}
	275
	276	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>(const __iterator& __x, const __iterator& __y)
	277	requires random_access_range<_Base>
	278	{
	279	return __y < __x;
	280	}
	281
	282	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator<=(const __iterator& __x, const __iterator& __y)
	283	requires random_access_range<_Base>
	284	{
	285	return !(__y < __x);
	286	}
	287
	288	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator>=(const __iterator& __x, const __iterator& __y)
	289	requires random_access_range<_Base>
	290	{
	291	return !(__x < __y);
	292	}
	293
	294	_LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
	295	requires random_access_range<_Base> && three_way_comparable<iterator_t<_Base>>
	296	{
	297	return __x.__current_.back() <=> __y.__current_.back();
	298	}
	299
	300	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __i, difference_type __n)
	301	requires random_access_range<_Base>
	302	{
	303	auto __r = __i;
	304	__r += __n;
	305	return __r;
	306	}
	307
	308	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, const __iterator& __i)
	309	requires random_access_range<_Base>
	310	{
	311	return __i + __n;
	312	}
	313
	314	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __i, difference_type __n)
	315	requires random_access_range<_Base>
	316	{
	317	auto __r = __i;
	318	__r -= __n;
	319	return __r;
	320	}
	321
	322	_LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
	323	requires sized_sentinel_for<iterator_t<_Base>, iterator_t<_Base>>
	324	{
	325	return __x.__current_.back() - __y.__current_.back();
	326	}
	327
	328	_LIBCPP_HIDE_FROM_ABI friend constexpr auto iter_move(const __iterator& __i) noexcept(
	329	noexcept(ranges::iter_move(std::declval<const iterator_t<_Base>&>())) &&
	330	is_nothrow_move_constructible_v<range_rvalue_reference_t<_Base>>) {
	331	return std::__tuple_transform(ranges::iter_move, __i.__current_);
	332	}
	333
	334	_LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap(const __iterator& __l, const __iterator& __r) noexcept(
	335	noexcept(ranges::iter_swap(std::declval<iterator_t<_Base>>(), std::declval<iterator_t<_Base>>())))
	336	requires indirectly_swappable<iterator_t<_Base>>
	337	{
	338	for (size_t __i = 0; __i < _Np; ++__i) {
	339	ranges::iter_swap(__l.__current_[__i], __r.__current_[__i]);
	340	}
	341	}
	342	};
	343
	344	template <forward_range _View, size_t _Np>
	345	requires view<_View> && (_Np > 0)
	346	template <bool _Const>
	347	class adjacent_view<_View, _Np>::__sentinel {
	348	friend adjacent_view;
	349	using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _View>;
	350	sentinel_t<_Base> __end_ = sentinel_t<_Base>();
	351
	352	_LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(sentinel_t<_Base> __end) { __end_ = std::move(__end); }
	353
	354	public:
	355	_LIBCPP_HIDE_FROM_ABI __sentinel() = default;
	356
	357	_LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __i)
	358	requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
	359	: __end_(std::move(__i.__end_)) {}
	360
	361	template <bool _OtherConst>
	362	requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
	363	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	364	return __x.__current_.back() == __y.__end_;
	365	}
	366
	367	template <bool _OtherConst>
	368	requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
	369	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
	370	operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	371	return __x.__current_.back() - __y.__end_;
	372	}
	373
	374	template <bool _OtherConst>
	375	requires sized_sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
	376	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _View>>
	377	operator-(const __sentinel& __y, const __iterator<_OtherConst>& __x) {
	378	return __y.__end_ - __x.__current_.back();
	379	}
	380	};
	381
	382	template <class _View, size_t _Np>
	383	constexpr bool enable_borrowed_range<adjacent_view<_View, _Np>> = enable_borrowed_range<_View>;
	384
	385	namespace views {
	386	namespace __adjacent {
	387
	388	template <size_t _Np>
	389	struct __fn : __range_adaptor_closure<__fn<_Np>> {
	390	template <class _Range>
	391	requires(_Np == 0 && forward_range<_Range &&>)
	392	_LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Range&&) noexcept {
	393	return empty_view<tuple<>>{};
	394	}
	395
	396	template <class _Ranges>
	397	_LIBCPP_HIDE_FROM_ABI static constexpr auto operator()(_Ranges&& __range) noexcept(
	398	noexcept(adjacent_view<views::all_t<_Ranges&&>, _Np>(std::forward<_Ranges>(__range))))
	399	-> decltype(adjacent_view<views::all_t<_Ranges&&>, _Np>(std::forward<_Ranges>(__range))) {
	400	return adjacent_view<views::all_t<_Ranges&&>, _Np>(std::forward<_Ranges>(__range));
	401	}
	402	};
	403
	404	} // namespace __adjacent
	405	inline namespace __cpo {
	406	template <size_t _Np>
	407	inline constexpr auto adjacent = __adjacent::__fn<_Np>{};
	408	inline constexpr auto pairwise = adjacent<2>;
	409	} // namespace __cpo
	410	} // namespace views
	411	} // namespace ranges
	412
	413	#endif // _LIBCPP_STD_VER >= 23
	414
	415	_LIBCPP_END_NAMESPACE_STD
	416
	417	_LIBCPP_POP_MACROS
	418
	419	#endif // _LIBCPP___RANGES_ADJACENT_VIEW_H

lib/libcxx/include/__ranges/as_rvalue_view.h+9-9

...	...	@@ -48,27 +48,27 @@ public:
48	48
49	49	_LIBCPP_HIDE_FROM_ABI constexpr explicit as_rvalue_view(_View __base) : __base_(std::move(__base)) {}
50	50
51		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	51	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
52	52	requires copy_constructible<_View>
53	53	{
54	54	return __base_;
55	55	}
56	56
57		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	57	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
58	58
59		_LIBCPP_HIDE_FROM_ABI constexpr auto begin()
	59	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
60	60	requires(!__simple_view<_View>)
61	61	{
62	62	return move_iterator(ranges::begin(__base_));
63	63	}
64	64
65		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	65	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
66	66	requires range<const _View>
67	67	{
68	68	return move_iterator(ranges::begin(__base_));
69	69	}
70	70
71		_LIBCPP_HIDE_FROM_ABI constexpr auto end()
	71	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end()
72	72	requires(!__simple_view<_View>)
73	73	{
74	74	if constexpr (common_range<_View>) {
...	...	@@ -78,7 +78,7 @@ public:
78	78	}
79	79	}
80	80
81		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	81	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
82	82	requires range<const _View>
83	83	{
84	84	if constexpr (common_range<const _View>) {
...	...	@@ -88,13 +88,13 @@ public:
88	88	}
89	89	}
90	90
91		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	91	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
92	92	requires sized_range<_View>
93	93	{
94	94	return ranges::size(__base_);
95	95	}
96	96
97		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	97	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
98	98	requires sized_range<const _View>
99	99	{
100	100	return ranges::size(__base_);
...	...	@@ -117,7 +117,7 @@ struct __fn : __range_adaptor_closure<__fn> {
117	117	return /---------------------------------/ as_rvalue_view(std::forward<_Range>(__range));
118	118	}
119	119
120		template <class _Range>
	120	template <input_range _Range>
121	121	requires same_as<range_rvalue_reference_t<_Range>, range_reference_t<_Range>>
122	122	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr auto
123	123	operator()(_Range&& __range) noexcept(noexcept(views::all(std::forward<_Range>(__range))))

lib/libcxx/include/__ranges/chunk_by_view.h+6-6

...	...	@@ -100,17 +100,17 @@ public:
100	100	_LIBCPP_HIDE_FROM_ABI constexpr explicit chunk_by_view(_View __base, _Pred __pred)
101	101	: __base_(std::move(__base)), __pred_(in_place, std::move(__pred)) {}
102	102
103		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	103	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
104	104	requires copy_constructible<_View>
105	105	{
106	106	return __base_;
107	107	}
108	108
109		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	109	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
110	110
111		_LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
	111	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
112	112
113		_LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
	113	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
114	114	// Note: this duplicates a check in `optional` but provides a better error message.
115	115	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
116	116	__pred_.__has_value(), "Trying to call begin() on a chunk_by_view that does not have a valid predicate.");
...	...	@@ -122,7 +122,7 @@ public:
122	122	return {this, std::move(__first), __cached_begin_};
123	123	}
124	124
125		_LIBCPP_HIDE_FROM_ABI constexpr auto end() {
	125	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
126	126	if constexpr (common_range<_View>) {
127	127	return __iterator{*this, ranges::end(__base_), ranges::end(__base_)};
128	128	} else {
...	...	@@ -155,7 +155,7 @@ public:
155	155
156	156	_LIBCPP_HIDE_FROM_ABI __iterator() = default;
157	157
158		_LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const {
	158	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const {
159	159	// If the iterator is at end, this would return an empty range which can be checked by the calling code and doesn't
160	160	// necessarily lead to a bad access.
161	161	_LIBCPP_ASSERT_PEDANTIC(__current_ != __next_, "Trying to dereference past-the-end chunk_by_view iterator.");

lib/libcxx/include/__ranges/common_view.h+7-7

...	...	@@ -56,16 +56,16 @@ public:
56	56	return __base_;
57	57	}
58	58
59		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	59	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
60	60
61		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
	61	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
62	62	if constexpr (random_access_range<_View> && sized_range<_View>)
63	63	return ranges::begin(__base_);
64	64	else
65	65	return common_iterator<iterator_t<_View>, sentinel_t<_View>>(ranges::begin(__base_));
66	66	}
67	67
68		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	68	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
69	69	requires range<const _View>
70	70	{
71	71	if constexpr (random_access_range<const _View> && sized_range<const _View>)
...	...	@@ -74,14 +74,14 @@ public:
74	74	return common_iterator<iterator_t<const _View>, sentinel_t<const _View>>(ranges::begin(__base_));
75	75	}
76	76
77		_LIBCPP_HIDE_FROM_ABI constexpr auto end() {
	77	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
78	78	if constexpr (random_access_range<_View> && sized_range<_View>)
79	79	return ranges::begin(__base_) + ranges::size(__base_);
80	80	else
81	81	return common_iterator<iterator_t<_View>, sentinel_t<_View>>(ranges::end(__base_));
82	82	}
83	83
84		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	84	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
85	85	requires range<const _View>
86	86	{
87	87	if constexpr (random_access_range<const _View> && sized_range<const _View>)
...	...	@@ -90,13 +90,13 @@ public:
90	90	return common_iterator<iterator_t<const _View>, sentinel_t<const _View>>(ranges::end(__base_));
91	91	}
92	92
93		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	93	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
94	94	requires sized_range<_View>
95	95	{
96	96	return ranges::size(__base_);
97	97	}
98	98
99		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	99	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
100	100	requires sized_range<const _View>
101	101	{
102	102	return ranges::size(__base_);

lib/libcxx/include/__ranges/drop_view.h+8-8

...	...	@@ -80,14 +80,14 @@ public:
80	80	_LIBCPP_ASSERT_UNCATEGORIZED(__count_ >= 0, "count must be greater than or equal to zero.");
81	81	}
82	82
83		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	83	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
84	84	requires copy_constructible<_View>
85	85	{
86	86	return __base_;
87	87	}
88		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	88	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
89	89
90		_LIBCPP_HIDE_FROM_ABI constexpr auto begin()
	90	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
91	91	requires(!(__simple_view<_View> && random_access_range<const _View> && sized_range<const _View>))
92	92	{
93	93	if constexpr (random_access_range<_View> && sized_range<_View>) {
...	...	@@ -104,20 +104,20 @@ public:
104	104	return __tmp;
105	105	}
106	106
107		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	107	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
108	108	requires random_access_range<const _View> && sized_range<const _View>
109	109	{
110	110	const auto __dist = std::min(ranges::distance(__base_), __count_);
111	111	return ranges::begin(__base_) + __dist;
112	112	}
113	113
114		_LIBCPP_HIDE_FROM_ABI constexpr auto end()
	114	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end()
115	115	requires(!__simple_view<_View>)
116	116	{
117	117	return ranges::end(__base_);
118	118	}
119	119
120		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	120	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
121	121	requires range<const _View>
122	122	{
123	123	return ranges::end(__base_);
...	...	@@ -129,13 +129,13 @@ public:
129	129	return __s < __c ? 0 : __s - __c;
130	130	}
131	131
132		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	132	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
133	133	requires sized_range<_View>
134	134	{
135	135	return __size(*this);
136	136	}
137	137
138		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	138	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
139	139	requires sized_range<const _View>
140	140	{
141	141	return __size(*this);

lib/libcxx/include/__ranges/drop_while_view.h+5-5

...	...	@@ -57,17 +57,17 @@ public:
57	57	_LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 drop_while_view(_View __base, _Pred __pred)
58	58	: __base_(std::move(__base)), __pred_(std::in_place, std::move(__pred)) {}
59	59
60		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	60	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
61	61	requires copy_constructible<_View>
62	62	{
63	63	return __base_;
64	64	}
65	65
66		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	66	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
67	67
68		_LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
	68	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Pred& pred() const { return *__pred_; }
69	69
70		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
	70	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() {
71	71	// Note: this duplicates a check in `optional` but provides a better error message.
72	72	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
73	73	__pred_.__has_value(),
...	...	@@ -83,7 +83,7 @@ public:
83	83	}
84	84	}
85	85
86		_LIBCPP_HIDE_FROM_ABI constexpr auto end() { return ranges::end(__base_); }
	86	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() { return ranges::end(__base_); }
87	87
88	88	private:
89	89	_LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View();

lib/libcxx/include/__ranges/elements_of.h created+49

...	...	@@ -0,0 +1,49 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___RANGES_ELEMENTS_OF_H
	11	#define _LIBCPP___RANGES_ELEMENTS_OF_H
	12
	13	#include <__config>
	14	#include <__cstddef/byte.h>
	15	#include <__memory/allocator.h>
	16	#include <__ranges/concepts.h>
	17	#include <__utility/forward.h>
	18
	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	20	# pragma GCC system_header
	21	#endif
	22
	23	_LIBCPP_PUSH_MACROS
	24	#include <__undef_macros>
	25
	26	_LIBCPP_BEGIN_NAMESPACE_STD
	27
	28	#if _LIBCPP_STD_VER >= 23
	29
	30	namespace ranges {
	31
	32	template <range _Range, class _Allocator = allocator<byte>>
	33	struct elements_of {
	34	_LIBCPP_NO_UNIQUE_ADDRESS _Range range;
	35	_LIBCPP_NO_UNIQUE_ADDRESS _Allocator allocator = _Allocator();
	36	};
	37
	38	template <class _Range, class _Allocator = allocator<byte>>
	39	elements_of(_Range&&, _Allocator = _Allocator()) -> elements_of<_Range&&, _Allocator>;
	40
	41	} // namespace ranges
	42
	43	#endif // _LIBCPP_STD_VER >= 23
	44
	45	_LIBCPP_END_NAMESPACE_STD
	46
	47	_LIBCPP_POP_MACROS
	48
	49	#endif // _LIBCPP___RANGES_ELEMENTS_OF_H

lib/libcxx/include/__ranges/empty_view.h+5-5

...	...	@@ -29,11 +29,11 @@ template <class _Tp>
29	29	requires is_object_v<_Tp>
30	30	class empty_view : public view_interface<empty_view<_Tp>> {
31	31	public:
32		_LIBCPP_HIDE_FROM_ABI static constexpr _Tp* begin() noexcept { return nullptr; }
33		_LIBCPP_HIDE_FROM_ABI static constexpr _Tp* end() noexcept { return nullptr; }
34		_LIBCPP_HIDE_FROM_ABI static constexpr _Tp* data() noexcept { return nullptr; }
35		_LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 0; }
36		_LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return true; }
	32	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* begin() noexcept { return nullptr; }
	33	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* end() noexcept { return nullptr; }
	34	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr _Tp* data() noexcept { return nullptr; }
	35	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 0; }
	36	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return true; }
37	37	};
38	38
39	39	template <class _Tp>

lib/libcxx/include/__ranges/filter_view.h+10-10

...	...	@@ -76,16 +76,16 @@ public:
76	76	: __base_(std::move(__base)), __pred_(in_place, std::move(__pred)) {}
77	77
78	78	template <class _Vp = _View>
79		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	79	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
80	80	requires copy_constructible<_Vp>
81	81	{
82	82	return __base_;
83	83	}
84		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	84	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
85	85
86		_LIBCPP_HIDE_FROM_ABI constexpr _Pred const& pred() const { return *__pred_; }
	86	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Pred const& pred() const { return *__pred_; }
87	87
88		_LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
	88	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() {
89	89	// Note: this duplicates a check in `optional` but provides a better error message.
90	90	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
91	91	__pred_.__has_value(), "Trying to call begin() on a filter_view that does not have a valid predicate.");
...	...	@@ -99,7 +99,7 @@ public:
99	99	}
100	100	}
101	101
102		_LIBCPP_HIDE_FROM_ABI constexpr auto end() {
	102	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() {
103	103	if constexpr (common_range<_View>)
104	104	return __iterator{*this, ranges::end(__base_)};
105	105	else
...	...	@@ -148,10 +148,10 @@ public:
148	148	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(filter_view& __parent, iterator_t<_View> __current)
149	149	: __current_(std::move(__current)), __parent_(std::addressof(__parent)) {}
150	150
151		_LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> const& base() const& noexcept { return __current_; }
152		_LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> base() && { return std::move(__current_); }
	151	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> const& base() const& noexcept { return __current_; }
	152	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> base() && { return std::move(__current_); }
153	153
154		_LIBCPP_HIDE_FROM_ABI constexpr range_reference_t<_View> operator() const { return __current_; }
	154	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr range_reference_t<_View> operator() const { return __current_; }
155	155	_LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_View> operator->() const
156	156	requires __has_arrow<iterator_t<_View>> && copyable<iterator_t<_View>>
157	157	{
...	...	@@ -194,7 +194,7 @@ public:
194	194	return __x.__current_ == __y.__current_;
195	195	}
196	196
197		_LIBCPP_HIDE_FROM_ABI friend constexpr range_rvalue_reference_t<_View>
	197	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr range_rvalue_reference_t<_View>
198	198	iter_move(__iterator const& __it) noexcept(noexcept(ranges::iter_move(__it.__current_))) {
199	199	return ranges::iter_move(__it.__current_);
200	200	}
...	...	@@ -218,7 +218,7 @@ public:
218	218
219	219	_LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(filter_view& __parent) : __end_(ranges::end(__parent.__base_)) {}
220	220
221		_LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_View> base() const { return __end_; }
	221	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_View> base() const { return __end_; }
222	222
223	223	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(__iterator const& __x, __sentinel const& __y) {
224	224	return __x.__current_ == __y.__end_;

lib/libcxx/include/__ranges/iota_view.h+41-22

...	...	@@ -30,6 +30,7 @@
30	30	#include <__ranges/movable_box.h>
31	31	#include <__ranges/view_interface.h>
32	32	#include <__type_traits/conditional.h>
	33	#include <__type_traits/decay.h>
33	34	#include <__type_traits/is_nothrow_constructible.h>
34	35	#include <__type_traits/make_unsigned.h>
35	36	#include <__type_traits/type_identity.h>
...	...	@@ -57,11 +58,17 @@ struct __get_wider_signed {
57	58	return type_identity<int>{};
58	59	else if constexpr (sizeof(_Int) < sizeof(long))
59	60	return type_identity<long>{};
60		else
	61	else if constexpr (sizeof(_Int) < sizeof(long long))
61	62	return type_identity<long long>{};
62
63		static_assert(
64		sizeof(_Int) <= sizeof(long long), "Found integer-like type that is bigger than largest integer like type.");
	63	# if _LIBCPP_HAS_INT128
	64	else if constexpr (sizeof(_Int) <= sizeof(__int128))
	65	return type_identity<__int128>{};
	66	# else
	67	else if constexpr (sizeof(_Int) <= sizeof(long long))
	68	return type_identity<long long>{};
	69	# endif
	70	else
	71	static_assert(false, "Found integer-like type that is bigger than the largest integer like type.");
65	72	}
66	73
67	74	using type = typename decltype(__call())::type;
...	...	@@ -125,7 +132,8 @@ class iota_view : public view_interface<iota_view<_Start, _BoundSentinel>> {
125	132
126	133	_LIBCPP_HIDE_FROM_ABI constexpr explicit __iterator(_Start __value) : __value_(std::move(__value)) {}
127	134
128		_LIBCPP_HIDE_FROM_ABI constexpr _Start operator*() const noexcept(is_nothrow_copy_constructible_v<_Start>) {
	135	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Start operator*() const
	136	noexcept(is_nothrow_copy_constructible_v<_Start>) {
129	137	return __value_;
130	138	}
131	139
...	...	@@ -189,7 +197,7 @@ class iota_view : public view_interface<iota_view<_Start, _BoundSentinel>> {
189	197	return *this;
190	198	}
191	199
192		_LIBCPP_HIDE_FROM_ABI constexpr _Start operator[](difference_type __n) const
	200	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Start operator[](difference_type __n) const
193	201	requires __advanceable<_Start>
194	202	{
195	203	return _Start(__value_ + __n);
...	...	@@ -231,27 +239,28 @@ class iota_view : public view_interface<iota_view<_Start, _BoundSentinel>> {
231	239	return __x.__value_ <=> __y.__value_;
232	240	}
233	241
234		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n)
	242	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n)
235	243	requires __advanceable<_Start>
236	244	{
237	245	__i += __n;
238	246	return __i;
239	247	}
240	248
241		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i)
	249	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i)
242	250	requires __advanceable<_Start>
243	251	{
244	252	return __i + __n;
245	253	}
246	254
247		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n)
	255	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n)
248	256	requires __advanceable<_Start>
249	257	{
250	258	__i -= __n;
251	259	return __i;
252	260	}
253	261
254		_LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
	262	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type
	263	operator-(const __iterator& __x, const __iterator& __y)
255	264	requires __advanceable<_Start>
256	265	{
257	266	if constexpr (__integer_like<_Start>) {
...	...	@@ -282,14 +291,14 @@ class iota_view : public view_interface<iota_view<_Start, _BoundSentinel>> {
282	291	return __x.__value_ == __y.__bound_sentinel_;
283	292	}
284	293
285		_LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
	294	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
286	295	operator-(const __iterator& __x, const __sentinel& __y)
287	296	requires sized_sentinel_for<_BoundSentinel, _Start>
288	297	{
289	298	return __x.__value_ - __y.__bound_sentinel_;
290	299	}
291	300
292		_LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
	301	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr iter_difference_t<_Start>
293	302	operator-(const __sentinel& __x, const __iterator& __y)
294	303	requires sized_sentinel_for<_BoundSentinel, _Start>
295	304	{
...	...	@@ -329,24 +338,24 @@ public:
329	338	requires(!same_as<_Start, _BoundSentinel> && !same_as<_BoundSentinel, unreachable_sentinel_t>)
330	339	: iota_view(std::move(__first.__value_), std::move(__last.__bound_sentinel_)) {}
331	340
332		_LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const { return __iterator{__value_}; }
	341	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const { return __iterator{__value_}; }
333	342
334		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const {
	343	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const {
335	344	if constexpr (same_as<_BoundSentinel, unreachable_sentinel_t>)
336	345	return unreachable_sentinel;
337	346	else
338	347	return __sentinel{__bound_sentinel_};
339	348	}
340	349
341		_LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
	350	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
342	351	requires same_as<_Start, _BoundSentinel>
343	352	{
344	353	return __iterator{__bound_sentinel_};
345	354	}
346	355
347		_LIBCPP_HIDE_FROM_ABI constexpr bool empty() const { return __value_ == __bound_sentinel_; }
	356	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const { return __value_ == __bound_sentinel_; }
348	357
349		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	358	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
350	359	requires(same_as<_Start, _BoundSentinel> && __advanceable<_Start>) \|\|
351	360	(integral<_Start> && integral<_BoundSentinel>) \|\| sized_sentinel_for<_BoundSentinel, _Start>
352	361	{
...	...	@@ -374,14 +383,15 @@ namespace views {
374	383	namespace __iota {
375	384	struct __fn {
376	385	template <class _Start>
377		_LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start) const
378		noexcept(noexcept(ranges::iota_view(std::forward<_Start>(__start))))
379		-> decltype(ranges::iota_view(std::forward<_Start>(__start))) {
380		return ranges::iota_view(std::forward<_Start>(__start));
	386	requires(requires(_Start __s) { ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__s)); })
	387	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start) const
	388	noexcept(noexcept(ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__start)))) {
	389	return ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__start));
381	390	}
382	391
383	392	template <class _Start, class _BoundSentinel>
384		_LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start, _BoundSentinel&& __bound_sentinel) const noexcept(
	393	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto
	394	operator()(_Start&& __start, _BoundSentinel&& __bound_sentinel) const noexcept(
385	395	noexcept(ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel))))
386	396	-> decltype(ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel))) {
387	397	return ranges::iota_view(std::forward<_Start>(__start), std::forward<_BoundSentinel>(__bound_sentinel));
...	...	@@ -392,6 +402,15 @@ struct __fn {
392	402	inline namespace __cpo {
393	403	inline constexpr auto iota = __iota::__fn{};
394	404	} // namespace __cpo
	405
	406	# if _LIBCPP_STD_VER >= 26
	407
	408	inline constexpr auto indices = [] [[nodiscard]] (__integer_like auto __size) static {
	409	return ranges::views::iota(decltype(__size){}, __size);
	410	};
	411
	412	# endif
	413
395	414	} // namespace views
396	415	} // namespace ranges
397	416

lib/libcxx/include/__ranges/owning_view.h+15-15

...	...	@@ -49,52 +49,52 @@ public:
49	49	_LIBCPP_HIDE_FROM_ABI owning_view(owning_view&&) = default;
50	50	_LIBCPP_HIDE_FROM_ABI owning_view& operator=(owning_view&&) = default;
51	51
52		_LIBCPP_HIDE_FROM_ABI constexpr _Rp& base() & noexcept { return __r_; }
53		_LIBCPP_HIDE_FROM_ABI constexpr const _Rp& base() const& noexcept { return __r_; }
54		_LIBCPP_HIDE_FROM_ABI constexpr _Rp&& base() && noexcept { return std::move(__r_); }
55		_LIBCPP_HIDE_FROM_ABI constexpr const _Rp&& base() const&& noexcept { return std::move(__r_); }
56
57		_LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Rp> begin() { return ranges::begin(__r_); }
58		_LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Rp> end() { return ranges::end(__r_); }
59		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	52	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Rp& base() & noexcept { return __r_; }
	53	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Rp& base() const& noexcept { return __r_; }
	54	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Rp&& base() && noexcept { return std::move(__r_); }
	55	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Rp&& base() const&& noexcept { return std::move(__r_); }
	56
	57	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Rp> begin() { return ranges::begin(__r_); }
	58	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Rp> end() { return ranges::end(__r_); }
	59	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
60	60	requires range<const _Rp>
61	61	{
62	62	return ranges::begin(__r_);
63	63	}
64		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	64	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
65	65	requires range<const _Rp>
66	66	{
67	67	return ranges::end(__r_);
68	68	}
69	69
70		_LIBCPP_HIDE_FROM_ABI constexpr bool empty()
	70	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty()
71	71	requires requires { ranges::empty(__r_); }
72	72	{
73	73	return ranges::empty(__r_);
74	74	}
75		_LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
	75	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
76	76	requires requires { ranges::empty(__r_); }
77	77	{
78	78	return ranges::empty(__r_);
79	79	}
80	80
81		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	81	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
82	82	requires sized_range<_Rp>
83	83	{
84	84	return ranges::size(__r_);
85	85	}
86		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	86	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
87	87	requires sized_range<const _Rp>
88	88	{
89	89	return ranges::size(__r_);
90	90	}
91	91
92		_LIBCPP_HIDE_FROM_ABI constexpr auto data()
	92	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto data()
93	93	requires contiguous_range<_Rp>
94	94	{
95	95	return ranges::data(__r_);
96	96	}
97		_LIBCPP_HIDE_FROM_ABI constexpr auto data() const
	97	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
98	98	requires contiguous_range<const _Rp>
99	99	{
100	100	return ranges::data(__r_);

lib/libcxx/include/__ranges/ref_view.h+6-6

...	...	@@ -51,24 +51,24 @@ public:
51	51	_LIBCPP_HIDE_FROM_ABI constexpr ref_view(_Tp&& __t)
52	52	: __range_(std::addressof(static_cast<_Range&>(std::forward<_Tp>(__t)))) {}
53	53
54		_LIBCPP_HIDE_FROM_ABI constexpr _Range& base() const { return *__range_; }
	54	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Range& base() const { return *__range_; }
55	55
56		_LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Range> begin() const { return ranges::begin(*__range_); }
57		_LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Range> end() const { return ranges::end(*__range_); }
	56	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Range> begin() const { return ranges::begin(*__range_); }
	57	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Range> end() const { return ranges::end(*__range_); }
58	58
59		_LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
	59	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const
60	60	requires requires { ranges::empty(*__range_); }
61	61	{
62	62	return ranges::empty(*__range_);
63	63	}
64	64
65		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	65	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
66	66	requires sized_range<_Range>
67	67	{
68	68	return ranges::size(*__range_);
69	69	}
70	70
71		_LIBCPP_HIDE_FROM_ABI constexpr auto data() const
	71	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
72	72	requires contiguous_range<_Range>
73	73	{
74	74	return ranges::data(*__range_);

lib/libcxx/include/__ranges/repeat_view.h+17-10

...	...	@@ -108,17 +108,21 @@ public:
108	108	__bound_ >= 0, "The behavior is undefined if Bound is not unreachable_sentinel_t and bound is negative");
109	109	}
110	110
111		_LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const { return __iterator(std::addressof(*__value_)); }
	111	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator begin() const {
	112	return __iterator(std::addressof(*__value_));
	113	}
112	114
113		_LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
	115	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __iterator end() const
114	116	requires(!same_as<_Bound, unreachable_sentinel_t>)
115	117	{
116	118	return __iterator(std::addressof(*__value_), __bound_);
117	119	}
118	120
119		_LIBCPP_HIDE_FROM_ABI constexpr unreachable_sentinel_t end() const noexcept { return unreachable_sentinel; }
	121	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr unreachable_sentinel_t end() const noexcept {
	122	return unreachable_sentinel;
	123	}
120	124
121		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	125	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
122	126	requires(!same_as<_Bound, unreachable_sentinel_t>)
123	127	{
124	128	return std::__to_unsigned_like(__bound_);
...	...	@@ -152,7 +156,7 @@ public:
152	156
153	157	_LIBCPP_HIDE_FROM_ABI __iterator() = default;
154	158
155		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator() const noexcept { return __value_; }
	159	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator() const noexcept { return __value_; }
156	160
157	161	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
158	162	++__current_;
...	...	@@ -192,7 +196,9 @@ public:
192	196	return *this;
193	197	}
194	198
195		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](difference_type __n) const noexcept { return (this + __n); }
	199	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator[](difference_type __n) const noexcept {
	200	return (this + __n);
	201	}
196	202
197	203	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y) {
198	204	return __x.__current_ == __y.__current_;
...	...	@@ -202,22 +208,23 @@ public:
202	208	return __x.__current_ <=> __y.__current_;
203	209	}
204	210
205		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n) {
	211	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(__iterator __i, difference_type __n) {
206	212	__i += __n;
207	213	return __i;
208	214	}
209	215
210		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i) {
	216	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, __iterator __i) {
211	217	__i += __n;
212	218	return __i;
213	219	}
214	220
215		_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n) {
	221	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(__iterator __i, difference_type __n) {
216	222	__i -= __n;
217	223	return __i;
218	224	}
219	225
220		_LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y) {
	226	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI friend constexpr difference_type
	227	operator-(const __iterator& __x, const __iterator& __y) {
221	228	return static_cast<difference_type>(__x.__current_) - static_cast<difference_type>(__y.__current_);
222	229	}
223	230

lib/libcxx/include/__ranges/single_view.h+8-8

...	...	@@ -63,21 +63,21 @@ public:
63	63	_LIBCPP_HIDE_FROM_ABI constexpr explicit single_view(in_place_t, _Args&&... __args)
64	64	: __value_{in_place, std::forward<_Args>(__args)...} {}
65	65
66		_LIBCPP_HIDE_FROM_ABI constexpr _Tp* begin() noexcept { return data(); }
	66	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* begin() noexcept { return data(); }
67	67
68		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp* begin() const noexcept { return data(); }
	68	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* begin() const noexcept { return data(); }
69	69
70		_LIBCPP_HIDE_FROM_ABI constexpr _Tp* end() noexcept { return data() + 1; }
	70	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* end() noexcept { return data() + 1; }
71	71
72		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp* end() const noexcept { return data() + 1; }
	72	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* end() const noexcept { return data() + 1; }
73	73
74		_LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return false; }
	74	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr bool empty() noexcept { return false; }
75	75
76		_LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 1; }
	76	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr size_t size() noexcept { return 1; }
77	77
78		_LIBCPP_HIDE_FROM_ABI constexpr _Tp* data() noexcept { return __value_.operator->(); }
	78	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* data() noexcept { return __value_.operator->(); }
79	79
80		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp* data() const noexcept { return __value_.operator->(); }
	80	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp* data() const noexcept { return __value_.operator->(); }
81	81	};
82	82
83	83	template <class _Tp>

lib/libcxx/include/__ranges/take_view.h+9-9

...	...	@@ -75,15 +75,15 @@ public:
75	75	_LIBCPP_ASSERT_UNCATEGORIZED(__count >= 0, "count has to be greater than or equal to zero");
76	76	}
77	77
78		_LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
	78	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() const&
79	79	requires copy_constructible<_View>
80	80	{
81	81	return __base_;
82	82	}
83	83
84		_LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
	84	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); }
85	85
86		_LIBCPP_HIDE_FROM_ABI constexpr auto begin()
	86	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin()
87	87	requires(!__simple_view<_View>)
88	88	{
89	89	if constexpr (sized_range<_View>) {
...	...	@@ -99,7 +99,7 @@ public:
99	99	}
100	100	}
101	101
102		_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	102	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
103	103	requires range<const _View>
104	104	{
105	105	if constexpr (sized_range<const _View>) {
...	...	@@ -115,7 +115,7 @@ public:
115	115	}
116	116	}
117	117
118		_LIBCPP_HIDE_FROM_ABI constexpr auto end()
	118	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end()
119	119	requires(!__simple_view<_View>)
120	120	{
121	121	if constexpr (sized_range<_View>) {
...	...	@@ -129,7 +129,7 @@ public:
129	129	}
130	130	}
131	131
132		_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	132	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const
133	133	requires range<const _View>
134	134	{
135	135	if constexpr (sized_range<const _View>) {
...	...	@@ -143,14 +143,14 @@ public:
143	143	}
144	144	}
145	145
146		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	146	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
147	147	requires sized_range<_View>
148	148	{
149	149	auto __n = ranges::size(__base_);
150	150	return ranges::min(__n, static_cast<decltype(__n)>(__count_));
151	151	}
152	152
153		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	153	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
154	154	requires sized_range<const _View>
155	155	{
156	156	auto __n = ranges::size(__base_);
...	...	@@ -178,7 +178,7 @@ public:
178	178	requires _Const && convertible_to<sentinel_t<_View>, sentinel_t<_Base>>
179	179	: __end_(std::move(__s.__end_)) {}
180	180
181		_LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
	181	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr sentinel_t<_Base> base() const { return __end_; }
182	182
183	183	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const _Iter<_Const>& __lhs, const __sentinel& __rhs) {
184	184	return __lhs.count() == 0 \|\| __lhs.base() == __rhs.__end_;

lib/libcxx/include/__ranges/transform_view.h+1-2

...	...	@@ -13,7 +13,6 @@
13	13	#include <__compare/three_way_comparable.h>
14	14	#include <__concepts/constructible.h>
15	15	#include <__concepts/convertible_to.h>
16		#include <__concepts/copyable.h>
17	16	#include <__concepts/derived_from.h>
18	17	#include <__concepts/equality_comparable.h>
19	18	#include <__concepts/invocable.h>
...	...	@@ -64,7 +63,7 @@ concept __regular_invocable_with_range_ref = regular_invocable<_Fn, range_refere
64	63	template <class _View, class _Fn>
65	64	concept __transform_view_constraints =
66	65	view<_View> && is_object_v<_Fn> && regular_invocable<_Fn&, range_reference_t<_View>> &&
67		__is_referenceable_v<invoke_result_t<_Fn&, range_reference_t<_View>>>;
	66	__referenceable<invoke_result_t<_Fn&, range_reference_t<_View>>>;
68	67
69	68	# if _LIBCPP_STD_VER >= 23
70	69	template <input_range _View, move_constructible _Fn>

lib/libcxx/include/__ranges/view_interface.h+10-10

...	...	@@ -87,35 +87,35 @@ public:
87	87	}
88	88
89	89	template <class _D2 = _Derived>
90		_LIBCPP_HIDE_FROM_ABI constexpr auto data()
	90	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto data()
91	91	requires contiguous_iterator<iterator_t<_D2>>
92	92	{
93	93	return std::to_address(ranges::begin(__derived()));
94	94	}
95	95
96	96	template <class _D2 = _Derived>
97		_LIBCPP_HIDE_FROM_ABI constexpr auto data() const
	97	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto data() const
98	98	requires range<const _D2> && contiguous_iterator<iterator_t<const _D2>>
99	99	{
100	100	return std::to_address(ranges::begin(__derived()));
101	101	}
102	102
103	103	template <class _D2 = _Derived>
104		_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	104	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size()
105	105	requires forward_range<_D2> && sized_sentinel_for<sentinel_t<_D2>, iterator_t<_D2>>
106	106	{
107	107	return std::__to_unsigned_like(ranges::end(__derived()) - ranges::begin(__derived()));
108	108	}
109	109
110	110	template <class _D2 = _Derived>
111		_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	111	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto size() const
112	112	requires forward_range<const _D2> && sized_sentinel_for<sentinel_t<const _D2>, iterator_t<const _D2>>
113	113	{
114	114	return std::__to_unsigned_like(ranges::end(__derived()) - ranges::begin(__derived()));
115	115	}
116	116
117	117	template <class _D2 = _Derived>
118		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front()
	118	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front()
119	119	requires forward_range<_D2>
120	120	{
121	121	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
...	...	@@ -124,7 +124,7 @@ public:
124	124	}
125	125
126	126	template <class _D2 = _Derived>
127		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front() const
	127	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) front() const
128	128	requires forward_range<const _D2>
129	129	{
130	130	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
...	...	@@ -133,7 +133,7 @@ public:
133	133	}
134	134
135	135	template <class _D2 = _Derived>
136		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back()
	136	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back()
137	137	requires bidirectional_range<_D2> && common_range<_D2>
138	138	{
139	139	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
...	...	@@ -142,7 +142,7 @@ public:
142	142	}
143	143
144	144	template <class _D2 = _Derived>
145		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back() const
	145	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) back() const
146	146	requires bidirectional_range<const _D2> && common_range<const _D2>
147	147	{
148	148	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
...	...	@@ -151,12 +151,12 @@ public:
151	151	}
152	152
153	153	template <random_access_range _RARange = _Derived>
154		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) {
	154	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) {
155	155	return ranges::begin(__derived())[__index];
156	156	}
157	157
158	158	template <random_access_range _RARange = const _Derived>
159		_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) const {
	159	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](range_difference_t<_RARange> __index) const {
160	160	return ranges::begin(__derived())[__index];
161	161	}
162	162	};

lib/libcxx/include/__ranges/zip_transform_view.h created+357

...	...	@@ -0,0 +1,357 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___RANGES_ZIP_TRANSFORM_VIEW_H
	11	#define _LIBCPP___RANGES_ZIP_TRANSFORM_VIEW_H
	12
	13	#include <__config>
	14
	15	#include <__concepts/constructible.h>
	16	#include <__concepts/convertible_to.h>
	17	#include <__concepts/derived_from.h>
	18	#include <__concepts/equality_comparable.h>
	19	#include <__concepts/invocable.h>
	20	#include <__functional/invoke.h>
	21	#include <__iterator/concepts.h>
	22	#include <__iterator/incrementable_traits.h>
	23	#include <__iterator/iterator_traits.h>
	24	#include <__memory/addressof.h>
	25	#include <__ranges/access.h>
	26	#include <__ranges/all.h>
	27	#include <__ranges/concepts.h>
	28	#include <__ranges/empty_view.h>
	29	#include <__ranges/movable_box.h>
	30	#include <__ranges/view_interface.h>
	31	#include <__ranges/zip_view.h>
	32	#include <__type_traits/decay.h>
	33	#include <__type_traits/invoke.h>
	34	#include <__type_traits/is_object.h>
	35	#include <__type_traits/is_reference.h>
	36	#include <__type_traits/is_referenceable.h>
	37	#include <__type_traits/maybe_const.h>
	38	#include <__type_traits/remove_cvref.h>
	39	#include <__utility/forward.h>
	40	#include <__utility/in_place.h>
	41	#include <__utility/move.h>
	42	#include <tuple> // for std::apply
	43
	44	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	45	# pragma GCC system_header
	46	#endif
	47
	48	_LIBCPP_PUSH_MACROS
	49	#include <__undef_macros>
	50	_LIBCPP_BEGIN_NAMESPACE_STD
	51
	52	#if _LIBCPP_STD_VER >= 23
	53
	54	namespace ranges {
	55
	56	template <move_constructible _Fn, input_range... _Views>
	57	requires(view<_Views> && ...) &&
	58	(sizeof...(_Views) > 0) && is_object_v<_Fn> && regular_invocable<_Fn&, range_reference_t<_Views>...> &&
	59	__referenceable<invoke_result_t<_Fn&, range_reference_t<_Views>...>>
	60	class zip_transform_view : public view_interface<zip_transform_view<_Fn, _Views...>> {
	61	_LIBCPP_NO_UNIQUE_ADDRESS zip_view<_Views...> __zip_;
	62	_LIBCPP_NO_UNIQUE_ADDRESS __movable_box<_Fn> __fun_;
	63
	64	using _InnerView _LIBCPP_NODEBUG = zip_view<_Views...>;
	65	template <bool _Const>
	66	using __ziperator _LIBCPP_NODEBUG = iterator_t<__maybe_const<_Const, _InnerView>>;
	67	template <bool _Const>
	68	using __zentinel _LIBCPP_NODEBUG = sentinel_t<__maybe_const<_Const, _InnerView>>;
	69
	70	template <bool>
	71	class __iterator;
	72
	73	template <bool>
	74	class __sentinel;
	75
	76	public:
	77	_LIBCPP_HIDE_FROM_ABI zip_transform_view() = default;
	78
	79	_LIBCPP_HIDE_FROM_ABI constexpr explicit zip_transform_view(_Fn __fun, _Views... __views)
	80	: __zip_(std::move(__views)...), __fun_(in_place, std::move(__fun)) {}
	81
	82	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() { return __iterator<false>(*this, __zip_.begin()); }
	83
	84	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
	85	requires range<const _InnerView> && regular_invocable<const _Fn&, range_reference_t<const _Views>...>
	86	{
	87	return __iterator<true>(*this, __zip_.begin());
	88	}
	89
	90	_LIBCPP_HIDE_FROM_ABI constexpr auto end() {
	91	if constexpr (common_range<_InnerView>) {
	92	return __iterator<false>(*this, __zip_.end());
	93	} else {
	94	return __sentinel<false>(__zip_.end());
	95	}
	96	}
	97
	98	_LIBCPP_HIDE_FROM_ABI constexpr auto end() const
	99	requires range<const _InnerView> && regular_invocable<const _Fn&, range_reference_t<const _Views>...>
	100	{
	101	if constexpr (common_range<const _InnerView>) {
	102	return __iterator<true>(*this, __zip_.end());
	103	} else {
	104	return __sentinel<true>(__zip_.end());
	105	}
	106	}
	107
	108	_LIBCPP_HIDE_FROM_ABI constexpr auto size()
	109	requires sized_range<_InnerView>
	110	{
	111	return __zip_.size();
	112	}
	113
	114	_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
	115	requires sized_range<const _InnerView>
	116	{
	117	return __zip_.size();
	118	}
	119	};
	120
	121	template <class _Fn, class... _Ranges>
	122	zip_transform_view(_Fn, _Ranges&&...) -> zip_transform_view<_Fn, views::all_t<_Ranges>...>;
	123
	124	template <bool _Const, class _Fn, class... _Views>
	125	struct __zip_transform_iterator_category_base {};
	126
	127	template <bool _Const, class _Fn, class... _Views>
	128	requires forward_range<__maybe_const<_Const, zip_view<_Views...>>>
	129	struct __zip_transform_iterator_category_base<_Const, _Fn, _Views...> {
	130	private:
	131	template <class _View>
	132	using __tag _LIBCPP_NODEBUG = typename iterator_traits<iterator_t<__maybe_const<_Const, _View>>>::iterator_category;
	133
	134	static consteval auto __get_iterator_category() {
	135	if constexpr (!is_reference_v<invoke_result_t<__maybe_const<_Const, _Fn>&,
	136	range_reference_t<__maybe_const<_Const, _Views>>...>>) {
	137	return input_iterator_tag();
	138	} else if constexpr ((derived_from<__tag<_Views>, random_access_iterator_tag> && ...)) {
	139	return random_access_iterator_tag();
	140	} else if constexpr ((derived_from<__tag<_Views>, bidirectional_iterator_tag> && ...)) {
	141	return bidirectional_iterator_tag();
	142	} else if constexpr ((derived_from<__tag<_Views>, forward_iterator_tag> && ...)) {
	143	return forward_iterator_tag();
	144	} else {
	145	return input_iterator_tag();
	146	}
	147	}
	148
	149	public:
	150	using iterator_category = decltype(__get_iterator_category());
	151	};
	152
	153	template <move_constructible _Fn, input_range... _Views>
	154	requires(view<_Views> && ...) &&
	155	(sizeof...(_Views) > 0) && is_object_v<_Fn> && regular_invocable<_Fn&, range_reference_t<_Views>...> &&
	156	__referenceable<invoke_result_t<_Fn&, range_reference_t<_Views>...>>
	157	template <bool _Const>
	158	class zip_transform_view<_Fn, _Views...>::__iterator
	159	: public __zip_transform_iterator_category_base<_Const, _Fn, _Views...> {
	160	using _Parent _LIBCPP_NODEBUG = __maybe_const<_Const, zip_transform_view>;
	161	using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _InnerView>;
	162
	163	friend zip_transform_view<_Fn, _Views...>;
	164
	165	_Parent* __parent_ = nullptr;
	166	__ziperator<_Const> __inner_;
	167
	168	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(_Parent& __parent, __ziperator<_Const> __inner)
	169	: __parent_(std::addressof(__parent)), __inner_(std::move(__inner)) {}
	170
	171	_LIBCPP_HIDE_FROM_ABI constexpr auto __get_deref_and_invoke() const noexcept {
	172	return [&__fun = *__parent_->__fun_](const auto&... __iters) noexcept(noexcept(std::invoke(
	173	__parent_->__fun_, __iters...))) -> decltype(auto) { return std::invoke(__fun, *__iters...); };
	174	}
	175
	176	public:
	177	using iterator_concept = typename __ziperator<_Const>::iterator_concept;
	178	using value_type =
	179	remove_cvref_t<invoke_result_t<__maybe_const<_Const, _Fn>&, range_reference_t<__maybe_const<_Const, _Views>>...>>;
	180	using difference_type = range_difference_t<_Base>;
	181
	182	_LIBCPP_HIDE_FROM_ABI __iterator() = default;
	183	_LIBCPP_HIDE_FROM_ABI constexpr __iterator(__iterator<!_Const> __i)
	184	requires _Const && convertible_to<__ziperator<false>, __ziperator<_Const>>
	185	: __parent_(__i.__parent_), __inner_(std::move(__i.__inner_)) {}
	186
	187	_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const
	188	noexcept(noexcept(std::apply(__get_deref_and_invoke(), __zip_view_iterator_access::__get_underlying(__inner_)))) {
	189	return std::apply(__get_deref_and_invoke(), __zip_view_iterator_access::__get_underlying(__inner_));
	190	}
	191
	192	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
	193	++__inner_;
	194	return *this;
	195	}
	196
	197	_LIBCPP_HIDE_FROM_ABI constexpr void operator++(int) { ++*this; }
	198
	199	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator++(int)
	200	requires forward_range<_Base>
	201	{
	202	auto __tmp = *this;
	203	++*this;
	204	return __tmp;
	205	}
	206
	207	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator--()
	208	requires bidirectional_range<_Base>
	209	{
	210	--__inner_;
	211	return *this;
	212	}
	213
	214	_LIBCPP_HIDE_FROM_ABI constexpr __iterator operator--(int)
	215	requires bidirectional_range<_Base>
	216	{
	217	auto __tmp = *this;
	218	--*this;
	219	return __tmp;
	220	}
	221
	222	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator+=(difference_type __x)
	223	requires random_access_range<_Base>
	224	{
	225	__inner_ += __x;
	226	return *this;
	227	}
	228
	229	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator-=(difference_type __x)
	230	requires random_access_range<_Base>
	231	{
	232	__inner_ -= __x;
	233	return *this;
	234	}
	235
	236	_LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator[](difference_type __n) const
	237	requires random_access_range<_Base>
	238	{
	239	return std::apply(
	240	[&]<class... _Is>(const _Is&... __iters) -> decltype(auto) {
	241	return std::invoke(*__parent_->__fun_, __iters[iter_difference_t<_Is>(__n)]...);
	242	},
	243	__zip_view_iterator_access::__get_underlying(__inner_));
	244	}
	245
	246	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator& __x, const __iterator& __y)
	247	requires equality_comparable<__ziperator<_Const>>
	248	{
	249	return __x.__inner_ == __y.__inner_;
	250	}
	251
	252	_LIBCPP_HIDE_FROM_ABI friend constexpr auto operator<=>(const __iterator& __x, const __iterator& __y)
	253	requires random_access_range<_Base>
	254	{
	255	return __x.__inner_ <=> __y.__inner_;
	256	}
	257
	258	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(const __iterator& __i, difference_type __n)
	259	requires random_access_range<_Base>
	260	{
	261	return __iterator(*__i.__parent_, __i.__inner_ + __n);
	262	}
	263
	264	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator+(difference_type __n, const __iterator& __i)
	265	requires random_access_range<_Base>
	266	{
	267	return __iterator(*__i.__parent_, __i.__inner_ + __n);
	268	}
	269
	270	_LIBCPP_HIDE_FROM_ABI friend constexpr __iterator operator-(const __iterator& __i, difference_type __n)
	271	requires random_access_range<_Base>
	272	{
	273	return __iterator(*__i.__parent_, __i.__inner_ - __n);
	274	}
	275
	276	_LIBCPP_HIDE_FROM_ABI friend constexpr difference_type operator-(const __iterator& __x, const __iterator& __y)
	277	requires sized_sentinel_for<__ziperator<_Const>, __ziperator<_Const>>
	278	{
	279	return __x.__inner_ - __y.__inner_;
	280	}
	281	};
	282
	283	template <move_constructible _Fn, input_range... _Views>
	284	requires(view<_Views> && ...) &&
	285	(sizeof...(_Views) > 0) && is_object_v<_Fn> && regular_invocable<_Fn&, range_reference_t<_Views>...> &&
	286	__referenceable<invoke_result_t<_Fn&, range_reference_t<_Views>...>>
	287	template <bool _Const>
	288	class zip_transform_view<_Fn, _Views...>::__sentinel {
	289	__zentinel<_Const> __inner_;
	290
	291	friend zip_transform_view<_Fn, _Views...>;
	292
	293	_LIBCPP_HIDE_FROM_ABI constexpr explicit __sentinel(__zentinel<_Const> __inner) : __inner_(__inner) {}
	294
	295	public:
	296	_LIBCPP_HIDE_FROM_ABI __sentinel() = default;
	297
	298	_LIBCPP_HIDE_FROM_ABI constexpr __sentinel(__sentinel<!_Const> __i)
	299	requires _Const && convertible_to<__zentinel<false>, __zentinel<_Const>>
	300	: __inner_(__i.__inner_) {}
	301
	302	template <bool _OtherConst>
	303	requires sentinel_for<__zentinel<_Const>, __ziperator<_OtherConst>>
	304	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	305	return __x.__inner_ == __y.__inner_;
	306	}
	307
	308	template <bool _OtherConst>
	309	requires sized_sentinel_for<__zentinel<_Const>, __ziperator<_OtherConst>>
	310	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _InnerView>>
	311	operator-(const __iterator<_OtherConst>& __x, const __sentinel& __y) {
	312	return __x.__inner_ - __y.__inner_;
	313	}
	314
	315	template <bool _OtherConst>
	316	requires sized_sentinel_for<__zentinel<_Const>, __ziperator<_OtherConst>>
	317	_LIBCPP_HIDE_FROM_ABI friend constexpr range_difference_t<__maybe_const<_OtherConst, _InnerView>>
	318	operator-(const __sentinel& __x, const __iterator<_OtherConst>& __y) {
	319	return __x.__inner_ - __y.__inner_;
	320	}
	321	};
	322
	323	namespace views {
	324	namespace __zip_transform {
	325
	326	struct __fn {
	327	template <class _Fn>
	328	requires(move_constructible<decay_t<_Fn>> && regular_invocable<decay_t<_Fn>&> &&
	329	is_object_v<invoke_result_t<decay_t<_Fn>&>>)
	330	_LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&&) const
	331	noexcept(noexcept(auto(views::empty<decay_t<invoke_result_t<decay_t<_Fn>&>>>))) {
	332	return views::empty<decay_t<invoke_result_t<decay_t<_Fn>&>>>;
	333	}
	334
	335	template <class _Fn, class... _Ranges>
	336	requires(sizeof...(_Ranges) > 0)
	337	_LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Fn&& __fun, _Ranges&&... __rs) const
	338	noexcept(noexcept(zip_transform_view(std::forward<_Fn>(__fun), std::forward<_Ranges>(__rs)...)))
	339	-> decltype(zip_transform_view(std::forward<_Fn>(__fun), std::forward<_Ranges>(__rs)...)) {
	340	return zip_transform_view(std::forward<_Fn>(__fun), std::forward<_Ranges>(__rs)...);
	341	}
	342	};
	343
	344	} // namespace __zip_transform
	345	inline namespace __cpo {
	346	inline constexpr auto zip_transform = __zip_transform::__fn{};
	347	} // namespace __cpo
	348	} // namespace views
	349	} // namespace ranges
	350
	351	#endif // _LIBCPP_STD_VER >= 23
	352
	353	_LIBCPP_END_NAMESPACE_STD
	354
	355	_LIBCPP_POP_MACROS
	356
	357	#endif // _LIBCPP___RANGES_ZIP_TRANSFORM_VIEW_H

lib/libcxx/include/__ranges/zip_view.h+20-20

...	...	@@ -31,6 +31,7 @@
31	31	#include <__ranges/enable_borrowed_range.h>
32	32	#include <__ranges/size.h>
33	33	#include <__ranges/view_interface.h>
	34	#include <__tuple/tuple_transform.h>
34	35	#include <__type_traits/is_nothrow_constructible.h>
35	36	#include <__type_traits/make_unsigned.h>
36	37	#include <__utility/declval.h>
...	...	@@ -58,15 +59,6 @@ concept __zip_is_common =
58	59	(!(bidirectional_range<_Ranges> && ...) && (common_range<_Ranges> && ...)) \|\|
59	60	((random_access_range<_Ranges> && ...) && (sized_range<_Ranges> && ...));
60	61
61		template <class _Fun, class _Tuple>
62		_LIBCPP_HIDE_FROM_ABI constexpr auto __tuple_transform(_Fun&& __f, _Tuple&& __tuple) {
63		return std::apply(
64		[&]<class... _Types>(_Types&&... __elements) {
65		return tuple<invoke_result_t<_Fun&, _Types>...>(std::invoke(__f, std::forward<_Types>(__elements))...);
66		},
67		std::forward<_Tuple>(__tuple));
68		}
69
70	62	template <class _Fun, class _Tuple>
71	63	_LIBCPP_HIDE_FROM_ABI constexpr void __tuple_for_each(_Fun&& __f, _Tuple&& __tuple) {
72	64	std::apply(
...	...	@@ -145,24 +137,24 @@ public:
145	137	_LIBCPP_HIDE_FROM_ABI constexpr auto begin()
146	138	requires(!(__simple_view<_Views> && ...))
147	139	{
148		return __iterator<false>(ranges::__tuple_transform(ranges::begin, __views_));
	140	return __iterator<false>(std::__tuple_transform(ranges::begin, __views_));
149	141	}
150	142
151	143	_LIBCPP_HIDE_FROM_ABI constexpr auto begin() const
152	144	requires(range<const _Views> && ...)
153	145	{
154		return __iterator<true>(ranges::__tuple_transform(ranges::begin, __views_));
	146	return __iterator<true>(std::__tuple_transform(ranges::begin, __views_));
155	147	}
156	148
157	149	_LIBCPP_HIDE_FROM_ABI constexpr auto end()
158	150	requires(!(__simple_view<_Views> && ...))
159	151	{
160	152	if constexpr (!__zip_is_common<_Views...>) {
161		return __sentinel<false>(ranges::__tuple_transform(ranges::end, __views_));
	153	return __sentinel<false>(std::__tuple_transform(ranges::end, __views_));
162	154	} else if constexpr ((random_access_range<_Views> && ...)) {
163	155	return begin() + iter_difference_t<__iterator<false>>(size());
164	156	} else {
165		return __iterator<false>(ranges::__tuple_transform(ranges::end, __views_));
	157	return __iterator<false>(std::__tuple_transform(ranges::end, __views_));
166	158	}
167	159	}
168	160
...	...	@@ -170,11 +162,11 @@ public:
170	162	requires(range<const _Views> && ...)
171	163	{
172	164	if constexpr (!__zip_is_common<const _Views...>) {
173		return __sentinel<true>(ranges::__tuple_transform(ranges::end, __views_));
	165	return __sentinel<true>(std::__tuple_transform(ranges::end, __views_));
174	166	} else if constexpr ((random_access_range<const _Views> && ...)) {
175	167	return begin() + iter_difference_t<__iterator<true>>(size());
176	168	} else {
177		return __iterator<true>(ranges::__tuple_transform(ranges::end, __views_));
	169	return __iterator<true>(std::__tuple_transform(ranges::end, __views_));
178	170	}
179	171	}
180	172
...	...	@@ -186,7 +178,7 @@ public:
186	178	using _CT = make_unsigned_t<common_type_t<decltype(__sizes)...>>;
187	179	return ranges::min({_CT(__sizes)...});
188	180	},
189		ranges::__tuple_transform(ranges::size, __views_));
	181	std::__tuple_transform(ranges::size, __views_));
190	182	}
191	183
192	184	_LIBCPP_HIDE_FROM_ABI constexpr auto size() const
...	...	@@ -197,7 +189,7 @@ public:
197	189	using _CT = make_unsigned_t<common_type_t<decltype(__sizes)...>>;
198	190	return ranges::min({_CT(__sizes)...});
199	191	},
200		ranges::__tuple_transform(ranges::size, __views_));
	192	std::__tuple_transform(ranges::size, __views_));
201	193	}
202	194	};
203	195
...	...	@@ -235,6 +227,13 @@ struct __zip_view_iterator_category_base<_Const, _Views...> {
235	227	using iterator_category = input_iterator_tag;
236	228	};
237	229
	230	struct __zip_view_iterator_access {
	231	template <class _Iter>
	232	_LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __get_underlying(_Iter& __iter) noexcept {
	233	return (__iter.__current_);
	234	}
	235	};
	236
238	237	template <input_range... _Views>
239	238	requires(view<_Views> && ...) && (sizeof...(_Views) > 0)
240	239	template <bool _Const>
...	...	@@ -255,6 +254,7 @@ class zip_view<_Views...>::__iterator : public __zip_view_iterator_category_base
255	254	static constexpr bool __is_zip_view_iterator = true;
256	255
257	256	friend struct __product_iterator_traits<__iterator>;
	257	friend __zip_view_iterator_access;
258	258
259	259	public:
260	260	using iterator_concept = decltype(ranges::__get_zip_view_iterator_tag<_Const, _Views...>());
...	...	@@ -268,7 +268,7 @@ public:
268	268	: __current_(std::move(__i.__current_)) {}
269	269
270	270	_LIBCPP_HIDE_FROM_ABI constexpr auto operator*() const {
271		return ranges::__tuple_transform([](auto& __i) -> decltype(auto) { return *__i; }, __current_);
	271	return std::__tuple_transform([](auto& __i) -> decltype(auto) { return *__i; }, __current_);
272	272	}
273	273
274	274	_LIBCPP_HIDE_FROM_ABI constexpr __iterator& operator++() {
...	...	@@ -318,7 +318,7 @@ public:
318	318	_LIBCPP_HIDE_FROM_ABI constexpr auto operator[](difference_type __n) const
319	319	requires __zip_all_random_access<_Const, _Views...>
320	320	{
321		return ranges::__tuple_transform(
	321	return std::__tuple_transform(
322	322	[&]<class _Iter>(_Iter& __i) -> decltype(auto) { return __i[iter_difference_t<_Iter>(__n)]; }, __current_);
323	323	}
324	324
...	...	@@ -377,7 +377,7 @@ public:
377	377	_LIBCPP_HIDE_FROM_ABI friend constexpr auto iter_move(const __iterator& __i) noexcept(
378	378	(noexcept(ranges::iter_move(std::declval<const iterator_t<__maybe_const<_Const, _Views>>&>())) && ...) &&
379	379	(is_nothrow_move_constructible_v<range_rvalue_reference_t<__maybe_const<_Const, _Views>>> && ...)) {
380		return ranges::__tuple_transform(ranges::iter_move, __i.__current_);
	380	return std::__tuple_transform(ranges::iter_move, __i.__current_);
381	381	}
382	382
383	383	_LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap(const __iterator& __l, const __iterator& __r) noexcept(

lib/libcxx/include/__split_buffer+612-240

...	...	@@ -13,10 +13,12 @@
13	13	#include <__algorithm/max.h>
14	14	#include <__algorithm/move.h>
15	15	#include <__algorithm/move_backward.h>
	16	#include <__assert>
16	17	#include <__config>
17	18	#include <__iterator/distance.h>
18	19	#include <__iterator/iterator_traits.h>
19	20	#include <__iterator/move_iterator.h>
	21	#include <__memory/addressof.h>
20	22	#include <__memory/allocate_at_least.h>
21	23	#include <__memory/allocator.h>
22	24	#include <__memory/allocator_traits.h>
...	...	@@ -28,11 +30,9 @@
28	30	#include <__type_traits/integral_constant.h>
29	31	#include <__type_traits/is_nothrow_assignable.h>
30	32	#include <__type_traits/is_nothrow_constructible.h>
31		#include <__type_traits/is_replaceable.h>
32	33	#include <__type_traits/is_swappable.h>
33	34	#include <__type_traits/is_trivially_destructible.h>
34	35	#include <__type_traits/is_trivially_relocatable.h>
35		#include <__type_traits/remove_reference.h>
36	36	#include <__utility/forward.h>
37	37	#include <__utility/move.h>
38	38
...	...	@@ -45,25 +45,430 @@ _LIBCPP_PUSH_MACROS
45	45
46	46	_LIBCPP_BEGIN_NAMESPACE_STD
47	47
48		// __split_buffer allocates a contiguous chunk of memory and stores objects in the range [__begin_, __end_).
49		// It has uninitialized memory in the ranges [__first_, __begin_) and [__end_, __cap_). That allows
50		// it to grow both in the front and back without having to move the data.
	48	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	49	class __split_buffer;
	50
	51	template <class _SplitBuffer, class _Tp, class _Allocator>
	52	class __split_buffer_pointer_layout {
	53	protected:
	54	using value_type = _Tp;
	55	using allocator_type = _Allocator;
	56	using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<allocator_type>;
	57	using reference = value_type&;
	58	using const_reference = const value_type&;
	59	using size_type = typename __alloc_traits::size_type;
	60	using difference_type = typename __alloc_traits::difference_type;
	61	using pointer = typename __alloc_traits::pointer;
	62	using const_pointer = typename __alloc_traits::const_pointer;
	63	using iterator = pointer;
	64	using const_iterator = const_pointer;
	65	using __sentinel_type _LIBCPP_NODEBUG = pointer;
51	66
52		template <class _Tp, class _Allocator = allocator<_Tp> >
53		struct __split_buffer {
54	67	public:
55		using value_type = _Tp;
56		using allocator_type = _Allocator;
57		using __alloc_rr _LIBCPP_NODEBUG = __libcpp_remove_reference_t<allocator_type>;
58		using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<__alloc_rr>;
59		using reference = value_type&;
60		using const_reference = const value_type&;
61		using size_type = typename __alloc_traits::size_type;
62		using difference_type = typename __alloc_traits::difference_type;
63		using pointer = typename __alloc_traits::pointer;
64		using const_pointer = typename __alloc_traits::const_pointer;
65		using iterator = pointer;
66		using const_iterator = const_pointer;
	68	// Can't be defaulted due to _LIBCPP_COMPRESSED_PAIR not being an aggregate in C++03 and C++11.
	69	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer_pointer_layout() : __back_cap_(nullptr) {}
	70
	71	_LIBCPP_CONSTEXPR_SINCE_CXX20
	72	_LIBCPP_HIDE_FROM_ABI explicit __split_buffer_pointer_layout(const allocator_type& __alloc)
	73	: __back_cap_(nullptr), __alloc_(__alloc) {}
	74
	75	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __front_cap() _NOEXCEPT { return __front_cap_; }
	76
	77	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __front_cap() const _NOEXCEPT {
	78	return __front_cap_;
	79	}
	80
	81	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT { return __begin_; }
	82
	83	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT { return __begin_; }
	84
	85	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() _NOEXCEPT { return __end_; }
	86
	87	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() const _NOEXCEPT { return __end_; }
	88
	89	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
	90	return static_cast<size_type>(__end_ - __begin_);
	91	}
	92
	93	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __begin_ == __end_; }
	94
	95	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
	96	return static_cast<size_type>(__back_cap_ - __front_cap_);
	97	}
	98
	99	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __get_allocator() _NOEXCEPT { return __alloc_; }
	100
	101	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type const& __get_allocator() const _NOEXCEPT {
	102	return __alloc_;
	103	}
	104
	105	// Returns the sentinel object directly. Should be used in conjunction with automatic type deduction,
	106	// not explicit types.
	107	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __sentinel_type __raw_sentinel() const _NOEXCEPT {
	108	return __end_;
	109	}
	110	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __sentinel_type __raw_capacity() const _NOEXCEPT {
	111	return __back_cap_;
	112	}
	113
	114	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_data(pointer __new_first) _NOEXCEPT {
	115	__front_cap_ = __new_first;
	116	}
	117
	118	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	119	__set_valid_range(pointer __new_begin, pointer __new_end) _NOEXCEPT {
	120	__begin_ = __new_begin;
	121	__end_ = __new_end;
	122	}
	123
	124	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	125	__set_valid_range(pointer __new_begin, size_type __new_size) _NOEXCEPT {
	126	__begin_ = __new_begin;
	127	__end_ = __begin_ + __new_size;
	128	}
	129
	130	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(pointer __new_end) _NOEXCEPT {
	131	_LIBCPP_ASSERT_INTERNAL(__front_cap_ <= __new_end, "__new_end cannot precede __front_cap_");
	132	__end_ = __new_end;
	133	}
	134
	135	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(size_type __new_size) _NOEXCEPT {
	136	__end_ = __begin_ + __new_size;
	137	}
	138
	139	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(size_type __new_capacity) _NOEXCEPT {
	140	__back_cap_ = __front_cap_ + __new_capacity;
	141	}
	142
	143	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(pointer __new_capacity) _NOEXCEPT {
	144	__back_cap_ = __new_capacity;
	145	}
	146
	147	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const _NOEXCEPT {
	148	return static_cast<size_type>(__begin_ - __front_cap_);
	149	}
	150
	151	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const _NOEXCEPT {
	152	return static_cast<size_type>(__back_cap_ - __end_);
	153	}
	154
	155	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT { return *(__end_ - 1); }
	156
	157	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT { return *(__end_ - 1); }
	158
	159	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(
	160	__split_buffer_pointer_layout<__split_buffer<value_type, allocator_type, __split_buffer_pointer_layout>,
	161	value_type,
	162	allocator_type>& __other) _NOEXCEPT {
	163	std::swap(__front_cap_, __other.__front_cap_);
	164	std::swap(__begin_, __other.__begin_);
	165	std::swap(__back_cap_, __other.__back_cap_);
	166	std::swap(__end_, __other.__end_);
	167	}
	168
	169	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer_pointer_layout& __other) _NOEXCEPT {
	170	std::swap(__front_cap_, __other.__front_cap_);
	171	std::swap(__begin_, __other.__begin_);
	172	std::swap(__back_cap_, __other.__back_cap_);
	173	std::swap(__end_, __other.__end_);
	174	std::__swap_allocator(__alloc_, __other.__alloc_);
	175	}
	176
	177	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset() _NOEXCEPT {
	178	__front_cap_ = nullptr;
	179	__begin_ = nullptr;
	180	__end_ = nullptr;
	181	__back_cap_ = nullptr;
	182	}
	183
	184	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	185	__copy_without_alloc(__split_buffer_pointer_layout const& __other)
	186	_NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value) {
	187	__front_cap_ = __other.__front_cap_;
	188	__begin_ = __other.__begin_;
	189	__end_ = __other.__end_;
	190	__back_cap_ = __other.__back_cap_;
	191	}
	192
	193	private:
	194	pointer __front_cap_ = nullptr;
	195	pointer __begin_ = nullptr;
	196	pointer __end_ = nullptr;
	197	_LIBCPP_COMPRESSED_PAIR(pointer, __back_cap_, allocator_type, __alloc_);
	198
	199	template <class, class, class>
	200	friend class __split_buffer_pointer_layout;
	201	};
	202
	203	template <class _SplitBuffer, class _Tp, class _Allocator>
	204	class __split_buffer_size_layout {
	205	protected:
	206	using value_type = _Tp;
	207	using allocator_type = _Allocator;
	208	using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<allocator_type>;
	209	using reference = value_type&;
	210	using const_reference = const value_type&;
	211	using size_type = typename __alloc_traits::size_type;
	212	using difference_type = typename __alloc_traits::difference_type;
	213	using pointer = typename __alloc_traits::pointer;
	214	using const_pointer = typename __alloc_traits::const_pointer;
	215	using iterator = pointer;
	216	using const_iterator = const_pointer;
	217	using __sentinel_type _LIBCPP_NODEBUG = size_type;
	218
	219	public:
	220	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer_size_layout() = default;
	221
	222	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer_size_layout(const allocator_type& __alloc)
	223	: __alloc_(__alloc) {}
	224
	225	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __front_cap() _NOEXCEPT { return __front_cap_; }
	226
	227	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer __front_cap() const _NOEXCEPT {
	228	return __front_cap_;
	229	}
	230
	231	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT { return __begin_; }
	232
	233	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT { return __begin_; }
	234
	235	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() _NOEXCEPT { return __begin_ + __size_; }
	236
	237	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() const _NOEXCEPT { return __begin_ + __size_; }
	238
	239	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
	240
	241	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __size_ == 0; }
	242
	243	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT { return __cap_; }
	244
	245	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type& __get_allocator() _NOEXCEPT { return __alloc_; }
	246
	247	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type const& __get_allocator() const _NOEXCEPT {
	248	return __alloc_;
	249	}
	250
	251	// Returns the sentinel object directly. Should be used in conjunction with automatic type deduction,
	252	// not explicit types.
	253	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __sentinel_type __raw_sentinel() const _NOEXCEPT {
	254	return __size_;
	255	}
	256	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __sentinel_type __raw_capacity() const _NOEXCEPT {
	257	return __cap_;
	258	}
	259
	260	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_data(pointer __new_first) _NOEXCEPT {
	261	__front_cap_ = __new_first;
	262	}
	263
	264	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	265	__set_valid_range(pointer __new_begin, pointer __new_end) _NOEXCEPT {
	266	// Size-based __split_buffers track their size directly: we need to explicitly update the size
	267	// when the front is adjusted.
	268	__size_ -= __new_begin - __begin_;
	269	__begin_ = __new_begin;
	270	__set_sentinel(__new_end);
	271	}
	272
	273	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	274	__set_valid_range(pointer __new_begin, size_type __new_size) _NOEXCEPT {
	275	// Size-based __split_buffers track their size directly: we need to explicitly update the size
	276	// when the front is adjusted.
	277	__size_ -= __new_begin - __begin_;
	278	__begin_ = __new_begin;
	279	__set_sentinel(__new_size);
	280	}
	281
	282	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(pointer __new_end) _NOEXCEPT {
	283	_LIBCPP_ASSERT_INTERNAL(__front_cap_ <= __new_end, "__new_end cannot precede __front_cap_");
	284	__size_ += __new_end - end();
	285	}
	286
	287	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_sentinel(size_type __new_size) _NOEXCEPT {
	288	__size_ = __new_size;
	289	}
	290
	291	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(size_type __new_capacity) _NOEXCEPT {
	292	__cap_ = __new_capacity;
	293	}
	294
	295	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __set_capacity(pointer __new_capacity) _NOEXCEPT {
	296	__cap_ = __new_capacity - __begin_;
	297	}
	298
	299	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const _NOEXCEPT {
	300	return static_cast<size_type>(__begin_ - __front_cap_);
	301	}
	302
	303	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const _NOEXCEPT {
	304	// `__cap_ - __end_` tells us the total number of spares when in size-mode. We need to remove
	305	// the __front_spare from the count.
	306	return __cap_ - __size_ - __front_spare();
	307	}
	308
	309	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT { return __begin_[__size_ - 1]; }
	310
	311	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
	312	return __begin_[__size_ - 1];
	313	}
	314
	315	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(
	316	__split_buffer_pointer_layout<__split_buffer<value_type, allocator_type, __split_buffer_pointer_layout>,
	317	value_type,
	318	allocator_type>& __other) _NOEXCEPT {
	319	std::swap(__front_cap_, __other.__front_cap_);
	320	std::swap(__begin_, __other.__begin_);
	321	std::swap(__cap_, __other.__cap_);
	322	std::swap(__size_, __other.__size_);
	323	}
	324
	325	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer_size_layout& __other) _NOEXCEPT {
	326	std::swap(__front_cap_, __other.__front_cap_);
	327	std::swap(__begin_, __other.__begin_);
	328	std::swap(__cap_, __other.__cap_);
	329	std::swap(__size_, __other.__size_);
	330	std::__swap_allocator(__alloc_, __other.__alloc_);
	331	}
	332
	333	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset() _NOEXCEPT {
	334	__front_cap_ = nullptr;
	335	__begin_ = nullptr;
	336	__size_ = 0;
	337	__cap_ = 0;
	338	}
	339
	340	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	341	__copy_without_alloc(__split_buffer_size_layout const& __other)
	342	_NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value) {
	343	__front_cap_ = __other.__front_cap_;
	344	__begin_ = __other.__begin_;
	345	__cap_ = __other.__cap_;
	346	__size_ = __other.__size_;
	347	}
	348
	349	private:
	350	pointer __front_cap_ = nullptr;
	351	pointer __begin_ = nullptr;
	352	size_type __size_ = 0;
	353	size_type __cap_ = 0;
	354	_LIBCPP_NO_UNIQUE_ADDRESS allocator_type __alloc_;
	355
	356	template <class, class, class>
	357	friend class __split_buffer_size_layout;
	358	};
	359
	360	// `__split_buffer` is a contiguous array data structure. It may hold spare capacity at both ends of
	361	// the sequence. This allows for a `__split_buffer` to grow from both the front and the back without
	362	// relocating its contents until it runs out of room. This characteristic sets it apart from
	363	// `std::vector`, which only holds spare capacity at its end. As such, `__split_buffer` is useful
	364	// for implementing both `std::vector` and `std::deque`.
	365	//
	366	// The sequence is stored as a contiguous chunk of memory delimited by the following "pointers" (`o` denotes
	367	// uninitialized memory and `x` denotes a valid object):
	368	//
	369	// \|oooooooooooooooooooxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxoooooooooooooooooooooooo\|
	370	// ^ ^ ^ ^
	371	// __front_cap_ __begin_ __end_ __back_cap_
	372	//
	373	// The range [__front_cap_, __begin_) contains uninitialized memory. It is referred to as the "front spare capacity".
	374	// The range [__begin_, __end_) contains valid objects. It is referred to as the "valid range".
	375	// The range [__end_, __back_cap_) contains uninitialized memory. It is referred to as the "back spare capacity".
	376	//
	377	// The layout of `__split_buffer` is determined by the `_Layout` template template parameter. This
	378	// `_Layout` allows the above pointers to be stored as different representations, such as integer
	379	// offsets. A layout class template must provide the following interface:
	380	//
	381	// template<class _Tp, class _Allocator, class _Layout>
	382	// class __layout {
	383	// protected:
	384	// using value_type = _Tp;
	385	// using allocator_type = _Allocator;
	386	// using __alloc_traits = allocator_traits<allocator_type>;
	387	// using reference = value_type&;
	388	// using const_reference = const value_type&;
	389	// using size_type = typename __alloc_traits::size_type;
	390	// using difference_type = typename __alloc_traits::difference_type;
	391	// using pointer = typename __alloc_traits::pointer;
	392	// using const_pointer = typename __alloc_traits::const_pointer;
	393	// using iterator = pointer;
	394	// using const_iterator = const_pointer;
	395	// using __sentinel_type = /* type that represents the layout's sentinel */;
	396	//
	397	// public:
	398	// __layout() = default;
	399	// explicit __layout(const allocator_type&);
	400	//
	401	// pointer __front_cap();
	402	// const_pointer __front_cap() const;
	403	//
	404	// pointer begin();
	405	// const_pointer begin() const;
	406	//
	407	// pointer end();
	408	// pointer end() const;
	409	//
	410	// size_type size() const;
	411	// bool empty() const;
	412	// size_type capacity() const;
	413	//
	414	// allocator_type& __get_allocator();
	415	// allocator_type const& __get_allocator() const;
	416	//
	417	// __sentinel_type __raw_sentinel() const;
	418	// __sentinel_type __raw_capacity() const;
	419	//
	420	// void __set_data(pointer);
	421	// void __set_valid_range(pointer __begin, pointer __end);
	422	// void __set_valid_range(pointer __begin, size_type __size);
	423	// void __set_sentinel(pointer __end);
	424	// void __set_sentinel(size_type __size);
	425	//
	426	// void __set_capacity(size_type __capacity);
	427	// void __set_capacity(pointer __capacity);
	428	//
	429	// size_type __front_spare() const;
	430	// size_type __back_spare() const;
	431	//
	432	// reference back();
	433	// const_reference back() const;
	434	//
	435	// template<class _OtherLayout>
	436	// void __swap_without_allocator(_OtherLayout&);
	437	// void swap(__layout&);
	438	//
	439	// void __reset();
	440	// void __copy_without_alloc(__layout const&);
	441	// };
	442	//
	443	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	444	class __split_buffer : _Layout<__split_buffer<_Tp, _Allocator, _Layout>, _Tp, _Allocator> {
	445	using __base_type _LIBCPP_NODEBUG = _Layout<__split_buffer<_Tp, _Allocator, _Layout>, _Tp, _Allocator>;
	446
	447	public:
	448	using __base_type::__back_spare;
	449	using __base_type::__copy_without_alloc;
	450	using __base_type::__front_cap;
	451	using __base_type::__front_spare;
	452	using __base_type::__get_allocator;
	453	using __base_type::__raw_capacity;
	454	using __base_type::__raw_sentinel;
	455	using __base_type::__reset;
	456	using __base_type::__set_capacity;
	457	using __base_type::__set_data;
	458	using __base_type::__set_sentinel;
	459	using __base_type::__set_valid_range;
	460
	461	using typename __base_type::__alloc_traits;
	462	using typename __base_type::allocator_type;
	463	using typename __base_type::const_iterator;
	464	using typename __base_type::const_pointer;
	465	using typename __base_type::const_reference;
	466	using typename __base_type::difference_type;
	467	using typename __base_type::iterator;
	468	using typename __base_type::pointer;
	469	using typename __base_type::reference;
	470	using typename __base_type::size_type;
	471	using typename __base_type::value_type;
67	472
68	473	// A __split_buffer contains the following members which may be trivially relocatable:
69	474	// - pointer: may be trivially relocatable, so it's checked
...	...	@@ -73,36 +478,24 @@ public:
73	478	__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
74	479	__split_buffer,
75	480	void>;
76		using __replaceable _LIBCPP_NODEBUG =
77		__conditional_t<__is_replaceable_v<pointer> && __container_allocator_is_replaceable<__alloc_traits>::value,
78		__split_buffer,
79		void>;
80
81		pointer __first_;
82		pointer __begin_;
83		pointer __end_;
84		_LIBCPP_COMPRESSED_PAIR(pointer, __cap_, allocator_type, __alloc_);
85	481
86	482	__split_buffer(const __split_buffer&) = delete;
87	483	__split_buffer& operator=(const __split_buffer&) = delete;
88	484
89		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer()
90		_NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
91		: __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr) {}
	485	_LIBCPP_HIDE_FROM_ABI __split_buffer() = default;
92	486
93		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a)
94		: __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
	487	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(allocator_type& __a) : __base_type(__a) {}
95	488
96		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const __alloc_rr& __a)
97		: __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
	489	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const allocator_type& __a)
	490	: __base_type(__a) {}
98	491
99	492	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
100		__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a);
	493	__split_buffer(size_type __cap, size_type __start, allocator_type& __a);
101	494
102	495	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer(__split_buffer&& __c)
103	496	_NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value);
104	497
105		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer(__split_buffer&& __c, const __alloc_rr& __a);
	498	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer(__split_buffer&& __c, const allocator_type& __a);
106	499
107	500	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer& operator=(__split_buffer&& __c)
108	501	_NOEXCEPT_((__alloc_traits::propagate_on_container_move_assignment::value &&
...	...	@@ -111,36 +504,16 @@ public:
111	504
112	505	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~__split_buffer();
113	506
114		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __begin_; }
115		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __begin_; }
116
117		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __end_; }
118		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __end_; }
119
120		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(__begin_); }
	507	using __base_type::back;
	508	using __base_type::begin;
	509	using __base_type::capacity;
	510	using __base_type::empty;
	511	using __base_type::end;
	512	using __base_type::size;
121	513
122		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const {
123		return static_cast<size_type>(__end_ - __begin_);
124		}
125
126		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const { return __end_ == __begin_; }
127
128		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const {
129		return static_cast<size_type>(__cap_ - __first_);
130		}
131
132		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const {
133		return static_cast<size_type>(__begin_ - __first_);
134		}
135
136		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const {
137		return static_cast<size_type>(__cap_ - __end_);
138		}
139
140		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *__begin_; }
141		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *__begin_; }
142		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() { return *(__end_ - 1); }
143		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const { return *(__end_ - 1); }
	514	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(begin()); }
	515	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *begin(); }
	516	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *begin(); }
144	517
145	518	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
146	519
...	...	@@ -149,8 +522,8 @@ public:
149	522	template <class... _Args>
150	523	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
151	524
152		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(__begin_ + 1); }
153		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(__end_ - 1); }
	525	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(begin() + 1); }
	526	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(end() - 1); }
154	527
155	528	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
156	529	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
...	...	@@ -182,244 +555,242 @@ public:
182	555	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __destruct_at_end(pointer __new_last, true_type) _NOEXCEPT;
183	556
184	557	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__split_buffer& __x)
185		_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value \|\| __is_nothrow_swappable_v<__alloc_rr>);
	558	_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value \|\| __is_nothrow_swappable_v<allocator_type>);
	559
	560	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const {
	561	if (__front_cap() == nullptr) {
	562	if (begin() != nullptr)
	563	return false;
	564
	565	if (!empty())
	566	return false;
	567
	568	if (capacity() != 0)
	569	return false;
	570
	571	return true;
	572	} else {
	573	if (begin() < __front_cap())
	574	return false;
186	575
187		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
	576	if (capacity() < size())
	577	return false;
	578
	579	if (end() < begin())
	580	return false;
	581
	582	return true;
	583	}
	584	}
	585
	586	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
	587	__swap_without_allocator(__split_buffer<value_type, allocator_type, _Layout>& __other) _NOEXCEPT {
	588	__base_type::__swap_without_allocator(__other);
	589	}
188	590
189	591	private:
190	592	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer& __c, true_type)
191	593	_NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
192		__alloc_ = std::move(__c.__alloc_);
	594	__get_allocator() = std::move(__c.__get_allocator());
193	595	}
194	596
195	597	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer&, false_type) _NOEXCEPT {}
196	598
197	599	struct _ConstructTransaction {
198	600	_LIBCPP_CONSTEXPR_SINCE_CXX20
199		_LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(pointer* __p, size_type __n) _NOEXCEPT
200		: __pos_(*__p),
201		__end_(*__p + __n),
202		__dest_(__p) {}
	601	_LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(__split_buffer* __parent, pointer __p, size_type __n) _NOEXCEPT
	602	: __pos_(__p),
	603	__end_(__p + __n),
	604	__parent_(__parent) {}
203	605
204		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { *__dest_ = __pos_; }
	606	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { __parent_->__set_sentinel(__pos_); }
205	607
206	608	pointer __pos_;
207	609	const pointer __end_;
208	610
209	611	private:
210		pointer* __dest_;
	612	__split_buffer* __parent_;
211	613	};
212		};
213	614
214		template <class _Tp, class _Allocator>
215		_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __split_buffer<_Tp, _Allocator>::__invariants() const {
216		if (__first_ == nullptr) {
217		if (__begin_ != nullptr)
218		return false;
219		if (__end_ != nullptr)
220		return false;
221		if (__cap_ != nullptr)
222		return false;
223		} else {
224		if (__begin_ < __first_)
225		return false;
226		if (__end_ < __begin_)
227		return false;
228		if (__cap_ < __end_)
229		return false;
230		}
231		return true;
232		}
	615	template <class _T2, class _A2, template <class, class, class> class _L2>
	616	friend class __split_buffer;
	617	};
233	618
234		// Default constructs __n objects starting at __end_
	619	// Default constructs __n objects starting at `end()`
235	620	// throws if construction throws
236	621	// Precondition: __n > 0
237	622	// Precondition: size() + __n <= capacity()
238	623	// Postcondition: size() == size() + __n
239		template <class _Tp, class _Allocator>
240		_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n) {
241		_ConstructTransaction __tx(std::addressof(this->__end_), __n);
	624	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	625	_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator, _Layout>::__construct_at_end(size_type __n) {
	626	_ConstructTransaction __tx(this, end(), __n);
242	627	for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
243		__alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_));
	628	__alloc_traits::construct(__get_allocator(), std::__to_address(__tx.__pos_));
244	629	}
245	630	}
246	631
247		// Copy constructs __n objects starting at __end_ from __x
	632	// Copy constructs __n objects starting at `end()` from __x
248	633	// throws if construction throws
249	634	// Precondition: __n > 0
250	635	// Precondition: size() + __n <= capacity()
251	636	// Postcondition: size() == old size() + __n
252	637	// Postcondition: [i] == __x for all i in [size() - __n, __n)
253		template <class _Tp, class _Allocator>
	638	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
254	639	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
255		__split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
256		_ConstructTransaction __tx(std::addressof(this->__end_), __n);
	640	__split_buffer<_Tp, _Allocator, _Layout>::__construct_at_end(size_type __n, const_reference __x) {
	641	_ConstructTransaction __tx(this, end(), __n);
257	642	for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
258		__alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_), __x);
	643	__alloc_traits::construct(__get_allocator(), std::__to_address(__tx.__pos_), __x);
259	644	}
260	645	}
261	646
262		template <class _Tp, class _Allocator>
	647	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
263	648	template <class _Iterator, class _Sentinel>
264	649	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
265		__split_buffer<_Tp, _Allocator>::__construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last) {
266		__alloc_rr& __a = __alloc_;
	650	__split_buffer<_Tp, _Allocator, _Layout>::__construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last) {
	651	allocator_type& __a = __get_allocator();
267	652	for (; __first != __last; ++__first) {
268		if (__end_ == __cap_) {
269		size_type __old_cap = __cap_ - __first_;
	653	if (__back_spare() == 0) {
	654	size_type __old_cap = capacity();
270	655	size_type __new_cap = std::max<size_type>(2 * __old_cap, 8);
271	656	__split_buffer __buf(__new_cap, 0, __a);
272		for (pointer __p = __begin_; __p != __end_; ++__p, (void)++__buf.__end_)
273		__alloc_traits::construct(__buf.__alloc_, std::__to_address(__buf.__end_), std::move(*__p));
	657	pointer __buf_end = __buf.end();
	658	pointer __end = end();
	659	for (pointer __p = begin(); __p != __end; ++__p) {
	660	__alloc_traits::construct(__buf.__get_allocator(), std::__to_address(__buf_end), std::move(*__p));
	661	__buf.__set_sentinel(++__buf_end);
	662	}
274	663	swap(__buf);
275	664	}
276		__alloc_traits::construct(__a, std::__to_address(this->__end_), *__first);
277		++this->__end_;
	665
	666	__alloc_traits::construct(__a, std::__to_address(end()), *__first);
	667	__set_sentinel(size() + 1);
278	668	}
279	669	}
280		template <class _Tp, class _Allocator>
	670
	671	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
281	672	template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
282	673	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
283		__split_buffer<_Tp, _Allocator>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last) {
	674	__split_buffer<_Tp, _Allocator, _Layout>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last) {
284	675	__construct_at_end_with_size(__first, std::distance(__first, __last));
285	676	}
286	677
287		template <class _Tp, class _Allocator>
	678	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
288	679	template <class _ForwardIterator>
289	680	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
290		__split_buffer<_Tp, _Allocator>::__construct_at_end_with_size(_ForwardIterator __first, size_type __n) {
291		_ConstructTransaction __tx(std::addressof(this->__end_), __n);
	681	__split_buffer<_Tp, _Allocator, _Layout>::__construct_at_end_with_size(_ForwardIterator __first, size_type __n) {
	682	_ConstructTransaction __tx(this, end(), __n);
292	683	for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_, (void)++__first) {
293		__alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_), *__first);
	684	__alloc_traits::construct(__get_allocator(), std::__to_address(__tx.__pos_), *__first);
294	685	}
295	686	}
296	687
297		template <class _Tp, class _Allocator>
	688	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
298	689	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
299		__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, false_type) {
300		while (__begin_ != __new_begin)
301		__alloc_traits::destroy(__alloc_, std::__to_address(__begin_++));
	690	__split_buffer<_Tp, _Allocator, _Layout>::__destruct_at_begin(pointer __new_begin, false_type) {
	691	pointer __begin = begin();
	692	// Updating begin at every iteration is unnecessary because destruction can't throw.
	693	while (__begin != __new_begin)
	694	__alloc_traits::destroy(__get_allocator(), std::__to_address(__begin++));
	695	__set_valid_range(__begin, end());
302	696	}
303	697
304		template <class _Tp, class _Allocator>
	698	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
305	699	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
306		__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, true_type) {
307		__begin_ = __new_begin;
308		}
309
310		template <class _Tp, class _Allocator>
311		_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
312		__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, false_type) _NOEXCEPT {
313		while (__new_last != __end_)
314		__alloc_traits::destroy(__alloc_, std::__to_address(--__end_));
	700	__split_buffer<_Tp, _Allocator, _Layout>::__destruct_at_begin(pointer __new_begin, true_type) {
	701	__set_valid_range(__new_begin, end());
315	702	}
316	703
317		template <class _Tp, class _Allocator>
	704	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
318	705	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
319		__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, true_type) _NOEXCEPT {
320		__end_ = __new_last;
	706	__split_buffer<_Tp, _Allocator, _Layout>::__destruct_at_end(pointer __new_last, false_type) _NOEXCEPT {
	707	pointer __end = end();
	708	// Updating begin at every iteration is unnecessary because destruction can't throw.
	709	while (__new_last != __end)
	710	__alloc_traits::destroy(__get_allocator(), std::__to_address(--__end));
	711	__set_sentinel(__end);
321	712	}
322	713
323		template <class _Tp, class _Allocator>
	714	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
324	715	_LIBCPP_CONSTEXPR_SINCE_CXX20
325		__split_buffer<_Tp, _Allocator>::__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a)
326		: __cap_(nullptr), __alloc_(__a) {
327		if (__cap == 0) {
328		__first_ = nullptr;
329		} else {
330		auto __allocation = std::__allocate_at_least(__alloc_, __cap);
331		__first_ = __allocation.ptr;
332		__cap = __allocation.count;
	716	__split_buffer<_Tp, _Allocator, _Layout>::__split_buffer(size_type __cap, size_type __start, allocator_type& __a)
	717	: __base_type(__a) {
	718	_LIBCPP_ASSERT_INTERNAL(__cap >= __start, "can't have a start point outside the capacity");
	719	if (__cap > 0) {
	720	auto __allocation = std::__allocate_at_least(__get_allocator(), __cap);
	721	__set_data(__allocation.ptr);
	722	__cap = __allocation.count;
333	723	}
334		__begin_ = __end_ = __first_ + __start;
335		__cap_ = __first_ + __cap;
	724
	725	pointer __begin = __front_cap() + __start;
	726	__set_valid_range(__begin, __begin);
	727	__set_capacity(__cap);
336	728	}
337	729
338		template <class _Tp, class _Allocator>
339		_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>::~__split_buffer() {
	730	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	731	_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator, _Layout>::~__split_buffer() {
340	732	clear();
341		if (__first_)
342		__alloc_traits::deallocate(__alloc_, __first_, capacity());
	733	if (__front_cap())
	734	__alloc_traits::deallocate(__get_allocator(), __front_cap(), capacity());
343	735	}
344	736
345		template <class _Tp, class _Allocator>
346		_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c)
	737	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	738	_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator, _Layout>::__split_buffer(__split_buffer&& __c)
347	739	_NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
348		: __first_(std::move(__c.__first_)),
349		__begin_(std::move(__c.__begin_)),
350		__end_(std::move(__c.__end_)),
351		__cap_(std::move(__c.__cap_)),
352		__alloc_(std::move(__c.__alloc_)) {
353		__c.__first_ = nullptr;
354		__c.__begin_ = nullptr;
355		__c.__end_ = nullptr;
356		__c.__cap_ = nullptr;
	740	: __base_type(std::move(__c)) {
	741	__c.__reset();
357	742	}
358	743
359		template <class _Tp, class _Allocator>
	744	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
360	745	_LIBCPP_CONSTEXPR_SINCE_CXX20
361		__split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c, const __alloc_rr& __a)
362		: __cap_(nullptr), __alloc_(__a) {
363		if (__a == __c.__alloc_) {
364		__first_ = __c.__first_;
365		__begin_ = __c.__begin_;
366		__end_ = __c.__end_;
367		__cap_ = __c.__cap_;
368		__c.__first_ = nullptr;
369		__c.__begin_ = nullptr;
370		__c.__end_ = nullptr;
371		__c.__cap_ = nullptr;
	746	__split_buffer<_Tp, _Allocator, _Layout>::__split_buffer(__split_buffer&& __c, const allocator_type& __a)
	747	: __base_type(__a) {
	748	if (__a == __c.__get_allocator()) {
	749	__set_data(__c.__front_cap());
	750	__set_valid_range(__c.begin(), __c.end());
	751	__set_capacity(__c.capacity());
	752	__c.__reset();
372	753	} else {
373		auto __allocation = std::__allocate_at_least(__alloc_, __c.size());
374		__first_ = __allocation.ptr;
375		__begin_ = __end_ = __first_;
376		__cap_ = __first_ + __allocation.count;
	754	auto __allocation = std::__allocate_at_least(__get_allocator(), __c.size());
	755	__set_data(__allocation.ptr);
	756	__set_valid_range(__front_cap(), __front_cap());
	757	__set_capacity(__allocation.count);
377	758	typedef move_iterator<iterator> _Ip;
378	759	__construct_at_end(_Ip(__c.begin()), _Ip(__c.end()));
379	760	}
380	761	}
381	762
382		template <class _Tp, class _Allocator>
383		_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator>&
384		__split_buffer<_Tp, _Allocator>::operator=(__split_buffer&& __c)
	763	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	764	_LIBCPP_CONSTEXPR_SINCE_CXX20 __split_buffer<_Tp, _Allocator, _Layout>&
	765	__split_buffer<_Tp, _Allocator, _Layout>::operator=(__split_buffer&& __c)
385	766	_NOEXCEPT_((__alloc_traits::propagate_on_container_move_assignment::value &&
386	767	is_nothrow_move_assignable<allocator_type>::value) \|\|
387	768	!__alloc_traits::propagate_on_container_move_assignment::value) {
388	769	clear();
389	770	shrink_to_fit();
390		__first_ = __c.__first_;
391		__begin_ = __c.__begin_;
392		__end_ = __c.__end_;
393		__cap_ = __c.__cap_;
	771	__copy_without_alloc(__c);
394	772	__move_assign_alloc(__c, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
395		__c.__first_ = __c.__begin_ = __c.__end_ = __c.__cap_ = nullptr;
	773	__c.__reset();
396	774	return *this;
397	775	}
398	776
399		template <class _Tp, class _Allocator>
400		_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::swap(__split_buffer& __x)
401		_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value \|\| __is_nothrow_swappable_v<__alloc_rr>) {
402		std::swap(__first_, __x.__first_);
403		std::swap(__begin_, __x.__begin_);
404		std::swap(__end_, __x.__end_);
405		std::swap(__cap_, __x.__cap_);
406		std::__swap_allocator(__alloc_, __x.__alloc_);
	777	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	778	_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator, _Layout>::swap(__split_buffer& __x)
	779	_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value \|\| __is_nothrow_swappable_v<allocator_type>) {
	780	__base_type::swap(__x);
407	781	}
408	782
409		template <class _Tp, class _Allocator>
410		_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::shrink_to_fit() _NOEXCEPT {
	783	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
	784	_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator, _Layout>::shrink_to_fit() _NOEXCEPT {
411	785	if (capacity() > size()) {
412	786	#if _LIBCPP_HAS_EXCEPTIONS
413	787	try {
414	788	#endif // _LIBCPP_HAS_EXCEPTIONS
415		__split_buffer<value_type, __alloc_rr&> __t(size(), 0, __alloc_);
	789	__split_buffer<value_type, allocator_type, _Layout> __t(size(), 0, __get_allocator());
416	790	if (__t.capacity() < capacity()) {
417		__t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
418		__t.__end_ = __t.__begin_ + (__end_ - __begin_);
419		std::swap(__first_, __t.__first_);
420		std::swap(__begin_, __t.__begin_);
421		std::swap(__end_, __t.__end_);
422		std::swap(__cap_, __t.__cap_);
	791	__t.__construct_at_end(move_iterator<pointer>(begin()), move_iterator<pointer>(end()));
	792	__t.__set_sentinel(size());
	793	__swap_without_allocator(__t);
423	794	}
424	795	#if _LIBCPP_HAS_EXCEPTIONS
425	796	} catch (...) {
...	...	@@ -428,55 +799,56 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::shrink_to_fi
428	799	}
429	800	}
430	801
431		template <class _Tp, class _Allocator>
	802	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
432	803	template <class... _Args>
433		_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_front(_Args&&... __args) {
434		if (__begin_ == __first_) {
435		if (__end_ < __cap_) {
436		difference_type __d = __cap_ - __end_;
	804	_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator, _Layout>::emplace_front(_Args&&... __args) {
	805	if (__front_spare() == 0) {
	806	pointer __end = end();
	807	if (__back_spare() > 0) {
	808	// The elements are pressed up against the front of the buffer: we need to move them back a
	809	// little bit to make `emplace_front` have amortised O(1) complexity.
	810	difference_type __d = __back_spare();
437	811	__d = (__d + 1) / 2;
438		__begin_ = std::move_backward(__begin_, __end_, __end_ + __d);
439		__end_ += __d;
	812	auto __new_end = __end + __d;
	813	__set_valid_range(std::move_backward(begin(), __end, __new_end), __new_end);
440	814	} else {
441		size_type __c = std::max<size_type>(2 * static_cast<size_type>(__cap_ - __first_), 1);
442		__split_buffer<value_type, __alloc_rr&> __t(__c, (__c + 3) / 4, __alloc_);
443		__t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
444		std::swap(__first_, __t.__first_);
445		std::swap(__begin_, __t.__begin_);
446		std::swap(__end_, __t.__end_);
447		std::swap(__cap_, __t.__cap_);
	815	size_type __c = std::max<size_type>(2 * capacity(), 1);
	816	__split_buffer<value_type, allocator_type, _Layout> __t(__c, (__c + 3) / 4, __get_allocator());
	817	__t.__construct_at_end(move_iterator<pointer>(begin()), move_iterator<pointer>(__end));
	818	__base_type::__swap_without_allocator(__t);
448	819	}
449	820	}
450		__alloc_traits::construct(__alloc_, std::__to_address(__begin_ - 1), std::forward<_Args>(__args)...);
451		--__begin_;
	821
	822	__alloc_traits::construct(__get_allocator(), std::__to_address(begin() - 1), std::forward<_Args>(__args)...);
	823	__set_valid_range(begin() - 1, size() + 1);
452	824	}
453	825
454		template <class _Tp, class _Allocator>
	826	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
455	827	template <class... _Args>
456		_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
457		if (__end_ == __cap_) {
458		if (__begin_ > __first_) {
459		difference_type __d = __begin_ - __first_;
	828	_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator, _Layout>::emplace_back(_Args&&... __args) {
	829	pointer __end = end();
	830	if (__back_spare() == 0) {
	831	if (__front_spare() > 0) {
	832	difference_type __d = __front_spare();
460	833	__d = (__d + 1) / 2;
461		__end_ = std::move(__begin_, __end_, __begin_ - __d);
462		__begin_ -= __d;
	834	__end = std::move(begin(), __end, begin() - __d);
	835	__set_valid_range(begin() - __d, __end);
463	836	} else {
464		size_type __c = std::max<size_type>(2 * static_cast<size_type>(__cap_ - __first_), 1);
465		__split_buffer<value_type, __alloc_rr&> __t(__c, __c / 4, __alloc_);
466		__t.__construct_at_end(move_iterator<pointer>(__begin_), move_iterator<pointer>(__end_));
467		std::swap(__first_, __t.__first_);
468		std::swap(__begin_, __t.__begin_);
469		std::swap(__end_, __t.__end_);
470		std::swap(__cap_, __t.__cap_);
	837	size_type __c = std::max<size_type>(2 * capacity(), 1);
	838	__split_buffer<value_type, allocator_type, _Layout> __t(__c, __c / 4, __get_allocator());
	839	__t.__construct_at_end(move_iterator<pointer>(begin()), move_iterator<pointer>(__end));
	840	__base_type::__swap_without_allocator(__t);
471	841	}
472	842	}
473		__alloc_traits::construct(__alloc_, std::__to_address(__end_), std::forward<_Args>(__args)...);
474		++__end_;
	843
	844	__alloc_traits::construct(__get_allocator(), std::__to_address(__end), std::forward<_Args>(__args)...);
	845	__set_sentinel(++__end);
475	846	}
476	847
477		template <class _Tp, class _Allocator>
	848	template <class _Tp, class _Allocator, template <class, class, class> class _Layout>
478	849	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
479		swap(__split_buffer<_Tp, _Allocator>& __x, __split_buffer<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
	850	swap(__split_buffer<_Tp, _Allocator, _Layout>& __x, __split_buffer<_Tp, _Allocator, _Layout>& __y)
	851	_NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
480	852	__x.swap(__y);
481	853	}
482	854

lib/libcxx/include/__stop_token/atomic_unique_lock.h+1-1

...	...	@@ -27,7 +27,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
27	27	// where State contains a lock bit and might contain other data,
28	28	// and LockedBit is the value of State when the lock bit is set, e.g 1 << 2
29	29	template <class _State, _State _LockedBit>
30		class _LIBCPP_AVAILABILITY_SYNC __atomic_unique_lock {
	30	class __atomic_unique_lock {
31	31	static_assert(std::__popcount(static_cast<unsigned long long>(_LockedBit)) == 1,
32	32	"LockedBit must be an integer where only one bit is set");
33	33

lib/libcxx/include/__stop_token/stop_callback.h+2-2

...	...	@@ -34,7 +34,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
34	34	#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
35	35
36	36	template <class _Callback>
37		class _LIBCPP_AVAILABILITY_SYNC stop_callback : private __stop_callback_base {
	37	class stop_callback : private __stop_callback_base {
38	38	static_assert(invocable<_Callback>,
39	39	"Mandates: stop_callback is instantiated with an argument for the template parameter Callback that "
40	40	"satisfies invocable.");
...	...	@@ -91,7 +91,7 @@ private:
91	91	};
92	92
93	93	template <class _Callback>
94		_LIBCPP_AVAILABILITY_SYNC stop_callback(stop_token, _Callback) -> stop_callback<_Callback>;
	94	stop_callback(stop_token, _Callback) -> stop_callback<_Callback>;
95	95
96	96	#endif // _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
97	97

lib/libcxx/include/__stop_token/stop_source.h+1-1

...	...	@@ -30,7 +30,7 @@ struct nostopstate_t {
30	30
31	31	inline constexpr nostopstate_t nostopstate{};
32	32
33		class _LIBCPP_AVAILABILITY_SYNC stop_source {
	33	class stop_source {
34	34	public:
35	35	_LIBCPP_HIDE_FROM_ABI stop_source() : __state_(new __stop_state()) { __state_->__increment_stop_source_counter(); }
36	36

lib/libcxx/include/__stop_token/stop_state.h+4-4

...	...	@@ -100,7 +100,7 @@ public:
100	100	return ((__curent_state & __stop_requested_bit) != 0) \|\| ((__curent_state >> __stop_source_counter_shift) != 0);
101	101	}
102	102
103		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __request_stop() noexcept {
	103	_LIBCPP_HIDE_FROM_ABI bool __request_stop() noexcept {
104	104	auto __cb_list_lock = __try_lock_for_request_stop();
105	105	if (!__cb_list_lock.__owns_lock()) {
106	106	return false;
...	...	@@ -137,7 +137,7 @@ public:
137	137	return true;
138	138	}
139	139
140		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __add_callback(__stop_callback_base* __cb) noexcept {
	140	_LIBCPP_HIDE_FROM_ABI bool __add_callback(__stop_callback_base* __cb) noexcept {
141	141	// If it is already stop_requested. Do not try to request it again.
142	142	const auto __give_up_trying_to_lock_condition = [__cb](__state_t __state) {
143	143	if ((__state & __stop_requested_bit) != 0) {
...	...	@@ -164,7 +164,7 @@ public:
164	164	}
165	165
166	166	// called by the destructor of stop_callback
167		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void __remove_callback(__stop_callback_base* __cb) noexcept {
	167	_LIBCPP_HIDE_FROM_ABI void __remove_callback(__stop_callback_base* __cb) noexcept {
168	168	__callback_list_lock __cb_list_lock(__state_);
169	169
170	170	// under below condition, the request_stop call just popped __cb from the list and could execute it now
...	...	@@ -192,7 +192,7 @@ public:
192	192	}
193	193
194	194	private:
195		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI __callback_list_lock __try_lock_for_request_stop() noexcept {
	195	_LIBCPP_HIDE_FROM_ABI __callback_list_lock __try_lock_for_request_stop() noexcept {
196	196	// If it is already stop_requested, do not try to request stop or lock the list again.
197	197	const auto __lock_fail_condition = [](__state_t __state) { return (__state & __stop_requested_bit) != 0; };
198	198

lib/libcxx/include/__stop_token/stop_token.h+1-1

...	...	@@ -22,7 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
22	22
23	23	#if _LIBCPP_STD_VER >= 20 && _LIBCPP_HAS_THREADS
24	24
25		class _LIBCPP_AVAILABILITY_SYNC stop_token {
	25	class stop_token {
26	26	public:
27	27	_LIBCPP_HIDE_FROM_ABI stop_token() noexcept = default;
28	28

lib/libcxx/include/__string/char_traits.h+51-31

...	...	@@ -94,16 +94,17 @@ struct char_traits<char> {
94	94	}
95	95
96	96	// TODO: Make this _LIBCPP_HIDE_FROM_ABI
97		static inline _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR bool eq(char_type __c1, char_type __c2) _NOEXCEPT {
	97	[[__nodiscard__]] static inline _LIBCPP_HIDDEN _LIBCPP_CONSTEXPR bool eq(char_type __c1, char_type __c2) _NOEXCEPT {
98	98	return __c1 == __c2;
99	99	}
100		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool lt(char_type __c1, char_type __c2) _NOEXCEPT {
	100	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
	101	lt(char_type __c1, char_type __c2) _NOEXCEPT {
101	102	return (unsigned char)__c1 < (unsigned char)__c2;
102	103	}
103	104
104	105	// __constexpr_memcmp requires a trivially lexicographically comparable type, but char is not when char is a signed
105	106	// type
106		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	107	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
107	108	compare(const char_type* __lhs, const char_type* __rhs, size_t __count) _NOEXCEPT {
108	109	if (__libcpp_is_constant_evaluated()) {
109	110	#ifdef _LIBCPP_COMPILER_CLANG_BASED
...	...	@@ -126,11 +127,12 @@ struct char_traits<char> {
126	127	}
127	128	}
128	129
129		static inline _LIBCPP_HIDE_FROM_ABI size_t _LIBCPP_CONSTEXPR_SINCE_CXX17 length(const char_type* __s) _NOEXCEPT {
	130	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI size_t _LIBCPP_CONSTEXPR_SINCE_CXX17
	131	length(const char_type* __s) _NOEXCEPT {
130	132	return std::__constexpr_strlen(__s);
131	133	}
132	134
133		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
	135	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
134	136	find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
135	137	return std::__constexpr_memchr(__s, __a, __n);
136	138	}
...	...	@@ -154,19 +156,24 @@ struct char_traits<char> {
154	156	return __s;
155	157	}
156	158
157		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
	159	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
158	160	return eq_int_type(__c, eof()) ? ~eof() : __c;
159	161	}
160		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR char_type to_char_type(int_type __c) _NOEXCEPT {
	162	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR char_type
	163	to_char_type(int_type __c) _NOEXCEPT {
161	164	return char_type(__c);
162	165	}
163		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type to_int_type(char_type __c) _NOEXCEPT {
	166	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type
	167	to_int_type(char_type __c) _NOEXCEPT {
164	168	return int_type((unsigned char)__c);
165	169	}
166		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool eq_int_type(int_type __c1, int_type __c2) _NOEXCEPT {
	170	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool
	171	eq_int_type(int_type __c1, int_type __c2) _NOEXCEPT {
167	172	return __c1 == __c2;
168	173	}
169		static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT { return int_type(EOF); }
	174	[[__nodiscard__]] static inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT {
	175	return int_type(EOF);
	176	}
170	177	};
171	178
172	179	template <class _CharT, class _IntT, _IntT _EOFVal>
...	...	@@ -187,11 +194,11 @@ struct __char_traits_base {
187	194	__lhs = __rhs;
188	195	}
189	196
190		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool eq(char_type __lhs, char_type __rhs) _NOEXCEPT {
	197	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool eq(char_type __lhs, char_type __rhs) _NOEXCEPT {
191	198	return __lhs == __rhs;
192	199	}
193	200
194		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool lt(char_type __lhs, char_type __rhs) _NOEXCEPT {
	201	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool lt(char_type __lhs, char_type __rhs) _NOEXCEPT {
195	202	return __lhs < __rhs;
196	203	}
197	204
...	...	@@ -213,19 +220,22 @@ struct __char_traits_base {
213	220	return __str;
214	221	}
215	222
216		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR char_type to_char_type(int_type __c) _NOEXCEPT {
	223	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR char_type to_char_type(int_type __c) _NOEXCEPT {
217	224	return char_type(__c);
218	225	}
219	226
220		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type to_int_type(char_type __c) _NOEXCEPT { return int_type(__c); }
	227	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type to_int_type(char_type __c) _NOEXCEPT {
	228	return int_type(__c);
	229	}
221	230
222		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool eq_int_type(int_type __lhs, int_type __rhs) _NOEXCEPT {
	231	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR bool
	232	eq_int_type(int_type __lhs, int_type __rhs) _NOEXCEPT {
223	233	return __lhs == __rhs;
224	234	}
225	235
226		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT { return _EOFVal; }
	236	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type eof() _NOEXCEPT { return _EOFVal; }
227	237
228		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
	238	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR int_type not_eof(int_type __c) _NOEXCEPT {
229	239	return eq_int_type(__c, eof()) ? static_cast<int_type>(~eof()) : __c;
230	240	}
231	241	};
...	...	@@ -235,18 +245,19 @@ struct __char_traits_base {
235	245	#if _LIBCPP_HAS_WIDE_CHARACTERS
236	246	template <>
237	247	struct char_traits<wchar_t> : __char_traits_base<wchar_t, wint_t, static_cast<wint_t>(WEOF)> {
238		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	248	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 int
239	249	compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
240	250	if (__n == 0)
241	251	return 0;
242	252	return std::__constexpr_wmemcmp(__s1, __s2, __n);
243	253	}
244	254
245		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT {
	255	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t
	256	length(const char_type* __s) _NOEXCEPT {
246	257	return std::__constexpr_wcslen(__s);
247	258	}
248	259
249		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
	260	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
250	261	find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
251	262	return std::__constexpr_wmemchr(__s, __a, __n);
252	263	}
...	...	@@ -257,16 +268,16 @@ struct char_traits<wchar_t> : __char_traits_base<wchar_t, wint_t, static_cast<wi
257	268
258	269	template <>
259	270	struct char_traits<char8_t> : __char_traits_base<char8_t, unsigned int, static_cast<unsigned int>(EOF)> {
260		static _LIBCPP_HIDE_FROM_ABI constexpr int
	271	[[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr int
261	272	compare(const char_type* __s1, const char_type* __s2, size_t __n) noexcept {
262	273	return std::__constexpr_memcmp(__s1, __s2, __element_count(__n));
263	274	}
264	275
265		static _LIBCPP_HIDE_FROM_ABI constexpr size_t length(const char_type* __str) noexcept {
	276	[[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr size_t length(const char_type* __str) noexcept {
266	277	return std::__constexpr_strlen(__str);
267	278	}
268	279
269		_LIBCPP_HIDE_FROM_ABI static constexpr const char_type*
	280	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI static constexpr const char_type*
270	281	find(const char_type* __s, size_t __n, const char_type& __a) noexcept {
271	282	return std::__constexpr_memchr(__s, __a, __n);
272	283	}
...	...	@@ -276,11 +287,11 @@ struct char_traits<char8_t> : __char_traits_base<char8_t, unsigned int, static_c
276	287
277	288	template <>
278	289	struct char_traits<char16_t> : __char_traits_base<char16_t, uint_least16_t, static_cast<uint_least16_t>(0xFFFF)> {
279		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	290	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
280	291	compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT;
281	292	_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT;
282	293
283		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
	294	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
284	295	find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
285	296	__identity __proj;
286	297	const char_type* __match = std::__find(__s, __s + __n, __a, __proj);
...	...	@@ -290,7 +301,7 @@ struct char_traits<char16_t> : __char_traits_base<char16_t, uint_least16_t, stat
290	301	}
291	302	};
292	303
293		inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	304	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
294	305	char_traits<char16_t>::compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
295	306	for (; __n; --__n, ++__s1, ++__s2) {
296	307	if (lt(__s1, __s2))
...	...	@@ -301,7 +312,8 @@ char_traits<char16_t>::compare(const char_type* __s1, const char_type* __s2, siz
301	312	return 0;
302	313	}
303	314
304		inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t char_traits<char16_t>::length(const char_type* __s) _NOEXCEPT {
	315	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t
	316	char_traits<char16_t>::length(const char_type* __s) _NOEXCEPT {
305	317	size_t __len = 0;
306	318	for (; !eq(*__s, char_type(0)); ++__s)
307	319	++__len;
...	...	@@ -310,11 +322,11 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t char_traits<char16_t>::length(const
310	322
311	323	template <>
312	324	struct char_traits<char32_t> : __char_traits_base<char32_t, uint_least32_t, static_cast<uint_least32_t>(0xFFFFFFFF)> {
313		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	325	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 int
314	326	compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT;
315	327	_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t length(const char_type* __s) _NOEXCEPT;
316	328
317		_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
	329	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX17 const char_type*
318	330	find(const char_type* __s, size_t __n, const char_type& __a) _NOEXCEPT {
319	331	__identity __proj;
320	332	const char_type* __match = std::__find(__s, __s + __n, __a, __proj);
...	...	@@ -324,7 +336,7 @@ struct char_traits<char32_t> : __char_traits_base<char32_t, uint_least32_t, stat
324	336	}
325	337	};
326	338
327		inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
	339	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX17 int
328	340	char_traits<char32_t>::compare(const char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT {
329	341	for (; __n; --__n, ++__s1, ++__s2) {
330	342	if (lt(__s1, __s2))
...	...	@@ -335,7 +347,8 @@ char_traits<char32_t>::compare(const char_type* __s1, const char_type* __s2, siz
335	347	return 0;
336	348	}
337	349
338		inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t char_traits<char32_t>::length(const char_type* __s) _NOEXCEPT {
	350	[[__nodiscard__]] inline _LIBCPP_CONSTEXPR_SINCE_CXX17 size_t
	351	char_traits<char32_t>::length(const char_type* __s) _NOEXCEPT {
339	352	size_t __len = 0;
340	353	for (; !eq(*__s, char_type(0)); ++__s)
341	354	++__len;
...	...	@@ -369,6 +382,13 @@ _LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR_SINCE_CXX14 const _CharT* __searc
369	382	if (__len1 < __len2)
370	383	return __last1;
371	384
	385	if (__builtin_constant_p(__len2 == 1) && __len2 == 1) {
	386	auto __res = _Traits::find(__first1, __len1, *__first2);
	387	if (__res == nullptr)
	388	return __last1;
	389	return __res;
	390	}
	391
372	392	// First element of __first2 is loop invariant.
373	393	_CharT __f2 = *__first2;
374	394	while (true) {

lib/libcxx/include/__string/constexpr_c_functions.h+5-5

...	...	@@ -22,7 +22,6 @@
22	22	#include <__type_traits/is_equality_comparable.h>
23	23	#include <__type_traits/is_integral.h>
24	24	#include <__type_traits/is_same.h>
25		#include <__type_traits/is_trivially_copyable.h>
26	25	#include <__type_traits/is_trivially_lexicographically_comparable.h>
27	26	#include <__type_traits/remove_cv.h>
28	27	#include <__utility/element_count.h>
...	...	@@ -96,14 +95,13 @@ __constexpr_memcmp(const _Tp* __lhs, const _Up* __rhs, __element_count __n) {
96	95	}
97	96	}
98	97
99		// Because of __libcpp_is_trivially_equality_comparable we know that comparing the object representations is equivalent
	98	// Because of __is_trivially_equality_comparable_v we know that comparing the object representations is equivalent
100	99	// to a std::memcmp(...) == 0. Since we have multiple objects contiguously in memory, we can call memcmp once instead
101	100	// of invoking it on every object individually.
102	101	template <class _Tp, class _Up>
103	102	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
104	103	__constexpr_memcmp_equal(const _Tp* __lhs, const _Up* __rhs, __element_count __n) {
105		static_assert(__libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
106		"_Tp and _Up have to be trivially equality comparable");
	104	static_assert(__is_trivially_equality_comparable_v<_Tp, _Up>, "_Tp and _Up have to be trivially equality comparable");
107	105
108	106	auto __count = static_cast<size_t>(__n);
109	107
...	...	@@ -128,7 +126,7 @@ __constexpr_memcmp_equal(const _Tp* __lhs, const _Up* __rhs, __element_count __n
128	126
129	127	template <class _Tp, class _Up>
130	128	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_memchr(_Tp* __str, _Up __value, size_t __count) {
131		static_assert(sizeof(_Tp) == 1 && __libcpp_is_trivially_equality_comparable<_Tp, _Up>::value,
	129	static_assert(sizeof(_Tp) == 1 && __is_trivially_equality_comparable_v<_Tp, _Up>,
132	130	"Calling memchr on non-trivially equality comparable types is unsafe.");
133	131
134	132	if (__libcpp_is_constant_evaluated()) {
...	...	@@ -225,6 +223,8 @@ __constexpr_memmove(_Tp* __dest, _Up* __src, __element_count __n) {
225	223	std::__assign_trivially_copyable(__dest[__i], __src[__i]);
226	224	}
227	225	}
	226	} else if _LIBCPP_CONSTEXPR (sizeof(_Tp) == __datasizeof_v<_Tp>) {
	227	::__builtin_memmove(__dest, __src, __count * sizeof(_Tp));
228	228	} else if (__count > 0) {
229	229	::__builtin_memmove(__dest, __src, (__count - 1) * sizeof(_Tp) + __datasizeof_v<_Tp>);
230	230	}

lib/libcxx/include/__support/xlocale/__strtonum_fallback.h-8

...	...	@@ -34,12 +34,4 @@ inline _LIBCPP_HIDE_FROM_ABI long double strtold_l(const char* __nptr, char** __
34	34	return ::strtold(__nptr, __endptr);
35	35	}
36	36
37		inline _LIBCPP_HIDE_FROM_ABI long long strtoll_l(const char* __nptr, char** __endptr, int __base, locale_t) {
38		return ::strtoll(__nptr, __endptr, __base);
39		}
40
41		inline _LIBCPP_HIDE_FROM_ABI unsigned long long strtoull_l(const char* __nptr, char** __endptr, int __base, locale_t) {
42		return ::strtoull(__nptr, __endptr, __base);
43		}
44
45	37	#endif // _LIBCPP___SUPPORT_XLOCALE_STRTONUM_FALLBACK_H

lib/libcxx/include/__system_error/error_category.h+8-8

...	...	@@ -20,7 +20,7 @@
20	20
21	21	_LIBCPP_BEGIN_NAMESPACE_STD
22	22
23		class _LIBCPP_EXPORTED_FROM_ABI error_condition;
	23	class error_condition;
24	24	class _LIBCPP_EXPORTED_FROM_ABI error_code;
25	25
26	26	class _LIBCPP_HIDDEN __do_message;
...	...	@@ -37,11 +37,11 @@ public:
37	37	error_category(const error_category&) = delete;
38	38	error_category& operator=(const error_category&) = delete;
39	39
40		virtual const char* name() const _NOEXCEPT = 0;
41		virtual error_condition default_error_condition(int __ev) const _NOEXCEPT;
42		virtual bool equivalent(int __code, const error_condition& __condition) const _NOEXCEPT;
43		virtual bool equivalent(const error_code& __code, int __condition) const _NOEXCEPT;
44		virtual string message(int __ev) const = 0;
	40	[[__nodiscard__]] virtual const char* name() const _NOEXCEPT = 0;
	41	[[__nodiscard__]] virtual error_condition default_error_condition(int __ev) const _NOEXCEPT;
	42	[[__nodiscard__]] virtual bool equivalent(int __code, const error_condition& __condition) const _NOEXCEPT;
	43	[[__nodiscard__]] virtual bool equivalent(const error_code& __code, int __condition) const _NOEXCEPT;
	44	[[__nodiscard__]] virtual string message(int __ev) const = 0;
45	45
46	46	_LIBCPP_HIDE_FROM_ABI bool operator==(const error_category& __rhs) const _NOEXCEPT { return this == &__rhs; }
47	47
...	...	@@ -67,8 +67,8 @@ public:
67	67	string message(int __ev) const override;
68	68	};
69	69
70		[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& generic_category() _NOEXCEPT;
71		[[__gnu__::__const__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& system_category() _NOEXCEPT;
	70	[[__gnu__::__const__]] [[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& generic_category() _NOEXCEPT;
	71	[[__gnu__::__const__]] [[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& system_category() _NOEXCEPT;
72	72
73	73	_LIBCPP_END_NAMESPACE_STD
74	74

lib/libcxx/include/__system_error/error_code.h+5-5

...	...	@@ -71,20 +71,20 @@ public:
71	71	__cat_ = &system_category();
72	72	}
73	73
74		_LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
	74	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
75	75
76		_LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
	76	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
77	77
78		_LIBCPP_HIDE_FROM_ABI error_condition default_error_condition() const _NOEXCEPT {
	78	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI error_condition default_error_condition() const _NOEXCEPT {
79	79	return __cat_->default_error_condition(__val_);
80	80	}
81	81
82		string message() const;
	82	[[__nodiscard__]] string message() const;
83	83
84	84	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __val_ != 0; }
85	85	};
86	86
87		inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(errc __e) _NOEXCEPT {
	87	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(errc __e) _NOEXCEPT {
88	88	return error_code(static_cast<int>(__e), generic_category());
89	89	}
90	90

lib/libcxx/include/__system_error/error_condition.h+4-4

...	...	@@ -80,15 +80,15 @@ public:
80	80	__cat_ = &generic_category();
81	81	}
82	82
83		_LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
	83	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int value() const _NOEXCEPT { return __val_; }
84	84
85		_LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
86		string message() const;
	85	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const error_category& category() const _NOEXCEPT { return *__cat_; }
	86	[[__nodiscard__]] string message() const;
87	87
88	88	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const _NOEXCEPT { return __val_ != 0; }
89	89	};
90	90
91		inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(errc __e) _NOEXCEPT {
	91	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(errc __e) _NOEXCEPT {
92	92	return error_condition(static_cast<int>(__e), generic_category());
93	93	}
94	94

lib/libcxx/include/__system_error/system_error.h+1-1

...	...	@@ -36,7 +36,7 @@ public:
36	36	_LIBCPP_HIDE_FROM_ABI system_error(const system_error&) _NOEXCEPT = default;
37	37	~system_error() _NOEXCEPT override;
38	38
39		_LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
	39	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
40	40	};
41	41
42	42	// __ev is expected to be an error in the generic_category domain (e.g. from

lib/libcxx/include/__thread/id.h+1-1

...	...	@@ -23,7 +23,7 @@
23	23	_LIBCPP_BEGIN_NAMESPACE_STD
24	24
25	25	#if _LIBCPP_HAS_THREADS
26		class _LIBCPP_EXPORTED_FROM_ABI __thread_id;
	26	class __thread_id;
27	27
28	28	namespace this_thread {
29	29

lib/libcxx/include/__thread/jthread.h+1-1

...	...	@@ -36,7 +36,7 @@ _LIBCPP_PUSH_MACROS
36	36
37	37	_LIBCPP_BEGIN_NAMESPACE_STD
38	38
39		class _LIBCPP_AVAILABILITY_SYNC jthread {
	39	class jthread {
40	40	public:
41	41	// types
42	42	using id = thread::id;

lib/libcxx/include/__thread/poll_with_backoff.h+27-8

...	...	@@ -22,33 +22,50 @@ _LIBCPP_BEGIN_NAMESPACE_STD
22	22
23	23	static _LIBCPP_CONSTEXPR const int __libcpp_polling_count = 64;
24	24
	25	enum class __backoff_results : unsigned char {
	26	__continue_poll = 1,
	27	__poll_success = 2,
	28	__timeout = 3,
	29	__backoff_failure = 4,
	30	};
	31
	32	enum class __poll_with_backoff_results : unsigned char {
	33	__poll_success = static_cast<unsigned char>(__backoff_results::__poll_success),
	34	__timeout = static_cast<unsigned char>(__backoff_results::__timeout),
	35	__backoff_failure = static_cast<unsigned char>(__backoff_results::__backoff_failure),
	36	};
	37
25	38	// Polls a thread for a condition given by a predicate, and backs off based on a backoff policy
26	39	// before polling again.
27	40	//
28	41	// - __poll is the "test function" that should return true if polling succeeded, and false if it failed.
29	42	//
30	43	// - __backoff is the "backoff policy", which is called with the duration since we started polling. It should
31		// return false in order to resume polling, and true if polling should stop entirely for some reason.
	44	// return __backoff_results::__continue_poll in order to resume polling, and other appropriate __backoff_results
	45	// if polling should stop entirely for some reason.
32	46	// In general, backoff policies sleep for some time before returning control to the polling loop.
33	47	//
34	48	// - __max_elapsed is the maximum duration to try polling for. If the maximum duration is exceeded,
35		// the polling loop will return false to report a timeout.
	49	// the polling loop will return __poll_with_backoff_results::__timeout to report a timeout.
	50
36	51	template <class _Poll, class _Backoff>
37		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_poll_with_backoff(
	52	_LIBCPP_HIDE_FROM_ABI __poll_with_backoff_results __libcpp_thread_poll_with_backoff(
38	53	_Poll&& __poll, _Backoff&& __backoff, chrono::nanoseconds __max_elapsed = chrono::nanoseconds::zero()) {
39	54	auto const __start = chrono::high_resolution_clock::now();
40	55	for (int __count = 0;;) {
41	56	if (__poll())
42		return true; // __poll completion means success
	57	return __poll_with_backoff_results::__poll_success;
43	58	if (__count < __libcpp_polling_count) {
44	59	__count += 1;
45	60	continue;
46	61	}
47	62	chrono::nanoseconds const __elapsed = chrono::high_resolution_clock::now() - __start;
48	63	if (__max_elapsed != chrono::nanoseconds::zero() && __max_elapsed < __elapsed)
49		return false; // timeout failure
50		if (__backoff(__elapsed))
51		return false; // __backoff completion means failure
	64	return __poll_with_backoff_results::__timeout;
	65	if (auto __backoff_res = __backoff(__elapsed); __backoff_res == __backoff_results::__continue_poll)
	66	continue;
	67	else
	68	return static_cast<__poll_with_backoff_results>(__backoff_res);
52	69	}
53	70	}
54	71
...	...	@@ -59,7 +76,9 @@ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __libcpp_thread_poll_with_b
59	76	// so this should most likely only be used on single-threaded systems where there
60	77	// are no other threads to compete with.
61	78	struct __spinning_backoff_policy {
62		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator()(chrono::nanoseconds const&) const { return false; }
	79	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __backoff_results operator()(chrono::nanoseconds const&) const {
	80	return __backoff_results::__continue_poll;
	81	}
63	82	};
64	83
65	84	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__thread/support/c11.h+8-8

...	...	@@ -39,17 +39,17 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursiv
39	39	return mtx_init(__m, mtx_plain \| mtx_recursive) == thrd_success ? 0 : EINVAL;
40	40	}
41	41
42		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	42	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
43	43	__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m) {
44	44	return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
45	45	}
46	46
47		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
	47	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI bool
48	48	__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m) {
49	49	return mtx_trylock(__m) == thrd_success;
50	50	}
51	51
52		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	52	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
53	53	__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m) {
54	54	return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
55	55	}
...	...	@@ -59,15 +59,15 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recur
59	59	return 0;
60	60	}
61	61
62		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
	62	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
63	63	return mtx_lock(__m) == thrd_success ? 0 : EINVAL;
64	64	}
65	65
66		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
	66	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
67	67	return mtx_trylock(__m) == thrd_success;
68	68	}
69	69
70		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
	70	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
71	71	return mtx_unlock(__m) == thrd_success ? 0 : EINVAL;
72	72	}
73	73
...	...	@@ -92,12 +92,12 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t*
92	92	return cnd_broadcast(__cv) == thrd_success ? 0 : EINVAL;
93	93	}
94	94
95		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	95	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
96	96	__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m) {
97	97	return cnd_wait(__cv, __m) == thrd_success ? 0 : EINVAL;
98	98	}
99	99
100		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	100	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
101	101	__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, timespec* __ts) {
102	102	int __ec = cnd_timedwait(__cv, __m, __ts);
103	103	return __ec == thrd_timedout ? ETIMEDOUT : __ec;

lib/libcxx/include/__thread/support/pthread.h+8-8

...	...	@@ -72,17 +72,17 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursiv
72	72	return 0;
73	73	}
74	74
75		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	75	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
76	76	__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m) {
77	77	return pthread_mutex_lock(__m);
78	78	}
79	79
80		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
	80	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI bool
81	81	__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m) {
82	82	return pthread_mutex_trylock(__m) == 0;
83	83	}
84	84
85		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	85	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
86	86	__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m) {
87	87	return pthread_mutex_unlock(__m);
88	88	}
...	...	@@ -91,15 +91,15 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recur
91	91	return pthread_mutex_destroy(__m);
92	92	}
93	93
94		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
	94	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_lock(__libcpp_mutex_t* __m) {
95	95	return pthread_mutex_lock(__m);
96	96	}
97	97
98		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
	98	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m) {
99	99	return pthread_mutex_trylock(__m) == 0;
100	100	}
101	101
102		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
	102	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int __libcpp_mutex_unlock(__libcpp_mutex_t* __m) {
103	103	return pthread_mutex_unlock(__m);
104	104	}
105	105
...	...	@@ -117,12 +117,12 @@ inline _LIBCPP_HIDE_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t*
117	117	return pthread_cond_broadcast(__cv);
118	118	}
119	119
120		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	120	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
121	121	__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m) {
122	122	return pthread_cond_wait(__cv, __m);
123	123	}
124	124
125		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	125	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI int
126	126	__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, __libcpp_timespec_t* __ts) {
127	127	return pthread_cond_timedwait(__cv, __m, __ts);
128	128	}

lib/libcxx/include/__thread/support/windows.h+8-8

...	...	@@ -36,22 +36,22 @@ typedef void* __libcpp_recursive_mutex_t[6];
36	36
37	37	_LIBCPP_EXPORTED_FROM_ABI int __libcpp_recursive_mutex_init(__libcpp_recursive_mutex_t* __m);
38	38
39		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	39	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int
40	40	__libcpp_recursive_mutex_lock(__libcpp_recursive_mutex_t* __m);
41	41
42		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool
	42	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI bool
43	43	__libcpp_recursive_mutex_trylock(__libcpp_recursive_mutex_t* __m);
44	44
45		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	45	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int
46	46	__libcpp_recursive_mutex_unlock(__libcpp_recursive_mutex_t* __m);
47	47
48	48	_LIBCPP_EXPORTED_FROM_ABI int __libcpp_recursive_mutex_destroy(__libcpp_recursive_mutex_t* __m);
49	49
50		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_lock(__libcpp_mutex_t* __m);
	50	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int __libcpp_mutex_lock(__libcpp_mutex_t* __m);
51	51
52		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m);
	52	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI bool __libcpp_mutex_trylock(__libcpp_mutex_t* __m);
53	53
54		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int __libcpp_mutex_unlock(__libcpp_mutex_t* __m);
	54	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int __libcpp_mutex_unlock(__libcpp_mutex_t* __m);
55	55
56	56	_LIBCPP_EXPORTED_FROM_ABI int __libcpp_mutex_destroy(__libcpp_mutex_t* __m);
57	57
...	...	@@ -65,10 +65,10 @@ _LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_signal(__libcpp_condvar_t* __cv);
65	65
66	66	_LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_broadcast(__libcpp_condvar_t* __cv);
67	67
68		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	68	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int
69	69	__libcpp_condvar_wait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m);
70	70
71		_LIBCPP_EXPORTED_FROM_ABI _LIBCPP_NO_THREAD_SAFETY_ANALYSIS int
	71	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_EXPORTED_FROM_ABI int
72	72	__libcpp_condvar_timedwait(__libcpp_condvar_t* __cv, __libcpp_mutex_t* __m, __libcpp_timespec_t* __ts);
73	73
74	74	_LIBCPP_EXPORTED_FROM_ABI int __libcpp_condvar_destroy(__libcpp_condvar_t* __cv);

lib/libcxx/include/__thread/thread.h+13-8

...	...	@@ -25,6 +25,8 @@
25	25	#include <__thread/support.h>
26	26	#include <__type_traits/decay.h>
27	27	#include <__type_traits/enable_if.h>
	28	#include <__type_traits/invoke.h>
	29	#include <__type_traits/is_constructible.h>
28	30	#include <__type_traits/is_same.h>
29	31	#include <__type_traits/remove_cvref.h>
30	32	#include <__utility/forward.h>
...	...	@@ -155,8 +157,8 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, __thread_id __id) {
155	157	# ifndef _LIBCPP_CXX03_LANG
156	158
157	159	template <class _TSp, class _Fp, class... _Args, size_t... _Indices>
158		inline _LIBCPP_HIDE_FROM_ABI void __thread_execute(tuple<_TSp, _Fp, _Args...>& __t, __tuple_indices<_Indices...>) {
159		std::__invoke(std::move(std::get<1>(__t)), std::move(std::get<_Indices>(__t))...);
	160	inline _LIBCPP_HIDE_FROM_ABI void __thread_execute(tuple<_TSp, _Fp, _Args...>& __t, __index_sequence<_Indices...>) {
	161	std::__invoke(std::move(std::get<_Indices + 1>(__t))...);
160	162	}
161	163
162	164	template <class _Fp>
...	...	@@ -164,8 +166,7 @@ _LIBCPP_HIDE_FROM_ABI void* __thread_proxy(void* __vp) {
164	166	// _Fp = tuple< unique_ptr<__thread_struct>, Functor, Args...>
165	167	unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
166	168	__thread_local_data().set_pointer(std::get<0>(*__p.get()).release());
167		typedef typename __make_tuple_indices<tuple_size<_Fp>::value, 2>::type _Index;
168		std::__thread_execute(*__p.get(), _Index());
	169	std::__thread_execute(*__p.get(), __make_index_sequence<tuple_size<_Fp>::value - 1>());
169	170	return nullptr;
170	171	}
171	172
...	...	@@ -206,6 +207,10 @@ public:
206	207	# ifndef _LIBCPP_CXX03_LANG
207	208	template <class _Fp, class... _Args, __enable_if_t<!is_same<__remove_cvref_t<_Fp>, thread>::value, int> = 0>
208	209	_LIBCPP_HIDE_FROM_ABI explicit thread(_Fp&& __f, _Args&&... __args) {
	210	static_assert(is_constructible<__decay_t<_Fp>, _Fp>::value, "");
	211	static_assert(_And<is_constructible<__decay_t<_Args>, _Args>...>::value, "");
	212	static_assert(__is_invocable_v<__decay_t<_Fp>, __decay_t<_Args>...>, "");
	213
209	214	typedef unique_ptr<__thread_struct> _TSPtr;
210	215	_TSPtr __tsp(new __thread_struct);
211	216	typedef tuple<_TSPtr, __decay_t<_Fp>, __decay_t<_Args>...> _Gp;
...	...	@@ -243,13 +248,13 @@ public:
243	248
244	249	_LIBCPP_HIDE_FROM_ABI void swap(thread& __t) _NOEXCEPT { std::swap(__t_, __t.__t_); }
245	250
246		_LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); }
	251	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool joinable() const _NOEXCEPT { return !__libcpp_thread_isnull(&__t_); }
247	252	void join();
248	253	void detach();
249		_LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); }
250		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; }
	254	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI id get_id() const _NOEXCEPT { return __libcpp_thread_get_id(&__t_); }
	255	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() _NOEXCEPT { return __t_; }
251	256
252		static unsigned hardware_concurrency() _NOEXCEPT;
	257	[[__nodiscard__]] static unsigned hardware_concurrency() _NOEXCEPT;
253	258	};
254	259
255	260	inline _LIBCPP_HIDE_FROM_ABI void swap(thread& __x, thread& __y) _NOEXCEPT { __x.swap(__y); }

lib/libcxx/include/__thread/timed_backoff_policy.h+3-2

...	...	@@ -11,6 +11,7 @@
11	11	#define _LIBCPP___THREAD_TIMED_BACKOFF_POLICY_H
12	12
13	13	#include <__config>
	14	#include <__thread/poll_with_backoff.h>
14	15
15	16	#if _LIBCPP_HAS_THREADS
16	17
...	...	@@ -24,7 +25,7 @@
24	25	_LIBCPP_BEGIN_NAMESPACE_STD
25	26
26	27	struct __libcpp_timed_backoff_policy {
27		_LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
	28	_LIBCPP_HIDE_FROM_ABI __backoff_results operator()(chrono::nanoseconds __elapsed) const {
28	29	if (__elapsed > chrono::milliseconds(128))
29	30	__libcpp_thread_sleep_for(chrono::milliseconds(8));
30	31	else if (__elapsed > chrono::microseconds(64))
...	...	@@ -33,7 +34,7 @@ struct __libcpp_timed_backoff_policy {
33	34	__libcpp_thread_yield();
34	35	else {
35	36	} // poll
36		return false;
	37	return __backoff_results::__continue_poll;
37	38	}
38	39	};
39	40

lib/libcxx/include/__tree+849-701

...	...	@@ -11,37 +11,39 @@
11	11	#define _LIBCPP___TREE
12	12
13	13	#include <__algorithm/min.h>
	14	#include <__algorithm/specialized_algorithms.h>
14	15	#include <__assert>
15	16	#include <__config>
16		#include <__fwd/map.h>
17	17	#include <__fwd/pair.h>
18		#include <__fwd/set.h>
19	18	#include <__iterator/distance.h>
20	19	#include <__iterator/iterator_traits.h>
21	20	#include <__iterator/next.h>
22	21	#include <__memory/addressof.h>
23	22	#include <__memory/allocator_traits.h>
24	23	#include <__memory/compressed_pair.h>
	24	#include <__memory/construct_at.h>
25	25	#include <__memory/pointer_traits.h>
26	26	#include <__memory/swap_allocator.h>
27	27	#include <__memory/unique_ptr.h>
28		#include <__type_traits/can_extract_key.h>
	28	#include <__new/launder.h>
29	29	#include <__type_traits/copy_cvref.h>
30	30	#include <__type_traits/enable_if.h>
31	31	#include <__type_traits/invoke.h>
32		#include <__type_traits/is_const.h>
33	32	#include <__type_traits/is_constructible.h>
34	33	#include <__type_traits/is_nothrow_assignable.h>
35	34	#include <__type_traits/is_nothrow_constructible.h>
36	35	#include <__type_traits/is_same.h>
	36	#include <__type_traits/is_specialization.h>
37	37	#include <__type_traits/is_swappable.h>
	38	#include <__type_traits/make_transparent.h>
38	39	#include <__type_traits/remove_const.h>
39		#include <__type_traits/remove_const_ref.h>
40	40	#include <__type_traits/remove_cvref.h>
41	41	#include <__utility/forward.h>
	42	#include <__utility/lazy_synth_three_way_comparator.h>
42	43	#include <__utility/move.h>
43	44	#include <__utility/pair.h>
44	45	#include <__utility/swap.h>
	46	#include <__utility/try_key_extraction.h>
45	47	#include <limits>
46	48
47	49	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
...	...	@@ -51,14 +53,31 @@
51	53	_LIBCPP_PUSH_MACROS
52	54	#include <__undef_macros>
53	55
54		_LIBCPP_BEGIN_NAMESPACE_STD
	56	_LIBCPP_DIAGNOSTIC_PUSH
	57	// GCC complains about the backslashes at the end, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121528
	58	_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wcomment")
	59	// __tree is a red-black-tree implementation used for the associative containers (i.e. (multi)map/set). It stores
	60	// - (1) a pointer to the node with the smallest (i.e. leftmost) element, namely __begin_node_
	61	// - (2) the number of nodes in the tree, namely __size_
	62	// - (3) a pointer to the root of the tree, namely __end_node_
	63	//
	64	// Storing (1) and (2) is required to allow for constant time lookups. A tree looks like this in memory:
	65	//
	66	// __end_node_
	67	// \|
	68	// root
	69	// / \
	70	// l1 r1
	71	// / \ / \
	72	// ... ... ... ...
	73	//
	74	// All nodes except __end_node_ have a __left_ and __right_ pointer as well as a __parent_ pointer.
	75	// __end_node_ only contains a __left_ pointer, which points to the root of the tree.
	76	// This layout allows for iteration through the tree without a need for special handling of the end node. See
	77	// __tree_next_iter and __tree_prev_iter for more details.
	78	_LIBCPP_DIAGNOSTIC_POP
55	79
56		template <class _Tp, class _Compare, class _Allocator>
57		class __tree;
58		template <class _Tp, class _NodePtr, class _DiffType>
59		class __tree_iterator;
60		template <class _Tp, class _ConstNodePtr, class _DiffType>
61		class __tree_const_iterator;
	80	_LIBCPP_BEGIN_NAMESPACE_STD
62	81
63	82	template <class _Pointer>
64	83	class __tree_end_node;
...	...	@@ -70,13 +89,6 @@ class __tree_node;
70	89	template <class _Key, class _Value>
71	90	struct __value_type;
72	91
73		template <class _Allocator>
74		class __map_node_destructor;
75		template <class _TreeIterator>
76		class __map_iterator;
77		template <class _TreeIterator>
78		class __map_const_iterator;
79
80	92	/*
81	93
82	94	_NodePtr algorithms
...	...	@@ -185,6 +197,11 @@ _LIBCPP_HIDE_FROM_ABI _NodePtr __tree_next(_NodePtr __x) _NOEXCEPT {
185	197	return __x->__parent_unsafe();
186	198	}
187	199
	200	// __tree_next_iter and __tree_prev_iter implement iteration through the tree. The order is as follows:
	201	// left sub-tree -> node -> right sub-tree. When the right-most node of a sub-tree is reached, we walk up the tree until
	202	// we find a node where we were in the left sub-tree. We are _always_ in a left sub-tree, since the __end_node_ points
	203	// to the actual root of the tree through a __left_ pointer. Incrementing the end() pointer is UB, so we can assume that
	204	// never happens.
188	205	template <class _EndNodePtr, class _NodePtr>
189	206	inline _LIBCPP_HIDE_FROM_ABI _EndNodePtr __tree_next_iter(_NodePtr __x) _NOEXCEPT {
190	207	_LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
...	...	@@ -494,16 +511,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree_remove(_NodePtr __root, _NodePtr __z) _NOEXCEP
494	511	// node traits
495	512
496	513	template <class _Tp>
497		struct __is_tree_value_type_imp : false_type {};
498
499		template <class _Key, class _Value>
500		struct __is_tree_value_type_imp<__value_type<_Key, _Value> > : true_type {};
501
502		template <class... _Args>
503		struct __is_tree_value_type : false_type {};
504
505		template <class _One>
506		struct __is_tree_value_type<_One> : __is_tree_value_type_imp<__remove_cvref_t<_One> > {};
	514	inline const bool __is_tree_value_type_v = __is_specialization_v<_Tp, __value_type>;
507	515
508	516	template <class _Tp>
509	517	struct __get_tree_key_type {
...	...	@@ -549,15 +557,14 @@ private:
549	557	template <class _Pointer>
550	558	class __tree_end_node {
551	559	public:
552		typedef _Pointer pointer;
	560	using pointer = _Pointer;
553	561	pointer __left_;
554	562
555	563	_LIBCPP_HIDE_FROM_ABI __tree_end_node() _NOEXCEPT : __left_() {}
556	564	};
557	565
558	566	template <class _VoidPtr>
559		class _LIBCPP_STANDALONE_DEBUG
560		__tree_node_base : public __tree_end_node<__rebind_pointer_t<_VoidPtr, __tree_node_base<_VoidPtr> > > {
	567	class __tree_node_base : public __tree_end_node<__rebind_pointer_t<_VoidPtr, __tree_node_base<_VoidPtr> > > {
561	568	public:
562	569	using pointer = __rebind_pointer_t<_VoidPtr, __tree_node_base>;
563	570	using __end_node_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<_VoidPtr, __tree_end_node<pointer> >;
...	...	@@ -570,20 +577,41 @@ public:
570	577
571	578	_LIBCPP_HIDE_FROM_ABI void __set_parent(pointer __p) { __parent_ = static_cast<__end_node_pointer>(__p); }
572	579
573		~__tree_node_base() = delete;
	580	_LIBCPP_HIDE_FROM_ABI __tree_node_base() = default;
574	581	__tree_node_base(__tree_node_base const&) = delete;
575	582	__tree_node_base& operator=(__tree_node_base const&) = delete;
576	583	};
577	584
578	585	template <class _Tp, class _VoidPtr>
579		class _LIBCPP_STANDALONE_DEBUG __tree_node : public __tree_node_base<_VoidPtr> {
	586	class __tree_node : public __tree_node_base<_VoidPtr> {
580	587	public:
581	588	using __node_value_type _LIBCPP_NODEBUG = __get_node_value_type_t<_Tp>;
582	589
583		__node_value_type __value_;
	590	// We use a union to avoid initialization during member initialization, which allows us
	591	// to use the allocator from the container to construct the `__node_value_type` in the
	592	// memory provided by the union member
	593	#ifndef _LIBCPP_CXX03_LANG
	594
	595	private:
	596	union {
	597	__node_value_type __value_;
	598	};
584	599
	600	public:
585	601	_LIBCPP_HIDE_FROM_ABI __node_value_type& __get_value() { return __value_; }
	602	#else
	603
	604	private:
	605	_ALIGNAS_TYPE(__node_value_type) unsigned char __buffer_[sizeof(__node_value_type)];
	606
	607	public:
	608	_LIBCPP_HIDE_FROM_ABI __node_value_type& __get_value() { return reinterpret_cast<__node_value_type>(__buffer_); }
	609	#endif
586	610
	611	template <class _Alloc, class... _Args>
	612	_LIBCPP_HIDE_FROM_ABI explicit __tree_node(_Alloc& __na, _Args&&... __args) {
	613	allocator_traits<_Alloc>::construct(__na, std::addressof(__get_value()), std::forward<_Args>(__args)...);
	614	}
587	615	~__tree_node() = delete;
588	616	__tree_node(__tree_node const&) = delete;
589	617	__tree_node& operator=(__tree_node const&) = delete;
...	...	@@ -591,11 +619,11 @@ public:
591	619
592	620	template <class _Allocator>
593	621	class __tree_node_destructor {
594		typedef _Allocator allocator_type;
595		typedef allocator_traits<allocator_type> __alloc_traits;
	622	using allocator_type = _Allocator;
	623	using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<allocator_type>;
596	624
597	625	public:
598		typedef typename __alloc_traits::pointer pointer;
	626	using pointer = typename __alloc_traits::pointer;
599	627
600	628	private:
601	629	allocator_type& __na_;
...	...	@@ -612,7 +640,7 @@ public:
612	640
613	641	_LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
614	642	if (__value_constructed)
615		__alloc_traits::destroy(__na_, std::addressof(__p->__value_));
	643	__alloc_traits::destroy(__na_, std::addressof(__p->__get_value()));
616	644	if (__p)
617	645	__alloc_traits::deallocate(__na_, __p, 1);
618	646	}
...	...	@@ -630,12 +658,60 @@ struct __generic_container_node_destructor<__tree_node<_Tp, _VoidPtr>, _Alloc> :
630	658	};
631	659	#endif
632	660
	661	// Do an in-order traversal of the tree until `__break` returns true. Takes the root node of the tree.
	662	template <class _Reference, class _Break, class _NodePtr, class _Func, class _Proj>
	663	#ifndef _LIBCPP_COMPILER_GCC // This function is recursive, so GCC complains about always_inline.
	664	_LIBCPP_HIDE_FROM_ABI
	665	#endif
	666	bool __tree_iterate_from_root(_Break __break, _NodePtr __root, _Func& __func, _Proj& __proj) {
	667	if (__root->__left_) {
	668	if (std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__left_), __func, __proj))
	669	return true;
	670	}
	671	if (__break(__root))
	672	return true;
	673	std::__invoke(__func, std::__invoke(__proj, static_cast<_Reference>(__root->__get_value())));
	674	if (__root->__right_)
	675	return std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__right_), __func, __proj);
	676	return false;
	677	}
	678
	679	// Do an in-order traversal of the tree from __first to __last.
	680	template <class _NodeIter, class _Func, class _Proj>
	681	_LIBCPP_HIDE_FROM_ABI void
	682	__tree_iterate_subrange(_NodeIter __first_it, _NodeIter __last_it, _Func& __func, _Proj& __proj) {
	683	using _NodePtr = typename _NodeIter::__node_pointer;
	684	using _Reference = typename _NodeIter::reference;
	685
	686	auto __first = __first_it.__ptr_;
	687	auto __last = __last_it.__ptr_;
	688
	689	while (true) {
	690	if (__first == __last)
	691	return;
	692	const auto __nfirst = static_cast<_NodePtr>(__first);
	693	std::__invoke(__func, std::__invoke(__proj, static_cast<_Reference>(__nfirst->__get_value())));
	694	if (__nfirst->__right_) {
	695	if (std::__tree_iterate_from_root<_Reference>(
	696	[&](_NodePtr __node) -> bool { return __node == __last; },
	697	static_cast<_NodePtr>(__nfirst->__right_),
	698	__func,
	699	__proj))
	700	return;
	701	}
	702	while (!std::__tree_is_left_child(static_cast<_NodePtr>(__first)))
	703	__first = static_cast<_NodePtr>(__first)->__parent_;
	704	__first = static_cast<_NodePtr>(__first)->__parent_;
	705	}
	706	}
	707
633	708	template <class _Tp, class _NodePtr, class _DiffType>
634	709	class __tree_iterator {
635		typedef __tree_node_types<_NodePtr> _NodeTypes;
636		typedef _NodePtr __node_pointer;
637		typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
638		typedef typename _NodeTypes::__end_node_pointer __end_node_pointer;
	710	using _NodeTypes _LIBCPP_NODEBUG = __tree_node_types<_NodePtr>;
	711	// NOLINTNEXTLINE(libcpp-nodebug-on-aliases) lldb relies on this alias for pretty printing
	712	using __node_pointer = _NodePtr;
	713	using __node_base_pointer _LIBCPP_NODEBUG = typename _NodeTypes::__node_base_pointer;
	714	using __end_node_pointer _LIBCPP_NODEBUG = typename _NodeTypes::__end_node_pointer;
639	715
640	716	__end_node_pointer __ptr_;
641	717
...	...	@@ -646,15 +722,12 @@ public:
646	722	using reference = value_type&;
647	723	using pointer = __rebind_pointer_t<_NodePtr, value_type>;
648	724
649		_LIBCPP_HIDE_FROM_ABI __tree_iterator() _NOEXCEPT
650		#if _LIBCPP_STD_VER >= 14
651		: __ptr_(nullptr)
652		#endif
653		{
654		}
	725	_LIBCPP_HIDE_FROM_ABI __tree_iterator() _NOEXCEPT : __ptr_(nullptr) {}
655	726
656		_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
657		_LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
	727	_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__get_value(); }
	728	_LIBCPP_HIDE_FROM_ABI pointer operator->() const {
	729	return pointer_traits<pointer>::pointer_to(__get_np()->__get_value());
	730	}
658	731
659	732	_LIBCPP_HIDE_FROM_ABI __tree_iterator& operator++() {
660	733	__ptr_ = std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_));
...	...	@@ -691,50 +764,54 @@ private:
691	764	friend class __tree;
692	765	template <class, class, class>
693	766	friend class __tree_const_iterator;
694		template <class>
695		friend class __map_iterator;
696		template <class, class, class, class>
697		friend class map;
698		template <class, class, class, class>
699		friend class multimap;
700		template <class, class, class>
701		friend class set;
702		template <class, class, class>
703		friend class multiset;
	767
	768	template <class _NodeIter, class _Func, class _Proj>
	769	friend void __tree_iterate_subrange(_NodeIter, _NodeIter, _Func&, _Proj&);
	770	};
	771
	772	#ifndef _LIBCPP_CXX03_LANG
	773	// This also handles {multi,}set::iterator, since they're just aliases to __tree::iterator
	774	template <class _Tp, class _NodePtr, class _DiffType>
	775	struct __specialized_algorithm<
	776	_Algorithm::__for_each,
	777	__iterator_pair<__tree_iterator<_Tp, _NodePtr, _DiffType>, __tree_iterator<_Tp, _NodePtr, _DiffType>>> {
	778	static const bool __has_algorithm = true;
	779
	780	using __iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, _NodePtr, _DiffType>;
	781
	782	template <class _Func, class _Proj>
	783	_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
	784	std::__tree_iterate_subrange(__first, __last, __func, __proj);
	785	}
704	786	};
	787	#endif
705	788
706	789	template <class _Tp, class _NodePtr, class _DiffType>
707	790	class __tree_const_iterator {
708		typedef __tree_node_types<_NodePtr> _NodeTypes;
	791	using _NodeTypes _LIBCPP_NODEBUG = __tree_node_types<_NodePtr>;
709	792	// NOLINTNEXTLINE(libcpp-nodebug-on-aliases) lldb relies on this alias for pretty printing
710		using __node_pointer = _NodePtr;
711		typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
712		typedef typename _NodeTypes::__end_node_pointer __end_node_pointer;
	793	using __node_pointer = _NodePtr;
	794	using __node_base_pointer _LIBCPP_NODEBUG = typename _NodeTypes::__node_base_pointer;
	795	using __end_node_pointer _LIBCPP_NODEBUG = typename _NodeTypes::__end_node_pointer;
713	796
714	797	__end_node_pointer __ptr_;
715	798
716	799	public:
717		using iterator_category = bidirectional_iterator_tag;
718		using value_type = __get_node_value_type_t<_Tp>;
719		using difference_type = _DiffType;
720		using reference = const value_type&;
721		using pointer = __rebind_pointer_t<_NodePtr, const value_type>;
	800	using iterator_category = bidirectional_iterator_tag;
	801	using value_type = __get_node_value_type_t<_Tp>;
	802	using difference_type = _DiffType;
	803	using reference = const value_type&;
	804	using pointer = __rebind_pointer_t<_NodePtr, const value_type>;
	805	using __non_const_iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, __node_pointer, difference_type>;
722	806
723		_LIBCPP_HIDE_FROM_ABI __tree_const_iterator() _NOEXCEPT
724		#if _LIBCPP_STD_VER >= 14
725		: __ptr_(nullptr)
726		#endif
727		{
728		}
729
730		private:
731		typedef __tree_iterator<_Tp, __node_pointer, difference_type> __non_const_iterator;
	807	_LIBCPP_HIDE_FROM_ABI __tree_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
732	808
733		public:
734	809	_LIBCPP_HIDE_FROM_ABI __tree_const_iterator(__non_const_iterator __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
735	810
736		_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
737		_LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
	811	_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__get_value(); }
	812	_LIBCPP_HIDE_FROM_ABI pointer operator->() const {
	813	return pointer_traits<pointer>::pointer_to(__get_np()->__get_value());
	814	}
738	815
739	816	_LIBCPP_HIDE_FROM_ABI __tree_const_iterator& operator++() {
740	817	__ptr_ = std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_));
...	...	@@ -772,18 +849,28 @@ private:
772	849
773	850	template <class, class, class>
774	851	friend class __tree;
775		template <class, class, class, class>
776		friend class map;
777		template <class, class, class, class>
778		friend class multimap;
779		template <class, class, class>
780		friend class set;
781		template <class, class, class>
782		friend class multiset;
783		template <class>
784		friend class __map_const_iterator;
	852
	853	template <class _NodeIter, class _Func, class _Proj>
	854	friend void __tree_iterate_subrange(_NodeIter, _NodeIter, _Func&, _Proj&);
785	855	};
786	856
	857	#ifndef _LIBCPP_CXX03_LANG
	858	// This also handles {multi,}set::const_iterator, since they're just aliases to __tree::iterator
	859	template <class _Tp, class _NodePtr, class _DiffType>
	860	struct __specialized_algorithm<
	861	_Algorithm::__for_each,
	862	__iterator_pair<__tree_const_iterator<_Tp, _NodePtr, _DiffType>, __tree_const_iterator<_Tp, _NodePtr, _DiffType>>> {
	863	static const bool __has_algorithm = true;
	864
	865	using __iterator _LIBCPP_NODEBUG = __tree_const_iterator<_Tp, _NodePtr, _DiffType>;
	866
	867	template <class _Func, class _Proj>
	868	_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
	869	std::__tree_iterate_subrange(__first, __last, __func, __proj);
	870	}
	871	};
	872	#endif
	873
787	874	template <class _Tp, class _Compare>
788	875	#ifndef _LIBCPP_CXX03_LANG
789	876	_LIBCPP_DIAGNOSE_WARNING(!__is_invocable_v<_Compare const&, _Tp const&, _Tp const&>,
...	...	@@ -794,21 +881,20 @@ int __diagnose_non_const_comparator();
794	881	template <class _Tp, class _Compare, class _Allocator>
795	882	class __tree {
796	883	public:
797		using value_type = __get_node_value_type_t<_Tp>;
798		typedef _Compare value_compare;
799		typedef _Allocator allocator_type;
	884	using value_type = __get_node_value_type_t<_Tp>;
	885	using value_compare = _Compare;
	886	using allocator_type = _Allocator;
800	887
801	888	private:
802		typedef allocator_traits<allocator_type> __alloc_traits;
803		using key_type = __get_tree_key_type_t<_Tp>;
	889	using __alloc_traits _LIBCPP_NODEBUG = allocator_traits<allocator_type>;
	890	using key_type = __get_tree_key_type_t<_Tp>;
804	891
805	892	public:
806		typedef typename __alloc_traits::pointer pointer;
807		typedef typename __alloc_traits::const_pointer const_pointer;
808		typedef typename __alloc_traits::size_type size_type;
809		typedef typename __alloc_traits::difference_type difference_type;
	893	using pointer = typename __alloc_traits::pointer;
	894	using const_pointer = typename __alloc_traits::const_pointer;
	895	using size_type = typename __alloc_traits::size_type;
	896	using difference_type = typename __alloc_traits::difference_type;
810	897
811		public:
812	898	using __void_pointer _LIBCPP_NODEBUG = typename __alloc_traits::void_pointer;
813	899
814	900	using __node _LIBCPP_NODEBUG = __tree_node<_Tp, __void_pointer>;
...	...	@@ -821,22 +907,8 @@ public:
821	907	using __end_node_t _LIBCPP_NODEBUG = __tree_end_node<__node_base_pointer>;
822	908	using __end_node_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<__void_pointer, __end_node_t>;
823	909
824		using __parent_pointer _LIBCPP_NODEBUG = __end_node_pointer; // TODO: Remove this once the uses in <map> are removed
825
826		typedef __rebind_alloc<__alloc_traits, __node> __node_allocator;
827		typedef allocator_traits<__node_allocator> __node_traits;
828
829		// TODO(LLVM 22): Remove this check
830		#ifndef _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
831		static_assert(sizeof(__node_base_pointer) == sizeof(__end_node_pointer) && _LIBCPP_ALIGNOF(__node_base_pointer) ==
832		_LIBCPP_ALIGNOF(__end_node_pointer),
833		"It looks like you are using std::__tree (an implementation detail for (multi)map/set) with a fancy "
834		"pointer type that thas a different representation depending on whether it points to a __tree base "
835		"pointer or a __tree node pointer (both of which are implementation details of the standard library). "
836		"This means that your ABI is being broken between LLVM 19 and LLVM 20. If you don't care about your "
837		"ABI being broken, define the _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB macro to silence this "
838		"diagnostic.");
839		#endif
	910	using __node_allocator _LIBCPP_NODEBUG = __rebind_alloc<__alloc_traits, __node>;
	911	using __node_traits _LIBCPP_NODEBUG = allocator_traits<__node_allocator>;
840	912
841	913	private:
842	914	// check for sane allocator pointer rebinding semantics. Rebinding the
...	...	@@ -844,8 +916,8 @@ private:
844	916	// the pointer using 'pointer_traits'.
845	917	static_assert(is_same<__node_pointer, typename __node_traits::pointer>::value,
846	918	"Allocator does not rebind pointers in a sane manner.");
847		typedef __rebind_alloc<__node_traits, __node_base> __node_base_allocator;
848		typedef allocator_traits<__node_base_allocator> __node_base_traits;
	919	using __node_base_allocator _LIBCPP_NODEBUG = __rebind_alloc<__node_traits, __node_base>;
	920	using __node_base_traits _LIBCPP_NODEBUG = allocator_traits<__node_base_allocator>;
849	921	static_assert(is_same<__node_base_pointer, typename __node_base_traits::pointer>::value,
850	922	"Allocator does not rebind pointers in a sane manner.");
851	923
...	...	@@ -865,17 +937,11 @@ public:
865	937
866	938	private:
867	939	_LIBCPP_HIDE_FROM_ABI const __node_allocator& __node_alloc() const _NOEXCEPT { return __node_alloc_; }
868		_LIBCPP_HIDE_FROM_ABI __end_node_pointer& __begin_node() _NOEXCEPT { return __begin_node_; }
869		_LIBCPP_HIDE_FROM_ABI const __end_node_pointer& __begin_node() const _NOEXCEPT { return __begin_node_; }
870	940
871	941	public:
872	942	_LIBCPP_HIDE_FROM_ABI allocator_type __alloc() const _NOEXCEPT { return allocator_type(__node_alloc()); }
873	943
874		private:
875		_LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __size_; }
876
877		public:
878		_LIBCPP_HIDE_FROM_ABI const size_type& size() const _NOEXCEPT { return __size_; }
	944	_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
879	945	_LIBCPP_HIDE_FROM_ABI value_compare& value_comp() _NOEXCEPT { return __value_comp_; }
880	946	_LIBCPP_HIDE_FROM_ABI const value_compare& value_comp() const _NOEXCEPT { return __value_comp_; }
881	947
...	...	@@ -888,32 +954,61 @@ public:
888	954	return std::addressof(__end_node()->__left_);
889	955	}
890	956
891		typedef __tree_iterator<_Tp, __node_pointer, difference_type> iterator;
892		typedef __tree_const_iterator<_Tp, __node_pointer, difference_type> const_iterator;
	957	using iterator = __tree_iterator<_Tp, __node_pointer, difference_type>;
	958	using const_iterator = __tree_const_iterator<_Tp, __node_pointer, difference_type>;
893	959
894	960	_LIBCPP_HIDE_FROM_ABI explicit __tree(const value_compare& __comp) _NOEXCEPT_(
895		is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_copy_constructible<value_compare>::value);
896		_LIBCPP_HIDE_FROM_ABI explicit __tree(const allocator_type& __a);
897		_LIBCPP_HIDE_FROM_ABI __tree(const value_compare& __comp, const allocator_type& __a);
	961	is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_copy_constructible<value_compare>::value)
	962	: __size_(0), __value_comp_(__comp) {
	963	__begin_node_ = __end_node();
	964	}
	965
	966	_LIBCPP_HIDE_FROM_ABI explicit __tree(const allocator_type& __a)
	967	: __begin_node_(), __node_alloc_(__node_allocator(__a)), __size_(0) {
	968	__begin_node_ = __end_node();
	969	}
	970
	971	_LIBCPP_HIDE_FROM_ABI __tree(const value_compare& __comp, const allocator_type& __a)
	972	: __begin_node_(), __node_alloc_(__node_allocator(__a)), __size_(0), __value_comp_(__comp) {
	973	__begin_node_ = __end_node();
	974	}
	975
898	976	_LIBCPP_HIDE_FROM_ABI __tree(const __tree& __t);
	977
	978	_LIBCPP_HIDE_FROM_ABI __tree(const __tree& __other, const allocator_type& __alloc)
	979	: __begin_node_(__end_node()), __node_alloc_(__alloc), __size_(0), __value_comp_(__other.value_comp()) {
	980	if (__other.size() == 0)
	981	return;
	982
	983	*__root_ptr() = static_cast<__node_base_pointer>(__copy_construct_tree(__other.__root()));
	984	__root()->__parent_ = __end_node();
	985	__begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
	986	__size_ = __other.size();
	987	}
	988
899	989	_LIBCPP_HIDE_FROM_ABI __tree& operator=(const __tree& __t);
900	990	template <class _ForwardIterator>
901	991	_LIBCPP_HIDE_FROM_ABI void __assign_unique(_ForwardIterator __first, _ForwardIterator __last);
902		template <class _InputIterator>
903		_LIBCPP_HIDE_FROM_ABI void __assign_multi(_InputIterator __first, _InputIterator __last);
904	992	_LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t) _NOEXCEPT_(
905	993	is_nothrow_move_constructible<__node_allocator>::value&& is_nothrow_move_constructible<value_compare>::value);
906	994	_LIBCPP_HIDE_FROM_ABI __tree(__tree&& __t, const allocator_type& __a);
	995
907	996	_LIBCPP_HIDE_FROM_ABI __tree& operator=(__tree&& __t)
908	997	_NOEXCEPT_(is_nothrow_move_assignable<value_compare>::value &&
909	998	((__node_traits::propagate_on_container_move_assignment::value &&
910	999	is_nothrow_move_assignable<__node_allocator>::value) \|\|
911		allocator_traits<__node_allocator>::is_always_equal::value));
	1000	allocator_traits<__node_allocator>::is_always_equal::value)) {
	1001	__move_assign(__t, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
	1002	return *this;
	1003	}
912	1004
913		_LIBCPP_HIDE_FROM_ABI ~__tree();
	1005	_LIBCPP_HIDE_FROM_ABI ~__tree() {
	1006	static_assert(is_copy_constructible<value_compare>::value, "Comparator must be copy-constructible.");
	1007	destroy(__root());
	1008	}
914	1009
915		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__begin_node()); }
916		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return const_iterator(__begin_node()); }
	1010	_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__begin_node_); }
	1011	_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return const_iterator(__begin_node_); }
917	1012	_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(__end_node()); }
918	1013	_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(__end_node()); }
919	1014
...	...	@@ -931,116 +1026,151 @@ public:
931	1026	_NOEXCEPT_(__is_nothrow_swappable_v<value_compare>);
932	1027	#endif
933	1028
934		template <class _Key, class... _Args>
935		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_key_args(_Key const&, _Args&&... __args);
936		template <class _Key, class... _Args>
937		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_hint_unique_key_args(const_iterator, _Key const&, _Args&&...);
938
939		template <class... _Args>
940		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_impl(_Args&&... __args);
941
942		template <class... _Args>
943		_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique_impl(const_iterator __p, _Args&&... __args);
944
945	1029	template <class... _Args>
946	1030	_LIBCPP_HIDE_FROM_ABI iterator __emplace_multi(_Args&&... __args);
947	1031
948	1032	template <class... _Args>
949	1033	_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_multi(const_iterator __p, _Args&&... __args);
950	1034
951		template <class _Pp>
952		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Pp&& __x) {
953		return __emplace_unique_extract_key(std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
954		}
955
956		template <class _First,
957		class _Second,
958		__enable_if_t<__can_extract_map_key<_First, key_type, value_type>::value, int> = 0>
959		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_First&& __f, _Second&& __s) {
960		return __emplace_unique_key_args(__f, std::forward<_First>(__f), std::forward<_Second>(__s));
961		}
962
963	1035	template <class... _Args>
964	1036	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique(_Args&&... __args) {
965		return __emplace_unique_impl(std::forward<_Args>(__args)...);
966		}
967
968		template <class _Pp>
969		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_fail_tag) {
970		return __emplace_unique_impl(std::forward<_Pp>(__x));
971		}
972
973		template <class _Pp>
974		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_self_tag) {
975		return __emplace_unique_key_args(__x, std::forward<_Pp>(__x));
976		}
977
978		template <class _Pp>
979		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_unique_extract_key(_Pp&& __x, __extract_key_first_tag) {
980		return __emplace_unique_key_args(__x.first, std::forward<_Pp>(__x));
981		}
982
983		template <class _Pp>
984		_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _Pp&& __x) {
985		return __emplace_hint_unique_extract_key(__p, std::forward<_Pp>(__x), __can_extract_key<_Pp, key_type>());
986		}
987
988		template <class _First,
989		class _Second,
990		__enable_if_t<__can_extract_map_key<_First, key_type, value_type>::value, int> = 0>
991		_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _First&& __f, _Second&& __s) {
992		return __emplace_hint_unique_key_args(__p, __f, std::forward<_First>(__f), std::forward<_Second>(__s)).first;
	1037	return std::__try_key_extraction<key_type>(
	1038	[this](const key_type& __key, _Args&&... __args2) {
	1039	auto [__parent, __child] = __find_equal(__key);
	1040	__node_pointer __r = static_cast<__node_pointer>(__child);
	1041	bool __inserted = false;
	1042	if (__child == nullptr) {
	1043	__node_holder __h = __construct_node(std::forward<_Args>(__args2)...);
	1044	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
	1045	__r = __h.release();
	1046	__inserted = true;
	1047	}
	1048	return pair<iterator, bool>(iterator(__r), __inserted);
	1049	},
	1050	[this](_Args&&... __args2) {
	1051	__node_holder __h = __construct_node(std::forward<_Args>(__args2)...);
	1052	auto [__parent, __child] = __find_equal(__h->__get_value());
	1053	__node_pointer __r = static_cast<__node_pointer>(__child);
	1054	bool __inserted = false;
	1055	if (__child == nullptr) {
	1056	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
	1057	__r = __h.release();
	1058	__inserted = true;
	1059	}
	1060	return pair<iterator, bool>(iterator(__r), __inserted);
	1061	},
	1062	std::forward<_Args>(__args)...);
993	1063	}
994	1064
995	1065	template <class... _Args>
996		_LIBCPP_HIDE_FROM_ABI iterator __emplace_hint_unique(const_iterator __p, _Args&&... __args) {
997		return __emplace_hint_unique_impl(__p, std::forward<_Args>(__args)...);
	1066	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace_hint_unique(const_iterator __p, _Args&&... __args) {
	1067	return std::__try_key_extraction<key_type>(
	1068	[this, __p](const key_type& __key, _Args&&... __args2) {
	1069	__node_base_pointer __dummy;
	1070	auto [__parent, __child] = __find_equal(__p, __dummy, __key);
	1071	__node_pointer __r = static_cast<__node_pointer>(__child);
	1072	bool __inserted = false;
	1073	if (__child == nullptr) {
	1074	__node_holder __h = __construct_node(std::forward<_Args>(__args2)...);
	1075	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
	1076	__r = __h.release();
	1077	__inserted = true;
	1078	}
	1079	return pair<iterator, bool>(iterator(__r), __inserted);
	1080	},
	1081	[this, __p](_Args&&... __args2) {
	1082	__node_holder __h = __construct_node(std::forward<_Args>(__args2)...);
	1083	__node_base_pointer __dummy;
	1084	auto [__parent, __child] = __find_equal(__p, __dummy, __h->__get_value());
	1085	__node_pointer __r = static_cast<__node_pointer>(__child);
	1086	if (__child == nullptr) {
	1087	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
	1088	__r = __h.release();
	1089	}
	1090	return pair<iterator, bool>(iterator(__r), __child == nullptr);
	1091	},
	1092	std::forward<_Args>(__args)...);
998	1093	}
999	1094
1000		template <class _Pp>
1001		_LIBCPP_HIDE_FROM_ABI iterator
1002		__emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_fail_tag) {
1003		return __emplace_hint_unique_impl(__p, std::forward<_Pp>(__x));
1004		}
	1095	template <class _InIter, class _Sent>
	1096	_LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
	1097	if (__first == __last)
	1098	return;
1005	1099
1006		template <class _Pp>
1007		_LIBCPP_HIDE_FROM_ABI iterator
1008		__emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_self_tag) {
1009		return __emplace_hint_unique_key_args(__p, __x, std::forward<_Pp>(__x)).first;
1010		}
	1100	if (__root() == nullptr) { // Make sure we always have a root node
	1101	__insert_node_at(
	1102	__end_node(), __end_node()->__left_, static_cast<__node_base_pointer>(__construct_node(*__first).release()));
	1103	++__first;
	1104	}
1011	1105
1012		template <class _Pp>
1013		_LIBCPP_HIDE_FROM_ABI iterator
1014		__emplace_hint_unique_extract_key(const_iterator __p, _Pp&& __x, __extract_key_first_tag) {
1015		return __emplace_hint_unique_key_args(__p, __x.first, std::forward<_Pp>(__x)).first;
1016		}
	1106	auto __max_node = static_cast<__node_pointer>(std::__tree_max(static_cast<__node_base_pointer>(__root())));
1017	1107
1018		template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
1019		_LIBCPP_HIDE_FROM_ABI void
1020		__insert_unique_from_orphaned_node(const_iterator __p, __get_node_value_type_t<_Tp>&& __value) {
1021		__emplace_hint_unique(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
	1108	for (; __first != __last; ++__first) {
	1109	__node_holder __nd = __construct_node(*__first);
	1110	// Always check the max node first. This optimizes for sorted ranges inserted at the end.
	1111	if (!value_comp()(__nd->__get_value(), __max_node->__get_value())) { // __node >= __max_val
	1112	__insert_node_at(static_cast<__end_node_pointer>(__max_node),
	1113	__max_node->__right_,
	1114	static_cast<__node_base_pointer>(__nd.get()));
	1115	__max_node = __nd.release();
	1116	} else {
	1117	__end_node_pointer __parent;
	1118	__node_base_pointer& __child = __find_leaf_high(__parent, __nd->__get_value());
	1119	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd.release()));
	1120	}
	1121	}
1022	1122	}
1023	1123
1024		template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
1025		_LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(const_iterator __p, _Tp&& __value) {
1026		__emplace_hint_unique(__p, std::move(__value));
1027		}
	1124	template <class _InIter, class _Sent>
	1125	_LIBCPP_HIDE_FROM_ABI void __insert_range_unique(_InIter __first, _Sent __last) {
	1126	if (__first == __last)
	1127	return;
1028	1128
1029		template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
1030		_LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, value_type&& __value) {
1031		__emplace_hint_multi(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
1032		}
	1129	if (__root() == nullptr) {
	1130	__insert_node_at(
	1131	__end_node(), __end_node()->__left_, static_cast<__node_base_pointer>(__construct_node(*__first).release()));
	1132	++__first;
	1133	}
1033	1134
1034		template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
1035		_LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, _Tp&& __value) {
1036		__emplace_hint_multi(__p, std::move(__value));
	1135	auto __max_node = static_cast<__node_pointer>(std::__tree_max(static_cast<__node_base_pointer>(__root())));
	1136
	1137	using __reference = decltype(*__first);
	1138
	1139	for (; __first != __last; ++__first) {
	1140	std::__try_key_extraction<key_type>(
	1141	[this, &__max_node](const key_type& __key, __reference&& __val) {
	1142	if (value_comp()(__max_node->__get_value(), __key)) { // __key > __max_node
	1143	__node_holder __nd = __construct_node(std::forward<__reference>(__val));
	1144	__insert_node_at(static_cast<__end_node_pointer>(__max_node),
	1145	__max_node->__right_,
	1146	static_cast<__node_base_pointer>(__nd.get()));
	1147	__max_node = __nd.release();
	1148	} else {
	1149	auto [__parent, __child] = __find_equal(__key);
	1150	if (__child == nullptr) {
	1151	__node_holder __nd = __construct_node(std::forward<__reference>(__val));
	1152	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd.release()));
	1153	}
	1154	}
	1155	},
	1156	[this, &__max_node](__reference&& __val) {
	1157	__node_holder __nd = __construct_node(std::forward<__reference>(__val));
	1158	if (value_comp()(__max_node->__get_value(), __nd->__get_value())) { // __node > __max_node
	1159	__insert_node_at(static_cast<__end_node_pointer>(__max_node),
	1160	__max_node->__right_,
	1161	static_cast<__node_base_pointer>(__nd.get()));
	1162	__max_node = __nd.release();
	1163	} else {
	1164	auto [__parent, __child] = __find_equal(__nd->__get_value());
	1165	if (__child == nullptr) {
	1166	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd.release()));
	1167	}
	1168	}
	1169	},
	1170	*__first);
	1171	}
1037	1172	}
1038	1173
1039		_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __node_assign_unique(const value_type& __v, __node_pointer __dest);
1040
1041		_LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
1042		_LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(const_iterator __p, __node_pointer __nd);
1043
1044	1174	_LIBCPP_HIDE_FROM_ABI iterator __remove_node_pointer(__node_pointer) _NOEXCEPT;
1045	1175
1046	1176	#if _LIBCPP_STD_VER >= 17
...	...	@@ -1048,15 +1178,15 @@ public:
1048	1178	_LIBCPP_HIDE_FROM_ABI _InsertReturnType __node_handle_insert_unique(_NodeHandle&&);
1049	1179	template <class _NodeHandle>
1050	1180	_LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_unique(const_iterator, _NodeHandle&&);
1051		template <class _Tree>
1052		_LIBCPP_HIDE_FROM_ABI void __node_handle_merge_unique(_Tree& __source);
	1181	template <class _Comp2>
	1182	_LIBCPP_HIDE_FROM_ABI void __node_handle_merge_unique(__tree<_Tp, _Comp2, _Allocator>& __source);
1053	1183
1054	1184	template <class _NodeHandle>
1055	1185	_LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(_NodeHandle&&);
1056	1186	template <class _NodeHandle>
1057	1187	_LIBCPP_HIDE_FROM_ABI iterator __node_handle_insert_multi(const_iterator, _NodeHandle&&);
1058		template <class _Tree>
1059		_LIBCPP_HIDE_FROM_ABI void __node_handle_merge_multi(_Tree& __source);
	1188	template <class _Comp2>
	1189	_LIBCPP_HIDE_FROM_ABI void __node_handle_merge_multi(__tree<_Tp, _Comp2, _Allocator>& __source);
1060	1190
1061	1191	template <class _NodeHandle>
1062	1192	_LIBCPP_HIDE_FROM_ABI _NodeHandle __node_handle_extract(key_type const&);
...	...	@@ -1075,41 +1205,157 @@ public:
1075	1205	__insert_node_at(__end_node_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT;
1076	1206
1077	1207	template <class _Key>
1078		_LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __v);
	1208	_LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __key) {
	1209	auto [__, __match] = __find_equal(__key);
	1210	if (__match == nullptr)
	1211	return end();
	1212	return iterator(static_cast<__node_pointer>(__match));
	1213	}
	1214
1079	1215	template <class _Key>
1080		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __v) const;
	1216	_LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __key) const {
	1217	auto [__, __match] = __find_equal(__key);
	1218	if (__match == nullptr)
	1219	return end();
	1220	return const_iterator(static_cast<__node_pointer>(__match));
	1221	}
1081	1222
1082	1223	template <class _Key>
1083	1224	_LIBCPP_HIDE_FROM_ABI size_type __count_unique(const _Key& __k) const;
1084	1225	template <class _Key>
1085	1226	_LIBCPP_HIDE_FROM_ABI size_type __count_multi(const _Key& __k) const;
1086	1227
	1228	template <bool _LowerBound, class _Key>
	1229	_LIBCPP_HIDE_FROM_ABI __end_node_pointer __lower_upper_bound_unique_impl(const _Key& __v) const {
	1230	auto __rt = __root();
	1231	auto __result = __end_node();
	1232	auto __comp = __lazy_synth_three_way_comparator<_Compare, _Key, value_type>(value_comp());
	1233	while (__rt != nullptr) {
	1234	auto __comp_res = __comp(__v, __rt->__get_value());
	1235
	1236	if (__comp_res.__less()) {
	1237	__result = static_cast<__end_node_pointer>(__rt);
	1238	__rt = static_cast<__node_pointer>(__rt->__left_);
	1239	} else if (__comp_res.__greater()) {
	1240	__rt = static_cast<__node_pointer>(__rt->__right_);
	1241	} else if _LIBCPP_CONSTEXPR (_LowerBound) {
	1242	return static_cast<__end_node_pointer>(__rt);
	1243	} else {
	1244	return __rt->__right_ ? static_cast<__end_node_pointer>(std::__tree_min(__rt->__right_)) : __result;
	1245	}
	1246	}
	1247	return __result;
	1248	}
	1249
	1250	// Compatibility escape hatch for comparators that are not strict weak orderings. This
	1251	// can be removed for the LLVM 23 release.
	1252	#if defined(_LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND)
1087	1253	template <class _Key>
1088		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Key& __v) {
1089		return __lower_bound(__v, __root(), __end_node());
	1254	_LIBCPP_HIDE_FROM_ABI __end_node_pointer __lower_bound_unique_compat_impl(const _Key& __v) const {
	1255	auto __rt = __root();
	1256	auto __result = __end_node();
	1257	while (__rt != nullptr) {
	1258	if (!value_comp()(__rt->__get_value(), __v)) {
	1259	__result = std::__static_fancy_pointer_cast<__end_node_pointer>(__rt);
	1260	__rt = std::__static_fancy_pointer_cast<__node_pointer>(__rt->__left_);
	1261	} else {
	1262	__rt = std::__static_fancy_pointer_cast<__node_pointer>(__rt->__right_);
	1263	}
	1264	}
	1265	return __result;
1090	1266	}
	1267
1091	1268	template <class _Key>
1092		_LIBCPP_HIDE_FROM_ABI iterator __lower_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result);
	1269	_LIBCPP_HIDE_FROM_ABI __end_node_pointer __upper_bound_unique_compat_impl(const _Key& __v) const {
	1270	auto __rt = __root();
	1271	auto __result = __end_node();
	1272	while (__rt != nullptr) {
	1273	if (value_comp()(__v, __rt->__get_value())) {
	1274	__result = std::__static_fancy_pointer_cast<__end_node_pointer>(__rt);
	1275	__rt = std::__static_fancy_pointer_cast<__node_pointer>(__rt->__left_);
	1276	} else {
	1277	__rt = std::__static_fancy_pointer_cast<__node_pointer>(__rt->__right_);
	1278	}
	1279	}
	1280	return __result;
	1281	}
	1282	#endif // _LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND
	1283
1093	1284	template <class _Key>
1094		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Key& __v) const {
1095		return __lower_bound(__v, __root(), __end_node());
	1285	_LIBCPP_HIDE_FROM_ABI iterator __lower_bound_unique(const _Key& __v) {
	1286	#if defined(_LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND)
	1287	return iterator(__lower_bound_unique_compat_impl(__v));
	1288	#else
	1289	return iterator(__lower_upper_bound_unique_impl<true>(__v));
	1290	#endif
1096	1291	}
	1292
	1293	template <class _Key>
	1294	_LIBCPP_HIDE_FROM_ABI const_iterator __lower_bound_unique(const _Key& __v) const {
	1295	#if defined(_LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND)
	1296	return const_iterator(__lower_bound_unique_compat_impl(__v));
	1297	#else
	1298	return const_iterator(__lower_upper_bound_unique_impl<true>(__v));
	1299	#endif
	1300	}
	1301
	1302	template <class _Key>
	1303	_LIBCPP_HIDE_FROM_ABI iterator __upper_bound_unique(const _Key& __v) {
	1304	#if defined(_LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND)
	1305	return iterator(__upper_bound_unique_compat_impl(__v));
	1306	#else
	1307	return iterator(__lower_upper_bound_unique_impl<false>(__v));
	1308	#endif
	1309	}
	1310
	1311	template <class _Key>
	1312	_LIBCPP_HIDE_FROM_ABI const_iterator __upper_bound_unique(const _Key& __v) const {
	1313	#if defined(_LIBCPP_ENABLE_LEGACY_TREE_LOWER_UPPER_BOUND)
	1314	return iterator(__upper_bound_unique_compat_impl(__v));
	1315	#else
	1316	return iterator(__lower_upper_bound_unique_impl<false>(__v));
	1317	#endif
	1318	}
	1319
	1320	private:
	1321	template <class _Key>
	1322	_LIBCPP_HIDE_FROM_ABI iterator
	1323	__lower_bound_multi(const _Key& __v, __node_pointer __root, __end_node_pointer __result);
	1324
1097	1325	template <class _Key>
1098	1326	_LIBCPP_HIDE_FROM_ABI const_iterator
1099		__lower_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) const;
	1327	__lower_bound_multi(const _Key& __v, __node_pointer __root, __end_node_pointer __result) const;
	1328
	1329	public:
1100	1330	template <class _Key>
1101		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Key& __v) {
1102		return __upper_bound(__v, __root(), __end_node());
	1331	_LIBCPP_HIDE_FROM_ABI iterator __lower_bound_multi(const _Key& __v) {
	1332	return __lower_bound_multi(__v, __root(), __end_node());
1103	1333	}
1104	1334	template <class _Key>
1105		_LIBCPP_HIDE_FROM_ABI iterator __upper_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result);
	1335	_LIBCPP_HIDE_FROM_ABI const_iterator __lower_bound_multi(const _Key& __v) const {
	1336	return __lower_bound_multi(__v, __root(), __end_node());
	1337	}
	1338
	1339	template <class _Key>
	1340	_LIBCPP_HIDE_FROM_ABI iterator __upper_bound_multi(const _Key& __v) {
	1341	return __upper_bound_multi(__v, __root(), __end_node());
	1342	}
	1343
1106	1344	template <class _Key>
1107		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Key& __v) const {
1108		return __upper_bound(__v, __root(), __end_node());
	1345	_LIBCPP_HIDE_FROM_ABI const_iterator __upper_bound_multi(const _Key& __v) const {
	1346	return __upper_bound_multi(__v, __root(), __end_node());
1109	1347	}
	1348
	1349	private:
	1350	template <class _Key>
	1351	_LIBCPP_HIDE_FROM_ABI iterator
	1352	__upper_bound_multi(const _Key& __v, __node_pointer __root, __end_node_pointer __result);
	1353
1110	1354	template <class _Key>
1111	1355	_LIBCPP_HIDE_FROM_ABI const_iterator
1112		__upper_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) const;
	1356	__upper_bound_multi(const _Key& __v, __node_pointer __root, __end_node_pointer __result) const;
	1357
	1358	public:
1113	1359	template <class _Key>
1114	1360	_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> __equal_range_unique(const _Key& __k);
1115	1361	template <class _Key>
...	...	@@ -1120,22 +1366,24 @@ public:
1120	1366	template <class _Key>
1121	1367	_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> __equal_range_multi(const _Key& __k) const;
1122	1368
1123		typedef __tree_node_destructor<__node_allocator> _Dp;
1124		typedef unique_ptr<__node, _Dp> __node_holder;
	1369	using _Dp _LIBCPP_NODEBUG = __tree_node_destructor<__node_allocator>;
	1370	using __node_holder _LIBCPP_NODEBUG = unique_ptr<__node, _Dp>;
1125	1371
1126	1372	_LIBCPP_HIDE_FROM_ABI __node_holder remove(const_iterator __p) _NOEXCEPT;
1127	1373
1128	1374	// FIXME: Make this function const qualified. Unfortunately doing so
1129	1375	// breaks existing code which uses non-const callable comparators.
1130	1376	template <class _Key>
1131		_LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__end_node_pointer& __parent, const _Key& __v);
	1377	_LIBCPP_HIDE_FROM_ABI pair<__end_node_pointer, __node_base_pointer&> __find_equal(const _Key& __v);
	1378
1132	1379	template <class _Key>
1133		_LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__end_node_pointer& __parent, const _Key& __v) const {
1134		return const_cast<__tree*>(this)->__find_equal(__parent, __v);
	1380	_LIBCPP_HIDE_FROM_ABI pair<__end_node_pointer, __node_base_pointer&> __find_equal(const _Key& __v) const {
	1381	return const_cast<__tree*>(this)->__find_equal(__v);
1135	1382	}
	1383
1136	1384	template <class _Key>
1137		_LIBCPP_HIDE_FROM_ABI __node_base_pointer&
1138		__find_equal(const_iterator __hint, __end_node_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v);
	1385	_LIBCPP_HIDE_FROM_ABI pair<__end_node_pointer, __node_base_pointer&>
	1386	__find_equal(const_iterator __hint, __node_base_pointer& __dummy, const _Key& __v);
1139	1387
1140	1388	_LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree& __t) {
1141	1389	__copy_assign_alloc(__t, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
...	...	@@ -1160,7 +1408,7 @@ private:
1160	1408	_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args);
1161	1409
1162	1410	// TODO: Make this _LIBCPP_HIDE_FROM_ABI
1163		_LIBCPP_HIDDEN void destroy(__node_pointer __nd) _NOEXCEPT;
	1411	_LIBCPP_HIDDEN void destroy(__node_pointer __nd) _NOEXCEPT { (__tree_deleter(__node_alloc_))(__nd); }
1164	1412
1165	1413	_LIBCPP_HIDE_FROM_ABI void __move_assign(__tree& __t, false_type);
1166	1414	_LIBCPP_HIDE_FROM_ABI void __move_assign(__tree& __t, true_type) _NOEXCEPT_(
...	...	@@ -1178,7 +1426,7 @@ private:
1178	1426	}
1179	1427	_LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__tree&, false_type) _NOEXCEPT {}
1180	1428
1181		template <class _From, class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
	1429	template <class _From, class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
1182	1430	_LIBCPP_HIDE_FROM_ABI static void __assign_value(__get_node_value_type_t<value_type>& __lhs, _From&& __rhs) {
1183	1431	using __key_type = __remove_const_t<typename value_type::first_type>;
1184	1432
...	...	@@ -1188,166 +1436,203 @@ private:
1188	1436	__lhs.second = std::forward<_From>(__rhs).second;
1189	1437	}
1190	1438
1191		template <class _To, class _From, class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
	1439	template <class _To, class _From, class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
1192	1440	_LIBCPP_HIDE_FROM_ABI static void __assign_value(_To& __lhs, _From&& __rhs) {
1193	1441	__lhs = std::forward<_From>(__rhs);
1194	1442	}
1195	1443
1196		struct _DetachedTreeCache {
1197		_LIBCPP_HIDE_FROM_ABI explicit _DetachedTreeCache(__tree* __t) _NOEXCEPT
1198		: __t_(__t),
1199		__cache_root_(__detach_from_tree(__t)) {
1200		__advance();
	1444	class __tree_deleter {
	1445	__node_allocator& __alloc_;
	1446
	1447	public:
	1448	using pointer = __node_pointer;
	1449
	1450	_LIBCPP_HIDE_FROM_ABI __tree_deleter(__node_allocator& __alloc) : __alloc_(__alloc) {}
	1451
	1452	#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
	1453	_LIBCPP_HIDE_FROM_ABI
	1454	#endif
	1455	void
	1456	operator()(__node_pointer __ptr) {
	1457	if (!__ptr)
	1458	return;
	1459
	1460	(*this)(static_cast<__node_pointer>(__ptr->__left_));
	1461
	1462	auto __right = __ptr->__right_;
	1463
	1464	__node_traits::destroy(__alloc_, std::addressof(__ptr->__get_value()));
	1465	__node_traits::deallocate(__alloc_, __ptr, 1);
	1466
	1467	(*this)(static_cast<__node_pointer>(__right));
1201	1468	}
	1469	};
	1470
	1471	// This copy construction will always produce a correct red-black-tree assuming the incoming tree is correct, since we
	1472	// copy the exact structure 1:1. Since this is for copy construction _only_ we know that we get a correct tree. If we
	1473	// didn't get a correct tree, the invariants of __tree are broken and we have a much bigger problem than an improperly
	1474	// balanced tree.
	1475	template <class _NodeConstructor>
	1476	#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
	1477	_LIBCPP_HIDE_FROM_ABI
	1478	#endif
	1479	__node_pointer __construct_from_tree(__node_pointer __src, _NodeConstructor __construct) {
	1480	if (!__src)
	1481	return nullptr;
1202	1482
1203		_LIBCPP_HIDE_FROM_ABI __node_pointer __get() const _NOEXCEPT { return __cache_elem_; }
	1483	__node_holder __new_node = __construct(__src->__get_value());
1204	1484
1205		_LIBCPP_HIDE_FROM_ABI void __advance() _NOEXCEPT {
1206		__cache_elem_ = __cache_root_;
1207		if (__cache_root_) {
1208		__cache_root_ = __detach_next(__cache_root_);
1209		}
	1485	unique_ptr<__node, __tree_deleter> __left(
	1486	__construct_from_tree(static_cast<__node_pointer>(__src->__left_), __construct), __node_alloc_);
	1487	__node_pointer __right = __construct_from_tree(static_cast<__node_pointer>(__src->__right_), __construct);
	1488
	1489	__node_pointer __new_node_ptr = __new_node.release();
	1490
	1491	__new_node_ptr->__is_black_ = __src->__is_black_;
	1492	__new_node_ptr->__left_ = static_cast<__node_base_pointer>(__left.release());
	1493	__new_node_ptr->__right_ = static_cast<__node_base_pointer>(__right);
	1494	if (__new_node_ptr->__left_)
	1495	__new_node_ptr->__left_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
	1496	if (__new_node_ptr->__right_)
	1497	__new_node_ptr->__right_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
	1498	return __new_node_ptr;
	1499	}
	1500
	1501	_LIBCPP_HIDE_FROM_ABI __node_pointer __copy_construct_tree(__node_pointer __src) {
	1502	return __construct_from_tree(__src, [this](const value_type& __val) { return __construct_node(__val); });
	1503	}
	1504
	1505	template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
	1506	_LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
	1507	return __construct_from_tree(__src, [this](value_type& __val) {
	1508	return __construct_node(const_cast<key_type&&>(__val.first), std::move(__val.second));
	1509	});
	1510	}
	1511
	1512	template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
	1513	_LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
	1514	return __construct_from_tree(__src, [this](value_type& __val) { return __construct_node(std::move(__val)); });
	1515	}
	1516
	1517	template <class _Assignment, class _ConstructionAlg>
	1518	// This copy assignment will always produce a correct red-black-tree assuming the incoming tree is correct, since our
	1519	// own tree is a red-black-tree and the incoming tree is a red-black-tree. The invariants of a red-black-tree are
	1520	// temporarily not met until all of the incoming red-black tree is copied.
	1521	#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
	1522	_LIBCPP_HIDE_FROM_ABI
	1523	#endif
	1524	__node_pointer __assign_from_tree(
	1525	__node_pointer __dest, __node_pointer __src, _Assignment __assign, _ConstructionAlg __construct_subtree) {
	1526	if (!__src) {
	1527	destroy(__dest);
	1528	return nullptr;
1210	1529	}
1211	1530
1212		_LIBCPP_HIDE_FROM_ABI ~_DetachedTreeCache() {
1213		__t_->destroy(__cache_elem_);
1214		if (__cache_root_) {
1215		while (__cache_root_->__parent_ != nullptr)
1216		__cache_root_ = static_cast<__node_pointer>(__cache_root_->__parent_);
1217		__t_->destroy(__cache_root_);
1218		}
	1531	__assign(__dest->__get_value(), __src->__get_value());
	1532	__dest->__is_black_ = __src->__is_black_;
	1533
	1534	// If we already have a left node in the destination tree, reuse it and copy-assign recursively
	1535	if (__dest->__left_) {
	1536	__dest->__left_ = static_cast<__node_base_pointer>(__assign_from_tree(
	1537	static_cast<__node_pointer>(__dest->__left_),
	1538	static_cast<__node_pointer>(__src->__left_),
	1539	__assign,
	1540	__construct_subtree));
	1541
	1542	// Otherwise, we must create new nodes; copy-construct from here on
	1543	} else if (__src->__left_) {
	1544	auto __new_left = __construct_subtree(static_cast<__node_pointer>(__src->__left_));
	1545	__dest->__left_ = static_cast<__node_base_pointer>(__new_left);
	1546	__new_left->__parent_ = static_cast<__end_node_pointer>(__dest);
1219	1547	}
1220	1548
1221		_DetachedTreeCache(_DetachedTreeCache const&) = delete;
1222		_DetachedTreeCache& operator=(_DetachedTreeCache const&) = delete;
	1549	// Identical to the left case above, just for the right nodes
	1550	if (__dest->__right_) {
	1551	__dest->__right_ = static_cast<__node_base_pointer>(__assign_from_tree(
	1552	static_cast<__node_pointer>(__dest->__right_),
	1553	static_cast<__node_pointer>(__src->__right_),
	1554	__assign,
	1555	__construct_subtree));
	1556	} else if (__src->__right_) {
	1557	auto __new_right = __construct_subtree(static_cast<__node_pointer>(__src->__right_));
	1558	__dest->__right_ = static_cast<__node_base_pointer>(__new_right);
	1559	__new_right->__parent_ = static_cast<__end_node_pointer>(__dest);
	1560	}
1223	1561
1224		private:
1225		_LIBCPP_HIDE_FROM_ABI static __node_pointer __detach_from_tree(__tree* __t) _NOEXCEPT;
1226		_LIBCPP_HIDE_FROM_ABI static __node_pointer __detach_next(__node_pointer) _NOEXCEPT;
	1562	return __dest;
	1563	}
1227	1564
1228		__tree* __t_;
1229		__node_pointer __cache_root_;
1230		__node_pointer __cache_elem_;
1231		};
1232		};
	1565	_LIBCPP_HIDE_FROM_ABI __node_pointer __copy_assign_tree(__node_pointer __dest, __node_pointer __src) {
	1566	return __assign_from_tree(
	1567	__dest,
	1568	__src,
	1569	[](value_type& __lhs, const value_type& __rhs) { __assign_value(__lhs, __rhs); },
	1570	[this](__node_pointer __nd) { return __copy_construct_tree(__nd); });
	1571	}
1233	1572
1234		template <class _Tp, class _Compare, class _Allocator>
1235		__tree<_Tp, _Compare, _Allocator>::__tree(const value_compare& __comp) _NOEXCEPT_(
1236		is_nothrow_default_constructible<__node_allocator>::value&& is_nothrow_copy_constructible<value_compare>::value)
1237		: __size_(0), __value_comp_(__comp) {
1238		__begin_node() = __end_node();
1239		}
	1573	_LIBCPP_HIDE_FROM_ABI __node_pointer __move_assign_tree(__node_pointer __dest, __node_pointer __src) {
	1574	return __assign_from_tree(
	1575	__dest,
	1576	__src,
	1577	[](value_type& __lhs, value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
	1578	[this](__node_pointer __nd) { return __move_construct_tree(__nd); });
	1579	}
1240	1580
1241		template <class _Tp, class _Compare, class _Allocator>
1242		__tree<_Tp, _Compare, _Allocator>::__tree(const allocator_type& __a)
1243		: __begin_node_(), __node_alloc_(__node_allocator(__a)), __size_(0) {
1244		__begin_node() = __end_node();
1245		}
	1581	friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<__tree> >;
	1582	};
1246	1583
	1584	#if _LIBCPP_STD_VER >= 14
1247	1585	template <class _Tp, class _Compare, class _Allocator>
1248		__tree<_Tp, _Compare, _Allocator>::__tree(const value_compare& __comp, const allocator_type& __a)
1249		: __begin_node_(), __node_alloc_(__node_allocator(__a)), __size_(0), __value_comp_(__comp) {
1250		__begin_node() = __end_node();
1251		}
	1586	struct __specialized_algorithm<_Algorithm::__for_each, __single_range<__tree<_Tp, _Compare, _Allocator> > > {
	1587	static const bool __has_algorithm = true;
1252	1588
1253		// Precondition: size() != 0
1254		template <class _Tp, class _Compare, class _Allocator>
1255		typename __tree<_Tp, _Compare, _Allocator>::__node_pointer
1256		__tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_from_tree(__tree* __t) _NOEXCEPT {
1257		__node_pointer __cache = static_cast<__node_pointer>(__t->__begin_node());
1258		__t->__begin_node() = __t->__end_node();
1259		__t->__end_node()->__left_->__parent_ = nullptr;
1260		__t->__end_node()->__left_ = nullptr;
1261		__t->size() = 0;
1262		// __cache->__left_ == nullptr
1263		if (__cache->__right_ != nullptr)
1264		__cache = static_cast<__node_pointer>(__cache->__right_);
1265		// __cache->__left_ == nullptr
1266		// __cache->__right_ == nullptr
1267		return __cache;
1268		}
	1589	using __node_pointer _LIBCPP_NODEBUG = typename __tree<_Tp, _Compare, _Allocator>::__node_pointer;
1269	1590
1270		// Precondition: __cache != nullptr
1271		// __cache->left_ == nullptr
1272		// __cache->right_ == nullptr
1273		// This is no longer a red-black tree
1274		template <class _Tp, class _Compare, class _Allocator>
1275		typename __tree<_Tp, _Compare, _Allocator>::__node_pointer
1276		__tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_next(__node_pointer __cache) _NOEXCEPT {
1277		if (__cache->__parent_ == nullptr)
1278		return nullptr;
1279		if (std::__tree_is_left_child(static_cast<__node_base_pointer>(__cache))) {
1280		__cache->__parent_->__left_ = nullptr;
1281		__cache = static_cast<__node_pointer>(__cache->__parent_);
1282		if (__cache->__right_ == nullptr)
1283		return __cache;
1284		return static_cast<__node_pointer>(std::__tree_leaf(__cache->__right_));
1285		}
1286		// __cache is right child
1287		__cache->__parent_unsafe()->__right_ = nullptr;
1288		__cache = static_cast<__node_pointer>(__cache->__parent_);
1289		if (__cache->__left_ == nullptr)
1290		return __cache;
1291		return static_cast<__node_pointer>(std::__tree_leaf(__cache->__left_));
1292		}
	1591	template <class _Tree, class _Func, class _Proj>
	1592	_LIBCPP_HIDE_FROM_ABI static auto operator()(_Tree&& __range, _Func __func, _Proj __proj) {
	1593	if (__range.size() != 0)
	1594	std::__tree_iterate_from_root<__copy_cvref_t<_Tree, typename __remove_cvref_t<_Tree>::value_type>>(
	1595	[](__node_pointer) { return false; }, __range.__root(), __func, __proj);
	1596	return std::make_pair(__range.end(), std::move(__func));
	1597	}
	1598	};
	1599	#endif
1293	1600
1294	1601	template <class _Tp, class _Compare, class _Allocator>
1295	1602	__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(const __tree& __t) {
1296		if (this != std::addressof(__t)) {
1297		value_comp() = __t.value_comp();
1298		__copy_assign_alloc(__t);
1299		__assign_multi(__t.begin(), __t.end());
1300		}
1301		return *this;
1302		}
	1603	if (this == std::addressof(__t))
	1604	return *this;
1303	1605
1304		template <class _Tp, class _Compare, class _Allocator>
1305		template <class _ForwardIterator>
1306		void __tree<_Tp, _Compare, _Allocator>::__assign_unique(_ForwardIterator __first, _ForwardIterator __last) {
1307		typedef iterator_traits<_ForwardIterator> _ITraits;
1308		typedef typename _ITraits::value_type _ItValueType;
1309		static_assert(
1310		is_same<_ItValueType, value_type>::value, "__assign_unique may only be called with the containers value type");
1311		static_assert(
1312		__has_forward_iterator_category<_ForwardIterator>::value, "__assign_unique requires a forward iterator");
1313		if (size() != 0) {
1314		_DetachedTreeCache __cache(this);
1315		for (; __cache.__get() != nullptr && __first != __last; ++__first) {
1316		if (__node_assign_unique(*__first, __cache.__get()).second)
1317		__cache.__advance();
1318		}
1319		}
1320		for (; __first != __last; ++__first)
1321		__emplace_unique(*__first);
1322		}
	1606	value_comp() = __t.value_comp();
	1607	__copy_assign_alloc(__t);
1323	1608
1324		template <class _Tp, class _Compare, class _Allocator>
1325		template <class _InputIterator>
1326		void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _InputIterator __last) {
1327		typedef iterator_traits<_InputIterator> _ITraits;
1328		typedef typename _ITraits::value_type _ItValueType;
1329		static_assert(
1330		is_same<_ItValueType, value_type>::value, "__assign_multi may only be called with the containers value_type");
1331		if (size() != 0) {
1332		_DetachedTreeCache __cache(this);
1333		for (; __cache.__get() && __first != __last; ++__first) {
1334		__assign_value(__cache.__get()->__value_, *__first);
1335		__node_insert_multi(__cache.__get());
1336		__cache.__advance();
1337		}
	1609	if (__size_ != 0) {
	1610	*__root_ptr() = static_cast<__node_base_pointer>(__copy_assign_tree(__root(), __t.__root()));
	1611	} else {
	1612	*__root_ptr() = static_cast<__node_base_pointer>(__copy_construct_tree(__t.__root()));
	1613	if (__root())
	1614	__root()->__parent_ = __end_node();
1338	1615	}
1339		const_iterator __e = end();
1340		for (; __first != __last; ++__first)
1341		__emplace_hint_multi(__e, *__first);
	1616	__begin_node_ =
	1617	__end_node()->__left_ ? static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_)) : __end_node();
	1618	__size_ = __t.size();
	1619
	1620	return *this;
1342	1621	}
1343	1622
1344	1623	template <class _Tp, class _Compare, class _Allocator>
1345	1624	__tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t)
1346		: __begin_node_(),
	1625	: __begin_node_(__end_node()),
1347	1626	__node_alloc_(__node_traits::select_on_container_copy_construction(__t.__node_alloc())),
1348	1627	__size_(0),
1349	1628	__value_comp_(__t.value_comp()) {
1350		__begin_node() = __end_node();
	1629	if (__t.size() == 0)
	1630	return;
	1631
	1632	*__root_ptr() = static_cast<__node_base_pointer>(__copy_construct_tree(__t.__root()));
	1633	__root()->__parent_ = __end_node();
	1634	__begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
	1635	__size_ = __t.size();
1351	1636	}
1352	1637
1353	1638	template <class _Tp, class _Compare, class _Allocator>
...	...	@@ -1358,33 +1643,38 @@ __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t) _NOEXCEPT_(
1358	1643	__node_alloc_(std::move(__t.__node_alloc_)),
1359	1644	__size_(__t.__size_),
1360	1645	__value_comp_(std::move(__t.__value_comp_)) {
1361		if (size() == 0)
1362		__begin_node() = __end_node();
	1646	if (__size_ == 0)
	1647	__begin_node_ = __end_node();
1363	1648	else {
1364	1649	__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
1365		__t.__begin_node() = __t.__end_node();
	1650	__t.__begin_node_ = __t.__end_node();
1366	1651	__t.__end_node()->__left_ = nullptr;
1367		__t.size() = 0;
	1652	__t.__size_ = 0;
1368	1653	}
1369	1654	}
1370	1655
1371	1656	template <class _Tp, class _Compare, class _Allocator>
1372	1657	__tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __a)
1373		: __node_alloc_(__node_allocator(__a)), __size_(0), __value_comp_(std::move(__t.value_comp())) {
	1658	: __begin_node_(__end_node()),
	1659	__node_alloc_(__node_allocator(__a)),
	1660	__size_(0),
	1661	__value_comp_(std::move(__t.value_comp())) {
	1662	if (__t.size() == 0)
	1663	return;
1374	1664	if (__a == __t.__alloc()) {
1375		if (__t.size() == 0)
1376		__begin_node() = __end_node();
1377		else {
1378		__begin_node() = __t.__begin_node();
1379		__end_node()->__left_ = __t.__end_node()->__left_;
1380		__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
1381		size() = __t.size();
1382		__t.__begin_node() = __t.__end_node();
1383		__t.__end_node()->__left_ = nullptr;
1384		__t.size() = 0;
1385		}
	1665	__begin_node_ = __t.__begin_node_;
	1666	__end_node()->__left_ = __t.__end_node()->__left_;
	1667	__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
	1668	__size_ = __t.__size_;
	1669	__t.__begin_node_ = __t.__end_node();
	1670	__t.__end_node()->__left_ = nullptr;
	1671	__t.__size_ = 0;
1386	1672	} else {
1387		__begin_node() = __end_node();
	1673	*__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
	1674	__root()->__parent_ = __end_node();
	1675	__begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
	1676	__size_ = __t.size();
	1677	__t.clear(); // Ensure that __t is in a valid state after moving out the keys
1388	1678	}
1389	1679	}
1390	1680
...	...	@@ -1397,61 +1687,33 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, true_type)
1397	1687	__move_assign_alloc(__t);
1398	1688	__size_ = __t.__size_;
1399	1689	__value_comp_ = std::move(__t.__value_comp_);
1400		if (size() == 0)
1401		__begin_node() = __end_node();
	1690	if (__size_ == 0)
	1691	__begin_node_ = __end_node();
1402	1692	else {
1403	1693	__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
1404		__t.__begin_node() = __t.__end_node();
	1694	__t.__begin_node_ = __t.__end_node();
1405	1695	__t.__end_node()->__left_ = nullptr;
1406		__t.size() = 0;
	1696	__t.__size_ = 0;
1407	1697	}
1408	1698	}
1409	1699
1410	1700	template <class _Tp, class _Compare, class _Allocator>
1411	1701	void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
1412		if (__node_alloc() == __t.__node_alloc())
	1702	if (__node_alloc() == __t.__node_alloc()) {
1413	1703	__move_assign(__t, true_type());
1414		else {
1415		value_comp() = std::move(__t.value_comp());
1416		const_iterator __e = end();
1417		if (size() != 0) {
1418		_DetachedTreeCache __cache(this);
1419		while (__cache.__get() != nullptr && __t.size() != 0) {
1420		__assign_value(__cache.__get()->__value_, std::move(__t.remove(__t.begin())->__value_));
1421		__node_insert_multi(__cache.__get());
1422		__cache.__advance();
1423		}
1424		}
1425		while (__t.size() != 0) {
1426		__insert_multi_from_orphaned_node(__e, std::move(__t.remove(__t.begin())->__value_));
	1704	} else {
	1705	value_comp() = std::move(__t.value_comp());
	1706	if (__size_ != 0) {
	1707	*__root_ptr() = static_cast<__node_base_pointer>(__move_assign_tree(__root(), __t.__root()));
	1708	} else {
	1709	*__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
	1710	if (__root())
	1711	__root()->__parent_ = __end_node();
1427	1712	}
1428		}
1429		}
1430
1431		template <class _Tp, class _Compare, class _Allocator>
1432		__tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(__tree&& __t)
1433		_NOEXCEPT_(is_nothrow_move_assignable<value_compare>::value &&
1434		((__node_traits::propagate_on_container_move_assignment::value &&
1435		is_nothrow_move_assignable<__node_allocator>::value) \|\|
1436		allocator_traits<__node_allocator>::is_always_equal::value)) {
1437		__move_assign(__t, integral_constant<bool, __node_traits::propagate_on_container_move_assignment::value>());
1438		return *this;
1439		}
1440
1441		template <class _Tp, class _Compare, class _Allocator>
1442		__tree<_Tp, _Compare, _Allocator>::~__tree() {
1443		static_assert(is_copy_constructible<value_compare>::value, "Comparator must be copy-constructible.");
1444		destroy(__root());
1445		}
1446
1447		template <class _Tp, class _Compare, class _Allocator>
1448		void __tree<_Tp, _Compare, _Allocator>::destroy(__node_pointer __nd) _NOEXCEPT {
1449		if (__nd != nullptr) {
1450		destroy(static_cast<__node_pointer>(__nd->__left_));
1451		destroy(static_cast<__node_pointer>(__nd->__right_));
1452		__node_allocator& __na = __node_alloc();
1453		__node_traits::destroy(__na, std::addressof(__nd->__value_));
1454		__node_traits::deallocate(__na, __nd, 1);
	1713	__begin_node_ =
	1714	__end_node()->__left_ ? static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_)) : __end_node();
	1715	__size_ = __t.size();
	1716	__t.clear(); // Ensure that __t is in a valid state after moving out the keys
1455	1717	}
1456	1718	}
1457	1719
...	...	@@ -1470,12 +1732,12 @@ void __tree<_Tp, _Compare, _Allocator>::swap(__tree& __t)
1470	1732	std::__swap_allocator(__node_alloc(), __t.__node_alloc());
1471	1733	swap(__size_, __t.__size_);
1472	1734	swap(__value_comp_, __t.__value_comp_);
1473		if (size() == 0)
1474		__begin_node() = __end_node();
	1735	if (__size_ == 0)
	1736	__begin_node_ = __end_node();
1475	1737	else
1476	1738	__end_node()->__left_->__parent_ = __end_node();
1477		if (__t.size() == 0)
1478		__t.__begin_node() = __t.__end_node();
	1739	if (__t.__size_ == 0)
	1740	__t.__begin_node_ = __t.__end_node();
1479	1741	else
1480	1742	__t.__end_node()->__left_->__parent_ = __t.__end_node();
1481	1743	}
...	...	@@ -1483,8 +1745,8 @@ void __tree<_Tp, _Compare, _Allocator>::swap(__tree& __t)
1483	1745	template <class _Tp, class _Compare, class _Allocator>
1484	1746	void __tree<_Tp, _Compare, _Allocator>::clear() _NOEXCEPT {
1485	1747	destroy(__root());
1486		size() = 0;
1487		__begin_node() = __end_node();
	1748	__size_ = 0;
	1749	__begin_node_ = __end_node();
1488	1750	__end_node()->__left_ = nullptr;
1489	1751	}
1490	1752
...	...	@@ -1497,7 +1759,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__end_node_pointer& __parent,
1497	1759	__node_pointer __nd = __root();
1498	1760	if (__nd != nullptr) {
1499	1761	while (true) {
1500		if (value_comp()(__nd->__value_, __v)) {
	1762	if (value_comp()(__nd->__get_value(), __v)) {
1501	1763	if (__nd->__right_ != nullptr)
1502	1764	__nd = static_cast<__node_pointer>(__nd->__right_);
1503	1765	else {
...	...	@@ -1527,7 +1789,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__end_node_pointer& __parent
1527	1789	__node_pointer __nd = __root();
1528	1790	if (__nd != nullptr) {
1529	1791	while (true) {
1530		if (value_comp()(__v, __nd->__value_)) {
	1792	if (value_comp()(__v, __nd->__get_value())) {
1531	1793	if (__nd->__left_ != nullptr)
1532	1794	__nd = static_cast<__node_pointer>(__nd->__left_);
1533	1795	else {
...	...	@@ -1578,92 +1840,91 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
1578	1840	return __find_leaf_low(__parent, __v);
1579	1841	}
1580	1842
1581		// Find place to insert if __v doesn't exist
1582		// Set __parent to parent of null leaf
1583		// Return reference to null leaf
1584		// If __v exists, set parent to node of __v and return reference to node of __v
	1843	// Find __v
	1844	// If __v exists, return the parent of the node of __v and a reference to the pointer to the node of __v.
	1845	// If __v doesn't exist, return the parent of the null leaf and a reference to the pointer to the null leaf.
1585	1846	template <class _Tp, class _Compare, class _Allocator>
1586	1847	template <class _Key>
1587		typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
1588		__tree<_Tp, _Compare, _Allocator>::__find_equal(__end_node_pointer& __parent, const _Key& __v) {
1589		__node_pointer __nd = __root();
1590		__node_base_pointer* __nd_ptr = __root_ptr();
1591		if (__nd != nullptr) {
1592		while (true) {
1593		if (value_comp()(__v, __nd->__value_)) {
1594		if (__nd->__left_ != nullptr) {
1595		__nd_ptr = std::addressof(__nd->__left_);
1596		__nd = static_cast<__node_pointer>(__nd->__left_);
1597		} else {
1598		__parent = static_cast<__end_node_pointer>(__nd);
1599		return __parent->__left_;
1600		}
1601		} else if (value_comp()(__nd->__value_, __v)) {
1602		if (__nd->__right_ != nullptr) {
1603		__nd_ptr = std::addressof(__nd->__right_);
1604		__nd = static_cast<__node_pointer>(__nd->__right_);
1605		} else {
1606		__parent = static_cast<__end_node_pointer>(__nd);
1607		return __nd->__right_;
1608		}
1609		} else {
1610		__parent = static_cast<__end_node_pointer>(__nd);
1611		return *__nd_ptr;
1612		}
	1848	_LIBCPP_HIDE_FROM_ABI pair<typename __tree<_Tp, _Compare, _Allocator>::__end_node_pointer,
	1849	typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&>
	1850	__tree<_Tp, _Compare, _Allocator>::__find_equal(const _Key& __v) {
	1851	using _Pair = pair<__end_node_pointer, __node_base_pointer&>;
	1852
	1853	__node_pointer __nd = __root();
	1854
	1855	if (__nd == nullptr) {
	1856	auto __end = __end_node();
	1857	return _Pair(__end, __end->__left_);
	1858	}
	1859
	1860	__node_base_pointer* __node_ptr = __root_ptr();
	1861	auto&& __transparent = std::__as_transparent<_Key>(value_comp());
	1862	auto __comp =
	1863	__lazy_synth_three_way_comparator<__make_transparent_t<_Key, _Compare>, _Key, value_type>(__transparent);
	1864
	1865	while (true) {
	1866	auto __comp_res = __comp(__v, __nd->__get_value());
	1867
	1868	if (__comp_res.__less()) {
	1869	if (__nd->__left_ == nullptr)
	1870	return _Pair(static_cast<__end_node_pointer>(__nd), __nd->__left_);
	1871
	1872	__node_ptr = std::addressof(__nd->__left_);
	1873	__nd = static_cast<__node_pointer>(__nd->__left_);
	1874	} else if (__comp_res.__greater()) {
	1875	if (__nd->__right_ == nullptr)
	1876	return _Pair(static_cast<__end_node_pointer>(__nd), __nd->__right_);
	1877
	1878	__node_ptr = std::addressof(__nd->__right_);
	1879	__nd = static_cast<__node_pointer>(__nd->__right_);
	1880	} else {
	1881	return _Pair(static_cast<__end_node_pointer>(__nd), *__node_ptr);
1613	1882	}
1614	1883	}
1615		__parent = __end_node();
1616		return __parent->__left_;
1617	1884	}
1618	1885
1619		// Find place to insert if __v doesn't exist
	1886	// Find __v
1620	1887	// First check prior to __hint.
1621	1888	// Next check after __hint.
1622	1889	// Next do O(log N) search.
1623		// Set __parent to parent of null leaf
1624		// Return reference to null leaf
1625		// If __v exists, set parent to node of __v and return reference to node of __v
	1890	// If __v exists, return the parent of the node of __v and a reference to the pointer to the node of __v.
	1891	// If __v doesn't exist, return the parent of the null leaf and a reference to the pointer to the null leaf.
1626	1892	template <class _Tp, class _Compare, class _Allocator>
1627	1893	template <class _Key>
1628		typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Compare, _Allocator>::__find_equal(
1629		const_iterator __hint, __end_node_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v) {
1630		if (__hint == end() \|\| value_comp()(__v, *__hint)) // check before
1631		{
	1894	_LIBCPP_HIDE_FROM_ABI pair<typename __tree<_Tp, _Compare, _Allocator>::__end_node_pointer,
	1895	typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&>
	1896	__tree<_Tp, _Compare, _Allocator>::__find_equal(const_iterator __hint, __node_base_pointer& __dummy, const _Key& __v) {
	1897	using _Pair = pair<__end_node_pointer, __node_base_pointer&>;
	1898
	1899	if (__hint == end() \|\| value_comp()(__v, *__hint)) { // check before
1632	1900	// __v < *__hint
1633	1901	const_iterator __prior = __hint;
1634	1902	if (__prior == begin() \|\| value_comp()(*--__prior, __v)) {
1635	1903	// prev(__hint) < __v < __hint
1636		if (__hint.__ptr_->__left_ == nullptr) {
1637		__parent = __hint.__ptr_;
1638		return __parent->__left_;
1639		} else {
1640		__parent = __prior.__ptr_;
1641		return static_cast<__node_base_pointer>(__prior.__ptr_)->__right_;
1642		}
	1904	if (__hint.__ptr_->__left_ == nullptr)
	1905	return _Pair(__hint.__ptr_, __hint.__ptr_->__left_);
	1906	return _Pair(__prior.__ptr_, static_cast<__node_pointer>(__prior.__ptr_)->__right_);
1643	1907	}
1644	1908	// __v <= *prev(__hint)
1645		return __find_equal(__parent, __v);
1646		} else if (value_comp()(*__hint, __v)) // check after
1647		{
	1909	return __find_equal(__v);
	1910	}
	1911
	1912	if (value_comp()(*__hint, __v)) { // check after
1648	1913	// *__hint < __v
1649	1914	const_iterator __next = std::next(__hint);
1650	1915	if (__next == end() \|\| value_comp()(__v, *__next)) {
1651	1916	// __hint < __v < std::next(__hint)
1652		if (__hint.__get_np()->__right_ == nullptr) {
1653		__parent = __hint.__ptr_;
1654		return static_cast<__node_base_pointer>(__hint.__ptr_)->__right_;
1655		} else {
1656		__parent = __next.__ptr_;
1657		return __parent->__left_;
1658		}
	1917	if (__hint.__get_np()->__right_ == nullptr)
	1918	return _Pair(__hint.__ptr_, static_cast<__node_pointer>(__hint.__ptr_)->__right_);
	1919	return _Pair(__next.__ptr_, __next.__ptr_->__left_);
1659	1920	}
1660	1921	// *next(__hint) <= __v
1661		return __find_equal(__parent, __v);
	1922	return __find_equal(__v);
1662	1923	}
	1924
1663	1925	// else __v == *__hint
1664		__parent = __hint.__ptr_;
1665		__dummy = static_cast<__node_base_pointer>(__hint.__ptr_);
1666		return __dummy;
	1926	__dummy = static_cast<__node_base_pointer>(__hint.__ptr_);
	1927	return _Pair(__hint.__ptr_, __dummy);
1667	1928	}
1668	1929
1669	1930	template <class _Tp, class _Compare, class _Allocator>
...	...	@@ -1674,46 +1935,10 @@ void __tree<_Tp, _Compare, _Allocator>::__insert_node_at(
1674	1935	__new_node->__parent_ = __parent;
1675	1936	// __new_node->__is_black_ is initialized in __tree_balance_after_insert
1676	1937	__child = __new_node;
1677		if (__begin_node()->__left_ != nullptr)
1678		__begin_node() = static_cast<__end_node_pointer>(__begin_node()->__left_);
	1938	if (__begin_node_->__left_ != nullptr)
	1939	__begin_node_ = static_cast<__end_node_pointer>(__begin_node_->__left_);
1679	1940	std::__tree_balance_after_insert(__end_node()->__left_, __child);
1680		++size();
1681		}
1682
1683		template <class _Tp, class _Compare, class _Allocator>
1684		template <class _Key, class... _Args>
1685		pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
1686		__tree<_Tp, _Compare, _Allocator>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) {
1687		__end_node_pointer __parent;
1688		__node_base_pointer& __child = __find_equal(__parent, __k);
1689		__node_pointer __r = static_cast<__node_pointer>(__child);
1690		bool __inserted = false;
1691		if (__child == nullptr) {
1692		__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1693		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1694		__r = __h.release();
1695		__inserted = true;
1696		}
1697		return pair<iterator, bool>(iterator(__r), __inserted);
1698		}
1699
1700		template <class _Tp, class _Compare, class _Allocator>
1701		template <class _Key, class... _Args>
1702		pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
1703		__tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_key_args(
1704		const_iterator __p, _Key const& __k, _Args&&... __args) {
1705		__end_node_pointer __parent;
1706		__node_base_pointer __dummy;
1707		__node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __k);
1708		__node_pointer __r = static_cast<__node_pointer>(__child);
1709		bool __inserted = false;
1710		if (__child == nullptr) {
1711		__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1712		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1713		__r = __h.release();
1714		__inserted = true;
1715		}
1716		return pair<iterator, bool>(iterator(__r), __inserted);
	1941	++__size_;
1717	1942	}
1718	1943
1719	1944	template <class _Tp, class _Compare, class _Allocator>
...	...	@@ -1722,51 +1947,18 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_holder
1722	1947	__tree<_Tp, _Compare, _Allocator>::__construct_node(_Args&&... __args) {
1723	1948	__node_allocator& __na = __node_alloc();
1724	1949	__node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
1725		__node_traits::construct(__na, std::addressof(__h->__value_), std::forward<_Args>(__args)...);
	1950	std::__construct_at(std::addressof(*__h), __na, std::forward<_Args>(__args)...);
1726	1951	__h.get_deleter().__value_constructed = true;
1727	1952	return __h;
1728	1953	}
1729	1954
1730		template <class _Tp, class _Compare, class _Allocator>
1731		template <class... _Args>
1732		pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
1733		__tree<_Tp, _Compare, _Allocator>::__emplace_unique_impl(_Args&&... __args) {
1734		__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1735		__end_node_pointer __parent;
1736		__node_base_pointer& __child = __find_equal(__parent, __h->__value_);
1737		__node_pointer __r = static_cast<__node_pointer>(__child);
1738		bool __inserted = false;
1739		if (__child == nullptr) {
1740		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1741		__r = __h.release();
1742		__inserted = true;
1743		}
1744		return pair<iterator, bool>(iterator(__r), __inserted);
1745		}
1746
1747		template <class _Tp, class _Compare, class _Allocator>
1748		template <class... _Args>
1749		typename __tree<_Tp, _Compare, _Allocator>::iterator
1750		__tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_impl(const_iterator __p, _Args&&... __args) {
1751		__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1752		__end_node_pointer __parent;
1753		__node_base_pointer __dummy;
1754		__node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __h->__value_);
1755		__node_pointer __r = static_cast<__node_pointer>(__child);
1756		if (__child == nullptr) {
1757		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1758		__r = __h.release();
1759		}
1760		return iterator(__r);
1761		}
1762
1763	1955	template <class _Tp, class _Compare, class _Allocator>
1764	1956	template <class... _Args>
1765	1957	typename __tree<_Tp, _Compare, _Allocator>::iterator
1766	1958	__tree<_Tp, _Compare, _Allocator>::__emplace_multi(_Args&&... __args) {
1767	1959	__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1768	1960	__end_node_pointer __parent;
1769		__node_base_pointer& __child = __find_leaf_high(__parent, __h->__value_);
	1961	__node_base_pointer& __child = __find_leaf_high(__parent, __h->__get_value());
1770	1962	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1771	1963	return iterator(static_cast<__node_pointer>(__h.release()));
1772	1964	}
...	...	@@ -1777,53 +1969,19 @@ typename __tree<_Tp, _Compare, _Allocator>::iterator
1777	1969	__tree<_Tp, _Compare, _Allocator>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) {
1778	1970	__node_holder __h = __construct_node(std::forward<_Args>(__args)...);
1779	1971	__end_node_pointer __parent;
1780		__node_base_pointer& __child = __find_leaf(__p, __parent, __h->__value_);
	1972	__node_base_pointer& __child = __find_leaf(__p, __parent, __h->__get_value());
1781	1973	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1782	1974	return iterator(static_cast<__node_pointer>(__h.release()));
1783	1975	}
1784	1976
1785		template <class _Tp, class _Compare, class _Allocator>
1786		pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
1787		__tree<_Tp, _Compare, _Allocator>::__node_assign_unique(const value_type& __v, __node_pointer __nd) {
1788		__end_node_pointer __parent;
1789		__node_base_pointer& __child = __find_equal(__parent, __v);
1790		__node_pointer __r = static_cast<__node_pointer>(__child);
1791		bool __inserted = false;
1792		if (__child == nullptr) {
1793		__assign_value(__nd->__value_, __v);
1794		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
1795		__r = __nd;
1796		__inserted = true;
1797		}
1798		return pair<iterator, bool>(iterator(__r), __inserted);
1799		}
1800
1801		template <class _Tp, class _Compare, class _Allocator>
1802		typename __tree<_Tp, _Compare, _Allocator>::iterator
1803		__tree<_Tp, _Compare, _Allocator>::__node_insert_multi(__node_pointer __nd) {
1804		__end_node_pointer __parent;
1805		__node_base_pointer& __child = __find_leaf_high(__parent, __nd->__value_);
1806		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
1807		return iterator(__nd);
1808		}
1809
1810		template <class _Tp, class _Compare, class _Allocator>
1811		typename __tree<_Tp, _Compare, _Allocator>::iterator
1812		__tree<_Tp, _Compare, _Allocator>::__node_insert_multi(const_iterator __p, __node_pointer __nd) {
1813		__end_node_pointer __parent;
1814		__node_base_pointer& __child = __find_leaf(__p, __parent, __nd->__value_);
1815		__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
1816		return iterator(__nd);
1817		}
1818
1819	1977	template <class _Tp, class _Compare, class _Allocator>
1820	1978	typename __tree<_Tp, _Compare, _Allocator>::iterator
1821	1979	__tree<_Tp, _Compare, _Allocator>::__remove_node_pointer(__node_pointer __ptr) _NOEXCEPT {
1822	1980	iterator __r(__ptr);
1823	1981	++__r;
1824		if (__begin_node() == __ptr)
1825		__begin_node() = __r.__ptr_;
1826		--size();
	1982	if (__begin_node_ == __ptr)
	1983	__begin_node_ = __r.__ptr_;
	1984	--__size_;
1827	1985	std::__tree_remove(__end_node()->__left_, static_cast<__node_base_pointer>(__ptr));
1828	1986	return __r;
1829	1987	}
...	...	@@ -1837,8 +1995,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(_NodeHandle&& __n
1837	1995	return _InsertReturnType{end(), false, _NodeHandle()};
1838	1996
1839	1997	__node_pointer __ptr = __nh.__ptr_;
1840		__end_node_pointer __parent;
1841		__node_base_pointer& __child = __find_equal(__parent, __ptr->__value_);
	1998	auto [__parent, __child] = __find_equal(__ptr->__get_value());
1842	1999	if (__child != nullptr)
1843	2000	return _InsertReturnType{iterator(static_cast<__node_pointer>(__child)), false, std::move(__nh)};
1844	2001
...	...	@@ -1855,10 +2012,9 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(const_iterator __
1855	2012	return end();
1856	2013
1857	2014	__node_pointer __ptr = __nh.__ptr_;
1858		__end_node_pointer __parent;
1859	2015	__node_base_pointer __dummy;
1860		__node_base_pointer& __child = __find_equal(__hint, __parent, __dummy, __ptr->__value_);
1861		__node_pointer __r = static_cast<__node_pointer>(__child);
	2016	auto [__parent, __child] = __find_equal(__hint, __dummy, __ptr->__get_value());
	2017	__node_pointer __r = static_cast<__node_pointer>(__child);
1862	2018	if (__child == nullptr) {
1863	2019	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
1864	2020	__r = __ptr;
...	...	@@ -1885,14 +2041,12 @@ _LIBCPP_HIDE_FROM_ABI _NodeHandle __tree<_Tp, _Compare, _Allocator>::__node_hand
1885	2041	}
1886	2042
1887	2043	template <class _Tp, class _Compare, class _Allocator>
1888		template <class _Tree>
1889		_LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merge_unique(_Tree& __source) {
1890		static_assert(is_same<typename _Tree::__node_pointer, __node_pointer>::value, "");
1891
1892		for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
	2044	template <class _Comp2>
	2045	_LIBCPP_HIDE_FROM_ABI void
	2046	__tree<_Tp, _Compare, _Allocator>::__node_handle_merge_unique(__tree<_Tp, _Comp2, _Allocator>& __source) {
	2047	for (iterator __i = __source.begin(); __i != __source.end();) {
1893	2048	__node_pointer __src_ptr = __i.__get_np();
1894		__end_node_pointer __parent;
1895		__node_base_pointer& __child = __find_equal(__parent, __src_ptr->__value_);
	2049	auto [__parent, __child] = __find_equal(__src_ptr->__get_value());
1896	2050	++__i;
1897	2051	if (__child != nullptr)
1898	2052	continue;
...	...	@@ -1909,7 +2063,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(_NodeHandle&& __nh
1909	2063	return end();
1910	2064	__node_pointer __ptr = __nh.__ptr_;
1911	2065	__end_node_pointer __parent;
1912		__node_base_pointer& __child = __find_leaf_high(__parent, __ptr->__value_);
	2066	__node_base_pointer& __child = __find_leaf_high(__parent, __ptr->__get_value());
1913	2067	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
1914	2068	__nh.__release_ptr();
1915	2069	return iterator(__ptr);
...	...	@@ -1924,21 +2078,20 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(const_iterator __h
1924	2078
1925	2079	__node_pointer __ptr = __nh.__ptr_;
1926	2080	__end_node_pointer __parent;
1927		__node_base_pointer& __child = __find_leaf(__hint, __parent, __ptr->__value_);
	2081	__node_base_pointer& __child = __find_leaf(__hint, __parent, __ptr->__get_value());
1928	2082	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
1929	2083	__nh.__release_ptr();
1930	2084	return iterator(__ptr);
1931	2085	}
1932	2086
1933	2087	template <class _Tp, class _Compare, class _Allocator>
1934		template <class _Tree>
1935		_LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merge_multi(_Tree& __source) {
1936		static_assert(is_same<typename _Tree::__node_pointer, __node_pointer>::value, "");
1937
1938		for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
	2088	template <class _Comp2>
	2089	_LIBCPP_HIDE_FROM_ABI void
	2090	__tree<_Tp, _Compare, _Allocator>::__node_handle_merge_multi(__tree<_Tp, _Comp2, _Allocator>& __source) {
	2091	for (iterator __i = __source.begin(); __i != __source.end();) {
1939	2092	__node_pointer __src_ptr = __i.__get_np();
1940	2093	__end_node_pointer __parent;
1941		__node_base_pointer& __child = __find_leaf_high(__parent, __src_ptr->__value_);
	2094	__node_base_pointer& __child = __find_leaf_high(__parent, __src_ptr->__get_value());
1942	2095	++__i;
1943	2096	__source.__remove_node_pointer(__src_ptr);
1944	2097	__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__src_ptr));
...	...	@@ -1987,34 +2140,17 @@ __tree<_Tp, _Compare, _Allocator>::__erase_multi(const _Key& __k) {
1987	2140	return __r;
1988	2141	}
1989	2142
1990		template <class _Tp, class _Compare, class _Allocator>
1991		template <class _Key>
1992		typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) {
1993		iterator __p = __lower_bound(__v, __root(), __end_node());
1994		if (__p != end() && !value_comp()(__v, *__p))
1995		return __p;
1996		return end();
1997		}
1998
1999		template <class _Tp, class _Compare, class _Allocator>
2000		template <class _Key>
2001		typename __tree<_Tp, _Compare, _Allocator>::const_iterator
2002		__tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) const {
2003		const_iterator __p = __lower_bound(__v, __root(), __end_node());
2004		if (__p != end() && !value_comp()(__v, *__p))
2005		return __p;
2006		return end();
2007		}
2008
2009	2143	template <class _Tp, class _Compare, class _Allocator>
2010	2144	template <class _Key>
2011	2145	typename __tree<_Tp, _Compare, _Allocator>::size_type
2012	2146	__tree<_Tp, _Compare, _Allocator>::__count_unique(const _Key& __k) const {
2013	2147	__node_pointer __rt = __root();
	2148	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2014	2149	while (__rt != nullptr) {
2015		if (value_comp()(__k, __rt->__value_)) {
	2150	auto __comp_res = __comp(__k, __rt->__get_value());
	2151	if (__comp_res.__less()) {
2016	2152	__rt = static_cast<__node_pointer>(__rt->__left_);
2017		} else if (value_comp()(__rt->__value_, __k))
	2153	} else if (__comp_res.__greater())
2018	2154	__rt = static_cast<__node_pointer>(__rt->__right_);
2019	2155	else
2020	2156	return 1;
...	...	@@ -2028,26 +2164,28 @@ typename __tree<_Tp, _Compare, _Allocator>::size_type
2028	2164	__tree<_Tp, _Compare, _Allocator>::__count_multi(const _Key& __k) const {
2029	2165	__end_node_pointer __result = __end_node();
2030	2166	__node_pointer __rt = __root();
	2167	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2031	2168	while (__rt != nullptr) {
2032		if (value_comp()(__k, __rt->__value_)) {
	2169	auto __comp_res = __comp(__k, __rt->__get_value());
	2170	if (__comp_res.__less()) {
2033	2171	__result = static_cast<__end_node_pointer>(__rt);
2034	2172	__rt = static_cast<__node_pointer>(__rt->__left_);
2035		} else if (value_comp()(__rt->__value_, __k))
	2173	} else if (__comp_res.__greater())
2036	2174	__rt = static_cast<__node_pointer>(__rt->__right_);
2037	2175	else
2038	2176	return std::distance(
2039		__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
2040		__upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
	2177	__lower_bound_multi(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
	2178	__upper_bound_multi(__k, static_cast<__node_pointer>(__rt->__right_), __result));
2041	2179	}
2042	2180	return 0;
2043	2181	}
2044	2182
2045	2183	template <class _Tp, class _Compare, class _Allocator>
2046	2184	template <class _Key>
2047		typename __tree<_Tp, _Compare, _Allocator>::iterator
2048		__tree<_Tp, _Compare, _Allocator>::__lower_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
	2185	typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::__lower_bound_multi(
	2186	const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
2049	2187	while (__root != nullptr) {
2050		if (!value_comp()(__root->__value_, __v)) {
	2188	if (!value_comp()(__root->__get_value(), __v)) {
2051	2189	__result = static_cast<__end_node_pointer>(__root);
2052	2190	__root = static_cast<__node_pointer>(__root->__left_);
2053	2191	} else
...	...	@@ -2058,10 +2196,10 @@ __tree<_Tp, _Compare, _Allocator>::__lower_bound(const _Key& __v, __node_pointer
2058	2196
2059	2197	template <class _Tp, class _Compare, class _Allocator>
2060	2198	template <class _Key>
2061		typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__lower_bound(
	2199	typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__lower_bound_multi(
2062	2200	const _Key& __v, __node_pointer __root, __end_node_pointer __result) const {
2063	2201	while (__root != nullptr) {
2064		if (!value_comp()(__root->__value_, __v)) {
	2202	if (!value_comp()(__root->__get_value(), __v)) {
2065	2203	__result = static_cast<__end_node_pointer>(__root);
2066	2204	__root = static_cast<__node_pointer>(__root->__left_);
2067	2205	} else
...	...	@@ -2072,10 +2210,10 @@ typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare,
2072	2210
2073	2211	template <class _Tp, class _Compare, class _Allocator>
2074	2212	template <class _Key>
2075		typename __tree<_Tp, _Compare, _Allocator>::iterator
2076		__tree<_Tp, _Compare, _Allocator>::__upper_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
	2213	typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::__upper_bound_multi(
	2214	const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
2077	2215	while (__root != nullptr) {
2078		if (value_comp()(__v, __root->__value_)) {
	2216	if (value_comp()(__v, __root->__get_value())) {
2079	2217	__result = static_cast<__end_node_pointer>(__root);
2080	2218	__root = static_cast<__node_pointer>(__root->__left_);
2081	2219	} else
...	...	@@ -2086,10 +2224,10 @@ __tree<_Tp, _Compare, _Allocator>::__upper_bound(const _Key& __v, __node_pointer
2086	2224
2087	2225	template <class _Tp, class _Compare, class _Allocator>
2088	2226	template <class _Key>
2089		typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__upper_bound(
	2227	typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__upper_bound_multi(
2090	2228	const _Key& __v, __node_pointer __root, __end_node_pointer __result) const {
2091	2229	while (__root != nullptr) {
2092		if (value_comp()(__v, __root->__value_)) {
	2230	if (value_comp()(__v, __root->__get_value())) {
2093	2231	__result = static_cast<__end_node_pointer>(__root);
2094	2232	__root = static_cast<__node_pointer>(__root->__left_);
2095	2233	} else
...	...	@@ -2102,14 +2240,16 @@ template <class _Tp, class _Compare, class _Allocator>
2102	2240	template <class _Key>
2103	2241	pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, typename __tree<_Tp, _Compare, _Allocator>::iterator>
2104	2242	__tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) {
2105		typedef pair<iterator, iterator> _Pp;
	2243	using _Pp = pair<iterator, iterator>;
2106	2244	__end_node_pointer __result = __end_node();
2107	2245	__node_pointer __rt = __root();
	2246	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2108	2247	while (__rt != nullptr) {
2109		if (value_comp()(__k, __rt->__value_)) {
	2248	auto __comp_res = __comp(__k, __rt->__get_value());
	2249	if (__comp_res.__less()) {
2110	2250	__result = static_cast<__end_node_pointer>(__rt);
2111	2251	__rt = static_cast<__node_pointer>(__rt->__left_);
2112		} else if (value_comp()(__rt->__value_, __k))
	2252	} else if (__comp_res.__greater())
2113	2253	__rt = static_cast<__node_pointer>(__rt->__right_);
2114	2254	else
2115	2255	return _Pp(iterator(__rt),
...	...	@@ -2124,14 +2264,16 @@ template <class _Key>
2124	2264	pair<typename __tree<_Tp, _Compare, _Allocator>::const_iterator,
2125	2265	typename __tree<_Tp, _Compare, _Allocator>::const_iterator>
2126	2266	__tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) const {
2127		typedef pair<const_iterator, const_iterator> _Pp;
	2267	using _Pp = pair<const_iterator, const_iterator>;
2128	2268	__end_node_pointer __result = __end_node();
2129	2269	__node_pointer __rt = __root();
	2270	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2130	2271	while (__rt != nullptr) {
2131		if (value_comp()(__k, __rt->__value_)) {
	2272	auto __comp_res = __comp(__k, __rt->__get_value());
	2273	if (__comp_res.__less()) {
2132	2274	__result = static_cast<__end_node_pointer>(__rt);
2133	2275	__rt = static_cast<__node_pointer>(__rt->__left_);
2134		} else if (value_comp()(__rt->__value_, __k))
	2276	} else if (__comp_res.__greater())
2135	2277	__rt = static_cast<__node_pointer>(__rt->__right_);
2136	2278	else
2137	2279	return _Pp(
...	...	@@ -2146,18 +2288,21 @@ template <class _Tp, class _Compare, class _Allocator>
2146	2288	template <class _Key>
2147	2289	pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, typename __tree<_Tp, _Compare, _Allocator>::iterator>
2148	2290	__tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) {
2149		typedef pair<iterator, iterator> _Pp;
	2291	using _Pp = pair<iterator, iterator>;
2150	2292	__end_node_pointer __result = __end_node();
2151		__node_pointer __rt = __root();
	2293	__node_pointer __rt = __root();
	2294	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2152	2295	while (__rt != nullptr) {
2153		if (value_comp()(__k, __rt->__value_)) {
	2296	auto __comp_res = __comp(__k, __rt->__get_value());
	2297	if (__comp_res.__less()) {
2154	2298	__result = static_cast<__end_node_pointer>(__rt);
2155	2299	__rt = static_cast<__node_pointer>(__rt->__left_);
2156		} else if (value_comp()(__rt->__value_, __k))
	2300	} else if (__comp_res.__greater())
2157	2301	__rt = static_cast<__node_pointer>(__rt->__right_);
2158	2302	else
2159		return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
2160		__upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
	2303	return _Pp(
	2304	__lower_bound_multi(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
	2305	__upper_bound_multi(__k, static_cast<__node_pointer>(__rt->__right_), __result));
2161	2306	}
2162	2307	return _Pp(iterator(__result), iterator(__result));
2163	2308	}
...	...	@@ -2167,18 +2312,21 @@ template <class _Key>
2167	2312	pair<typename __tree<_Tp, _Compare, _Allocator>::const_iterator,
2168	2313	typename __tree<_Tp, _Compare, _Allocator>::const_iterator>
2169	2314	__tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) const {
2170		typedef pair<const_iterator, const_iterator> _Pp;
	2315	using _Pp = pair<const_iterator, const_iterator>;
2171	2316	__end_node_pointer __result = __end_node();
2172		__node_pointer __rt = __root();
	2317	__node_pointer __rt = __root();
	2318	auto __comp = __lazy_synth_three_way_comparator<value_compare, _Key, value_type>(value_comp());
2173	2319	while (__rt != nullptr) {
2174		if (value_comp()(__k, __rt->__value_)) {
	2320	auto __comp_res = __comp(__k, __rt->__get_value());
	2321	if (__comp_res.__less()) {
2175	2322	__result = static_cast<__end_node_pointer>(__rt);
2176	2323	__rt = static_cast<__node_pointer>(__rt->__left_);
2177		} else if (value_comp()(__rt->__value_, __k))
	2324	} else if (__comp_res.__greater())
2178	2325	__rt = static_cast<__node_pointer>(__rt->__right_);
2179	2326	else
2180		return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
2181		__upper_bound(__k, static_cast<__node_pointer>(__rt->__right_), __result));
	2327	return _Pp(
	2328	__lower_bound_multi(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
	2329	__upper_bound_multi(__k, static_cast<__node_pointer>(__rt->__right_), __result));
2182	2330	}
2183	2331	return _Pp(const_iterator(__result), const_iterator(__result));
2184	2332	}
...	...	@@ -2187,13 +2335,13 @@ template <class _Tp, class _Compare, class _Allocator>
2187	2335	typename __tree<_Tp, _Compare, _Allocator>::__node_holder
2188	2336	__tree<_Tp, _Compare, _Allocator>::remove(const_iterator __p) _NOEXCEPT {
2189	2337	__node_pointer __np = __p.__get_np();
2190		if (__begin_node() == __p.__ptr_) {
	2338	if (__begin_node_ == __p.__ptr_) {
2191	2339	if (__np->__right_ != nullptr)
2192		__begin_node() = static_cast<__end_node_pointer>(__np->__right_);
	2340	__begin_node_ = static_cast<__end_node_pointer>(__np->__right_);
2193	2341	else
2194		__begin_node() = static_cast<__end_node_pointer>(__np->__parent_);
	2342	__begin_node_ = static_cast<__end_node_pointer>(__np->__parent_);
2195	2343	}
2196		--size();
	2344	--__size_;
2197	2345	std::__tree_remove(__end_node()->__left_, static_cast<__node_base_pointer>(__np));
2198	2346	return __node_holder(__np, _Dp(__node_alloc(), true));
2199	2347	}

lib/libcxx/include/__tuple/sfinae_helpers.h+1-44

...	...	@@ -10,20 +10,6 @@
10	10	#define _LIBCPP___TUPLE_SFINAE_HELPERS_H
11	11
12	12	#include <__config>
13		#include <__cstddef/size_t.h>
14		#include <__fwd/tuple.h>
15		#include <__tuple/make_tuple_types.h>
16		#include <__tuple/tuple_element.h>
17		#include <__tuple/tuple_like_ext.h>
18		#include <__tuple/tuple_size.h>
19		#include <__tuple/tuple_types.h>
20		#include <__type_traits/conjunction.h>
21		#include <__type_traits/enable_if.h>
22		#include <__type_traits/integral_constant.h>
23		#include <__type_traits/is_constructible.h>
24		#include <__type_traits/is_same.h>
25		#include <__type_traits/remove_cvref.h>
26		#include <__type_traits/remove_reference.h>
27	13
28	14	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
29	15	# pragma GCC system_header
...	...	@@ -33,36 +19,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
33	19
34	20	#ifndef _LIBCPP_CXX03_LANG
35	21
36		struct __tuple_sfinae_base {
37		template <template <class, class...> class _Trait, class... _LArgs, class... _RArgs>
38		static auto __do_test(__tuple_types<_LArgs...>,
39		__tuple_types<_RArgs...>) -> __all<__enable_if_t<_Trait<_LArgs, _RArgs>::value, bool>{true}...>;
40		template <template <class...> class>
41		static auto __do_test(...) -> false_type;
42
43		template <class _FromArgs, class _ToArgs>
44		using __constructible _LIBCPP_NODEBUG = decltype(__do_test<is_constructible>(_ToArgs{}, _FromArgs{}));
45		};
46
47		// __tuple_constructible
48
49		template <class _Tp,
50		class _Up,
51		bool = __tuple_like_ext<__libcpp_remove_reference_t<_Tp> >::value,
52		bool = __tuple_like_ext<_Up>::value>
53		struct __tuple_constructible : public false_type {};
54
55		template <class _Tp, class _Up>
56		struct __tuple_constructible<_Tp, _Up, true, true>
57		: public __tuple_sfinae_base::__constructible< typename __make_tuple_types<_Tp>::type,
58		typename __make_tuple_types<_Up>::type > {};
59
60		template <size_t _Ip, class... _Tp>
61		struct tuple_element<_Ip, tuple<_Tp...> > {
62		using type _LIBCPP_NODEBUG = typename tuple_element<_Ip, __tuple_types<_Tp...> >::type;
63		};
64
65		struct _LIBCPP_EXPORTED_FROM_ABI __check_tuple_constructor_fail {
	22	struct __check_tuple_constructor_fail {
66	23	static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_explicit_default() { return false; }
67	24	static _LIBCPP_HIDE_FROM_ABI constexpr bool __enable_implicit_default() { return false; }
68	25	template <class...>

lib/libcxx/include/__tuple/tuple_element.h-12

...	...	@@ -11,8 +11,6 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
14		#include <__tuple/tuple_indices.h>
15		#include <__tuple/tuple_types.h>
16	14
17	15	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	16	# pragma GCC system_header
...	...	@@ -38,21 +36,11 @@ struct tuple_element<_Ip, const volatile _Tp> {
38	36	using type _LIBCPP_NODEBUG = const volatile typename tuple_element<_Ip, _Tp>::type;
39	37	};
40	38
41		#ifndef _LIBCPP_CXX03_LANG
42
43		template <size_t _Ip, class... _Types>
44		struct tuple_element<_Ip, __tuple_types<_Types...> > {
45		static_assert(_Ip < sizeof...(_Types), "tuple_element index out of range");
46		using type _LIBCPP_NODEBUG = __type_pack_element<_Ip, _Types...>;
47		};
48
49	39	# if _LIBCPP_STD_VER >= 14
50	40	template <size_t _Ip, class... _Tp>
51	41	using tuple_element_t _LIBCPP_NODEBUG = typename tuple_element<_Ip, _Tp...>::type;
52	42	# endif
53	43
54		#endif // _LIBCPP_CXX03_LANG
55
56	44	_LIBCPP_END_NAMESPACE_STD
57	45
58	46	#endif // _LIBCPP___TUPLE_TUPLE_ELEMENT_H

lib/libcxx/include/__tuple/tuple_size.h-4

...	...	@@ -12,7 +12,6 @@
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
14	14	#include <__fwd/tuple.h>
15		#include <__tuple/tuple_types.h>
16	15	#include <__type_traits/enable_if.h>
17	16	#include <__type_traits/integral_constant.h>
18	17	#include <__type_traits/is_const.h>
...	...	@@ -59,9 +58,6 @@ struct tuple_size<const volatile _Tp> : public tuple_size<_Tp> {};
59	58	template <class... _Tp>
60	59	struct tuple_size<tuple<_Tp...> > : public integral_constant<size_t, sizeof...(_Tp)> {};
61	60
62		template <class... _Tp>
63		struct tuple_size<__tuple_types<_Tp...> > : public integral_constant<size_t, sizeof...(_Tp)> {};
64
65	61	# if _LIBCPP_STD_VER >= 17
66	62	template <class _Tp>
67	63	inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;

lib/libcxx/include/__tuple/tuple_transform.h created+45

...	...	@@ -0,0 +1,45 @@
	1	// -- C++ --
	2	//===----------------------------------------------------------------------===//
	3	//
	4	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	5	// See https://llvm.org/LICENSE.txt for license information.
	6	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	7	//
	8	//===----------------------------------------------------------------------===//
	9
	10	#ifndef _LIBCPP___TUPLE_TUPLE_TRANSFORM_H
	11	#define _LIBCPP___TUPLE_TUPLE_TRANSFORM_H
	12
	13	#include <__config>
	14
	15	#include <__functional/invoke.h>
	16	#include <__utility/forward.h>
	17	#include <tuple>
	18
	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	20	# pragma GCC system_header
	21	#endif
	22
	23	_LIBCPP_PUSH_MACROS
	24	#include <__undef_macros>
	25
	26	_LIBCPP_BEGIN_NAMESPACE_STD
	27
	28	#if _LIBCPP_STD_VER >= 23
	29
	30	template <class _Fun, class _Tuple>
	31	_LIBCPP_HIDE_FROM_ABI constexpr auto __tuple_transform(_Fun&& __f, _Tuple&& __tuple) {
	32	return std::apply(
	33	[&]<class... _Types>(_Types&&... __elements) {
	34	return tuple<invoke_result_t<_Fun&, _Types>...>(std::invoke(__f, std::forward<_Types>(__elements))...);
	35	},
	36	std::forward<_Tuple>(__tuple));
	37	}
	38
	39	#endif // _LIBCPP_STD_VER >= 23
	40
	41	_LIBCPP_END_NAMESPACE_STD
	42
	43	_LIBCPP_POP_MACROS
	44
	45	#endif // _LIBCPP___TUPLE_TUPLE_TRANSFORM_H

lib/libcxx/include/__type_traits/aligned_storage.h+13-40

...	...	@@ -11,8 +11,6 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
14		#include <__type_traits/integral_constant.h>
15		#include <__type_traits/type_list.h>
16	14
17	15	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
18	16	# pragma GCC system_header
...	...	@@ -21,10 +19,10 @@
21	19	_LIBCPP_BEGIN_NAMESPACE_STD
22	20
23	21	template <class _Tp>
24		struct __align_type {
25		static const size_t value = _LIBCPP_PREFERRED_ALIGNOF(_Tp);
26		typedef _Tp type;
27		};
	22	struct _ALIGNAS(_LIBCPP_PREFERRED_ALIGNOF(_Tp)) _AlignedAsT {};
	23
	24	template <class... _Args>
	25	struct __max_align_impl : _AlignedAsT<_Args>... {};
28	26
29	27	struct __struct_double {
30	28	long double __lx;
...	...	@@ -33,41 +31,16 @@ struct __struct_double4 {
33	31	double __lx[4];
34	32	};
35	33
36		using __all_types _LIBCPP_NODEBUG =
37		__type_list<__align_type<unsigned char>,
38		__align_type<unsigned short>,
39		__align_type<unsigned int>,
40		__align_type<unsigned long>,
41		__align_type<unsigned long long>,
42		__align_type<double>,
43		__align_type<long double>,
44		__align_type<__struct_double>,
45		__align_type<__struct_double4>,
46		__align_type<int*> >;
47
48		template <class _TL, size_t _Len>
49		struct __find_max_align;
50
51		template <class _Head, size_t _Len>
52		struct __find_max_align<__type_list<_Head>, _Len> : public integral_constant<size_t, _Head::value> {};
53
54		template <size_t _Len, size_t _A1, size_t _A2>
55		struct __select_align {
56		private:
57		static const size_t __min = _A2 < _A1 ? _A2 : _A1;
58		static const size_t __max = _A1 < _A2 ? _A2 : _A1;
59
60		public:
61		static const size_t value = _Len < __max ? __min : __max;
62		};
	34	inline const size_t __aligned_storage_max_align =
	35	_LIBCPP_ALIGNOF(__max_align_impl<unsigned long long, double, long double, __struct_double, __struct_double4, int*>);
63	36
64		template <class _Head, class... _Tail, size_t _Len>
65		struct __find_max_align<__type_list<_Head, _Tail...>, _Len>
66		: public integral_constant<
67		size_t,
68		__select_align<_Len, _Head::value, __find_max_align<__type_list<_Tail...>, _Len>::value>::value> {};
	37	template <size_t _Len>
	38	inline const size_t __aligned_storage_alignment =
	39	_Len > __aligned_storage_max_align
	40	? __aligned_storage_max_align
	41	: size_t(1) << ((sizeof(size_t) * __CHAR_BIT__) - __builtin_clzg(_Len) - 1);
69	42
70		template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
	43	template <size_t _Len, size_t _Align = __aligned_storage_alignment<_Len> >
71	44	struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_NO_SPECIALIZATIONS aligned_storage {
72	45	union _ALIGNAS(_Align) type {
73	46	unsigned char __data[(_Len + _Align - 1) / _Align * _Align];
...	...	@@ -77,7 +50,7 @@ struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_NO_SPECIALIZATIONS aligned_storage {
77	50	#if _LIBCPP_STD_VER >= 14
78	51
79	52	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
80		template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
	53	template <size_t _Len, size_t _Align = __aligned_storage_alignment<_Len> >
81	54	using aligned_storage_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_storage<_Len, _Align>::type;
82	55	_LIBCPP_SUPPRESS_DEPRECATED_POP
83	56

lib/libcxx/include/__type_traits/desugars_to.h+4

...	...	@@ -10,6 +10,7 @@
10	10	#define _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H
11	11
12	12	#include <__config>
	13	#include <__type_traits/integral_constant.h>
13	14
14	15	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
15	16	# pragma GCC system_header
...	...	@@ -64,6 +65,9 @@ template <class _CanonicalTag, class _Operation, class... _Args>
64	65	inline const bool __desugars_to_v<_CanonicalTag, _Operation&&, _Args...> =
65	66	__desugars_to_v<_CanonicalTag, _Operation, _Args...>;
66	67
	68	template <class _CanonicalTag, class _Operation, class... _Args>
	69	struct __desugars_to : integral_constant<bool, __desugars_to_v<_CanonicalTag, _Operation, _Args...> > {};
	70
67	71	_LIBCPP_END_NAMESPACE_STD
68	72
69	73	#endif // _LIBCPP___TYPE_TRAITS_DESUGARS_TO_H

lib/libcxx/include/__type_traits/invoke.h+8

...	...	@@ -62,6 +62,9 @@
62	62	//
63	63	// template <class Func, class... Args>
64	64	// using __invoke_result_t = invoke_result_t<Func, Args...>;
	65	//
	66	// template <class Ret, class Func, class... Args>
	67	// struct __is_invocable_r : is_invocable_r<Ret, Func, Args...> {};
65	68
66	69	_LIBCPP_BEGIN_NAMESPACE_STD
67	70
...	...	@@ -112,8 +115,10 @@ inline const bool __is_invocable_r_v = __is_invocable_r_impl<_Ret, __is_invocabl
112	115	template <bool __is_invocable, class... _Args>
113	116	inline const bool __is_nothrow_invocable_impl = false;
114	117
	118	# ifndef _LIBCPP_CXX03_LANG
115	119	template <class... _Args>
116	120	inline const bool __is_nothrow_invocable_impl<true, _Args...> = noexcept(__builtin_invoke(std::declval<_Args>()...));
	121	# endif
117	122
118	123	template <class... _Args>
119	124	inline const bool __is_nothrow_invocable_v = __is_nothrow_invocable_impl<__is_invocable_v<_Args...>, _Args...>;
...	...	@@ -327,6 +332,9 @@ using __invoke_result_t _LIBCPP_NODEBUG = typename __invoke_result<_Func, _Args.
327	332
328	333	#endif // __has_builtin(__builtin_invoke_r)
329	334
	335	template <class _Ret, class _Func, class... _Args>
	336	struct __is_invocable_r : integral_constant<bool, __is_invocable_r_v<_Ret, _Func, _Args...> > {};
	337
330	338	template <class _Ret, bool = is_void<_Ret>::value>
331	339	struct __invoke_void_return_wrapper {
332	340	template <class... _Args>

lib/libcxx/include/__type_traits/is_allocator.h+6-7

...	...	@@ -11,7 +11,6 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
14		#include <__type_traits/integral_constant.h>
15	14	#include <__type_traits/void_t.h>
16	15	#include <__utility/declval.h>
17	16
...	...	@@ -21,13 +20,13 @@
21	20
22	21	_LIBCPP_BEGIN_NAMESPACE_STD
23	22
24		template <typename _Alloc, typename = void, typename = void>
25		struct __is_allocator : false_type {};
	23	template <class _Alloc, class = void, class = void>
	24	inline const bool __is_allocator_v = false;
26	25
27		template <typename _Alloc>
28		struct __is_allocator<_Alloc,
29		__void_t<typename _Alloc::value_type>,
30		__void_t<decltype(std::declval<_Alloc&>().allocate(size_t(0)))> > : true_type {};
	26	template <class _Alloc>
	27	inline const bool __is_allocator_v<_Alloc,
	28	__void_t<typename _Alloc::value_type>,
	29	__void_t<decltype(std::declval<_Alloc&>().allocate(size_t()))> > = true;
31	30
32	31	_LIBCPP_END_NAMESPACE_STD
33	32

lib/libcxx/include/__type_traits/is_array.h+26

...	...	@@ -26,6 +26,32 @@ template <class _Tp>
26	26	_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_array_v = __is_array(_Tp);
27	27	#endif
28	28
	29	template <class _Tp>
	30	inline const bool __is_bounded_array_v = __is_bounded_array(_Tp);
	31
	32	#if _LIBCPP_STD_VER >= 20
	33
	34	template <class _Tp>
	35	struct _LIBCPP_NO_SPECIALIZATIONS is_bounded_array : bool_constant<__is_bounded_array(_Tp)> {};
	36
	37	template <class _Tp>
	38	_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_bounded_array_v = __is_bounded_array(_Tp);
	39
	40	#endif
	41
	42	template <class _Tp>
	43	inline const bool __is_unbounded_array_v = __is_unbounded_array(_Tp);
	44
	45	#if _LIBCPP_STD_VER >= 20
	46
	47	template <class _Tp>
	48	struct _LIBCPP_NO_SPECIALIZATIONS is_unbounded_array : bool_constant<__is_unbounded_array(_Tp)> {};
	49
	50	template <class _Tp>
	51	_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_unbounded_array_v = __is_unbounded_array(_Tp);
	52
	53	#endif
	54
29	55	_LIBCPP_END_NAMESPACE_STD
30	56
31	57	#endif // _LIBCPP___TYPE_TRAITS_IS_ARRAY_H

lib/libcxx/include/__type_traits/is_equality_comparable.h+16-21

...	...	@@ -27,11 +27,11 @@
27	27	_LIBCPP_BEGIN_NAMESPACE_STD
28	28
29	29	template <class _Tp, class _Up, class = void>
30		struct __is_equality_comparable : false_type {};
	30	inline const bool __is_equality_comparable_v = false;
31	31
32	32	template <class _Tp, class _Up>
33		struct __is_equality_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>() == std::declval<_Up>())> > : true_type {
34		};
	33	inline const bool
	34	__is_equality_comparable_v<_Tp, _Up, __void_t<decltype(std::declval<_Tp>() == std::declval<_Up>())> > = true;
35	35
36	36	// A type is_trivially_equality_comparable if the expression `a == b` is equivalent to `std::memcmp(&a, &b, sizeof(T))`
37	37	// (with `a` and `b` being of type `T`). For the case where we compare two object of the same type, we can use
...	...	@@ -48,40 +48,35 @@ struct __is_equality_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>()
48	48	// representation may not be equivalent.
49	49
50	50	template <class _Tp, class _Up, class = void>
51		struct __libcpp_is_trivially_equality_comparable_impl : false_type {};
	51	inline const bool __is_trivially_equality_comparable_impl = false;
52	52
53	53	template <class _Tp>
54		struct __libcpp_is_trivially_equality_comparable_impl<_Tp, _Tp>
	54	inline const bool __is_trivially_equality_comparable_impl<_Tp, _Tp>
55	55	#if __has_builtin(__is_trivially_equality_comparable)
56		: integral_constant<bool, __is_trivially_equality_comparable(_Tp) && __is_equality_comparable<_Tp, _Tp>::value> {
57		};
	56	= __is_trivially_equality_comparable(_Tp) && __is_equality_comparable_v<_Tp, _Tp>;
58	57	#else
59		: is_integral<_Tp> {
60		};
	58	= is_integral<_Tp>::value;
61	59	#endif // __has_builtin(__is_trivially_equality_comparable)
62	60
63	61	template <class _Tp, class _Up>
64		struct __libcpp_is_trivially_equality_comparable_impl<
	62	inline const bool __is_trivially_equality_comparable_impl<
65	63	_Tp,
66	64	_Up,
67		__enable_if_t<is_integral<_Tp>::value && is_integral<_Up>::value && !is_same<_Tp, _Up>::value &&
68		is_signed<_Tp>::value == is_signed<_Up>::value && sizeof(_Tp) == sizeof(_Up)> > : true_type {};
	65	__enable_if_t<is_integral<_Tp>::value && is_integral<_Up>::value && !is_same<_Tp, _Up>::value> > =
	66	is_signed<_Tp>::value == is_signed<_Up>::value && sizeof(_Tp) == sizeof(_Up);
69	67
70	68	template <class _Tp>
71		struct __libcpp_is_trivially_equality_comparable_impl<_Tp, _Tp> : true_type {};
	69	inline const bool __is_trivially_equality_comparable_impl<_Tp, _Tp> = true;
72	70
73	71	// TODO: Use is_pointer_inverconvertible_base_of
74	72	template <class _Tp, class _Up>
75		struct __libcpp_is_trivially_equality_comparable_impl<_Tp, _Up>
76		: integral_constant<
77		bool,
78		__is_equality_comparable<_Tp, _Up>::value &&
79		(is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value \|\| is_void<_Tp>::value \|\| is_void<_Up>::value)> {
80		};
	73	inline const bool __is_trivially_equality_comparable_impl<_Tp, _Up> =
	74	__is_equality_comparable_v<_Tp, _Up> &&
	75	(is_same<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >::value \|\| is_void<_Tp>::value \|\| is_void<_Up>::value);
81	76
82	77	template <class _Tp, class _Up>
83		using __libcpp_is_trivially_equality_comparable _LIBCPP_NODEBUG =
84		__libcpp_is_trivially_equality_comparable_impl<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >;
	78	inline const bool __is_trivially_equality_comparable_v =
	79	__is_trivially_equality_comparable_impl<__remove_cv_t<_Tp>, __remove_cv_t<_Up> >;
85	80
86	81	_LIBCPP_END_NAMESPACE_STD
87	82

lib/libcxx/include/__type_traits/is_final.h+1-1

...	...	@@ -19,7 +19,7 @@
19	19	_LIBCPP_BEGIN_NAMESPACE_STD
20	20
21	21	template <class _Tp>
22		struct __libcpp_is_final : integral_constant<bool, __is_final(_Tp)> {};
	22	inline const bool __is_final_v = __is_final(_Tp);
23	23
24	24	#if _LIBCPP_STD_VER >= 14
25	25	template <class _Tp>

lib/libcxx/include/__type_traits/is_floating_point.h+7-6

...	...	@@ -20,18 +20,19 @@
20	20	_LIBCPP_BEGIN_NAMESPACE_STD
21	21
22	22	// clang-format off
23		template <class _Tp> struct __libcpp_is_floating_point : false_type {};
24		template <> struct __libcpp_is_floating_point<float> : true_type {};
25		template <> struct __libcpp_is_floating_point<double> : true_type {};
26		template <> struct __libcpp_is_floating_point<long double> : true_type {};
	23	template <class _Tp> inline const bool __is_floating_point_impl = false;
	24	template <> inline const bool __is_floating_point_impl<float> = true;
	25	template <> inline const bool __is_floating_point_impl<double> = true;
	26	template <> inline const bool __is_floating_point_impl<long double> = true;
27	27	// clang-format on
28	28
29	29	template <class _Tp>
30		struct _LIBCPP_NO_SPECIALIZATIONS is_floating_point : __libcpp_is_floating_point<__remove_cv_t<_Tp> > {};
	30	struct _LIBCPP_NO_SPECIALIZATIONS is_floating_point
	31	: integral_constant<bool, __is_floating_point_impl<__remove_cv_t<_Tp> > > {};
31	32
32	33	#if _LIBCPP_STD_VER >= 17
33	34	template <class _Tp>
34		_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_floating_point_v = is_floating_point<_Tp>::value;
	35	_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_floating_point_v = __is_floating_point_impl<__remove_cv_t<_Tp>>;
35	36	#endif
36	37
37	38	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/__type_traits/is_generic_transparent_comparator.h created+30

...	...	@@ -0,0 +1,30 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___TYPE_TRAITS_IS_GENERIC_TRANSPARENT_COMPARATOR_H
	10	#define _LIBCPP___TYPE_TRAITS_IS_GENERIC_TRANSPARENT_COMPARATOR_H
	11
	12	#include <__config>
	13
	14	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	15	# pragma GCC system_header
	16	#endif
	17
	18	_LIBCPP_BEGIN_NAMESPACE_STD
	19
	20	// This trait returns true if the given _Comparator is known to accept any two types for comparison. This is separate
	21	// from `__is_transparent_v`, since that only enables overloads of specific functions, but doesn't give any semantic
	22	// guarantees. This trait guarantess that the comparator simply calls the appropriate comparison functions for any two
	23	// types.
	24
	25	template <class _Comparator>
	26	inline const bool __is_generic_transparent_comparator_v = false;
	27
	28	_LIBCPP_END_NAMESPACE_STD
	29
	30	#endif // _LIBCPP___TYPE_TRAITS_IS_GENERIC_TRANSPARENT_COMPARATOR_H

lib/libcxx/include/__type_traits/is_specialization.h+2-6

...	...	@@ -30,15 +30,11 @@
30	30
31	31	_LIBCPP_BEGIN_NAMESPACE_STD
32	32
33		#if _LIBCPP_STD_VER >= 17
34
35	33	template <class _Tp, template <class...> class _Template>
36		inline constexpr bool __is_specialization_v = false; // true if and only if _Tp is a specialization of _Template
	34	inline const bool __is_specialization_v = false; // true if and only if _Tp is a specialization of _Template
37	35
38	36	template <template <class...> class _Template, class... _Args>
39		inline constexpr bool __is_specialization_v<_Template<_Args...>, _Template> = true;
40
41		#endif // _LIBCPP_STD_VER >= 17
	37	inline const bool __is_specialization_v<_Template<_Args...>, _Template> = true;
42	38
43	39	_LIBCPP_END_NAMESPACE_STD
44	40

lib/libcxx/include/__type_traits/is_within_lifetime.h created+29

...	...	@@ -0,0 +1,29 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___TYPE_TRAITS_IS_WITHIN_LIFETIME_H
	10	#define _LIBCPP___TYPE_TRAITS_IS_WITHIN_LIFETIME_H
	11
	12	#include <__config>
	13
	14	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	15	# pragma GCC system_header
	16	#endif
	17
	18	_LIBCPP_BEGIN_NAMESPACE_STD
	19
	20	#if _LIBCPP_STD_VER >= 26 && __has_builtin(__builtin_is_within_lifetime)
	21	template <class _Tp>
	22	_LIBCPP_HIDE_FROM_ABI consteval bool is_within_lifetime(const _Tp* __p) noexcept {
	23	return __builtin_is_within_lifetime(__p);
	24	}
	25	#endif
	26
	27	_LIBCPP_END_NAMESPACE_STD
	28
	29	#endif // _LIBCPP___TYPE_TRAITS_IS_WITHIN_LIFETIME_H

lib/libcxx/include/__type_traits/make_transparent.h created+52

...	...	@@ -0,0 +1,52 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___TYPE_TRAITS_MAKE_TRANSPARENT_H
	10	#define _LIBCPP___TYPE_TRAITS_MAKE_TRANSPARENT_H
	11
	12	#include <__config>
	13	#include <__type_traits/enable_if.h>
	14	#include <__type_traits/is_empty.h>
	15	#include <__type_traits/is_same.h>
	16
	17	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	18	# pragma GCC system_header
	19	#endif
	20
	21	_LIBCPP_BEGIN_NAMESPACE_STD
	22
	23	// __make_transparent tries to create a transparent comparator from its non-transparent counterpart, e.g. obtain
	24	// `less<>` from `less<T>`. This is useful in cases where conversions can be avoided (e.g. a string literal to a
	25	// std::string).
	26
	27	template <class _Tp, class _Comparator>
	28	struct __make_transparent {
	29	using type _LIBCPP_NODEBUG = _Comparator;
	30	};
	31
	32	template <class _Tp, class _Comparator>
	33	using __make_transparent_t _LIBCPP_NODEBUG = typename __make_transparent<_Tp, _Comparator>::type;
	34
	35	template <class _Tp,
	36	class _Comparator,
	37	__enable_if_t<is_same<_Comparator, __make_transparent_t<_Tp, _Comparator> >::value, int> = 0>
	38	_LIBCPP_HIDE_FROM_ABI _Comparator& __as_transparent(_Comparator& __comp) {
	39	return __comp;
	40	}
	41
	42	template <class _Tp,
	43	class _Comparator,
	44	__enable_if_t<!is_same<_Comparator, __make_transparent_t<_Tp, _Comparator> >::value, int> = 0>
	45	_LIBCPP_HIDE_FROM_ABI __make_transparent_t<_Tp, _Comparator> __as_transparent(_Comparator&) {
	46	static_assert(is_empty<_Comparator>::value);
	47	return __make_transparent_t<_Tp, _Comparator>();
	48	}
	49
	50	_LIBCPP_END_NAMESPACE_STD
	51
	52	#endif // _LIBCPP___TYPE_TRAITS_MAKE_TRANSPARENT_H

lib/libcxx/include/__type_traits/reference_constructs_from_temporary.h+1-7

...	...	@@ -18,7 +18,7 @@
18	18
19	19	_LIBCPP_BEGIN_NAMESPACE_STD
20	20
21		#if _LIBCPP_STD_VER >= 23 && __has_builtin(__reference_constructs_from_temporary)
	21	#if _LIBCPP_STD_VER >= 23
22	22
23	23	template <class _Tp, class _Up>
24	24	struct _LIBCPP_NO_SPECIALIZATIONS reference_constructs_from_temporary
...	...	@@ -30,14 +30,8 @@ _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool reference_constructs_from_tempo
30	30
31	31	#endif
32	32
33		#if __has_builtin(__reference_constructs_from_temporary)
34	33	template <class _Tp, class _Up>
35	34	inline const bool __reference_constructs_from_temporary_v = __reference_constructs_from_temporary(_Tp, _Up);
36		#else
37		// TODO(LLVM 22): Remove this as all supported compilers should have __reference_constructs_from_temporary implemented.
38		template <class _Tp, class _Up>
39		inline const bool __reference_constructs_from_temporary_v = __reference_binds_to_temporary(_Tp, _Up);
40		#endif
41	35
42	36	_LIBCPP_END_NAMESPACE_STD
43	37

lib/libcxx/include/__type_traits/reference_converts_from_temporary.h+1-1

...	...	@@ -18,7 +18,7 @@
18	18
19	19	_LIBCPP_BEGIN_NAMESPACE_STD
20	20
21		#if _LIBCPP_STD_VER >= 23 && __has_builtin(__reference_converts_from_temporary)
	21	#if _LIBCPP_STD_VER >= 23
22	22
23	23	template <class _Tp, class _Up>
24	24	struct _LIBCPP_NO_SPECIALIZATIONS reference_converts_from_temporary

lib/libcxx/include/__utility/cmp.h+19-7

...	...	@@ -26,10 +26,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
26	26
27	27	#if _LIBCPP_STD_VER >= 20
28	28
	29	template <typename _Tp, typename _Ip>
	30	concept __comparison_can_promote_to =
	31	sizeof(_Tp) < sizeof(_Ip) \|\| (sizeof(_Tp) == sizeof(_Ip) && __signed_integer<_Tp>);
	32
29	33	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
30		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
	34	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
31	35	if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
32	36	return __t == __u;
	37	else if constexpr (__comparison_can_promote_to<_Tp, int> && __comparison_can_promote_to<_Up, int>)
	38	return static_cast<int>(__t) == static_cast<int>(__u);
	39	else if constexpr (__comparison_can_promote_to<_Tp, long long> && __comparison_can_promote_to<_Up, long long>)
	40	return static_cast<long long>(__t) == static_cast<long long>(__u);
33	41	else if constexpr (is_signed_v<_Tp>)
34	42	return __t < 0 ? false : make_unsigned_t<_Tp>(__t) == __u;
35	43	else
...	...	@@ -37,14 +45,18 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_equal(_Tp __t, _Up __u) noexcept {
37	45	}
38	46
39	47	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
40		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_not_equal(_Tp __t, _Up __u) noexcept {
	48	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_not_equal(_Tp __t, _Up __u) noexcept {
41	49	return !std::cmp_equal(__t, __u);
42	50	}
43	51
44	52	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
45		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
	53	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
46	54	if constexpr (is_signed_v<_Tp> == is_signed_v<_Up>)
47	55	return __t < __u;
	56	else if constexpr (__comparison_can_promote_to<_Tp, int> && __comparison_can_promote_to<_Up, int>)
	57	return static_cast<int>(__t) < static_cast<int>(__u);
	58	else if constexpr (__comparison_can_promote_to<_Tp, long long> && __comparison_can_promote_to<_Up, long long>)
	59	return static_cast<long long>(__t) < static_cast<long long>(__u);
48	60	else if constexpr (is_signed_v<_Tp>)
49	61	return __t < 0 ? true : make_unsigned_t<_Tp>(__t) < __u;
50	62	else
...	...	@@ -52,22 +64,22 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less(_Tp __t, _Up __u) noexcept {
52	64	}
53	65
54	66	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
55		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater(_Tp __t, _Up __u) noexcept {
	67	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater(_Tp __t, _Up __u) noexcept {
56	68	return std::cmp_less(__u, __t);
57	69	}
58	70
59	71	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
60		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less_equal(_Tp __t, _Up __u) noexcept {
	72	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_less_equal(_Tp __t, _Up __u) noexcept {
61	73	return !std::cmp_greater(__t, __u);
62	74	}
63	75
64	76	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
65		_LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater_equal(_Tp __t, _Up __u) noexcept {
	77	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool cmp_greater_equal(_Tp __t, _Up __u) noexcept {
66	78	return !std::cmp_less(__t, __u);
67	79	}
68	80
69	81	template <__signed_or_unsigned_integer _Tp, __signed_or_unsigned_integer _Up>
70		_LIBCPP_HIDE_FROM_ABI constexpr bool in_range(_Up __u) noexcept {
	82	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool in_range(_Up __u) noexcept {
71	83	return std::cmp_less_equal(__u, numeric_limits<_Tp>::max()) &&
72	84	std::cmp_greater_equal(__u, numeric_limits<_Tp>::min());
73	85	}

lib/libcxx/include/__utility/default_three_way_comparator.h created+70

...	...	@@ -0,0 +1,70 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___UTILITY_DEFAULT_THREE_WAY_COMPARATOR_H
	10	#define _LIBCPP___UTILITY_DEFAULT_THREE_WAY_COMPARATOR_H
	11
	12	#include <__config>
	13	#include <__type_traits/enable_if.h>
	14	#include <__type_traits/is_arithmetic.h>
	15
	16	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	17	# pragma GCC system_header
	18	#endif
	19
	20	_LIBCPP_BEGIN_NAMESPACE_STD
	21
	22	// This struct can be specialized to provide a three way comparator between _LHS and _RHS.
	23	// The return value should be
	24	// - less than zero if (lhs_val < rhs_val)
	25	// - greater than zero if (rhs_val < lhs_val)
	26	// - zero otherwise
	27	template <class _LHS, class _RHS, class = void>
	28	struct __default_three_way_comparator;
	29
	30	template <class _LHS, class _RHS>
	31	struct __default_three_way_comparator<_LHS,
	32	_RHS,
	33	__enable_if_t<is_arithmetic<_LHS>::value && is_arithmetic<_RHS>::value> > {
	34	_LIBCPP_HIDE_FROM_ABI static int operator()(_LHS __lhs, _RHS __rhs) {
	35	if (__lhs < __rhs)
	36	return -1;
	37	if (__lhs > __rhs)
	38	return 1;
	39	return 0;
	40	}
	41	};
	42
	43	#if _LIBCPP_STD_VER >= 20 && __has_builtin(__builtin_lt_synthesizes_from_spaceship)
	44	template <class _LHS, class _RHS>
	45	struct __default_three_way_comparator<
	46	_LHS,
	47	_RHS,
	48	__enable_if_t<!(is_arithmetic<_LHS>::value && is_arithmetic<_RHS>::value) &&
	49	__builtin_lt_synthesizes_from_spaceship(const _LHS&, const _RHS&)>> {
	50	_LIBCPP_HIDE_FROM_ABI static int operator()(const _LHS& __lhs, const _RHS& __rhs) {
	51	auto __res = __lhs <=> __rhs;
	52	if (__res < 0)
	53	return -1;
	54	if (__res > 0)
	55	return 1;
	56	return 0;
	57	}
	58	};
	59	#endif
	60
	61	template <class _LHS, class _RHS, bool = true>
	62	struct __has_default_three_way_comparator : false_type {};
	63
	64	template <class _LHS, class _RHS>
	65	struct __has_default_three_way_comparator<_LHS, _RHS, sizeof(__default_three_way_comparator<_LHS, _RHS>) >= 0>
	66	: true_type {};
	67
	68	_LIBCPP_END_NAMESPACE_STD
	69
	70	#endif // _LIBCPP___UTILITY_DEFAULT_THREE_WAY_COMPARATOR_H

lib/libcxx/include/__utility/in_place.h+1-1

...	...	@@ -22,7 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
22	22
23	23	#if _LIBCPP_STD_VER >= 17
24	24
25		struct _LIBCPP_EXPORTED_FROM_ABI in_place_t {
	25	struct in_place_t {
26	26	explicit in_place_t() = default;
27	27	};
28	28	inline constexpr in_place_t in_place{};

lib/libcxx/include/__utility/integer_sequence.h+56-41

...	...	@@ -11,63 +11,52 @@
11	11
12	12	#include <__config>
13	13	#include <__cstddef/size_t.h>
	14	#include <__tuple/tuple_element.h>
	15	#include <__tuple/tuple_size.h>
14	16	#include <__type_traits/is_integral.h>
15	17
16	18	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
17	19	# pragma GCC system_header
18	20	#endif
19	21
	22	#ifndef _LIBCPP_CXX03_LANG
	23
20	24	_LIBCPP_BEGIN_NAMESPACE_STD
21	25
22		template <size_t...>
23		struct __tuple_indices;
	26	# if __has_builtin(__make_integer_seq)
	27	template <template <class _Tp, _Tp...> class _BaseType, class _Tp, _Tp _SequenceSize>
	28	using __make_integer_sequence_impl _LIBCPP_NODEBUG = __make_integer_seq<_BaseType, _Tp, _SequenceSize>;
	29	# else
	30	template <template <class _Tp, _Tp...> class _BaseType, class _Tp, _Tp _SequenceSize>
	31	using __make_integer_sequence_impl _LIBCPP_NODEBUG = _BaseType<_Tp, __integer_pack(_SequenceSize)...>;
	32	# endif
24	33
25		template <class _IdxType, _IdxType... _Values>
	34	template <class _Tp, _Tp... _Indices>
26	35	struct __integer_sequence {
27		template <template <class _OIdxType, _OIdxType...> class _ToIndexSeq, class _ToIndexType>
28		using __convert _LIBCPP_NODEBUG = _ToIndexSeq<_ToIndexType, _Values...>;
29
30		template <size_t _Sp>
31		using __to_tuple_indices _LIBCPP_NODEBUG = __tuple_indices<(_Values + _Sp)...>;
	36	using value_type = _Tp;
	37	static_assert(is_integral<_Tp>::value, "std::integer_sequence can only be instantiated with an integral type");
	38	[[__nodiscard__]] static _LIBCPP_HIDE_FROM_ABI constexpr size_t size() noexcept { return sizeof...(_Indices); }
32	39	};
33	40
34		#if __has_builtin(__make_integer_seq)
35		template <size_t _Ep, size_t _Sp>
36		using __make_indices_imp _LIBCPP_NODEBUG =
37		typename __make_integer_seq<__integer_sequence, size_t, _Ep - _Sp>::template __to_tuple_indices<_Sp>;
38		#elif __has_builtin(__integer_pack)
39		template <size_t _Ep, size_t _Sp>
40		using __make_indices_imp _LIBCPP_NODEBUG =
41		typename __integer_sequence<size_t, __integer_pack(_Ep - _Sp)...>::template __to_tuple_indices<_Sp>;
42		#else
43		# error "No known way to get an integer pack from the compiler"
44		#endif
	41	template <size_t... _Indices>
	42	using __index_sequence _LIBCPP_NODEBUG = __integer_sequence<size_t, _Indices...>;
45	43
46		#if _LIBCPP_STD_VER >= 14
	44	template <size_t _SequenceSize>
	45	using __make_index_sequence _LIBCPP_NODEBUG = __make_integer_sequence_impl<__integer_sequence, size_t, _SequenceSize>;
47	46
48		template <class _Tp, _Tp... _Ip>
49		struct integer_sequence {
50		typedef _Tp value_type;
51		static_assert(is_integral<_Tp>::value, "std::integer_sequence can only be instantiated with an integral type");
52		static _LIBCPP_HIDE_FROM_ABI constexpr size_t size() noexcept { return sizeof...(_Ip); }
53		};
	47	template <class... _Args>
	48	using __index_sequence_for _LIBCPP_NODEBUG = __make_index_sequence<sizeof...(_Args)>;
	49
	50	# if _LIBCPP_STD_VER >= 14
	51
	52	template <class _Tp, _Tp... _Indices>
	53	struct integer_sequence : __integer_sequence<_Tp, _Indices...> {};
54	54
55	55	template <size_t... _Ip>
56	56	using index_sequence = integer_sequence<size_t, _Ip...>;
57	57
58		# if __has_builtin(__make_integer_seq)
59
60	58	template <class _Tp, _Tp _Ep>
61		using make_integer_sequence _LIBCPP_NODEBUG = __make_integer_seq<integer_sequence, _Tp, _Ep>;
62
63		# elif __has_builtin(__integer_pack)
64
65		template <class _Tp, _Tp _SequenceSize>
66		using make_integer_sequence _LIBCPP_NODEBUG = integer_sequence<_Tp, __integer_pack(_SequenceSize)...>;
67
68		# else
69		# error "No known way to get an integer pack from the compiler"
70		# endif
	59	using make_integer_sequence _LIBCPP_NODEBUG = __make_integer_sequence_impl<integer_sequence, _Tp, _Ep>;
71	60
72	61	template <size_t _Np>
73	62	using make_index_sequence = make_integer_sequence<size_t, _Np>;
...	...	@@ -75,16 +64,42 @@ using make_index_sequence = make_integer_sequence<size_t, _Np>;
75	64	template <class... _Tp>
76	65	using index_sequence_for = make_index_sequence<sizeof...(_Tp)>;
77	66
78		# if _LIBCPP_STD_VER >= 20
	67	# if _LIBCPP_STD_VER >= 20
79	68	// Executes __func for every element in an index_sequence.
80	69	template <size_t... _Index, class _Function>
81	70	_LIBCPP_HIDE_FROM_ABI constexpr void __for_each_index_sequence(index_sequence<_Index...>, _Function __func) {
82	71	(__func.template operator()<_Index>(), ...);
83	72	}
84		# endif // _LIBCPP_STD_VER >= 20
	73	# endif // _LIBCPP_STD_VER >= 20
	74
	75	# if _LIBCPP_STD_VER >= 26
	76	// [intseq.binding], structured binding support
	77	template <class _Tp, _Tp... _Indices>
	78	struct tuple_size<integer_sequence<_Tp, _Indices...>> : integral_constant<size_t, sizeof...(_Indices)> {};
85	79
86		#endif // _LIBCPP_STD_VER >= 14
	80	template <size_t _Ip, class _Tp, _Tp... _Indices>
	81	struct tuple_element<_Ip, integer_sequence<_Tp, _Indices...>> {
	82	static_assert(_Ip < sizeof...(_Indices), "Index out of bounds in std::tuple_element<> (std::integer_sequence)");
	83	using type _LIBCPP_NODEBUG = _Tp;
	84	};
	85
	86	template <size_t _Ip, class _Tp, _Tp... _Indices>
	87	struct tuple_element<_Ip, const integer_sequence<_Tp, _Indices...>> {
	88	static_assert(_Ip < sizeof...(_Indices), "Index out of bounds in std::tuple_element<> (const std::integer_sequence)");
	89	using type _LIBCPP_NODEBUG = _Tp;
	90	};
	91
	92	template <size_t _Ip, class _Tp, _Tp... _Indices>
	93	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp get(integer_sequence<_Tp, _Indices...>) noexcept {
	94	static_assert(_Ip < sizeof...(_Indices), "Index out of bounds in std::get<> (std::integer_sequence)");
	95	return _Indices...[_Ip];
	96	}
	97	# endif // _LIBCPP_STD_VER >= 26
	98
	99	# endif // _LIBCPP_STD_VER >= 14
87	100
88	101	_LIBCPP_END_NAMESPACE_STD
89	102
	103	#endif // _LIBCPP_CXX03_LANG
	104
90	105	#endif // _LIBCPP___UTILITY_INTEGER_SEQUENCE_H

lib/libcxx/include/__utility/lazy_synth_three_way_comparator.h created+120

...	...	@@ -0,0 +1,120 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___UTILITY_LAZY_SYNTH_THREE_WAY_COMPARATOR_H
	10	#define _LIBCPP___UTILITY_LAZY_SYNTH_THREE_WAY_COMPARATOR_H
	11
	12	#include <__assert>
	13	#include <__config>
	14	#include <__type_traits/conjunction.h>
	15	#include <__type_traits/desugars_to.h>
	16	#include <__type_traits/enable_if.h>
	17	#include <__utility/default_three_way_comparator.h>
	18
	19	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	20	# pragma GCC system_header
	21	#endif
	22
	23	// This file implements a __lazy_synth_three_way_comparator, which tries to build an efficient three way comparison from
	24	// a binary comparator. That is done in multiple steps:
	25	// 1) Check whether the comparator desugars to a less-than operator
	26	// If that is the case, check whether there exists a specialization of `__default_three_way_comparator`, which
	27	// can be specialized to implement a three way comparator for the specific types.
	28	// 2) Fall back to doing a lazy less than/greater than comparison
	29
	30	_LIBCPP_BEGIN_NAMESPACE_STD
	31
	32	template <class _Comparator, class _LHS, class _RHS>
	33	struct __lazy_compare_result {
	34	const _Comparator& __comp_;
	35	const _LHS& __lhs_;
	36	const _RHS& __rhs_;
	37
	38	_LIBCPP_HIDE_FROM_ABI
	39	__lazy_compare_result(_LIBCPP_CTOR_LIFETIMEBOUND const _Comparator& __comp,
	40	_LIBCPP_CTOR_LIFETIMEBOUND const _LHS& __lhs,
	41	_LIBCPP_CTOR_LIFETIMEBOUND const _RHS& __rhs)
	42	: __comp_(__comp), __lhs_(__lhs), __rhs_(__rhs) {}
	43
	44	_LIBCPP_HIDE_FROM_ABI bool __less() const {
	45	bool __result = __comp_(__lhs_, __rhs_);
	46	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__result ? !static_cast<bool>(__comp_(__rhs_, __lhs_)) : true,
	47	"Comparator does not induce a strict weak ordering");
	48	return __result;
	49	}
	50
	51	_LIBCPP_HIDE_FROM_ABI bool __greater() const {
	52	bool __result = __comp_(__rhs_, __lhs_);
	53	_LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(__result ? !static_cast<bool>(__comp_(__lhs_, __rhs_)) : true,
	54	"Comparator does not induce a strict weak ordering");
	55	return __result;
	56	}
	57	};
	58
	59	// This class provides three way comparison between _LHS and _RHS as efficiently as possible. This can be specialized if
	60	// a comparator only compares part of the object, potentially allowing an efficient three way comparison between the
	61	// subobjects. The specialization should use the __lazy_synth_three_way_comparator for the subobjects to achieve this.
	62	template <class _Comparator, class _LHS, class _RHS, class = void>
	63	struct __lazy_synth_three_way_comparator {
	64	const _Comparator& __comp_;
	65
	66	_LIBCPP_HIDE_FROM_ABI __lazy_synth_three_way_comparator(_LIBCPP_CTOR_LIFETIMEBOUND const _Comparator& __comp)
	67	: __comp_(__comp) {}
	68
	69	_LIBCPP_HIDE_FROM_ABI __lazy_compare_result<_Comparator, _LHS, _RHS>
	70	operator()(_LIBCPP_LIFETIMEBOUND const _LHS& __lhs, _LIBCPP_LIFETIMEBOUND const _RHS& __rhs) const {
	71	return __lazy_compare_result<_Comparator, _LHS, _RHS>(__comp_, __lhs, __rhs);
	72	}
	73	};
	74
	75	struct __eager_compare_result {
	76	int __res_;
	77
	78	_LIBCPP_HIDE_FROM_ABI explicit __eager_compare_result(int __res) : __res_(__res) {}
	79
	80	_LIBCPP_HIDE_FROM_ABI bool __less() const { return __res_ < 0; }
	81	_LIBCPP_HIDE_FROM_ABI bool __greater() const { return __res_ > 0; }
	82	};
	83
	84	template <class _Comparator, class _LHS, class _RHS>
	85	struct __lazy_synth_three_way_comparator<_Comparator,
	86	_LHS,
	87	_RHS,
	88	__enable_if_t<_And<__desugars_to<__less_tag, _Comparator, _LHS, _RHS>,
	89	__has_default_three_way_comparator<_LHS, _RHS> >::value> > {
	90	// This lifetimebound annotation is technically incorrect, but other specializations actually capture the lifetime of
	91	// the comparator.
	92	_LIBCPP_HIDE_FROM_ABI __lazy_synth_three_way_comparator(_LIBCPP_CTOR_LIFETIMEBOUND const _Comparator&) {}
	93
	94	// Same comment as above.
	95	_LIBCPP_HIDE_FROM_ABI static __eager_compare_result
	96	operator()(_LIBCPP_LIFETIMEBOUND const _LHS& __lhs, _LIBCPP_LIFETIMEBOUND const _RHS& __rhs) {
	97	return __eager_compare_result(__default_three_way_comparator<_LHS, _RHS>()(__lhs, __rhs));
	98	}
	99	};
	100
	101	template <class _Comparator, class _LHS, class _RHS>
	102	struct __lazy_synth_three_way_comparator<_Comparator,
	103	_LHS,
	104	_RHS,
	105	__enable_if_t<_And<__desugars_to<__greater_tag, _Comparator, _LHS, _RHS>,
	106	__has_default_three_way_comparator<_LHS, _RHS> >::value> > {
	107	// This lifetimebound annotation is technically incorrect, but other specializations actually capture the lifetime of
	108	// the comparator.
	109	_LIBCPP_HIDE_FROM_ABI __lazy_synth_three_way_comparator(_LIBCPP_CTOR_LIFETIMEBOUND const _Comparator&) {}
	110
	111	// Same comment as above.
	112	_LIBCPP_HIDE_FROM_ABI static __eager_compare_result
	113	operator()(_LIBCPP_LIFETIMEBOUND const _LHS& __lhs, _LIBCPP_LIFETIMEBOUND const _RHS& __rhs) {
	114	return __eager_compare_result(-__default_three_way_comparator<_LHS, _RHS>()(__lhs, __rhs));
	115	}
	116	};
	117
	118	_LIBCPP_END_NAMESPACE_STD
	119
	120	#endif // _LIBCPP___UTILITY_LAZY_SYNTH_THREE_WAY_COMPARATOR_H

lib/libcxx/include/__utility/pair.h+22-25

...	...	@@ -18,7 +18,6 @@
18	18	#include <__fwd/array.h>
19	19	#include <__fwd/pair.h>
20	20	#include <__fwd/tuple.h>
21		#include <__tuple/tuple_indices.h>
22	21	#include <__tuple/tuple_like_no_subrange.h>
23	22	#include <__tuple/tuple_size.h>
24	23	#include <__type_traits/common_reference.h>
...	...	@@ -32,14 +31,13 @@
32	31	#include <__type_traits/is_implicitly_default_constructible.h>
33	32	#include <__type_traits/is_nothrow_assignable.h>
34	33	#include <__type_traits/is_nothrow_constructible.h>
35		#include <__type_traits/is_replaceable.h>
36		#include <__type_traits/is_same.h>
37	34	#include <__type_traits/is_swappable.h>
38	35	#include <__type_traits/is_trivially_relocatable.h>
39	36	#include <__type_traits/nat.h>
40	37	#include <__type_traits/unwrap_ref.h>
41	38	#include <__utility/declval.h>
42	39	#include <__utility/forward.h>
	40	#include <__utility/integer_sequence.h>
43	41	#include <__utility/move.h>
44	42	#include <__utility/piecewise_construct.h>
45	43
...	...	@@ -102,7 +100,6 @@ struct pair
102	100	__conditional_t<__libcpp_is_trivially_relocatable<_T1>::value && __libcpp_is_trivially_relocatable<_T2>::value,
103	101	pair,
104	102	void>;
105		using __replaceable _LIBCPP_NODEBUG = __conditional_t<__is_replaceable_v<_T1> && __is_replaceable_v<_T2>, pair, void>;
106	103
107	104	_LIBCPP_HIDE_FROM_ABI pair(pair const&) = default;
108	105	_LIBCPP_HIDE_FROM_ABI pair(pair&&) = default;
...	...	@@ -222,11 +219,7 @@ struct pair
222	219	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
223	220	pair(piecewise_construct_t __pc, tuple<_Args1...> __first_args, tuple<_Args2...> __second_args) noexcept(
224	221	is_nothrow_constructible<first_type, _Args1...>::value && is_nothrow_constructible<second_type, _Args2...>::value)
225		: pair(__pc,
226		__first_args,
227		__second_args,
228		typename __make_tuple_indices<sizeof...(_Args1)>::type(),
229		typename __make_tuple_indices<sizeof...(_Args2) >::type()) {}
	222	: pair(__pc, __first_args, __second_args, __index_sequence_for<_Args1...>(), __index_sequence_for<_Args2...>()) {}
230	223
231	224	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair&
232	225	operator=(__conditional_t<is_copy_assignable<first_type>::value && is_copy_assignable<second_type>::value,
...	...	@@ -440,8 +433,8 @@ private:
440	433	pair(piecewise_construct_t,
441	434	tuple<_Args1...>& __first_args,
442	435	tuple<_Args2...>& __second_args,
443		__tuple_indices<_I1...>,
444		__tuple_indices<_I2...>)
	436	__index_sequence<_I1...>,
	437	__index_sequence<_I2...>)
445	438	: first(std::forward<_Args1>(std::get<_I1>(__first_args))...),
446	439	second(std::forward<_Args2>(std::get<_I2>(__second_args))...) {}
447	440	#endif
...	...	@@ -546,8 +539,8 @@ swap(const pair<_T1, _T2>& __x, const pair<_T1, _T2>& __y) noexcept(noexcept(__x
546	539	#endif
547	540
548	541	template <class _T1, class _T2>
549		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<__unwrap_ref_decay_t<_T1>, __unwrap_ref_decay_t<_T2> >
550		make_pair(_T1&& __t1, _T2&& __t2) {
	542	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	543	pair<__unwrap_ref_decay_t<_T1>, __unwrap_ref_decay_t<_T2> > make_pair(_T1&& __t1, _T2&& __t2) {
551	544	return pair<__unwrap_ref_decay_t<_T1>, __unwrap_ref_decay_t<_T2> >(std::forward<_T1>(__t1), std::forward<_T2>(__t2));
552	545	}
553	546
...	...	@@ -619,67 +612,71 @@ struct __get_pair<1> {
619	612	};
620	613
621	614	template <size_t _Ip, class _T1, class _T2>
622		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&
	615	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	616	typename tuple_element<_Ip, pair<_T1, _T2> >::type&
623	617	get(pair<_T1, _T2>& __p) _NOEXCEPT {
624	618	return __get_pair<_Ip>::get(__p);
625	619	}
626	620
627	621	template <size_t _Ip, class _T1, class _T2>
628		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&
	622	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	623	_LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&
629	624	get(const pair<_T1, _T2>& __p) _NOEXCEPT {
630	625	return __get_pair<_Ip>::get(__p);
631	626	}
632	627
633	628	template <size_t _Ip, class _T1, class _T2>
634		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
	629	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	630	typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
635	631	get(pair<_T1, _T2>&& __p) _NOEXCEPT {
636	632	return __get_pair<_Ip>::get(std::move(__p));
637	633	}
638	634
639	635	template <size_t _Ip, class _T1, class _T2>
640		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
	636	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	637	_LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, pair<_T1, _T2> >::type&&
641	638	get(const pair<_T1, _T2>&& __p) _NOEXCEPT {
642	639	return __get_pair<_Ip>::get(std::move(__p));
643	640	}
644	641
645	642	#if _LIBCPP_STD_VER >= 14
646	643	template <class _T1, class _T2>
647		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(pair<_T1, _T2>& __p) _NOEXCEPT {
	644	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(pair<_T1, _T2>& __p) _NOEXCEPT {
648	645	return __p.first;
649	646	}
650	647
651	648	template <class _T1, class _T2>
652		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(pair<_T1, _T2> const& __p) _NOEXCEPT {
	649	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(pair<_T1, _T2> const& __p) _NOEXCEPT {
653	650	return __p.first;
654	651	}
655	652
656	653	template <class _T1, class _T2>
657		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
	654	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
658	655	return std::forward<_T1&&>(__p.first);
659	656	}
660	657
661	658	template <class _T1, class _T2>
662		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(pair<_T1, _T2> const&& __p) _NOEXCEPT {
	659	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(pair<_T1, _T2> const&& __p) _NOEXCEPT {
663	660	return std::forward<_T1 const&&>(__p.first);
664	661	}
665	662
666	663	template <class _T2, class _T1>
667		inline _LIBCPP_HIDE_FROM_ABI constexpr _T2& get(pair<_T1, _T2>& __p) _NOEXCEPT {
	664	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T2& get(pair<_T1, _T2>& __p) _NOEXCEPT {
668	665	return __p.second;
669	666	}
670	667
671	668	template <class _T2, class _T1>
672		inline _LIBCPP_HIDE_FROM_ABI constexpr _T2 const& get(pair<_T1, _T2> const& __p) _NOEXCEPT {
	669	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T2 const& get(pair<_T1, _T2> const& __p) _NOEXCEPT {
673	670	return __p.second;
674	671	}
675	672
676	673	template <class _T2, class _T1>
677		inline _LIBCPP_HIDE_FROM_ABI constexpr _T2&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
	674	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T2&& get(pair<_T1, _T2>&& __p) _NOEXCEPT {
678	675	return std::forward<_T2&&>(__p.second);
679	676	}
680	677
681	678	template <class _T2, class _T1>
682		inline _LIBCPP_HIDE_FROM_ABI constexpr _T2 const&& get(pair<_T1, _T2> const&& __p) _NOEXCEPT {
	679	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T2 const&& get(pair<_T1, _T2> const&& __p) _NOEXCEPT {
683	680	return std::forward<_T2 const&&>(__p.second);
684	681	}
685	682

lib/libcxx/include/__utility/scope_guard.h+2

...	...	@@ -43,6 +43,8 @@ public:
43	43	#endif
44	44	};
45	45
	46	_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(__scope_guard);
	47
46	48	template <class _Func>
47	49	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __scope_guard<_Func> __make_scope_guard(_Func __func) {
48	50	return __scope_guard<_Func>(std::move(__func));

lib/libcxx/include/__utility/try_key_extraction.h created+114

...	...	@@ -0,0 +1,114 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP___UTILITY_TRY_EXTRACT_KEY_H
	10	#define _LIBCPP___UTILITY_TRY_EXTRACT_KEY_H
	11
	12	#include <__config>
	13	#include <__fwd/pair.h>
	14	#include <__fwd/tuple.h>
	15	#include <__type_traits/enable_if.h>
	16	#include <__type_traits/is_same.h>
	17	#include <__type_traits/remove_const.h>
	18	#include <__type_traits/remove_const_ref.h>
	19	#include <__utility/declval.h>
	20	#include <__utility/forward.h>
	21	#include <__utility/piecewise_construct.h>
	22	#include <__utility/priority_tag.h>
	23
	24	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	25	# pragma GCC system_header
	26	#endif
	27
	28	_LIBCPP_BEGIN_NAMESPACE_STD
	29
	30	template <class _KeyT, class _Ret, class _WithKey, class _WithoutKey, class... _Args>
	31	_LIBCPP_HIDE_FROM_ABI _Ret
	32	__try_key_extraction_impl(__priority_tag<0>, _WithKey, _WithoutKey __without_key, _Args&&... __args) {
	33	return __without_key(std::forward<_Args>(__args)...);
	34	}
	35
	36	template <class _KeyT,
	37	class _Ret,
	38	class _WithKey,
	39	class _WithoutKey,
	40	class _Arg,
	41	__enable_if_t<is_same<_KeyT, __remove_const_ref_t<_Arg> >::value, int> = 0>
	42	_LIBCPP_HIDE_FROM_ABI _Ret
	43	__try_key_extraction_impl(__priority_tag<1>, _WithKey __with_key, _WithoutKey, _Arg&& __arg) {
	44	return __with_key(__arg, std::forward<_Arg>(__arg));
	45	}
	46
	47	template <class _KeyT,
	48	class _Ret,
	49	class _WithKey,
	50	class _WithoutKey,
	51	class _Arg,
	52	__enable_if_t<__is_pair_v<__remove_const_ref_t<_Arg> > &&
	53	is_same<__remove_const_t<typename __remove_const_ref_t<_Arg>::first_type>, _KeyT>::value,
	54	int> = 0>
	55	_LIBCPP_HIDE_FROM_ABI _Ret
	56	__try_key_extraction_impl(__priority_tag<1>, _WithKey __with_key, _WithoutKey, _Arg&& __arg) {
	57	return __with_key(__arg.first, std::forward<_Arg>(__arg));
	58	}
	59
	60	template <class _KeyT,
	61	class _Ret,
	62	class _WithKey,
	63	class _WithoutKey,
	64	class _Arg1,
	65	class _Arg2,
	66	__enable_if_t<is_same<_KeyT, __remove_const_ref_t<_Arg1> >::value, int> = 0>
	67	_LIBCPP_HIDE_FROM_ABI _Ret
	68	__try_key_extraction_impl(__priority_tag<1>, _WithKey __with_key, _WithoutKey, _Arg1&& __arg1, _Arg2&& __arg2) {
	69	return __with_key(__arg1, std::forward<_Arg1>(__arg1), std::forward<_Arg2>(__arg2));
	70	}
	71
	72	#ifndef _LIBCPP_CXX03_LANG
	73	template <class _KeyT,
	74	class _Ret,
	75	class _WithKey,
	76	class _WithoutKey,
	77	class _PiecewiseConstruct,
	78	class _Tuple1,
	79	class _Tuple2,
	80	__enable_if_t<is_same<__remove_const_ref_t<_PiecewiseConstruct>, piecewise_construct_t>::value &&
	81	__is_tuple_v<_Tuple1> && tuple_size<_Tuple1>::value == 1 &&
	82	is_same<__remove_const_ref_t<typename tuple_element<0, _Tuple1>::type>, _KeyT>::value,
	83	int> = 0>
	84	_LIBCPP_HIDE_FROM_ABI _Ret __try_key_extraction_impl(
	85	__priority_tag<1>,
	86	_WithKey __with_key,
	87	_WithoutKey,
	88	_PiecewiseConstruct&& __pc,
	89	_Tuple1&& __tuple1,
	90	_Tuple2&& __tuple2) {
	91	return __with_key(
	92	std::get<0>(__tuple1),
	93	std::forward<_PiecewiseConstruct>(__pc),
	94	std::forward<_Tuple1>(__tuple1),
	95	std::forward<_Tuple2>(__tuple2));
	96	}
	97	#endif // _LIBCPP_CXX03_LANG
	98
	99	// This function tries extracting the given _KeyT from _Args...
	100	// If it succeeds to extract the key, it calls the `__with_key` function with the extracted key and all of the
	101	// arguments. Otherwise it calls the `__without_key` function with all of the arguments.
	102	//
	103	// Both `__with_key` and `__without_key` must take all arguments by reference.
	104	template <class _KeyT, class _WithKey, class _WithoutKey, class... _Args>
	105	_LIBCPP_HIDE_FROM_ABI decltype(std::declval<_WithoutKey>()(std::declval<_Args>()...))
	106	__try_key_extraction(_WithKey __with_key, _WithoutKey __without_key, _Args&&... __args) {
	107	using _Ret = decltype(__without_key(std::forward<_Args>(__args)...));
	108	return std::__try_key_extraction_impl<_KeyT, _Ret>(
	109	__priority_tag<1>(), __with_key, __without_key, std::forward<_Args>(__args)...);
	110	}
	111
	112	_LIBCPP_END_NAMESPACE_STD
	113
	114	#endif // _LIBCPP___UTILITY_TRY_EXTRACT_KEY_H

lib/libcxx/include/__vector/vector.h+187-160

...	...	@@ -12,18 +12,17 @@
12	12	#include <__algorithm/copy.h>
13	13	#include <__algorithm/copy_n.h>
14	14	#include <__algorithm/fill_n.h>
	15	#include <__algorithm/iterator_operations.h>
15	16	#include <__algorithm/max.h>
16	17	#include <__algorithm/min.h>
17	18	#include <__algorithm/move.h>
18	19	#include <__algorithm/move_backward.h>
19		#include <__algorithm/ranges_copy_n.h>
20	20	#include <__algorithm/rotate.h>
21	21	#include <__assert>
22	22	#include <__config>
23	23	#include <__debug_utils/sanitizers.h>
24	24	#include <__format/enable_insertable.h>
25	25	#include <__fwd/vector.h>
26		#include <__iterator/advance.h>
27	26	#include <__iterator/bounded_iter.h>
28	27	#include <__iterator/concepts.h>
29	28	#include <__iterator/distance.h>
...	...	@@ -43,6 +42,7 @@
43	42	#include <__memory/temp_value.h>
44	43	#include <__memory/uninitialized_algorithms.h>
45	44	#include <__ranges/access.h>
	45	#include <__ranges/as_rvalue_view.h>
46	46	#include <__ranges/concepts.h>
47	47	#include <__ranges/container_compatible_range.h>
48	48	#include <__ranges/from_range.h>
...	...	@@ -55,7 +55,6 @@
55	55	#include <__type_traits/is_nothrow_assignable.h>
56	56	#include <__type_traits/is_nothrow_constructible.h>
57	57	#include <__type_traits/is_pointer.h>
58		#include <__type_traits/is_replaceable.h>
59	58	#include <__type_traits/is_same.h>
60	59	#include <__type_traits/is_trivially_relocatable.h>
61	60	#include <__type_traits/type_identity.h>
...	...	@@ -86,6 +85,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
86	85
87	86	template <class _Tp, class _Allocator /* = allocator<_Tp> */>
88	87	class vector {
	88	template <class _Up, class _Alloc>
	89	using __split_buffer _LIBCPP_NODEBUG = std::__split_buffer<_Up, _Alloc, __split_buffer_pointer_layout>;
	90
89	91	public:
90	92	//
91	93	// Types
...	...	@@ -113,7 +115,7 @@ public:
113	115	using reverse_iterator = std::reverse_iterator<iterator>;
114	116	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
115	117
116		// A vector containers the following members which may be trivially relocatable:
	118	// A vector contains the following members which may be trivially relocatable:
117	119	// - pointer: may be trivially relocatable, so it's checked
118	120	// - allocator_type: may be trivially relocatable, so it's checked
119	121	// vector doesn't contain any self-references, so it's trivially relocatable if its members are.
...	...	@@ -121,10 +123,6 @@ public:
121	123	__libcpp_is_trivially_relocatable<pointer>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
122	124	vector,
123	125	void>;
124		using __replaceable _LIBCPP_NODEBUG =
125		__conditional_t<__is_replaceable_v<pointer> && __container_allocator_is_replaceable<__alloc_traits>::value,
126		vector,
127		void>;
128	126
129	127	static_assert(__check_valid_allocator<allocator_type>::value, "");
130	128	static_assert(is_same<typename allocator_type::value_type, value_type>::value,
...	...	@@ -174,7 +172,7 @@ public:
174	172	__guard.__complete();
175	173	}
176	174
177		template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
	175	template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0>
178	176	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
179	177	vector(size_type __n, const value_type& __x, const allocator_type& __a)
180	178	: __alloc_(__a) {
...	...	@@ -317,7 +315,7 @@ public:
317	315	is_constructible<value_type, typename iterator_traits<_ForwardIterator>::reference>::value,
318	316	int> = 0>
319	317	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void assign(_ForwardIterator __first, _ForwardIterator __last) {
320		__assign_with_size(__first, __last, std::distance(__first, __last));
	318	__assign_with_size<_ClassicAlgPolicy>(__first, __last, std::distance(__first, __last));
321	319	}
322	320
323	321	#if _LIBCPP_STD_VER >= 23
...	...	@@ -325,7 +323,7 @@ public:
325	323	_LIBCPP_HIDE_FROM_ABI constexpr void assign_range(_Range&& __range) {
326	324	if constexpr (ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>) {
327	325	auto __n = static_cast<size_type>(ranges::distance(__range));
328		__assign_with_size(ranges::begin(__range), ranges::end(__range), __n);
	326	__assign_with_size<_RangeAlgPolicy>(ranges::begin(__range), ranges::end(__range), __n);
329	327
330	328	} else {
331	329	__assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
...	...	@@ -341,59 +339,67 @@ public:
341	339	}
342	340	#endif
343	341
344		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	342	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
345	343	return this->__alloc_;
346	344	}
347	345
348	346	//
349	347	// Iterators
350	348	//
351		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
	349	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
352	350	return __make_iter(__add_alignment_assumption(this->__begin_));
353	351	}
354		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
	352	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
355	353	return __make_iter(__add_alignment_assumption(this->__begin_));
356	354	}
357		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
	355	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
358	356	return __make_iter(__add_alignment_assumption(this->__end_));
359	357	}
360		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
	358	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
361	359	return __make_iter(__add_alignment_assumption(this->__end_));
362	360	}
363	361
364		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
	362	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
365	363	return reverse_iterator(end());
366	364	}
367		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	365	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	366	rbegin() const _NOEXCEPT {
368	367	return const_reverse_iterator(end());
369	368	}
370		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
	369	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
371	370	return reverse_iterator(begin());
372	371	}
373		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	372	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
374	373	return const_reverse_iterator(begin());
375	374	}
376	375
377		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
378		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
379		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
	376	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
	377	return begin();
	378	}
	379	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
	380	return end();
	381	}
	382	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	383	crbegin() const _NOEXCEPT {
380	384	return rbegin();
381	385	}
382		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	386	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
	387	return rend();
	388	}
383	389
384	390	//
385	391	// [vector.capacity], capacity
386	392	//
387		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
	393	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
388	394	return static_cast<size_type>(this->__end_ - this->__begin_);
389	395	}
390		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
	396	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const _NOEXCEPT {
391	397	return static_cast<size_type>(this->__cap_ - this->__begin_);
392	398	}
393	399	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
394	400	return this->__begin_ == this->__end_;
395	401	}
396		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	402	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
397	403	return std::min<size_type>(__alloc_traits::max_size(this->__alloc_), numeric_limits<difference_type>::max());
398	404	}
399	405	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve(size_type __n);
...	...	@@ -402,38 +408,39 @@ public:
402	408	//
403	409	// element access
404	410	//
405		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT {
	411	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __n) _NOEXCEPT {
406	412	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
407	413	return this->__begin_[__n];
408	414	}
409		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const _NOEXCEPT {
	415	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference
	416	operator[](size_type __n) const _NOEXCEPT {
410	417	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector[] index out of bounds");
411	418	return this->__begin_[__n];
412	419	}
413		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n) {
	420	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference at(size_type __n) {
414	421	if (__n >= size())
415	422	this->__throw_out_of_range();
416	423	return this->__begin_[__n];
417	424	}
418		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const {
	425	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __n) const {
419	426	if (__n >= size())
420	427	this->__throw_out_of_range();
421	428	return this->__begin_[__n];
422	429	}
423	430
424		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT {
	431	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT {
425	432	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
426	433	return *this->__begin_;
427	434	}
428		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
	435	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
429	436	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "front() called on an empty vector");
430	437	return *this->__begin_;
431	438	}
432		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT {
	439	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT {
433	440	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
434	441	return *(this->__end_ - 1);
435	442	}
436		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
	443	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
437	444	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "back() called on an empty vector");
438	445	return *(this->__end_ - 1);
439	446	}
...	...	@@ -441,11 +448,11 @@ public:
441	448	//
442	449	// [vector.data], data access
443	450	//
444		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT {
	451	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI value_type* data() _NOEXCEPT {
445	452	return std::__to_address(this->__begin_);
446	453	}
447	454
448		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT {
	455	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const value_type* data() const _NOEXCEPT {
449	456	return std::__to_address(this->__begin_);
450	457	}
451	458
...	...	@@ -478,7 +485,21 @@ public:
478	485	#if _LIBCPP_STD_VER >= 23
479	486	template <_ContainerCompatibleRange<_Tp> _Range>
480	487	_LIBCPP_HIDE_FROM_ABI constexpr void append_range(_Range&& __range) {
481		insert_range(end(), std::forward<_Range>(__range));
	488	if constexpr (ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>) {
	489	auto __len = ranges::distance(__range);
	490	if (__len <= __cap_ - __end_) {
	491	__construct_at_end(ranges::begin(__range), ranges::end(__range), __len);
	492	} else {
	493	__split_buffer<value_type, allocator_type> __buffer(__recommend(size() + __len), size(), __alloc_);
	494	__buffer.__construct_at_end_with_size(ranges::begin(__range), __len);
	495	__swap_out_circular_buffer(__buffer);
	496	}
	497	} else {
	498	vector __buffer(__alloc_);
	499	for (auto&& __val : __range)
	500	__buffer.emplace_back(std::forward<decltype(__val)>(__val));
	501	append_range(ranges::as_rvalue_view(__buffer));
	502	}
482	503	}
483	504	#endif
484	505
...	...	@@ -512,7 +533,7 @@ public:
512	533	int> = 0>
513	534	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
514	535	insert(const_iterator __position, _ForwardIterator __first, _ForwardIterator __last) {
515		return __insert_with_size(__position, __first, __last, std::distance(__first, __last));
	536	return __insert_with_size<_ClassicAlgPolicy>(__position, __first, __last, std::distance(__first, __last));
516	537	}
517	538
518	539	#if _LIBCPP_STD_VER >= 23
...	...	@@ -520,7 +541,7 @@ public:
520	541	_LIBCPP_HIDE_FROM_ABI constexpr iterator insert_range(const_iterator __position, _Range&& __range) {
521	542	if constexpr (ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>) {
522	543	auto __n = static_cast<size_type>(ranges::distance(__range));
523		return __insert_with_size(__position, ranges::begin(__range), ranges::end(__range), __n);
	544	return __insert_with_size<_RangeAlgPolicy>(__position, ranges::begin(__range), ranges::end(__range), __n);
524	545
525	546	} else {
526	547	return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
...	...	@@ -613,12 +634,13 @@ private:
613	634	// The `_Iterator` in `_with_size` functions can be input-only only if called from `_range` (since C++23).
614	635	// Otherwise, `_Iterator` is a forward iterator.
615	636
616		template <class _Iterator, class _Sentinel>
	637	template <class _AlgPolicy, class _Iterator, class _Sentinel>
617	638	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
618	639	__assign_with_size(_Iterator __first, _Sentinel __last, difference_type __n);
619	640
620		template <class _Iterator,
621		__enable_if_t<!is_same<decltype(*std::declval<_Iterator&>())&&, value_type&&>::value, int> = 0>
	641	template <class _AlgPolicy,
	642	class _Iterator,
	643	__enable_if_t<!is_same<__policy_value_type<_AlgPolicy, _Iterator>, value_type>::value, int> = 0>
622	644	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
623	645	__insert_assign_n_unchecked(_Iterator __first, difference_type __n, pointer __position) {
624	646	for (pointer __end_position = __position + __n; __position != __end_position; ++__position, (void)++__first) {
...	...	@@ -627,25 +649,19 @@ private:
627	649	}
628	650	}
629	651
630		template <class _Iterator,
631		__enable_if_t<is_same<decltype(*std::declval<_Iterator&>())&&, value_type&&>::value, int> = 0>
	652	template <class _AlgPolicy,
	653	class _Iterator,
	654	__enable_if_t<is_same<__policy_value_type<_AlgPolicy, _Iterator>, value_type>::value, int> = 0>
632	655	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
633	656	__insert_assign_n_unchecked(_Iterator __first, difference_type __n, pointer __position) {
634		#if _LIBCPP_STD_VER >= 23
635		if constexpr (!forward_iterator<_Iterator>) { // Handles input-only sized ranges for insert_range
636		ranges::copy_n(std::move(__first), __n, __position);
637		} else
638		#endif
639		{
640		std::copy_n(__first, __n, __position);
641		}
	657	std::__copy_n<_AlgPolicy>(std::move(__first), __n, __position);
642	658	}
643	659
644	660	template <class _InputIterator, class _Sentinel>
645	661	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
646	662	__insert_with_sentinel(const_iterator __position, _InputIterator __first, _Sentinel __last);
647	663
648		template <class _Iterator, class _Sentinel>
	664	template <class _AlgPolicy, class _Iterator, class _Sentinel>
649	665	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
650	666	__insert_with_size(const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n);
651	667
...	...	@@ -653,9 +669,6 @@ private:
653	669	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
654	670	__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
655	671
656		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n);
657		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __append(size_type __n, const_reference __x);
658
659	672	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator __make_iter(pointer __p) _NOEXCEPT {
660	673	#ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
661	674	// Bound the iterator according to the capacity, rather than the size.
...	...	@@ -689,9 +702,9 @@ private:
689	702	}
690	703
691	704	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
692		__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v);
	705	__swap_out_circular_buffer(__split_buffer<value_type, allocator_type>& __v);
693	706	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer
694		__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p);
	707	__swap_out_circular_buffer(__split_buffer<value_type, allocator_type>& __v, pointer __p);
695	708	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
696	709	__move_range(pointer __from_s, pointer __from_e, pointer __to);
697	710	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign(vector& __c, true_type)
...	...	@@ -811,26 +824,44 @@ private:
811	824	__add_alignment_assumption(_Ptr __p) _NOEXCEPT {
812	825	return __p;
813	826	}
	827
	828	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_layouts(__split_buffer<_Tp, allocator_type>& __sb) {
	829	auto __vector_begin = __begin_;
	830	auto __vector_sentinel = __end_;
	831	auto __vector_cap = __cap_;
	832
	833	auto __sb_begin = __sb.begin();
	834	auto __sb_sentinel = __sb.__raw_sentinel();
	835	auto __sb_cap = __sb.__raw_capacity();
	836
	837	// TODO: replace with __set_valid_range and __set_capacity when vector supports it.
	838	__begin_ = __sb_begin;
	839	__end_ = __sb_sentinel;
	840	__cap_ = __sb_cap;
	841
	842	__sb.__set_valid_range(__vector_begin, __vector_sentinel);
	843	__sb.__set_capacity(__vector_cap);
	844	}
814	845	};
815	846
816	847	#if _LIBCPP_STD_VER >= 17
817	848	template <class _InputIterator,
818		class _Alloc = allocator<__iter_value_type<_InputIterator>>,
	849	class _Alloc = allocator<__iterator_value_type<_InputIterator>>,
819	850	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
820		class = enable_if_t<__is_allocator<_Alloc>::value> >
821		vector(_InputIterator, _InputIterator) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
	851	class = enable_if_t<__is_allocator_v<_Alloc>>>
	852	vector(_InputIterator, _InputIterator) -> vector<__iterator_value_type<_InputIterator>, _Alloc>;
822	853
823	854	template <class _InputIterator,
824	855	class _Alloc,
825	856	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
826		class = enable_if_t<__is_allocator<_Alloc>::value> >
827		vector(_InputIterator, _InputIterator, _Alloc) -> vector<__iter_value_type<_InputIterator>, _Alloc>;
	857	class = enable_if_t<__is_allocator_v<_Alloc>>>
	858	vector(_InputIterator, _InputIterator, _Alloc) -> vector<__iterator_value_type<_InputIterator>, _Alloc>;
828	859	#endif
829	860
830	861	#if _LIBCPP_STD_VER >= 23
831	862	template <ranges::input_range _Range,
832	863	class _Alloc = allocator<ranges::range_value_t<_Range>>,
833		class = enable_if_t<__is_allocator<_Alloc>::value> >
	864	class = enable_if_t<__is_allocator_v<_Alloc>>>
834	865	vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_t<_Range>, _Alloc>;
835	866	#endif
836	867
...	...	@@ -839,17 +870,16 @@ vector(from_range_t, _Range&&, _Alloc = _Alloc()) -> vector<ranges::range_value_
839	870	// function has a strong exception guarantee.
840	871	template <class _Tp, class _Allocator>
841	872	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
842		vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v) {
	873	vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type>& __v) {
843	874	__annotate_delete();
844		auto __new_begin = __v.__begin_ - (__end_ - __begin_);
	875	auto __new_begin = __v.begin() - size();
845	876	std::__uninitialized_allocator_relocate(
846	877	this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin));
847		__v.__begin_ = __new_begin;
	878	__v.__set_valid_range(__new_begin, __v.end());
848	879	__end_ = __begin_; // All the objects have been destroyed by relocating them.
849		std::swap(this->__begin_, __v.__begin_);
850		std::swap(this->__end_, __v.__end_);
851		std::swap(this->__cap_, __v.__cap_);
852		__v.__first_ = __v.__begin_;
	880
	881	__swap_layouts(__v);
	882	__v.__set_data(__v.begin());
853	883	__annotate_new(size());
854	884	}
855	885
...	...	@@ -859,27 +889,25 @@ vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, a
859	889	// function has a strong exception guarantee if __begin_ == __p \|\| __end_ == __p.
860	890	template <class _Tp, class _Allocator>
861	891	_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
862		vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type&>& __v, pointer __p) {
	892	vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer<value_type, allocator_type>& __v, pointer __p) {
863	893	__annotate_delete();
864		pointer __ret = __v.__begin_;
	894	pointer __ret = __v.begin();
865	895
866	896	// Relocate [__p, __end_) first to avoid having a hole in [__begin_, __end_)
867	897	// in case something in [__begin_, __p) throws.
868	898	std::__uninitialized_allocator_relocate(
869		this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_));
870		__v.__end_ += (__end_ - __p);
	899	this->__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.end()));
	900	auto __relocated_so_far = __end_ - __p;
	901	__v.__set_sentinel(__v.end() + __relocated_so_far);
871	902	__end_ = __p; // The objects in [__p, __end_) have been destroyed by relocating them.
872		auto __new_begin = __v.__begin_ - (__p - __begin_);
	903	auto __new_begin = __v.begin() - (__p - __begin_);
873	904
874	905	std::__uninitialized_allocator_relocate(
875	906	this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin));
876		__v.__begin_ = __new_begin;
877		__end_ = __begin_; // All the objects have been destroyed by relocating them.
878
879		std::swap(this->__begin_, __v.__begin_);
880		std::swap(this->__end_, __v.__end_);
881		std::swap(this->__cap_, __v.__cap_);
882		__v.__first_ = __v.__begin_;
	907	__v.__set_valid_range(__new_begin, __v.end());
	908	__end_ = __begin_; // All the objects have been destroyed by relocating them.
	909	__swap_layouts(__v);
	910	__v.__set_data(__v.begin());
883	911	__annotate_new(size());
884	912	return __ret;
885	913	}
...	...	@@ -945,36 +973,6 @@ vector<_Tp, _Allocator>::__construct_at_end(_InputIterator __first, _Sentinel __
945	973	__tx.__pos_ = std::__uninitialized_allocator_copy(this->__alloc_, std::move(__first), std::move(__last), __tx.__pos_);
946	974	}
947	975
948		// Default constructs __n objects starting at __end_
949		// throws if construction throws
950		// Postcondition: size() == size() + __n
951		// Exception safety: strong.
952		template <class _Tp, class _Allocator>
953		_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n) {
954		if (static_cast<size_type>(this->__cap_ - this->__end_) >= __n)
955		this->__construct_at_end(__n);
956		else {
957		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), this->__alloc_);
958		__v.__construct_at_end(__n);
959		__swap_out_circular_buffer(__v);
960		}
961		}
962
963		// Default constructs __n objects starting at __end_
964		// throws if construction throws
965		// Postcondition: size() == size() + __n
966		// Exception safety: strong.
967		template <class _Tp, class _Allocator>
968		_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__append(size_type __n, const_reference __x) {
969		if (static_cast<size_type>(this->__cap_ - this->__end_) >= __n)
970		this->__construct_at_end(__n, __x);
971		else {
972		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), size(), this->__alloc_);
973		__v.__construct_at_end(__n, __x);
974		__swap_out_circular_buffer(__v);
975		}
976		}
977
978	976	template <class _Tp, class _Allocator>
979	977	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI vector<_Tp, _Allocator>::vector(vector&& __x)
980	978	#if _LIBCPP_STD_VER >= 17
...	...	@@ -1051,20 +1049,14 @@ vector<_Tp, _Allocator>::__assign_with_sentinel(_Iterator __first, _Sentinel __l
1051	1049	}
1052	1050
1053	1051	template <class _Tp, class _Allocator>
1054		template <class _Iterator, class _Sentinel>
	1052	template <class _AlgPolicy, class _Iterator, class _Sentinel>
1055	1053	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
1056	1054	vector<_Tp, _Allocator>::__assign_with_size(_Iterator __first, _Sentinel __last, difference_type __n) {
1057	1055	size_type __new_size = static_cast<size_type>(__n);
1058	1056	if (__new_size <= capacity()) {
1059	1057	if (__new_size > size()) {
1060		#if _LIBCPP_STD_VER >= 23
1061		auto __mid = ranges::copy_n(std::move(__first), size(), this->__begin_).in;
	1058	auto __mid = std::__copy_n<_AlgPolicy>(std::move(__first), size(), this->__begin_).first;
1062	1059	__construct_at_end(std::move(__mid), std::move(__last), __new_size - size());
1063		#else
1064		_Iterator __mid = std::next(__first, size());
1065		std::copy(__first, __mid, this->__begin_);
1066		__construct_at_end(__mid, __last, __new_size - size());
1067		#endif
1068	1060	} else {
1069	1061	pointer __m = std::__copy(std::move(__first), __last, this->__begin_).second;
1070	1062	this->__destruct_at_end(__m);
...	...	@@ -1097,7 +1089,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::reserve(size_type __
1097	1089	if (__n > capacity()) {
1098	1090	if (__n > max_size())
1099	1091	this->__throw_length_error();
1100		__split_buffer<value_type, allocator_type&> __v(__n, size(), this->__alloc_);
	1092	__split_buffer<value_type, allocator_type> __v(__n, size(), this->__alloc_);
1101	1093	__swap_out_circular_buffer(__v);
1102	1094	}
1103	1095	}
...	...	@@ -1108,7 +1100,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::shrink_to_fit() _NOE
1108	1100	#if _LIBCPP_HAS_EXCEPTIONS
1109	1101	try {
1110	1102	#endif // _LIBCPP_HAS_EXCEPTIONS
1111		__split_buffer<value_type, allocator_type&> __v(size(), size(), this->__alloc_);
	1103	__split_buffer<value_type, allocator_type> __v(size(), size(), this->__alloc_);
1112	1104	// The Standard mandates shrink_to_fit() does not increase the capacity.
1113	1105	// With equal capacity keep the existing buffer. This avoids extra work
1114	1106	// due to swapping the elements.
...	...	@@ -1125,14 +1117,33 @@ template <class _Tp, class _Allocator>
1125	1117	template <class... _Args>
1126	1118	_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::pointer
1127	1119	vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) {
1128		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), size(), this->__alloc_);
	1120	__split_buffer<value_type, allocator_type> __v(__recommend(size() + 1), size(), this->__alloc_);
1129	1121	// __v.emplace_back(std::forward<_Args>(__args)...);
1130		__alloc_traits::construct(this->__alloc_, std::__to_address(__v.__end_), std::forward<_Args>(__args)...);
1131		__v.__end_++;
	1122	pointer __end = __v.end();
	1123	__alloc_traits::construct(this->__alloc_, std::__to_address(__end), std::forward<_Args>(__args)...);
	1124	__v.__set_sentinel(++__end);
1132	1125	__swap_out_circular_buffer(__v);
1133	1126	return this->__end_;
1134	1127	}
1135	1128
	1129	// This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without
	1130	// the `__builtin_constant_p`, since it considers `__else` unlikely even through it's known to be run.
	1131	// See https://llvm.org/PR154292
	1132	template <class _If, class _Else>
	1133	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool __cond, _If __if, _Else __else) {
	1134	if (__builtin_constant_p(__cond)) {
	1135	if (__cond)
	1136	__if();
	1137	else
	1138	__else();
	1139	} else {
	1140	if (__cond) [[__likely__]]
	1141	__if();
	1142	else
	1143	__else();
	1144	}
	1145	}
	1146
1136	1147	template <class _Tp, class _Allocator>
1137	1148	template <class... _Args>
1138	1149	_LIBCPP_CONSTEXPR_SINCE_CXX20 inline
...	...	@@ -1143,12 +1154,14 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 inline
1143	1154	#endif
1144	1155	vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
1145	1156	pointer __end = this->__end_;
1146		if (__end < this->__cap_) {
1147		__emplace_back_assume_capacity(std::forward<_Args>(__args)...);
1148		++__end;
1149		} else {
1150		__end = __emplace_back_slow_path(std::forward<_Args>(__args)...);
1151		}
	1157	std::__if_likely_else(
	1158	__end < this->__cap_,
	1159	[&] {
	1160	__emplace_back_assume_capacity(std::forward<_Args>(__args)...);
	1161	++__end;
	1162	},
	1163	[&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); });
	1164
1152	1165	this->__end_ = __end;
1153	1166	#if _LIBCPP_STD_VER >= 17
1154	1167	return *(__end - 1);
...	...	@@ -1207,7 +1220,7 @@ vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x)
1207	1220	__p = __xr;
1208	1221	}
1209	1222	} else {
1210		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
	1223	__split_buffer<value_type, allocator_type> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
1211	1224	__v.emplace_back(__x);
1212	1225	__p = __swap_out_circular_buffer(__v, __p);
1213	1226	}
...	...	@@ -1226,7 +1239,7 @@ vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
1226	1239	*__p = std::move(__x);
1227	1240	}
1228	1241	} else {
1229		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
	1242	__split_buffer<value_type, allocator_type> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
1230	1243	__v.emplace_back(std::move(__x));
1231	1244	__p = __swap_out_circular_buffer(__v, __p);
1232	1245	}
...	...	@@ -1247,7 +1260,7 @@ vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
1247	1260	*__p = std::move(__tmp.get());
1248	1261	}
1249	1262	} else {
1250		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
	1263	__split_buffer<value_type, allocator_type> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
1251	1264	__v.emplace_back(std::forward<_Args>(__args)...);
1252	1265	__p = __swap_out_circular_buffer(__v, __p);
1253	1266	}
...	...	@@ -1275,7 +1288,7 @@ vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_
1275	1288	std::fill_n(__p, __n, *__xr);
1276	1289	}
1277	1290	} else {
1278		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_);
	1291	__split_buffer<value_type, allocator_type> __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_);
1279	1292	__v.__construct_at_end(__n, __x);
1280	1293	__p = __swap_out_circular_buffer(__v, __p);
1281	1294	}
...	...	@@ -1296,28 +1309,28 @@ vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inpu
1296	1309	if (__first == __last)
1297	1310	(void)std::rotate(__p, __old_last, this->__end_);
1298	1311	else {
1299		__split_buffer<value_type, allocator_type&> __v(__alloc_);
	1312	__split_buffer<value_type, allocator_type> __v(__alloc_);
1300	1313	auto __guard = std::__make_exception_guard(
1301	1314	_AllocatorDestroyRangeReverse<allocator_type, pointer>(__alloc_, __old_last, this->__end_));
1302	1315	__v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
1303		__split_buffer<value_type, allocator_type&> __merged(
	1316	__split_buffer<value_type, allocator_type> __merged(
1304	1317	__recommend(size() + __v.size()), __off, __alloc_); // has `__off` positions available at the front
1305	1318	std::__uninitialized_allocator_relocate(
1306		__alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.__end_));
	1319	__alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.end()));
1307	1320	__guard.__complete(); // Release the guard once objects in [__old_last_, __end_) have been successfully relocated.
1308		__merged.__end_ += this->__end_ - __old_last;
	1321	__merged.__set_sentinel(__merged.end() + (this->__end_ - __old_last));
1309	1322	this->__end_ = __old_last;
1310	1323	std::__uninitialized_allocator_relocate(
1311		__alloc_, std::__to_address(__v.__begin_), std::__to_address(__v.__end_), std::__to_address(__merged.__end_));
1312		__merged.__end_ += __v.size();
1313		__v.__end_ = __v.__begin_;
	1324	__alloc_, std::__to_address(__v.begin()), std::__to_address(__v.end()), std::__to_address(__merged.end()));
	1325	__merged.__set_sentinel(__merged.size() + __v.size());
	1326	__v.__set_sentinel(__v.begin());
1314	1327	__p = __swap_out_circular_buffer(__merged, __p);
1315	1328	}
1316	1329	return __make_iter(__p);
1317	1330	}
1318	1331
1319	1332	template <class _Tp, class _Allocator>
1320		template <class _Iterator, class _Sentinel>
	1333	template <class _AlgPolicy, class _Iterator, class _Sentinel>
1321	1334	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
1322	1335	vector<_Tp, _Allocator>::__insert_with_size(
1323	1336	const_iterator __position, _Iterator __first, _Sentinel __last, difference_type __n) {
...	...	@@ -1338,15 +1351,15 @@ vector<_Tp, _Allocator>::__insert_with_size(
1338	1351	__construct_at_end(__m, __last, __n - __dx);
1339	1352	if (__dx > 0) {
1340	1353	__move_range(__p, __old_last, __p + __n);
1341		__insert_assign_n_unchecked(__first, __dx, __p);
	1354	__insert_assign_n_unchecked<_AlgPolicy>(__first, __dx, __p);
1342	1355	}
1343	1356	}
1344	1357	} else {
1345	1358	__move_range(__p, __old_last, __p + __n);
1346		__insert_assign_n_unchecked(std::move(__first), __n, __p);
	1359	__insert_assign_n_unchecked<_AlgPolicy>(std::move(__first), __n, __p);
1347	1360	}
1348	1361	} else {
1349		__split_buffer<value_type, allocator_type&> __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_);
	1362	__split_buffer<value_type, allocator_type> __v(__recommend(size() + __n), __p - this->__begin_, this->__alloc_);
1350	1363	__v.__construct_at_end_with_size(std::move(__first), __n);
1351	1364	__p = __swap_out_circular_buffer(__v, __p);
1352	1365	}
...	...	@@ -1355,21 +1368,35 @@ vector<_Tp, _Allocator>::__insert_with_size(
1355	1368	}
1356	1369
1357	1370	template <class _Tp, class _Allocator>
1358		_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz) {
1359		size_type __cs = size();
1360		if (__cs < __sz)
1361		this->__append(__sz - __cs);
1362		else if (__cs > __sz)
1363		this->__destruct_at_end(this->__begin_ + __sz);
	1371	_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __new_size) {
	1372	size_type __current_size = size();
	1373	if (__current_size < __new_size) {
	1374	if (__new_size <= capacity()) {
	1375	__construct_at_end(__new_size - __current_size);
	1376	} else {
	1377	__split_buffer<value_type, allocator_type> __v(__recommend(__new_size), __current_size, __alloc_);
	1378	__v.__construct_at_end(__new_size - __current_size);
	1379	__swap_out_circular_buffer(__v);
	1380	}
	1381	} else if (__current_size > __new_size) {
	1382	this->__destruct_at_end(this->__begin_ + __new_size);
	1383	}
1364	1384	}
1365	1385
1366	1386	template <class _Tp, class _Allocator>
1367		_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __sz, const_reference __x) {
1368		size_type __cs = size();
1369		if (__cs < __sz)
1370		this->__append(__sz - __cs, __x);
1371		else if (__cs > __sz)
1372		this->__destruct_at_end(this->__begin_ + __sz);
	1387	_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::resize(size_type __new_size, const_reference __x) {
	1388	size_type __current_size = size();
	1389	if (__current_size < __new_size) {
	1390	if (__new_size <= capacity())
	1391	__construct_at_end(__new_size - __current_size, __x);
	1392	else {
	1393	__split_buffer<value_type, allocator_type> __v(__recommend(__new_size), __current_size, __alloc_);
	1394	__v.__construct_at_end(__new_size - __current_size, __x);
	1395	__swap_out_circular_buffer(__v);
	1396	}
	1397	} else if (__current_size > __new_size) {
	1398	this->__destruct_at_end(this->__begin_ + __new_size);
	1399	}
1373	1400	}
1374	1401
1375	1402	template <class _Tp, class _Allocator>

lib/libcxx/include/__vector/vector_bool.h+76-83

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__algorithm/copy.h>
13	13	#include <__algorithm/copy_backward.h>
	14	#include <__algorithm/copy_n.h>
14	15	#include <__algorithm/fill_n.h>
15	16	#include <__algorithm/iterator_operations.h>
16	17	#include <__algorithm/max.h>
...	...	@@ -19,7 +20,7 @@
19	20	#include <__bit_reference>
20	21	#include <__config>
21	22	#include <__functional/unary_function.h>
22		#include <__fwd/bit_reference.h> // TODO: This is a workaround for https://github.com/llvm/llvm-project/issues/131814
	23	#include <__fwd/bit_reference.h> // TODO: This is a workaround for https://llvm.org/PR131814
23	24	#include <__fwd/functional.h>
24	25	#include <__fwd/vector.h>
25	26	#include <__iterator/distance.h>
...	...	@@ -234,74 +235,89 @@ public:
234	235	}
235	236	#endif
236	237
237		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
	238	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
238	239	return allocator_type(this->__alloc_);
239	240	}
240	241
241		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT;
242		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
	242	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT;
	243	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
243	244	return __internal_cap_to_external(__cap_);
244	245	}
245		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT { return __size_; }
	246	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT {
	247	return __size_;
	248	}
246	249	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool empty() const _NOEXCEPT {
247	250	return __size_ == 0;
248	251	}
249	252	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reserve(size_type __n);
250	253	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void shrink_to_fit() _NOEXCEPT;
251	254
252		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT { return __make_iter(0); }
253		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT { return __make_iter(0); }
254		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT { return __make_iter(__size_); }
255		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
	255	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT {
	256	return __make_iter(0);
	257	}
	258	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT {
	259	return __make_iter(0);
	260	}
	261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT {
	262	return __make_iter(__size_);
	263	}
	264	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
256	265	return __make_iter(__size_);
257	266	}
258	267
259		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
	268	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
260	269	return reverse_iterator(end());
261	270	}
262		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
	271	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator
	272	rbegin() const _NOEXCEPT {
263	273	return const_reverse_iterator(end());
264	274	}
265		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
	275	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
266	276	return reverse_iterator(begin());
267	277	}
268		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
	278	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
269	279	return const_reverse_iterator(begin());
270	280	}
271	281
272		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return __make_iter(0); }
273		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
	282	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT {
	283	return __make_iter(0);
	284	}
	285	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
274	286	return __make_iter(__size_);
275	287	}
276		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
	288	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator
	289	crbegin() const _NOEXCEPT {
277	290	return rbegin();
278	291	}
279		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	292	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT {
	293	return rend();
	294	}
280	295
281		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __n) {
	296	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __n) {
282	297	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector<bool>::operator[] index out of bounds");
283	298	return __make_ref(__n);
284	299	}
285		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __n) const {
	300	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference
	301	operator[](size_type __n) const {
286	302	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < size(), "vector<bool>::operator[] index out of bounds");
287	303	return __make_ref(__n);
288	304	}
289		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference at(size_type __n);
290		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference at(size_type __n) const;
	305	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference at(size_type __n);
	306	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference at(size_type __n) const;
291	307
292		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() {
	308	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() {
293	309	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector<bool>::front() called on an empty vector");
294	310	return __make_ref(0);
295	311	}
296		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const {
	312	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const {
297	313	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector<bool>::front() called on an empty vector");
298	314	return __make_ref(0);
299	315	}
300		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() {
	316	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() {
301	317	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector<bool>::back() called on an empty vector");
302	318	return __make_ref(__size_ - 1);
303	319	}
304		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const {
	320	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const {
305	321	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "vector<bool>::back() called on an empty vector");
306	322	return __make_ref(__size_ - 1);
307	323	}
...	...	@@ -463,7 +479,6 @@ private:
463	479	return (__new_size + (__bits_per_word - 1)) & ~((size_type)__bits_per_word - 1);
464	480	}
465	481	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __new_size) const;
466		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct_at_end(size_type __n, bool __x);
467	482	template <class _InputIterator, class _Sentinel>
468	483	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
469	484	__construct_at_end(_InputIterator __first, _Sentinel __last, size_type __n);
...	...	@@ -552,20 +567,6 @@ vector<bool, _Allocator>::__recommend(size_type __new_size) const {
552	567	return std::max<size_type>(2 * __cap, __align_it(__new_size));
553	568	}
554	569
555		// Default constructs __n objects starting at __end_
556		// Precondition: size() + __n <= capacity()
557		// Postcondition: size() == size() + __n
558		template <class _Allocator>
559		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
560		vector<bool, _Allocator>::__construct_at_end(size_type __n, bool __x) {
561		_LIBCPP_ASSERT_INTERNAL(
562		capacity() >= size() + __n, "vector<bool>::__construct_at_end called with insufficient capacity");
563		std::fill_n(end(), __n, __x);
564		this->__size_ += __n;
565		if (end().__ctz_ != 0) // Ensure uninitialized leading bits in the last word are set to zero
566		std::fill_n(end(), __bits_per_word - end().__ctz_, 0);
567		}
568
569	570	template <class _Allocator>
570	571	template <class _InputIterator, class _Sentinel>
571	572	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
...	...	@@ -598,7 +599,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n)
598	599	: __begin_(nullptr), __size_(0), __cap_(0) {
599	600	if (__n > 0) {
600	601	__vallocate(__n);
601		__construct_at_end(__n, false);
	602	std::fill_n(__begin_, __external_cap_to_internal(__n), __storage_type(0));
	603	__size_ = __n;
602	604	}
603	605	}
604	606
...	...	@@ -608,7 +610,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, co
608	610	: __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
609	611	if (__n > 0) {
610	612	__vallocate(__n);
611		__construct_at_end(__n, false);
	613	std::fill_n(__begin_, __external_cap_to_internal(__n), __storage_type(0));
	614	__size_ = __n;
612	615	}
613	616	}
614	617	#endif
...	...	@@ -618,7 +621,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(size_type __n, co
618	621	: __begin_(nullptr), __size_(0), __cap_(0) {
619	622	if (__n > 0) {
620	623	__vallocate(__n);
621		__construct_at_end(__n, __x);
	624	std::fill_n(__begin_, __external_cap_to_internal(__n), __storage_type(0) - __x);
	625	__size_ = __n;
622	626	}
623	627	}
624	628
...	...	@@ -628,7 +632,8 @@ vector<bool, _Allocator>::vector(size_type __n, const value_type& __x, const all
628	632	: __begin_(nullptr), __size_(0), __cap_(0), __alloc_(static_cast<__storage_allocator>(__a)) {
629	633	if (__n > 0) {
630	634	__vallocate(__n);
631		__construct_at_end(__n, __x);
	635	std::fill_n(__begin_, __external_cap_to_internal(__n), __storage_type(0) - __x);
	636	__size_ = __n;
632	637	}
633	638	}
634	639
...	...	@@ -697,7 +702,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
697	702	__alloc_(__storage_traits::select_on_container_copy_construction(__v.__alloc_)) {
698	703	if (__v.size() > 0) {
699	704	__vallocate(__v.size());
700		__construct_at_end(__v.begin(), __v.end(), __v.size());
	705	std::copy_n(__v.__begin_, __external_cap_to_internal(__v.size()), __begin_);
	706	__size_ = __v.size();
701	707	}
702	708	}
703	709
...	...	@@ -706,7 +712,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
706	712	: __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
707	713	if (__v.size() > 0) {
708	714	__vallocate(__v.size());
709		__construct_at_end(__v.begin(), __v.end(), __v.size());
	715	std::copy_n(__v.__begin_, __external_cap_to_internal(__v.size()), __begin_);
	716	__size_ = __v.size();
710	717	}
711	718	}
712	719
...	...	@@ -719,7 +726,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>
719	726	__vdeallocate();
720	727	__vallocate(__v.__size_);
721	728	}
722		std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
	729	std::copy_n(__v.__begin_, __external_cap_to_internal(__v.size()), __begin_);
723	730	}
724	731	__size_ = __v.__size_;
725	732	}
...	...	@@ -754,7 +761,8 @@ vector<bool, _Allocator>::vector(vector&& __v, const __type_identity_t<allocator
754	761	__v.__cap_ = __v.__size_ = 0;
755	762	} else if (__v.size() > 0) {
756	763	__vallocate(__v.size());
757		__construct_at_end(__v.begin(), __v.end(), __v.size());
	764	__size_ = __v.__size_;
	765	std::copy_n(__v.__begin_, __external_cap_to_internal(__v.size()), __begin_);
758	766	}
759	767	}
760	768
...	...	@@ -849,7 +857,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::reserve(size_type _
849	857	this->__throw_length_error();
850	858	vector __v(this->get_allocator());
851	859	__v.__vallocate(__n);
852		__v.__construct_at_end(this->begin(), this->end(), this->size());
	860	__v.__size_ = __size_;
	861	std::copy_n(__begin_, __external_cap_to_internal(__size_), __v.__begin_);
853	862	swap(__v);
854	863	}
855	864	}
...	...	@@ -956,21 +965,14 @@ vector<bool, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inp
956	965	}
957	966	vector __v(get_allocator());
958	967	if (__first != __last) {
959		#if _LIBCPP_HAS_EXCEPTIONS
960		try {
961		#endif // _LIBCPP_HAS_EXCEPTIONS
962		__v.__assign_with_sentinel(std::move(__first), std::move(__last));
963		difference_type __old_size = static_cast<difference_type>(__old_end - begin());
964		difference_type __old_p = __p - begin();
965		reserve(__recommend(size() + __v.size()));
966		__p = begin() + __old_p;
967		__old_end = begin() + __old_size;
968		#if _LIBCPP_HAS_EXCEPTIONS
969		} catch (...) {
970		erase(__old_end, end());
971		throw;
972		}
973		#endif // _LIBCPP_HAS_EXCEPTIONS
	968	auto __guard = std::__make_exception_guard([&] { erase(__old_end, end()); });
	969	__v.__assign_with_sentinel(std::move(__first), std::move(__last));
	970	difference_type __old_size = static_cast<difference_type>(__old_end - begin());
	971	difference_type __old_p = __p - begin();
	972	reserve(__recommend(size() + __v.size()));
	973	__p = begin() + __old_p;
	974	__old_end = begin() + __old_size;
	975	__guard.__complete();
974	976	}
975	977	__p = std::rotate(__p, __old_end, end());
976	978	insert(__p, __v.begin(), __v.end());
...	...	@@ -1048,25 +1050,16 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::swap(vector& __x)
1048	1050	}
1049	1051
1050	1052	template <class _Allocator>
1051		_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::resize(size_type __sz, value_type __x) {
1052		size_type __cs = size();
1053		if (__cs < __sz) {
1054		iterator __r;
1055		size_type __c = capacity();
1056		size_type __n = __sz - __cs;
1057		if (__n <= __c && __cs <= __c - __n) {
1058		__r = end();
1059		__size_ += __n;
1060		} else {
1061		vector __v(get_allocator());
1062		__v.reserve(__recommend(__size_ + __n));
1063		__v.__size_ = __size_ + __n;
1064		__r = std::copy(cbegin(), cend(), __v.begin());
1065		swap(__v);
1066		}
1067		std::fill_n(__r, __n, __x);
1068		} else
1069		__size_ = __sz;
	1053	_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::resize(size_type __new_size, value_type __x) {
	1054	size_type __current_size = size();
	1055	if (__new_size < __current_size) {
	1056	__size_ = __new_size;
	1057	return;
	1058	}
	1059
	1060	reserve(__new_size);
	1061	std::fill_n(end(), __new_size - __current_size, __x);
	1062	__size_ = __new_size;
1070	1063	}
1071	1064
1072	1065	template <class _Allocator>

lib/libcxx/include/any+118-155

...	...	@@ -84,14 +84,12 @@ namespace std {
84	84	# include <__cxx03/__config>
85	85	#else
86	86	# include <__config>
87		# include <__memory/allocator.h>
88		# include <__memory/allocator_destructor.h>
89		# include <__memory/allocator_traits.h>
90		# include <__memory/unique_ptr.h>
	87	# include <__memory/construct_at.h>
	88	# include <__new/allocate.h>
91	89	# include <__type_traits/add_cv_quals.h>
92	90	# include <__type_traits/add_pointer.h>
93		# include <__type_traits/aligned_storage.h>
94	91	# include <__type_traits/conditional.h>
	92	# include <__type_traits/conjunction.h>
95	93	# include <__type_traits/decay.h>
96	94	# include <__type_traits/enable_if.h>
97	95	# include <__type_traits/is_constructible.h>
...	...	@@ -100,9 +98,11 @@ namespace std {
100	98	# include <__type_traits/is_reference.h>
101	99	# include <__type_traits/is_same.h>
102	100	# include <__type_traits/is_void.h>
	101	# include <__type_traits/negation.h>
103	102	# include <__type_traits/remove_cv.h>
104	103	# include <__type_traits/remove_cvref.h>
105	104	# include <__type_traits/remove_reference.h>
	105	# include <__utility/exception_guard.h>
106	106	# include <__utility/forward.h>
107	107	# include <__utility/in_place.h>
108	108	# include <__utility/move.h>
...	...	@@ -142,21 +142,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD
142	142	class any;
143	143
144	144	template <class _ValueType>
145		_LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) _NOEXCEPT;
	145	_LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) noexcept;
146	146
147	147	template <class _ValueType>
148		_LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any*) _NOEXCEPT;
	148	_LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any*) noexcept;
149	149
150	150	namespace __any_imp {
151		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
152		using _Buffer _LIBCPP_NODEBUG = aligned_storage_t<3 * sizeof(void), alignof(void)>;
153		_LIBCPP_SUPPRESS_DEPRECATED_POP
	151	inline constexpr size_t __small_buffer_size = 3 * sizeof(void*);
	152	inline constexpr size_t __small_buffer_alignment = alignof(void*);
154	153
155	154	template <class _Tp>
156	155	using _IsSmallObject _LIBCPP_NODEBUG =
157	156	integral_constant<bool,
158		sizeof(_Tp) <= sizeof(_Buffer) && alignof(_Buffer) % alignof(_Tp) == 0 &&
159		is_nothrow_move_constructible<_Tp>::value >;
	157	sizeof(_Tp) <= __small_buffer_size && alignof(_Tp) <= __small_buffer_alignment &&
	158	is_nothrow_move_constructible_v<_Tp>>;
160	159
161	160	enum class _Action { _Destroy, _Copy, _Move, _Get, _TypeInfo };
162	161
...	...	@@ -192,37 +191,44 @@ using _Handler _LIBCPP_NODEBUG = conditional_t< _IsSmallObject<_Tp>::value, _Sma
192	191	class any {
193	192	public:
194	193	// construct/destruct
195		_LIBCPP_HIDE_FROM_ABI constexpr any() _NOEXCEPT : __h_(nullptr) {}
	194	_LIBCPP_HIDE_FROM_ABI constexpr any() noexcept : __h_(nullptr) {}
196	195
197	196	_LIBCPP_HIDE_FROM_ABI any(any const& __other) : __h_(nullptr) {
198	197	if (__other.__h_)
199	198	__other.__call(_Action::_Copy, this);
200	199	}
201	200
202		_LIBCPP_HIDE_FROM_ABI any(any&& __other) _NOEXCEPT : __h_(nullptr) {
	201	_LIBCPP_HIDE_FROM_ABI any(any&& __other) noexcept : __h_(nullptr) {
203	202	if (__other.__h_)
204	203	__other.__call(_Action::_Move, this);
205	204	}
206	205
207		template < class _ValueType,
208		class _Tp = decay_t<_ValueType>,
209		class = enable_if_t< !is_same<_Tp, any>::value && !__is_inplace_type<_ValueType>::value &&
210		is_copy_constructible<_Tp>::value> >
211		_LIBCPP_HIDE_FROM_ABI any(_ValueType&& __value);
	206	template <
	207	class _ValueType,
	208	class _Tp = decay_t<_ValueType>,
	209	enable_if_t<_And<_Not<is_same<_Tp, any>>, _Not<__is_inplace_type<_ValueType>>, is_copy_constructible<_Tp>>::value,
	210	int> = 0>
	211	_LIBCPP_HIDE_FROM_ABI any(_ValueType&& __value) : __h_(nullptr) {
	212	__any_imp::_Handler<_Tp>::__create(*this, std::forward<_ValueType>(__value));
	213	}
212	214
213	215	template <class _ValueType,
214	216	class... _Args,
215		class _Tp = decay_t<_ValueType>,
216		class = enable_if_t< is_constructible<_Tp, _Args...>::value && is_copy_constructible<_Tp>::value > >
217		_LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, _Args&&... __args);
	217	class _Tp = decay_t<_ValueType>,
	218	enable_if_t<is_constructible_v<_Tp, _Args...> && is_copy_constructible_v<_Tp>, int> = 0>
	219	_LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, _Args&&... __args) {
	220	__any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
	221	}
218	222
219		template <class _ValueType,
220		class _Up,
221		class... _Args,
222		class _Tp = decay_t<_ValueType>,
223		class = enable_if_t< is_constructible<_Tp, initializer_list<_Up>&, _Args...>::value &&
224		is_copy_constructible<_Tp>::value> >
225		_LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, initializer_list<_Up>, _Args&&... __args);
	223	template <
	224	class _ValueType,
	225	class _Up,
	226	class... _Args,
	227	class _Tp = decay_t<_ValueType>,
	228	enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...> && is_copy_constructible_v<_Tp>, int> = 0>
	229	_LIBCPP_HIDE_FROM_ABI explicit any(in_place_type_t<_ValueType>, initializer_list<_Up> __il, _Args&&... __args) {
	230	__any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
	231	}
226	232
227	233	_LIBCPP_HIDE_FROM_ABI ~any() { this->reset(); }
228	234
...	...	@@ -232,43 +238,65 @@ public:
232	238	return *this;
233	239	}
234	240
235		_LIBCPP_HIDE_FROM_ABI any& operator=(any&& __rhs) _NOEXCEPT {
	241	_LIBCPP_HIDE_FROM_ABI any& operator=(any&& __rhs) noexcept {
236	242	any(std::move(__rhs)).swap(*this);
237	243	return *this;
238	244	}
239	245
240		template < class _ValueType,
241		class _Tp = decay_t<_ValueType>,
242		class = enable_if_t< !is_same<_Tp, any>::value && is_copy_constructible<_Tp>::value> >
243		_LIBCPP_HIDE_FROM_ABI any& operator=(_ValueType&& __rhs);
244
245	246	template <class _ValueType,
246		class... _Args,
247		class _Tp = decay_t<_ValueType>,
248		class = enable_if_t< is_constructible<_Tp, _Args...>::value && is_copy_constructible<_Tp>::value> >
249		_LIBCPP_HIDE_FROM_ABI _Tp& emplace(_Args&&...);
	247	class _Tp = decay_t<_ValueType>,
	248	enable_if_t<!is_same_v<_Tp, any> && is_copy_constructible_v<_Tp>, int> = 0>
	249	_LIBCPP_HIDE_FROM_ABI any& operator=(_ValueType&& __rhs) {
	250	any(std::forward<_ValueType>(__rhs)).swap(*this);
	251	return *this;
	252	}
250	253
251	254	template <class _ValueType,
252		class _Up,
253	255	class... _Args,
254		class _Tp = decay_t<_ValueType>,
255		class = enable_if_t< is_constructible<_Tp, initializer_list<_Up>&, _Args...>::value &&
256		is_copy_constructible<_Tp>::value> >
257		_LIBCPP_HIDE_FROM_ABI _Tp& emplace(initializer_list<_Up>, _Args&&...);
	256	class _Tp = decay_t<_ValueType>,
	257	enable_if_t<is_constructible_v<_Tp, _Args...> && is_copy_constructible_v<_Tp>, int> = 0>
	258	_LIBCPP_HIDE_FROM_ABI _Tp& emplace(_Args&&... __args) {
	259	reset();
	260	return __any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
	261	}
	262
	263	template <
	264	class _ValueType,
	265	class _Up,
	266	class... _Args,
	267	class _Tp = decay_t<_ValueType>,
	268	enable_if_t<is_constructible_v<_Tp, initializer_list<_Up>&, _Args...> && is_copy_constructible_v<_Tp>, int> = 0>
	269	_LIBCPP_HIDE_FROM_ABI _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
	270	reset();
	271	return __any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
	272	}
258	273
259	274	// 6.3.3 any modifiers
260		_LIBCPP_HIDE_FROM_ABI void reset() _NOEXCEPT {
	275	_LIBCPP_HIDE_FROM_ABI void reset() noexcept {
261	276	if (__h_)
262	277	this->__call(_Action::_Destroy);
263	278	}
264	279
265		_LIBCPP_HIDE_FROM_ABI void swap(any& __rhs) _NOEXCEPT;
	280	_LIBCPP_HIDE_FROM_ABI void swap(any& __rhs) noexcept {
	281	if (this == &__rhs)
	282	return;
	283	if (__h_ && __rhs.__h_) {
	284	any __tmp;
	285	__rhs.__call(_Action::_Move, &__tmp);
	286	this->__call(_Action::_Move, &__rhs);
	287	__tmp.__call(_Action::_Move, this);
	288	} else if (__h_) {
	289	this->__call(_Action::_Move, &__rhs);
	290	} else if (__rhs.__h_) {
	291	__rhs.__call(_Action::_Move, this);
	292	}
	293	}
266	294
267	295	// 6.3.4 any observers
268		_LIBCPP_HIDE_FROM_ABI bool has_value() const _NOEXCEPT { return __h_ != nullptr; }
	296	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool has_value() const noexcept { return __h_ != nullptr; }
269	297
270	298	# if _LIBCPP_HAS_RTTI
271		_LIBCPP_HIDE_FROM_ABI const type_info& type() const _NOEXCEPT {
	299	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const type_info& type() const noexcept {
272	300	if (__h_) {
273	301	return static_cast<type_info const>(this->__call(_Action::_TypeInfo));
274	302	} else {
...	...	@@ -285,7 +313,7 @@ private:
285	313	union _Storage {
286	314	_LIBCPP_HIDE_FROM_ABI constexpr _Storage() : __ptr(nullptr) {}
287	315	void* __ptr;
288		__any_imp::_Buffer __buf;
	316	alignas(__any_imp::__small_buffer_alignment) char __buf[__any_imp::__small_buffer_size];
289	317	};
290	318
291	319	_LIBCPP_HIDE_FROM_ABI void*
...	...	@@ -305,10 +333,10 @@ private:
305	333	friend struct __any_imp::_LargeHandler;
306	334
307	335	template <class _ValueType>
308		friend add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) _NOEXCEPT;
	336	friend add_pointer_t<add_const_t<_ValueType>> any_cast(any const*) noexcept;
309	337
310	338	template <class _ValueType>
311		friend add_pointer_t<_ValueType> any_cast(any*) _NOEXCEPT;
	339	friend add_pointer_t<_ValueType> any_cast(any*) noexcept;
312	340
313	341	_HandleFuncPtr __h_ = nullptr;
314	342	_Storage __s_;
...	...	@@ -339,22 +367,14 @@ struct _SmallHandler {
339	367
340	368	template <class... _Args>
341	369	_LIBCPP_HIDE_FROM_ABI static _Tp& __create(any& __dest, _Args&&... __args) {
342		typedef allocator<_Tp> _Alloc;
343		typedef allocator_traits<_Alloc> _ATraits;
344		_Alloc __a;
345		_Tp* __ret = static_cast<_Tp>(static_cast<void>(&__dest.__s_.__buf));
346		_ATraits::construct(__a, __ret, std::forward<_Args>(__args)...);
	370	auto __ret = std::__construct_at(reinterpret_cast<_Tp*>(&__dest.__s_.__buf), std::forward<_Args>(__args)...);
347	371	__dest.__h_ = &_SmallHandler::__handle;
348	372	return *__ret;
349	373	}
350	374
351	375	private:
352	376	_LIBCPP_HIDE_FROM_ABI static void __destroy(any& __this) {
353		typedef allocator<_Tp> _Alloc;
354		typedef allocator_traits<_Alloc> _ATraits;
355		_Alloc __a;
356		_Tp* __p = static_cast<_Tp>(static_cast<void>(&__this.__s_.__buf));
357		_ATraits::destroy(__a, __p);
	377	std::__destroy_at(reinterpret_cast<_Tp*>(&__this.__s_.__buf));
358	378	__this.__h_ = nullptr;
359	379	}
360	380
...	...	@@ -406,26 +426,20 @@ struct _LargeHandler {
406	426
407	427	template <class... _Args>
408	428	_LIBCPP_HIDE_FROM_ABI static _Tp& __create(any& __dest, _Args&&... __args) {
409		typedef allocator<_Tp> _Alloc;
410		typedef allocator_traits<_Alloc> _ATraits;
411		typedef __allocator_destructor<_Alloc> _Dp;
412		_Alloc __a;
413		unique_ptr<_Tp, _Dp> __hold(_ATraits::allocate(__a, 1), _Dp(__a, 1));
414		_Tp* __ret = __hold.get();
415		_ATraits::construct(__a, __ret, std::forward<_Args>(__args)...);
416		__dest.__s_.__ptr = __hold.release();
	429	_Tp* __ptr = static_cast<_Tp*>(std::__libcpp_allocate<_Tp>(__element_count(1)));
	430	std::__exception_guard __guard([&] { std::__libcpp_deallocate<_Tp>(__ptr, __element_count(1)); });
	431	std::__construct_at(__ptr, std::forward<_Args>(__args)...);
	432	__guard.__complete();
	433	__dest.__s_.__ptr = __ptr;
417	434	__dest.__h_ = &_LargeHandler::__handle;
418		return *__ret;
	435	return *__ptr;
419	436	}
420	437
421	438	private:
422	439	_LIBCPP_HIDE_FROM_ABI static void __destroy(any& __this) {
423		typedef allocator<_Tp> _Alloc;
424		typedef allocator_traits<_Alloc> _ATraits;
425		_Alloc __a;
426	440	_Tp* __p = static_cast<_Tp*>(__this.__s_.__ptr);
427		_ATraits::destroy(__a, __p);
428		_ATraits::deallocate(__a, __p, 1);
	441	std::__destroy_at(__p);
	442	std::__libcpp_deallocate<_Tp>(__p, __element_count(1));
429	443	__this.__h_ = nullptr;
430	444	}
431	445
...	...	@@ -456,72 +470,24 @@ private:
456	470
457	471	} // namespace __any_imp
458	472
459		template <class _ValueType, class _Tp, class>
460		any::any(_ValueType&& __v) : __h_(nullptr) {
461		__any_imp::_Handler<_Tp>::__create(*this, std::forward<_ValueType>(__v));
462		}
463
464		template <class _ValueType, class... _Args, class _Tp, class>
465		any::any(in_place_type_t<_ValueType>, _Args&&... __args) {
466		__any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
467		}
468
469		template <class _ValueType, class _Up, class... _Args, class _Tp, class>
470		any::any(in_place_type_t<_ValueType>, initializer_list<_Up> __il, _Args&&... __args) {
471		__any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
472		}
473
474		template <class _ValueType, class, class>
475		inline _LIBCPP_HIDE_FROM_ABI any& any::operator=(_ValueType&& __v) {
476		any(std::forward<_ValueType>(__v)).swap(*this);
477		return *this;
478		}
479
480		template <class _ValueType, class... _Args, class _Tp, class>
481		inline _LIBCPP_HIDE_FROM_ABI _Tp& any::emplace(_Args&&... __args) {
482		reset();
483		return __any_imp::_Handler<_Tp>::__create(*this, std::forward<_Args>(__args)...);
484		}
485
486		template <class _ValueType, class _Up, class... _Args, class _Tp, class>
487		inline _LIBCPP_HIDE_FROM_ABI _Tp& any::emplace(initializer_list<_Up> __il, _Args&&... __args) {
488		reset();
489		return __any_imp::_Handler<_Tp>::__create(*this, __il, std::forward<_Args>(__args)...);
490		}
491
492		inline _LIBCPP_HIDE_FROM_ABI void any::swap(any& __rhs) _NOEXCEPT {
493		if (this == &__rhs)
494		return;
495		if (__h_ && __rhs.__h_) {
496		any __tmp;
497		__rhs.__call(_Action::_Move, &__tmp);
498		this->__call(_Action::_Move, &__rhs);
499		__tmp.__call(_Action::_Move, this);
500		} else if (__h_) {
501		this->__call(_Action::_Move, &__rhs);
502		} else if (__rhs.__h_) {
503		__rhs.__call(_Action::_Move, this);
504		}
505		}
506
507	473	// 6.4 Non-member functions
508	474
509		inline _LIBCPP_HIDE_FROM_ABI void swap(any& __lhs, any& __rhs) _NOEXCEPT { __lhs.swap(__rhs); }
	475	inline _LIBCPP_HIDE_FROM_ABI void swap(any& __lhs, any& __rhs) noexcept { __lhs.swap(__rhs); }
510	476
511	477	template <class _Tp, class... _Args>
512		inline _LIBCPP_HIDE_FROM_ABI any make_any(_Args&&... __args) {
	478	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI any make_any(_Args&&... __args) {
513	479	return any(in_place_type<_Tp>, std::forward<_Args>(__args)...);
514	480	}
515	481
516	482	template <class _Tp, class _Up, class... _Args>
517		inline _LIBCPP_HIDE_FROM_ABI any make_any(initializer_list<_Up> __il, _Args&&... __args) {
	483	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI any make_any(initializer_list<_Up> __il, _Args&&... __args) {
518	484	return any(in_place_type<_Tp>, __il, std::forward<_Args>(__args)...);
519	485	}
520	486
521	487	template <class _ValueType>
522		inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any const& __v) {
	488	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any const& __v) {
523	489	using _RawValueType = __remove_cvref_t<_ValueType>;
524		static_assert(is_constructible<_ValueType, _RawValueType const&>::value,
	490	static_assert(is_constructible_v<_ValueType, _RawValueType const&>,
525	491	"ValueType is required to be a const lvalue reference "
526	492	"or a CopyConstructible type");
527	493	auto __tmp = std::any_cast<add_const_t<_RawValueType>>(&__v);
...	...	@@ -531,9 +497,9 @@ inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any const& __v) {
531	497	}
532	498
533	499	template <class _ValueType>
534		inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any& __v) {
	500	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any& __v) {
535	501	using _RawValueType = __remove_cvref_t<_ValueType>;
536		static_assert(is_constructible<_ValueType, _RawValueType&>::value,
	502	static_assert(is_constructible_v<_ValueType, _RawValueType&>,
537	503	"ValueType is required to be an lvalue reference "
538	504	"or a CopyConstructible type");
539	505	auto __tmp = std::any_cast<_RawValueType>(&__v);
...	...	@@ -543,9 +509,9 @@ inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any& __v) {
543	509	}
544	510
545	511	template <class _ValueType>
546		inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any&& __v) {
	512	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any&& __v) {
547	513	using _RawValueType = __remove_cvref_t<_ValueType>;
548		static_assert(is_constructible<_ValueType, _RawValueType>::value,
	514	static_assert(is_constructible_v<_ValueType, _RawValueType>,
549	515	"ValueType is required to be an rvalue reference "
550	516	"or a CopyConstructible type");
551	517	auto __tmp = std::any_cast<_RawValueType>(&__v);
...	...	@@ -555,39 +521,31 @@ inline _LIBCPP_HIDE_FROM_ABI _ValueType any_cast(any&& __v) {
555	521	}
556	522
557	523	template <class _ValueType>
558		inline _LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const* __any) _NOEXCEPT {
	524	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI add_pointer_t<add_const_t<_ValueType>> any_cast(any const* __any) noexcept {
559	525	static_assert(!is_void_v<_ValueType>, "_ValueType may not be void.");
560		static_assert(!is_reference<_ValueType>::value, "_ValueType may not be a reference.");
	526	static_assert(!is_reference_v<_ValueType>, "_ValueType may not be a reference.");
561	527	return std::any_cast<_ValueType>(const_cast<any*>(__any));
562	528	}
563	529
564		template <class _RetType>
565		inline _LIBCPP_HIDE_FROM_ABI _RetType __pointer_or_func_cast(void* __p, /IsFunction/ false_type) noexcept {
566		return static_cast<_RetType>(__p);
567		}
568
569		template <class _RetType>
570		inline _LIBCPP_HIDE_FROM_ABI _RetType __pointer_or_func_cast(void, /IsFunction*/ true_type) noexcept {
571		return nullptr;
572		}
573
574	530	template <class _ValueType>
575		_LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any* __any) _NOEXCEPT {
	531	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI add_pointer_t<_ValueType> any_cast(any* __any) noexcept {
576	532	using __any_imp::_Action;
577	533	static_assert(!is_void_v<_ValueType>, "_ValueType may not be void.");
578		static_assert(!is_reference<_ValueType>::value, "_ValueType may not be a reference.");
579		typedef add_pointer_t<_ValueType> _ReturnType;
580		if (__any && __any->__h_) {
581		void* __p = __any->__call(
582		_Action::_Get,
583		nullptr,
	534	static_assert(!is_reference_v<_ValueType>, "_ValueType may not be a reference.");
	535	if constexpr (!is_function_v<_ValueType>) {
	536	using _ReturnType = add_pointer_t<_ValueType>;
	537	if (__any && __any->__h_) {
	538	void* __p = __any->__call(
	539	_Action::_Get,
	540	nullptr,
584	541	# if _LIBCPP_HAS_RTTI
585		&typeid(_ValueType),
	542	&typeid(_ValueType),
586	543	# else
587		nullptr,
	544	nullptr,
588	545	# endif
589		__any_imp::__get_fallback_typeid<_ValueType>());
590		return std::__pointer_or_func_cast<_ReturnType>(__p, is_function<_ValueType>{});
	546	__any_imp::__get_fallback_typeid<_ValueType>());
	547	return static_cast<_ReturnType>(__p);
	548	}
591	549	}
592	550	return nullptr;
593	551	}
...	...	@@ -613,6 +571,11 @@ _LIBCPP_POP_MACROS
613	571	# include <type_traits>
614	572	# include <variant>
615	573	# endif
	574
	575	# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
	576	# include <cstring>
	577	# include <limits>
	578	# endif
616	579	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
617	580
618	581	#endif // _LIBCPP_ANY

lib/libcxx/include/array+88-56

...	...	@@ -134,7 +134,6 @@ template <size_t I, class T, size_t N> const T&& get(const array<T, N>&&) noexce
134	134	# include <__type_traits/is_const.h>
135	135	# include <__type_traits/is_constructible.h>
136	136	# include <__type_traits/is_nothrow_constructible.h>
137		# include <__type_traits/is_replaceable.h>
138	137	# include <__type_traits/is_same.h>
139	138	# include <__type_traits/is_swappable.h>
140	139	# include <__type_traits/is_trivially_relocatable.h>
...	...	@@ -176,7 +175,6 @@ template <class _Tp, size_t _Size>
176	175	struct array {
177	176	using __trivially_relocatable _LIBCPP_NODEBUG =
178	177	__conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, array, void>;
179		using __replaceable _LIBCPP_NODEBUG = __conditional_t<__is_replaceable_v<_Tp>, array, void>;
180	178
181	179	// types:
182	180	using __self _LIBCPP_NODEBUG = array;
...	...	@@ -212,28 +210,28 @@ struct array {
212	210	}
213	211
214	212	// iterators:
215		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
	213	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
216	214	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
217	215	return std::__make_static_bounded_iter<_Size>(data(), data());
218	216	# else
219	217	return iterator(data());
220	218	# endif
221	219	}
222		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
	220	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
223	221	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
224	222	return std::__make_static_bounded_iter<_Size>(data(), data());
225	223	# else
226	224	return const_iterator(data());
227	225	# endif
228	226	}
229		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
	227	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
230	228	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
231	229	return std::__make_static_bounded_iter<_Size>(data() + _Size, data());
232	230	# else
233	231	return iterator(data() + _Size);
234	232	# endif
235	233	}
236		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
	234	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
237	235	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
238	236	return std::__make_static_bounded_iter<_Size>(data() + _Size, data());
239	237	# else
...	...	@@ -241,62 +239,81 @@ struct array {
241	239	# endif
242	240	}
243	241
244		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
	242	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
245	243	return reverse_iterator(end());
246	244	}
247		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rbegin() const _NOEXCEPT {
	245	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator
	246	rbegin() const _NOEXCEPT {
248	247	return const_reverse_iterator(end());
249	248	}
250		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
	249	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
251	250	return reverse_iterator(begin());
252	251	}
253		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
	252	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
254	253	return const_reverse_iterator(begin());
255	254	}
256	255
257		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT { return begin(); }
258		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT { return end(); }
259		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crbegin() const _NOEXCEPT {
	256	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT {
	257	return begin();
	258	}
	259	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT {
	260	return end();
	261	}
	262	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator
	263	crbegin() const _NOEXCEPT {
260	264	return rbegin();
261	265	}
262		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	266	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT {
	267	return rend();
	268	}
263	269
264	270	// capacity:
265		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return _Size; }
266		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return _Size; }
	271	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return _Size; }
	272	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return _Size; }
267	273	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return _Size == 0; }
268	274
269	275	// element access:
270		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type __n) _NOEXCEPT {
	276	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type __n) _NOEXCEPT {
271	277	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < _Size, "out-of-bounds access in std::array<T, N>");
272	278	return __elems_[__n];
273	279	}
274		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference operator[](size_type __n) const _NOEXCEPT {
	280	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference
	281	operator[](size_type __n) const _NOEXCEPT {
275	282	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__n < _Size, "out-of-bounds access in std::array<T, N>");
276	283	return __elems_[__n];
277	284	}
278	285
279		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type __n) {
	286	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type __n) {
280	287	if (__n >= _Size)
281	288	std::__throw_out_of_range("array::at");
282	289	return __elems_[__n];
283	290	}
284	291
285		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type __n) const {
	292	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type __n) const {
286	293	if (__n >= _Size)
287	294	std::__throw_out_of_range("array::at");
288	295	return __elems_[__n];
289	296	}
290	297
291		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT { return (*this)[0]; }
292		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT { return (*this)[0]; }
293		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT { return (*this)[_Size - 1]; }
294		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
	298	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT {
	299	return (*this)[0];
	300	}
	301	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT {
	302	return (*this)[0];
	303	}
	304	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT {
	305	return (*this)[_Size - 1];
	306	}
	307	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
295	308	return (*this)[_Size - 1];
296	309	}
297	310
298		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT { return __elems_; }
299		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT { return __elems_; }
	311	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT {
	312	return __elems_;
	313	}
	314	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT {
	315	return __elems_;
	316	}
300	317	};
301	318
302	319	template <class _Tp>
...	...	@@ -330,8 +347,10 @@ struct array<_Tp, 0> {
330	347	};
331	348	_ALIGNAS_TYPE(_ArrayInStructT) _EmptyType __elems_[sizeof(_ArrayInStructT)];
332	349
333		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT { return nullptr; }
334		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT { return nullptr; }
	350	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 value_type* data() _NOEXCEPT { return nullptr; }
	351	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const value_type* data() const _NOEXCEPT {
	352	return nullptr;
	353	}
335	354
336	355	// No explicit construct/copy/destroy for aggregate type
337	356	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void fill(const value_type&) {
...	...	@@ -343,28 +362,28 @@ struct array<_Tp, 0> {
343	362	}
344	363
345	364	// iterators:
346		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
	365	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator begin() _NOEXCEPT {
347	366	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
348	367	return std::__make_static_bounded_iter<0>(data(), data());
349	368	# else
350	369	return iterator(data());
351	370	# endif
352	371	}
353		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
	372	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator begin() const _NOEXCEPT {
354	373	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
355	374	return std::__make_static_bounded_iter<0>(data(), data());
356	375	# else
357	376	return const_iterator(data());
358	377	# endif
359	378	}
360		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
	379	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 iterator end() _NOEXCEPT {
361	380	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
362	381	return std::__make_static_bounded_iter<0>(data(), data());
363	382	# else
364	383	return iterator(data());
365	384	# endif
366	385	}
367		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
	386	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator end() const _NOEXCEPT {
368	387	# if defined(_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY)
369	388	return std::__make_static_bounded_iter<0>(data(), data());
370	389	# else
...	...	@@ -372,68 +391,77 @@ struct array<_Tp, 0> {
372	391	# endif
373	392	}
374	393
375		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
	394	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rbegin() _NOEXCEPT {
376	395	return reverse_iterator(end());
377	396	}
378		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rbegin() const _NOEXCEPT {
	397	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator
	398	rbegin() const _NOEXCEPT {
379	399	return const_reverse_iterator(end());
380	400	}
381		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
	401	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reverse_iterator rend() _NOEXCEPT {
382	402	return reverse_iterator(begin());
383	403	}
384		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
	404	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator rend() const _NOEXCEPT {
385	405	return const_reverse_iterator(begin());
386	406	}
387	407
388		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT { return begin(); }
389		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT { return end(); }
390		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crbegin() const _NOEXCEPT {
	408	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cbegin() const _NOEXCEPT {
	409	return begin();
	410	}
	411	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_iterator cend() const _NOEXCEPT {
	412	return end();
	413	}
	414	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator
	415	crbegin() const _NOEXCEPT {
391	416	return rbegin();
392	417	}
393		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	418	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 const_reverse_iterator crend() const _NOEXCEPT {
	419	return rend();
	420	}
394	421
395	422	// capacity:
396		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return 0; }
397		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return 0; }
	423	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type size() const _NOEXCEPT { return 0; }
	424	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_type max_size() const _NOEXCEPT { return 0; }
398	425	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return true; }
399	426
400	427	// element access:
401		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type) _NOEXCEPT {
	428	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference operator[](size_type) _NOEXCEPT {
402	429	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::operator[] on a zero-sized array");
403	430	__libcpp_unreachable();
404	431	}
405	432
406		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference operator[](size_type) const _NOEXCEPT {
	433	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference
	434	operator[](size_type) const _NOEXCEPT {
407	435	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::operator[] on a zero-sized array");
408	436	__libcpp_unreachable();
409	437	}
410	438
411		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type) {
	439	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference at(size_type) {
412	440	std::__throw_out_of_range("array<T, 0>::at");
413	441	__libcpp_unreachable();
414	442	}
415	443
416		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type) const {
	444	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference at(size_type) const {
417	445	std::__throw_out_of_range("array<T, 0>::at");
418	446	__libcpp_unreachable();
419	447	}
420	448
421		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT {
	449	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference front() _NOEXCEPT {
422	450	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::front() on a zero-sized array");
423	451	__libcpp_unreachable();
424	452	}
425	453
426		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT {
	454	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference front() const _NOEXCEPT {
427	455	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::front() on a zero-sized array");
428	456	__libcpp_unreachable();
429	457	}
430	458
431		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT {
	459	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 reference back() _NOEXCEPT {
432	460	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::back() on a zero-sized array");
433	461	__libcpp_unreachable();
434	462	}
435	463
436		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
	464	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const_reference back() const _NOEXCEPT {
437	465	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(false, "cannot call array<T, 0>::back() on a zero-sized array");
438	466	__libcpp_unreachable();
439	467	}
...	...	@@ -503,25 +531,29 @@ struct tuple_element<_Ip, array<_Tp, _Size> > {
503	531	};
504	532
505	533	template <size_t _Ip, class _Tp, size_t _Size>
506		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp& get(array<_Tp, _Size>& __a) _NOEXCEPT {
	534	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&
	535	get(array<_Tp, _Size>& __a) _NOEXCEPT {
507	536	static_assert(_Ip < _Size, "Index out of bounds in std::get<> (std::array)");
508	537	return __a.__elems_[_Ip];
509	538	}
510	539
511	540	template <size_t _Ip, class _Tp, size_t _Size>
512		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp& get(const array<_Tp, _Size>& __a) _NOEXCEPT {
	541	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&
	542	get(const array<_Tp, _Size>& __a) _NOEXCEPT {
513	543	static_assert(_Ip < _Size, "Index out of bounds in std::get<> (const std::array)");
514	544	return __a.__elems_[_Ip];
515	545	}
516	546
517	547	template <size_t _Ip, class _Tp, size_t _Size>
518		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&& get(array<_Tp, _Size>&& __a) _NOEXCEPT {
	548	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp&&
	549	get(array<_Tp, _Size>&& __a) _NOEXCEPT {
519	550	static_assert(_Ip < _Size, "Index out of bounds in std::get<> (std::array &&)");
520	551	return std::move(__a.__elems_[_Ip]);
521	552	}
522	553
523	554	template <size_t _Ip, class _Tp, size_t _Size>
524		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&& get(const array<_Tp, _Size>&& __a) _NOEXCEPT {
	555	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Tp&&
	556	get(const array<_Tp, _Size>&& __a) _NOEXCEPT {
525	557	static_assert(_Ip < _Size, "Index out of bounds in std::get<> (const std::array &&)");
526	558	return std::move(__a.__elems_[_Ip]);
527	559	}
...	...	@@ -541,7 +573,7 @@ __to_array_rvalue_impl(_Tp (&&__arr)[_Size], index_sequence<_Index...>) {
541	573	}
542	574
543	575	template <typename _Tp, size_t _Size>
544		_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
	576	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
545	577	to_array(_Tp (&__arr)[_Size]) noexcept(is_nothrow_constructible_v<_Tp, _Tp&>) {
546	578	static_assert(!is_array_v<_Tp>, "[array.creation]/1: to_array does not accept multidimensional arrays.");
547	579	static_assert(is_constructible_v<_Tp, _Tp&>, "[array.creation]/1: to_array requires copy constructible elements.");
...	...	@@ -549,7 +581,7 @@ to_array(_Tp (&__arr)[_Size]) noexcept(is_nothrow_constructible_v<_Tp, _Tp&>) {
549	581	}
550	582
551	583	template <typename _Tp, size_t _Size>
552		_LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
	584	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr array<remove_cv_t<_Tp>, _Size>
553	585	to_array(_Tp (&&__arr)[_Size]) noexcept(is_nothrow_move_constructible_v<_Tp>) {
554	586	static_assert(!is_array_v<_Tp>, "[array.creation]/4: to_array does not accept multidimensional arrays.");
555	587	static_assert(is_move_constructible_v<_Tp>, "[array.creation]/4: to_array requires move constructible elements.");

lib/libcxx/include/atomic+2

...	...	@@ -608,6 +608,8 @@ template <class T>
608	608	# include <__atomic/atomic_init.h>
609	609	# include <__atomic/atomic_lock_free.h>
610	610	# include <__atomic/atomic_sync.h>
	611	# include <__atomic/atomic_sync_timed.h>
	612	# include <__atomic/atomic_waitable_traits.h>
611	613	# include <__atomic/check_memory_order.h>
612	614	# include <__atomic/contention_t.h>
613	615	# include <__atomic/fence.h>

lib/libcxx/include/barrier+20-24

...	...	@@ -57,8 +57,6 @@ namespace std
57	57	# include <__atomic/memory_order.h>
58	58	# include <__cstddef/ptrdiff_t.h>
59	59	# include <__memory/unique_ptr.h>
60		# include <__thread/poll_with_backoff.h>
61		# include <__thread/timed_backoff_policy.h>
62	60	# include <__utility/move.h>
63	61	# include <cstdint>
64	62	# include <limits>
...	...	@@ -97,19 +95,20 @@ using __barrier_phase_t _LIBCPP_NODEBUG = uint8_t;
97	95
98	96	class __barrier_algorithm_base;
99	97
100		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base*
101		__construct_barrier_algorithm_base(ptrdiff_t& __expected);
	98	[[__gnu__::__returns_nonnull__, __gnu__::__malloc__]]
	99	_LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base* __construct_barrier_algorithm_base(ptrdiff_t& __expected);
102	100
103		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI bool
104		__arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept;
	101	_LIBCPP_EXPORTED_FROM_ABI bool
	102	__arrive_barrier_algorithm_base([[__gnu__::__nonnull__]] _LIBCPP_NOESCAPE __barrier_algorithm_base* __barrier,
	103	__barrier_phase_t __old_phase) noexcept;
105	104
106		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_EXPORTED_FROM_ABI void
107		__destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) noexcept;
	105	_LIBCPP_EXPORTED_FROM_ABI void __destroy_barrier_algorithm_base(
	106	[[__gnu__::__nonnull__]] _LIBCPP_NOESCAPE __barrier_algorithm_base* __barrier) noexcept;
108	107
109	108	template <class _CompletionF>
110	109	class __barrier_base {
111	110	ptrdiff_t __expected_;
112		unique_ptr<__barrier_algorithm_base, void ()(__barrier_algorithm_base)> __base_;
	111	unique_ptr<__barrier_algorithm_base, void ()(_LIBCPP_NOESCAPE __barrier_algorithm_base)> __base_;
113	112	atomic<ptrdiff_t> __expected_adjustment_;
114	113	_CompletionF __completion_;
115	114	atomic<__barrier_phase_t> __phase_;
...	...	@@ -119,14 +118,13 @@ public:
119	118
120	119	static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
121	120
122		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI
123		__barrier_base(ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
	121	_LIBCPP_HIDE_FROM_ABI __barrier_base(ptrdiff_t __expected, _CompletionF __completion = _CompletionF())
124	122	: __expected_(__expected),
125	123	__base_(std::__construct_barrier_algorithm_base(this->__expected_), &__destroy_barrier_algorithm_base),
126	124	__expected_adjustment_(0),
127	125	__completion_(std::move(__completion)),
128	126	__phase_(0) {}
129		[[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update) {
	127	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update) {
130	128	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
131	129	__update <= __expected_, "update is greater than the expected count for the current barrier phase");
132	130
...	...	@@ -141,11 +139,10 @@ public:
141	139	}
142	140	return __old_phase;
143	141	}
144		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __old_phase) const {
145		auto const __test_fn = [this, __old_phase]() -> bool { return __phase_.load(memory_order_acquire) != __old_phase; };
146		std::__libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy());
	142	_LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __old_phase) const {
	143	__phase_.wait(__old_phase, std::memory_order_acquire);
147	144	}
148		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
	145	_LIBCPP_HIDE_FROM_ABI void arrive_and_drop() {
149	146	__expected_adjustment_.fetch_sub(1, memory_order_relaxed);
150	147	(void)arrive(1);
151	148	}
...	...	@@ -158,9 +155,10 @@ class barrier {
158	155	public:
159	156	using arrival_token = typename __barrier_base<_CompletionF>::arrival_token;
160	157
161		static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return __barrier_base<_CompletionF>::max(); }
	158	[[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept {
	159	return __barrier_base<_CompletionF>::max();
	160	}
162	161
163		_LIBCPP_AVAILABILITY_SYNC
164	162	_LIBCPP_HIDE_FROM_ABI explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF())
165	163	: __b_(__count, std::move(__completion)) {
166	164	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
...	...	@@ -175,15 +173,13 @@ public:
175	173	barrier(barrier const&) = delete;
176	174	barrier& operator=(barrier const&) = delete;
177	175
178		[[nodiscard]] _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update = 1) {
	176	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI arrival_token arrive(ptrdiff_t __update = 1) {
179	177	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update > 0, "barrier:arrive must be called with a value greater than 0");
180	178	return __b_.arrive(__update);
181	179	}
182		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __phase) const {
183		__b_.wait(std::move(__phase));
184		}
185		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait() { wait(arrive()); }
186		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_drop() { __b_.arrive_and_drop(); }
	180	_LIBCPP_HIDE_FROM_ABI void wait(arrival_token&& __phase) const { __b_.wait(std::move(__phase)); }
	181	_LIBCPP_HIDE_FROM_ABI void arrive_and_wait() { wait(arrive()); }
	182	_LIBCPP_HIDE_FROM_ABI void arrive_and_drop() { __b_.arrive_and_drop(); }
187	183	};
188	184
189	185	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/bitset+49-30

...	...	@@ -147,7 +147,6 @@ template <size_t N> struct hash<std::bitset<N>>;
147	147	# include <__functional/hash.h>
148	148	# include <__functional/identity.h>
149	149	# include <__functional/unary_function.h>
150		# include <__tuple/tuple_indices.h>
151	150	# include <__type_traits/enable_if.h>
152	151	# include <__type_traits/integral_constant.h>
153	152	# include <__type_traits/is_char_like_type.h>
...	...	@@ -314,7 +313,7 @@ private:
314	313	_LIBCPP_HIDE_FROM_ABI void __init(unsigned long long __v, true_type) _NOEXCEPT;
315	314	# else
316	315	template <size_t... _Indices>
317		_LIBCPP_HIDE_FROM_ABI constexpr __bitset(unsigned long long __v, std::__tuple_indices<_Indices...>) _NOEXCEPT
	316	_LIBCPP_HIDE_FROM_ABI constexpr __bitset(unsigned long long __v, __index_sequence<_Indices...>) _NOEXCEPT
318	317	: __first_{static_cast<__storage_type>(__v >> (_Indices * __bits_per_word))...} {}
319	318	# endif // _LIBCPP_CXX03_LANG
320	319	};
...	...	@@ -352,10 +351,9 @@ template <size_t _N_words, size_t _Size>
352	351	inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
353	352	# ifndef _LIBCPP_CXX03_LANG
354	353	: __bitset(__v,
355		std::__make_indices_imp< (_N_words < (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1)
356		? _N_words
357		: (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1,
358		0>{})
	354	__make_index_sequence<(_N_words < (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1)
	355	? _N_words
	356	: (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1>())
359	357	# endif
360	358	{
361	359	# ifdef _LIBCPP_CXX03_LANG
...	...	@@ -677,53 +675,63 @@ public:
677	675	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& set(size_t __pos, bool __val = true);
678	676	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& reset() _NOEXCEPT;
679	677	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& reset(size_t __pos);
680		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator~() const _NOEXCEPT;
	678	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator~() const _NOEXCEPT;
681	679	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& flip() _NOEXCEPT;
682	680	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset& flip(size_t __pos);
683	681
684	682	// element access:
685		# ifdef _LIBCPP_ABI_BITSET_VECTOR_BOOL_CONST_SUBSCRIPT_RETURN_BOOL
686		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator[](size_t __p) const {
	683	// TODO(LLVM 24): Remove the opt-out
	684	# ifndef _LIBCPP_DEPRECATED_ABI_BITSET_CONST_SUBSCRIPT_RETURN_REF
	685	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool operator[](size_t __p) const {
687	686	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p < _Size, "bitset::operator[] index out of bounds");
688	687	return __base::__make_ref(__p);
689	688	}
690	689	# else
691		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __const_reference operator[](size_t __p) const {
	690	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __const_reference operator[](size_t __p) const {
692	691	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p < _Size, "bitset::operator[] index out of bounds");
693	692	return __base::__make_ref(__p);
694	693	}
695	694	# endif
696		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference operator[](size_t __p) {
	695	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 reference operator[](size_t __p) {
697	696	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p < _Size, "bitset::operator[] index out of bounds");
698	697	return __base::__make_ref(__p);
699	698	}
700		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const { return __base::to_ulong(); }
701		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
	699	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const {
	700	return __base::to_ulong();
	701	}
	702	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
702	703	return __base::to_ullong();
703	704	}
704	705	template <class _CharT, class _Traits, class _Allocator>
705		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
	706	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
706	707	to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
707	708	template <class _CharT, class _Traits>
708		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, allocator<_CharT> >
	709	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
	710	_LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, allocator<_CharT> >
709	711	to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
710	712	template <class _CharT>
711		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, char_traits<_CharT>, allocator<_CharT> >
	713	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
	714	_LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, char_traits<_CharT>, allocator<_CharT> >
712	715	to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
713		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<char, char_traits<char>, allocator<char> >
	716	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
	717	_LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<char, char_traits<char>, allocator<char> >
714	718	to_string(char __zero = '0', char __one = '1') const;
715		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t count() const _NOEXCEPT;
716		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_t size() const _NOEXCEPT { return _Size; }
	719	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t count() const _NOEXCEPT;
	720	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR size_t size() const _NOEXCEPT { return _Size; }
717	721	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const bitset& __rhs) const _NOEXCEPT;
718	722	# if _LIBCPP_STD_VER <= 17
719	723	_LIBCPP_HIDE_FROM_ABI bool operator!=(const bitset& __rhs) const _NOEXCEPT;
720	724	# endif
721		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool test(size_t __pos) const;
722		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT { return __base::all(); }
723		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT { return __base::any(); }
724		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool none() const _NOEXCEPT { return !any(); }
725		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator<<(size_t __pos) const _NOEXCEPT;
726		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator>>(size_t __pos) const _NOEXCEPT;
	725	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool test(size_t __pos) const;
	726	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT {
	727	return __base::all();
	728	}
	729	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool any() const _NOEXCEPT {
	730	return __base::any();
	731	}
	732	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool none() const _NOEXCEPT { return !any(); }
	733	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator<<(size_t __pos) const _NOEXCEPT;
	734	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset operator>>(size_t __pos) const _NOEXCEPT;
727	735
728	736	private:
729	737	template <class _CharT, class _Traits>
...	...	@@ -869,7 +877,16 @@ bitset<_Size>::to_string(char __zero, char __one) const {
869	877
870	878	template <size_t _Size>
871	879	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 size_t bitset<_Size>::count() const _NOEXCEPT {
872		return static_cast<size_t>(std::count(__base::__make_iter(0), __base::__make_iter(_Size), true));
	880	# if defined(_LIBCPP_COMPILER_CLANG_BASED) && !defined(_LIBCPP_CXX03_LANG)
	881	if constexpr (_Size == 0) {
	882	return 0;
	883	} else if constexpr (_Size <= __base::__bits_per_word) {
	884	return __builtin_popcountg(static_cast<unsigned _BitInt(_Size)>(__base::__first_));
	885	} else
	886	# endif
	887	{
	888	return static_cast<size_t>(std::count(__base::__make_iter(0), __base::__make_iter(_Size), true));
	889	}
873	890	}
874	891
875	892	template <size_t _Size>
...	...	@@ -912,7 +929,7 @@ bitset<_Size>::operator>>(size_t __pos) const _NOEXCEPT {
912	929	}
913	930
914	931	template <size_t _Size>
915		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
	932	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
916	933	operator&(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
917	934	bitset<_Size> __r = __x;
918	935	__r &= __y;
...	...	@@ -920,7 +937,7 @@ operator&(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
920	937	}
921	938
922	939	template <size_t _Size>
923		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
	940	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
924	941	operator\|(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
925	942	bitset<_Size> __r = __x;
926	943	__r \|= __y;
...	...	@@ -928,7 +945,7 @@ operator\|(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
928	945	}
929	946
930	947	template <size_t _Size>
931		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
	948	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>
932	949	operator^(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
933	950	bitset<_Size> __r = __x;
934	951	__r ^= __y;
...	...	@@ -937,7 +954,9 @@ operator^(const bitset<_Size>& __x, const bitset<_Size>& __y) _NOEXCEPT {
937	954
938	955	template <size_t _Size>
939	956	struct hash<bitset<_Size> > : public __unary_function<bitset<_Size>, size_t> {
940		_LIBCPP_HIDE_FROM_ABI size_t operator()(const bitset<_Size>& __bs) const _NOEXCEPT { return __bs.__hash_code(); }
	957	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const bitset<_Size>& __bs) const _NOEXCEPT {
	958	return __bs.__hash_code();
	959	}
941	960	};
942	961
943	962	template <class _CharT, class _Traits, size_t _Size>

lib/libcxx/include/ccomplex+3-11

...	...	@@ -26,18 +26,10 @@
26	26	# pragma GCC system_header
27	27	# endif
28	28
29		# if _LIBCPP_STD_VER >= 20
30
31		using __standard_header_ccomplex
32		_LIBCPP_DEPRECATED_("removed in C++20. Include <complex> instead.") _LIBCPP_NODEBUG = void;
33		using __use_standard_header_ccomplex _LIBCPP_NODEBUG = __standard_header_ccomplex;
34
35		# elif _LIBCPP_STD_VER >= 17
36
37		using __standard_header_ccomplex _LIBCPP_DEPRECATED_("Include <complex> instead.") _LIBCPP_NODEBUG = void;
38		using __use_standard_header_ccomplex _LIBCPP_NODEBUG = __standard_header_ccomplex;
39
	29	# if _LIBCPP_STD_VER >= 17 && !__building_module(std) && _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS
	30	# warning <ccomplex> is deprecated in C++17 and removed in C++20. Include <complex> instead.
40	31	# endif
	32
41	33	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
42	34
43	35	#endif // _LIBCPP_CCOMPLEX

lib/libcxx/include/chrono+47

...	...	@@ -218,6 +218,9 @@ template <class ToDuration, class Rep, class Period>
218	218	template <class ToDuration, class Rep, class Period>
219	219	constexpr ToDuration round(const duration<Rep, Period>& d); // C++17
220	220
	221	template <class T> struct is_clock; // C++20
	222	template <class T> inline constexpr bool is_clock_v = is_clock<T>::value; // C++20
	223
221	224	// duration I/O
222	225	template<class charT, class traits, class Rep, class Period> // C++20
223	226	basic_ostream<charT, traits>&
...	...	@@ -1035,6 +1038,49 @@ constexpr chrono::year operator ""y(unsigned lo
1035	1038	} // chrono_literals
1036	1039	} // literals
1037	1040
	1041	namespace std {
	1042	template<class T>
	1043	struct hash; // C++26
	1044	template<class Rep, class Period>
	1045	struct hash<chrono::duration<Rep, Period>>; // C++26
	1046	template<class Clock, class Duration>
	1047	struct hash<chrono::time_point<Clock, Duration>>; // C++26
	1048	template<>
	1049	struct hash<chrono::day>; // C++26
	1050	template<>
	1051	struct hash<chrono::month>; // C++26
	1052	template<>
	1053	struct hash<chrono::year>; // C++26
	1054	template<>
	1055	struct hash<chrono::weekday>; // C++26
	1056	template<>
	1057	struct hash<chrono::weekday_indexed>; // C++26
	1058	template<>
	1059	struct hash<chrono::weekday_last>; // C++26
	1060	template<>
	1061	struct hash<chrono::month_day>; // C++26
	1062	template<>
	1063	struct hash<chrono::month_day_last>; // C++26
	1064	template<>
	1065	struct hash<chrono::month_weekday>; // C++26
	1066	template<>
	1067	struct hash<chrono::month_weekday_last>; // C++26
	1068	template<>
	1069	struct hash<chrono::year_month>; // C++26
	1070	template<>
	1071	struct hash<chrono::year_month_day>; // C++26
	1072	template<>
	1073	struct hash<chrono::year_month_day_last>; // C++26
	1074	template<>
	1075	struct hash<chrono::year_month_weekday>; // C++26
	1076	template<>
	1077	struct hash<chrono::year_month_weekday_last>; // C++26
	1078	template<class Duration, class TimeZonePtr>
	1079	struct hash<chrono::zoned_time<Duration, TimeZonePtr>>; // C++26
	1080	template<>
	1081	struct hash<chrono::leap_second>; // C++26
	1082	} // namespace std
	1083
1038	1084	} // std
1039	1085	*/
1040	1086
...	...	@@ -1057,6 +1103,7 @@ constexpr chrono::year operator ""y(unsigned lo
1057	1103	# include <__chrono/day.h>
1058	1104	# include <__chrono/exception.h>
1059	1105	# include <__chrono/hh_mm_ss.h>
	1106	# include <__chrono/is_clock.h>
1060	1107	# include <__chrono/literals.h>
1061	1108	# include <__chrono/local_info.h>
1062	1109	# include <__chrono/month.h>

lib/libcxx/include/ciso646+3-6

...	...	@@ -24,13 +24,10 @@
24	24	# pragma GCC system_header
25	25	# endif
26	26
27		# if _LIBCPP_STD_VER >= 20
28
29		using __standard_header_ciso646
30		_LIBCPP_DEPRECATED_("removed in C++20. Include <version> instead.") _LIBCPP_NODEBUG = void;
31		using __use_standard_header_ciso646 _LIBCPP_NODEBUG = __standard_header_ciso646;
32
	27	# if _LIBCPP_STD_VER >= 20 && !__building_module(std) && _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS
	28	# warning <ciso646> is removed in C++20. Include <version> instead.
33	29	# endif
	30
34	31	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
35	32
36	33	#endif // _LIBCPP_CISO646

lib/libcxx/include/complex+77-65

...	...	@@ -319,8 +319,8 @@ public:
319	319	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 complex(const complex<_Xp>& __c)
320	320	: __re_(__c.real()), __im_(__c.imag()) {}
321	321
322		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type real() const { return __re_; }
323		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type imag() const { return __im_; }
	322	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type real() const { return __re_; }
	323	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 value_type imag() const { return __im_; }
324	324
325	325	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
326	326	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
...	...	@@ -432,8 +432,8 @@ public:
432	432	_LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<double>& __c);
433	433	_LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<long double>& __c);
434	434
435		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float real() const { return __re_; }
436		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float imag() const { return __im_; }
	435	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float real() const { return __re_; }
	436	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR float imag() const { return __im_; }
437	437
438	438	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
439	439	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
...	...	@@ -529,8 +529,8 @@ public:
529	529	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<float>& __c);
530	530	_LIBCPP_HIDE_FROM_ABI explicit _LIBCPP_CONSTEXPR complex(const complex<long double>& __c);
531	531
532		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double real() const { return __re_; }
533		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double imag() const { return __im_; }
	532	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double real() const { return __re_; }
	533	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR double imag() const { return __im_; }
534	534
535	535	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
536	536	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
...	...	@@ -630,8 +630,8 @@ public:
630	630	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<float>& __c);
631	631	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR complex(const complex<double>& __c);
632	632
633		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double real() const { return __re_; }
634		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double imag() const { return __im_; }
	633	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double real() const { return __re_; }
	634	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR long double imag() const { return __im_; }
635	635
636	636	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void real(value_type __re) { __re_ = __re; }
637	637	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void imag(value_type __im) { __im_ = __im; }
...	...	@@ -732,7 +732,7 @@ inline _LIBCPP_CONSTEXPR complex<long double>::complex(const complex<double>& __
732	732	// 26.3.6 operators:
733	733
734	734	template <class _Tp>
735		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	735	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
736	736	operator+(const complex<_Tp>& __x, const complex<_Tp>& __y) {
737	737	complex<_Tp> __t(__x);
738	738	__t += __y;
...	...	@@ -740,7 +740,7 @@ operator+(const complex<_Tp>& __x, const complex<_Tp>& __y) {
740	740	}
741	741
742	742	template <class _Tp>
743		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	743	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
744	744	operator+(const complex<_Tp>& __x, const _Tp& __y) {
745	745	complex<_Tp> __t(__x);
746	746	__t += __y;
...	...	@@ -748,7 +748,7 @@ operator+(const complex<_Tp>& __x, const _Tp& __y) {
748	748	}
749	749
750	750	template <class _Tp>
751		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	751	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
752	752	operator+(const _Tp& __x, const complex<_Tp>& __y) {
753	753	complex<_Tp> __t(__y);
754	754	__t += __x;
...	...	@@ -756,7 +756,7 @@ operator+(const _Tp& __x, const complex<_Tp>& __y) {
756	756	}
757	757
758	758	template <class _Tp>
759		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	759	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
760	760	operator-(const complex<_Tp>& __x, const complex<_Tp>& __y) {
761	761	complex<_Tp> __t(__x);
762	762	__t -= __y;
...	...	@@ -764,7 +764,7 @@ operator-(const complex<_Tp>& __x, const complex<_Tp>& __y) {
764	764	}
765	765
766	766	template <class _Tp>
767		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	767	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
768	768	operator-(const complex<_Tp>& __x, const _Tp& __y) {
769	769	complex<_Tp> __t(__x);
770	770	__t -= __y;
...	...	@@ -772,7 +772,7 @@ operator-(const complex<_Tp>& __x, const _Tp& __y) {
772	772	}
773	773
774	774	template <class _Tp>
775		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	775	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
776	776	operator-(const _Tp& __x, const complex<_Tp>& __y) {
777	777	complex<_Tp> __t(-__y);
778	778	__t += __x;
...	...	@@ -780,13 +780,13 @@ operator-(const _Tp& __x, const complex<_Tp>& __y) {
780	780	}
781	781
782	782	template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> >
783		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	783	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
784	784	operator*(const complex<_Tp>& __lhs, const complex<_Tp>& __rhs) {
785	785	return complex<_Tp>(__from_builtin_tag(), __lhs.__builtin() * __rhs.__builtin());
786	786	}
787	787
788	788	template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> >
789		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	789	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
790	790	operator*(const complex<_Tp>& __z, const complex<_Tp>& __w) {
791	791	_Tp __a = __z.real();
792	792	_Tp __b = __z.imag();
...	...	@@ -797,7 +797,7 @@ operator*(const complex<_Tp>& __z, const complex<_Tp>& __w) {
797	797	}
798	798
799	799	template <class _Tp>
800		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	800	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
801	801	operator*(const complex<_Tp>& __x, const _Tp& __y) {
802	802	complex<_Tp> __t(__x);
803	803	__t *= __y;
...	...	@@ -805,7 +805,7 @@ operator*(const complex<_Tp>& __x, const _Tp& __y) {
805	805	}
806	806
807	807	template <class _Tp>
808		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	808	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
809	809	operator*(const _Tp& __x, const complex<_Tp>& __y) {
810	810	complex<_Tp> __t(__y);
811	811	__t *= __x;
...	...	@@ -813,13 +813,13 @@ operator*(const _Tp& __x, const complex<_Tp>& __y) {
813	813	}
814	814
815	815	template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> >
816		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	816	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
817	817	operator/(const complex<_Tp>& __lhs, const complex<_Tp>& __rhs) {
818	818	return complex<_Tp>(__from_builtin_tag(), __lhs.__builtin() / __rhs.__builtin());
819	819	}
820	820
821	821	template <class _Tp, __enable_if_t<!is_floating_point<_Tp>::value, int> >
822		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	822	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
823	823	operator/(const complex<_Tp>& __z, const complex<_Tp>& __w) {
824	824	_Tp __a = __z.real();
825	825	_Tp __b = __z.imag();
...	...	@@ -831,13 +831,13 @@ operator/(const complex<_Tp>& __z, const complex<_Tp>& __w) {
831	831	}
832	832
833	833	template <class _Tp>
834		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	834	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
835	835	operator/(const complex<_Tp>& __x, const _Tp& __y) {
836	836	return complex<_Tp>(__x.real() / __y, __x.imag() / __y);
837	837	}
838	838
839	839	template <class _Tp>
840		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	840	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
841	841	operator/(const _Tp& __x, const complex<_Tp>& __y) {
842	842	complex<_Tp> __t(__x);
843	843	__t /= __y;
...	...	@@ -845,12 +845,14 @@ operator/(const _Tp& __x, const complex<_Tp>& __y) {
845	845	}
846	846
847	847	template <class _Tp>
848		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> operator+(const complex<_Tp>& __x) {
	848	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	849	operator+(const complex<_Tp>& __x) {
849	850	return __x;
850	851	}
851	852
852	853	template <class _Tp>
853		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> operator-(const complex<_Tp>& __x) {
	854	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	855	operator-(const complex<_Tp>& __x) {
854	856	return complex<_Tp>(-__x.real(), -__x.imag());
855	857	}
856	858
...	...	@@ -912,12 +914,13 @@ struct __libcpp_complex_overload_traits<_Tp, false, true> {
912	914	// real
913	915
914	916	template <class _Tp>
915		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp real(const complex<_Tp>& __c) {
	917	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp real(const complex<_Tp>& __c) {
916	918	return __c.real();
917	919	}
918	920
919	921	template <class _Tp>
920		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
	922	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	923	typename __libcpp_complex_overload_traits<_Tp>::_ValueType
921	924	real(_Tp __re) {
922	925	return __re;
923	926	}
...	...	@@ -925,12 +928,13 @@ real(_Tp __re) {
925	928	// imag
926	929
927	930	template <class _Tp>
928		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp imag(const complex<_Tp>& __c) {
	931	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp imag(const complex<_Tp>& __c) {
929	932	return __c.imag();
930	933	}
931	934
932	935	template <class _Tp>
933		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
	936	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	937	typename __libcpp_complex_overload_traits<_Tp>::_ValueType
934	938	imag(_Tp) {
935	939	return 0;
936	940	}
...	...	@@ -938,36 +942,36 @@ imag(_Tp) {
938	942	// abs
939	943
940	944	template <class _Tp>
941		inline _LIBCPP_HIDE_FROM_ABI _Tp abs(const complex<_Tp>& __c) {
	945	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _Tp abs(const complex<_Tp>& __c) {
942	946	return std::hypot(__c.real(), __c.imag());
943	947	}
944	948
945	949	// arg
946	950
947	951	template <class _Tp>
948		inline _LIBCPP_HIDE_FROM_ABI _Tp arg(const complex<_Tp>& __c) {
	952	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _Tp arg(const complex<_Tp>& __c) {
949	953	return std::atan2(__c.imag(), __c.real());
950	954	}
951	955
952	956	template <class _Tp, __enable_if_t<is_same<_Tp, long double>::value, int> = 0>
953		inline _LIBCPP_HIDE_FROM_ABI long double arg(_Tp __re) {
	957	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long double arg(_Tp __re) {
954	958	return std::atan2l(0.L, __re);
955	959	}
956	960
957	961	template <class _Tp, __enable_if_t<is_integral<_Tp>::value \|\| is_same<_Tp, double>::value, int> = 0>
958		inline _LIBCPP_HIDE_FROM_ABI double arg(_Tp __re) {
	962	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI double arg(_Tp __re) {
959	963	return std::atan2(0., __re);
960	964	}
961	965
962	966	template <class _Tp, __enable_if_t<is_same<_Tp, float>::value, int> = 0>
963		inline _LIBCPP_HIDE_FROM_ABI float arg(_Tp __re) {
	967	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI float arg(_Tp __re) {
964	968	return std::atan2f(0.F, __re);
965	969	}
966	970
967	971	// norm
968	972
969	973	template <class _Tp>
970		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp norm(const complex<_Tp>& __c) {
	974	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp norm(const complex<_Tp>& __c) {
971	975	if (std::__constexpr_isinf(__c.real()))
972	976	return std::abs(__c.real());
973	977	if (std::__constexpr_isinf(__c.imag()))
...	...	@@ -976,7 +980,8 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp norm(const comple
976	980	}
977	981
978	982	template <class _Tp>
979		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __libcpp_complex_overload_traits<_Tp>::_ValueType
	983	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
	984	typename __libcpp_complex_overload_traits<_Tp>::_ValueType
980	985	norm(_Tp __re) {
981	986	typedef typename __libcpp_complex_overload_traits<_Tp>::_ValueType _ValueType;
982	987	return static_cast<_ValueType>(__re) * __re;
...	...	@@ -985,12 +990,14 @@ norm(_Tp __re) {
985	990	// conj
986	991
987	992	template <class _Tp>
988		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp> conj(const complex<_Tp>& __c) {
	993	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 complex<_Tp>
	994	conj(const complex<_Tp>& __c) {
989	995	return complex<_Tp>(__c.real(), -__c.imag());
990	996	}
991	997
992	998	template <class _Tp>
993		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 typename __libcpp_complex_overload_traits<_Tp>::_ComplexType
	999	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
	1000	typename __libcpp_complex_overload_traits<_Tp>::_ComplexType
994	1001	conj(_Tp __re) {
995	1002	typedef typename __libcpp_complex_overload_traits<_Tp>::_ComplexType _ComplexType;
996	1003	return _ComplexType(__re);
...	...	@@ -999,7 +1006,7 @@ conj(_Tp __re) {
999	1006	// proj
1000	1007
1001	1008	template <class _Tp>
1002		inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> proj(const complex<_Tp>& __c) {
	1009	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> proj(const complex<_Tp>& __c) {
1003	1010	complex<_Tp> __r = __c;
1004	1011	if (std::isinf(__c.real()) \|\| std::isinf(__c.imag()))
1005	1012	__r = complex<_Tp>(INFINITY, std::copysign(_Tp(0), __c.imag()));
...	...	@@ -1007,14 +1014,16 @@ inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> proj(const complex<_Tp>& __c) {
1007	1014	}
1008	1015
1009	1016	template <class _Tp, __enable_if_t<is_floating_point<_Tp>::value, int> = 0>
1010		inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType proj(_Tp __re) {
	1017	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType
	1018	proj(_Tp __re) {
1011	1019	if (std::isinf(__re))
1012	1020	__re = std::abs(__re);
1013	1021	return complex<_Tp>(__re);
1014	1022	}
1015	1023
1016	1024	template <class _Tp, __enable_if_t<is_integral<_Tp>::value, int> = 0>
1017		inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType proj(_Tp __re) {
	1025	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_ComplexType
	1026	proj(_Tp __re) {
1018	1027	typedef typename __libcpp_complex_overload_traits<_Tp>::_ComplexType _ComplexType;
1019	1028	return _ComplexType(__re);
1020	1029	}
...	...	@@ -1022,7 +1031,7 @@ inline _LIBCPP_HIDE_FROM_ABI typename __libcpp_complex_overload_traits<_Tp>::_Co
1022	1031	// polar
1023	1032
1024	1033	template <class _Tp>
1025		_LIBCPP_HIDE_FROM_ABI complex<_Tp> polar(const _Tp& __rho, const _Tp& __theta = _Tp()) {
	1034	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> polar(const _Tp& __rho, const _Tp& __theta = _Tp()) {
1026	1035	if (std::isnan(__rho) \|\| std::signbit(__rho))
1027	1036	return complex<_Tp>(_Tp(NAN), _Tp(NAN));
1028	1037	if (std::isnan(__theta)) {
...	...	@@ -1047,21 +1056,21 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> polar(const _Tp& __rho, const _Tp& __theta =
1047	1056	// log
1048	1057
1049	1058	template <class _Tp>
1050		inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log(const complex<_Tp>& __x) {
	1059	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log(const complex<_Tp>& __x) {
1051	1060	return complex<_Tp>(std::log(std::abs(__x)), std::arg(__x));
1052	1061	}
1053	1062
1054	1063	// log10
1055	1064
1056	1065	template <class _Tp>
1057		inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log10(const complex<_Tp>& __x) {
	1066	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> log10(const complex<_Tp>& __x) {
1058	1067	return std::log(__x) / std::log(_Tp(10));
1059	1068	}
1060	1069
1061	1070	// sqrt
1062	1071
1063	1072	template <class _Tp>
1064		_LIBCPP_HIDE_FROM_ABI complex<_Tp> sqrt(const complex<_Tp>& __x) {
	1073	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> sqrt(const complex<_Tp>& __x) {
1065	1074	if (std::isinf(__x.imag()))
1066	1075	return complex<_Tp>(_Tp(INFINITY), __x.imag());
1067	1076	if (std::isinf(__x.real())) {
...	...	@@ -1075,7 +1084,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> sqrt(const complex<_Tp>& __x) {
1075	1084	// exp
1076	1085
1077	1086	template <class _Tp>
1078		_LIBCPP_HIDE_FROM_ABI complex<_Tp> exp(const complex<_Tp>& __x) {
	1087	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> exp(const complex<_Tp>& __x) {
1079	1088	_Tp __i = __x.imag();
1080	1089	if (__i == 0) {
1081	1090	return complex<_Tp>(std::exp(__x.real()), std::copysign(_Tp(0), __x.imag()));
...	...	@@ -1097,24 +1106,27 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> exp(const complex<_Tp>& __x) {
1097	1106	// pow
1098	1107
1099	1108	template <class _Tp>
1100		inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> pow(const complex<_Tp>& __x, const complex<_Tp>& __y) {
	1109	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> pow(const complex<_Tp>& __x, const complex<_Tp>& __y) {
1101	1110	return std::exp(__y * std::log(__x));
1102	1111	}
1103	1112
1104	1113	template <class _Tp, class _Up, __enable_if_t<is_floating_point<_Tp>::value && is_floating_point<_Up>::value, int> = 0>
1105		inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> > pow(const complex<_Tp>& __x, const complex<_Up>& __y) {
	1114	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> >
	1115	pow(const complex<_Tp>& __x, const complex<_Up>& __y) {
1106	1116	typedef complex<__promote_t<_Tp, _Up> > result_type;
1107	1117	return std::pow(result_type(__x), result_type(__y));
1108	1118	}
1109	1119
1110	1120	template <class _Tp, class _Up, __enable_if_t<is_floating_point<_Tp>::value && is_arithmetic<_Up>::value, int> = 0>
1111		inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> > pow(const complex<_Tp>& __x, const _Up& __y) {
	1121	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> >
	1122	pow(const complex<_Tp>& __x, const _Up& __y) {
1112	1123	typedef complex<__promote_t<_Tp, _Up> > result_type;
1113	1124	return std::pow(result_type(__x), result_type(__y));
1114	1125	}
1115	1126
1116	1127	template <class _Tp, class _Up, __enable_if_t<is_arithmetic<_Tp>::value && is_floating_point<_Up>::value, int> = 0>
1117		inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> > pow(const _Tp& __x, const complex<_Up>& __y) {
	1128	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<__promote_t<_Tp, _Up> >
	1129	pow(const _Tp& __x, const complex<_Up>& __y) {
1118	1130	typedef complex<__promote_t<_Tp, _Up> > result_type;
1119	1131	return std::pow(result_type(__x), result_type(__y));
1120	1132	}
...	...	@@ -1129,7 +1141,7 @@ inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> __sqr(const complex<_Tp>& __x) {
1129	1141	// asinh
1130	1142
1131	1143	template <class _Tp>
1132		_LIBCPP_HIDE_FROM_ABI complex<_Tp> asinh(const complex<_Tp>& __x) {
	1144	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> asinh(const complex<_Tp>& __x) {
1133	1145	const _Tp __pi(atan2(+0., -0.));
1134	1146	if (std::isinf(__x.real())) {
1135	1147	if (std::isnan(__x.imag()))
...	...	@@ -1154,7 +1166,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> asinh(const complex<_Tp>& __x) {
1154	1166	// acosh
1155	1167
1156	1168	template <class _Tp>
1157		_LIBCPP_HIDE_FROM_ABI complex<_Tp> acosh(const complex<_Tp>& __x) {
	1169	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> acosh(const complex<_Tp>& __x) {
1158	1170	const _Tp __pi(atan2(+0., -0.));
1159	1171	if (std::isinf(__x.real())) {
1160	1172	if (std::isnan(__x.imag()))
...	...	@@ -1183,7 +1195,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> acosh(const complex<_Tp>& __x) {
1183	1195	// atanh
1184	1196
1185	1197	template <class _Tp>
1186		_LIBCPP_HIDE_FROM_ABI complex<_Tp> atanh(const complex<_Tp>& __x) {
	1198	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> atanh(const complex<_Tp>& __x) {
1187	1199	const _Tp __pi(atan2(+0., -0.));
1188	1200	if (std::isinf(__x.imag())) {
1189	1201	return complex<_Tp>(std::copysign(_Tp(0), __x.real()), std::copysign(__pi / _Tp(2), __x.imag()));
...	...	@@ -1209,7 +1221,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> atanh(const complex<_Tp>& __x) {
1209	1221	// sinh
1210	1222
1211	1223	template <class _Tp>
1212		_LIBCPP_HIDE_FROM_ABI complex<_Tp> sinh(const complex<_Tp>& __x) {
	1224	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> sinh(const complex<_Tp>& __x) {
1213	1225	if (std::isinf(__x.real()) && !std::isfinite(__x.imag()))
1214	1226	return complex<_Tp>(__x.real(), _Tp(NAN));
1215	1227	if (__x.real() == 0 && !std::isfinite(__x.imag()))
...	...	@@ -1222,7 +1234,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> sinh(const complex<_Tp>& __x) {
1222	1234	// cosh
1223	1235
1224	1236	template <class _Tp>
1225		_LIBCPP_HIDE_FROM_ABI complex<_Tp> cosh(const complex<_Tp>& __x) {
	1237	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> cosh(const complex<_Tp>& __x) {
1226	1238	if (std::isinf(__x.real()) && !std::isfinite(__x.imag()))
1227	1239	return complex<_Tp>(std::abs(__x.real()), _Tp(NAN));
1228	1240	if (__x.real() == 0 && !std::isfinite(__x.imag()))
...	...	@@ -1237,7 +1249,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> cosh(const complex<_Tp>& __x) {
1237	1249	// tanh
1238	1250
1239	1251	template <class _Tp>
1240		_LIBCPP_HIDE_FROM_ABI complex<_Tp> tanh(const complex<_Tp>& __x) {
	1252	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> tanh(const complex<_Tp>& __x) {
1241	1253	if (std::isinf(__x.real())) {
1242	1254	if (!std::isfinite(__x.imag()))
1243	1255	return complex<_Tp>(std::copysign(_Tp(1), __x.real()), _Tp(0));
...	...	@@ -1257,7 +1269,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> tanh(const complex<_Tp>& __x) {
1257	1269	// asin
1258	1270
1259	1271	template <class _Tp>
1260		_LIBCPP_HIDE_FROM_ABI complex<_Tp> asin(const complex<_Tp>& __x) {
	1272	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> asin(const complex<_Tp>& __x) {
1261	1273	complex<_Tp> __z = std::asinh(complex<_Tp>(-__x.imag(), __x.real()));
1262	1274	return complex<_Tp>(__z.imag(), -__z.real());
1263	1275	}
...	...	@@ -1265,7 +1277,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> asin(const complex<_Tp>& __x) {
1265	1277	// acos
1266	1278
1267	1279	template <class _Tp>
1268		_LIBCPP_HIDE_FROM_ABI complex<_Tp> acos(const complex<_Tp>& __x) {
	1280	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> acos(const complex<_Tp>& __x) {
1269	1281	const _Tp __pi(atan2(+0., -0.));
1270	1282	if (std::isinf(__x.real())) {
1271	1283	if (std::isnan(__x.imag()))
...	...	@@ -1297,7 +1309,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> acos(const complex<_Tp>& __x) {
1297	1309	// atan
1298	1310
1299	1311	template <class _Tp>
1300		_LIBCPP_HIDE_FROM_ABI complex<_Tp> atan(const complex<_Tp>& __x) {
	1312	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> atan(const complex<_Tp>& __x) {
1301	1313	complex<_Tp> __z = std::atanh(complex<_Tp>(-__x.imag(), __x.real()));
1302	1314	return complex<_Tp>(__z.imag(), -__z.real());
1303	1315	}
...	...	@@ -1305,7 +1317,7 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> atan(const complex<_Tp>& __x) {
1305	1317	// sin
1306	1318
1307	1319	template <class _Tp>
1308		_LIBCPP_HIDE_FROM_ABI complex<_Tp> sin(const complex<_Tp>& __x) {
	1320	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> sin(const complex<_Tp>& __x) {
1309	1321	complex<_Tp> __z = std::sinh(complex<_Tp>(-__x.imag(), __x.real()));
1310	1322	return complex<_Tp>(__z.imag(), -__z.real());
1311	1323	}
...	...	@@ -1313,14 +1325,14 @@ _LIBCPP_HIDE_FROM_ABI complex<_Tp> sin(const complex<_Tp>& __x) {
1313	1325	// cos
1314	1326
1315	1327	template <class _Tp>
1316		inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> cos(const complex<_Tp>& __x) {
	1328	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI complex<_Tp> cos(const complex<_Tp>& __x) {
1317	1329	return std::cosh(complex<_Tp>(-__x.imag(), __x.real()));
1318	1330	}
1319	1331
1320	1332	// tan
1321	1333
1322	1334	template <class _Tp>
1323		_LIBCPP_HIDE_FROM_ABI complex<_Tp> tan(const complex<_Tp>& __x) {
	1335	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI complex<_Tp> tan(const complex<_Tp>& __x) {
1324	1336	complex<_Tp> __z = std::tanh(complex<_Tp>(-__x.imag(), __x.real()));
1325	1337	return complex<_Tp>(__z.imag(), -__z.real());
1326	1338	}
...	...	@@ -1398,7 +1410,7 @@ struct tuple_element<_Ip, complex<_Tp>> {
1398	1410	};
1399	1411
1400	1412	template <size_t _Ip, class _Xp>
1401		_LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>& __z) noexcept {
	1413	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>& __z) noexcept {
1402	1414	static_assert(_Ip < 2, "Index value is out of range.");
1403	1415	if constexpr (_Ip == 0) {
1404	1416	return __z.__re_;
...	...	@@ -1408,7 +1420,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Xp& get(complex<_Xp>& __z) noexcept {
1408	1420	}
1409	1421
1410	1422	template <size_t _Ip, class _Xp>
1411		_LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&& __z) noexcept {
	1423	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&& __z) noexcept {
1412	1424	static_assert(_Ip < 2, "Index value is out of range.");
1413	1425	if constexpr (_Ip == 0) {
1414	1426	return std::move(__z.__re_);
...	...	@@ -1418,7 +1430,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr _Xp&& get(complex<_Xp>&& __z) noexcept {
1418	1430	}
1419	1431
1420	1432	template <size_t _Ip, class _Xp>
1421		_LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>& __z) noexcept {
	1433	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>& __z) noexcept {
1422	1434	static_assert(_Ip < 2, "Index value is out of range.");
1423	1435	if constexpr (_Ip == 0) {
1424	1436	return __z.__re_;
...	...	@@ -1428,7 +1440,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr const _Xp& get(const complex<_Xp>& __z) noexcept
1428	1440	}
1429	1441
1430	1442	template <size_t _Ip, class _Xp>
1431		_LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&& __z) noexcept {
	1443	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI constexpr const _Xp&& get(const complex<_Xp>&& __z) noexcept {
1432	1444	static_assert(_Ip < 2, "Index value is out of range.");
1433	1445	if constexpr (_Ip == 0) {
1434	1446	return std::move(__z.__re_);

lib/libcxx/include/complex.h+10-10

...	...	@@ -18,19 +18,19 @@
18	18	*/
19	19
20	20	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
21		# include <__cxx03/complex.h>
	21	# include <__cxx03/__config>
22	22	#else
23	23	# include <__config>
	24	#endif
24	25
25		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
26		# pragma GCC system_header
27		# endif
	26	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	27	# pragma GCC system_header
	28	#endif
28	29
29		# ifdef __cplusplus
30		# include <complex>
31		# elif __has_include_next(<complex.h>)
32		# include_next <complex.h>
33		# endif
34		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	30	#ifdef __cplusplus
	31	# include <complex>
	32	#elif __has_include_next(<complex.h>)
	33	# include_next <complex.h>
	34	#endif
35	35
36	36	#endif // _LIBCPP_COMPLEX_H

lib/libcxx/include/condition_variable+3-3

...	...	@@ -206,14 +206,14 @@ public:
206	206	# if _LIBCPP_STD_VER >= 20
207	207
208	208	template <class _Lock, class _Predicate>
209		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool wait(_Lock& __lock, stop_token __stoken, _Predicate __pred);
	209	_LIBCPP_HIDE_FROM_ABI bool wait(_Lock& __lock, stop_token __stoken, _Predicate __pred);
210	210
211	211	template <class _Lock, class _Clock, class _Duration, class _Predicate>
212		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool wait_until(
	212	_LIBCPP_HIDE_FROM_ABI bool wait_until(
213	213	_Lock& __lock, stop_token __stoken, const chrono::time_point<_Clock, _Duration>& __abs_time, _Predicate __pred);
214	214
215	215	template <class _Lock, class _Rep, class _Period, class _Predicate>
216		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
	216	_LIBCPP_HIDE_FROM_ABI bool
217	217	wait_for(_Lock& __lock, stop_token __stoken, const chrono::duration<_Rep, _Period>& __rel_time, _Predicate __pred);
218	218
219	219	# endif // _LIBCPP_STD_VER >= 20

lib/libcxx/include/cstdalign+3-10

...	...	@@ -43,17 +43,10 @@ Macros:
43	43	# undef __alignof_is_defined
44	44	# define __alignof_is_defined 1
45	45
46		# if _LIBCPP_STD_VER >= 20
47
48		using __standard_header_cstdalign _LIBCPP_DEPRECATED_("removed in C++20.") _LIBCPP_NODEBUG = void;
49		using __use_standard_header_cstdalign _LIBCPP_NODEBUG = __standard_header_cstdalign;
50
51		# elif _LIBCPP_STD_VER >= 17
52
53		using __standard_header_cstdalign _LIBCPP_DEPRECATED _LIBCPP_NODEBUG = void;
54		using __use_standard_header_cstdalign _LIBCPP_NODEBUG = __standard_header_cstdalign;
55
	46	# if _LIBCPP_STD_VER >= 17 && !__building_module(std) && _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS
	47	# warning <cstdalign> is deprecated in C++17 and removed in C++20.
56	48	# endif
	49
57	50	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
58	51
59	52	#endif // _LIBCPP_CSTDALIGN

lib/libcxx/include/cstdbool+3-10

...	...	@@ -31,17 +31,10 @@ Macros:
31	31	# undef __bool_true_false_are_defined
32	32	# define __bool_true_false_are_defined 1
33	33
34		# if _LIBCPP_STD_VER >= 20
35
36		using __standard_header_cstdbool _LIBCPP_DEPRECATED_("removed in C++20.") _LIBCPP_NODEBUG = void;
37		using __use_standard_header_cstdbool _LIBCPP_NODEBUG = __standard_header_cstdbool;
38
39		# elif _LIBCPP_STD_VER >= 17
40
41		using __standard_header_cstdbool _LIBCPP_DEPRECATED _LIBCPP_NODEBUG = void;
42		using __use_standard_header_cstdbool _LIBCPP_NODEBUG = __standard_header_cstdbool;
43
	34	# if _LIBCPP_STD_VER >= 17 && !__building_module(std) && _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS
	35	# warning <cstdbool> is deprecated in C++17 and removed in C++20.
44	36	# endif
	37
45	38	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
46	39
47	40	#endif // _LIBCPP_CSTDBOOL

lib/libcxx/include/ctgmath+2-11

...	...	@@ -28,17 +28,8 @@
28	28	# pragma GCC system_header
29	29	# endif
30	30
31		# if _LIBCPP_STD_VER >= 20
32
33		using __standard_header_ctgmath
34		_LIBCPP_DEPRECATED_("removed in C++20. Include <cmath> and <complex> instead.") _LIBCPP_NODEBUG = void;
35		using __use_standard_header_ctgmath _LIBCPP_NODEBUG = __standard_header_ctgmath;
36
37		# elif _LIBCPP_STD_VER >= 17
38
39		using __standard_header_ctgmath _LIBCPP_DEPRECATED_("Include <cmath> and <complex> instead.") _LIBCPP_NODEBUG = void;
40		using __use_standard_header_ctgmath _LIBCPP_NODEBUG = __standard_header_ctgmath;
41
	31	# if _LIBCPP_STD_VER >= 17 && !__building_module(std) && _LIBCPP_DIAGNOSE_DEPRECATED_HEADERS
	32	# warning <ctgmath> is deprecated in C++17 and removed in C++20. Include <cmath> and <complex> instead.
42	33	# endif
43	34
44	35	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)

lib/libcxx/include/ctype.h+24-24

...	...	@@ -30,36 +30,36 @@ int toupper(int c);
30	30	*/
31	31
32	32	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
33		# include <__cxx03/ctype.h>
	33	# include <__cxx03/__config>
34	34	#else
35	35	# include <__config>
	36	#endif
36	37
37		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
38		# pragma GCC system_header
39		# endif
	38	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	39	# pragma GCC system_header
	40	#endif
40	41
41		# if __has_include_next(<ctype.h>)
42		# include_next <ctype.h>
43		# endif
	42	#if __has_include_next(<ctype.h>)
	43	# include_next <ctype.h>
	44	#endif
44	45
45		# ifdef __cplusplus
	46	#ifdef __cplusplus
46	47
47		# undef isalnum
48		# undef isalpha
49		# undef isblank
50		# undef iscntrl
51		# undef isdigit
52		# undef isgraph
53		# undef islower
54		# undef isprint
55		# undef ispunct
56		# undef isspace
57		# undef isupper
58		# undef isxdigit
59		# undef tolower
60		# undef toupper
	48	# undef isalnum
	49	# undef isalpha
	50	# undef isblank
	51	# undef iscntrl
	52	# undef isdigit
	53	# undef isgraph
	54	# undef islower
	55	# undef isprint
	56	# undef ispunct
	57	# undef isspace
	58	# undef isupper
	59	# undef isxdigit
	60	# undef tolower
	61	# undef toupper
61	62
62		# endif
63		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	63	#endif
64	64
65	65	#endif // _LIBCPP_CTYPE_H

lib/libcxx/include/cwchar+2-2

...	...	@@ -231,8 +231,8 @@ __constexpr_wmemcmp(const wchar_t* __lhs, const wchar_t* __rhs, size_t __count)
231	231
232	232	template <class _Tp, class _Up>
233	233	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _Tp* __constexpr_wmemchr(_Tp* __str, _Up __value, size_t __count) {
234		static_assert(sizeof(_Tp) == sizeof(wchar_t)&& _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t) &&
235		__libcpp_is_trivially_equality_comparable<_Tp, _Tp>::value,
	234	static_assert(sizeof(_Tp) == sizeof(wchar_t) && _LIBCPP_ALIGNOF(_Tp) >= _LIBCPP_ALIGNOF(wchar_t) &&
	235	__is_trivially_equality_comparable_v<_Tp, _Tp>,
236	236	"Calling wmemchr on non-trivially equality comparable types is unsafe.");
237	237
238	238	# if __has_builtin(__builtin_wmemchr)

lib/libcxx/include/deque+109-225

...	...	@@ -191,9 +191,9 @@ template <class T, class Allocator, class Predicate>
191	191	# include <__algorithm/min.h>
192	192	# include <__algorithm/move.h>
193	193	# include <__algorithm/move_backward.h>
	194	# include <__algorithm/ranges_copy_n.h>
194	195	# include <__algorithm/remove.h>
195	196	# include <__algorithm/remove_if.h>
196		# include <__algorithm/unwrap_iter.h>
197	197	# include <__assert>
198	198	# include <__config>
199	199	# include <__debug_utils/sanitizers.h>
...	...	@@ -220,21 +220,19 @@ template <class T, class Allocator, class Predicate>
220	220	# include <__ranges/concepts.h>
221	221	# include <__ranges/container_compatible_range.h>
222	222	# include <__ranges/from_range.h>
223		# include <__ranges/size.h>
224	223	# include <__split_buffer>
225	224	# include <__type_traits/conditional.h>
226	225	# include <__type_traits/container_traits.h>
227		# include <__type_traits/disjunction.h>
228	226	# include <__type_traits/enable_if.h>
229	227	# include <__type_traits/is_allocator.h>
230	228	# include <__type_traits/is_convertible.h>
231	229	# include <__type_traits/is_nothrow_assignable.h>
232	230	# include <__type_traits/is_nothrow_constructible.h>
233		# include <__type_traits/is_replaceable.h>
234	231	# include <__type_traits/is_same.h>
235	232	# include <__type_traits/is_swappable.h>
236	233	# include <__type_traits/is_trivially_relocatable.h>
237	234	# include <__type_traits/type_identity.h>
	235	# include <__utility/exception_guard.h>
238	236	# include <__utility/forward.h>
239	237	# include <__utility/move.h>
240	238	# include <__utility/pair.h>
...	...	@@ -461,9 +459,8 @@ private:
461	459	__deque_iterator<_ValueType, _Pointer, _Reference, _MapPointer, _DiffType, _BlockSize>;
462	460
463	461	public:
464		using __is_segmented_iterator _LIBCPP_NODEBUG = true_type;
465		using __segment_iterator _LIBCPP_NODEBUG = _MapPointer;
466		using __local_iterator _LIBCPP_NODEBUG = _Pointer;
	462	using __segment_iterator _LIBCPP_NODEBUG = _MapPointer;
	463	using __local_iterator _LIBCPP_NODEBUG = _Pointer;
467	464
468	465	static _LIBCPP_HIDE_FROM_ABI __segment_iterator __segment(_Iterator __iter) { return __iter.__m_iter_; }
469	466	static _LIBCPP_HIDE_FROM_ABI __local_iterator __local(_Iterator __iter) { return __iter.__ptr_; }
...	...	@@ -488,6 +485,9 @@ const _DiffType __deque_iterator<_ValueType, _Pointer, _Reference, _MapPointer,
488	485
489	486	template <class _Tp, class _Allocator /= allocator<_Tp>/>
490	487	class deque {
	488	template <class _Up, class _Alloc>
	489	using __split_buffer _LIBCPP_NODEBUG = std::__split_buffer<_Up, _Alloc, __split_buffer_pointer_layout>;
	490
491	491	public:
492	492	// types:
493	493
...	...	@@ -531,10 +531,6 @@ public:
531	531	__libcpp_is_trivially_relocatable<__map>::value && __libcpp_is_trivially_relocatable<allocator_type>::value,
532	532	deque,
533	533	void>;
534		using __replaceable _LIBCPP_NODEBUG =
535		__conditional_t<__is_replaceable_v<__map> && __container_allocator_is_replaceable<__alloc_traits>::value,
536		deque,
537		void>;
538	534
539	535	static_assert(is_nothrow_default_constructible<allocator_type>::value ==
540	536	is_nothrow_default_constructible<__pointer_allocator>::value,
...	...	@@ -635,7 +631,7 @@ public:
635	631	# endif
636	632	_LIBCPP_HIDE_FROM_ABI deque(size_type __n, const value_type& __v);
637	633
638		template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
	634	template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0>
639	635	_LIBCPP_HIDE_FROM_ABI deque(size_type __n, const value_type& __v, const allocator_type& __a)
640	636	: __map_(__pointer_allocator(__a)), __start_(0), __size_(0), __alloc_(__a) {
641	637	__annotate_new(0);
...	...	@@ -703,8 +699,16 @@ public:
703	699	__assign_with_size_random_access(ranges::begin(__range), __n);
704	700
705	701	} else if constexpr (ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>) {
706		auto __n = static_cast<size_type>(ranges::distance(__range));
707		__assign_with_size(ranges::begin(__range), __n);
	702	auto __n = ranges::distance(__range);
	703	auto __first = ranges::begin(__range);
	704
	705	auto __result = std::ranges::copy_n(std::move(__first), std::min<size_t>(__n, size()), begin());
	706
	707	if (static_cast<size_type>(__n) > size()) {
	708	__append_with_size(std::move(__result.in), __n - size());
	709	} else {
	710	__erase_to_end(__result.out);
	711	}
708	712
709	713	} else {
710	714	__assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
...	...	@@ -720,45 +724,53 @@ public:
720	724
721	725	// iterators:
722	726
723		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
	727	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
724	728	__map_pointer __mp = __map_.begin() + __start_ / __block_size;
725	729	return iterator(__mp, __map_.empty() ? 0 : *__mp + __start_ % __block_size);
726	730	}
727	731
728		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
	732	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
729	733	__map_const_pointer __mp = static_cast<__map_const_pointer>(__map_.begin() + __start_ / __block_size);
730	734	return const_iterator(__mp, __map_.empty() ? 0 : *__mp + __start_ % __block_size);
731	735	}
732	736
733		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
	737	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
734	738	size_type __p = size() + __start_;
735	739	__map_pointer __mp = __map_.begin() + __p / __block_size;
736	740	return iterator(__mp, __map_.empty() ? 0 : *__mp + __p % __block_size);
737	741	}
738	742
739		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
	743	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
740	744	size_type __p = size() + __start_;
741	745	__map_const_pointer __mp = static_cast<__map_const_pointer>(__map_.begin() + __p / __block_size);
742	746	return const_iterator(__mp, __map_.empty() ? 0 : *__mp + __p % __block_size);
743	747	}
744	748
745		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
746		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
747		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
748		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	749	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
	750	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	751	return const_reverse_iterator(end());
	752	}
	753	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
	754	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	755	return const_reverse_iterator(begin());
	756	}
749	757
750		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
751		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
752		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
753		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	758	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
	759	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
	760	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
	761	return const_reverse_iterator(end());
	762	}
	763	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
	764	return const_reverse_iterator(begin());
	765	}
754	766
755	767	// capacity:
756		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size(); }
	768	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size(); }
757	769
758	770	_LIBCPP_HIDE_FROM_ABI size_type& __size() _NOEXCEPT { return __size_; }
759	771	_LIBCPP_HIDE_FROM_ABI const size_type& __size() const _NOEXCEPT { return __size_; }
760	772
761		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	773	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
762	774	return std::min<size_type>(__alloc_traits::max_size(__alloc()), numeric_limits<difference_type>::max());
763	775	}
764	776	_LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
...	...	@@ -767,18 +779,22 @@ public:
767	779	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return size() == 0; }
768	780
769	781	// element access:
770		_LIBCPP_HIDE_FROM_ABI reference operator[](size_type __i) _NOEXCEPT;
771		_LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __i) const _NOEXCEPT;
772		_LIBCPP_HIDE_FROM_ABI reference at(size_type __i);
773		_LIBCPP_HIDE_FROM_ABI const_reference at(size_type __i) const;
774		_LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT;
775		_LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT;
776		_LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT;
777		_LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT;
	782	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference operator[](size_type __i) _NOEXCEPT;
	783	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __i) const _NOEXCEPT;
	784	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference at(size_type __i);
	785	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __i) const;
	786	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference front() _NOEXCEPT;
	787	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT;
	788	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference back() _NOEXCEPT;
	789	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT;
778	790
779	791	// 23.2.2.3 modifiers:
780	792	_LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __v);
781	793	_LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __v);
	794
	795	template <class... _Args>
	796	_LIBCPP_HIDE_FROM_ABI iterator __emplace(const_iterator __p, _Args&&... __args);
	797
782	798	# ifndef _LIBCPP_CXX03_LANG
783	799	# if _LIBCPP_STD_VER >= 17
784	800	template <class... _Args>
...	...	@@ -791,8 +807,11 @@ public:
791	807	template <class... _Args>
792	808	_LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
793	809	# endif
	810
794	811	template <class... _Args>
795		_LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
	812	_LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args) {
	813	return __emplace(__p, std::forward<_Args>(__args)...);
	814	}
796	815
797	816	_LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __v);
798	817	_LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __v);
...	...	@@ -805,17 +824,21 @@ public:
805	824
806	825	template <_ContainerCompatibleRange<_Tp> _Range>
807	826	_LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
808		insert_range(end(), std::forward<_Range>(__range));
	827	if constexpr (ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>) {
	828	__append_with_size(ranges::begin(__range), ranges::distance(__range));
	829	} else {
	830	__append_with_sentinel(ranges::begin(__range), ranges::end(__range));
	831	}
809	832	}
810	833	# endif
811	834
812		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v);
	835	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) { return __emplace(__p, std::move(__v)); }
813	836
814	837	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
815	838	return insert(__p, __il.begin(), __il.end());
816	839	}
817	840	# endif // _LIBCPP_CXX03_LANG
818		_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v);
	841	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) { return __emplace(__p, __v); }
819	842	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __v);
820	843	template <class _InputIter, __enable_if_t<__has_exactly_input_iterator_category<_InputIter>::value, int> = 0>
821	844	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InputIter __f, _InputIter __l);
...	...	@@ -1109,7 +1132,7 @@ public:
1109	1132	// This function tests deque object annotations.
1110	1133	if (empty()) {
1111	1134	for (__map_const_iterator __it = __map_.begin(); __it != __map_.end(); ++__it) {
1112		if (!__sanitizer_verify_double_ended_contiguous_container(
	1135	if (!::__sanitizer_verify_double_ended_contiguous_container(
1113	1136	std::__to_address(*__it),
1114	1137	std::__to_address(*__it),
1115	1138	std::__to_address(*__it),
...	...	@@ -1141,7 +1164,7 @@ public:
1141	1164
1142	1165	// Is the block before or after deque blocks that contain elements?
1143	1166	if (__it < __first_mp \|\| __it > __last_mp) {
1144		if (!__sanitizer_verify_double_ended_contiguous_container(
	1167	if (!::__sanitizer_verify_double_ended_contiguous_container(
1145	1168	std::__to_address(*__it),
1146	1169	std::__to_address(*__it),
1147	1170	std::__to_address(*__it),
...	...	@@ -1151,7 +1174,7 @@ public:
1151	1174	const void* __containers_buffer_beg = (__it == __first_mp) ? __p_beg : (const void)std::__to_address(__it);
1152	1175	const void* __containers_buffer_end =
1153	1176	(__it == __last_mp) ? __p_end : (const void)std::__to_address(__it + __block_size);
1154		if (!__sanitizer_verify_double_ended_contiguous_container(
	1177	if (!::__sanitizer_verify_double_ended_contiguous_container(
1155	1178	std::__to_address(*__it),
1156	1179	__containers_buffer_beg,
1157	1180	__containers_buffer_end,
...	...	@@ -1191,7 +1214,7 @@ private:
1191	1214
1192	1215	template <class _RandomAccessIterator>
1193	1216	_LIBCPP_HIDE_FROM_ABI void __assign_with_size_random_access(_RandomAccessIterator __f, difference_type __n);
1194		template <class _Iterator>
	1217	template <class _AlgPolicy, class _Iterator>
1195	1218	_LIBCPP_HIDE_FROM_ABI void __assign_with_size(_Iterator __f, difference_type __n);
1196	1219
1197	1220	template <class _Iterator, class _Sentinel>
...	...	@@ -1239,8 +1262,8 @@ private:
1239	1262	clear();
1240	1263	shrink_to_fit();
1241	1264	}
1242		__alloc() = __c.__alloc();
1243		__map_.__alloc_ = __c.__map_.__alloc_;
	1265	__alloc() = __c.__alloc();
	1266	__map_.__get_allocator() = __c.__map_.__get_allocator();
1244	1267	}
1245	1268
1246	1269	_LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const deque&, false_type) {}
...	...	@@ -1256,22 +1279,22 @@ _LIBCPP_CONSTEXPR const typename allocator_traits<_Alloc>::difference_type deque
1256	1279
1257	1280	# if _LIBCPP_STD_VER >= 17
1258	1281	template <class _InputIterator,
1259		class _Alloc = allocator<__iter_value_type<_InputIterator>>,
	1282	class _Alloc = allocator<__iterator_value_type<_InputIterator>>,
1260	1283	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1261		class = enable_if_t<__is_allocator<_Alloc>::value> >
1262		deque(_InputIterator, _InputIterator) -> deque<__iter_value_type<_InputIterator>, _Alloc>;
	1284	class = enable_if_t<__is_allocator_v<_Alloc>>>
	1285	deque(_InputIterator, _InputIterator) -> deque<__iterator_value_type<_InputIterator>, _Alloc>;
1263	1286
1264	1287	template <class _InputIterator,
1265	1288	class _Alloc,
1266	1289	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1267		class = enable_if_t<__is_allocator<_Alloc>::value> >
1268		deque(_InputIterator, _InputIterator, _Alloc) -> deque<__iter_value_type<_InputIterator>, _Alloc>;
	1290	class = enable_if_t<__is_allocator_v<_Alloc>>>
	1291	deque(_InputIterator, _InputIterator, _Alloc) -> deque<__iterator_value_type<_InputIterator>, _Alloc>;
1269	1292	# endif
1270	1293
1271	1294	# if _LIBCPP_STD_VER >= 23
1272	1295	template <ranges::input_range _Range,
1273	1296	class _Alloc = allocator<ranges::range_value_t<_Range>>,
1274		class = enable_if_t<__is_allocator<_Alloc>::value> >
	1297	class = enable_if_t<__is_allocator_v<_Alloc>>>
1275	1298	deque(from_range_t, _Range&&, _Alloc = _Alloc()) -> deque<ranges::range_value_t<_Range>, _Alloc>;
1276	1299	# endif
1277	1300
...	...	@@ -1319,7 +1342,7 @@ deque<_Tp, _Allocator>::deque(const deque& __c)
1319	1342	: __map_(__pointer_allocator(__alloc_traits::select_on_container_copy_construction(__c.__alloc()))),
1320	1343	__start_(0),
1321	1344	__size_(0),
1322		__alloc_(__map_.__alloc_) {
	1345	__alloc_(__map_.__get_allocator()) {
1323	1346	__annotate_new(0);
1324	1347	__append(__c.begin(), __c.end());
1325	1348	}
...	...	@@ -1451,24 +1474,6 @@ deque<_Tp, _Allocator>::__assign_with_size_random_access(_RandomAccessIterator _
1451	1474	__erase_to_end(std::copy_n(__f, __n, begin()));
1452	1475	}
1453	1476
1454		template <class _Tp, class _Allocator>
1455		template <class _Iterator>
1456		_LIBCPP_HIDE_FROM_ABI void deque<_Tp, _Allocator>::__assign_with_size(_Iterator __f, difference_type __n) {
1457		if (static_cast<size_type>(__n) > size()) {
1458		auto __added_size = __n - size();
1459
1460		auto __i = begin();
1461		for (auto __count = size(); __count != 0; --__count) {
1462		__i++ = __f++;
1463		}
1464
1465		__append_with_size(__f, __added_size);
1466
1467		} else {
1468		__erase_to_end(std::copy_n(__f, __n, begin()));
1469		}
1470		}
1471
1472	1477	template <class _Tp, class _Allocator>
1473	1478	void deque<_Tp, _Allocator>::assign(size_type __n, const value_type& __v) {
1474	1479	if (__n > size()) {
...	...	@@ -1661,56 +1666,11 @@ deque<_Tp, _Allocator>::emplace_front(_Args&&... __args) {
1661	1666	return *begin();
1662	1667	# endif
1663	1668	}
1664
1665		template <class _Tp, class _Allocator>
1666		typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, value_type&& __v) {
1667		size_type __pos = __p - begin();
1668		size_type __to_end = size() - __pos;
1669		allocator_type& __a = __alloc();
1670		if (__pos < __to_end) { // insert by shifting things backward
1671		if (__front_spare() == 0)
1672		__add_front_capacity();
1673		// __front_spare() >= 1
1674		__annotate_increase_front(1);
1675		if (__pos == 0) {
1676		__alloc_traits::construct(__a, std::addressof(*--begin()), std::move(__v));
1677		--__start_;
1678		++__size();
1679		} else {
1680		iterator __b = begin();
1681		iterator __bm1 = std::prev(__b);
1682		__alloc_traits::construct(__a, std::addressof(__bm1), std::move(__b));
1683		--__start_;
1684		++__size();
1685		if (__pos > 1)
1686		__b = std::move(std::next(__b), __b + __pos, __b);
1687		*__b = std::move(__v);
1688		}
1689		} else { // insert by shifting things forward
1690		if (__back_spare() == 0)
1691		__add_back_capacity();
1692		// __back_capacity >= 1
1693		__annotate_increase_back(1);
1694		size_type __de = size() - __pos;
1695		if (__de == 0) {
1696		__alloc_traits::construct(__a, std::addressof(*end()), std::move(__v));
1697		++__size();
1698		} else {
1699		iterator __e = end();
1700		iterator __em1 = std::prev(__e);
1701		__alloc_traits::construct(__a, std::addressof(__e), std::move(__em1));
1702		++__size();
1703		if (__de > 1)
1704		__e = std::move_backward(__e - __de, __em1, __e);
1705		*--__e = std::move(__v);
1706		}
1707		}
1708		return begin() + __pos;
1709		}
	1669	# endif // _LIBCPP_CXX03_LANG
1710	1670
1711	1671	template <class _Tp, class _Allocator>
1712	1672	template <class... _Args>
1713		typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::emplace(const_iterator __p, _Args&&... __args) {
	1673	typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::__emplace(const_iterator __p, _Args&&... __args) {
1714	1674	size_type __pos = __p - begin();
1715	1675	size_type __to_end = size() - __pos;
1716	1676	allocator_type& __a = __alloc();
...	...	@@ -1757,60 +1717,6 @@ typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::emplace(const_
1757	1717	return begin() + __pos;
1758	1718	}
1759	1719
1760		# endif // _LIBCPP_CXX03_LANG
1761
1762		template <class _Tp, class _Allocator>
1763		typename deque<_Tp, _Allocator>::iterator deque<_Tp, _Allocator>::insert(const_iterator __p, const value_type& __v) {
1764		size_type __pos = __p - begin();
1765		size_type __to_end = size() - __pos;
1766		allocator_type& __a = __alloc();
1767		if (__pos < __to_end) { // insert by shifting things backward
1768		if (__front_spare() == 0)
1769		__add_front_capacity();
1770		// __front_spare() >= 1
1771		__annotate_increase_front(1);
1772		if (__pos == 0) {
1773		__alloc_traits::construct(__a, std::addressof(*--begin()), __v);
1774		--__start_;
1775		++__size();
1776		} else {
1777		const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
1778		iterator __b = begin();
1779		iterator __bm1 = std::prev(__b);
1780		if (__vt == pointer_traits<const_pointer>::pointer_to(*__b))
1781		__vt = pointer_traits<const_pointer>::pointer_to(*__bm1);
1782		__alloc_traits::construct(__a, std::addressof(__bm1), std::move(__b));
1783		--__start_;
1784		++__size();
1785		if (__pos > 1)
1786		__b = __move_and_check(std::next(__b), __b + __pos, __b, __vt);
1787		__b = __vt;
1788		}
1789		} else { // insert by shifting things forward
1790		if (__back_spare() == 0)
1791		__add_back_capacity();
1792		// __back_capacity >= 1
1793		__annotate_increase_back(1);
1794		size_type __de = size() - __pos;
1795		if (__de == 0) {
1796		__alloc_traits::construct(__a, std::addressof(*end()), __v);
1797		++__size();
1798		} else {
1799		const_pointer __vt = pointer_traits<const_pointer>::pointer_to(__v);
1800		iterator __e = end();
1801		iterator __em1 = std::prev(__e);
1802		if (__vt == pointer_traits<const_pointer>::pointer_to(*__em1))
1803		__vt = pointer_traits<const_pointer>::pointer_to(*__e);
1804		__alloc_traits::construct(__a, std::addressof(__e), std::move(__em1));
1805		++__size();
1806		if (__de > 1)
1807		__e = __move_backward_and_check(__e - __de, __em1, __e, __vt);
1808		--__e = __vt;
1809		}
1810		}
1811		return begin() + __pos;
1812		}
1813
1814	1720	template <class _Tp, class _Allocator>
1815	1721	typename deque<_Tp, _Allocator>::iterator
1816	1722	deque<_Tp, _Allocator>::insert(const_iterator __p, size_type __n, const value_type& __v) {
...	...	@@ -1874,9 +1780,9 @@ template <class _Tp, class _Allocator>
1874	1780	template <class _Iterator, class _Sentinel>
1875	1781	_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
1876	1782	deque<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l) {
1877		__split_buffer<value_type, allocator_type&> __buf(__alloc());
	1783	__split_buffer<value_type, allocator_type> __buf(__alloc());
1878	1784	__buf.__construct_at_end_with_sentinel(std::move(__f), std::move(__l));
1879		typedef typename __split_buffer<value_type, allocator_type&>::iterator __bi;
	1785	typedef typename __split_buffer<value_type, allocator_type>::iterator __bi;
1880	1786	return insert(__p, move_iterator<__bi>(__buf.begin()), move_iterator<__bi>(__buf.end()));
1881	1787	}
1882	1788
...	...	@@ -1891,9 +1797,9 @@ template <class _Tp, class _Allocator>
1891	1797	template <class _Iterator>
1892	1798	_LIBCPP_HIDE_FROM_ABI typename deque<_Tp, _Allocator>::iterator
1893	1799	deque<_Tp, _Allocator>::__insert_with_size(const_iterator __p, _Iterator __f, size_type __n) {
1894		__split_buffer<value_type, allocator_type&> __buf(__n, 0, __alloc());
1895		__buf.__construct_at_end_with_size(__f, __n);
1896		typedef typename __split_buffer<value_type, allocator_type&>::iterator __fwd;
	1800	__split_buffer<value_type, allocator_type> __buf(__n, 0, __alloc());
	1801	__buf.__construct_at_end_with_size(std::move(__f), __n);
	1802	typedef typename __split_buffer<value_type, allocator_type>::iterator __fwd;
1897	1803	return insert(__p, move_iterator<__fwd>(__buf.begin()), move_iterator<__fwd>(__buf.end()));
1898	1804	}
1899	1805
...	...	@@ -2071,8 +1977,8 @@ void deque<_Tp, _Allocator>::__add_front_capacity() {
2071	1977	}
2072	1978	// Else need to allocate 1 buffer, and we need to reallocate __map_.
2073	1979	else {
2074		__split_buffer<pointer, __pointer_allocator&> __buf(
2075		std::max<size_type>(2 * __map_.capacity(), 1), 0, __map_.__alloc_);
	1980	__split_buffer<pointer, __pointer_allocator> __buf(
	1981	std::max<size_type>(2 * __map_.capacity(), 1), 0, __map_.__get_allocator());
2076	1982
2077	1983	typedef __allocator_destructor<_Allocator> _Dp;
2078	1984	unique_ptr<pointer, _Dp> __hold(__alloc_traits::allocate(__a, __block_size), _Dp(__a, __block_size));
...	...	@@ -2081,10 +1987,7 @@ void deque<_Tp, _Allocator>::__add_front_capacity() {
2081	1987
2082	1988	for (__map_pointer __i = __map_.begin(); __i != __map_.end(); ++__i)
2083	1989	__buf.emplace_back(*__i);
2084		std::swap(__map_.__first_, __buf.__first_);
2085		std::swap(__map_.__begin_, __buf.__begin_);
2086		std::swap(__map_.__end_, __buf.__end_);
2087		std::swap(__map_.__cap_, __buf.__cap_);
	1990	__map_.__swap_without_allocator(__buf);
2088	1991	__start_ = __map_.size() == 1 ? __block_size / 2 : __start_ + __block_size;
2089	1992	}
2090	1993	__annotate_whole_block(0, __asan_poison);
...	...	@@ -2134,34 +2037,26 @@ void deque<_Tp, _Allocator>::__add_front_capacity(size_type __n) {
2134	2037	// Else need to allocate __nb buffers, and we need to reallocate __map_.
2135	2038	else {
2136	2039	size_type __ds = (__nb + __back_capacity) * __block_size - __map_.empty();
2137		__split_buffer<pointer, __pointer_allocator&> __buf(
2138		std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()), 0, __map_.__alloc_);
2139		# if _LIBCPP_HAS_EXCEPTIONS
2140		try {
2141		# endif // _LIBCPP_HAS_EXCEPTIONS
2142		for (; __nb > 0; --__nb) {
2143		__buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
2144		// ASan: this is empty container, we have to poison whole block
2145		__annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
2146		}
2147		# if _LIBCPP_HAS_EXCEPTIONS
2148		} catch (...) {
	2040	__split_buffer<pointer, __pointer_allocator> __buf(
	2041	std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()), 0, __map_.__get_allocator());
	2042	auto __guard = std::__make_exception_guard([&] {
2149	2043	__annotate_delete();
2150	2044	for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
2151	2045	__alloc_traits::deallocate(__a, *__i, __block_size);
2152		throw;
	2046	});
	2047	for (; __nb > 0; --__nb) {
	2048	__buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
	2049	// ASan: this is empty container, we have to poison whole block
	2050	__annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
2153	2051	}
2154		# endif // _LIBCPP_HAS_EXCEPTIONS
	2052	__guard.__complete();
2155	2053	for (; __back_capacity > 0; --__back_capacity) {
2156	2054	__buf.emplace_back(__map_.back());
2157	2055	__map_.pop_back();
2158	2056	}
2159	2057	for (__map_pointer __i = __map_.begin(); __i != __map_.end(); ++__i)
2160	2058	__buf.emplace_back(*__i);
2161		std::swap(__map_.__first_, __buf.__first_);
2162		std::swap(__map_.__begin_, __buf.__begin_);
2163		std::swap(__map_.__end_, __buf.__end_);
2164		std::swap(__map_.__cap_, __buf.__cap_);
	2059	__map_.__swap_without_allocator(__buf);
2165	2060	__start_ += __ds;
2166	2061	}
2167	2062	}
...	...	@@ -2194,8 +2089,8 @@ void deque<_Tp, _Allocator>::__add_back_capacity() {
2194	2089	}
2195	2090	// Else need to allocate 1 buffer, and we need to reallocate __map_.
2196	2091	else {
2197		__split_buffer<pointer, __pointer_allocator&> __buf(
2198		std::max<size_type>(2 * __map_.capacity(), 1), __map_.size(), __map_.__alloc_);
	2092	__split_buffer<pointer, __pointer_allocator> __buf(
	2093	std::max<size_type>(2 * __map_.capacity(), 1), __map_.size(), __map_.__get_allocator());
2199	2094
2200	2095	typedef __allocator_destructor<_Allocator> _Dp;
2201	2096	unique_ptr<pointer, _Dp> __hold(__alloc_traits::allocate(__a, __block_size), _Dp(__a, __block_size));
...	...	@@ -2204,10 +2099,7 @@ void deque<_Tp, _Allocator>::__add_back_capacity() {
2204	2099
2205	2100	for (__map_pointer __i = __map_.end(); __i != __map_.begin();)
2206	2101	__buf.emplace_front(*--__i);
2207		std::swap(__map_.__first_, __buf.__first_);
2208		std::swap(__map_.__begin_, __buf.__begin_);
2209		std::swap(__map_.__end_, __buf.__end_);
2210		std::swap(__map_.__cap_, __buf.__cap_);
	2102	__map_.__swap_without_allocator(__buf);
2211	2103	__annotate_whole_block(__map_.size() - 1, __asan_poison);
2212	2104	}
2213	2105	}
...	...	@@ -2257,36 +2149,28 @@ void deque<_Tp, _Allocator>::__add_back_capacity(size_type __n) {
2257	2149	// Else need to allocate __nb buffers, and we need to reallocate __map_.
2258	2150	else {
2259	2151	size_type __ds = __front_capacity * __block_size;
2260		__split_buffer<pointer, __pointer_allocator&> __buf(
	2152	__split_buffer<pointer, __pointer_allocator> __buf(
2261	2153	std::max<size_type>(2 * __map_.capacity(), __nb + __map_.size()),
2262	2154	__map_.size() - __front_capacity,
2263		__map_.__alloc_);
2264		# if _LIBCPP_HAS_EXCEPTIONS
2265		try {
2266		# endif // _LIBCPP_HAS_EXCEPTIONS
2267		for (; __nb > 0; --__nb) {
2268		__buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
2269		// ASan: this is an empty container, we have to poison the whole block
2270		__annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
2271		}
2272		# if _LIBCPP_HAS_EXCEPTIONS
2273		} catch (...) {
	2155	__map_.__get_allocator());
	2156	auto __guard = std::__make_exception_guard([&] {
2274	2157	__annotate_delete();
2275	2158	for (__map_pointer __i = __buf.begin(); __i != __buf.end(); ++__i)
2276	2159	__alloc_traits::deallocate(__a, *__i, __block_size);
2277		throw;
	2160	});
	2161	for (; __nb > 0; --__nb) {
	2162	__buf.emplace_back(__alloc_traits::allocate(__a, __block_size));
	2163	// ASan: this is an empty container, we have to poison the whole block
	2164	__annotate_poison_block(std::__to_address(__buf.back()), std::__to_address(__buf.back() + __block_size));
2278	2165	}
2279		# endif // _LIBCPP_HAS_EXCEPTIONS
	2166	__guard.__complete();
2280	2167	for (; __front_capacity > 0; --__front_capacity) {
2281	2168	__buf.emplace_back(__map_.front());
2282	2169	__map_.pop_front();
2283	2170	}
2284	2171	for (__map_pointer __i = __map_.end(); __i != __map_.begin();)
2285	2172	__buf.emplace_front(*--__i);
2286		std::swap(__map_.__first_, __buf.__first_);
2287		std::swap(__map_.__begin_, __buf.__begin_);
2288		std::swap(__map_.__end_, __buf.__end_);
2289		std::swap(__map_.__cap_, __buf.__cap_);
	2173	__map_.__swap_without_allocator(__buf);
2290	2174	__start_ -= __ds;
2291	2175	}
2292	2176	}

lib/libcxx/include/errno.h+269-269

...	...	@@ -23,381 +23,381 @@ Macros:
23	23	*/
24	24
25	25	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
26		# include <__cxx03/errno.h>
	26	# include <__cxx03/__config>
27	27	#else
28	28	# include <__config>
	29	#endif
29	30
30		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
31		# pragma GCC system_header
32		# endif
	31	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	32	# pragma GCC system_header
	33	#endif
33	34
34		# if __has_include_next(<errno.h>)
35		# include_next <errno.h>
36		# endif
	35	#if __has_include_next(<errno.h>)
	36	# include_next <errno.h>
	37	#endif
37	38
38		# ifdef __cplusplus
	39	#ifdef __cplusplus
39	40
40		# if !defined(EOWNERDEAD) \|\| !defined(ENOTRECOVERABLE)
	41	# if !defined(EOWNERDEAD) \|\| !defined(ENOTRECOVERABLE)
41	42
42		# ifdef ELAST
	43	# ifdef ELAST
43	44
44	45	static const int __elast1 = ELAST + 1;
45	46	static const int __elast2 = ELAST + 2;
46	47
47		# else
	48	# else
48	49
49	50	static const int __elast1 = 104;
50	51	static const int __elast2 = 105;
51	52
52		# endif
	53	# endif
53	54
54		# ifdef ENOTRECOVERABLE
	55	# ifdef ENOTRECOVERABLE
55	56
56		# define EOWNERDEAD __elast1
	57	# define EOWNERDEAD __elast1
57	58
58		# ifdef ELAST
59		# undef ELAST
60		# define ELAST EOWNERDEAD
61		# endif
	59	# ifdef ELAST
	60	# undef ELAST
	61	# define ELAST EOWNERDEAD
	62	# endif
62	63
63		# elif defined(EOWNERDEAD)
	64	# elif defined(EOWNERDEAD)
64	65
65		# define ENOTRECOVERABLE __elast1
66		# ifdef ELAST
67		# undef ELAST
68		# define ELAST ENOTRECOVERABLE
69		# endif
	66	# define ENOTRECOVERABLE __elast1
	67	# ifdef ELAST
	68	# undef ELAST
	69	# define ELAST ENOTRECOVERABLE
	70	# endif
70	71
71		# else // defined(EOWNERDEAD)
	72	# else // defined(EOWNERDEAD)
72	73
73		# define EOWNERDEAD __elast1
74		# define ENOTRECOVERABLE __elast2
75		# ifdef ELAST
76		# undef ELAST
77		# define ELAST ENOTRECOVERABLE
78		# endif
	74	# define EOWNERDEAD __elast1
	75	# define ENOTRECOVERABLE __elast2
	76	# ifdef ELAST
	77	# undef ELAST
	78	# define ELAST ENOTRECOVERABLE
	79	# endif
79	80
80		# endif // defined(EOWNERDEAD)
	81	# endif // defined(EOWNERDEAD)
81	82
82		# endif // !defined(EOWNERDEAD) \|\| !defined(ENOTRECOVERABLE)
	83	# endif // !defined(EOWNERDEAD) \|\| !defined(ENOTRECOVERABLE)
83	84
84	85	// supply errno values likely to be missing, particularly on Windows
85	86
86		# ifndef EAFNOSUPPORT
87		# define EAFNOSUPPORT 9901
88		# endif
	87	# ifndef EAFNOSUPPORT
	88	# define EAFNOSUPPORT 9901
	89	# endif
89	90
90		# ifndef EADDRINUSE
91		# define EADDRINUSE 9902
92		# endif
	91	# ifndef EADDRINUSE
	92	# define EADDRINUSE 9902
	93	# endif
93	94
94		# ifndef EADDRNOTAVAIL
95		# define EADDRNOTAVAIL 9903
96		# endif
	95	# ifndef EADDRNOTAVAIL
	96	# define EADDRNOTAVAIL 9903
	97	# endif
97	98
98		# ifndef EISCONN
99		# define EISCONN 9904
100		# endif
	99	# ifndef EISCONN
	100	# define EISCONN 9904
	101	# endif
101	102
102		# ifndef EBADMSG
103		# define EBADMSG 9905
104		# endif
	103	# ifndef EBADMSG
	104	# define EBADMSG 9905
	105	# endif
105	106
106		# ifndef ECONNABORTED
107		# define ECONNABORTED 9906
108		# endif
	107	# ifndef ECONNABORTED
	108	# define ECONNABORTED 9906
	109	# endif
109	110
110		# ifndef EALREADY
111		# define EALREADY 9907
112		# endif
	111	# ifndef EALREADY
	112	# define EALREADY 9907
	113	# endif
113	114
114		# ifndef ECONNREFUSED
115		# define ECONNREFUSED 9908
116		# endif
	115	# ifndef ECONNREFUSED
	116	# define ECONNREFUSED 9908
	117	# endif
117	118
118		# ifndef ECONNRESET
119		# define ECONNRESET 9909
120		# endif
	119	# ifndef ECONNRESET
	120	# define ECONNRESET 9909
	121	# endif
121	122
122		# ifndef EDESTADDRREQ
123		# define EDESTADDRREQ 9910
124		# endif
	123	# ifndef EDESTADDRREQ
	124	# define EDESTADDRREQ 9910
	125	# endif
125	126
126		# ifndef EHOSTUNREACH
127		# define EHOSTUNREACH 9911
128		# endif
	127	# ifndef EHOSTUNREACH
	128	# define EHOSTUNREACH 9911
	129	# endif
129	130
130		# ifndef EIDRM
131		# define EIDRM 9912
132		# endif
	131	# ifndef EIDRM
	132	# define EIDRM 9912
	133	# endif
133	134
134		# ifndef EMSGSIZE
135		# define EMSGSIZE 9913
136		# endif
	135	# ifndef EMSGSIZE
	136	# define EMSGSIZE 9913
	137	# endif
137	138
138		# ifndef ENETDOWN
139		# define ENETDOWN 9914
140		# endif
	139	# ifndef ENETDOWN
	140	# define ENETDOWN 9914
	141	# endif
141	142
142		# ifndef ENETRESET
143		# define ENETRESET 9915
144		# endif
	143	# ifndef ENETRESET
	144	# define ENETRESET 9915
	145	# endif
145	146
146		# ifndef ENETUNREACH
147		# define ENETUNREACH 9916
148		# endif
	147	# ifndef ENETUNREACH
	148	# define ENETUNREACH 9916
	149	# endif
149	150
150		# ifndef ENOBUFS
151		# define ENOBUFS 9917
152		# endif
	151	# ifndef ENOBUFS
	152	# define ENOBUFS 9917
	153	# endif
153	154
154		# ifndef ENOLINK
155		# define ENOLINK 9918
156		# endif
	155	# ifndef ENOLINK
	156	# define ENOLINK 9918
	157	# endif
157	158
158		# ifndef ENODATA
159		# define ENODATA 9919
160		# endif
	159	# ifndef ENODATA
	160	# define ENODATA 9919
	161	# endif
161	162
162		# ifndef ENOMSG
163		# define ENOMSG 9920
164		# endif
	163	# ifndef ENOMSG
	164	# define ENOMSG 9920
	165	# endif
165	166
166		# ifndef ENOPROTOOPT
167		# define ENOPROTOOPT 9921
168		# endif
	167	# ifndef ENOPROTOOPT
	168	# define ENOPROTOOPT 9921
	169	# endif
169	170
170		# ifndef ENOSR
171		# define ENOSR 9922
172		# endif
	171	# ifndef ENOSR
	172	# define ENOSR 9922
	173	# endif
173	174
174		# ifndef ENOTSOCK
175		# define ENOTSOCK 9923
176		# endif
	175	# ifndef ENOTSOCK
	176	# define ENOTSOCK 9923
	177	# endif
177	178
178		# ifndef ENOSTR
179		# define ENOSTR 9924
180		# endif
	179	# ifndef ENOSTR
	180	# define ENOSTR 9924
	181	# endif
181	182
182		# ifndef ENOTCONN
183		# define ENOTCONN 9925
184		# endif
	183	# ifndef ENOTCONN
	184	# define ENOTCONN 9925
	185	# endif
185	186
186		# ifndef ENOTSUP
187		# define ENOTSUP 9926
188		# endif
	187	# ifndef ENOTSUP
	188	# define ENOTSUP 9926
	189	# endif
189	190
190		# ifndef ECANCELED
191		# define ECANCELED 9927
192		# endif
	191	# ifndef ECANCELED
	192	# define ECANCELED 9927
	193	# endif
193	194
194		# ifndef EINPROGRESS
195		# define EINPROGRESS 9928
196		# endif
	195	# ifndef EINPROGRESS
	196	# define EINPROGRESS 9928
	197	# endif
197	198
198		# ifndef EOPNOTSUPP
199		# define EOPNOTSUPP 9929
200		# endif
	199	# ifndef EOPNOTSUPP
	200	# define EOPNOTSUPP 9929
	201	# endif
201	202
202		# ifndef EWOULDBLOCK
203		# define EWOULDBLOCK 9930
204		# endif
	203	# ifndef EWOULDBLOCK
	204	# define EWOULDBLOCK 9930
	205	# endif
205	206
206		# ifndef EOWNERDEAD
207		# define EOWNERDEAD 9931
208		# endif
	207	# ifndef EOWNERDEAD
	208	# define EOWNERDEAD 9931
	209	# endif
209	210
210		# ifndef EPROTO
211		# define EPROTO 9932
212		# endif
	211	# ifndef EPROTO
	212	# define EPROTO 9932
	213	# endif
213	214
214		# ifndef EPROTONOSUPPORT
215		# define EPROTONOSUPPORT 9933
216		# endif
	215	# ifndef EPROTONOSUPPORT
	216	# define EPROTONOSUPPORT 9933
	217	# endif
217	218
218		# ifndef ENOTRECOVERABLE
219		# define ENOTRECOVERABLE 9934
220		# endif
	219	# ifndef ENOTRECOVERABLE
	220	# define ENOTRECOVERABLE 9934
	221	# endif
221	222
222		# ifndef ETIME
223		# define ETIME 9935
224		# endif
	223	# ifndef ETIME
	224	# define ETIME 9935
	225	# endif
225	226
226		# ifndef ETXTBSY
227		# define ETXTBSY 9936
228		# endif
	227	# ifndef ETXTBSY
	228	# define ETXTBSY 9936
	229	# endif
229	230
230		# ifndef ETIMEDOUT
231		# define ETIMEDOUT 9938
232		# endif
	231	# ifndef ETIMEDOUT
	232	# define ETIMEDOUT 9938
	233	# endif
233	234
234		# ifndef ELOOP
235		# define ELOOP 9939
236		# endif
	235	# ifndef ELOOP
	236	# define ELOOP 9939
	237	# endif
237	238
238		# ifndef EOVERFLOW
239		# define EOVERFLOW 9940
240		# endif
	239	# ifndef EOVERFLOW
	240	# define EOVERFLOW 9940
	241	# endif
241	242
242		# ifndef EPROTOTYPE
243		# define EPROTOTYPE 9941
244		# endif
	243	# ifndef EPROTOTYPE
	244	# define EPROTOTYPE 9941
	245	# endif
245	246
246		# ifndef ENOSYS
247		# define ENOSYS 9942
248		# endif
	247	# ifndef ENOSYS
	248	# define ENOSYS 9942
	249	# endif
249	250
250		# ifndef EINVAL
251		# define EINVAL 9943
252		# endif
	251	# ifndef EINVAL
	252	# define EINVAL 9943
	253	# endif
253	254
254		# ifndef ERANGE
255		# define ERANGE 9944
256		# endif
	255	# ifndef ERANGE
	256	# define ERANGE 9944
	257	# endif
257	258
258		# ifndef EILSEQ
259		# define EILSEQ 9945
260		# endif
	259	# ifndef EILSEQ
	260	# define EILSEQ 9945
	261	# endif
261	262
262	263	// Windows Mobile doesn't appear to define these:
263	264
264		# ifndef E2BIG
265		# define E2BIG 9946
266		# endif
	265	# ifndef E2BIG
	266	# define E2BIG 9946
	267	# endif
267	268
268		# ifndef EDOM
269		# define EDOM 9947
270		# endif
	269	# ifndef EDOM
	270	# define EDOM 9947
	271	# endif
271	272
272		# ifndef EFAULT
273		# define EFAULT 9948
274		# endif
	273	# ifndef EFAULT
	274	# define EFAULT 9948
	275	# endif
275	276
276		# ifndef EBADF
277		# define EBADF 9949
278		# endif
	277	# ifndef EBADF
	278	# define EBADF 9949
	279	# endif
279	280
280		# ifndef EPIPE
281		# define EPIPE 9950
282		# endif
	281	# ifndef EPIPE
	282	# define EPIPE 9950
	283	# endif
283	284
284		# ifndef EXDEV
285		# define EXDEV 9951
286		# endif
	285	# ifndef EXDEV
	286	# define EXDEV 9951
	287	# endif
287	288
288		# ifndef EBUSY
289		# define EBUSY 9952
290		# endif
	289	# ifndef EBUSY
	290	# define EBUSY 9952
	291	# endif
291	292
292		# ifndef ENOTEMPTY
293		# define ENOTEMPTY 9953
294		# endif
	293	# ifndef ENOTEMPTY
	294	# define ENOTEMPTY 9953
	295	# endif
295	296
296		# ifndef ENOEXEC
297		# define ENOEXEC 9954
298		# endif
	297	# ifndef ENOEXEC
	298	# define ENOEXEC 9954
	299	# endif
299	300
300		# ifndef EEXIST
301		# define EEXIST 9955
302		# endif
	301	# ifndef EEXIST
	302	# define EEXIST 9955
	303	# endif
303	304
304		# ifndef EFBIG
305		# define EFBIG 9956
306		# endif
	305	# ifndef EFBIG
	306	# define EFBIG 9956
	307	# endif
307	308
308		# ifndef ENAMETOOLONG
309		# define ENAMETOOLONG 9957
310		# endif
	309	# ifndef ENAMETOOLONG
	310	# define ENAMETOOLONG 9957
	311	# endif
311	312
312		# ifndef ENOTTY
313		# define ENOTTY 9958
314		# endif
	313	# ifndef ENOTTY
	314	# define ENOTTY 9958
	315	# endif
315	316
316		# ifndef EINTR
317		# define EINTR 9959
318		# endif
	317	# ifndef EINTR
	318	# define EINTR 9959
	319	# endif
319	320
320		# ifndef ESPIPE
321		# define ESPIPE 9960
322		# endif
	321	# ifndef ESPIPE
	322	# define ESPIPE 9960
	323	# endif
323	324
324		# ifndef EIO
325		# define EIO 9961
326		# endif
	325	# ifndef EIO
	326	# define EIO 9961
	327	# endif
327	328
328		# ifndef EISDIR
329		# define EISDIR 9962
330		# endif
	329	# ifndef EISDIR
	330	# define EISDIR 9962
	331	# endif
331	332
332		# ifndef ECHILD
333		# define ECHILD 9963
334		# endif
	333	# ifndef ECHILD
	334	# define ECHILD 9963
	335	# endif
335	336
336		# ifndef ENOLCK
337		# define ENOLCK 9964
338		# endif
	337	# ifndef ENOLCK
	338	# define ENOLCK 9964
	339	# endif
339	340
340		# ifndef ENOSPC
341		# define ENOSPC 9965
342		# endif
	341	# ifndef ENOSPC
	342	# define ENOSPC 9965
	343	# endif
343	344
344		# ifndef ENXIO
345		# define ENXIO 9966
346		# endif
	345	# ifndef ENXIO
	346	# define ENXIO 9966
	347	# endif
347	348
348		# ifndef ENODEV
349		# define ENODEV 9967
350		# endif
	349	# ifndef ENODEV
	350	# define ENODEV 9967
	351	# endif
351	352
352		# ifndef ENOENT
353		# define ENOENT 9968
354		# endif
	353	# ifndef ENOENT
	354	# define ENOENT 9968
	355	# endif
355	356
356		# ifndef ESRCH
357		# define ESRCH 9969
358		# endif
	357	# ifndef ESRCH
	358	# define ESRCH 9969
	359	# endif
359	360
360		# ifndef ENOTDIR
361		# define ENOTDIR 9970
362		# endif
	361	# ifndef ENOTDIR
	362	# define ENOTDIR 9970
	363	# endif
363	364
364		# ifndef ENOMEM
365		# define ENOMEM 9971
366		# endif
	365	# ifndef ENOMEM
	366	# define ENOMEM 9971
	367	# endif
367	368
368		# ifndef EPERM
369		# define EPERM 9972
370		# endif
	369	# ifndef EPERM
	370	# define EPERM 9972
	371	# endif
371	372
372		# ifndef EACCES
373		# define EACCES 9973
374		# endif
	373	# ifndef EACCES
	374	# define EACCES 9973
	375	# endif
375	376
376		# ifndef EROFS
377		# define EROFS 9974
378		# endif
	377	# ifndef EROFS
	378	# define EROFS 9974
	379	# endif
379	380
380		# ifndef EDEADLK
381		# define EDEADLK 9975
382		# endif
	381	# ifndef EDEADLK
	382	# define EDEADLK 9975
	383	# endif
383	384
384		# ifndef EAGAIN
385		# define EAGAIN 9976
386		# endif
	385	# ifndef EAGAIN
	386	# define EAGAIN 9976
	387	# endif
387	388
388		# ifndef ENFILE
389		# define ENFILE 9977
390		# endif
	389	# ifndef ENFILE
	390	# define ENFILE 9977
	391	# endif
391	392
392		# ifndef EMFILE
393		# define EMFILE 9978
394		# endif
	393	# ifndef EMFILE
	394	# define EMFILE 9978
	395	# endif
395	396
396		# ifndef EMLINK
397		# define EMLINK 9979
398		# endif
	397	# ifndef EMLINK
	398	# define EMLINK 9979
	399	# endif
399	400
400		# endif // __cplusplus
401		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	401	#endif // __cplusplus
402	402
403	403	#endif // _LIBCPP_ERRNO_H

lib/libcxx/include/exception+4-1

...	...	@@ -93,10 +93,13 @@ template <class E> void rethrow_if_nested(const E& e);
93	93
94	94	# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
95	95	# include <cstddef>
96		# include <cstdlib>
97	96	# include <new>
98	97	# include <type_traits>
99	98	# endif
	99
	100	# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
	101	# include <cstdlib>
	102	# endif
100	103	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
101	104
102	105	#endif // _LIBCPP_EXCEPTION

lib/libcxx/include/ext/hash_map+7-48

...	...	@@ -206,6 +206,7 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
206	206	#else
207	207	# include <__config>
208	208	# include <__hash_table>
	209	# include <__memory/compressed_pair.h>
209	210	# include <algorithm>
210	211	# include <ext/__hash>
211	212	# include <functional>
...	...	@@ -224,21 +225,9 @@ _LIBCPP_WARNING("Use of the header <ext/hash_map> is deprecated. Migrate to <un
224	225
225	226	namespace __gnu_cxx {
226	227
227		template <class _Tp, class _Hash, bool = std::is_empty<_Hash>::value && !std::__libcpp_is_final<_Hash>::value >
228		class __hash_map_hasher : private _Hash {
229		public:
230		_LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : _Hash() {}
231		_LIBCPP_HIDE_FROM_ABI __hash_map_hasher(const _Hash& __h) : _Hash(__h) {}
232		_LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const { return *this; }
233		_LIBCPP_HIDE_FROM_ABI size_t operator()(const _Tp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
234		_LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const {
235		return static_cast<const _Hash&>(*this)(__x);
236		}
237		};
238
239	228	template <class _Tp, class _Hash>
240		class __hash_map_hasher<_Tp, _Hash, false> {
241		_Hash __hash_;
	229	class __hash_map_hasher {
	230	_LIBCPP_COMPRESSED_ELEMENT(_Hash, __hash_);
242	231
243	232	public:
244	233	_LIBCPP_HIDE_FROM_ABI __hash_map_hasher() : __hash_() {}
...	...	@@ -248,30 +237,9 @@ public:
248	237	_LIBCPP_HIDE_FROM_ABI size_t operator()(const typename _Tp::first_type& __x) const { return __hash_(__x); }
249	238	};
250	239
251		template <class _Tp, class _Pred, bool = std::is_empty<_Pred>::value && !std::__libcpp_is_final<_Pred>::value >
252		class __hash_map_equal : private _Pred {
253		public:
254		_LIBCPP_HIDE_FROM_ABI __hash_map_equal() : _Pred() {}
255		_LIBCPP_HIDE_FROM_ABI __hash_map_equal(const _Pred& __p) : _Pred(__p) {}
256		_LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const { return *this; }
257		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const _Tp& __y) const {
258		return static_cast<const _Pred&>(*this)(__x.first, __y.first);
259		}
260		_LIBCPP_HIDE_FROM_ABI bool operator()(const typename _Tp::first_type& __x, const _Tp& __y) const {
261		return static_cast<const _Pred&>(*this)(__x, __y.first);
262		}
263		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Tp& __x, const typename _Tp::first_type& __y) const {
264		return static_cast<const _Pred&>(*this)(__x.first, __y);
265		}
266		_LIBCPP_HIDE_FROM_ABI bool
267		operator()(const typename _Tp::first_type& __x, const typename _Tp::first_type& __y) const {
268		return static_cast<const _Pred&>(*this)(__x, __y);
269		}
270		};
271
272	240	template <class _Tp, class _Pred>
273		class __hash_map_equal<_Tp, _Pred, false> {
274		_Pred __pred_;
	241	class __hash_map_equal {
	242	_LIBCPP_COMPRESSED_ELEMENT(_Pred, __pred_);
275	243
276	244	public:
277	245	_LIBCPP_HIDE_FROM_ABI __hash_map_equal() : __pred_() {}
...	...	@@ -467,8 +435,6 @@ private:
467	435
468	436	__table __table_;
469	437
470		typedef typename __table::__node_pointer __node_pointer;
471		typedef typename __table::__node_const_pointer __node_const_pointer;
472	438	typedef typename __table::__node_traits __node_traits;
473	439	typedef typename __table::__node_allocator __node_allocator;
474	440	typedef typename __table::__node __node;
...	...	@@ -604,10 +570,7 @@ hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(
604	570	}
605	571
606	572	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
607		hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(const hash_map& __u) : __table_(__u.__table_) {
608		__table_.__rehash_unique(__u.bucket_count());
609		insert(__u.begin(), __u.end());
610		}
	573	hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_map(const hash_map& __u) : __table_(__u.__table_) {}
611	574
612	575	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
613	576	typename hash_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
...	...	@@ -693,7 +656,6 @@ private:
693	656
694	657	__table __table_;
695	658
696		typedef typename __table::__node_traits __node_traits;
697	659	typedef typename __table::__node_allocator __node_allocator;
698	660	typedef typename __table::__node __node;
699	661	typedef __hash_map_node_destructor<__node_allocator> _Dp;
...	...	@@ -822,10 +784,7 @@ hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(
822	784	}
823	785
824	786	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
825		hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(const hash_multimap& __u) : __table_(__u.__table_) {
826		__table_.__rehash_multi(__u.bucket_count());
827		insert(__u.begin(), __u.end());
828		}
	787	hash_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::hash_multimap(const hash_multimap& __u) : __table_(__u.__table_) {}
829	788
830	789	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
831	790	template <class _InputIterator>

lib/libcxx/include/ext/hash_set+2-8

...	...	@@ -356,10 +356,7 @@ hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(
356	356	}
357	357
358	358	template <class _Value, class _Hash, class _Pred, class _Alloc>
359		hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(const hash_set& __u) : __table_(__u.__table_) {
360		__table_.__rehash_unique(__u.bucket_count());
361		insert(__u.begin(), __u.end());
362		}
	359	hash_set<_Value, _Hash, _Pred, _Alloc>::hash_set(const hash_set& __u) : __table_(__u.__table_) {}
363	360
364	361	template <class _Value, class _Hash, class _Pred, class _Alloc>
365	362	template <class _InputIterator>
...	...	@@ -534,10 +531,7 @@ hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(
534	531	}
535	532
536	533	template <class _Value, class _Hash, class _Pred, class _Alloc>
537		hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(const hash_multiset& __u) : __table_(__u.__table_) {
538		__table_.__rehash_multi(__u.bucket_count());
539		insert(__u.begin(), __u.end());
540		}
	534	hash_multiset<_Value, _Hash, _Pred, _Alloc>::hash_multiset(const hash_multiset& __u) : __table_(__u.__table_) {}
541	535
542	536	template <class _Value, class _Hash, class _Pred, class _Alloc>
543	537	template <class _InputIterator>

lib/libcxx/include/fenv.h+43-43

...	...	@@ -50,69 +50,69 @@ int feupdateenv(const fenv_t* envp);
50	50	*/
51	51
52	52	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
53		# include <__cxx03/fenv.h>
	53	# include <__cxx03/__config>
54	54	#else
55	55	# include <__config>
	56	#endif
56	57
57		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
58		# pragma GCC system_header
59		# endif
	58	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	59	# pragma GCC system_header
	60	#endif
60	61
61		# if __has_include_next(<fenv.h>)
62		# include_next <fenv.h>
63		# endif
	62	#if __has_include_next(<fenv.h>)
	63	# include_next <fenv.h>
	64	#endif
64	65
65		# ifdef __cplusplus
	66	#ifdef __cplusplus
66	67
67	68	extern "C++" {
68	69
69		# ifdef feclearexcept
70		# undef feclearexcept
71		# endif
	70	# ifdef feclearexcept
	71	# undef feclearexcept
	72	# endif
72	73
73		# ifdef fegetexceptflag
74		# undef fegetexceptflag
75		# endif
	74	# ifdef fegetexceptflag
	75	# undef fegetexceptflag
	76	# endif
76	77
77		# ifdef feraiseexcept
78		# undef feraiseexcept
79		# endif
	78	# ifdef feraiseexcept
	79	# undef feraiseexcept
	80	# endif
80	81
81		# ifdef fesetexceptflag
82		# undef fesetexceptflag
83		# endif
	82	# ifdef fesetexceptflag
	83	# undef fesetexceptflag
	84	# endif
84	85
85		# ifdef fetestexcept
86		# undef fetestexcept
87		# endif
	86	# ifdef fetestexcept
	87	# undef fetestexcept
	88	# endif
88	89
89		# ifdef fegetround
90		# undef fegetround
91		# endif
	90	# ifdef fegetround
	91	# undef fegetround
	92	# endif
92	93
93		# ifdef fesetround
94		# undef fesetround
95		# endif
	94	# ifdef fesetround
	95	# undef fesetround
	96	# endif
96	97
97		# ifdef fegetenv
98		# undef fegetenv
99		# endif
	98	# ifdef fegetenv
	99	# undef fegetenv
	100	# endif
100	101
101		# ifdef feholdexcept
102		# undef feholdexcept
103		# endif
	102	# ifdef feholdexcept
	103	# undef feholdexcept
	104	# endif
104	105
105		# ifdef fesetenv
106		# undef fesetenv
107		# endif
	106	# ifdef fesetenv
	107	# undef fesetenv
	108	# endif
108	109
109		# ifdef feupdateenv
110		# undef feupdateenv
111		# endif
	110	# ifdef feupdateenv
	111	# undef feupdateenv
	112	# endif
112	113
113	114	} // extern "C++"
114	115
115		# endif // defined(__cplusplus)
116		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	116	#endif // defined(__cplusplus)
117	117
118	118	#endif // _LIBCPP_FENV_H

lib/libcxx/include/flat_map+663-11

...	...	@@ -17,18 +17,359 @@
17	17	#include <initializer_list> // see [initializer.list.syn]
18	18
19	19	namespace std {
	20	struct sorted_unique_t { explicit sorted_unique_t() = default; };
	21	inline constexpr sorted_unique_t sorted_unique{};
	22
20	23	// [flat.map], class template flat_map
21	24	template<class Key, class T, class Compare = less<Key>,
22	25	class KeyContainer = vector<Key>, class MappedContainer = vector<T>>
23		class flat_map;
	26	class flat_map {
	27	public:
	28	// types
	29	using key_type = Key;
	30	using mapped_type = T;
	31	using value_type = pair<key_type, mapped_type>;
	32	using key_compare = Compare;
	33	using reference = pair<const key_type&, mapped_type&>;
	34	using const_reference = pair<const key_type&, const mapped_type&>;
	35	using size_type = size_t;
	36	using difference_type = ptrdiff_t;
	37	using iterator = implementation-defined; // see [container.requirements]
	38	using const_iterator = implementation-defined; // see [container.requirements]
	39	using reverse_iterator = std::reverse_iterator<iterator>;
	40	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
	41	using key_container_type = KeyContainer;
	42	using mapped_container_type = MappedContainer;
24	43
25		struct sorted_unique_t { explicit sorted_unique_t() = default; };
26		inline constexpr sorted_unique_t sorted_unique{};
	44	class value_compare {
	45	private:
	46	key_compare comp; // exposition only
	47	constexpr value_compare(key_compare c) : comp(c) { } // exposition only
	48
	49	public:
	50	constexpr bool operator()(const_reference x, const_reference y) const {
	51	return comp(x.first, y.first);
	52	}
	53	};
	54
	55	struct containers {
	56	key_container_type keys;
	57	mapped_container_type values;
	58	};
	59
	60	// [flat.map.cons], constructors
	61	constexpr flat_map() : flat_map(key_compare()) { }
	62
	63	constexpr flat_map(const flat_map&);
	64	constexpr flat_map(flat_map&&);
	65	constexpr flat_map& operator=(const flat_map&);
	66	constexpr flat_map& operator=(flat_map&&);
	67
	68	constexpr explicit flat_map(const key_compare& comp)
	69	: c(), compare(comp) { }
	70
	71	constexpr flat_map(key_container_type key_cont, mapped_container_type mapped_cont,
	72	const key_compare& comp = key_compare());
	73
	74	constexpr flat_map(sorted_unique_t, key_container_type key_cont,
	75	mapped_container_type mapped_cont,
	76	const key_compare& comp = key_compare());
	77
	78	template<class InputIterator>
	79	constexpr flat_map(InputIterator first, InputIterator last,
	80	const key_compare& comp = key_compare())
	81	: c(), compare(comp) { insert(first, last); }
	82
	83	template<class InputIterator>
	84	constexpr flat_map(sorted_unique_t, InputIterator first, InputIterator last,
	85	const key_compare& comp = key_compare())
	86	: c(), compare(comp) { insert(sorted_unique, first, last); }
	87
	88	template<container-compatible-range<value_type> R>
	89	constexpr flat_map(from_range_t, R&& rg)
	90	: flat_map(from_range, std::forward<R>(rg), key_compare()) { }
	91	template<container-compatible-range<value_type> R>
	92	constexpr flat_map(from_range_t, R&& rg, const key_compare& comp)
	93	: flat_map(comp) { insert_range(std::forward<R>(rg)); }
	94
	95	constexpr flat_map(initializer_list<value_type> il, const key_compare& comp = key_compare())
	96	: flat_map(il.begin(), il.end(), comp) { }
	97
	98	constexpr flat_map(sorted_unique_t, initializer_list<value_type> il,
	99	const key_compare& comp = key_compare())
	100	: flat_map(sorted_unique, il.begin(), il.end(), comp) { }
	101
	102	// [flat.map.cons.alloc], constructors with allocators
	103
	104	template<class Alloc>
	105	constexpr explicit flat_map(const Alloc& a);
	106	template<class Alloc>
	107	constexpr flat_map(const key_compare& comp, const Alloc& a);
	108	template<class Alloc>
	109	constexpr flat_map(const key_container_type& key_cont,
	110	const mapped_container_type& mapped_cont,
	111	const Alloc& a);
	112	template<class Alloc>
	113	constexpr flat_map(const key_container_type& key_cont,
	114	const mapped_container_type& mapped_cont,
	115	const key_compare& comp, const Alloc& a);
	116	template<class Alloc>
	117	constexpr flat_map(sorted_unique_t, const key_container_type& key_cont,
	118	const mapped_container_type& mapped_cont, const Alloc& a);
	119	template<class Alloc>
	120	constexpr flat_map(sorted_unique_t, const key_container_type& key_cont,
	121	const mapped_container_type& mapped_cont, const key_compare& comp,
	122	const Alloc& a);
	123	template<class Alloc>
	124	constexpr flat_map(const flat_map&, const Alloc& a);
	125	template<class Alloc>
	126	constexpr flat_map(flat_map&&, const Alloc& a);
	127	template<class InputIterator, class Alloc>
	128	constexpr flat_map(InputIterator first, InputIterator last, const Alloc& a);
	129	template<class InputIterator, class Alloc>
	130	constexpr flat_map(InputIterator first, InputIterator last,
	131	const key_compare& comp, const Alloc& a);
	132	template<class InputIterator, class Alloc>
	133	constexpr flat_map(sorted_unique_t, InputIterator first, InputIterator last,
	134	const Alloc& a);
	135	template<class InputIterator, class Alloc>
	136	constexpr flat_map(sorted_unique_t, InputIterator first, InputIterator last,
	137	const key_compare& comp, const Alloc& a);
	138	template<container-compatible-range<value_type> R, class Alloc>
	139	constexpr flat_map(from_range_t, R&& rg, const Alloc& a);
	140	template<container-compatible-range<value_type> R, class Alloc>
	141	constexpr flat_map(from_range_t, R&& rg, const key_compare& comp, const Alloc& a);
	142	template<class Alloc>
	143	constexpr flat_map(initializer_list<value_type> il, const Alloc& a);
	144	template<class Alloc>
	145	constexpr flat_map(initializer_list<value_type> il, const key_compare& comp,
	146	const Alloc& a);
	147	template<class Alloc>
	148	constexpr flat_map(sorted_unique_t, initializer_list<value_type> il, const Alloc& a);
	149	template<class Alloc>
	150	constexpr flat_map(sorted_unique_t, initializer_list<value_type> il,
	151	const key_compare& comp, const Alloc& a);
	152
	153	constexpr flat_map& operator=(initializer_list<value_type>);
	154
	155	// iterators
	156	constexpr iterator begin() noexcept;
	157	constexpr const_iterator begin() const noexcept;
	158	constexpr iterator end() noexcept;
	159	constexpr const_iterator end() const noexcept;
	160
	161	constexpr reverse_iterator rbegin() noexcept;
	162	constexpr const_reverse_iterator rbegin() const noexcept;
	163	constexpr reverse_iterator rend() noexcept;
	164	constexpr const_reverse_iterator rend() const noexcept;
	165
	166	constexpr const_iterator cbegin() const noexcept;
	167	constexpr const_iterator cend() const noexcept;
	168	constexpr const_reverse_iterator crbegin() const noexcept;
	169	constexpr const_reverse_iterator crend() const noexcept;
	170
	171	// [flat.map.capacity], capacity
	172	constexpr bool empty() const noexcept;
	173	constexpr size_type size() const noexcept;
	174	constexpr size_type max_size() const noexcept;
	175
	176	// [flat.map.access], element access
	177	constexpr mapped_type& operator[](const key_type& x);
	178	constexpr mapped_type& operator[](key_type&& x);
	179	template<class K> constexpr mapped_type& operator[](K&& x);
	180	constexpr mapped_type& at(const key_type& x);
	181	constexpr const mapped_type& at(const key_type& x) const;
	182	template<class K> constexpr mapped_type& at(const K& x);
	183	template<class K> constexpr const mapped_type& at(const K& x) const;
	184
	185	// [flat.map.modifiers], modifiers
	186	template<class... Args> constexpr pair<iterator, bool> emplace(Args&&... args);
	187	template<class... Args>
	188	constexpr iterator emplace_hint(const_iterator position, Args&&... args);
	189
	190	constexpr pair<iterator, bool> insert(const value_type& x)
	191	{ return emplace(x); }
	192	constexpr pair<iterator, bool> insert(value_type&& x)
	193	{ return emplace(std::move(x)); }
	194	constexpr iterator insert(const_iterator position, const value_type& x)
	195	{ return emplace_hint(position, x); }
	196	constexpr iterator insert(const_iterator position, value_type&& x)
	197	{ return emplace_hint(position, std::move(x)); }
	198
	199	template<class P> constexpr pair<iterator, bool> insert(P&& x);
	200	template<class P>
	201	constexpr iterator insert(const_iterator position, P&&);
	202	template<class InputIterator>
	203	constexpr void insert(InputIterator first, InputIterator last);
	204	template<class InputIterator>
	205	constexpr void insert(sorted_unique_t, InputIterator first, InputIterator last);
	206	template<container-compatible-range<value_type> R>
	207	constexpr void insert_range(R&& rg);
	208	template<container-compatible-range<value_type> R>
	209	constexpr void insert_range(sorted_unique_t, R&& rg);
	210
	211	constexpr void insert(initializer_list<value_type> il)
	212	{ insert(il.begin(), il.end()); }
	213	constexpr void insert(sorted_unique_t, initializer_list<value_type> il)
	214	{ insert(sorted_unique, il.begin(), il.end()); }
	215
	216	constexpr containers extract() &&;
	217	constexpr void replace(key_container_type&& key_cont, mapped_container_type&& mapped_cont);
	218
	219	template<class... Args>
	220	constexpr pair<iterator, bool> try_emplace(const key_type& k, Args&&... args);
	221	template<class... Args>
	222	constexpr pair<iterator, bool> try_emplace(key_type&& k, Args&&... args);
	223	template<class K, class... Args>
	224	constexpr pair<iterator, bool> try_emplace(K&& k, Args&&... args);
	225	template<class... Args>
	226	constexpr iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args);
	227	template<class... Args>
	228	constexpr iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args);
	229	template<class K, class... Args>
	230	constexpr iterator try_emplace(const_iterator hint, K&& k, Args&&... args);
	231	template<class M>
	232	constexpr pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj);
	233	template<class M>
	234	constexpr pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj);
	235	template<class K, class M>
	236	constexpr pair<iterator, bool> insert_or_assign(K&& k, M&& obj);
	237	template<class M>
	238	constexpr iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj);
	239	template<class M>
	240	constexpr iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj);
	241	template<class K, class M>
	242	constexpr iterator insert_or_assign(const_iterator hint, K&& k, M&& obj);
	243
	244	constexpr iterator erase(iterator position);
	245	constexpr iterator erase(const_iterator position);
	246	constexpr size_type erase(const key_type& x);
	247	template<class K> constexpr size_type erase(K&& x);
	248	constexpr iterator erase(const_iterator first, const_iterator last);
	249
	250	constexpr void swap(flat_map& y) noexcept(see below);
	251	constexpr void clear() noexcept;
	252
	253	// observers
	254	constexpr key_compare key_comp() const;
	255	constexpr value_compare value_comp() const;
	256
	257	constexpr const key_container_type& keys() const noexcept { return c.keys; }
	258	constexpr const mapped_container_type& values() const noexcept { return c.values; }
	259
	260	// map operations
	261	constexpr iterator find(const key_type& x);
	262	constexpr const_iterator find(const key_type& x) const;
	263	template<class K> constexpr iterator find(const K& x);
	264	template<class K> constexpr const_iterator find(const K& x) const;
	265
	266	constexpr size_type count(const key_type& x) const;
	267	template<class K> constexpr size_type count(const K& x) const;
	268
	269	constexpr bool contains(const key_type& x) const;
	270	template<class K> constexpr bool contains(const K& x) const;
	271
	272	constexpr iterator lower_bound(const key_type& x);
	273	constexpr const_iterator lower_bound(const key_type& x) const;
	274	template<class K> constexpr iterator lower_bound(const K& x);
	275	template<class K> constexpr const_iterator lower_bound(const K& x) const;
	276
	277	constexpr iterator upper_bound(const key_type& x);
	278	constexpr const_iterator upper_bound(const key_type& x) const;
	279	template<class K> constexpr iterator upper_bound(const K& x);
	280	template<class K> constexpr const_iterator upper_bound(const K& x) const;
	281
	282	constexpr pair<iterator, iterator> equal_range(const key_type& x);
	283	constexpr pair<const_iterator, const_iterator> equal_range(const key_type& x) const;
	284	template<class K> constexpr pair<iterator, iterator> equal_range(const K& x);
	285	template<class K>
	286	constexpr pair<const_iterator, const_iterator> equal_range(const K& x) const;
	287
	288	friend constexpr bool operator==(const flat_map& x, const flat_map& y);
	289
	290	friend constexpr synth-three-way-result<value_type>
	291	operator<=>(const flat_map& x, const flat_map& y);
	292
	293	friend constexpr void swap(flat_map& x, flat_map& y) noexcept(noexcept(x.swap(y)))
	294	{ x.swap(y); }
	295
	296	private:
	297	containers c; // exposition only
	298	key_compare compare; // exposition only
	299
	300	struct key-equiv { // exposition only
	301	constexpr key-equiv(key_compare c) : comp(c) { }
	302	constexpr bool operator()(const_reference x, const_reference y) const {
	303	return !comp(x.first, y.first) && !comp(y.first, x.first);
	304	}
	305	key_compare comp;
	306	};
	307	};
	308
	309	template<class KeyContainer, class MappedContainer,
	310	class Compare = less<typename KeyContainer::value_type>>
	311	flat_map(KeyContainer, MappedContainer, Compare = Compare())
	312	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	313	Compare, KeyContainer, MappedContainer>;
	314
	315	template<class KeyContainer, class MappedContainer, class Allocator>
	316	flat_map(KeyContainer, MappedContainer, Allocator)
	317	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	318	less<typename KeyContainer::value_type>, KeyContainer, MappedContainer>;
	319	template<class KeyContainer, class MappedContainer, class Compare, class Allocator>
	320	flat_map(KeyContainer, MappedContainer, Compare, Allocator)
	321	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	322	Compare, KeyContainer, MappedContainer>;
	323
	324	template<class KeyContainer, class MappedContainer,
	325	class Compare = less<typename KeyContainer::value_type>>
	326	flat_map(sorted_unique_t, KeyContainer, MappedContainer, Compare = Compare())
	327	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	328	Compare, KeyContainer, MappedContainer>;
	329
	330	template<class KeyContainer, class MappedContainer, class Allocator>
	331	flat_map(sorted_unique_t, KeyContainer, MappedContainer, Allocator)
	332	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	333	less<typename KeyContainer::value_type>, KeyContainer, MappedContainer>;
	334	template<class KeyContainer, class MappedContainer, class Compare, class Allocator>
	335	flat_map(sorted_unique_t, KeyContainer, MappedContainer, Compare, Allocator)
	336	-> flat_map<typename KeyContainer::value_type, typename MappedContainer::value_type,
	337	Compare, KeyContainer, MappedContainer>;
	338
	339	template<class InputIterator, class Compare = less<iter-key-type<InputIterator>>>
	340	flat_map(InputIterator, InputIterator, Compare = Compare())
	341	-> flat_map<iter-key-type<InputIterator>, iter-mapped-type<InputIterator>, Compare>;
	342
	343	template<class InputIterator, class Compare = less<iter-key-type<InputIterator>>>
	344	flat_map(sorted_unique_t, InputIterator, InputIterator, Compare = Compare())
	345	-> flat_map<iter-key-type<InputIterator>, iter-mapped-type<InputIterator>, Compare>;
	346
	347	template<ranges::input_range R, class Compare = less<range-key-type<R>>,
	348	class Allocator = allocator<byte>>
	349	flat_map(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
	350	-> flat_map<range-key-type<R>, range-mapped-type<R>, Compare,
	351	vector<range-key-type<R>, alloc-rebind<Allocator, range-key-type<R>>>,
	352	vector<range-mapped-type<R>, alloc-rebind<Allocator, range-mapped-type<R>>>>;
	353
	354	template<ranges::input_range R, class Allocator>
	355	flat_map(from_range_t, R&&, Allocator)
	356	-> flat_map<range-key-type<R>, range-mapped-type<R>, less<range-key-type<R>>,
	357	vector<range-key-type<R>, alloc-rebind<Allocator, range-key-type<R>>>,
	358	vector<range-mapped-type<R>, alloc-rebind<Allocator, range-mapped-type<R>>>>;
	359
	360	template<class Key, class T, class Compare = less<Key>>
	361	flat_map(initializer_list<pair<Key, T>>, Compare = Compare())
	362	-> flat_map<Key, T, Compare>;
	363
	364	template<class Key, class T, class Compare = less<Key>>
	365	flat_map(sorted_unique_t, initializer_list<pair<Key, T>>, Compare = Compare())
	366	-> flat_map<Key, T, Compare>;
27	367
28	368	template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
29		class Allocator>
30		struct uses_allocator<flat_map<Key, T, Compare, KeyContainer, MappedContainer>,
31		Allocator>;
	369	class Allocator>
	370	struct uses_allocator<flat_map<Key, T, Compare, KeyContainer, MappedContainer>, Allocator>
	371	: bool_constant<uses_allocator_v<KeyContainer, Allocator> &&
	372	uses_allocator_v<MappedContainer, Allocator>> { };
32	373
33	374	// [flat.map.erasure], erasure for flat_map
34	375	template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
...	...	@@ -36,18 +377,329 @@ namespace std {
36	377	typename flat_map<Key, T, Compare, KeyContainer, MappedContainer>::size_type
37	378	erase_if(flat_map<Key, T, Compare, KeyContainer, MappedContainer>& c, Predicate pred);
38	379
	380	struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
	381	inline constexpr sorted_equivalent_t sorted_equivalent{};
	382
39	383	// [flat.multimap], class template flat_multimap
40	384	template<class Key, class T, class Compare = less<Key>,
41	385	class KeyContainer = vector<Key>, class MappedContainer = vector<T>>
42		class flat_multimap;
	386	class flat_multimap {
	387	public:
	388	// types
	389	using key_type = Key;
	390	using mapped_type = T;
	391	using value_type = pair<key_type, mapped_type>;
	392	using key_compare = Compare;
	393	using reference = pair<const key_type&, mapped_type&>;
	394	using const_reference = pair<const key_type&, const mapped_type&>;
	395	using size_type = size_t;
	396	using difference_type = ptrdiff_t;
	397	using iterator = implementation-defined; // see [container.requirements]
	398	using const_iterator = implementation-defined; // see [container.requirements]
	399	using reverse_iterator = std::reverse_iterator<iterator>;
	400	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
	401	using key_container_type = KeyContainer;
	402	using mapped_container_type = MappedContainer;
43	403
44		struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
45		inline constexpr sorted_equivalent_t sorted_equivalent{};
	404	class value_compare {
	405	private:
	406	key_compare comp; // exposition only
	407	constexpr value_compare(key_compare c) : comp(c) { } // exposition only
	408
	409	public:
	410	constexpr bool operator()(const_reference x, const_reference y) const {
	411	return comp(x.first, y.first);
	412	}
	413	};
	414
	415	struct containers {
	416	key_container_type keys;
	417	mapped_container_type values;
	418	};
	419
	420	// [flat.multimap.cons], constructors
	421	constexpr flat_multimap() : flat_multimap(key_compare()) { }
	422
	423	constexpr flat_multimap(const flat_multimap&);
	424	constexpr flat_multimap(flat_multimap&&);
	425	constexpr flat_multimap& operator=(const flat_multimap&);
	426	constexpr flat_multimap& operator=(flat_multimap&&);
	427
	428	constexpr explicit flat_multimap(const key_compare& comp)
	429	: c(), compare(comp) { }
	430
	431	constexpr flat_multimap(key_container_type key_cont, mapped_container_type mapped_cont,
	432	const key_compare& comp = key_compare());
	433
	434	constexpr flat_multimap(sorted_equivalent_t,
	435	key_container_type key_cont, mapped_container_type mapped_cont,
	436	const key_compare& comp = key_compare());
	437
	438	template<class InputIterator>
	439	constexpr flat_multimap(InputIterator first, InputIterator last,
	440	const key_compare& comp = key_compare())
	441	: c(), compare(comp)
	442	{ insert(first, last); }
	443
	444	template<class InputIterator>
	445	constexpr flat_multimap(sorted_equivalent_t, InputIterator first, InputIterator last,
	446	const key_compare& comp = key_compare())
	447	: c(), compare(comp) { insert(sorted_equivalent, first, last); }
	448
	449	template<container-compatible-range<value_type> R>
	450	constexpr flat_multimap(from_range_t, R&& rg)
	451	: flat_multimap(from_range, std::forward<R>(rg), key_compare()) { }
	452	template<container-compatible-range<value_type> R>
	453	constexpr flat_multimap(from_range_t, R&& rg, const key_compare& comp)
	454	: flat_multimap(comp) { insert_range(std::forward<R>(rg)); }
	455
	456	constexpr flat_multimap(initializer_list<value_type> il,
	457	const key_compare& comp = key_compare())
	458	: flat_multimap(il.begin(), il.end(), comp) { }
	459
	460	constexpr flat_multimap(sorted_equivalent_t, initializer_list<value_type> il,
	461	const key_compare& comp = key_compare())
	462	: flat_multimap(sorted_equivalent, il.begin(), il.end(), comp) { }
	463
	464	// [flat.multimap.cons.alloc], constructors with allocators
	465
	466	template<class Alloc>
	467	constexpr explicit flat_multimap(const Alloc& a);
	468	template<class Alloc>
	469	constexpr flat_multimap(const key_compare& comp, const Alloc& a);
	470	template<class Alloc>
	471	constexpr flat_multimap(const key_container_type& key_cont,
	472	const mapped_container_type& mapped_cont, const Alloc& a);
	473	template<class Alloc>
	474	constexpr flat_multimap(const key_container_type& key_cont,
	475	const mapped_container_type& mapped_cont,
	476	const key_compare& comp, const Alloc& a);
	477	template<class Alloc>
	478	constexpr flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
	479	const mapped_container_type& mapped_cont, const Alloc& a);
	480	template<class Alloc>
	481	constexpr flat_multimap(sorted_equivalent_t, const key_container_type& key_cont,
	482	const mapped_container_type& mapped_cont,
	483	const key_compare& comp, const Alloc& a);
	484	template<class Alloc>
	485	constexpr flat_multimap(const flat_multimap&, const Alloc& a);
	486	template<class Alloc>
	487	constexpr flat_multimap(flat_multimap&&, const Alloc& a);
	488	template<class InputIterator, class Alloc>
	489	constexpr flat_multimap(InputIterator first, InputIterator last, const Alloc& a);
	490	template<class InputIterator, class Alloc>
	491	constexpr flat_multimap(InputIterator first, InputIterator last,
	492	const key_compare& comp, const Alloc& a);
	493	template<class InputIterator, class Alloc>
	494	constexpr flat_multimap(sorted_equivalent_t, InputIterator first, InputIterator last,
	495	const Alloc& a);
	496	template<class InputIterator, class Alloc>
	497	constexpr flat_multimap(sorted_equivalent_t, InputIterator first, InputIterator last,
	498	const key_compare& comp, const Alloc& a);
	499	template<container-compatible-range<value_type> R, class Alloc>
	500	constexpr flat_multimap(from_range_t, R&& rg, const Alloc& a);
	501	template<container-compatible-range<value_type> R, class Alloc>
	502	constexpr flat_multimap(from_range_t, R&& rg, const key_compare& comp, const Alloc& a);
	503	template<class Alloc>
	504	constexpr flat_multimap(initializer_list<value_type> il, const Alloc& a);
	505	template<class Alloc>
	506	constexpr flat_multimap(initializer_list<value_type> il, const key_compare& comp,
	507	const Alloc& a);
	508	template<class Alloc>
	509	constexpr flat_multimap(sorted_equivalent_t, initializer_list<value_type> il,
	510	const Alloc& a);
	511	template<class Alloc>
	512	constexpr flat_multimap(sorted_equivalent_t, initializer_list<value_type> il,
	513	const key_compare& comp, const Alloc& a);
	514
	515	flat_multimap& operator=(initializer_list<value_type>);
	516
	517	// iterators
	518	constexpr iterator begin() noexcept;
	519	constexpr const_iterator begin() const noexcept;
	520	constexpr iterator end() noexcept;
	521	constexpr const_iterator end() const noexcept;
	522
	523	constexpr reverse_iterator rbegin() noexcept;
	524	constexpr const_reverse_iterator rbegin() const noexcept;
	525	constexpr reverse_iterator rend() noexcept;
	526	constexpr const_reverse_iterator rend() const noexcept;
	527
	528	constexpr const_iterator cbegin() const noexcept;
	529	constexpr const_iterator cend() const noexcept;
	530	constexpr const_reverse_iterator crbegin() const noexcept;
	531	constexpr const_reverse_iterator crend() const noexcept;
	532
	533	// capacity
	534	constexpr bool empty() const noexcept;
	535	constexpr size_type size() const noexcept;
	536	constexpr size_type max_size() const noexcept;
	537
	538	// modifiers
	539	template<class... Args> constexpr iterator emplace(Args&&... args);
	540	template<class... Args>
	541	constexpr iterator emplace_hint(const_iterator position, Args&&... args);
	542
	543	constexpr iterator insert(const value_type& x)
	544	{ return emplace(x); }
	545	constexpr iterator insert(value_type&& x)
	546	{ return emplace(std::move(x)); }
	547	constexpr iterator insert(const_iterator position, const value_type& x)
	548	{ return emplace_hint(position, x); }
	549	constexpr iterator insert(const_iterator position, value_type&& x)
	550	{ return emplace_hint(position, std::move(x)); }
	551
	552	template<class P> constexpr iterator insert(P&& x);
	553	template<class P>
	554	constexpr iterator insert(const_iterator position, P&&);
	555	template<class InputIterator>
	556	constexpr void insert(InputIterator first, InputIterator last);
	557	template<class InputIterator>
	558	constexpr void insert(sorted_equivalent_t, InputIterator first, InputIterator last);
	559	template<container-compatible-range<value_type> R>
	560	constexpr void insert_range(R&& rg);
	561	template<container-compatible-range<value_type> R>
	562	constexpr void insert_range(sorted_equivalent_t, R&& rg);
	563
	564	constexpr void insert(initializer_list<value_type> il)
	565	{ insert(il.begin(), il.end()); }
	566	constexpr void insert(sorted_equivalent_t, initializer_list<value_type> il)
	567	{ insert(sorted_equivalent, il.begin(), il.end()); }
	568
	569	constexpr containers extract() &&;
	570	constexpr void replace(key_container_type&& key_cont, mapped_container_type&& mapped_cont);
	571
	572	constexpr iterator erase(iterator position);
	573	constexpr iterator erase(const_iterator position);
	574	constexpr size_type erase(const key_type& x);
	575	template<class K> constexpr size_type erase(K&& x);
	576	constexpr iterator erase(const_iterator first, const_iterator last);
	577
	578	constexpr void swap(flat_multimap&)
	579	noexcept(is_nothrow_swappable_v<key_container_type> &&
	580	is_nothrow_swappable_v<mapped_container_type> &&
	581	is_nothrow_swappable_v<key_compare>);
	582	constexpr void clear() noexcept;
	583
	584	// observers
	585	constexpr key_compare key_comp() const;
	586	constexpr value_compare value_comp() const;
	587
	588	constexpr const key_container_type& keys() const noexcept { return c.keys; }
	589	constexpr const mapped_container_type& values() const noexcept { return c.values; }
	590
	591	// map operations
	592	constexpr iterator find(const key_type& x);
	593	constexpr const_iterator find(const key_type& x) const;
	594	template<class K> constexpr iterator find(const K& x);
	595	template<class K> constexpr const_iterator find(const K& x) const;
	596
	597	constexpr size_type count(const key_type& x) const;
	598	template<class K> constexpr size_type count(const K& x) const;
	599
	600	constexpr bool contains(const key_type& x) const;
	601	template<class K> constexpr bool contains(const K& x) const;
	602
	603	constexpr iterator lower_bound(const key_type& x);
	604	constexpr const_iterator lower_bound(const key_type& x) const;
	605	template<class K> constexpr iterator lower_bound(const K& x);
	606	template<class K> constexpr const_iterator lower_bound(const K& x) const;
	607
	608	constexpr iterator upper_bound(const key_type& x);
	609	constexpr const_iterator upper_bound(const key_type& x) const;
	610	template<class K> constexpr iterator upper_bound(const K& x);
	611	template<class K> constexpr const_iterator upper_bound(const K& x) const;
	612
	613	constexpr pair<iterator, iterator> equal_range(const key_type& x);
	614	constexpr pair<const_iterator, const_iterator> equal_range(const key_type& x) const;
	615	template<class K>
	616	constexpr pair<iterator, iterator> equal_range(const K& x);
	617	template<class K>
	618	constexpr pair<const_iterator, const_iterator> equal_range(const K& x) const;
	619
	620	friend constexpr bool operator==(const flat_multimap& x, const flat_multimap& y);
	621
	622	friend constexpr synth-three-way-result<value_type>
	623	operator<=>(const flat_multimap& x, const flat_multimap& y);
	624
	625	friend constexpr void swap(flat_multimap& x, flat_multimap& y)
	626	noexcept(noexcept(x.swap(y)))
	627	{ x.swap(y); }
	628
	629	private:
	630	containers c; // exposition only
	631	key_compare compare; // exposition only
	632	};
	633
	634	template<class KeyContainer, class MappedContainer,
	635	class Compare = less<typename KeyContainer::value_type>>
	636	flat_multimap(KeyContainer, MappedContainer, Compare = Compare())
	637	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	638	Compare, KeyContainer, MappedContainer>;
	639
	640	template<class KeyContainer, class MappedContainer, class Allocator>
	641	flat_multimap(KeyContainer, MappedContainer, Allocator)
	642	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	643	less<typename KeyContainer::value_type>, KeyContainer, MappedContainer>;
	644	template<class KeyContainer, class MappedContainer, class Compare, class Allocator>
	645	flat_multimap(KeyContainer, MappedContainer, Compare, Allocator)
	646	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	647	Compare, KeyContainer, MappedContainer>;
	648
	649	template<class KeyContainer, class MappedContainer,
	650	class Compare = less<typename KeyContainer::value_type>>
	651	flat_multimap(sorted_equivalent_t, KeyContainer, MappedContainer, Compare = Compare())
	652	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	653	Compare, KeyContainer, MappedContainer>;
	654
	655	template<class KeyContainer, class MappedContainer, class Allocator>
	656	flat_multimap(sorted_equivalent_t, KeyContainer, MappedContainer, Allocator)
	657	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	658	less<typename KeyContainer::value_type>, KeyContainer, MappedContainer>;
	659	template<class KeyContainer, class MappedContainer, class Compare, class Allocator>
	660	flat_multimap(sorted_equivalent_t, KeyContainer, MappedContainer, Compare, Allocator)
	661	-> flat_multimap<typename KeyContainer::value_type, typename MappedContainer::value_type,
	662	Compare, KeyContainer, MappedContainer>;
	663
	664	template<class InputIterator, class Compare = less<iter-key-type<InputIterator>>>
	665	flat_multimap(InputIterator, InputIterator, Compare = Compare())
	666	-> flat_multimap<iter-key-type<InputIterator>, iter-mapped-type<InputIterator>, Compare>;
	667
	668	template<class InputIterator, class Compare = less<iter-key-type<InputIterator>>>
	669	flat_multimap(sorted_equivalent_t, InputIterator, InputIterator, Compare = Compare())
	670	-> flat_multimap<iter-key-type<InputIterator>, iter-mapped-type<InputIterator>, Compare>;
	671
	672	template<ranges::input_range R, class Compare = less<range-key-type<R>>,
	673	class Allocator = allocator<byte>>
	674	flat_multimap(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
	675	-> flat_multimap<range-key-type<R>, range-mapped-type<R>, Compare,
	676	vector<range-key-type<R>,
	677	alloc-rebind<Allocator, range-key-type<R>>>,
	678	vector<range-mapped-type<R>,
	679	alloc-rebind<Allocator, range-mapped-type<R>>>>;
	680
	681	template<ranges::input_range R, class Allocator>
	682	flat_multimap(from_range_t, R&&, Allocator)
	683	-> flat_multimap<range-key-type<R>, range-mapped-type<R>, less<range-key-type<R>>,
	684	vector<range-key-type<R>,
	685	alloc-rebind<Allocator, range-key-type<R>>>,
	686	vector<range-mapped-type<R>,
	687	alloc-rebind<Allocator, range-mapped-type<R>>>>;
	688
	689	template<class Key, class T, class Compare = less<Key>>
	690	flat_multimap(initializer_list<pair<Key, T>>, Compare = Compare())
	691	-> flat_multimap<Key, T, Compare>;
	692
	693	template<class Key, class T, class Compare = less<Key>>
	694	flat_multimap(sorted_equivalent_t, initializer_list<pair<Key, T>>, Compare = Compare())
	695	-> flat_multimap<Key, T, Compare>;
46	696
47	697	template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,
48		class Allocator>
	698	class Allocator>
49	699	struct uses_allocator<flat_multimap<Key, T, Compare, KeyContainer, MappedContainer>,
50		Allocator>;
	700	Allocator>
	701	: bool_constant<uses_allocator_v<KeyContainer, Allocator> &&
	702	uses_allocator_v<MappedContainer, Allocator>> { };
51	703
52	704	// [flat.multimap.erasure], erasure for flat_multimap
53	705	template<class Key, class T, class Compare, class KeyContainer, class MappedContainer,

lib/libcxx/include/flat_set+543-8

...	...	@@ -17,30 +17,565 @@
17	17	#include <initializer_list> // see [initializer.list.syn]
18	18
19	19	namespace std {
	20	struct sorted_unique_t { explicit sorted_unique_t() = default; };
	21	inline constexpr sorted_unique_t sorted_unique{};
	22
20	23	// [flat.set], class template flat_set
21	24	template<class Key, class Compare = less<Key>, class KeyContainer = vector<Key>>
22		class flat_set;
	25	class flat_set {
	26	public:
	27	// types
	28	using key_type = Key;
	29	using value_type = Key;
	30	using key_compare = Compare;
	31	using value_compare = Compare;
	32	using reference = value_type&;
	33	using const_reference = const value_type&;
	34	using size_type = KeyContainer::size_type;
	35	using difference_type = KeyContainer::difference_type;
	36	using iterator = implementation-defined; // see [container.requirements]
	37	using const_iterator = implementation-defined; // see [container.requirements]
	38	using reverse_iterator = std::reverse_iterator<iterator>;
	39	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
	40	using container_type = KeyContainer;
23	41
24		struct sorted_unique_t { explicit sorted_unique_t() = default; };
25		inline constexpr sorted_unique_t sorted_unique{};
	42	// [flat.set.cons], constructors
	43	constexpr flat_set() : flat_set(key_compare()) { }
	44
	45	constexpr flat_set(const flat_set&);
	46	constexpr flat_set(flat_set&&);
	47	constexpr flat_set& operator=(const flat_set&);
	48	constexpr flat_set& operator=(flat_set&&);
	49
	50	constexpr explicit flat_set(const key_compare& comp)
	51	: c(), compare(comp) { }
	52
	53	constexpr explicit flat_set(container_type cont, const key_compare& comp = key_compare());
	54
	55	constexpr flat_set(sorted_unique_t, container_type cont,
	56	const key_compare& comp = key_compare())
	57	: c(std::move(cont)), compare(comp) { }
	58
	59	template<class InputIterator>
	60	constexpr flat_set(InputIterator first, InputIterator last,
	61	const key_compare& comp = key_compare())
	62	: c(), compare(comp)
	63	{ insert(first, last); }
	64
	65	template<class InputIterator>
	66	constexpr flat_set(sorted_unique_t, InputIterator first, InputIterator last,
	67	const key_compare& comp = key_compare())
	68	: c(first, last), compare(comp) { }
	69
	70	template<container-compatible-range<value_type> R>
	71	constexpr flat_set(from_range_t, R&& rg)
	72	: flat_set(from_range, std::forward<R>(rg), key_compare()) { }
	73	template<container-compatible-range<value_type> R>
	74	constexpr flat_set(from_range_t, R&& rg, const key_compare& comp)
	75	: flat_set(comp)
	76	{ insert_range(std::forward<R>(rg)); }
	77
	78	constexpr flat_set(initializer_list<value_type> il, const key_compare& comp = key_compare())
	79	: flat_set(il.begin(), il.end(), comp) { }
	80
	81	constexpr flat_set(sorted_unique_t, initializer_list<value_type> il,
	82	const key_compare& comp = key_compare())
	83	: flat_set(sorted_unique, il.begin(), il.end(), comp) { }
	84
	85	// [flat.set.cons.alloc], constructors with allocators
	86
	87	template<class Alloc>
	88	constexpr explicit flat_set(const Alloc& a);
	89	template<class Alloc>
	90	constexpr flat_set(const key_compare& comp, const Alloc& a);
	91	template<class Alloc>
	92	constexpr flat_set(const container_type& cont, const Alloc& a);
	93	template<class Alloc>
	94	constexpr flat_set(const container_type& cont, const key_compare& comp, const Alloc& a);
	95	template<class Alloc>
	96	constexpr flat_set(sorted_unique_t, const container_type& cont, const Alloc& a);
	97	template<class Alloc>
	98	constexpr flat_set(sorted_unique_t, const container_type& cont,
	99	const key_compare& comp, const Alloc& a);
	100	template<class Alloc>
	101	constexpr flat_set(const flat_set&, const Alloc& a);
	102	template<class Alloc>
	103	constexpr flat_set(flat_set&&, const Alloc& a);
	104	template<class InputIterator, class Alloc>
	105	constexpr flat_set(InputIterator first, InputIterator last, const Alloc& a);
	106	template<class InputIterator, class Alloc>
	107	constexpr flat_set(InputIterator first, InputIterator last,
	108	const key_compare& comp, const Alloc& a);
	109	template<class InputIterator, class Alloc>
	110	constexpr flat_set(sorted_unique_t, InputIterator first, InputIterator last,
	111	const Alloc& a);
	112	template<class InputIterator, class Alloc>
	113	constexpr flat_set(sorted_unique_t, InputIterator first, InputIterator last,
	114	const key_compare& comp, const Alloc& a);
	115	template<container-compatible-range<value_type> R, class Alloc>
	116	constexpr flat_set(from_range_t, R&& rg, const Alloc& a);
	117	template<container-compatible-range<value_type> R, class Alloc>
	118	constexpr flat_set(from_range_t, R&& rg, const key_compare& comp, const Alloc& a);
	119	template<class Alloc>
	120	constexpr flat_set(initializer_list<value_type> il, const Alloc& a);
	121	template<class Alloc>
	122	constexpr flat_set(initializer_list<value_type> il, const key_compare& comp,
	123	const Alloc& a);
	124	template<class Alloc>
	125	constexpr flat_set(sorted_unique_t, initializer_list<value_type> il, const Alloc& a);
	126	template<class Alloc>
	127	constexpr flat_set(sorted_unique_t, initializer_list<value_type> il,
	128	const key_compare& comp, const Alloc& a);
	129
	130	constexpr flat_set& operator=(initializer_list<value_type>);
	131
	132	// iterators
	133	constexpr iterator begin() noexcept;
	134	constexpr const_iterator begin() const noexcept;
	135	constexpr iterator end() noexcept;
	136	constexpr const_iterator end() const noexcept;
	137
	138	constexpr reverse_iterator rbegin() noexcept;
	139	constexpr const_reverse_iterator rbegin() const noexcept;
	140	constexpr reverse_iterator rend() noexcept;
	141	constexpr const_reverse_iterator rend() const noexcept;
	142
	143	constexpr const_iterator cbegin() const noexcept;
	144	constexpr const_iterator cend() const noexcept;
	145	constexpr const_reverse_iterator crbegin() const noexcept;
	146	constexpr const_reverse_iterator crend() const noexcept;
	147
	148	// capacity
	149	constexpr bool empty() const noexcept;
	150	constexpr size_type size() const noexcept;
	151	constexpr size_type max_size() const noexcept;
	152
	153	// [flat.set.modifiers], modifiers
	154	template<class... Args> constexpr pair<iterator, bool> emplace(Args&&... args);
	155	template<class... Args>
	156	constexpr iterator emplace_hint(const_iterator position, Args&&... args);
	157
	158	constexpr pair<iterator, bool> insert(const value_type& x)
	159	{ return emplace(x); }
	160	constexpr pair<iterator, bool> insert(value_type&& x)
	161	{ return emplace(std::move(x)); }
	162	template<class K> constexpr pair<iterator, bool> insert(K&& x);
	163	constexpr iterator insert(const_iterator position, const value_type& x)
	164	{ return emplace_hint(position, x); }
	165	constexpr iterator insert(const_iterator position, value_type&& x)
	166	{ return emplace_hint(position, std::move(x)); }
	167	template<class K> constexpr iterator insert(const_iterator hint, K&& x);
	168
	169	template<class InputIterator>
	170	constexpr void insert(InputIterator first, InputIterator last);
	171	template<class InputIterator>
	172	constexpr void insert(sorted_unique_t, InputIterator first, InputIterator last);
	173	template<container-compatible-range<value_type> R>
	174	constexpr void insert_range(R&& rg);
	175	template<container-compatible-range<value_type> R>
	176	constexpr void insert_range(sorted_unique_t, R&& rg);
	177
	178	constexpr void insert(initializer_list<value_type> il)
	179	{ insert(il.begin(), il.end()); }
	180	constexpr void insert(sorted_unique_t, initializer_list<value_type> il)
	181	{ insert(sorted_unique, il.begin(), il.end()); }
	182
	183	constexpr container_type extract() &&;
	184	constexpr void replace(container_type&&);
	185
	186	constexpr iterator erase(iterator position) requires (!same_as<iterator, const_iterator>);
	187	constexpr iterator erase(const_iterator position);
	188	constexpr size_type erase(const key_type& x);
	189	template<class K> constexpr size_type erase(K&& x);
	190	constexpr iterator erase(const_iterator first, const_iterator last);
	191
	192	constexpr void swap(flat_set& y) noexcept(see below);
	193	constexpr void clear() noexcept;
	194
	195	// observers
	196	constexpr key_compare key_comp() const;
	197	constexpr value_compare value_comp() const;
	198
	199	// set operations
	200	constexpr iterator find(const key_type& x);
	201	constexpr const_iterator find(const key_type& x) const;
	202	template<class K> constexpr iterator find(const K& x);
	203	template<class K> constexpr const_iterator find(const K& x) const;
	204
	205	constexpr size_type count(const key_type& x) const;
	206	template<class K> constexpr size_type count(const K& x) const;
	207
	208	constexpr bool contains(const key_type& x) const;
	209	template<class K> constexpr bool contains(const K& x) const;
	210
	211	constexpr iterator lower_bound(const key_type& x);
	212	constexpr const_iterator lower_bound(const key_type& x) const;
	213	template<class K> constexpr iterator lower_bound(const K& x);
	214	template<class K> constexpr const_iterator lower_bound(const K& x) const;
	215
	216	constexpr iterator upper_bound(const key_type& x);
	217	constexpr const_iterator upper_bound(const key_type& x) const;
	218	template<class K> constexpr iterator upper_bound(const K& x);
	219	template<class K> constexpr const_iterator upper_bound(const K& x) const;
	220
	221	constexpr pair<iterator, iterator> equal_range(const key_type& x);
	222	constexpr pair<const_iterator, const_iterator> equal_range(const key_type& x) const;
	223	template<class K>
	224	constexpr pair<iterator, iterator> equal_range(const K& x);
	225	template<class K>
	226	constexpr pair<const_iterator, const_iterator> equal_range(const K& x) const;
	227
	228	friend constexpr bool operator==(const flat_set& x, const flat_set& y);
	229
	230	friend constexpr synth-three-way-result<value_type>
	231	operator<=>(const flat_set& x, const flat_set& y);
	232
	233	friend constexpr void swap(flat_set& x, flat_set& y) noexcept(noexcept(x.swap(y)))
	234	{ x.swap(y); }
	235
	236	private:
	237	container_type c; // exposition only
	238	key_compare compare; // exposition only
	239	};
	240
	241	template<class KeyContainer, class Compare = less<typename KeyContainer::value_type>>
	242	flat_set(KeyContainer, Compare = Compare())
	243	-> flat_set<typename KeyContainer::value_type, Compare, KeyContainer>;
	244	template<class KeyContainer, class Allocator>
	245	flat_set(KeyContainer, Allocator)
	246	-> flat_set<typename KeyContainer::value_type,
	247	less<typename KeyContainer::value_type>, KeyContainer>;
	248	template<class KeyContainer, class Compare, class Allocator>
	249	flat_set(KeyContainer, Compare, Allocator)
	250	-> flat_set<typename KeyContainer::value_type, Compare, KeyContainer>;
	251
	252	template<class KeyContainer, class Compare = less<typename KeyContainer::value_type>>
	253	flat_set(sorted_unique_t, KeyContainer, Compare = Compare())
	254	-> flat_set<typename KeyContainer::value_type, Compare, KeyContainer>;
	255	template<class KeyContainer, class Allocator>
	256	flat_set(sorted_unique_t, KeyContainer, Allocator)
	257	-> flat_set<typename KeyContainer::value_type,
	258	less<typename KeyContainer::value_type>, KeyContainer>;
	259	template<class KeyContainer, class Compare, class Allocator>
	260	flat_set(sorted_unique_t, KeyContainer, Compare, Allocator)
	261	-> flat_set<typename KeyContainer::value_type, Compare, KeyContainer>;
	262
	263	template<class InputIterator, class Compare = less<iter-value-type<InputIterator>>>
	264	flat_set(InputIterator, InputIterator, Compare = Compare())
	265	-> flat_set<iter-value-type<InputIterator>, Compare>;
	266
	267	template<class InputIterator, class Compare = less<iter-value-type<InputIterator>>>
	268	flat_set(sorted_unique_t, InputIterator, InputIterator, Compare = Compare())
	269	-> flat_set<iter-value-type<InputIterator>, Compare>;
	270
	271	template<ranges::input_range R, class Compare = less<ranges::range_value_t<R>>,
	272	class Allocator = allocator<ranges::range_value_t<R>>>
	273	flat_set(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
	274	-> flat_set<ranges::range_value_t<R>, Compare,
	275	vector<ranges::range_value_t<R>,
	276	alloc-rebind<Allocator, ranges::range_value_t<R>>>>;
	277
	278	template<ranges::input_range R, class Allocator>
	279	flat_set(from_range_t, R&&, Allocator)
	280	-> flat_set<ranges::range_value_t<R>, less<ranges::range_value_t<R>>,
	281	vector<ranges::range_value_t<R>,
	282	alloc-rebind<Allocator, ranges::range_value_t<R>>>>;
	283
	284	template<class Key, class Compare = less<Key>>
	285	flat_set(initializer_list<Key>, Compare = Compare())
	286	-> flat_set<Key, Compare>;
	287
	288	template<class Key, class Compare = less<Key>>
	289	flat_set(sorted_unique_t, initializer_list<Key>, Compare = Compare())
	290	-> flat_set<Key, Compare>;
26	291
27	292	template<class Key, class Compare, class KeyContainer, class Allocator>
28		struct uses_allocator<flat_set<Key, Compare, KeyContainer>, Allocator>;
	293	struct uses_allocator<flat_set<Key, Compare, KeyContainer>, Allocator>
	294	: bool_constant<uses_allocator_v<KeyContainer, Allocator>> { };
29	295
30	296	// [flat.set.erasure], erasure for flat_set
31	297	template<class Key, class Compare, class KeyContainer, class Predicate>
32	298	typename flat_set<Key, Compare, KeyContainer>::size_type
33	299	erase_if(flat_set<Key, Compare, KeyContainer>& c, Predicate pred);
34	300
	301	struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
	302	inline constexpr sorted_equivalent_t sorted_equivalent{};
	303
35	304	// [flat.multiset], class template flat_multiset
36	305	template<class Key, class Compare = less<Key>, class KeyContainer = vector<Key>>
37		class flat_multiset;
	306	class flat_multiset {
	307	public:
	308	// types
	309	using key_type = Key;
	310	using value_type = Key;
	311	using key_compare = Compare;
	312	using value_compare = Compare;
	313	using reference = value_type&;
	314	using const_reference = const value_type&;
	315	using size_type = KeyContainer::size_type;
	316	using difference_type = KeyContainer::difference_type;
	317	using iterator = implementation-defined; // see [container.requirements]
	318	using const_iterator = implementation-defined; // see [container.requirements]
	319	using reverse_iterator = std::reverse_iterator<iterator>;
	320	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
	321	using container_type = KeyContainer;
38	322
39		struct sorted_equivalent_t { explicit sorted_equivalent_t() = default; };
40		inline constexpr sorted_equivalent_t sorted_equivalent{};
	323	// [flat.multiset.cons], constructors
	324	constexpr flat_multiset() : flat_multiset(key_compare()) { }
	325
	326	constexpr flat_multiset(const flat_multiset&);
	327	constexpr flat_multiset(flat_multiset&&);
	328	constexpr flat_multiset& operator=(const flat_multiset&);
	329	constexpr flat_multiset& operator=(flat_multiset&&);
	330
	331	constexpr explicit flat_multiset(const key_compare& comp)
	332	: c(), compare(comp) { }
	333
	334	constexpr explicit flat_multiset(container_type cont,
	335	const key_compare& comp = key_compare());
	336
	337	constexpr flat_multiset(sorted_equivalent_t, container_type cont,
	338	const key_compare& comp = key_compare())
	339	: c(std::move(cont)), compare(comp) { }
	340
	341	template<class InputIterator>
	342	constexpr flat_multiset(InputIterator first, InputIterator last,
	343	const key_compare& comp = key_compare())
	344	: c(), compare(comp)
	345	{ insert(first, last); }
	346
	347	template<class InputIterator>
	348	constexpr flat_multiset(sorted_equivalent_t, InputIterator first, InputIterator last,
	349	const key_compare& comp = key_compare())
	350	: c(first, last), compare(comp) { }
	351
	352	template<container-compatible-range<value_type> R>
	353	constexpr flat_multiset(from_range_t, R&& rg)
	354	: flat_multiset(from_range, std::forward<R>(rg), key_compare()) { }
	355	template<container-compatible-range<value_type> R>
	356	constexpr flat_multiset(from_range_t, R&& rg, const key_compare& comp)
	357	: flat_multiset(comp)
	358	{ insert_range(std::forward<R>(rg)); }
	359
	360	constexpr flat_multiset(initializer_list<value_type> il,
	361	const key_compare& comp = key_compare())
	362	: flat_multiset(il.begin(), il.end(), comp) { }
	363
	364	constexpr flat_multiset(sorted_equivalent_t, initializer_list<value_type> il,
	365	const key_compare& comp = key_compare())
	366	: flat_multiset(sorted_equivalent, il.begin(), il.end(), comp) { }
	367
	368	// [flat.multiset.cons.alloc], constructors with allocators
	369
	370	template<class Alloc>
	371	constexpr explicit flat_multiset(const Alloc& a);
	372	template<class Alloc>
	373	constexpr flat_multiset(const key_compare& comp, const Alloc& a);
	374	template<class Alloc>
	375	constexpr flat_multiset(const container_type& cont, const Alloc& a);
	376	template<class Alloc>
	377	constexpr flat_multiset(const container_type& cont, const key_compare& comp,
	378	const Alloc& a);
	379	template<class Alloc>
	380	constexpr flat_multiset(sorted_equivalent_t, const container_type& cont, const Alloc& a);
	381	template<class Alloc>
	382	constexpr flat_multiset(sorted_equivalent_t, const container_type& cont,
	383	const key_compare& comp, const Alloc& a);
	384	template<class Alloc>
	385	constexpr flat_multiset(const flat_multiset&, const Alloc& a);
	386	template<class Alloc>
	387	constexpr flat_multiset(flat_multiset&&, const Alloc& a);
	388	template<class InputIterator, class Alloc>
	389	constexpr flat_multiset(InputIterator first, InputIterator last, const Alloc& a);
	390	template<class InputIterator, class Alloc>
	391	constexpr flat_multiset(InputIterator first, InputIterator last,
	392	const key_compare& comp, const Alloc& a);
	393	template<class InputIterator, class Alloc>
	394	constexpr flat_multiset(sorted_equivalent_t, InputIterator first, InputIterator last,
	395	const Alloc& a);
	396	template<class InputIterator, class Alloc>
	397	constexpr flat_multiset(sorted_equivalent_t, InputIterator first, InputIterator last,
	398	const key_compare& comp, const Alloc& a);
	399	template<container-compatible-range<value_type> R, class Alloc>
	400	constexpr flat_multiset(from_range_t, R&& rg, const Alloc& a);
	401	template<container-compatible-range<value_type> R, class Alloc>
	402	constexpr flat_multiset(from_range_t, R&& rg, const key_compare& comp, const Alloc& a);
	403	template<class Alloc>
	404	constexpr flat_multiset(initializer_list<value_type> il, const Alloc& a);
	405	template<class Alloc>
	406	constexpr flat_multiset(initializer_list<value_type> il, const key_compare& comp,
	407	const Alloc& a);
	408	template<class Alloc>
	409	constexpr flat_multiset(sorted_equivalent_t, initializer_list<value_type> il,
	410	const Alloc& a);
	411	template<class Alloc>
	412	constexpr flat_multiset(sorted_equivalent_t, initializer_list<value_type> il,
	413	const key_compare& comp, const Alloc& a);
	414
	415	constexpr flat_multiset& operator=(initializer_list<value_type>);
	416
	417	// iterators
	418	constexpr iterator begin() noexcept;
	419	constexpr const_iterator begin() const noexcept;
	420	constexpr iterator end() noexcept;
	421	constexpr const_iterator end() const noexcept;
	422
	423	constexpr reverse_iterator rbegin() noexcept;
	424	constexpr const_reverse_iterator rbegin() const noexcept;
	425	constexpr reverse_iterator rend() noexcept;
	426	constexpr const_reverse_iterator rend() const noexcept;
	427
	428	constexpr const_iterator cbegin() const noexcept;
	429	constexpr const_iterator cend() const noexcept;
	430	constexpr const_reverse_iterator crbegin() const noexcept;
	431	constexpr const_reverse_iterator crend() const noexcept;
	432
	433	// capacity
	434	constexpr bool empty() const noexcept;
	435	constexpr size_type size() const noexcept;
	436	constexpr size_type max_size() const noexcept;
	437
	438	// [flat.multiset.modifiers], modifiers
	439	template<class... Args> constexpr iterator emplace(Args&&... args);
	440	template<class... Args>
	441	constexpr iterator emplace_hint(const_iterator position, Args&&... args);
	442
	443	constexpr iterator insert(const value_type& x)
	444	{ return emplace(x); }
	445	constexpr iterator insert(value_type&& x)
	446	{ return emplace(std::move(x)); }
	447	constexpr iterator insert(const_iterator position, const value_type& x)
	448	{ return emplace_hint(position, x); }
	449	constexpr iterator insert(const_iterator position, value_type&& x)
	450	{ return emplace_hint(position, std::move(x)); }
	451
	452	template<class InputIterator>
	453	constexpr void insert(InputIterator first, InputIterator last);
	454	template<class InputIterator>
	455	constexpr void insert(sorted_equivalent_t, InputIterator first, InputIterator last);
	456	template<container-compatible-range<value_type> R>
	457	constexpr void insert_range(R&& rg);
	458	template<container-compatible-range<value_type> R>
	459	constexpr void insert_range(sorted_equivalent_t, R&& rg);
	460
	461	constexpr void insert(initializer_list<value_type> il)
	462	{ insert(il.begin(), il.end()); }
	463	constexpr void insert(sorted_equivalent_t, initializer_list<value_type> il)
	464	{ insert(sorted_equivalent, il.begin(), il.end()); }
	465
	466	constexpr container_type extract() &&;
	467	constexpr void replace(container_type&&);
	468
	469	constexpr iterator erase(iterator position) requires (!same_as<iterator, const_iterator>);
	470	constexpr iterator erase(const_iterator position);
	471	constexpr size_type erase(const key_type& x);
	472	template<class K> constexpr size_type erase(K&& x);
	473	constexpr iterator erase(const_iterator first, const_iterator last);
	474
	475	constexpr void swap(flat_multiset& y) noexcept(see below);
	476	constexpr void clear() noexcept;
	477
	478	// observers
	479	constexpr key_compare key_comp() const;
	480	constexpr value_compare value_comp() const;
	481
	482	// set operations
	483	constexpr iterator find(const key_type& x);
	484	constexpr const_iterator find(const key_type& x) const;
	485	template<class K> constexpr iterator find(const K& x);
	486	template<class K> constexpr const_iterator find(const K& x) const;
	487
	488	constexpr size_type count(const key_type& x) const;
	489	template<class K> constexpr size_type count(const K& x) const;
	490
	491	constexpr bool contains(const key_type& x) const;
	492	template<class K> constexpr bool contains(const K& x) const;
	493
	494	constexpr iterator lower_bound(const key_type& x);
	495	constexpr const_iterator lower_bound(const key_type& x) const;
	496	template<class K> constexpr iterator lower_bound(const K& x);
	497	template<class K> constexpr const_iterator lower_bound(const K& x) const;
	498
	499	constexpr iterator upper_bound(const key_type& x);
	500	constexpr const_iterator upper_bound(const key_type& x) const;
	501	template<class K> constexpr iterator upper_bound(const K& x);
	502	template<class K> constexpr const_iterator upper_bound(const K& x) const;
	503
	504	constexpr pair<iterator, iterator> equal_range(const key_type& x);
	505	constexpr pair<const_iterator, const_iterator> equal_range(const key_type& x) const;
	506	template<class K>
	507	constexpr pair<iterator, iterator> equal_range(const K& x);
	508	template<class K>
	509	constexpr pair<const_iterator, const_iterator> equal_range(const K& x) const;
	510
	511	friend constexpr bool operator==(const flat_multiset& x, const flat_multiset& y);
	512
	513	friend constexpr synth-three-way-result<value_type>
	514	operator<=>(const flat_multiset& x, const flat_multiset& y);
	515
	516	friend constexpr void swap(flat_multiset& x, flat_multiset& y)
	517	noexcept(noexcept(x.swap(y)))
	518	{ x.swap(y); }
	519
	520	private:
	521	container_type c; // exposition only
	522	key_compare compare; // exposition only
	523	};
	524
	525	template<class KeyContainer, class Compare = less<typename KeyContainer::value_type>>
	526	flat_multiset(KeyContainer, Compare = Compare())
	527	-> flat_multiset<typename KeyContainer::value_type, Compare, KeyContainer>;
	528	template<class KeyContainer, class Allocator>
	529	flat_multiset(KeyContainer, Allocator)
	530	-> flat_multiset<typename KeyContainer::value_type,
	531	less<typename KeyContainer::value_type>, KeyContainer>;
	532	template<class KeyContainer, class Compare, class Allocator>
	533	flat_multiset(KeyContainer, Compare, Allocator)
	534	-> flat_multiset<typename KeyContainer::value_type, Compare, KeyContainer>;
	535
	536	template<class KeyContainer, class Compare = less<typename KeyContainer::value_type>>
	537	flat_multiset(sorted_equivalent_t, KeyContainer, Compare = Compare())
	538	-> flat_multiset<typename KeyContainer::value_type, Compare, KeyContainer>;
	539	template<class KeyContainer, class Allocator>
	540	flat_multiset(sorted_equivalent_t, KeyContainer, Allocator)
	541	-> flat_multiset<typename KeyContainer::value_type,
	542	less<typename KeyContainer::value_type>, KeyContainer>;
	543	template<class KeyContainer, class Compare, class Allocator>
	544	flat_multiset(sorted_equivalent_t, KeyContainer, Compare, Allocator)
	545	-> flat_multiset<typename KeyContainer::value_type, Compare, KeyContainer>;
	546
	547	template<class InputIterator, class Compare = less<iter-value-type<InputIterator>>>
	548	flat_multiset(InputIterator, InputIterator, Compare = Compare())
	549	-> flat_multiset<iter-value-type<InputIterator>, Compare>;
	550
	551	template<class InputIterator, class Compare = less<iter-value-type<InputIterator>>>
	552	flat_multiset(sorted_equivalent_t, InputIterator, InputIterator, Compare = Compare())
	553	-> flat_multiset<iter-value-type<InputIterator>, Compare>;
	554
	555	template<ranges::input_range R, class Compare = less<ranges::range_value_t<R>>,
	556	class Allocator = allocator<ranges::range_value_t<R>>>
	557	flat_multiset(from_range_t, R&&, Compare = Compare(), Allocator = Allocator())
	558	-> flat_multiset<ranges::range_value_t<R>, Compare,
	559	vector<ranges::range_value_t<R>,
	560	alloc-rebind<Allocator, ranges::range_value_t<R>>>>;
	561
	562	template<ranges::input_range R, class Allocator>
	563	flat_multiset(from_range_t, R&&, Allocator)
	564	-> flat_multiset<ranges::range_value_t<R>, less<ranges::range_value_t<R>>,
	565	vector<ranges::range_value_t<R>,
	566	alloc-rebind<Allocator, ranges::range_value_t<R>>>>;
	567
	568	template<class Key, class Compare = less<Key>>
	569	flat_multiset(initializer_list<Key>, Compare = Compare())
	570	-> flat_multiset<Key, Compare>;
	571
	572	template<class Key, class Compare = less<Key>>
	573	flat_multiset(sorted_equivalent_t, initializer_list<Key>, Compare = Compare())
	574	-> flat_multiset<Key, Compare>;
41	575
42	576	template<class Key, class Compare, class KeyContainer, class Allocator>
43		struct uses_allocator<flat_multiset<Key, Compare, KeyContainer>, Allocator>;
	577	struct uses_allocator<flat_multiset<Key, Compare, KeyContainer>, Allocator>
	578	: bool_constant<uses_allocator_v<KeyContainer, Allocator>> { };
44	579
45	580	// [flat.multiset.erasure], erasure for flat_multiset
46	581	template<class Key, class Compare, class KeyContainer, class Predicate>

lib/libcxx/include/float.h+16-16

...	...	@@ -71,29 +71,29 @@ Macros:
71	71	*/
72	72
73	73	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
74		# include <__cxx03/float.h>
	74	# include <__cxx03/__config>
75	75	#else
76	76	# include <__config>
	77	#endif
77	78
78		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
79		# pragma GCC system_header
80		# endif
	79	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	80	# pragma GCC system_header
	81	#endif
81	82
82		# if __has_include_next(<float.h>)
83		# include_next <float.h>
84		# endif
	83	#if __has_include_next(<float.h>)
	84	# include_next <float.h>
	85	#endif
85	86
86		# ifdef __cplusplus
	87	#ifdef __cplusplus
87	88
88		# ifndef FLT_EVAL_METHOD
89		# define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
90		# endif
	89	# ifndef FLT_EVAL_METHOD
	90	# define FLT_EVAL_METHOD __FLT_EVAL_METHOD__
	91	# endif
91	92
92		# ifndef DECIMAL_DIG
93		# define DECIMAL_DIG __DECIMAL_DIG__
94		# endif
	93	# ifndef DECIMAL_DIG
	94	# define DECIMAL_DIG __DECIMAL_DIG__
	95	# endif
95	96
96		# endif // __cplusplus
97		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	97	#endif // __cplusplus
98	98
99	99	#endif // _LIBCPP_FLOAT_H

lib/libcxx/include/forward_list+33-53

...	...	@@ -223,18 +223,17 @@ template <class T, class Allocator, class Predicate>
223	223	# include <__ranges/concepts.h>
224	224	# include <__ranges/container_compatible_range.h>
225	225	# include <__ranges/from_range.h>
226		# include <__type_traits/conditional.h>
227	226	# include <__type_traits/container_traits.h>
228	227	# include <__type_traits/enable_if.h>
229	228	# include <__type_traits/is_allocator.h>
230	229	# include <__type_traits/is_const.h>
231	230	# include <__type_traits/is_nothrow_assignable.h>
232	231	# include <__type_traits/is_nothrow_constructible.h>
233		# include <__type_traits/is_pointer.h>
234	232	# include <__type_traits/is_same.h>
235	233	# include <__type_traits/is_swappable.h>
236	234	# include <__type_traits/remove_cv.h>
237	235	# include <__type_traits/type_identity.h>
	236	# include <__utility/exception_guard.h>
238	237	# include <__utility/forward.h>
239	238	# include <__utility/move.h>
240	239	# include <__utility/swap.h>
...	...	@@ -283,17 +282,6 @@ struct __forward_node_traits {
283	282	typedef _NodePtr __node_pointer;
284	283	typedef __forward_begin_node<_NodePtr> __begin_node;
285	284	typedef __rebind_pointer_t<_NodePtr, __begin_node> __begin_node_pointer;
286
287		// TODO(LLVM 22): Remove this check
288		# ifndef _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
289		static_assert(sizeof(__begin_node_pointer) == sizeof(__node_pointer) && _LIBCPP_ALIGNOF(__begin_node_pointer) ==
290		_LIBCPP_ALIGNOF(__node_pointer),
291		"It looks like you are using std::forward_list with a fancy pointer type that thas a different "
292		"representation depending on whether it points to a forward_list base pointer or a forward_list node "
293		"pointer (both of which are implementation details of the standard library). This means that your ABI "
294		"is being broken between LLVM 19 and LLVM 20. If you don't care about your ABI being broken, define "
295		"the _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
296		# endif
297	285	};
298	286
299	287	template <class _NodePtr>
...	...	@@ -680,7 +668,7 @@ public:
680	668	# endif
681	669	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI forward_list(size_type __n, const value_type& __v);
682	670
683		template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
	671	template <__enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
684	672	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
685	673	forward_list(size_type __n, const value_type& __v, const allocator_type& __a)
686	674	: __base(__a) {
...	...	@@ -744,50 +732,52 @@ public:
744	732
745	733	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __v);
746	734
747		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	735	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
748	736	return allocator_type(this->__alloc_);
749	737	}
750	738
751		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
	739	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
752	740	return iterator(__base::__before_begin()->__next_);
753	741	}
754		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
	742	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
755	743	return const_iterator(__base::__before_begin()->__next_);
756	744	}
757		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(nullptr); }
758		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
	745	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
	746	return iterator(nullptr);
	747	}
	748	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
759	749	return const_iterator(nullptr);
760	750	}
761	751
762		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
	752	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
763	753	return const_iterator(__base::__before_begin()->__next_);
764	754	}
765		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
	755	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
766	756	return const_iterator(nullptr);
767	757	}
768	758
769		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator before_begin() _NOEXCEPT {
	759	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator before_begin() _NOEXCEPT {
770	760	return iterator(__base::__before_begin());
771	761	}
772		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator before_begin() const _NOEXCEPT {
	762	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator before_begin() const _NOEXCEPT {
773	763	return const_iterator(__base::__before_begin());
774	764	}
775		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbefore_begin() const _NOEXCEPT {
	765	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbefore_begin() const _NOEXCEPT {
776	766	return const_iterator(__base::__before_begin());
777	767	}
778	768
779	769	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
780	770	return __base::__before_begin()->__next_ == nullptr;
781	771	}
782		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	772	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
783	773	return std::min<size_type>(__node_traits::max_size(this->__alloc_), numeric_limits<difference_type>::max());
784	774	}
785	775
786		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
	776	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
787	777	_LIBCPP_ASSERT_NON_NULL(!empty(), "forward_list::front called on an empty list");
788	778	return __base::__before_begin()->__next_->__get_value();
789	779	}
790		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
	780	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
791	781	_LIBCPP_ASSERT_NON_NULL(!empty(), "forward_list::front called on an empty list");
792	782	return __base::__before_begin()->__next_->__get_value();
793	783	}
...	...	@@ -918,22 +908,22 @@ private:
918	908
919	909	# if _LIBCPP_STD_VER >= 17
920	910	template <class _InputIterator,
921		class _Alloc = allocator<__iter_value_type<_InputIterator>>,
	911	class _Alloc = allocator<__iterator_value_type<_InputIterator>>,
922	912	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
923		class = enable_if_t<__is_allocator<_Alloc>::value> >
924		forward_list(_InputIterator, _InputIterator) -> forward_list<__iter_value_type<_InputIterator>, _Alloc>;
	913	class = enable_if_t<__is_allocator_v<_Alloc>>>
	914	forward_list(_InputIterator, _InputIterator) -> forward_list<__iterator_value_type<_InputIterator>, _Alloc>;
925	915
926	916	template <class _InputIterator,
927	917	class _Alloc,
928	918	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
929		class = enable_if_t<__is_allocator<_Alloc>::value> >
930		forward_list(_InputIterator, _InputIterator, _Alloc) -> forward_list<__iter_value_type<_InputIterator>, _Alloc>;
	919	class = enable_if_t<__is_allocator_v<_Alloc>>>
	920	forward_list(_InputIterator, _InputIterator, _Alloc) -> forward_list<__iterator_value_type<_InputIterator>, _Alloc>;
931	921	# endif
932	922
933	923	# if _LIBCPP_STD_VER >= 23
934	924	template <ranges::input_range _Range,
935	925	class _Alloc = allocator<ranges::range_value_t<_Range>>,
936		class = enable_if_t<__is_allocator<_Alloc>::value> >
	926	class = enable_if_t<__is_allocator_v<_Alloc>>>
937	927	forward_list(from_range_t, _Range&&, _Alloc = _Alloc()) -> forward_list<ranges::range_value_t<_Range>, _Alloc>;
938	928	# endif
939	929
...	...	@@ -1180,22 +1170,17 @@ forward_list<_Tp, _Alloc>::__insert_after(const_iterator __p, size_type __n, _Ar
1180	1170	if (__n > 0) {
1181	1171	__node_pointer __first = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
1182	1172	__node_pointer __last = __first;
1183		# if _LIBCPP_HAS_EXCEPTIONS
1184		try {
1185		# endif // _LIBCPP_HAS_EXCEPTIONS
1186		for (--__n; __n != 0; --__n, __last = __last->__next_) {
1187		__last->__next_ = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
1188		}
1189		# if _LIBCPP_HAS_EXCEPTIONS
1190		} catch (...) {
	1173	auto __guard = std::__make_exception_guard([&] {
1191	1174	while (__first != nullptr) {
1192	1175	__node_pointer __next = __first->__next_;
1193	1176	this->__delete_node(__first);
1194	1177	__first = __next;
1195	1178	}
1196		throw;
	1179	});
	1180	for (--__n; __n != 0; --__n, __last = __last->__next_) {
	1181	__last->__next_ = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
1197	1182	}
1198		# endif // _LIBCPP_HAS_EXCEPTIONS
	1183	__guard.__complete();
1199	1184	__last->__next_ = __r->__next_;
1200	1185	__r->__next_ = __first;
1201	1186	__r = std::__static_fancy_pointer_cast<__begin_node_pointer>(__last);
...	...	@@ -1220,22 +1205,17 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
1220	1205	__node_pointer __first = this->__create_node(/* next = / nullptr, __f);
1221	1206	__node_pointer __last = __first;
1222	1207
1223		# if _LIBCPP_HAS_EXCEPTIONS
1224		try {
1225		# endif // _LIBCPP_HAS_EXCEPTIONS
1226		for (++__f; __f != __l; ++__f, ((void)(__last = __last->__next_))) {
1227		__last->__next_ = this->__create_node(/* next = / nullptr, __f);
1228		}
1229		# if _LIBCPP_HAS_EXCEPTIONS
1230		} catch (...) {
	1208	auto __guard = std::__make_exception_guard([&] {
1231	1209	while (__first != nullptr) {
1232	1210	__node_pointer __next = __first->__next_;
1233	1211	this->__delete_node(__first);
1234	1212	__first = __next;
1235	1213	}
1236		throw;
	1214	});
	1215	for (++__f; __f != __l; ++__f, ((void)(__last = __last->__next_))) {
	1216	__last->__next_ = this->__create_node(/* next = / nullptr, __f);
1237	1217	}
1238		# endif // _LIBCPP_HAS_EXCEPTIONS
	1218	__guard.__complete();
1239	1219
1240	1220	__last->__next_ = __r->__next_;
1241	1221	__r->__next_ = __first;

lib/libcxx/include/fstream+57-42

...	...	@@ -221,7 +221,7 @@ _LIBCPP_PUSH_MACROS
221	221
222	222	_LIBCPP_BEGIN_NAMESPACE_STD
223	223
224		# if _LIBCPP_STD_VER >= 26 && defined(_LIBCPP_WIN32API)
	224	# if _LIBCPP_STD_VER >= 23 && defined(_LIBCPP_WIN32API)
225	225	_LIBCPP_EXPORTED_FROM_ABI void* __filebuf_windows_native_handle(FILE* __file) noexcept;
226	226	# endif
227	227
...	...	@@ -254,7 +254,7 @@ public:
254	254	void swap(basic_filebuf& __rhs);
255	255
256	256	// 27.9.1.4 Members:
257		_LIBCPP_HIDE_FROM_ABI bool is_open() const;
	257	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_open() const;
258	258	basic_filebuf* open(const char* __s, ios_base::openmode __mode);
259	259	# if _LIBCPP_HAS_OPEN_WITH_WCHAR
260	260	basic_filebuf* open(const wchar_t* __s, ios_base::openmode __mode);
...	...	@@ -262,15 +262,14 @@ public:
262	262	_LIBCPP_HIDE_FROM_ABI basic_filebuf* open(const string& __s, ios_base::openmode __mode);
263	263
264	264	# if _LIBCPP_STD_VER >= 17
265		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI basic_filebuf*
266		open(const filesystem::path& __p, ios_base::openmode __mode) {
	265	_LIBCPP_HIDE_FROM_ABI basic_filebuf* open(const filesystem::path& __p, ios_base::openmode __mode) {
267	266	return open(__p.c_str(), __mode);
268	267	}
269	268	# endif
270	269	_LIBCPP_HIDE_FROM_ABI basic_filebuf* __open(int __fd, ios_base::openmode __mode);
271	270	basic_filebuf* close();
272	271	# if _LIBCPP_STD_VER >= 26
273		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
	272	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
274	273	_LIBCPP_ASSERT_UNCATEGORIZED(this->is_open(), "File must be opened");
275	274	# if defined(_LIBCPP_WIN32API)
276	275	return std::__filebuf_windows_native_handle(__file_);
...	...	@@ -299,6 +298,16 @@ protected:
299	298	int sync() override;
300	299	void imbue(const locale& __loc) override;
301	300
	301	_LIBCPP_HIDE_FROM_ABI_VIRTUAL streamsize xsputn(const char_type* __str, streamsize __len) override {
	302	if (__always_noconv_ && __len >= (this->epptr() - this->pbase())) {
	303	if (traits_type::eq_int_type(overflow(), traits_type::eof()))
	304	return 0;
	305
	306	return std::fwrite(__str, sizeof(char_type), __len, __file_);
	307	}
	308	return basic_streambuf<_CharT, _Traits>::xsputn(__str, __len);
	309	}
	310
302	311	private:
303	312	char* __extbuf_;
304	313	const char* __extbufnext_;
...	...	@@ -401,6 +410,14 @@ private:
401	410	}
402	411	}
403	412	}
	413
	414	_LIBCPP_HIDE_FROM_ABI typename traits_type::int_type __overflow_failed() {
	415	if (this->pptr() == this->epptr() + 1) {
	416	this->pbump(-1); // lose the character we overflowed above -- we don't really have a
	417	// choice since we couldn't commit the contents of the put area
	418	}
	419	return traits_type::eof();
	420	}
404	421	};
405	422
406	423	template <class _CharT, class _Traits>
...	...	@@ -742,7 +759,12 @@ basic_filebuf<_CharT, _Traits>* basic_filebuf<_CharT, _Traits>::close() {
742	759	if (fclose(__h.release()))
743	760	__rt = nullptr;
744	761	__file_ = nullptr;
745		setbuf(0, 0);
	762	// Reset the get and the put areas without getting rid of the underlying buffers,
	763	// which might have been configured by the user. Make sure to keep the buffers
	764	// since the user may re-open the stream.
	765	this->setg(nullptr, nullptr, nullptr);
	766	this->setp(nullptr, nullptr);
	767	__cm_ = __no_io_operations;
746	768	}
747	769	return __rt;
748	770	}
...	...	@@ -821,14 +843,6 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
821	843
822	844	template <class _CharT, class _Traits>
823	845	typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>::overflow(int_type __c) {
824		auto __failed = [this]() {
825		if (this->pptr() == this->epptr() + 1) {
826		this->pbump(-1); // lose the character we overflowed above -- we don't really have a
827		// choice since we couldn't commit the contents of the put area
828		}
829		return traits_type::eof();
830		};
831
832	846	if (__file_ == nullptr)
833	847	return traits_type::eof();
834	848	__write_mode();
...	...	@@ -850,7 +864,7 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
850	864	if (__always_noconv_) {
851	865	size_t __n = static_cast<size_t>(this->pptr() - this->pbase());
852	866	if (std::fwrite(this->pbase(), sizeof(char_type), __n, __file_) != __n) {
853		return __failed();
	867	return __overflow_failed();
854	868	}
855	869	} else {
856	870	if (!__cv_)
...	...	@@ -864,14 +878,14 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
864	878	do {
865	879	codecvt_base::result __r = __cv_->out(__st_, __b, __p, __end, __extbuf_, __extbuf_ + __ebs_, __extbuf_end);
866	880	if (__end == __b) {
867		return __failed();
	881	return __overflow_failed();
868	882	}
869	883
870	884	// No conversion needed: output characters directly to the file, done.
871	885	if (__r == codecvt_base::noconv) {
872	886	size_t __n = static_cast<size_t>(__p - __b);
873	887	if (std::fwrite(__b, 1, __n, __file_) != __n) {
874		return __failed();
	888	return __overflow_failed();
875	889	}
876	890	break;
877	891
...	...	@@ -879,7 +893,7 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
879	893	} else if (__r == codecvt_base::ok) {
880	894	size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_);
881	895	if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) {
882		return __failed();
	896	return __overflow_failed();
883	897	}
884	898	break;
885	899
...	...	@@ -888,13 +902,13 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
888	902	} else if (__r == codecvt_base::partial) {
889	903	size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_);
890	904	if (std::fwrite(__extbuf_, 1, __n, __file_) != __n) {
891		return __failed();
	905	return __overflow_failed();
892	906	}
893	907	__b = const_cast<char_type*>(__end);
894	908	continue;
895	909
896	910	} else {
897		return __failed();
	911	return __overflow_failed();
898	912	}
899	913	} while (true);
900	914	}
...	...	@@ -977,7 +991,7 @@ template <class _CharT, class _Traits>
977	991	int basic_filebuf<_CharT, _Traits>::__fseek(FILE* __file, pos_type __offset, int __whence) {
978	992	# if defined(_LIBCPP_MSVCRT_LIKE)
979	993	return _fseeki64(__file, __offset, __whence);
980		# elif defined(_NEWLIB_VERSION)
	994	# elif _LIBCPP_LIBC_NEWLIB
981	995	return fseek(__file, __offset, __whence);
982	996	# else
983	997	return ::fseeko(__file, __offset, __whence);
...	...	@@ -988,7 +1002,7 @@ template <class _CharT, class _Traits>
988	1002	typename basic_filebuf<_CharT, _Traits>::pos_type basic_filebuf<_CharT, _Traits>::__ftell(FILE* __file) {
989	1003	# if defined(_LIBCPP_MSVCRT_LIKE)
990	1004	return _ftelli64(__file);
991		# elif defined(_NEWLIB_VERSION)
	1005	# elif _LIBCPP_LIBC_NEWLIB
992	1006	return ftell(__file);
993	1007	# else
994	1008	return ftello(__file);
...	...	@@ -1147,27 +1161,27 @@ public:
1147	1161	_LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const string& __s, ios_base::openmode __mode = ios_base::in);
1148	1162	# if _LIBCPP_STD_VER >= 17
1149	1163	template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
1150		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
1151		_LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const _Tp& __p, ios_base::openmode __mode = ios_base::in)
	1164	_LIBCPP_HIDE_FROM_ABI explicit basic_ifstream(const _Tp& __p, ios_base::openmode __mode = ios_base::in)
1152	1165	: basic_ifstream(__p.c_str(), __mode) {}
1153	1166	# endif // _LIBCPP_STD_VER >= 17
1154	1167	_LIBCPP_HIDE_FROM_ABI basic_ifstream(basic_ifstream&& __rhs);
1155	1168	_LIBCPP_HIDE_FROM_ABI basic_ifstream& operator=(basic_ifstream&& __rhs);
1156	1169	_LIBCPP_HIDE_FROM_ABI void swap(basic_ifstream& __rhs);
1157	1170
1158		_LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
	1171	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
1159	1172	# if _LIBCPP_STD_VER >= 26
1160		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
	1173	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
	1174	return rdbuf()->native_handle();
	1175	}
1161	1176	# endif
1162		_LIBCPP_HIDE_FROM_ABI bool is_open() const;
	1177	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_open() const;
1163	1178	void open(const char* __s, ios_base::openmode __mode = ios_base::in);
1164	1179	# if _LIBCPP_HAS_OPEN_WITH_WCHAR
1165	1180	void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in);
1166	1181	# endif
1167	1182	void open(const string& __s, ios_base::openmode __mode = ios_base::in);
1168	1183	# if _LIBCPP_STD_VER >= 17
1169		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
1170		open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in) {
	1184	_LIBCPP_HIDE_FROM_ABI void open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in) {
1171	1185	return open(__p.c_str(), __mode);
1172	1186	}
1173	1187	# endif // _LIBCPP_STD_VER >= 17
...	...	@@ -1304,8 +1318,7 @@ public:
1304	1318
1305	1319	# if _LIBCPP_STD_VER >= 17
1306	1320	template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
1307		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY
1308		_LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const _Tp& __p, ios_base::openmode __mode = ios_base::out)
	1321	_LIBCPP_HIDE_FROM_ABI explicit basic_ofstream(const _Tp& __p, ios_base::openmode __mode = ios_base::out)
1309	1322	: basic_ofstream(__p.c_str(), __mode) {}
1310	1323	# endif // _LIBCPP_STD_VER >= 17
1311	1324
...	...	@@ -1313,11 +1326,13 @@ public:
1313	1326	_LIBCPP_HIDE_FROM_ABI basic_ofstream& operator=(basic_ofstream&& __rhs);
1314	1327	_LIBCPP_HIDE_FROM_ABI void swap(basic_ofstream& __rhs);
1315	1328
1316		_LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
	1329	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
1317	1330	# if _LIBCPP_STD_VER >= 26
1318		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
	1331	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
	1332	return rdbuf()->native_handle();
	1333	}
1319	1334	# endif
1320		_LIBCPP_HIDE_FROM_ABI bool is_open() const;
	1335	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_open() const;
1321	1336	void open(const char* __s, ios_base::openmode __mode = ios_base::out);
1322	1337	# if _LIBCPP_HAS_OPEN_WITH_WCHAR
1323	1338	void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::out);
...	...	@@ -1325,8 +1340,7 @@ public:
1325	1340	void open(const string& __s, ios_base::openmode __mode = ios_base::out);
1326	1341
1327	1342	# if _LIBCPP_STD_VER >= 17
1328		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
1329		open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::out) {
	1343	_LIBCPP_HIDE_FROM_ABI void open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::out) {
1330	1344	return open(__p.c_str(), __mode);
1331	1345	}
1332	1346	# endif // _LIBCPP_STD_VER >= 17
...	...	@@ -1466,8 +1480,7 @@ public:
1466	1480
1467	1481	# if _LIBCPP_STD_VER >= 17
1468	1482	template <class _Tp, class = enable_if_t<is_same_v<_Tp, filesystem::path>>>
1469		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI explicit basic_fstream(
1470		const _Tp& __p, ios_base::openmode __mode = ios_base::in \| ios_base::out)
	1483	_LIBCPP_HIDE_FROM_ABI explicit basic_fstream(const _Tp& __p, ios_base::openmode __mode = ios_base::in \| ios_base::out)
1471	1484	: basic_fstream(__p.c_str(), __mode) {}
1472	1485	# endif // _LIBCPP_STD_VER >= 17
1473	1486
...	...	@@ -1477,11 +1490,13 @@ public:
1477	1490
1478	1491	_LIBCPP_HIDE_FROM_ABI void swap(basic_fstream& __rhs);
1479	1492
1480		_LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
	1493	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_filebuf<char_type, traits_type>* rdbuf() const;
1481	1494	# if _LIBCPP_STD_VER >= 26
1482		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept { return rdbuf()->native_handle(); }
	1495	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() const noexcept {
	1496	return rdbuf()->native_handle();
	1497	}
1483	1498	# endif
1484		_LIBCPP_HIDE_FROM_ABI bool is_open() const;
	1499	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool is_open() const;
1485	1500	_LIBCPP_HIDE_FROM_ABI void open(const char* __s, ios_base::openmode __mode = ios_base::in \| ios_base::out);
1486	1501	# if _LIBCPP_HAS_OPEN_WITH_WCHAR
1487	1502	void open(const wchar_t* __s, ios_base::openmode __mode = ios_base::in \| ios_base::out);
...	...	@@ -1489,7 +1504,7 @@ public:
1489	1504	_LIBCPP_HIDE_FROM_ABI void open(const string& __s, ios_base::openmode __mode = ios_base::in \| ios_base::out);
1490	1505
1491	1506	# if _LIBCPP_STD_VER >= 17
1492		_LIBCPP_AVAILABILITY_FILESYSTEM_LIBRARY _LIBCPP_HIDE_FROM_ABI void
	1507	_LIBCPP_HIDE_FROM_ABI void
1493	1508	open(const filesystem::path& __p, ios_base::openmode __mode = ios_base::in \| ios_base::out) {
1494	1509	return open(__p.c_str(), __mode);
1495	1510	}

lib/libcxx/include/future+41-51

...	...	@@ -397,12 +397,14 @@ template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
397	397	# include <__type_traits/decay.h>
398	398	# include <__type_traits/enable_if.h>
399	399	# include <__type_traits/invoke.h>
	400	# include <__type_traits/is_constructible.h>
400	401	# include <__type_traits/is_same.h>
401	402	# include <__type_traits/remove_cvref.h>
402	403	# include <__type_traits/remove_reference.h>
403	404	# include <__type_traits/strip_signature.h>
404	405	# include <__type_traits/underlying_type.h>
405	406	# include <__utility/auto_cast.h>
	407	# include <__utility/exception_guard.h>
406	408	# include <__utility/forward.h>
407	409	# include <__utility/move.h>
408	410	# include <__utility/swap.h>
...	...	@@ -477,13 +479,13 @@ inline _LIBCPP_HIDE_FROM_ABI launch& operator^=(launch& __x, launch __y) {
477	479	_LIBCPP_DECLARE_STRONG_ENUM(future_status){ready, timeout, deferred};
478	480	_LIBCPP_DECLARE_STRONG_ENUM_EPILOG(future_status)
479	481
480		_LIBCPP_EXPORTED_FROM_ABI const error_category& future_category() _NOEXCEPT;
	482	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& future_category() _NOEXCEPT;
481	483
482		inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(future_errc __e) _NOEXCEPT {
	484	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(future_errc __e) _NOEXCEPT {
483	485	return error_code(static_cast<int>(__e), future_category());
484	486	}
485	487
486		inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(future_errc __e) _NOEXCEPT {
	488	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(future_errc __e) _NOEXCEPT {
487	489	return error_condition(static_cast<int>(__e), future_category());
488	490	}
489	491
...	...	@@ -502,7 +504,7 @@ public:
502	504	_LIBCPP_HIDE_FROM_ABI explicit future_error(future_errc __ec) : future_error(std::make_error_code(__ec)) {}
503	505	# endif
504	506
505		_LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
	507	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const error_code& code() const _NOEXCEPT { return __ec_; }
506	508
507	509	_LIBCPP_HIDE_FROM_ABI future_error(const future_error&) _NOEXCEPT = default;
508	510	~future_error() _NOEXCEPT override;
...	...	@@ -583,12 +585,9 @@ inline future_status __assoc_sub_state::wait_for(const chrono::duration<_Rep, _P
583	585	template <class _Rp>
584	586	class _LIBCPP_HIDDEN __assoc_state : public __assoc_sub_state {
585	587	typedef __assoc_sub_state base;
586		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
587		typedef typename aligned_storage<sizeof(_Rp), _LIBCPP_ALIGNOF(_Rp)>::type _Up;
588		_LIBCPP_SUPPRESS_DEPRECATED_POP
589	588
590	589	protected:
591		_Up __value_;
	590	_ALIGNAS_TYPE(_Rp) char __value_[sizeof(_Rp)];
592	591
593	592	_LIBCPP_HIDE_FROM_ABI_VIRTUAL void __on_zero_shared() _NOEXCEPT override;
594	593
...	...	@@ -943,15 +942,15 @@ public:
943	942	}
944	943
945	944	_LIBCPP_HIDE_FROM_ABI ~future();
946		_LIBCPP_HIDE_FROM_ABI shared_future<_Rp> share() _NOEXCEPT;
	945	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_future<_Rp> share() _NOEXCEPT;
947	946
948	947	// retrieving the value
949		_LIBCPP_HIDE_FROM_ABI _Rp get();
	948	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Rp get();
950	949
951	950	_LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
952	951
953	952	// functions to check state
954		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	953	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
955	954
956	955	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
957	956	template <class _Rep, class _Period>
...	...	@@ -1014,15 +1013,15 @@ public:
1014	1013	}
1015	1014
1016	1015	_LIBCPP_HIDE_FROM_ABI ~future();
1017		_LIBCPP_HIDE_FROM_ABI shared_future<_Rp&> share() _NOEXCEPT;
	1016	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI shared_future<_Rp&> share() _NOEXCEPT;
1018	1017
1019	1018	// retrieving the value
1020		_LIBCPP_HIDE_FROM_ABI _Rp& get();
	1019	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Rp& get();
1021	1020
1022	1021	_LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1023	1022
1024	1023	// functions to check state
1025		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	1024	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
1026	1025
1027	1026	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
1028	1027	template <class _Rep, class _Period>
...	...	@@ -1089,7 +1088,7 @@ public:
1089	1088	_LIBCPP_HIDE_FROM_ABI void swap(future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1090	1089
1091	1090	// functions to check state
1092		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	1091	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
1093	1092
1094	1093	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
1095	1094	template <class _Rep, class _Period>
...	...	@@ -1139,7 +1138,7 @@ public:
1139	1138	_LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1140	1139
1141	1140	// retrieving the result
1142		_LIBCPP_HIDE_FROM_ABI future<_Rp> get_future();
	1141	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI future<_Rp> get_future();
1143	1142
1144	1143	// setting the result
1145	1144	_LIBCPP_HIDE_FROM_ABI void set_value(const _Rp& __r);
...	...	@@ -1256,7 +1255,7 @@ public:
1256	1255	_LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1257	1256
1258	1257	// retrieving the result
1259		_LIBCPP_HIDE_FROM_ABI future<_Rp&> get_future();
	1258	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI future<_Rp&> get_future();
1260	1259
1261	1260	// setting the result
1262	1261	_LIBCPP_HIDE_FROM_ABI void set_value(_Rp& __r);
...	...	@@ -1366,7 +1365,7 @@ public:
1366	1365	_LIBCPP_HIDE_FROM_ABI void swap(promise& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1367	1366
1368	1367	// retrieving the result
1369		future<void> get_future();
	1368	[[__nodiscard__]] future<void> get_future();
1370	1369
1371	1370	// setting the result
1372	1371	void set_value();
...	...	@@ -1639,10 +1638,10 @@ public:
1639	1638	__p_.swap(__other.__p_);
1640	1639	}
1641	1640
1642		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
	1641	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
1643	1642
1644	1643	// result retrieval
1645		_LIBCPP_HIDE_FROM_ABI future<_Rp> get_future() { return __p_.get_future(); }
	1644	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI future<_Rp> get_future() { return __p_.get_future(); }
1646	1645
1647	1646	// execution
1648	1647	_LIBCPP_HIDE_FROM_ABI void operator()(_ArgTypes... __args);
...	...	@@ -1728,10 +1727,10 @@ public:
1728	1727	__p_.swap(__other.__p_);
1729	1728	}
1730	1729
1731		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
	1730	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __p_.__state_ != nullptr; }
1732	1731
1733	1732	// result retrieval
1734		_LIBCPP_HIDE_FROM_ABI future<void> get_future() { return __p_.get_future(); }
	1733	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI future<void> get_future() { return __p_.get_future(); }
1735	1734
1736	1735	// execution
1737	1736	_LIBCPP_HIDE_FROM_ABI void operator()(_ArgTypes... __args);
...	...	@@ -1815,16 +1814,9 @@ template <class _Rp, class _Fp>
1815	1814	_LIBCPP_HIDE_FROM_ABI future<_Rp> __make_async_assoc_state(_Fp&& __f) {
1816	1815	unique_ptr<__async_assoc_state<_Rp, _Fp>, __release_shared_count> __h(
1817	1816	new __async_assoc_state<_Rp, _Fp>(std::forward<_Fp>(__f)));
1818		# if _LIBCPP_HAS_EXCEPTIONS
1819		try {
1820		# endif
1821		std::thread(&__async_assoc_state<_Rp, _Fp>::__execute, __h.get()).detach();
1822		# if _LIBCPP_HAS_EXCEPTIONS
1823		} catch (...) {
1824		__h->__make_ready();
1825		throw;
1826		}
1827		# endif
	1817	auto __guard = std::__make_exception_guard([&] { __h->__make_ready(); });
	1818	std::thread(&__async_assoc_state<_Rp, _Fp>::__execute, __h.get()).detach();
	1819	__guard.__complete();
1828	1820	return future<_Rp>(__h.get());
1829	1821	}
1830	1822
...	...	@@ -1837,20 +1829,16 @@ class _LIBCPP_HIDDEN __async_func {
1837	1829	public:
1838	1830	using _Rp _LIBCPP_NODEBUG = __invoke_result_t<_Fp, _Args...>;
1839	1831
1840		_LIBCPP_HIDE_FROM_ABI explicit __async_func(_Fp&& __f, _Args&&... __args)
1841		: __f_(std::move(__f), std::move(__args)...) {}
	1832	template <class _Gp, class... _BArgs>
	1833	_LIBCPP_HIDE_FROM_ABI explicit __async_func(_Gp&& __g, _BArgs&&... __bargs)
	1834	: __f_(std::forward<_Gp>(__g), std::forward<_BArgs>(__bargs)...) {}
1842	1835
1843	1836	_LIBCPP_HIDE_FROM_ABI __async_func(__async_func&& __f) : __f_(std::move(__f.__f_)) {}
1844	1837
1845	1838	_LIBCPP_HIDE_FROM_ABI _Rp operator()() {
1846		typedef typename __make_tuple_indices<1 + sizeof...(_Args), 1>::type _Index;
1847		return __execute(_Index());
1848		}
1849
1850		private:
1851		template <size_t... _Indices>
1852		_LIBCPP_HIDE_FROM_ABI _Rp __execute(__tuple_indices<_Indices...>) {
1853		return std::__invoke(std::move(std::get<0>(__f_)), std::move(std::get<_Indices>(__f_))...);
	1839	return [&]<size_t... _Indices>(__index_sequence<_Indices...>) -> _Rp {
	1840	return std::__invoke(std::move(std::get<_Indices>(__f_))...);
	1841	}(__index_sequence_for<_Fp, _Args...>{});
1854	1842	}
1855	1843	};
1856	1844
...	...	@@ -1861,6 +1849,10 @@ inline _LIBCPP_HIDE_FROM_ABI bool __does_policy_contain(launch __policy, launch
1861	1849	template <class _Fp, class... _Args>
1862	1850	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI future<__invoke_result_t<__decay_t<_Fp>, __decay_t<_Args>...> >
1863	1851	async(launch __policy, _Fp&& __f, _Args&&... __args) {
	1852	static_assert(is_constructible<__decay_t<_Fp>, _Fp>::value, "");
	1853	static_assert(_And<is_constructible<__decay_t<_Args>, _Args>...>::value, "");
	1854	static_assert(__is_invocable_v<__decay_t<_Fp>, __decay_t<_Args>...>, "");
	1855
1864	1856	typedef __async_func<__decay_t<_Fp>, __decay_t<_Args>...> _BF;
1865	1857	typedef typename _BF::_Rp _Rp;
1866	1858
...	...	@@ -1868,8 +1860,7 @@ async(launch __policy, _Fp&& __f, _Args&&... __args) {
1868	1860	try {
1869	1861	# endif
1870	1862	if (__does_policy_contain(__policy, launch::async))
1871		return std::__make_async_assoc_state<_Rp>(
1872		_BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
	1863	return std::__make_async_assoc_state<_Rp>(_BF(std::forward<_Fp>(__f), std::forward<_Args>(__args)...));
1873	1864	# if _LIBCPP_HAS_EXCEPTIONS
1874	1865	} catch (...) {
1875	1866	if (__policy == launch::async)
...	...	@@ -1878,8 +1869,7 @@ async(launch __policy, _Fp&& __f, _Args&&... __args) {
1878	1869	# endif
1879	1870
1880	1871	if (__does_policy_contain(__policy, launch::deferred))
1881		return std::__make_deferred_assoc_state<_Rp>(
1882		_BF(_LIBCPP_AUTO_CAST(std::forward<_Fp>(__f)), _LIBCPP_AUTO_CAST(std::forward<_Args>(__args))...));
	1872	return std::__make_deferred_assoc_state<_Rp>(_BF(std::forward<_Fp>(__f), std::forward<_Args>(__args)...));
1883	1873	return future<_Rp>{};
1884	1874	}
1885	1875
...	...	@@ -1915,12 +1905,12 @@ public:
1915	1905	}
1916	1906
1917	1907	// retrieving the value
1918		_LIBCPP_HIDE_FROM_ABI const _Rp& get() const { return __state_->copy(); }
	1908	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const _Rp& get() const { return __state_->copy(); }
1919	1909
1920	1910	_LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1921	1911
1922	1912	// functions to check state
1923		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	1913	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
1924	1914
1925	1915	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
1926	1916	template <class _Rep, class _Period>
...	...	@@ -1971,12 +1961,12 @@ public:
1971	1961	}
1972	1962
1973	1963	// retrieving the value
1974		_LIBCPP_HIDE_FROM_ABI _Rp& get() const { return __state_->copy(); }
	1964	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _Rp& get() const { return __state_->copy(); }
1975	1965
1976	1966	_LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
1977	1967
1978	1968	// functions to check state
1979		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	1969	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
1980	1970
1981	1971	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
1982	1972	template <class _Rep, class _Period>
...	...	@@ -2032,7 +2022,7 @@ public:
2032	2022	_LIBCPP_HIDE_FROM_ABI void swap(shared_future& __rhs) _NOEXCEPT { std::swap(__state_, __rhs.__state_); }
2033	2023
2034	2024	// functions to check state
2035		_LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
	2025	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool valid() const _NOEXCEPT { return __state_ != nullptr; }
2036	2026
2037	2027	_LIBCPP_HIDE_FROM_ABI void wait() const { __state_->wait(); }
2038	2028	template <class _Rep, class _Period>

lib/libcxx/include/initializer_list+9-3

...	...	@@ -78,11 +78,17 @@ public:
78	78
79	79	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 initializer_list() _NOEXCEPT : __begin_(nullptr), __size_(0) {}
80	80
81		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t size() const _NOEXCEPT { return __size_; }
	81	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 size_t size() const _NOEXCEPT {
	82	return __size_;
	83	}
82	84
83		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* begin() const _NOEXCEPT { return __begin_; }
	85	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* begin() const _NOEXCEPT {
	86	return __begin_;
	87	}
84	88
85		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end() const _NOEXCEPT { return __begin_ + __size_; }
	89	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const _Ep* end() const _NOEXCEPT {
	90	return __begin_ + __size_;
	91	}
86	92	};
87	93
88	94	template <class _Ep>

lib/libcxx/include/inttypes.h+16-16

...	...	@@ -236,33 +236,33 @@ uintmax_t wcstoumax(const wchar_t* restrict nptr, wchar_t** restrict endptr, int
236	236	*/
237	237
238	238	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
239		# include <__cxx03/inttypes.h>
	239	# include <__cxx03/__config>
240	240	#else
241	241	# include <__config>
	242	#endif
242	243
243		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
244		# pragma GCC system_header
245		# endif
	244	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	245	# pragma GCC system_header
	246	#endif
246	247
247	248	/* C99 stdlib (e.g. glibc < 2.18) does not provide format macros needed
248	249	for C++11 unless __STDC_FORMAT_MACROS is defined
249	250	*/
250		# if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS)
251		# define __STDC_FORMAT_MACROS
252		# endif
	251	#if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS)
	252	# define __STDC_FORMAT_MACROS
	253	#endif
253	254
254		# if __has_include_next(<inttypes.h>)
255		# include_next <inttypes.h>
256		# endif
	255	#if __has_include_next(<inttypes.h>)
	256	# include_next <inttypes.h>
	257	#endif
257	258
258		# ifdef __cplusplus
	259	#ifdef __cplusplus
259	260
260		# include <stdint.h>
	261	# include <stdint.h>
261	262
262		# undef imaxabs
263		# undef imaxdiv
	263	# undef imaxabs
	264	# undef imaxdiv
264	265
265		# endif // __cplusplus
266		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	266	#endif // __cplusplus
267	267
268	268	#endif // _LIBCPP_INTTYPES_H

lib/libcxx/include/ios+28-28

...	...	@@ -305,25 +305,25 @@ public:
305	305	class _LIBCPP_EXPORTED_FROM_ABI Init;
306	306
307	307	// 27.5.2.2 fmtflags state:
308		_LIBCPP_HIDE_FROM_ABI fmtflags flags() const;
	308	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI fmtflags flags() const;
309	309	_LIBCPP_HIDE_FROM_ABI fmtflags flags(fmtflags __fmtfl);
310	310	_LIBCPP_HIDE_FROM_ABI fmtflags setf(fmtflags __fmtfl);
311	311	_LIBCPP_HIDE_FROM_ABI fmtflags setf(fmtflags __fmtfl, fmtflags __mask);
312	312	_LIBCPP_HIDE_FROM_ABI void unsetf(fmtflags __mask);
313	313
314		_LIBCPP_HIDE_FROM_ABI streamsize precision() const;
	314	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI streamsize precision() const;
315	315	_LIBCPP_HIDE_FROM_ABI streamsize precision(streamsize __prec);
316		_LIBCPP_HIDE_FROM_ABI streamsize width() const;
	316	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI streamsize width() const;
317	317	_LIBCPP_HIDE_FROM_ABI streamsize width(streamsize __wide);
318	318
319	319	// 27.5.2.3 locales:
320	320	locale imbue(const locale& __loc);
321		locale getloc() const;
	321	[[__nodiscard__]] locale getloc() const;
322	322
323	323	// 27.5.2.5 storage:
324		static int xalloc();
325		long& iword(int __index);
326		void*& pword(int __index);
	324	[[__nodiscard__]] static int xalloc();
	325	[[__nodiscard__]] long& iword(int __index);
	326	[[__nodiscard__]] void*& pword(int __index);
327	327
328	328	// destructor
329	329	virtual ~ios_base();
...	...	@@ -338,16 +338,16 @@ public:
338	338
339	339	static bool sync_with_stdio(bool __sync = true);
340	340
341		_LIBCPP_HIDE_FROM_ABI iostate rdstate() const;
	341	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iostate rdstate() const;
342	342	void clear(iostate __state = goodbit);
343	343	_LIBCPP_HIDE_FROM_ABI void setstate(iostate __state);
344	344
345		_LIBCPP_HIDE_FROM_ABI bool good() const;
346		_LIBCPP_HIDE_FROM_ABI bool eof() const;
347		_LIBCPP_HIDE_FROM_ABI bool fail() const;
348		_LIBCPP_HIDE_FROM_ABI bool bad() const;
	345	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool good() const;
	346	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool eof() const;
	347	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool fail() const;
	348	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool bad() const;
349	349
350		_LIBCPP_HIDE_FROM_ABI iostate exceptions() const;
	350	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iostate exceptions() const;
351	351	_LIBCPP_HIDE_FROM_ABI void exceptions(iostate __iostate);
352	352
353	353	void __set_badbit_and_consider_rethrow();
...	...	@@ -425,13 +425,13 @@ template <>
425	425	struct is_error_code_enum<io_errc::__lx> : public true_type {};
426	426	# endif
427	427
428		_LIBCPP_EXPORTED_FROM_ABI const error_category& iostream_category() _NOEXCEPT;
	428	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI const error_category& iostream_category() _NOEXCEPT;
429	429
430		inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(io_errc __e) _NOEXCEPT {
	430	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_code make_error_code(io_errc __e) _NOEXCEPT {
431	431	return error_code(static_cast<int>(__e), iostream_category());
432	432	}
433	433
434		inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(io_errc __e) _NOEXCEPT {
	434	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI error_condition make_error_condition(io_errc __e) _NOEXCEPT {
435	435	return error_condition(static_cast<int>(__e), iostream_category());
436	436	}
437	437
...	...	@@ -580,16 +580,16 @@ public:
580	580	_LIBCPP_HIDE_FROM_ABI explicit operator bool() const { return !fail(); }
581	581	# endif
582	582
583		_LIBCPP_HIDE_FROM_ABI bool operator!() const { return fail(); }
584		_LIBCPP_HIDE_FROM_ABI iostate rdstate() const { return ios_base::rdstate(); }
	583	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool operator!() const { return fail(); }
	584	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iostate rdstate() const { return ios_base::rdstate(); }
585	585	_LIBCPP_HIDE_FROM_ABI void clear(iostate __state = goodbit) { ios_base::clear(__state); }
586	586	_LIBCPP_HIDE_FROM_ABI void setstate(iostate __state) { ios_base::setstate(__state); }
587		_LIBCPP_HIDE_FROM_ABI bool good() const { return ios_base::good(); }
588		_LIBCPP_HIDE_FROM_ABI bool eof() const { return ios_base::eof(); }
589		_LIBCPP_HIDE_FROM_ABI bool fail() const { return ios_base::fail(); }
590		_LIBCPP_HIDE_FROM_ABI bool bad() const { return ios_base::bad(); }
	587	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool good() const { return ios_base::good(); }
	588	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool eof() const { return ios_base::eof(); }
	589	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool fail() const { return ios_base::fail(); }
	590	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool bad() const { return ios_base::bad(); }
591	591
592		_LIBCPP_HIDE_FROM_ABI iostate exceptions() const { return ios_base::exceptions(); }
	592	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iostate exceptions() const { return ios_base::exceptions(); }
593	593	_LIBCPP_HIDE_FROM_ABI void exceptions(iostate __iostate) { ios_base::exceptions(__iostate); }
594	594
595	595	// 27.5.4.1 Constructor/destructor:
...	...	@@ -597,21 +597,21 @@ public:
597	597	~basic_ios() override;
598	598
599	599	// 27.5.4.2 Members:
600		_LIBCPP_HIDE_FROM_ABI basic_ostream<char_type, traits_type>* tie() const;
	600	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_ostream<char_type, traits_type>* tie() const;
601	601	_LIBCPP_HIDE_FROM_ABI basic_ostream<char_type, traits_type>* tie(basic_ostream<char_type, traits_type>* __tiestr);
602	602
603		_LIBCPP_HIDE_FROM_ABI basic_streambuf<char_type, traits_type>* rdbuf() const;
	603	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_streambuf<char_type, traits_type>* rdbuf() const;
604	604	_LIBCPP_HIDE_FROM_ABI basic_streambuf<char_type, traits_type>* rdbuf(basic_streambuf<char_type, traits_type>* __sb);
605	605
606	606	basic_ios& copyfmt(const basic_ios& __rhs);
607	607
608		_LIBCPP_HIDE_FROM_ABI char_type fill() const;
	608	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char_type fill() const;
609	609	_LIBCPP_HIDE_FROM_ABI char_type fill(char_type __ch);
610	610
611	611	_LIBCPP_HIDE_FROM_ABI locale imbue(const locale& __loc);
612	612
613		_LIBCPP_HIDE_FROM_ABI char narrow(char_type __c, char __dfault) const;
614		_LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const;
	613	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char narrow(char_type __c, char __dfault) const;
	614	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char_type widen(char __c) const;
615	615
616	616	protected:
617	617	_LIBCPP_HIDE_FROM_ABI basic_ios() {

lib/libcxx/include/istream+19-13

...	...	@@ -70,6 +70,7 @@ public:
70	70	basic_istream& getline(char_type* s, streamsize n, char_type delim);
71	71
72	72	basic_istream& ignore(streamsize n = 1, int_type delim = traits_type::eof());
	73	basic_istream& ignore(streamsize n, char_type delim); // Since C++26, implemented as a DR
73	74	int_type peek();
74	75	basic_istream& read (char_type* s, streamsize n);
75	76	streamsize readsome(char_type* s, streamsize n);
...	...	@@ -172,6 +173,7 @@ template <class Stream, class T>
172	173	# include <__type_traits/conjunction.h>
173	174	# include <__type_traits/enable_if.h>
174	175	# include <__type_traits/is_base_of.h>
	176	# include <__type_traits/is_same.h>
175	177	# include <__type_traits/make_unsigned.h>
176	178	# include <__utility/declval.h>
177	179	# include <__utility/forward.h>
...	...	@@ -207,7 +209,7 @@ public:
207	209	typedef typename traits_type::off_type off_type;
208	210
209	211	// 27.7.1.1.1 Constructor/destructor:
210		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_istream(basic_streambuf<char_type, traits_type>* __sb)
	212	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 explicit basic_istream(basic_streambuf<char_type, traits_type>* __sb)
211	213	: __gc_(0) {
212	214	this->init(__sb);
213	215	}
...	...	@@ -219,7 +221,7 @@ protected:
219	221	// 27.7.1.1.2 Assign/swap:
220	222	inline _LIBCPP_HIDE_FROM_ABI basic_istream& operator=(basic_istream&& __rhs);
221	223
222		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_istream& __rhs) {
	224	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void swap(basic_istream& __rhs) {
223	225	std::swap(__gc_, __rhs.__gc_);
224	226	basic_ios<char_type, traits_type>::swap(__rhs);
225	227	}
...	...	@@ -232,17 +234,17 @@ public:
232	234	class sentry;
233	235
234	236	// 27.7.1.2 Formatted input:
235		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(basic_istream& (*__pf)(basic_istream&)) {
	237	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& operator>>(basic_istream& (*__pf)(basic_istream&)) {
236	238	return __pf(*this);
237	239	}
238	240
239		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream&
	241	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream&
240	242	operator>>(basic_ios<char_type, traits_type>& (*__pf)(basic_ios<char_type, traits_type>&)) {
241	243	__pf(*this);
242	244	return *this;
243	245	}
244	246
245		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& operator>>(ios_base& (*__pf)(ios_base&)) {
	247	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& operator>>(ios_base& (*__pf)(ios_base&)) {
246	248	__pf(*this);
247	249	return *this;
248	250	}
...	...	@@ -263,35 +265,39 @@ public:
263	265	basic_istream& operator>>(void*& __p);
264	266
265	267	// 27.7.1.3 Unformatted input:
266		_LIBCPP_HIDE_FROM_ABI streamsize gcount() const { return __gc_; }
	268	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI streamsize gcount() const { return __gc_; }
267	269	int_type get();
268	270
269		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type& __c) {
	271	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& get(char_type& __c) {
270	272	int_type __ch = get();
271	273	if (__ch != traits_type::eof())
272	274	__c = traits_type::to_char_type(__ch);
273	275	return *this;
274	276	}
275	277
276		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(char_type* __s, streamsize __n) {
	278	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& get(char_type* __s, streamsize __n) {
277	279	return get(__s, __n, this->widen('\n'));
278	280	}
279	281
280	282	basic_istream& get(char_type* __s, streamsize __n, char_type __dlm);
281	283
282		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& get(basic_streambuf<char_type, traits_type>& __sb) {
	284	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& get(basic_streambuf<char_type, traits_type>& __sb) {
283	285	return get(__sb, this->widen('\n'));
284	286	}
285	287
286	288	basic_istream& get(basic_streambuf<char_type, traits_type>& __sb, char_type __dlm);
287	289
288		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_istream& getline(char_type* __s, streamsize __n) {
	290	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_istream& getline(char_type* __s, streamsize __n) {
289	291	return getline(__s, __n, this->widen('\n'));
290	292	}
291	293
292	294	basic_istream& getline(char_type* __s, streamsize __n, char_type __dlm);
293	295
294	296	basic_istream& ignore(streamsize __n = 1, int_type __dlm = traits_type::eof());
	297	template <class _Tp = char_type, __enable_if_t<is_same<_Tp, char>::value, int> = 0>
	298	_LIBCPP_HIDE_FROM_ABI basic_istream& ignore(streamsize __n, char_type __delim) {
	299	return ignore(__n, traits_type::to_int_type(__delim));
	300	}
295	301	int_type peek();
296	302	basic_istream& read(char_type* __s, streamsize __n);
297	303	streamsize readsome(char_type* __s, streamsize __n);
...	...	@@ -300,7 +306,7 @@ public:
300	306	basic_istream& unget();
301	307	int sync();
302	308
303		pos_type tellg();
	309	[[__nodiscard__]] pos_type tellg();
304	310	basic_istream& seekg(pos_type __pos);
305	311	basic_istream& seekg(off_type __off, ios_base::seekdir __dir);
306	312	};
...	...	@@ -1178,7 +1184,7 @@ public:
1178	1184	typedef typename traits_type::off_type off_type;
1179	1185
1180	1186	// constructor/destructor
1181		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit basic_iostream(basic_streambuf<char_type, traits_type>* __sb)
	1187	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 explicit basic_iostream(basic_streambuf<char_type, traits_type>* __sb)
1182	1188	: basic_istream<_CharT, _Traits>(__sb) {}
1183	1189
1184	1190	~basic_iostream() override;
...	...	@@ -1189,7 +1195,7 @@ protected:
1189	1195	// assign/swap
1190	1196	inline _LIBCPP_HIDE_FROM_ABI basic_iostream& operator=(basic_iostream&& __rhs);
1191	1197
1192		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void swap(basic_iostream& __rhs) {
	1198	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void swap(basic_iostream& __rhs) {
1193	1199	basic_istream<char_type, traits_type>::swap(__rhs);
1194	1200	}
1195	1201	};

lib/libcxx/include/iterator+10

...	...	@@ -737,6 +737,16 @@ template <class E> constexpr const E* data(initializer_list<E> il) noexcept;
737	737	# include <compare>
738	738	# include <concepts>
739	739
	740	// [range.access.general]
	741	# if _LIBCPP_STD_VER >= 20
	742	# include <__ranges/access.h>
	743	# include <__ranges/data.h>
	744	# include <__ranges/empty.h>
	745	# include <__ranges/rbegin.h>
	746	# include <__ranges/rend.h>
	747	# include <__ranges/size.h>
	748	# endif
	749
740	750	# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
741	751	# pragma GCC system_header
742	752	# endif

lib/libcxx/include/latch+7-5

...	...	@@ -70,7 +70,9 @@ class latch {
70	70	atomic<ptrdiff_t> __a_;
71	71
72	72	public:
73		static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept { return numeric_limits<ptrdiff_t>::max(); }
	73	[[nodiscard]] static _LIBCPP_HIDE_FROM_ABI constexpr ptrdiff_t max() noexcept {
	74	return numeric_limits<ptrdiff_t>::max();
	75	}
74	76
75	77	inline _LIBCPP_HIDE_FROM_ABI constexpr explicit latch(ptrdiff_t __expected) : __a_(__expected) {
76	78	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
...	...	@@ -87,7 +89,7 @@ public:
87	89	latch(const latch&) = delete;
88	90	latch& operator=(const latch&) = delete;
89	91
90		inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void count_down(ptrdiff_t __update = 1) {
	92	inline _LIBCPP_HIDE_FROM_ABI void count_down(ptrdiff_t __update = 1) {
91	93	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::count_down called with a negative value");
92	94	auto const __old = __a_.fetch_sub(__update, memory_order_release);
93	95	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
...	...	@@ -97,16 +99,16 @@ public:
97	99	if (__old == __update)
98	100	__a_.notify_all();
99	101	}
100		inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept {
	102	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI bool try_wait() const noexcept {
101	103	auto __value = __a_.load(memory_order_acquire);
102	104	return try_wait_impl(__value);
103	105	}
104		inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait() const {
	106	inline _LIBCPP_HIDE_FROM_ABI void wait() const {
105	107	std::__atomic_wait_unless(__a_, memory_order_acquire, [this](ptrdiff_t& __value) -> bool {
106	108	return try_wait_impl(__value);
107	109	});
108	110	}
109		inline _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) {
	111	inline _LIBCPP_HIDE_FROM_ABI void arrive_and_wait(ptrdiff_t __update = 1) {
110	112	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "latch::arrive_and_wait called with a negative value");
111	113	// other preconditions on __update are checked in count_down()
112	114

lib/libcxx/include/limits+4-4

...	...	@@ -107,7 +107,7 @@ template<> class numeric_limits<cv long double>;
107	107	#else
108	108	# include <__config>
109	109	# include <__type_traits/is_arithmetic.h>
110		# include <__type_traits/is_signed.h>
	110	# include <__type_traits/is_same.h>
111	111
112	112	# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
113	113	# pragma GCC system_header
...	...	@@ -217,10 +217,10 @@ protected:
217	217
218	218	static _LIBCPP_CONSTEXPR const bool is_iec559 = false;
219	219	static _LIBCPP_CONSTEXPR const bool is_bounded = true;
220		static _LIBCPP_CONSTEXPR const bool is_modulo = !std::is_signed<_Tp>::value;
	220	static _LIBCPP_CONSTEXPR const bool is_modulo = !is_signed;
221	221
222		# if defined(__i386__) \|\| defined(__x86_64__) \|\| defined(__pnacl__) \|\| defined(__wasm__)
223		static _LIBCPP_CONSTEXPR const bool traps = true;
	222	# if defined(__i386__) \|\| defined(__x86_64__) \|\| defined(__wasm__)
	223	static _LIBCPP_CONSTEXPR const bool traps = is_same<decltype(+_Tp(0)), _Tp>::value;
224	224	# else
225	225	static _LIBCPP_CONSTEXPR const bool traps = false;
226	226	# endif

lib/libcxx/include/list+61-79

...	...	@@ -228,15 +228,14 @@ template <class T, class Allocator, class Predicate>
228	228	# include <__ranges/concepts.h>
229	229	# include <__ranges/container_compatible_range.h>
230	230	# include <__ranges/from_range.h>
231		# include <__type_traits/conditional.h>
232	231	# include <__type_traits/container_traits.h>
233	232	# include <__type_traits/enable_if.h>
234	233	# include <__type_traits/is_allocator.h>
235	234	# include <__type_traits/is_nothrow_assignable.h>
236	235	# include <__type_traits/is_nothrow_constructible.h>
237		# include <__type_traits/is_pointer.h>
238	236	# include <__type_traits/is_same.h>
239	237	# include <__type_traits/type_identity.h>
	238	# include <__utility/exception_guard.h>
240	239	# include <__utility/forward.h>
241	240	# include <__utility/move.h>
242	241	# include <__utility/swap.h>
...	...	@@ -275,17 +274,6 @@ template <class _Tp, class _VoidPtr>
275	274	struct __list_node_pointer_traits {
276	275	typedef __rebind_pointer_t<_VoidPtr, __list_node<_Tp, _VoidPtr> > __node_pointer;
277	276	typedef __rebind_pointer_t<_VoidPtr, __list_node_base<_Tp, _VoidPtr> > __base_pointer;
278
279		// TODO(LLVM 22): Remove this check
280		# ifndef _LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB
281		static_assert(sizeof(__node_pointer) == sizeof(__node_pointer) && _LIBCPP_ALIGNOF(__base_pointer) ==
282		_LIBCPP_ALIGNOF(__node_pointer),
283		"It looks like you are using std::list with a fancy pointer type that thas a different representation "
284		"depending on whether it points to a list base pointer or a list node pointer (both of which are "
285		"implementation details of the standard library). This means that your ABI is being broken between "
286		"LLVM 19 and LLVM 20. If you don't care about your ABI being broken, define the "
287		"_LIBCPP_ABI_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
288		# endif
289	277	};
290	278
291	279	template <class _Tp, class _VoidPtr>
...	...	@@ -724,7 +712,7 @@ public:
724	712	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n, const allocator_type& __a);
725	713	# endif
726	714	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x);
727		template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
	715	template <__enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
728	716	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
729	717	list(size_type __n, const value_type& __x, const allocator_type& __a)
730	718	: __base(__a) {
...	...	@@ -786,57 +774,71 @@ public:
786	774
787	775	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __x);
788	776
789		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
	777	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
790	778
791		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return this->__size_; }
	779	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT {
	780	return this->__size_;
	781	}
792	782	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
793	783	return __base::empty();
794	784	}
795		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	785	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
796	786	return std::min<size_type>(this->__node_alloc_max_size(), numeric_limits<difference_type >::max());
797	787	}
798	788
799		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __base::begin(); }
800		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __base::begin(); }
801		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __base::end(); }
802		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __base::end(); }
803		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
	789	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT {
	790	return __base::begin();
	791	}
	792	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
	793	return __base::begin();
	794	}
	795	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT {
	796	return __base::end();
	797	}
	798	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
	799	return __base::end();
	800	}
	801	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
804	802	return __base::begin();
805	803	}
806		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __base::end(); }
	804	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
	805	return __base::end();
	806	}
807	807
808		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
	808	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
809	809	return reverse_iterator(end());
810	810	}
811		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	811	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	812	rbegin() const _NOEXCEPT {
812	813	return const_reverse_iterator(end());
813	814	}
814		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
	815	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
815	816	return reverse_iterator(begin());
816	817	}
817		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	818	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
818	819	return const_reverse_iterator(begin());
819	820	}
820		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
	821	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	822	crbegin() const _NOEXCEPT {
821	823	return const_reverse_iterator(end());
822	824	}
823		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
	825	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
824	826	return const_reverse_iterator(begin());
825	827	}
826	828
827		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
	829	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
828	830	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
829	831	return __base::__end_.__next_->__as_node()->__get_value();
830	832	}
831		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
	833	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
832	834	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
833	835	return __base::__end_.__next_->__as_node()->__get_value();
834	836	}
835		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference back() {
	837	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference back() {
836	838	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
837	839	return __base::__end_.__prev_->__as_node()->__get_value();
838	840	}
839		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference back() const {
	841	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference back() const {
840	842	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
841	843	return __base::__end_.__prev_->__as_node()->__get_value();
842	844	}
...	...	@@ -1000,22 +1002,22 @@ private:
1000	1002
1001	1003	# if _LIBCPP_STD_VER >= 17
1002	1004	template <class _InputIterator,
1003		class _Alloc = allocator<__iter_value_type<_InputIterator>>,
	1005	class _Alloc = allocator<__iterator_value_type<_InputIterator>>,
1004	1006	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1005		class = enable_if_t<__is_allocator<_Alloc>::value> >
1006		list(_InputIterator, _InputIterator) -> list<__iter_value_type<_InputIterator>, _Alloc>;
	1007	class = enable_if_t<__is_allocator_v<_Alloc>>>
	1008	list(_InputIterator, _InputIterator) -> list<__iterator_value_type<_InputIterator>, _Alloc>;
1007	1009
1008	1010	template <class _InputIterator,
1009	1011	class _Alloc,
1010	1012	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1011		class = enable_if_t<__is_allocator<_Alloc>::value> >
1012		list(_InputIterator, _InputIterator, _Alloc) -> list<__iter_value_type<_InputIterator>, _Alloc>;
	1013	class = enable_if_t<__is_allocator_v<_Alloc>>>
	1014	list(_InputIterator, _InputIterator, _Alloc) -> list<__iterator_value_type<_InputIterator>, _Alloc>;
1013	1015	# endif
1014	1016
1015	1017	# if _LIBCPP_STD_VER >= 23
1016	1018	template <ranges::input_range _Range,
1017	1019	class _Alloc = allocator<ranges::range_value_t<_Range>>,
1018		class = enable_if_t<__is_allocator<_Alloc>::value> >
	1020	class = enable_if_t<__is_allocator_v<_Alloc>>>
1019	1021	list(from_range_t, _Range&&, _Alloc = _Alloc()) -> list<ranges::range_value_t<_Range>, _Alloc>;
1020	1022	# endif
1021	1023
...	...	@@ -1233,14 +1235,7 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
1233	1235	++__ds;
1234	1236	__r = iterator(__node->__as_link());
1235	1237	iterator __e = __r;
1236		# if _LIBCPP_HAS_EXCEPTIONS
1237		try {
1238		# endif // _LIBCPP_HAS_EXCEPTIONS
1239		for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
1240		__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr, __x)->__as_link();
1241		}
1242		# if _LIBCPP_HAS_EXCEPTIONS
1243		} catch (...) {
	1238	auto __guard = std::__make_exception_guard([&] {
1244	1239	while (true) {
1245	1240	__base_pointer __prev = __e.__ptr_->__prev_;
1246	1241	__node_pointer __current = __e.__ptr_->__as_node();
...	...	@@ -1249,9 +1244,11 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
1249	1244	break;
1250	1245	__e = iterator(__prev);
1251	1246	}
1252		throw;
	1247	});
	1248	for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
	1249	__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr, __x)->__as_link();
1253	1250	}
1254		# endif // _LIBCPP_HAS_EXCEPTIONS
	1251	__guard.__complete();
1255	1252	__link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
1256	1253	this->__size_ += __ds;
1257	1254	}
...	...	@@ -1276,14 +1273,7 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
1276	1273	++__ds;
1277	1274	__r = iterator(__node->__as_link());
1278	1275	iterator __e = __r;
1279		# if _LIBCPP_HAS_EXCEPTIONS
1280		try {
1281		# endif // _LIBCPP_HAS_EXCEPTIONS
1282		for (++__f; __f != __l; ++__f, (void)++__e, ++__ds) {
1283		__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = / nullptr, __f)->__as_link();
1284		}
1285		# if _LIBCPP_HAS_EXCEPTIONS
1286		} catch (...) {
	1276	auto __guard = std::__make_exception_guard([&] {
1287	1277	while (true) {
1288	1278	__base_pointer __prev = __e.__ptr_->__prev_;
1289	1279	__node_pointer __current = __e.__ptr_->__as_node();
...	...	@@ -1292,9 +1282,11 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
1292	1282	break;
1293	1283	__e = iterator(__prev);
1294	1284	}
1295		throw;
	1285	});
	1286	for (++__f; __f != __l; ++__f, (void)++__e, ++__ds) {
	1287	__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = / nullptr, __f)->__as_link();
1296	1288	}
1297		# endif // _LIBCPP_HAS_EXCEPTIONS
	1289	__guard.__complete();
1298	1290	__link_nodes(__p.__ptr_, __r.__ptr_, __e.__ptr_);
1299	1291	this->__size_ += __ds;
1300	1292	}
...	...	@@ -1452,14 +1444,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n) {
1452	1444	++__ds;
1453	1445	iterator __r = iterator(__node->__as_link());
1454	1446	iterator __e = __r;
1455		# if _LIBCPP_HAS_EXCEPTIONS
1456		try {
1457		# endif // _LIBCPP_HAS_EXCEPTIONS
1458		for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
1459		__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr)->__as_link();
1460		}
1461		# if _LIBCPP_HAS_EXCEPTIONS
1462		} catch (...) {
	1447	auto __guard = std::__make_exception_guard([&] {
1463	1448	while (true) {
1464	1449	__base_pointer __prev = __e.__ptr_->__prev_;
1465	1450	__node_pointer __current = __e.__ptr_->__as_node();
...	...	@@ -1468,9 +1453,11 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n) {
1468	1453	break;
1469	1454	__e = iterator(__prev);
1470	1455	}
1471		throw;
	1456	});
	1457	for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
	1458	__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr)->__as_link();
1472	1459	}
1473		# endif // _LIBCPP_HAS_EXCEPTIONS
	1460	__guard.__complete();
1474	1461	__link_nodes_at_back(__r.__ptr_, __e.__ptr_);
1475	1462	this->__size_ += __ds;
1476	1463	}
...	...	@@ -1488,14 +1475,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n, cons
1488	1475	__base_pointer __nl = __node->__as_link();
1489	1476	iterator __r = iterator(__nl);
1490	1477	iterator __e = __r;
1491		# if _LIBCPP_HAS_EXCEPTIONS
1492		try {
1493		# endif // _LIBCPP_HAS_EXCEPTIONS
1494		for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
1495		__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr, __x)->__as_link();
1496		}
1497		# if _LIBCPP_HAS_EXCEPTIONS
1498		} catch (...) {
	1478	auto __guard = std::__make_exception_guard([&] {
1499	1479	while (true) {
1500	1480	__base_pointer __prev = __e.__ptr_->__prev_;
1501	1481	__node_pointer __current = __e.__ptr_->__as_node();
...	...	@@ -1504,9 +1484,11 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n, cons
1504	1484	break;
1505	1485	__e = iterator(__prev);
1506	1486	}
1507		throw;
	1487	});
	1488	for (--__n; __n != 0; --__n, (void)++__e, ++__ds) {
	1489	__e.__ptr_->__next_ = this->__create_node(/* prev = / __e.__ptr_, / next = */ nullptr, __x)->__as_link();
1508	1490	}
1509		# endif // _LIBCPP_HAS_EXCEPTIONS
	1491	__guard.__complete();
1510	1492	__link_nodes(__base::__end_as_link(), __r.__ptr_, __e.__ptr_);
1511	1493	this->__size_ += __ds;
1512	1494	}

lib/libcxx/include/map+386-274

...	...	@@ -577,12 +577,12 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred); // C++20
577	577	# include <__algorithm/equal.h>
578	578	# include <__algorithm/lexicographical_compare.h>
579	579	# include <__algorithm/lexicographical_compare_three_way.h>
	580	# include <__algorithm/specialized_algorithms.h>
580	581	# include <__assert>
581	582	# include <__config>
582	583	# include <__functional/binary_function.h>
583	584	# include <__functional/is_transparent.h>
584	585	# include <__functional/operations.h>
585		# include <__fwd/map.h>
586	586	# include <__iterator/erase_if_container.h>
587	587	# include <__iterator/iterator_traits.h>
588	588	# include <__iterator/ranges_iterator_traits.h>
...	...	@@ -590,19 +590,23 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred); // C++20
590	590	# include <__memory/addressof.h>
591	591	# include <__memory/allocator.h>
592	592	# include <__memory/allocator_traits.h>
	593	# include <__memory/compressed_pair.h>
593	594	# include <__memory/pointer_traits.h>
594	595	# include <__memory/unique_ptr.h>
595	596	# include <__memory_resource/polymorphic_allocator.h>
596	597	# include <__node_handle>
	598	# include <__ranges/access.h>
597	599	# include <__ranges/concepts.h>
598	600	# include <__ranges/container_compatible_range.h>
599	601	# include <__ranges/from_range.h>
600	602	# include <__tree>
601	603	# include <__type_traits/container_traits.h>
602	604	# include <__type_traits/is_allocator.h>
	605	# include <__type_traits/make_transparent.h>
603	606	# include <__type_traits/remove_const.h>
604	607	# include <__type_traits/type_identity.h>
605	608	# include <__utility/forward.h>
	609	# include <__utility/lazy_synth_three_way_comparator.h>
606	610	# include <__utility/pair.h>
607	611	# include <__utility/piecewise_construct.h>
608	612	# include <__utility/swap.h>
...	...	@@ -632,47 +636,9 @@ _LIBCPP_PUSH_MACROS
632	636
633	637	_LIBCPP_BEGIN_NAMESPACE_STD
634	638
635		template <class _Key,
636		class _CP,
637		class _Compare,
638		bool = is_empty<_Compare>::value && !__libcpp_is_final<_Compare>::value>
639		class __map_value_compare : private _Compare {
640		public:
641		_LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
642		: _Compare() {}
643		_LIBCPP_HIDE_FROM_ABI __map_value_compare(_Compare __c) _NOEXCEPT_(is_nothrow_copy_constructible<_Compare>::value)
644		: _Compare(__c) {}
645		_LIBCPP_HIDE_FROM_ABI const _Compare& key_comp() const _NOEXCEPT { return *this; }
646		_LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const {
647		return static_cast<const _Compare&>(*this)(__x.first, __y.first);
648		}
649		_LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const {
650		return static_cast<const _Compare&>(*this)(__x.first, __y);
651		}
652		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const {
653		return static_cast<const _Compare&>(*this)(__x, __y.first);
654		}
655		_LIBCPP_HIDE_FROM_ABI void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
656		using std::swap;
657		swap(static_cast<_Compare&>(*this), static_cast<_Compare&>(__y));
658		}
659
660		# if _LIBCPP_STD_VER >= 14
661		template <typename _K2>
662		_LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
663		return static_cast<const _Compare&>(*this)(__x, __y.first);
664		}
665
666		template <typename _K2>
667		_LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
668		return static_cast<const _Compare&>(*this)(__x.first, __y);
669		}
670		# endif
671		};
672
673	639	template <class _Key, class _CP, class _Compare>
674		class __map_value_compare<_Key, _CP, _Compare, false> {
675		_Compare __comp_;
	640	class __map_value_compare {
	641	_LIBCPP_COMPRESSED_ELEMENT(_Compare, __comp_);
676	642
677	643	public:
678	644	_LIBCPP_HIDE_FROM_ABI __map_value_compare() _NOEXCEPT_(is_nothrow_default_constructible<_Compare>::value)
...	...	@@ -684,7 +650,7 @@ public:
684	650	_LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _CP& __y) const { return __comp_(__x.first, __y.first); }
685	651	_LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _Key& __y) const { return __comp_(__x.first, __y); }
686	652	_LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _CP& __y) const { return __comp_(__x, __y.first); }
687		void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
	653	_LIBCPP_HIDE_FROM_ABI void swap(__map_value_compare& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Compare>) {
688	654	using std::swap;
689	655	swap(__comp_, __y.__comp_);
690	656	}
...	...	@@ -702,9 +668,58 @@ public:
702	668	# endif
703	669	};
704	670
705		template <class _Key, class _CP, class _Compare, bool __b>
	671	template <class _Key, class _MapValueT, class _Compare>
	672	struct __make_transparent<_Key, __map_value_compare<_Key, _MapValueT, _Compare> > {
	673	using type _LIBCPP_NODEBUG = __map_value_compare<_Key, _MapValueT, __make_transparent_t<_Key, _Compare> >;
	674	};
	675
	676	# if _LIBCPP_STD_VER >= 14
	677	template <class _MapValueT, class _Key, class _Compare>
	678	struct __lazy_synth_three_way_comparator<__map_value_compare<_Key, _MapValueT, _Compare>, _MapValueT, _MapValueT> {
	679	__lazy_synth_three_way_comparator<_Compare, _Key, _Key> __comp_;
	680
	681	__lazy_synth_three_way_comparator(
	682	_LIBCPP_CTOR_LIFETIMEBOUND const __map_value_compare<_Key, _MapValueT, _Compare>& __comp)
	683	: __comp_(__comp.key_comp()) {}
	684
	685	_LIBCPP_HIDE_FROM_ABI auto
	686	operator()(_LIBCPP_LIFETIMEBOUND const _MapValueT& __lhs, _LIBCPP_LIFETIMEBOUND const _MapValueT& __rhs) const {
	687	return __comp_(__lhs.first, __rhs.first);
	688	}
	689	};
	690
	691	template <class _MapValueT, class _Key, class _TransparentKey, class _Compare>
	692	struct __lazy_synth_three_way_comparator<__map_value_compare<_Key, _MapValueT, _Compare>, _TransparentKey, _MapValueT> {
	693	__lazy_synth_three_way_comparator<_Compare, _TransparentKey, _Key> __comp_;
	694
	695	__lazy_synth_three_way_comparator(
	696	_LIBCPP_CTOR_LIFETIMEBOUND const __map_value_compare<_Key, _MapValueT, _Compare>& __comp)
	697	: __comp_(__comp.key_comp()) {}
	698
	699	_LIBCPP_HIDE_FROM_ABI auto
	700	operator()(_LIBCPP_LIFETIMEBOUND const _TransparentKey& __lhs, _LIBCPP_LIFETIMEBOUND const _MapValueT& __rhs) const {
	701	return __comp_(__lhs, __rhs.first);
	702	}
	703	};
	704
	705	template <class _MapValueT, class _Key, class _TransparentKey, class _Compare>
	706	struct __lazy_synth_three_way_comparator<__map_value_compare<_Key, _MapValueT, _Compare>, _MapValueT, _TransparentKey> {
	707	__lazy_synth_three_way_comparator<_Compare, _Key, _TransparentKey> __comp_;
	708
	709	__lazy_synth_three_way_comparator(
	710	_LIBCPP_CTOR_LIFETIMEBOUND const __map_value_compare<_Key, _MapValueT, _Compare>& __comp)
	711	: __comp_(__comp.key_comp()) {}
	712
	713	_LIBCPP_HIDE_FROM_ABI auto
	714	operator()(_LIBCPP_LIFETIMEBOUND const _MapValueT& __lhs, _LIBCPP_LIFETIMEBOUND const _TransparentKey& __rhs) const {
	715	return __comp_(__lhs.first, __rhs);
	716	}
	717	};
	718	# endif // _LIBCPP_STD_VER >= 14
	719
	720	template <class _Key, class _CP, class _Compare>
706	721	inline _LIBCPP_HIDE_FROM_ABI void
707		swap(__map_value_compare<_Key, _CP, _Compare, __b>& __x, __map_value_compare<_Key, _CP, _Compare, __b>& __y)
	722	swap(__map_value_compare<_Key, _CP, _Compare>& __x, __map_value_compare<_Key, _CP, _Compare>& __y)
708	723	_NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
709	724	__x.swap(__y);
710	725	}
...	...	@@ -742,9 +757,9 @@ public:
742	757
743	758	_LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
744	759	if (__second_constructed)
745		__alloc_traits::destroy(__na_, std::addressof(__p->__value_.second));
	760	__alloc_traits::destroy(__na_, std::addressof(__p->__get_value().second));
746	761	if (__first_constructed)
747		__alloc_traits::destroy(__na_, std::addressof(__p->__value_.first));
	762	__alloc_traits::destroy(__na_, std::addressof(__p->__get_value().first));
748	763	if (__p)
749	764	__alloc_traits::deallocate(__na_, __p, 1);
750	765	}
...	...	@@ -804,7 +819,26 @@ public:
804	819	friend class multimap;
805	820	template <class>
806	821	friend class __map_const_iterator;
	822
	823	template <class, class...>
	824	friend struct __specialized_algorithm;
	825	};
	826
	827	# ifndef _LIBCPP_CXX03_LANG
	828	template <class _Alg, class _TreeIterator>
	829	struct __specialized_algorithm<_Alg, __iterator_pair<__map_iterator<_TreeIterator>, __map_iterator<_TreeIterator>>> {
	830	using __base _LIBCPP_NODEBUG = __specialized_algorithm<_Alg, __iterator_pair<_TreeIterator, _TreeIterator>>;
	831
	832	static const bool __has_algorithm = __base::__has_algorithm;
	833
	834	using __iterator _LIBCPP_NODEBUG = __map_iterator<_TreeIterator>;
	835
	836	template <class... _Args>
	837	_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Args&&... __args) {
	838	__base()(__first.__i_, __last.__i_, std::forward<_Args>(__args)...);
	839	}
807	840	};
	841	# endif
808	842
809	843	template <class _TreeIterator>
810	844	class __map_const_iterator {
...	...	@@ -859,9 +893,33 @@ public:
859	893	friend class multimap;
860	894	template <class, class, class>
861	895	friend class __tree_const_iterator;
	896
	897	template <class, class...>
	898	friend struct __specialized_algorithm;
862	899	};
863	900
864		template <class _Key, class _Tp, class _Compare, class _Allocator>
	901	# ifndef _LIBCPP_CXX03_LANG
	902	template <class _Alg, class _TreeIterator>
	903	struct __specialized_algorithm<
	904	_Alg,
	905	__iterator_pair<__map_const_iterator<_TreeIterator>, __map_const_iterator<_TreeIterator>>> {
	906	using __base _LIBCPP_NODEBUG = __specialized_algorithm<_Alg, __iterator_pair<_TreeIterator, _TreeIterator>>;
	907
	908	static const bool __has_algorithm = __base::__has_algorithm;
	909
	910	using __iterator _LIBCPP_NODEBUG = __map_const_iterator<_TreeIterator>;
	911
	912	template <class... _Args>
	913	_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Args&&... __args) {
	914	__base()(__first.__i_, __last.__i_, std::forward<_Args>(__args)...);
	915	}
	916	};
	917	# endif
	918
	919	template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
	920	class multimap;
	921
	922	template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
865	923	class map {
866	924	public:
867	925	// types:
...	...	@@ -970,17 +1028,17 @@ public:
970	1028	: map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
971	1029	# endif
972	1030
973		_LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
	1031	_LIBCPP_HIDE_FROM_ABI map(const map& __m) = default;
974	1032
975	1033	_LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
976	1034
977	1035	# ifndef _LIBCPP_CXX03_LANG
978	1036
979		_LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
	1037	_LIBCPP_HIDE_FROM_ABI map(map&& __m) = default;
980	1038
981		_LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
	1039	_LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}
982	1040
983		_LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
	1041	_LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) = default;
984	1042
985	1043	_LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
986	1044	: __tree_(__vc(__comp)) {
...	...	@@ -998,7 +1056,8 @@ public:
998	1056	# endif
999	1057
1000	1058	_LIBCPP_HIDE_FROM_ABI map& operator=(initializer_list<value_type> __il) {
1001		__tree_.__assign_unique(__il.begin(), __il.end());
	1059	clear();
	1060	insert(__il.begin(), __il.end());
1002	1061	return *this;
1003	1062	}
1004	1063
...	...	@@ -1006,43 +1065,66 @@ public:
1006	1065
1007	1066	_LIBCPP_HIDE_FROM_ABI explicit map(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
1008	1067
1009		_LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __a)
1010		: __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
1011		insert(__m.begin(), __m.end());
1012		}
	1068	_LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __alloc) : __tree_(__m.__tree_, __alloc) {}
1013	1069
1014	1070	_LIBCPP_HIDE_FROM_ABI ~map() { static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
1015	1071
1016		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
1017		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
1018		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
1019		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
	1072	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
	1073	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
	1074	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
	1075	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
1020	1076
1021		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
1022		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
1023		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
1024		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	1077	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
	1078	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	1079	return const_reverse_iterator(end());
	1080	}
	1081	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
	1082	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	1083	return const_reverse_iterator(begin());
	1084	}
1025	1085
1026		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
1027		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
1028		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
1029		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	1086	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
	1087	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
	1088	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
	1089	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
1030	1090
1031	1091	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
1032		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
1033		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
	1092	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
	1093	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
1034	1094
1035	1095	_LIBCPP_HIDE_FROM_ABI mapped_type& operator[](const key_type& __k);
1036	1096	# ifndef _LIBCPP_CXX03_LANG
1037	1097	_LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
1038	1098	# endif
1039	1099
1040		_LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
1041		_LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
	1100	template <class _Arg,
	1101	__enable_if_t<__is_transparently_comparable_v<_Compare, key_type, __remove_cvref_t<_Arg> >, int> = 0>
	1102	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI mapped_type& at(_Arg&& __arg) {
	1103	auto [_, __child] = __tree_.__find_equal(__arg);
	1104	if (__child == nullptr)
	1105	std::__throw_out_of_range("map::at: key not found");
	1106	return static_cast<__node_pointer>(__child)->__get_value().second;
	1107	}
	1108
	1109	template <class _Arg,
	1110	__enable_if_t<__is_transparently_comparable_v<_Compare, key_type, __remove_cvref_t<_Arg> >, int> = 0>
	1111	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const mapped_type& at(_Arg&& __arg) const {
	1112	auto [_, __child] = __tree_.__find_equal(__arg);
	1113	if (__child == nullptr)
	1114	std::__throw_out_of_range("map::at: key not found");
	1115	return static_cast<__node_pointer>(__child)->__get_value().second;
	1116	}
1042	1117
1043		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(__tree_.__alloc()); }
1044		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
1045		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
	1118	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
	1119	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
	1120
	1121	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	1122	return allocator_type(__tree_.__alloc());
	1123	}
	1124	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
	1125	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const {
	1126	return value_compare(__tree_.value_comp().key_comp());
	1127	}
1046	1128
1047	1129	# ifndef _LIBCPP_CXX03_LANG
1048	1130	template <class... _Args>
...	...	@@ -1052,7 +1134,7 @@ public:
1052	1134
1053	1135	template <class... _Args>
1054	1136	_LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
1055		return __tree_.__emplace_hint_unique(__p.__i_, std::forward<_Args>(__args)...);
	1137	return __tree_.__emplace_hint_unique(__p.__i_, std::forward<_Args>(__args)...).first;
1056	1138	}
1057	1139
1058	1140	template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
...	...	@@ -1062,7 +1144,7 @@ public:
1062	1144
1063	1145	template <class _Pp, __enable_if_t<is_constructible<value_type, _Pp>::value, int> = 0>
1064	1146	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __pos, _Pp&& __p) {
1065		return __tree_.__emplace_hint_unique(__pos.__i_, std::forward<_Pp>(__p));
	1147	return __tree_.__emplace_hint_unique(__pos.__i_, std::forward<_Pp>(__p)).first;
1066	1148	}
1067	1149
1068	1150	# endif // _LIBCPP_CXX03_LANG
...	...	@@ -1070,7 +1152,7 @@ public:
1070	1152	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__emplace_unique(__v); }
1071	1153
1072	1154	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
1073		return __tree_.__emplace_hint_unique(__p.__i_, __v);
	1155	return __tree_.__emplace_hint_unique(__p.__i_, __v).first;
1074	1156	}
1075	1157
1076	1158	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -1079,25 +1161,21 @@ public:
1079	1161	}
1080	1162
1081	1163	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
1082		return __tree_.__emplace_hint_unique(__p.__i_, std::move(__v));
	1164	return __tree_.__emplace_hint_unique(__p.__i_, std::move(__v)).first;
1083	1165	}
1084	1166
1085	1167	_LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
1086	1168	# endif
1087	1169
1088	1170	template <class _InputIterator>
1089		_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
1090		for (const_iterator __e = cend(); __f != __l; ++__f)
1091		insert(__e.__i_, *__f);
	1171	_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
	1172	__tree_.__insert_range_unique(__first, __last);
1092	1173	}
1093	1174
1094	1175	# if _LIBCPP_STD_VER >= 23
1095	1176	template <_ContainerCompatibleRange<value_type> _Range>
1096	1177	_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
1097		const_iterator __end = cend();
1098		for (auto&& __element : __range) {
1099		insert(__end.__i_, std::forward<decltype(__element)>(__element));
1100		}
	1178	__tree_.__insert_range_unique(ranges::begin(__range), ranges::end(__range));
1101	1179	}
1102	1180	# endif
1103	1181
...	...	@@ -1105,17 +1183,13 @@ public:
1105	1183
1106	1184	template <class... _Args>
1107	1185	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
1108		return __tree_.__emplace_unique_key_args(
1109		__k,
1110		std::piecewise_construct,
1111		std::forward_as_tuple(__k),
1112		std::forward_as_tuple(std::forward<_Args>(__args)...));
	1186	return __tree_.__emplace_unique(
	1187	std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple(std::forward<_Args>(__args)...));
1113	1188	}
1114	1189
1115	1190	template <class... _Args>
1116	1191	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(key_type&& __k, _Args&&... __args) {
1117		return __tree_.__emplace_unique_key_args(
1118		__k,
	1192	return __tree_.__emplace_unique(
1119	1193	std::piecewise_construct,
1120	1194	std::forward_as_tuple(std::move(__k)),
1121	1195	std::forward_as_tuple(std::forward<_Args>(__args)...));
...	...	@@ -1124,9 +1198,8 @@ public:
1124	1198	template <class... _Args>
1125	1199	_LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __h, const key_type& __k, _Args&&... __args) {
1126	1200	return __tree_
1127		.__emplace_hint_unique_key_args(
	1201	.__emplace_hint_unique(
1128	1202	__h.__i_,
1129		__k,
1130	1203	std::piecewise_construct,
1131	1204	std::forward_as_tuple(__k),
1132	1205	std::forward_as_tuple(std::forward<_Args>(__args)...))
...	...	@@ -1136,9 +1209,8 @@ public:
1136	1209	template <class... _Args>
1137	1210	_LIBCPP_HIDE_FROM_ABI iterator try_emplace(const_iterator __h, key_type&& __k, _Args&&... __args) {
1138	1211	return __tree_
1139		.__emplace_hint_unique_key_args(
	1212	.__emplace_hint_unique(
1140	1213	__h.__i_,
1141		__k,
1142	1214	std::piecewise_construct,
1143	1215	std::forward_as_tuple(std::move(__k)),
1144	1216	std::forward_as_tuple(std::forward<_Args>(__args)...))
...	...	@@ -1147,27 +1219,25 @@ public:
1147	1219
1148	1220	template <class _Vp>
1149	1221	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(const key_type& __k, _Vp&& __v) {
1150		iterator __p = lower_bound(__k);
1151		if (__p != end() && !key_comp()(__k, __p->first)) {
1152		__p->second = std::forward<_Vp>(__v);
1153		return std::make_pair(__p, false);
1154		}
1155		return std::make_pair(emplace_hint(__p, __k, std::forward<_Vp>(__v)), true);
	1222	auto __result = __tree_.__emplace_unique(__k, std::forward<_Vp>(__v));
	1223	auto& [__iter, __inserted] = __result;
	1224	if (!__inserted)
	1225	__iter->second = std::forward<_Vp>(__v);
	1226	return __result;
1156	1227	}
1157	1228
1158	1229	template <class _Vp>
1159	1230	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(key_type&& __k, _Vp&& __v) {
1160		iterator __p = lower_bound(__k);
1161		if (__p != end() && !key_comp()(__k, __p->first)) {
1162		__p->second = std::forward<_Vp>(__v);
1163		return std::make_pair(__p, false);
1164		}
1165		return std::make_pair(emplace_hint(__p, std::move(__k), std::forward<_Vp>(__v)), true);
	1231	auto __result = __tree_.__emplace_unique(std::move(__k), std::forward<_Vp>(__v));
	1232	auto& [__iter, __inserted] = __result;
	1233	if (!__inserted)
	1234	__iter->second = std::forward<_Vp>(__v);
	1235	return __result;
1166	1236	}
1167	1237
1168	1238	template <class _Vp>
1169	1239	_LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __h, const key_type& __k, _Vp&& __v) {
1170		auto [__r, __inserted] = __tree_.__emplace_hint_unique_key_args(__h.__i_, __k, __k, std::forward<_Vp>(__v));
	1240	auto [__r, __inserted] = __tree_.__emplace_hint_unique(__h.__i_, __k, std::forward<_Vp>(__v));
1171	1241
1172	1242	if (!__inserted)
1173	1243	__r->second = std::forward<_Vp>(__v);
...	...	@@ -1177,8 +1247,7 @@ public:
1177	1247
1178	1248	template <class _Vp>
1179	1249	_LIBCPP_HIDE_FROM_ABI iterator insert_or_assign(const_iterator __h, key_type&& __k, _Vp&& __v) {
1180		auto [__r, __inserted] =
1181		__tree_.__emplace_hint_unique_key_args(__h.__i_, __k, std::move(__k), std::forward<_Vp>(__v));
	1250	auto [__r, __inserted] = __tree_.__emplace_hint_unique(__h.__i_, std::move(__k), std::forward<_Vp>(__v));
1182	1251
1183	1252	if (!__inserted)
1184	1253	__r->second = std::forward<_Vp>(__v);
...	...	@@ -1207,10 +1276,10 @@ public:
1207	1276	"node_type with incompatible allocator passed to map::insert()");
1208	1277	return __tree_.template __node_handle_insert_unique<node_type>(__hint.__i_, std::move(__nh));
1209	1278	}
1210		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1279	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
1211	1280	return __tree_.template __node_handle_extract<node_type>(__key);
1212	1281	}
1213		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	1282	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
1214	1283	return __tree_.template __node_handle_extract<node_type>(__it.__i_);
1215	1284	}
1216	1285	template <class _Compare2>
...	...	@@ -1241,75 +1310,105 @@ public:
1241	1310
1242	1311	_LIBCPP_HIDE_FROM_ABI void swap(map& __m) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__m.__tree_); }
1243	1312
1244		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
1245		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
	1313	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
	1314	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
1246	1315	# if _LIBCPP_STD_VER >= 14
1247		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1248		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1316	template <typename _K2,
	1317	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1318	int> = 0>
	1319	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
1249	1320	return __tree_.find(__k);
1250	1321	}
1251		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1252		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1322	template <typename _K2,
	1323	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1324	int> = 0>
	1325	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
1253	1326	return __tree_.find(__k);
1254	1327	}
1255	1328	# endif
1256	1329
1257		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
	1330	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1331	return __tree_.__count_unique(__k);
	1332	}
1258	1333	# if _LIBCPP_STD_VER >= 14
1259	1334	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1260		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1335	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
1261	1336	return __tree_.__count_multi(__k);
1262	1337	}
1263	1338	# endif
1264	1339
1265	1340	# if _LIBCPP_STD_VER >= 20
1266		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
1267		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1268		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1341	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1342	template <typename _K2,
	1343	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1344	int> = 0>
	1345	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
1269	1346	return find(__k) != end();
1270	1347	}
1271	1348	# endif // _LIBCPP_STD_VER >= 20
1272	1349
1273		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
1274		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
	1350	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) {
	1351	return __tree_.__lower_bound_unique(__k);
	1352	}
	1353
	1354	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const {
	1355	return __tree_.__lower_bound_unique(__k);
	1356	}
	1357
	1358	// The transparent versions of the lookup functions use the _multi version, since a non-element key is allowed to
	1359	// match multiple elements.
1275	1360	# if _LIBCPP_STD_VER >= 14
1276		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1277		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
1278		return __tree_.lower_bound(__k);
	1361	template <typename _K2,
	1362	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1363	int> = 0>
	1364	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
	1365	return __tree_.__lower_bound_multi(__k);
1279	1366	}
1280	1367
1281		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1282		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
1283		return __tree_.lower_bound(__k);
	1368	template <typename _K2,
	1369	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1370	int> = 0>
	1371	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
	1372	return __tree_.__lower_bound_multi(__k);
1284	1373	}
1285	1374	# endif
1286	1375
1287		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
1288		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
1289		# if _LIBCPP_STD_VER >= 14
1290		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1291		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
1292		return __tree_.upper_bound(__k);
	1376	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) {
	1377	return __tree_.__upper_bound_unique(__k);
1293	1378	}
1294		template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1295		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
1296		return __tree_.upper_bound(__k);
	1379
	1380	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const {
	1381	return __tree_.__upper_bound_unique(__k);
	1382	}
	1383
	1384	# if _LIBCPP_STD_VER >= 14
	1385	template <typename _K2,
	1386	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1387	int> = 0>
	1388	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
	1389	return __tree_.__upper_bound_multi(__k);
	1390	}
	1391	template <typename _K2,
	1392	enable_if_t<__is_transparent_v<_Compare, _K2> \|\| __is_transparently_comparable_v<_Compare, key_type, _K2>,
	1393	int> = 0>
	1394	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
	1395	return __tree_.__upper_bound_multi(__k);
1297	1396	}
1298	1397	# endif
1299	1398
1300		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1399	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
1301	1400	return __tree_.__equal_range_unique(__k);
1302	1401	}
1303		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1402	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
1304	1403	return __tree_.__equal_range_unique(__k);
1305	1404	}
1306	1405	# if _LIBCPP_STD_VER >= 14
1307	1406	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1308		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1407	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
1309	1408	return __tree_.__equal_range_multi(__k);
1310	1409	}
1311	1410	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1312		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1411	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
1313	1412	return __tree_.__equal_range_multi(__k);
1314	1413	}
1315	1414	# endif
...	...	@@ -1319,7 +1418,6 @@ private:
1319	1418	typedef typename __base::__node_allocator __node_allocator;
1320	1419	typedef typename __base::__node_pointer __node_pointer;
1321	1420	typedef typename __base::__node_base_pointer __node_base_pointer;
1322		typedef typename __base::__parent_pointer __parent_pointer;
1323	1421
1324	1422	typedef __map_node_destructor<__node_allocator> _Dp;
1325	1423	typedef unique_ptr<__node, _Dp> __node_holder;
...	...	@@ -1327,6 +1425,8 @@ private:
1327	1425	# ifdef _LIBCPP_CXX03_LANG
1328	1426	_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
1329	1427	# endif
	1428
	1429	friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<map> >;
1330	1430	};
1331	1431
1332	1432	# if _LIBCPP_STD_VER >= 17
...	...	@@ -1334,8 +1434,8 @@ template <class _InputIterator,
1334	1434	class _Compare = less<__iter_key_type<_InputIterator>>,
1335	1435	class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
1336	1436	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1337		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1338		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1437	class = enable_if_t<!__is_allocator_v<_Compare>>,
	1438	class = enable_if_t<__is_allocator_v<_Allocator>>>
1339	1439	map(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
1340	1440	-> map<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
1341	1441
...	...	@@ -1343,8 +1443,8 @@ map(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocat
1343	1443	template <ranges::input_range _Range,
1344	1444	class _Compare = less<__range_key_type<_Range>>,
1345	1445	class _Allocator = allocator<__range_to_alloc_type<_Range>>,
1346		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1347		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1446	class = enable_if_t<!__is_allocator_v<_Compare>>,
	1447	class = enable_if_t<__is_allocator_v<_Allocator>>>
1348	1448	map(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
1349	1449	-> map<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
1350	1450	# endif
...	...	@@ -1353,16 +1453,15 @@ template <class _Key,
1353	1453	class _Tp,
1354	1454	class _Compare = less<remove_const_t<_Key>>,
1355	1455	class _Allocator = allocator<pair<const _Key, _Tp>>,
1356		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1357		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
1358		map(initializer_list<pair<_Key, _Tp>>,
1359		_Compare = _Compare(),
1360		_Allocator = _Allocator()) -> map<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
	1456	class = enable_if_t<!__is_allocator_v<_Compare>>,
	1457	class = enable_if_t<__is_allocator_v<_Allocator>>>
	1458	map(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare(), _Allocator = _Allocator())
	1459	-> map<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
1361	1460
1362	1461	template <class _InputIterator,
1363	1462	class _Allocator,
1364	1463	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1365		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1464	class = enable_if_t<__is_allocator_v<_Allocator>>>
1366	1465	map(_InputIterator, _InputIterator, _Allocator)
1367	1466	-> map<__iter_key_type<_InputIterator>,
1368	1467	__iter_mapped_type<_InputIterator>,
...	...	@@ -1370,44 +1469,44 @@ map(_InputIterator, _InputIterator, _Allocator)
1370	1469	_Allocator>;
1371	1470
1372	1471	# if _LIBCPP_STD_VER >= 23
1373		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1472	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1374	1473	map(from_range_t, _Range&&, _Allocator)
1375	1474	-> map<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
1376	1475	# endif
1377	1476
1378		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
1379		map(initializer_list<pair<_Key, _Tp>>,
1380		_Allocator) -> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
	1477	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
	1478	map(initializer_list<pair<_Key, _Tp>>, _Allocator)
	1479	-> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
1381	1480	# endif
1382	1481
1383		# ifndef _LIBCPP_CXX03_LANG
	1482	# if _LIBCPP_STD_VER >= 14
1384	1483	template <class _Key, class _Tp, class _Compare, class _Allocator>
1385		map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
1386		: __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
1387		if (__a != __m.get_allocator()) {
1388		const_iterator __e = cend();
1389		while (!__m.empty()) {
1390		__tree_.__insert_unique_from_orphaned_node(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_));
1391		}
	1484	struct __specialized_algorithm<_Algorithm::__for_each, __single_range<map<_Key, _Tp, _Compare, _Allocator>>> {
	1485	using __map _LIBCPP_NODEBUG = map<_Key, _Tp, _Compare, _Allocator>;
	1486
	1487	static const bool __has_algorithm = true;
	1488
	1489	template <class _Map, class _Func, class _Proj>
	1490	_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func, _Proj __proj) {
	1491	auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, __single_range<typename __map::__base>>()(
	1492	__map.__tree_, std::move(__func), std::move(__proj));
	1493	return std::make_pair(__map.end(), std::move(__func2));
1392	1494	}
1393		}
	1495	};
	1496	# endif
1394	1497
	1498	# ifndef _LIBCPP_CXX03_LANG
1395	1499	template <class _Key, class _Tp, class _Compare, class _Allocator>
1396	1500	_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
1397		return __tree_
1398		.__emplace_unique_key_args(__k, std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
	1501	return __tree_.__emplace_unique(std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
1399	1502	.first->second;
1400	1503	}
1401	1504
1402	1505	template <class _Key, class _Tp, class _Compare, class _Allocator>
1403	1506	_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](key_type&& __k) {
1404		// TODO investigate this clang-tidy warning.
1405		// NOLINTBEGIN(bugprone-use-after-move)
1406	1507	return __tree_
1407		.__emplace_unique_key_args(
1408		__k, std::piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
	1508	.__emplace_unique(std::piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
1409	1509	.first->second;
1410		// NOLINTEND(bugprone-use-after-move)
1411	1510	}
1412	1511
1413	1512	# else // _LIBCPP_CXX03_LANG
...	...	@@ -1417,44 +1516,41 @@ typename map<_Key, _Tp, _Compare, _Allocator>::__node_holder
1417	1516	map<_Key, _Tp, _Compare, _Allocator>::__construct_node_with_key(const key_type& __k) {
1418	1517	__node_allocator& __na = __tree_.__node_alloc();
1419	1518	__node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
1420		__node_traits::construct(__na, std::addressof(__h->__value_.first), __k);
	1519	__node_traits::construct(__na, std::addressof(__h->__get_value().first), __k);
1421	1520	__h.get_deleter().__first_constructed = true;
1422		__node_traits::construct(__na, std::addressof(__h->__value_.second));
	1521	__node_traits::construct(__na, std::addressof(__h->__get_value().second));
1423	1522	__h.get_deleter().__second_constructed = true;
1424	1523	return __h;
1425	1524	}
1426	1525
1427	1526	template <class _Key, class _Tp, class _Compare, class _Allocator>
1428	1527	_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
1429		__parent_pointer __parent;
1430		__node_base_pointer& __child = __tree_.__find_equal(__parent, __k);
1431		__node_pointer __r = static_cast<__node_pointer>(__child);
	1528	auto [__parent, __child] = __tree_.__find_equal(__k);
	1529	__node_pointer __r = static_cast<__node_pointer>(__child);
1432	1530	if (__child == nullptr) {
1433	1531	__node_holder __h = __construct_node_with_key(__k);
1434	1532	__tree_.__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
1435	1533	__r = __h.release();
1436	1534	}
1437		return __r->__value_.second;
	1535	return __r->__get_value().second;
1438	1536	}
1439	1537
1440	1538	# endif // _LIBCPP_CXX03_LANG
1441	1539
1442	1540	template <class _Key, class _Tp, class _Compare, class _Allocator>
1443	1541	_Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) {
1444		__parent_pointer __parent;
1445		__node_base_pointer& __child = __tree_.__find_equal(__parent, __k);
	1542	auto [_, __child] = __tree_.__find_equal(__k);
1446	1543	if (__child == nullptr)
1447	1544	std::__throw_out_of_range("map::at: key not found");
1448		return static_cast<__node_pointer>(__child)->__value_.second;
	1545	return static_cast<__node_pointer>(__child)->__get_value().second;
1449	1546	}
1450	1547
1451	1548	template <class _Key, class _Tp, class _Compare, class _Allocator>
1452	1549	const _Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) const {
1453		__parent_pointer __parent;
1454		__node_base_pointer __child = __tree_.__find_equal(__parent, __k);
	1550	auto [_, __child] = __tree_.__find_equal(__k);
1455	1551	if (__child == nullptr)
1456	1552	std::__throw_out_of_range("map::at: key not found");
1457		return static_cast<__node_pointer>(__child)->__value_.second;
	1553	return static_cast<__node_pointer>(__child)->__get_value().second;
1458	1554	}
1459	1555
1460	1556	template <class _Key, class _Tp, class _Compare, class _Allocator>
...	...	@@ -1637,22 +1733,17 @@ public:
1637	1733	: multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
1638	1734	# endif
1639	1735
1640		_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
1641		: __tree_(__m.__tree_.value_comp(),
1642		__alloc_traits::select_on_container_copy_construction(__m.__tree_.__alloc())) {
1643		insert(__m.begin(), __m.end());
1644		}
	1736	_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m) = default;
1645	1737
1646	1738	_LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
1647	1739
1648	1740	# ifndef _LIBCPP_CXX03_LANG
1649	1741
1650		_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
	1742	_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) = default;
1651	1743
1652		_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
	1744	_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}
1653	1745
1654		_LIBCPP_HIDE_FROM_ABI multimap&
1655		operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
	1746	_LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) = default;
1656	1747
1657	1748	_LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
1658	1749	: __tree_(__vc(__comp)) {
...	...	@@ -1671,7 +1762,8 @@ public:
1671	1762	# endif
1672	1763
1673	1764	_LIBCPP_HIDE_FROM_ABI multimap& operator=(initializer_list<value_type> __il) {
1674		__tree_.__assign_multi(__il.begin(), __il.end());
	1765	clear();
	1766	insert(__il.begin(), __il.end());
1675	1767	return *this;
1676	1768	}
1677	1769
...	...	@@ -1679,37 +1771,42 @@ public:
1679	1771
1680	1772	_LIBCPP_HIDE_FROM_ABI explicit multimap(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}
1681	1773
1682		_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a)
1683		: __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
1684		insert(__m.begin(), __m.end());
1685		}
	1774	_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a) : __tree_(__m.__tree_, __a) {}
1686	1775
1687	1776	_LIBCPP_HIDE_FROM_ABI ~multimap() {
1688	1777	static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), "");
1689	1778	}
1690	1779
1691		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
1692		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
1693		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
1694		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
	1780	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
	1781	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
	1782	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
	1783	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
1695	1784
1696		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
1697		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
1698		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
1699		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	1785	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
	1786	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	1787	return const_reverse_iterator(end());
	1788	}
	1789	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
	1790	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	1791	return const_reverse_iterator(begin());
	1792	}
1700	1793
1701		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
1702		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
1703		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
1704		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	1794	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
	1795	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
	1796	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
	1797	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
1705	1798
1706	1799	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
1707		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
1708		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
	1800	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
	1801	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
1709	1802
1710		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return allocator_type(__tree_.__alloc()); }
1711		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
1712		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return value_compare(__tree_.value_comp().key_comp()); }
	1803	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	1804	return allocator_type(__tree_.__alloc());
	1805	}
	1806	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp().key_comp(); }
	1807	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const {
	1808	return value_compare(__tree_.value_comp().key_comp());
	1809	}
1713	1810
1714	1811	# ifndef _LIBCPP_CXX03_LANG
1715	1812
...	...	@@ -1751,17 +1848,13 @@ public:
1751	1848
1752	1849	template <class _InputIterator>
1753	1850	_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
1754		for (const_iterator __e = cend(); __f != __l; ++__f)
1755		__tree_.__emplace_hint_multi(__e.__i_, *__f);
	1851	__tree_.__insert_range_multi(__f, __l);
1756	1852	}
1757	1853
1758	1854	# if _LIBCPP_STD_VER >= 23
1759	1855	template <_ContainerCompatibleRange<value_type> _Range>
1760	1856	_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
1761		const_iterator __end = cend();
1762		for (auto&& __element : __range) {
1763		__tree_.__emplace_hint_multi(__end.__i_, std::forward<decltype(__element)>(__element));
1764		}
	1857	__tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
1765	1858	}
1766	1859	# endif
1767	1860
...	...	@@ -1783,10 +1876,10 @@ public:
1783	1876	"node_type with incompatible allocator passed to multimap::insert()");
1784	1877	return __tree_.template __node_handle_insert_multi<node_type>(__hint.__i_, std::move(__nh));
1785	1878	}
1786		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1879	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
1787	1880	return __tree_.template __node_handle_extract<node_type>(__key);
1788	1881	}
1789		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	1882	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
1790	1883	return __tree_.template __node_handle_extract<node_type>(__it.__i_);
1791	1884	}
1792	1885	template <class _Compare2>
...	...	@@ -1821,75 +1914,89 @@ public:
1821	1914	__tree_.swap(__m.__tree_);
1822	1915	}
1823	1916
1824		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
1825		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
	1917	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
	1918	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
1826	1919	# if _LIBCPP_STD_VER >= 14
1827	1920	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1828		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1921	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
1829	1922	return __tree_.find(__k);
1830	1923	}
1831	1924	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1832		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1925	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
1833	1926	return __tree_.find(__k);
1834	1927	}
1835	1928	# endif
1836	1929
1837		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
	1930	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1931	return __tree_.__count_multi(__k);
	1932	}
1838	1933	# if _LIBCPP_STD_VER >= 14
1839	1934	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1840		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1935	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
1841	1936	return __tree_.__count_multi(__k);
1842	1937	}
1843	1938	# endif
1844	1939
1845	1940	# if _LIBCPP_STD_VER >= 20
1846		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1941	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
1847	1942	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1848		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1943	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
1849	1944	return find(__k) != end();
1850	1945	}
1851	1946	# endif // _LIBCPP_STD_VER >= 20
1852	1947
1853		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
1854		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
	1948	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) {
	1949	return __tree_.__lower_bound_multi(__k);
	1950	}
	1951
	1952	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const {
	1953	return __tree_.__lower_bound_multi(__k);
	1954	}
	1955
1855	1956	# if _LIBCPP_STD_VER >= 14
1856	1957	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1857		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
1858		return __tree_.lower_bound(__k);
	1958	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
	1959	return __tree_.__lower_bound_multi(__k);
1859	1960	}
1860	1961
1861	1962	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1862		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
1863		return __tree_.lower_bound(__k);
	1963	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
	1964	return __tree_.__lower_bound_multi(__k);
1864	1965	}
1865	1966	# endif
1866	1967
1867		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
1868		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
	1968	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) {
	1969	return __tree_.__upper_bound_multi(__k);
	1970	}
	1971
	1972	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const {
	1973	return __tree_.__upper_bound_multi(__k);
	1974	}
	1975
1869	1976	# if _LIBCPP_STD_VER >= 14
1870	1977	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1871		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
1872		return __tree_.upper_bound(__k);
	1978	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
	1979	return __tree_.__upper_bound_multi(__k);
1873	1980	}
1874	1981	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1875		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
1876		return __tree_.upper_bound(__k);
	1982	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
	1983	return __tree_.__upper_bound_multi(__k);
1877	1984	}
1878	1985	# endif
1879	1986
1880		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1987	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
1881	1988	return __tree_.__equal_range_multi(__k);
1882	1989	}
1883		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1990	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
1884	1991	return __tree_.__equal_range_multi(__k);
1885	1992	}
1886	1993	# if _LIBCPP_STD_VER >= 14
1887	1994	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1888		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1995	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
1889	1996	return __tree_.__equal_range_multi(__k);
1890	1997	}
1891	1998	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1892		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1999	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
1893	2000	return __tree_.__equal_range_multi(__k);
1894	2001	}
1895	2002	# endif
...	...	@@ -1901,6 +2008,8 @@ private:
1901	2008
1902	2009	typedef __map_node_destructor<__node_allocator> _Dp;
1903	2010	typedef unique_ptr<__node, _Dp> __node_holder;
	2011
	2012	friend struct __specialized_algorithm<_Algorithm::__for_each, __single_range<multimap> >;
1904	2013	};
1905	2014
1906	2015	# if _LIBCPP_STD_VER >= 17
...	...	@@ -1908,8 +2017,8 @@ template <class _InputIterator,
1908	2017	class _Compare = less<__iter_key_type<_InputIterator>>,
1909	2018	class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
1910	2019	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1911		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1912		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	2020	class = enable_if_t<!__is_allocator_v<_Compare>>,
	2021	class = enable_if_t<__is_allocator_v<_Allocator>>>
1913	2022	multimap(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
1914	2023	-> multimap<__iter_key_type<_InputIterator>, __iter_mapped_type<_InputIterator>, _Compare, _Allocator>;
1915	2024
...	...	@@ -1917,8 +2026,8 @@ multimap(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Al
1917	2026	template <ranges::input_range _Range,
1918	2027	class _Compare = less<__range_key_type<_Range>>,
1919	2028	class _Allocator = allocator<__range_to_alloc_type<_Range>>,
1920		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1921		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	2029	class = enable_if_t<!__is_allocator_v<_Compare>>,
	2030	class = enable_if_t<__is_allocator_v<_Allocator>>>
1922	2031	multimap(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
1923	2032	-> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, _Compare, _Allocator>;
1924	2033	# endif
...	...	@@ -1927,16 +2036,15 @@ template <class _Key,
1927	2036	class _Tp,
1928	2037	class _Compare = less<remove_const_t<_Key>>,
1929	2038	class _Allocator = allocator<pair<const _Key, _Tp>>,
1930		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
1931		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
1932		multimap(initializer_list<pair<_Key, _Tp>>,
1933		_Compare = _Compare(),
1934		_Allocator = _Allocator()) -> multimap<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
	2039	class = enable_if_t<!__is_allocator_v<_Compare>>,
	2040	class = enable_if_t<__is_allocator_v<_Allocator>>>
	2041	multimap(initializer_list<pair<_Key, _Tp>>, _Compare = _Compare(), _Allocator = _Allocator())
	2042	-> multimap<remove_const_t<_Key>, _Tp, _Compare, _Allocator>;
1935	2043
1936	2044	template <class _InputIterator,
1937	2045	class _Allocator,
1938	2046	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1939		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	2047	class = enable_if_t<__is_allocator_v<_Allocator>>>
1940	2048	multimap(_InputIterator, _InputIterator, _Allocator)
1941	2049	-> multimap<__iter_key_type<_InputIterator>,
1942	2050	__iter_mapped_type<_InputIterator>,
...	...	@@ -1944,26 +2052,30 @@ multimap(_InputIterator, _InputIterator, _Allocator)
1944	2052	_Allocator>;
1945	2053
1946	2054	# if _LIBCPP_STD_VER >= 23
1947		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	2055	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1948	2056	multimap(from_range_t, _Range&&, _Allocator)
1949	2057	-> multimap<__range_key_type<_Range>, __range_mapped_type<_Range>, less<__range_key_type<_Range>>, _Allocator>;
1950	2058	# endif
1951	2059
1952		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
1953		multimap(initializer_list<pair<_Key, _Tp>>,
1954		_Allocator) -> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
	2060	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
	2061	multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
	2062	-> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
1955	2063	# endif
1956	2064
1957		# ifndef _LIBCPP_CXX03_LANG
	2065	# if _LIBCPP_STD_VER >= 14
1958	2066	template <class _Key, class _Tp, class _Compare, class _Allocator>
1959		multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const allocator_type& __a)
1960		: __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
1961		if (__a != __m.get_allocator()) {
1962		const_iterator __e = cend();
1963		while (!__m.empty())
1964		__tree_.__insert_multi_from_orphaned_node(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_));
	2067	struct __specialized_algorithm<_Algorithm::__for_each, __single_range<multimap<_Key, _Tp, _Compare, _Allocator>>> {
	2068	using __map _LIBCPP_NODEBUG = multimap<_Key, _Tp, _Compare, _Allocator>;
	2069
	2070	static const bool __has_algorithm = true;
	2071
	2072	template <class _Map, class _Func, class _Proj>
	2073	_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func, _Proj __proj) {
	2074	auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, __single_range<typename __map::__base>>()(
	2075	__map.__tree_, std::move(__func), std::move(__proj));
	2076	return std::make_pair(__map.end(), std::move(__func2));
1965	2077	}
1966		}
	2078	};
1967	2079	# endif
1968	2080
1969	2081	template <class _Key, class _Tp, class _Compare, class _Allocator>

lib/libcxx/include/math.h+19

...	...	@@ -429,6 +429,25 @@ using std::__math::isnormal;
429	429	using std::__math::isunordered;
430	430	# endif // _LIBCPP_MSVCRT
431	431
	432	# if defined(_LIBCPP_MSVCRT) && _LIBCPP_STD_VER >= 20
	433	// MS UCRT incorrectly defines some functions in a way not working with integer types. Until C++20, this was worked
	434	// around by -fdelayed-template-parsing. Since C++20, we can use standard feature "requires" instead.
	435
	436	// TODO: Remove the workaround once UCRT fixes these functions. Note that this doesn't seem planned as of 2025-07 per
	437	// https://developercommunity.visualstudio.com/t/10294165.
	438
	439	using std::__math::__ucrt::isfinite;
	440	using std::__math::__ucrt::isgreater;
	441	using std::__math::__ucrt::isgreaterequal;
	442	using std::__math::__ucrt::isinf;
	443	using std::__math::__ucrt::isless;
	444	using std::__math::__ucrt::islessequal;
	445	using std::__math::__ucrt::islessgreater;
	446	using std::__math::__ucrt::isnan;
	447	using std::__math::__ucrt::isnormal;
	448	using std::__math::__ucrt::isunordered;
	449	# endif // defined(_LIBCPP_MSVCRT) && _LIBCPP_STD_VER >= 20
	450
432	451	// We have to provide double overloads for <math.h> to work on platforms that don't provide the full set of math
433	452	// functions. To make the overload set work with multiple functions that take the same arguments, we make our overloads
434	453	// templates. Functions are preferred over function templates during overload resolution, which means that our overload

lib/libcxx/include/mdspan+1-5

...	...	@@ -450,11 +450,7 @@ namespace std {
450	450	# include <__config>
451	451
452	452	# if _LIBCPP_STD_VER >= 23
453		# include <__fwd/mdspan.h> // TODO(boomanaiden154): This is currently a
454		// non-standard extension to include
455		// std::dynamic_extent tracked by LWG issue 4275.
456		// This comment should be deleted or the include
457		// deleted upon resolution.
	453	# include <__fwd/mdspan.h>
458	454	# include <__fwd/span.h>
459	455	# include <__mdspan/default_accessor.h>
460	456	# include <__mdspan/extents.h>

lib/libcxx/include/mutex+29-19

...	...	@@ -229,12 +229,12 @@ public:
229	229	recursive_mutex& operator=(const recursive_mutex&) = delete;
230	230
231	231	void lock();
232		bool try_lock() _NOEXCEPT;
	232	[[__nodiscard__]] bool try_lock() _NOEXCEPT;
233	233	void unlock() _NOEXCEPT;
234	234
235	235	typedef __libcpp_recursive_mutex_t* native_handle_type;
236	236
237		_LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
	237	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
238	238	};
239	239
240	240	class _LIBCPP_EXPORTED_FROM_ABI timed_mutex {
...	...	@@ -251,14 +251,14 @@ public:
251	251
252	252	public:
253	253	void lock();
254		bool try_lock() _NOEXCEPT;
	254	[[__nodiscard__]] bool try_lock() _NOEXCEPT;
255	255	template <class _Rep, class _Period>
256		_LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
	256	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
257	257	return try_lock_until(chrono::steady_clock::now() + __d);
258	258	}
259	259
260	260	template <class _Clock, class _Duration>
261		_LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
	261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
262	262	using namespace chrono;
263	263	unique_lock<mutex> __lk(__m_);
264	264	bool __no_timeout = _Clock::now() < __t;
...	...	@@ -288,14 +288,14 @@ public:
288	288	recursive_timed_mutex& operator=(const recursive_timed_mutex&) = delete;
289	289
290	290	void lock();
291		bool try_lock() _NOEXCEPT;
	291	[[__nodiscard__]] bool try_lock() _NOEXCEPT;
292	292	template <class _Rep, class _Period>
293		_LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
	293	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __d) {
294	294	return try_lock_until(chrono::steady_clock::now() + __d);
295	295	}
296	296
297	297	template <class _Clock, class _Duration>
298		_LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
	298	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __t) {
299	299	using namespace chrono;
300	300	__thread_id __id = this_thread::get_id();
301	301	unique_lock<mutex> __lk(__m_);
...	...	@@ -320,7 +320,7 @@ public:
320	320	};
321	321
322	322	template <class _L0, class _L1>
323		_LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) {
	323	[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) {
324	324	unique_lock<_L0> __u0(__l0, try_to_lock_t());
325	325	if (__u0.owns_lock()) {
326	326	if (__l1.try_lock()) {
...	...	@@ -335,7 +335,8 @@ _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1) {
335	335	# ifndef _LIBCPP_CXX03_LANG
336	336
337	337	template <class _L0, class _L1, class _L2, class... _L3>
338		_LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
	338	[[__nodiscard__]] _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
	339	_LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
339	340	int __r = 0;
340	341	unique_lock<_L0> __u0(__l0, try_to_lock);
341	342	if (__u0.owns_lock()) {
...	...	@@ -350,8 +351,11 @@ _LIBCPP_HIDE_FROM_ABI int try_lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3
350	351
351	352	# endif // _LIBCPP_CXX03_LANG
352	353
	354	// We're using unique_lock to implement the functions, which thread annotations don't support. So we have to disable
	355	// the analysis inside the function.
353	356	template <class _L0, class _L1>
354		_LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1) {
	357	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1)
	358	_LIBCPP_ACQUIRE_CAPABILITY(__l0, __l1) {
355	359	while (true) {
356	360	{
357	361	unique_lock<_L0> __u0(__l0);
...	...	@@ -375,7 +379,7 @@ _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1) {
375	379	# ifndef _LIBCPP_CXX03_LANG
376	380
377	381	template <class _L0, class _L1, class _L2, class... _L3>
378		void __lock_first(int __i, _L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
	382	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS void __lock_first(int __i, _L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
379	383	while (true) {
380	384	switch (__i) {
381	385	case 0: {
...	...	@@ -410,8 +414,14 @@ void __lock_first(int __i, _L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
410	414	}
411	415	}
412	416
	417	// We're using unique_lock to implement the functions, which thread annotations don't support. So we have to disable
	418	// the analysis inside the function.
413	419	template <class _L0, class _L1, class _L2, class... _L3>
414		inline _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3) {
	420	_LIBCPP_NO_THREAD_SAFETY_ANALYSIS inline _LIBCPP_HIDE_FROM_ABI void lock(_L0& __l0, _L1& __l1, _L2& __l2, _L3&... __l3)
	421	# if defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 2101
	422	_LIBCPP_ACQUIRE_CAPABILITY(__l0, __l1, __l2, __l3...)
	423	# endif
	424	{
415	425	std::__lock_first(0, __l0, __l1, __l2, __l3...);
416	426	}
417	427
...	...	@@ -469,17 +479,14 @@ public:
469	479
470	480	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI scoped_lock(adopt_lock_t, _MArgs&... __margs) : __t_(__margs...) {}
471	481
472		_LIBCPP_HIDE_FROM_ABI ~scoped_lock() {
473		typedef typename __make_tuple_indices<sizeof...(_MArgs)>::type _Indices;
474		__unlock_unpack(_Indices{}, __t_);
475		}
	482	_LIBCPP_HIDE_FROM_ABI ~scoped_lock() { __unlock_unpack(make_index_sequence<sizeof...(_MArgs)>(), __t_); }
476	483
477	484	scoped_lock(scoped_lock const&) = delete;
478	485	scoped_lock& operator=(scoped_lock const&) = delete;
479	486
480	487	private:
481	488	template <size_t... _Indx>
482		_LIBCPP_HIDE_FROM_ABI static void __unlock_unpack(__tuple_indices<_Indx...>, _MutexTuple& __mt) {
	489	_LIBCPP_HIDE_FROM_ABI static void __unlock_unpack(index_sequence<_Indx...>, _MutexTuple& __mt) {
483	490	(std::get<_Indx>(__mt).unlock(), ...);
484	491	}
485	492
...	...	@@ -494,6 +501,10 @@ _LIBCPP_END_NAMESPACE_STD
494	501
495	502	_LIBCPP_POP_MACROS
496	503
	504	# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
	505	# include <typeinfo>
	506	# endif
	507
497	508	# if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
498	509	# include <atomic>
499	510	# include <concepts>
...	...	@@ -507,7 +518,6 @@ _LIBCPP_POP_MACROS
507	518	# include <stdexcept>
508	519	# include <system_error>
509	520	# include <type_traits>
510		# include <typeinfo>
511	521	# endif
512	522	#endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
513	523

lib/libcxx/include/optional+644-149

...	...	@@ -20,6 +20,11 @@ namespace std {
20	20	template <class T>
21	21	class optional;
22	22
	23	template<class T>
	24	constexpr bool ranges::enable_view<optional<T>> = true;
	25	template<class T>
	26	constexpr auto format_kind<optional<T>> = range_format::disabled;
	27
23	28	template<class T>
24	29	concept is-derived-from-optional = requires(const T& t) { // exposition only
25	30	[]<class U>(const optional<U>&){ }(t);
...	...	@@ -102,6 +107,8 @@ namespace std {
102	107	class optional {
103	108	public:
104	109	using value_type = T;
	110	using iterator = implementation-defined; // see [optional.iterators]
	111	using const_iterator = implementation-defined; // see [optional.iterators]
105	112
106	113	// [optional.ctor], constructors
107	114	constexpr optional() noexcept;
...	...	@@ -112,7 +119,7 @@ namespace std {
112	119	constexpr explicit optional(in_place_t, Args &&...);
113	120	template<class U, class... Args>
114	121	constexpr explicit optional(in_place_t, initializer_list<U>, Args &&...);
115		template<class U = T>
	122	template<class U = remove_cv_t<T>>
116	123	constexpr explicit(see-below) optional(U &&);
117	124	template<class U>
118	125	explicit(see-below) optional(const optional<U> &); // constexpr in C++20
...	...	@@ -126,7 +133,7 @@ namespace std {
126	133	optional &operator=(nullopt_t) noexcept; // constexpr in C++20
127	134	constexpr optional &operator=(const optional &);
128	135	constexpr optional &operator=(optional &&) noexcept(see below);
129		template<class U = T> optional &operator=(U &&); // constexpr in C++20
	136	template<class U = remove_cv_t<T>> optional &operator=(U &&); // constexpr in C++20
130	137	template<class U> optional &operator=(const optional<U> &); // constexpr in C++20
131	138	template<class U> optional &operator=(optional<U> &&); // constexpr in C++20
132	139	template<class... Args> T& emplace(Args &&...); // constexpr in C++20
...	...	@@ -135,6 +142,12 @@ namespace std {
135	142	// [optional.swap], swap
136	143	void swap(optional &) noexcept(see below ); // constexpr in C++20
137	144
	145	// [optional.iterators], iterator support
	146	constexpr iterator begin() noexcept;
	147	constexpr const_iterator begin() const noexcept;
	148	constexpr iterator end() noexcept;
	149	constexpr const_iterator end() const noexcept;
	150
138	151	// [optional.observe], observers
139	152	constexpr T const *operator->() const noexcept;
140	153	constexpr T *operator->() noexcept;
...	...	@@ -148,8 +161,8 @@ namespace std {
148	161	constexpr T &value() &;
149	162	constexpr T &&value() &&;
150	163	constexpr const T &&value() const &&;
151		template<class U> constexpr T value_or(U &&) const &;
152		template<class U> constexpr T value_or(U &&) &&;
	164	template<class U = remove_cv_t<T>> constexpr T value_or(U &&) const &;
	165	template<class U = remove_cv_t<T>> constexpr T value_or(U &&) &&;
153	166
154	167	// [optional.monadic], monadic operations
155	168	template<class F> constexpr auto and_then(F&& f) &; // since C++23
...	...	@@ -173,6 +186,71 @@ namespace std {
173	186	template<class T>
174	187	optional(T) -> optional<T>;
175	188
	189	template<class T>
	190	class optional<T&> { // since C++26
	191	public:
	192	using value_type = T;
	193	using iterator = implementation-defined; // see [optional.ref.iterators]
	194
	195	public:
	196	// [optional.ref.ctor], constructors
	197	constexpr optional() noexcept = default;
	198	constexpr optional(nullopt_t) noexcept : optional() {}
	199	constexpr optional(const optional& rhs) noexcept = default;
	200
	201	template<class Arg>
	202	constexpr explicit optional(in_place_t, Arg&& arg);
	203	template<class U>
	204	constexpr explicit(see below) optional(U&& u) noexcept(see below);
	205	template<class U>
	206	constexpr explicit(see below) optional(optional<U>& rhs) noexcept(see below);
	207	template<class U>
	208	constexpr explicit(see below) optional(const optional<U>& rhs) noexcept(see below);
	209	template<class U>
	210	constexpr explicit(see below) optional(optional<U>&& rhs) noexcept(see below);
	211	template<class U>
	212	constexpr explicit(see below) optional(const optional<U>&& rhs) noexcept(see below);
	213
	214	constexpr ~optional() = default;
	215
	216	// [optional.ref.assign], assignment
	217	constexpr optional& operator=(nullopt_t) noexcept;
	218	constexpr optional& operator=(const optional& rhs) noexcept = default;
	219
	220	template<class U> constexpr T& emplace(U&& u) noexcept(see below);
	221
	222	// [optional.ref.swap], swap
	223	constexpr void swap(optional& rhs) noexcept;
	224
	225	// [optional.ref.iterators], iterator support
	226	constexpr iterator begin() const noexcept;
	227	constexpr iterator end() const noexcept;
	228
	229	// [optional.ref.observe], observers
	230	constexpr T* operator->() const noexcept;
	231	constexpr T& operator*() const noexcept;
	232	constexpr explicit operator bool() const noexcept;
	233	constexpr bool has_value() const noexcept;
	234	constexpr T& value() const; // freestanding-deleted
	235	template<class U = remove_cv_t<T>>
	236	constexpr remove_cv_t<T> value_or(U&& u) const;
	237
	238	// [optional.ref.monadic], monadic operations
	239	template<class F> constexpr auto and_then(F&& f) const;
	240	template<class F> constexpr optional<invoke_result_t<F, T&>> transform(F&& f) const;
	241	template<class F> constexpr optional or_else(F&& f) const;
	242
	243	// [optional.ref.mod], modifiers
	244	constexpr void reset() noexcept;
	245
	246	private:
	247	T* val = nullptr; // exposition only
	248
	249	// [optional.ref.expos], exposition only helper functions
	250	template<class U>
	251	constexpr void convert-ref-init-val(U&& u); // exposition only
	252	};
	253
176	254	} // namespace std
177	255
178	256	*/
...	...	@@ -186,13 +264,19 @@ namespace std {
186	264	# include <__compare/three_way_comparable.h>
187	265	# include <__concepts/invocable.h>
188	266	# include <__config>
	267	# include <__cstddef/ptrdiff_t.h>
189	268	# include <__exception/exception.h>
	269	# include <__format/range_format.h>
190	270	# include <__functional/hash.h>
191	271	# include <__functional/invoke.h>
192	272	# include <__functional/unary_function.h>
193	273	# include <__fwd/functional.h>
	274	# include <__iterator/bounded_iter.h>
	275	# include <__iterator/wrap_iter.h>
194	276	# include <__memory/addressof.h>
195	277	# include <__memory/construct_at.h>
	278	# include <__ranges/enable_borrowed_range.h>
	279	# include <__ranges/enable_view.h>
196	280	# include <__tuple/sfinae_helpers.h>
197	281	# include <__type_traits/add_pointer.h>
198	282	# include <__type_traits/conditional.h>
...	...	@@ -200,6 +284,7 @@ namespace std {
200	284	# include <__type_traits/decay.h>
201	285	# include <__type_traits/disjunction.h>
202	286	# include <__type_traits/enable_if.h>
	287	# include <__type_traits/integral_constant.h>
203	288	# include <__type_traits/invoke.h>
204	289	# include <__type_traits/is_array.h>
205	290	# include <__type_traits/is_assignable.h>
...	...	@@ -207,11 +292,11 @@ namespace std {
207	292	# include <__type_traits/is_convertible.h>
208	293	# include <__type_traits/is_core_convertible.h>
209	294	# include <__type_traits/is_destructible.h>
	295	# include <__type_traits/is_function.h>
210	296	# include <__type_traits/is_nothrow_assignable.h>
211	297	# include <__type_traits/is_nothrow_constructible.h>
212	298	# include <__type_traits/is_object.h>
213	299	# include <__type_traits/is_reference.h>
214		# include <__type_traits/is_replaceable.h>
215	300	# include <__type_traits/is_same.h>
216	301	# include <__type_traits/is_scalar.h>
217	302	# include <__type_traits/is_swappable.h>
...	...	@@ -220,6 +305,7 @@ namespace std {
220	305	# include <__type_traits/is_trivially_destructible.h>
221	306	# include <__type_traits/is_trivially_relocatable.h>
222	307	# include <__type_traits/negation.h>
	308	# include <__type_traits/reference_constructs_from_temporary.h>
223	309	# include <__type_traits/remove_const.h>
224	310	# include <__type_traits/remove_cv.h>
225	311	# include <__type_traits/remove_cvref.h>
...	...	@@ -255,7 +341,7 @@ public:
255	341	_LIBCPP_HIDE_FROM_ABI bad_optional_access& operator=(const bad_optional_access&) _NOEXCEPT = default;
256	342	// Get the key function ~bad_optional_access() into the dylib
257	343	~bad_optional_access() _NOEXCEPT override;
258		const char* what() const _NOEXCEPT override;
	344	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
259	345	};
260	346
261	347	} // namespace std
...	...	@@ -358,7 +444,7 @@ struct __optional_storage_base : __optional_destruct_base<_Tp> {
358	444	using value_type = _Tp;
359	445	using __base::__base;
360	446
361		_LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__engaged_; }
	447	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return this->__engaged_; }
362	448
363	449	_LIBCPP_HIDE_FROM_ABI constexpr value_type& __get() & noexcept { return this->__val_; }
364	450	_LIBCPP_HIDE_FROM_ABI constexpr const value_type& __get() const& noexcept { return this->__val_; }
...	...	@@ -390,58 +476,60 @@ struct __optional_storage_base : __optional_destruct_base<_Tp> {
390	476	__construct(std::forward<_That>(__opt).__get());
391	477	}
392	478	}
	479
	480	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
	481	__swap(__optional_storage_base& __rhs) noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_swappable_v<_Tp>) {
	482	using std::swap;
	483	if (this->has_value() == __rhs.has_value()) {
	484	if (this->has_value())
	485	swap(this->__get(), __rhs.__get());
	486	} else {
	487	if (this->has_value()) {
	488	__rhs.__construct(std::move(this->__get()));
	489	this->reset();
	490	} else {
	491	this->__construct(std::move(__rhs.__get()));
	492	__rhs.reset();
	493	}
	494	}
	495	}
393	496	};
394	497
395		// optional<T&> is currently required to be ill-formed. However, it may
396		// be allowed in the future. For this reason, it has already been implemented
397		// to ensure we can make the change in an ABI-compatible manner.
398	498	template <class _Tp>
399	499	struct __optional_storage_base<_Tp, true> {
400	500	using value_type = _Tp;
401	501	using __raw_type _LIBCPP_NODEBUG = remove_reference_t<_Tp>;
402	502	__raw_type* __value_;
403	503
	504	_LIBCPP_HIDE_FROM_ABI constexpr __optional_storage_base() noexcept : __value_(nullptr) {}
	505
404	506	template <class _Up>
405		static _LIBCPP_HIDE_FROM_ABI constexpr bool __can_bind_reference() {
406		using _RawUp = __libcpp_remove_reference_t<_Up>;
407		using _UpPtr = _RawUp*;
408		using _RawTp = __libcpp_remove_reference_t<_Tp>;
409		using _TpPtr = _RawTp*;
410		using _CheckLValueArg =
411		integral_constant<bool,
412		(is_lvalue_reference<_Up>::value && is_convertible<_UpPtr, _TpPtr>::value) \|\|
413		is_same<_RawUp, reference_wrapper<_RawTp>>::value \|\|
414		is_same<_RawUp, reference_wrapper<__remove_const_t<_RawTp>>>::value >;
415		return (is_lvalue_reference<_Tp>::value && _CheckLValueArg::value) \|\|
416		(is_rvalue_reference<_Tp>::value && !is_lvalue_reference<_Up>::value &&
417		is_convertible<_UpPtr, _TpPtr>::value);
	507	_LIBCPP_HIDE_FROM_ABI constexpr void __convert_init_ref_val(_Up&& __val) noexcept {
	508	_Tp& __r(std::forward<_Up>(__val));
	509	__value_ = std::addressof(__r);
418	510	}
419	511
420		_LIBCPP_HIDE_FROM_ABI constexpr __optional_storage_base() noexcept : __value_(nullptr) {}
421
422	512	template <class _UArg>
423		_LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_storage_base(in_place_t, _UArg&& __uarg)
424		: __value_(std::addressof(__uarg)) {
425		static_assert(__can_bind_reference<_UArg>(),
426		"Attempted to construct a reference element in tuple from a "
	513	_LIBCPP_HIDE_FROM_ABI constexpr explicit __optional_storage_base(in_place_t, _UArg&& __uarg) {
	514	static_assert(!__reference_constructs_from_temporary_v<_Tp, _UArg>,
	515	"Attempted to construct a reference element in optional from a "
427	516	"possible temporary");
	517	__convert_init_ref_val(std::forward<_UArg>(__uarg));
428	518	}
429	519
430	520	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void reset() noexcept { __value_ = nullptr; }
431	521
432		_LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return __value_ != nullptr; }
433
434		_LIBCPP_HIDE_FROM_ABI constexpr value_type& __get() const& noexcept { return *__value_; }
	522	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool has_value() const noexcept { return __value_ != nullptr; }
435	523
436		_LIBCPP_HIDE_FROM_ABI constexpr value_type&& __get() const&& noexcept { return std::forward<value_type>(*__value_); }
	524	_LIBCPP_HIDE_FROM_ABI constexpr value_type& __get() const noexcept { return *__value_; }
437	525
438	526	template <class _UArg>
439	527	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __construct(_UArg&& __val) {
440	528	_LIBCPP_ASSERT_INTERNAL(!has_value(), "__construct called for engaged __optional_storage");
441		static_assert(__can_bind_reference<_UArg>(),
	529	static_assert(!__reference_constructs_from_temporary_v<_Tp, _UArg>,
442	530	"Attempted to construct a reference element in tuple from a "
443	531	"possible temporary");
444		__value_ = std::addressof(__val);
	532	__convert_init_ref_val(std::forward<_UArg>(__val));
445	533	}
446	534
447	535	template <class _That>
...	...	@@ -462,9 +550,13 @@ struct __optional_storage_base<_Tp, true> {
462	550	__construct(std::forward<_That>(__opt).__get());
463	551	}
464	552	}
	553
	554	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __swap(__optional_storage_base& __rhs) noexcept {
	555	std::swap(__value_, __rhs.__value_);
	556	}
465	557	};
466	558
467		template <class _Tp, bool = is_trivially_copy_constructible<_Tp>::value>
	559	template <class _Tp, bool = is_trivially_copy_constructible_v<_Tp> \|\| is_lvalue_reference_v<_Tp>>
468	560	struct __optional_copy_base : __optional_storage_base<_Tp> {
469	561	using __optional_storage_base<_Tp>::__optional_storage_base;
470	562	};
...	...	@@ -484,7 +576,7 @@ struct __optional_copy_base<_Tp, false> : __optional_storage_base<_Tp> {
484	576	_LIBCPP_HIDE_FROM_ABI __optional_copy_base& operator=(__optional_copy_base&&) = default;
485	577	};
486	578
487		template <class _Tp, bool = is_trivially_move_constructible<_Tp>::value>
	579	template <class _Tp, bool = is_trivially_move_constructible_v<_Tp> \|\| is_lvalue_reference_v<_Tp>>
488	580	struct __optional_move_base : __optional_copy_base<_Tp> {
489	581	using __optional_copy_base<_Tp>::__optional_copy_base;
490	582	};
...	...	@@ -507,8 +599,9 @@ struct __optional_move_base<_Tp, false> : __optional_copy_base<_Tp> {
507	599	};
508	600
509	601	template <class _Tp,
510		bool = is_trivially_destructible<_Tp>::value && is_trivially_copy_constructible<_Tp>::value &&
511		is_trivially_copy_assignable<_Tp>::value>
	602	bool = (is_trivially_destructible_v<_Tp> && is_trivially_copy_constructible_v<_Tp> &&
	603	is_trivially_copy_assignable_v<_Tp>) \|\|
	604	is_lvalue_reference_v<_Tp>>
512	605	struct __optional_copy_assign_base : __optional_move_base<_Tp> {
513	606	using __optional_move_base<_Tp>::__optional_move_base;
514	607	};
...	...	@@ -531,8 +624,9 @@ struct __optional_copy_assign_base<_Tp, false> : __optional_move_base<_Tp> {
531	624	};
532	625
533	626	template <class _Tp,
534		bool = is_trivially_destructible<_Tp>::value && is_trivially_move_constructible<_Tp>::value &&
535		is_trivially_move_assignable<_Tp>::value>
	627	bool = (is_trivially_destructible_v<_Tp> && is_trivially_move_constructible_v<_Tp> &&
	628	is_trivially_move_assignable_v<_Tp>) \|\|
	629	is_lvalue_reference_v<_Tp>>
536	630	struct __optional_move_assign_base : __optional_copy_assign_base<_Tp> {
537	631	using __optional_copy_assign_base<_Tp>::__optional_copy_assign_base;
538	632	};
...	...	@@ -561,12 +655,24 @@ using __optional_sfinae_ctor_base_t _LIBCPP_NODEBUG =
561	655
562	656	template <class _Tp>
563	657	using __optional_sfinae_assign_base_t _LIBCPP_NODEBUG =
564		__sfinae_assign_base< (is_copy_constructible<_Tp>::value && is_copy_assignable<_Tp>::value),
565		(is_move_constructible<_Tp>::value && is_move_assignable<_Tp>::value) >;
	658	__sfinae_assign_base< (is_copy_constructible_v<_Tp> && is_copy_assignable_v<_Tp>) \|\| is_lvalue_reference_v<_Tp>,
	659	(is_move_constructible_v<_Tp> && is_move_assignable_v<_Tp>) \|\| is_lvalue_reference_v<_Tp>>;
566	660
567	661	template <class _Tp>
568	662	class optional;
569	663
	664	# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR
	665	template <class _Tp>
	666	constexpr bool ranges::enable_view<optional<_Tp>> = true;
	667
	668	template <class _Tp>
	669	constexpr range_format format_kind<optional<_Tp>> = range_format::disabled;
	670
	671	template <class _Tp>
	672	constexpr bool ranges::enable_borrowed_range<optional<_Tp&>> = true;
	673
	674	# endif
	675
570	676	# if _LIBCPP_STD_VER >= 20
571	677
572	678	template <class _Tp>
...	...	@@ -579,29 +685,150 @@ struct __is_std_optional : false_type {};
579	685	template <class _Tp>
580	686	struct __is_std_optional<optional<_Tp>> : true_type {};
581	687
	688	template <class _Tp, class... _Args>
	689	inline constexpr bool __is_constructible_for_optional_v = is_constructible_v<_Tp, _Args...>;
	690
	691	template <class _Tp, class... _Args>
	692	struct __is_constructible_for_optional : bool_constant<__is_constructible_for_optional_v<_Tp, _Args...>> {};
	693
	694	template <class _Tp, class _Up, class... _Args>
	695	inline constexpr bool __is_constructible_for_optional_initializer_list_v =
	696	is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>;
	697
	698	# if _LIBCPP_STD_VER >= 26
	699	template <class _Tp, class... _Args>
	700	inline constexpr bool __is_constructible_for_optional_v<_Tp&, _Args...> = false;
	701	template <class _Tp, class _Arg>
	702	inline constexpr bool __is_constructible_for_optional_v<_Tp&, _Arg> =
	703	is_constructible_v<_Tp&, _Arg> && !reference_constructs_from_temporary_v<_Tp&, _Arg>;
	704
	705	template <class _Tp, class _Up, class... _Args>
	706	inline constexpr bool __is_constructible_for_optional_initializer_list_v<_Tp&, _Up, _Args...> = false;
	707	# endif
	708
	709	template <class _Tp, class = void>
	710	struct __optional_iterator {};
	711
	712	# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR
	713
	714	template <class _Tp>
	715	struct __optional_iterator<_Tp, enable_if_t<is_object_v<_Tp>>> {
	716	private:
	717	using __pointer _LIBCPP_NODEBUG = add_pointer_t<_Tp>;
	718	using __const_pointer _LIBCPP_NODEBUG = add_pointer_t<const _Tp>;
	719
	720	public:
	721	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
	722	using iterator = __bounded_iter<__wrap_iter<__pointer>>;
	723	using const_iterator = __bounded_iter<__wrap_iter<__const_pointer>>;
	724	# else
	725	using iterator = __wrap_iter<__pointer>;
	726	using const_iterator = __wrap_iter<__const_pointer>;
	727	# endif
	728
	729	// [optional.iterators], iterator support
	730	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() noexcept {
	731	auto& __derived_self = static_cast<optional<_Tp>&>(*this);
	732	auto* __ptr = std::addressof(__derived_self.__get());
	733
	734	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
	735	return std::__make_bounded_iter(
	736	__wrap_iter<__pointer>(__ptr),
	737	__wrap_iter<__pointer>(__ptr),
	738	__wrap_iter<__pointer>(__ptr) + (__derived_self.has_value() ? 1 : 0));
	739	# else
	740	return iterator(__ptr);
	741	# endif
	742	}
	743
	744	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept {
	745	auto& __derived_self = static_cast<const optional<_Tp>&>(*this);
	746	auto* __ptr = std::addressof(__derived_self.__get());
	747
	748	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
	749	return std::__make_bounded_iter(
	750	__wrap_iter<__const_pointer>(__ptr),
	751	__wrap_iter<__const_pointer>(__ptr),
	752	__wrap_iter<__const_pointer>(__ptr) + (__derived_self.has_value() ? 1 : 0));
	753	# else
	754	return const_iterator(__ptr);
	755	# endif
	756	}
	757
	758	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator end() noexcept {
	759	return begin() + (static_cast<optional<_Tp>&>(*this).has_value() ? 1 : 0);
	760	}
	761	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept {
	762	return begin() + (static_cast<const optional<_Tp>&>(*this).has_value() ? 1 : 0);
	763	}
	764	};
	765
	766	template <class _Tp>
	767	struct __optional_iterator<_Tp&, enable_if_t<is_object_v<_Tp> && !__is_unbounded_array_v<_Tp> >> {
	768	private:
	769	using __pointer _LIBCPP_NODEBUG = add_pointer_t<_Tp>;
	770
	771	public:
	772	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
	773	using iterator = __bounded_iter<__wrap_iter<__pointer>>;
	774	# else
	775	using iterator = __wrap_iter<__pointer>;
	776	# endif
	777
	778	// [optional.ref.iterators], iterator support
	779
	780	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const noexcept {
	781	auto& __derived_self = static_cast<const optional<_Tp&>&>(*this);
	782	auto* __ptr = __derived_self.has_value() ? std::addressof(__derived_self.__get()) : nullptr;
	783
	784	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
	785	return std::__make_bounded_iter(
	786	__wrap_iter<__pointer>(__ptr),
	787	__wrap_iter<__pointer>(__ptr),
	788	__wrap_iter<__pointer>(__ptr) + (__derived_self.has_value() ? 1 : 0));
	789	# else
	790	return iterator(__ptr);
	791	# endif
	792	}
	793
	794	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto end() const noexcept {
	795	return begin() + (static_cast<const optional<_Tp&>&>(*this).has_value() ? 1 : 0);
	796	}
	797	};
	798
	799	# endif // _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR
	800
582	801	template <class _Tp>
583	802	class _LIBCPP_DECLSPEC_EMPTY_BASES optional
584	803	: private __optional_move_assign_base<_Tp>,
585	804	private __optional_sfinae_ctor_base_t<_Tp>,
586		private __optional_sfinae_assign_base_t<_Tp> {
	805	private __optional_sfinae_assign_base_t<_Tp>,
	806	public __optional_iterator<_Tp> {
587	807	using __base _LIBCPP_NODEBUG = __optional_move_assign_base<_Tp>;
588	808
589	809	public:
590		using value_type = _Tp;
	810	using value_type = __libcpp_remove_reference_t<_Tp>;
591	811
592	812	using __trivially_relocatable _LIBCPP_NODEBUG =
593	813	conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>;
594		using __replaceable _LIBCPP_NODEBUG = conditional_t<__is_replaceable_v<_Tp>, optional, void>;
595	814
596	815	private:
597		// Disable the reference extension using this static assert.
598		static_assert(!is_same_v<__remove_cvref_t<value_type>, in_place_t>,
	816	static_assert(!is_same_v<__remove_cvref_t<_Tp>, in_place_t>,
599	817	"instantiation of optional with in_place_t is ill-formed");
600		static_assert(!is_same_v<__remove_cvref_t<value_type>, nullopt_t>,
601		"instantiation of optional with nullopt_t is ill-formed");
602		static_assert(!is_reference_v<value_type>, "instantiation of optional with a reference type is ill-formed");
603		static_assert(is_destructible_v<value_type>, "instantiation of optional with a non-destructible type is ill-formed");
604		static_assert(!is_array_v<value_type>, "instantiation of optional with an array type is ill-formed");
	818	static_assert(!is_same_v<__remove_cvref_t<_Tp>, nullopt_t>, "instantiation of optional with nullopt_t is ill-formed");
	819	# if _LIBCPP_STD_VER >= 26
	820	static_assert(!is_rvalue_reference_v<_Tp>, "instantiation of optional with an rvalue reference type is ill-formed");
	821	# else
	822	static_assert(!is_reference_v<_Tp>, "instantiation of optional with a reference type is ill-formed");
	823	# endif
	824	static_assert(is_destructible_v<_Tp>, "instantiation of optional with a non-destructible type is ill-formed");
	825	static_assert(!is_array_v<_Tp>, "instantiation of optional with an array type is ill-formed");
	826
	827	# if _LIBCPP_STD_VER >= 26
	828	template <class _Up>
	829	constexpr static bool __libcpp_opt_ref_ctor_deleted =
	830	is_lvalue_reference_v<_Tp> && reference_constructs_from_temporary_v<_Tp, _Up>;
	831	# endif
605	832
606	833	// LWG2756: conditionally explicit conversion from _Up
607	834	struct _CheckOptionalArgsConstructor {
...	...	@@ -674,45 +901,131 @@ public:
674	901	_LIBCPP_HIDE_FROM_ABI constexpr optional(optional&&) = default;
675	902	_LIBCPP_HIDE_FROM_ABI constexpr optional(nullopt_t) noexcept {}
676	903
677		template <class _InPlaceT,
678		class... _Args,
679		enable_if_t<_And<_IsSame<_InPlaceT, in_place_t>, is_constructible<value_type, _Args...>>::value, int> = 0>
	904	template <
	905	class _InPlaceT,
	906	class... _Args,
	907	enable_if_t<_And<_IsSame<_InPlaceT, in_place_t>, __is_constructible_for_optional<_Tp, _Args...>>::value, int> = 0>
680	908	_LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_InPlaceT, _Args&&... __args)
681	909	: __base(in_place, std::forward<_Args>(__args)...) {}
682	910
683	911	template <class _Up,
684	912	class... _Args,
685		enable_if_t<is_constructible_v<value_type, initializer_list<_Up>&, _Args...>, int> = 0>
	913	enable_if_t<__is_constructible_for_optional_initializer_list_v<_Tp, _Up, _Args...>, int> = 0>
686	914	_LIBCPP_HIDE_FROM_ABI constexpr explicit optional(in_place_t, initializer_list<_Up> __il, _Args&&... __args)
687	915	: __base(in_place, __il, std::forward<_Args>(__args)...) {}
688	916
689		template <class _Up = value_type,
690		enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_implicit<_Up>(), int> = 0>
691		_LIBCPP_HIDE_FROM_ABI constexpr optional(_Up&& __v) : __base(in_place, std::forward<_Up>(__v)) {}
	917	template <class _Up = _Tp, enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_implicit<_Up>(), int> = 0>
	918	_LIBCPP_HIDE_FROM_ABI constexpr optional(_Up&& __v)
	919	# if _LIBCPP_STD_VER >= 26
	920	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up>)
	921	# endif
	922	: __base(in_place, std::forward<_Up>(__v)) {
	923	}
692	924
693		template <class _Up, enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_explicit<_Up>(), int> = 0>
694		_LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_Up&& __v) : __base(in_place, std::forward<_Up>(__v)) {}
	925	template <class _Up = remove_cv_t<_Tp>,
	926	enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_explicit<_Up>(), int> = 0>
	927	_LIBCPP_HIDE_FROM_ABI constexpr explicit optional(_Up&& __v)
	928	# if _LIBCPP_STD_VER >= 26
	929	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up>)
	930	# endif
	931	: __base(in_place, std::forward<_Up>(__v)) {
	932	}
695	933
696	934	// LWG2756: conditionally explicit conversion from const optional<_Up>&
697	935	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_implicit<_Up>(), int> = 0>
698		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(const optional<_Up>& __v) {
	936	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(const optional<_Up>& __v)
	937	# if _LIBCPP_STD_VER >= 26
	938	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up&>)
	939	# endif
	940	{
699	941	this->__construct_from(__v);
700	942	}
701	943	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_explicit<_Up>(), int> = 0>
702		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(const optional<_Up>& __v) {
	944	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(const optional<_Up>& __v)
	945	# if _LIBCPP_STD_VER >= 26
	946	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up&>)
	947	# endif
	948	{
703	949	this->__construct_from(__v);
704	950	}
705	951
706	952	// LWG2756: conditionally explicit conversion from optional<_Up>&&
707	953	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_implicit<_Up>(), int> = 0>
708		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(optional<_Up>&& __v) {
	954	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(optional<_Up>&& __v)
	955	# if _LIBCPP_STD_VER >= 26
	956	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up>)
	957	# endif
	958	{
709	959	this->__construct_from(std::move(__v));
710	960	}
711	961	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_explicit<_Up>(), int> = 0>
712		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(optional<_Up>&& __v) {
	962	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit optional(optional<_Up>&& __v)
	963	# if _LIBCPP_STD_VER >= 26
	964	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp&, _Up>)
	965	# endif
	966	{
713	967	this->__construct_from(std::move(__v));
714	968	}
715	969
	970	// deleted optional<T&> constructors and additional optional<T&> constructors
	971	# if _LIBCPP_STD_VER >= 26
	972	// optional(U&&)
	973	template <class _Up = _Tp, enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_implicit<_Up>(), int> = 0>
	974	requires __libcpp_opt_ref_ctor_deleted<_Up>
	975	optional(_Up&&) = delete;
	976
	977	template <class _Up = remove_cv_t<_Tp>,
	978	enable_if_t<_CheckOptionalArgsCtor<_Up>::template __enable_explicit<_Up>(), int> = 0>
	979	requires __libcpp_opt_ref_ctor_deleted<_Up>
	980	explicit optional(_Up&&) = delete;
	981
	982	// optional(optional<U>& rhs)
	983	template <class _Up>
	984	requires(!__libcpp_opt_ref_ctor_deleted<_Up>) && (!is_same_v<remove_cvref_t<_Tp>, optional<_Up>>) &&
	985	(!is_same_v<_Tp&, _Up>) && is_constructible_v<_Tp&, _Up&>
	986	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up&, _Tp&>)
	987	optional(optional<_Up>& __rhs) noexcept(is_nothrow_constructible_v<_Tp&, _Up&>) {
	988	this->__construct_from(__rhs);
	989	}
	990
	991	template <class _Up>
	992	requires __libcpp_opt_ref_ctor_deleted<_Up> && (!is_same_v<remove_cvref_t<_Tp>, optional<_Up>>) &&
	993	(!is_same_v<_Tp&, _Up>) && is_constructible_v<_Tp&, _Up&>
	994	constexpr explicit optional(optional<_Up>& __rhs) noexcept = delete;
	995
	996	// optional(const optional<U>&)
	997	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_implicit<_Up>(), int> = 0>
	998	requires __libcpp_opt_ref_ctor_deleted<_Up>
	999	optional(const optional<_Up>&) = delete;
	1000
	1001	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up const&>::template __enable_explicit<_Up>(), int> = 0>
	1002	requires __libcpp_opt_ref_ctor_deleted<_Up>
	1003	explicit optional(const optional<_Up>&) = delete;
	1004
	1005	// optional(optional<U>&&)
	1006	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_implicit<_Up>(), int> = 0>
	1007	requires __libcpp_opt_ref_ctor_deleted<_Up>
	1008	optional(optional<_Up>&&) = delete;
	1009
	1010	template <class _Up, enable_if_t<_CheckOptionalLikeCtor<_Up, _Up&&>::template __enable_explicit<_Up>(), int> = 0>
	1011	requires __libcpp_opt_ref_ctor_deleted<_Up>
	1012	explicit optional(optional<_Up>&&) = delete;
	1013
	1014	// optional(const optional<U>&&)
	1015	template <class _Up>
	1016	requires(!__libcpp_opt_ref_ctor_deleted<_Up>) && (!is_same_v<remove_cvref_t<_Tp>, optional<_Up>>) &&
	1017	(!is_same_v<_Tp&, _Up>) && is_constructible_v<_Tp&, _Up>
	1018	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(!is_convertible_v<const _Up, _Tp&>)
	1019	optional(const optional<_Up>&& __v) noexcept(is_nothrow_constructible_v<_Tp&, const _Up>) {
	1020	this->__construct_from(std::move(__v));
	1021	}
	1022
	1023	template <class _Up>
	1024	requires __libcpp_opt_ref_ctor_deleted<_Up> && (!is_same_v<remove_cvref_t<_Tp>, optional<_Up>>) &&
	1025	(!is_same_v<_Tp&, _Up>) && is_constructible_v<_Tp&, _Up>
	1026	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional(const optional<_Up>&& __v) noexcept = delete;
	1027	# endif
	1028
716	1029	# if _LIBCPP_STD_VER >= 23
717	1030	template <class _Tag,
718	1031	class _Fp,
...	...	@@ -731,11 +1044,12 @@ public:
731	1044	_LIBCPP_HIDE_FROM_ABI constexpr optional& operator=(optional&&) = default;
732	1045
733	1046	// LWG2756
734		template <class _Up = value_type,
	1047	template <class _Up = remove_cv_t<_Tp>,
735	1048	enable_if_t<_And<_IsNotSame<__remove_cvref_t<_Up>, optional>,
736		_Or<_IsNotSame<__remove_cvref_t<_Up>, value_type>, _Not<is_scalar<value_type>>>,
737		is_constructible<value_type, _Up>,
738		is_assignable<value_type&, _Up>>::value,
	1049	_Or<_IsNotSame<__remove_cvref_t<_Up>, _Tp>, _Not<is_scalar<_Tp>>>,
	1050	is_constructible<_Tp, _Up>,
	1051	is_assignable<_Tp&, _Up>,
	1052	is_object<_Tp>>::value,
739	1053	int> = 0>
740	1054	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 optional& operator=(_Up&& __v) {
741	1055	if (this->has_value())
...	...	@@ -759,8 +1073,12 @@ public:
759	1073	return *this;
760	1074	}
761	1075
762		template <class... _Args, enable_if_t<is_constructible_v<value_type, _Args...>, int> = 0>
763		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args) {
	1076	template <class... _Args, enable_if_t<__is_constructible_for_optional_v<_Tp, _Args...>, int> = 0>
	1077	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(_Args&&... __args)
	1078	# if _LIBCPP_STD_VER >= 26
	1079	noexcept(is_lvalue_reference_v<_Tp> && is_nothrow_constructible_v<_Tp, _Args...>)
	1080	# endif
	1081	{
764	1082	reset();
765	1083	this->__construct(std::forward<_Args>(__args)...);
766	1084	return this->__get();
...	...	@@ -768,56 +1086,72 @@ public:
768	1086
769	1087	template <class _Up,
770	1088	class... _Args,
771		enable_if_t<is_constructible_v<value_type, initializer_list<_Up>&, _Args...>, int> = 0>
	1089	enable_if_t<__is_constructible_for_optional_initializer_list_v<_Tp, _Up, _Args...>, int> = 0>
772	1090	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp& emplace(initializer_list<_Up> __il, _Args&&... __args) {
773	1091	reset();
774	1092	this->__construct(__il, std::forward<_Args>(__args)...);
775	1093	return this->__get();
776	1094	}
777	1095
778		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
779		swap(optional& __opt) noexcept(is_nothrow_move_constructible_v<value_type> && is_nothrow_swappable_v<value_type>) {
780		if (this->has_value() == __opt.has_value()) {
781		using std::swap;
782		if (this->has_value())
783		swap(this->__get(), __opt.__get());
784		} else {
785		if (this->has_value()) {
786		__opt.__construct(std::move(this->__get()));
787		reset();
788		} else {
789		this->__construct(std::move(__opt.__get()));
790		__opt.reset();
791		}
792		}
	1096	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(optional& __opt) noexcept(
	1097	(is_nothrow_move_constructible_v<_Tp> && is_nothrow_swappable_v<_Tp>)
	1098	# if _LIBCPP_STD_VER >= 26
	1099	\|\| is_lvalue_reference_v<_Tp>
	1100	# endif
	1101	) {
	1102	this->__swap(__opt);
793	1103	}
794	1104
795		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type const> operator->() const noexcept {
	1105	_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp const> operator->() const noexcept
	1106	# if _LIBCPP_STD_VER >= 26
	1107	requires(is_object_v<_Tp>)
	1108	# endif
	1109	{
796	1110	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
797	1111	return std::addressof(this->__get());
798	1112	}
799	1113
800		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type> operator->() noexcept {
	1114	_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp> operator->() noexcept
	1115	# if _LIBCPP_STD_VER >= 26
	1116	requires(is_object_v<_Tp>)
	1117	# endif
	1118	{
801	1119	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
802	1120	return std::addressof(this->__get());
803	1121	}
804	1122
805		_LIBCPP_HIDE_FROM_ABI constexpr const value_type& operator*() const& noexcept {
	1123	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& operator*() const& noexcept
	1124	# if _LIBCPP_STD_VER >= 26
	1125	requires(is_object_v<_Tp>)
	1126	# endif
	1127	{
806	1128	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
807	1129	return this->__get();
808	1130	}
809	1131
810		_LIBCPP_HIDE_FROM_ABI constexpr value_type& operator*() & noexcept {
	1132	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() & noexcept
	1133	# if _LIBCPP_STD_VER >= 26
	1134	requires(is_object_v<_Tp>)
	1135	# endif
	1136	{
811	1137	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
812	1138	return this->__get();
813	1139	}
814	1140
815		_LIBCPP_HIDE_FROM_ABI constexpr value_type&& operator*() && noexcept {
	1141	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& operator*() && noexcept
	1142	# if _LIBCPP_STD_VER >= 26
	1143	requires(is_object_v<_Tp>)
	1144	# endif
	1145	{
816	1146	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
817	1147	return std::move(this->__get());
818	1148	}
819	1149
820		_LIBCPP_HIDE_FROM_ABI constexpr const value_type&& operator*() const&& noexcept {
	1150	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& operator*() const&& noexcept
	1151	# if _LIBCPP_STD_VER >= 26
	1152	requires(is_object_v<_Tp>)
	1153	# endif
	1154	{
821	1155	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
822	1156	return std::move(this->__get());
823	1157	}
...	...	@@ -827,48 +1161,73 @@ public:
827	1161	using __base::__get;
828	1162	using __base::has_value;
829	1163
830		_LIBCPP_HIDE_FROM_ABI constexpr value_type const& value() const& {
	1164	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp const& value() const&
	1165	# if _LIBCPP_STD_VER >= 26
	1166	requires(is_object_v<_Tp>)
	1167	# endif
	1168	{
831	1169	if (!this->has_value())
832	1170	std::__throw_bad_optional_access();
833	1171	return this->__get();
834	1172	}
835	1173
836		_LIBCPP_HIDE_FROM_ABI constexpr value_type& value() & {
	1174	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& value() &
	1175	# if _LIBCPP_STD_VER >= 26
	1176	requires(is_object_v<_Tp>)
	1177	# endif
	1178	{
837	1179	if (!this->has_value())
838	1180	std::__throw_bad_optional_access();
839	1181	return this->__get();
840	1182	}
841	1183
842		_LIBCPP_HIDE_FROM_ABI constexpr value_type&& value() && {
	1184	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& value() &&
	1185	# if _LIBCPP_STD_VER >= 26
	1186	requires(is_object_v<_Tp>)
	1187	# endif
	1188	{
843	1189	if (!this->has_value())
844	1190	std::__throw_bad_optional_access();
845	1191	return std::move(this->__get());
846	1192	}
847	1193
848		_LIBCPP_HIDE_FROM_ABI constexpr value_type const&& value() const&& {
	1194	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp const&& value() const&&
	1195	# if _LIBCPP_STD_VER >= 26
	1196	requires(is_object_v<_Tp>)
	1197	# endif
	1198	{
849	1199	if (!this->has_value())
850	1200	std::__throw_bad_optional_access();
851	1201	return std::move(this->__get());
852	1202	}
853	1203
854		template <class _Up>
855		_LIBCPP_HIDE_FROM_ABI constexpr value_type value_or(_Up&& __v) const& {
856		static_assert(is_copy_constructible_v<value_type>, "optional<T>::value_or: T must be copy constructible");
857		static_assert(is_convertible_v<_Up, value_type>, "optional<T>::value_or: U must be convertible to T");
858		return this->has_value() ? this->__get() : static_cast<value_type>(std::forward<_Up>(__v));
	1204	template <class _Up = remove_cv_t<_Tp>>
	1205	# if _LIBCPP_STD_VER >= 26
	1206	requires(is_object_v<_Tp>)
	1207	# endif
	1208	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) const& {
	1209	static_assert(is_copy_constructible_v<_Tp>, "optional<T>::value_or: T must be copy constructible");
	1210	static_assert(is_convertible_v<_Up, _Tp>, "optional<T>::value_or: U must be convertible to T");
	1211	return this->has_value() ? this->__get() : static_cast<_Tp>(std::forward<_Up>(__v));
859	1212	}
860	1213
861		template <class _Up>
862		_LIBCPP_HIDE_FROM_ABI constexpr value_type value_or(_Up&& __v) && {
863		static_assert(is_move_constructible_v<value_type>, "optional<T>::value_or: T must be move constructible");
864		static_assert(is_convertible_v<_Up, value_type>, "optional<T>::value_or: U must be convertible to T");
865		return this->has_value() ? std::move(this->__get()) : static_cast<value_type>(std::forward<_Up>(__v));
	1214	template <class _Up = remove_cv_t<_Tp>>
	1215	# if _LIBCPP_STD_VER >= 26
	1216	requires(is_object_v<_Tp>)
	1217	# endif
	1218	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp value_or(_Up&& __v) && {
	1219	static_assert(is_move_constructible_v<_Tp>, "optional<T>::value_or: T must be move constructible");
	1220	static_assert(is_convertible_v<_Up, _Tp>, "optional<T>::value_or: U must be convertible to T");
	1221	return this->has_value() ? std::move(this->__get()) : static_cast<_Tp>(std::forward<_Up>(__v));
866	1222	}
867	1223
868	1224	# if _LIBCPP_STD_VER >= 23
869	1225	template <class _Func>
870		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
871		using _Up = invoke_result_t<_Func, value_type&>;
	1226	# if _LIBCPP_STD_VER >= 26
	1227	requires(is_object_v<_Tp>)
	1228	# endif
	1229	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) & {
	1230	using _Up = invoke_result_t<_Func, _Tp&>;
872	1231	static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
873	1232	"Result of f(value()) must be a specialization of std::optional");
874	1233	if (*this)
...	...	@@ -877,8 +1236,11 @@ public:
877	1236	}
878	1237
879	1238	template <class _Func>
880		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
881		using _Up = invoke_result_t<_Func, const value_type&>;
	1239	# if _LIBCPP_STD_VER >= 26
	1240	requires(is_object_v<_Tp>)
	1241	# endif
	1242	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const& {
	1243	using _Up = invoke_result_t<_Func, const _Tp&>;
882	1244	static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
883	1245	"Result of f(value()) must be a specialization of std::optional");
884	1246	if (*this)
...	...	@@ -887,8 +1249,11 @@ public:
887	1249	}
888	1250
889	1251	template <class _Func>
890		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
891		using _Up = invoke_result_t<_Func, value_type&&>;
	1252	# if _LIBCPP_STD_VER >= 26
	1253	requires(is_object_v<_Tp>)
	1254	# endif
	1255	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) && {
	1256	using _Up = invoke_result_t<_Func, _Tp&&>;
892	1257	static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
893	1258	"Result of f(std::move(value())) must be a specialization of std::optional");
894	1259	if (*this)
...	...	@@ -897,8 +1262,11 @@ public:
897	1262	}
898	1263
899	1264	template <class _Func>
900		_LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
901		using _Up = invoke_result_t<_Func, const value_type&&>;
	1265	# if _LIBCPP_STD_VER >= 26
	1266	requires(is_object_v<_Tp>)
	1267	# endif
	1268	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const&& {
	1269	using _Up = invoke_result_t<_Func, const _Tp&&>;
902	1270	static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
903	1271	"Result of f(std::move(value())) must be a specialization of std::optional");
904	1272	if (*this)
...	...	@@ -907,8 +1275,11 @@ public:
907	1275	}
908	1276
909	1277	template <class _Func>
910		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
911		using _Up = remove_cv_t<invoke_result_t<_Func, value_type&>>;
	1278	# if _LIBCPP_STD_VER >= 26
	1279	requires(is_object_v<_Tp>)
	1280	# endif
	1281	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) & {
	1282	using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&>>;
912	1283	static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
913	1284	static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
914	1285	static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(value()) should not be std::nullopt_t");
...	...	@@ -919,8 +1290,11 @@ public:
919	1290	}
920	1291
921	1292	template <class _Func>
922		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
923		using _Up = remove_cv_t<invoke_result_t<_Func, const value_type&>>;
	1293	# if _LIBCPP_STD_VER >= 26
	1294	requires(is_object_v<_Tp>)
	1295	# endif
	1296	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const& {
	1297	using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&>>;
924	1298	static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
925	1299	static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
926	1300	static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(value()) should not be std::nullopt_t");
...	...	@@ -931,8 +1305,11 @@ public:
931	1305	}
932	1306
933	1307	template <class _Func>
934		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
935		using _Up = remove_cv_t<invoke_result_t<_Func, value_type&&>>;
	1308	# if _LIBCPP_STD_VER >= 26
	1309	requires(is_object_v<_Tp>)
	1310	# endif
	1311	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) && {
	1312	using _Up = remove_cv_t<invoke_result_t<_Func, _Tp&&>>;
936	1313	static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
937	1314	static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
938	1315	static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(std::move(value())) should not be std::nullopt_t");
...	...	@@ -943,8 +1320,11 @@ public:
943	1320	}
944	1321
945	1322	template <class _Func>
946		_LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
947		using _Up = remove_cvref_t<invoke_result_t<_Func, const value_type&&>>;
	1323	# if _LIBCPP_STD_VER >= 26
	1324	requires(is_object_v<_Tp>)
	1325	# endif
	1326	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto transform(_Func&& __f) const&& {
	1327	using _Up = remove_cv_t<invoke_result_t<_Func, const _Tp&&>>;
948	1328	static_assert(!is_array_v<_Up>, "Result of f(std::move(value())) should not be an Array");
949	1329	static_assert(!is_same_v<_Up, in_place_t>, "Result of f(std::move(value())) should not be std::in_place_t");
950	1330	static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(std::move(value())) should not be std::nullopt_t");
...	...	@@ -955,8 +1335,11 @@ public:
955	1335	}
956	1336
957	1337	template <invocable _Func>
958		_LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) const&
959		requires is_copy_constructible_v<value_type>
	1338	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) const&
	1339	requires is_copy_constructible_v<_Tp>
	1340	# if _LIBCPP_STD_VER >= 26
	1341	&& (is_object_v<_Tp>)
	1342	# endif
960	1343	{
961	1344	static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
962	1345	"Result of f() should be the same type as this optional");
...	...	@@ -966,8 +1349,11 @@ public:
966	1349	}
967	1350
968	1351	template <invocable _Func>
969		_LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) &&
970		requires is_move_constructible_v<value_type>
	1352	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) &&
	1353	requires is_move_constructible_v<_Tp>
	1354	# if _LIBCPP_STD_VER >= 26
	1355	&& (is_object_v<_Tp>)
	1356	# endif
971	1357	{
972	1358	static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
973	1359	"Result of f() should be the same type as this optional");
...	...	@@ -978,6 +1364,78 @@ public:
978	1364	# endif // _LIBCPP_STD_VER >= 23
979	1365
980	1366	using __base::reset;
	1367
	1368	// optional<T&> overloads
	1369	# if _LIBCPP_STD_VER >= 26
	1370
	1371	_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp> operator->() const noexcept
	1372	requires(is_lvalue_reference_v<_Tp>)
	1373	{
	1374	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
	1375	return std::addressof(this->__get());
	1376	}
	1377
	1378	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& operator*() const noexcept
	1379	requires(is_lvalue_reference_v<_Tp>)
	1380	{
	1381	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator* called on a disengaged value");
	1382	return this->__get();
	1383	}
	1384
	1385	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& value() const
	1386	requires(is_lvalue_reference_v<_Tp>)
	1387	{
	1388	if (!this->has_value())
	1389	std::__throw_bad_optional_access();
	1390	return this->__get();
	1391	}
	1392
	1393	template <class _Up = remove_cvref_t<_Tp>>
	1394	requires(is_lvalue_reference_v<_Tp> && is_object_v<__libcpp_remove_reference_t<_Tp>> &&
	1395	!is_array_v<__libcpp_remove_reference_t<_Tp>>)
	1396	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr decay_t<_Tp> value_or(_Up&& __v) const {
	1397	static_assert(
	1398	is_constructible_v<remove_cvref_t<_Tp>, _Tp&>, "optional<T&>::value_or: remove_cv_t<T> must be constructible");
	1399	static_assert(
	1400	is_convertible_v<_Up, remove_cvref_t<_Tp>>, "optional<T&>::value_or: U must be convertible to remove_cv_t<T>");
	1401	return this->has_value() ? this->__get() : static_cast<remove_cvref_t<_Tp>>(std::forward<_Up>(__v));
	1402	}
	1403
	1404	template <class _Func>
	1405	requires(is_lvalue_reference_v<_Tp>)
	1406	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto and_then(_Func&& __f) const {
	1407	using _Up = invoke_result_t<_Func, _Tp&>;
	1408	static_assert(__is_std_optional<remove_cvref_t<_Up>>::value,
	1409	"Result of f(value()) must be a specialization of std::optional");
	1410	if (*this)
	1411	return std::invoke(std::forward<_Func>(__f), value());
	1412	return remove_cvref_t<_Up>();
	1413	}
	1414
	1415	template <class _Func>
	1416	requires(is_lvalue_reference_v<_Tp>)
	1417	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional<remove_cv_t<invoke_result_t<_Func, _Tp&>>>
	1418	transform(_Func&& __f) const {
	1419	using _Up = remove_cvref_t<invoke_result_t<_Func, _Tp&>>;
	1420	static_assert(!is_array_v<_Up>, "Result of f(value()) should not be an Array");
	1421	static_assert(!is_same_v<_Up, in_place_t>, "Result of f(value()) should not be std::in_place_t");
	1422	static_assert(!is_same_v<_Up, nullopt_t>, "Result of f(value()) should not be std::nullopt_t");
	1423	static_assert(is_object_v<_Up>, "Result of f(value()) should be an object type");
	1424	if (*this)
	1425	return optional<_Up>(__optional_construct_from_invoke_tag{}, std::forward<_Func>(__f), value());
	1426	return optional<_Up>();
	1427	}
	1428
	1429	template <invocable _Func>
	1430	requires(is_lvalue_reference_v<_Tp>)
	1431	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional or_else(_Func&& __f) const {
	1432	static_assert(is_same_v<remove_cvref_t<invoke_result_t<_Func>>, optional>,
	1433	"Result of f() should be the same type as this optional");
	1434	if (*this)
	1435	return *this;
	1436	return std::forward<_Func>(__f)();
	1437	}
	1438	# endif
981	1439	};
982	1440
983	1441	template <class _Tp>
...	...	@@ -1154,7 +1612,9 @@ template <
1154	1612	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() == std::declval<const _Up&>()), bool>,
1155	1613	int> = 0>
1156	1614	_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const optional<_Tp>& __x, const _Up& __v) {
1157		return static_cast<bool>(__x) ? *__x == __v : false;
	1615	if (__x.has_value())
	1616	return *__x == __v;
	1617	return false;
1158	1618	}
1159	1619
1160	1620	template <
...	...	@@ -1163,7 +1623,9 @@ template <
1163	1623	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() == std::declval<const _Up&>()), bool>,
1164	1624	int> = 0>
1165	1625	_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const _Tp& __v, const optional<_Up>& __x) {
1166		return static_cast<bool>(__x) ? __v == *__x : false;
	1626	if (__x.has_value())
	1627	return __v == *__x;
	1628	return false;
1167	1629	}
1168	1630
1169	1631	template <
...	...	@@ -1172,7 +1634,9 @@ template <
1172	1634	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() != std::declval<const _Up&>()), bool>,
1173	1635	int> = 0>
1174	1636	_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const optional<_Tp>& __x, const _Up& __v) {
1175		return static_cast<bool>(__x) ? *__x != __v : true;
	1637	if (__x.has_value())
	1638	return *__x != __v;
	1639	return true;
1176	1640	}
1177	1641
1178	1642	template <
...	...	@@ -1181,7 +1645,9 @@ template <
1181	1645	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() != std::declval<const _Up&>()), bool>,
1182	1646	int> = 0>
1183	1647	_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const _Tp& __v, const optional<_Up>& __x) {
1184		return static_cast<bool>(__x) ? __v != *__x : true;
	1648	if (__x.has_value())
	1649	return __v != *__x;
	1650	return true;
1185	1651	}
1186	1652
1187	1653	template < class _Tp,
...	...	@@ -1189,7 +1655,9 @@ template < class _Tp,
1189	1655	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() < std::declval<const _Up&>()), bool>,
1190	1656	int> = 0>
1191	1657	_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const optional<_Tp>& __x, const _Up& __v) {
1192		return static_cast<bool>(__x) ? *__x < __v : true;
	1658	if (__x.has_value())
	1659	return *__x < __v;
	1660	return true;
1193	1661	}
1194	1662
1195	1663	template < class _Tp,
...	...	@@ -1197,7 +1665,9 @@ template < class _Tp,
1197	1665	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() < std::declval<const _Up&>()), bool>,
1198	1666	int> = 0>
1199	1667	_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const _Tp& __v, const optional<_Up>& __x) {
1200		return static_cast<bool>(__x) ? __v < *__x : false;
	1668	if (__x.has_value())
	1669	return __v < *__x;
	1670	return false;
1201	1671	}
1202	1672
1203	1673	template <
...	...	@@ -1206,7 +1676,9 @@ template <
1206	1676	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>()), bool>,
1207	1677	int> = 0>
1208	1678	_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const optional<_Tp>& __x, const _Up& __v) {
1209		return static_cast<bool>(__x) ? *__x <= __v : true;
	1679	if (__x.has_value())
	1680	return *__x <= __v;
	1681	return true;
1210	1682	}
1211	1683
1212	1684	template <
...	...	@@ -1215,7 +1687,9 @@ template <
1215	1687	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() <= std::declval<const _Up&>()), bool>,
1216	1688	int> = 0>
1217	1689	_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const _Tp& __v, const optional<_Up>& __x) {
1218		return static_cast<bool>(__x) ? __v <= *__x : false;
	1690	if (__x.has_value())
	1691	return __v <= *__x;
	1692	return false;
1219	1693	}
1220	1694
1221	1695	template < class _Tp,
...	...	@@ -1223,7 +1697,9 @@ template < class _Tp,
1223	1697	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() > std::declval<const _Up&>()), bool>,
1224	1698	int> = 0>
1225	1699	_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const optional<_Tp>& __x, const _Up& __v) {
1226		return static_cast<bool>(__x) ? *__x > __v : false;
	1700	if (__x.has_value())
	1701	return *__x > __v;
	1702	return false;
1227	1703	}
1228	1704
1229	1705	template < class _Tp,
...	...	@@ -1231,7 +1707,9 @@ template < class _Tp,
1231	1707	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() > std::declval<const _Up&>()), bool>,
1232	1708	int> = 0>
1233	1709	_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const _Tp& __v, const optional<_Up>& __x) {
1234		return static_cast<bool>(__x) ? __v > *__x : true;
	1710	if (__x.has_value())
	1711	return __v > *__x;
	1712	return true;
1235	1713	}
1236	1714
1237	1715	template <
...	...	@@ -1240,7 +1718,9 @@ template <
1240	1718	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>()), bool>,
1241	1719	int> = 0>
1242	1720	_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const optional<_Tp>& __x, const _Up& __v) {
1243		return static_cast<bool>(__x) ? *__x >= __v : false;
	1721	if (__x.has_value())
	1722	return *__x >= __v;
	1723	return false;
1244	1724	}
1245	1725
1246	1726	template <
...	...	@@ -1249,7 +1729,9 @@ template <
1249	1729	enable_if_t<__is_core_convertible_v<decltype(std::declval<const _Tp&>() >= std::declval<const _Up&>()), bool>,
1250	1730	int> = 0>
1251	1731	_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const _Tp& __v, const optional<_Up>& __x) {
1252		return static_cast<bool>(__x) ? __v >= *__x : true;
	1732	if (__x.has_value())
	1733	return __v >= *__x;
	1734	return true;
1253	1735	}
1254	1736
1255	1737	# if _LIBCPP_STD_VER >= 20
...	...	@@ -1264,23 +1746,36 @@ operator<=>(const optional<_Tp>& __x, const _Up& __v) {
1264	1746	# endif // _LIBCPP_STD_VER >= 20
1265	1747
1266	1748	template <class _Tp, enable_if_t< is_move_constructible_v<_Tp> && is_swappable_v<_Tp>, int> = 0>
1267		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
1268		swap(optional<_Tp>& __x, optional<_Tp>& __y) noexcept(noexcept(__x.swap(__y))) {
	1749	inline _LIBCPP_HIDE_FROM_ABI
	1750	_LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(optional<_Tp>& __x, optional<_Tp>& __y) noexcept(noexcept(__x.swap(__y))) {
1269	1751	__x.swap(__y);
1270	1752	}
1271	1753
1272		template <class _Tp>
1273		_LIBCPP_HIDE_FROM_ABI constexpr optional<decay_t<_Tp>> make_optional(_Tp&& __v) {
	1754	struct __make_optional_barrier_tag {
	1755	explicit __make_optional_barrier_tag() = default;
	1756	};
	1757
	1758	template <
	1759	# if _LIBCPP_STD_VER >= 26
	1760	__make_optional_barrier_tag = __make_optional_barrier_tag{},
	1761	# endif
	1762	class _Tp,
	1763	enable_if_t<is_constructible_v<decay_t<_Tp>, _Tp>, int> = 0>
	1764	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional<decay_t<_Tp>> make_optional(_Tp&& __v) {
1274	1765	return optional<decay_t<_Tp>>(std::forward<_Tp>(__v));
1275	1766	}
1276	1767
1277		template <class _Tp, class... _Args>
1278		_LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(_Args&&... __args) {
	1768	template <class _Tp, class... _Args, enable_if_t<__is_constructible_for_optional_v<_Tp, _Args...>, int> = 0>
	1769	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(_Args&&... __args) {
1279	1770	return optional<_Tp>(in_place, std::forward<_Args>(__args)...);
1280	1771	}
1281	1772
1282		template <class _Tp, class _Up, class... _Args>
1283		_LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp> make_optional(initializer_list<_Up> __il, _Args&&... __args) {
	1773	template <class _Tp,
	1774	class _Up,
	1775	class... _Args,
	1776	enable_if_t<__is_constructible_for_optional_initializer_list_v<_Tp, _Up, _Args...>, int> = 0>
	1777	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr optional<_Tp>
	1778	make_optional(initializer_list<_Up> __il, _Args&&... __args) {
1284	1779	return optional<_Tp>(in_place, __il, std::forward<_Args>(__args)...);
1285	1780	}
1286	1781
...	...	@@ -1291,7 +1786,7 @@ struct hash< __enable_hash_helper<optional<_Tp>, remove_const_t<_Tp>> > {
1291	1786	_LIBCPP_DEPRECATED_IN_CXX17 typedef size_t result_type;
1292	1787	# endif
1293	1788
1294		_LIBCPP_HIDE_FROM_ABI size_t operator()(const optional<_Tp>& __opt) const {
	1789	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const optional<_Tp>& __opt) const {
1295	1790	return static_cast<bool>(__opt) ? hash<remove_const_t<_Tp>>()(*__opt) : 0;
1296	1791	}
1297	1792	};

lib/libcxx/include/print+9-5

...	...	@@ -329,7 +329,8 @@ __vprint_unicode([[maybe_unused]] FILE* __stream,
329	329	} // namespace __print
330	330
331	331	template <class... _Args>
332		_LIBCPP_HIDE_FROM_ABI void print(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
	332	_LIBCPP_HIDE_FROM_ABI void
	333	print(FILE* _LIBCPP_DIAGNOSE_NULLPTR __stream, format_string<_Args...> __fmt, _Args&&... __args) {
333	334	# if _LIBCPP_HAS_UNICODE
334	335	if constexpr (__print::__use_unicode_execution_charset)
335	336	__print::__vprint_unicode(__stream, __fmt.get(), std::make_format_args(__args...), false);
...	...	@@ -346,7 +347,8 @@ _LIBCPP_HIDE_FROM_ABI void print(format_string<_Args...> __fmt, _Args&&... __arg
346	347	}
347	348
348	349	template <class... _Args>
349		_LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, format_string<_Args...> __fmt, _Args&&... __args) {
	350	_LIBCPP_HIDE_FROM_ABI void
	351	println(FILE* _LIBCPP_DIAGNOSE_NULLPTR __stream, format_string<_Args...> __fmt, _Args&&... __args) {
350	352	# if _LIBCPP_HAS_UNICODE
351	353	// Note the wording in the Standard is inefficient. The output of
352	354	// std::format is a std::string which is then copied. This solution
...	...	@@ -361,7 +363,7 @@ _LIBCPP_HIDE_FROM_ABI void println(FILE* __stream, format_string<_Args...> __fmt
361	363	}
362	364
363	365	template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
364		_LIBCPP_HIDE_FROM_ABI inline void println(FILE* __stream) {
	366	_LIBCPP_HIDE_FROM_ABI inline void println(FILE* _LIBCPP_DIAGNOSE_NULLPTR __stream) {
365	367	std::print(__stream, "\n");
366	368	}
367	369
...	...	@@ -377,7 +379,8 @@ _LIBCPP_HIDE_FROM_ABI void println(format_string<_Args...> __fmt, _Args&&... __a
377	379
378	380	# if _LIBCPP_HAS_UNICODE
379	381	template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
380		_LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(FILE* __stream, string_view __fmt, format_args __args) {
	382	_LIBCPP_HIDE_FROM_ABI inline void
	383	vprint_unicode(FILE* _LIBCPP_DIAGNOSE_NULLPTR __stream, string_view __fmt, format_args __args) {
381	384	__print::__vprint_unicode(__stream, __fmt, __args, false);
382	385	}
383	386
...	...	@@ -389,7 +392,8 @@ _LIBCPP_HIDE_FROM_ABI inline void vprint_unicode(string_view __fmt, format_args
389	392	# endif // _LIBCPP_HAS_UNICODE
390	393
391	394	template <class = void> // TODO PRINT template or availability markup fires too eagerly (http://llvm.org/PR61563).
392		_LIBCPP_HIDE_FROM_ABI inline void vprint_nonunicode(FILE* __stream, string_view __fmt, format_args __args) {
	395	_LIBCPP_HIDE_FROM_ABI inline void
	396	vprint_nonunicode(FILE* _LIBCPP_DIAGNOSE_NULLPTR __stream, string_view __fmt, format_args __args) {
393	397	__print::__vprint_nonunicode(__stream, __fmt, __args, false);
394	398	}
395	399

lib/libcxx/include/queue+39-37

...	...	@@ -376,12 +376,12 @@ public:
376	376	# endif // _LIBCPP_CXX03_LANG
377	377
378	378	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
379		_LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
	379	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
380	380
381		_LIBCPP_HIDE_FROM_ABI reference front() { return c.front(); }
382		_LIBCPP_HIDE_FROM_ABI const_reference front() const { return c.front(); }
383		_LIBCPP_HIDE_FROM_ABI reference back() { return c.back(); }
384		_LIBCPP_HIDE_FROM_ABI const_reference back() const { return c.back(); }
	381	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference front() { return c.front(); }
	382	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference front() const { return c.front(); }
	383	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference back() { return c.back(); }
	384	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference back() const { return c.back(); }
385	385
386	386	_LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
387	387	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -437,19 +437,19 @@ public:
437	437	};
438	438
439	439	# if _LIBCPP_STD_VER >= 17
440		template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
	440	template <class _Container, class = enable_if_t<!__is_allocator_v<_Container>>>
441	441	queue(_Container) -> queue<typename _Container::value_type, _Container>;
442	442
443	443	template <class _Container,
444	444	class _Alloc,
445		class = enable_if_t<!__is_allocator<_Container>::value>,
	445	class = enable_if_t<!__is_allocator_v<_Container>>,
446	446	class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
447	447	queue(_Container, _Alloc) -> queue<typename _Container::value_type, _Container>;
448	448	# endif
449	449
450	450	# if _LIBCPP_STD_VER >= 23
451	451	template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
452		queue(_InputIterator, _InputIterator) -> queue<__iter_value_type<_InputIterator>>;
	452	queue(_InputIterator, _InputIterator) -> queue<__iterator_value_type<_InputIterator>>;
453	453
454	454	template <ranges::input_range _Range>
455	455	queue(from_range_t, _Range&&) -> queue<ranges::range_value_t<_Range>>;
...	...	@@ -457,11 +457,11 @@ queue(from_range_t, _Range&&) -> queue<ranges::range_value_t<_Range>>;
457	457	template <class _InputIterator,
458	458	class _Alloc,
459	459	__enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
460		__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
	460	__enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
461	461	queue(_InputIterator, _InputIterator, _Alloc)
462		-> queue<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
	462	-> queue<__iterator_value_type<_InputIterator>, deque<__iterator_value_type<_InputIterator>, _Alloc>>;
463	463
464		template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
	464	template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
465	465	queue(from_range_t, _Range&&, _Alloc)
466	466	-> queue<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
467	467	# endif
...	...	@@ -664,8 +664,10 @@ public:
664	664	# endif
665	665
666	666	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
667		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
668		_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.front(); }
	667	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
	668	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference top() const {
	669	return c.front();
	670	}
669	671
670	672	_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push(const value_type& __v);
671	673	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -700,45 +702,45 @@ public:
700	702	# if _LIBCPP_STD_VER >= 17
701	703	template <class _Compare,
702	704	class _Container,
703		class = enable_if_t<!__is_allocator<_Compare>::value>,
704		class = enable_if_t<!__is_allocator<_Container>::value> >
	705	class = enable_if_t<!__is_allocator_v<_Compare>>,
	706	class = enable_if_t<!__is_allocator_v<_Container>>>
705	707	priority_queue(_Compare, _Container) -> priority_queue<typename _Container::value_type, _Container, _Compare>;
706	708
707	709	template <class _InputIterator,
708		class _Compare = less<__iter_value_type<_InputIterator>>,
709		class _Container = vector<__iter_value_type<_InputIterator>>,
	710	class _Compare = less<__iterator_value_type<_InputIterator>>,
	711	class _Container = vector<__iterator_value_type<_InputIterator>>,
710	712	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
711		class = enable_if_t<!__is_allocator<_Compare>::value>,
712		class = enable_if_t<!__is_allocator<_Container>::value> >
	713	class = enable_if_t<!__is_allocator_v<_Compare>>,
	714	class = enable_if_t<!__is_allocator_v<_Container>>>
713	715	priority_queue(_InputIterator, _InputIterator, _Compare = _Compare(), _Container = _Container())
714		-> priority_queue<__iter_value_type<_InputIterator>, _Container, _Compare>;
	716	-> priority_queue<__iterator_value_type<_InputIterator>, _Container, _Compare>;
715	717
716	718	template <class _Compare,
717	719	class _Container,
718	720	class _Alloc,
719		class = enable_if_t<!__is_allocator<_Compare>::value>,
720		class = enable_if_t<!__is_allocator<_Container>::value>,
721		class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
	721	class = enable_if_t<!__is_allocator_v<_Compare>>,
	722	class = enable_if_t<!__is_allocator_v<_Container>>,
	723	class = enable_if_t<uses_allocator<_Container, _Alloc>::value>>
722	724	priority_queue(_Compare, _Container, _Alloc) -> priority_queue<typename _Container::value_type, _Container, _Compare>;
723	725
724	726	template <class _InputIterator,
725	727	class _Allocator,
726	728	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
727		class = enable_if_t<__is_allocator<_Allocator>::value> >
	729	class = enable_if_t<__is_allocator_v<_Allocator>>>
728	730	priority_queue(_InputIterator, _InputIterator, _Allocator)
729		-> priority_queue<__iter_value_type<_InputIterator>,
730		vector<__iter_value_type<_InputIterator>, _Allocator>,
731		less<__iter_value_type<_InputIterator>>>;
	731	-> priority_queue<__iterator_value_type<_InputIterator>,
	732	vector<__iterator_value_type<_InputIterator>, _Allocator>,
	733	less<__iterator_value_type<_InputIterator>>>;
732	734
733	735	template <class _InputIterator,
734	736	class _Compare,
735	737	class _Allocator,
736	738	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
737		class = enable_if_t<!__is_allocator<_Compare>::value>,
738		class = enable_if_t<__is_allocator<_Allocator>::value> >
	739	class = enable_if_t<!__is_allocator_v<_Compare>>,
	740	class = enable_if_t<__is_allocator_v<_Allocator>>>
739	741	priority_queue(_InputIterator, _InputIterator, _Compare, _Allocator)
740		-> priority_queue<__iter_value_type<_InputIterator>,
741		vector<__iter_value_type<_InputIterator>, _Allocator>,
	742	-> priority_queue<__iterator_value_type<_InputIterator>,
	743	vector<__iterator_value_type<_InputIterator>, _Allocator>,
742	744	_Compare>;
743	745
744	746	template <class _InputIterator,
...	...	@@ -746,8 +748,8 @@ template <class _InputIterator,
746	748	class _Container,
747	749	class _Alloc,
748	750	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
749		class = enable_if_t<!__is_allocator<_Compare>::value>,
750		class = enable_if_t<!__is_allocator<_Container>::value>,
	751	class = enable_if_t<!__is_allocator_v<_Compare>>,
	752	class = enable_if_t<!__is_allocator_v<_Container>>,
751	753	class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
752	754	priority_queue(_InputIterator, _InputIterator, _Compare, _Container, _Alloc)
753	755	-> priority_queue<typename _Container::value_type, _Container, _Compare>;
...	...	@@ -757,19 +759,19 @@ priority_queue(_InputIterator, _InputIterator, _Compare, _Container, _Alloc)
757	759
758	760	template <ranges::input_range _Range,
759	761	class _Compare = less<ranges::range_value_t<_Range>>,
760		class = enable_if_t<!__is_allocator<_Compare>::value>>
	762	class = enable_if_t<!__is_allocator_v<_Compare>>>
761	763	priority_queue(from_range_t, _Range&&, _Compare = _Compare())
762	764	-> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>>, _Compare>;
763	765
764	766	template <ranges::input_range _Range,
765	767	class _Compare,
766	768	class _Alloc,
767		class = enable_if_t<!__is_allocator<_Compare>::value>,
768		class = enable_if_t<__is_allocator<_Alloc>::value>>
	769	class = enable_if_t<!__is_allocator_v<_Compare>>,
	770	class = enable_if_t<__is_allocator_v<_Alloc>>>
769	771	priority_queue(from_range_t, _Range&&, _Compare, _Alloc)
770	772	-> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>, _Alloc>, _Compare>;
771	773
772		template <ranges::input_range _Range, class _Alloc, class = enable_if_t<__is_allocator<_Alloc>::value>>
	774	template <ranges::input_range _Range, class _Alloc, class = enable_if_t<__is_allocator_v<_Alloc>>>
773	775	priority_queue(from_range_t, _Range&&, _Alloc)
774	776	-> priority_queue<ranges::range_value_t<_Range>, vector<ranges::range_value_t<_Range>, _Alloc>>;
775	777

lib/libcxx/include/ranges+48

...	...	@@ -116,6 +116,10 @@ namespace std::ranges {
116	116	// [range.dangling], dangling iterator handling
117	117	struct dangling;
118	118
	119	// [range.elementsof], class template elements_of
	120	template<range R, class Allocator = allocator<byte>>
	121	struct elements_of;
	122
119	123	template<range R>
120	124	using borrowed_iterator_t = see below;
121	125
...	...	@@ -267,6 +271,11 @@ namespace std::ranges {
267	271	template<class W, class Bound>
268	272	inline constexpr bool enable_borrowed_range<iota_view<W, Bound>> = true;
269	273
	274	namespace views {
	275	inline constexpr unspecified iota = unspecified;
	276	inline constexpr unspecified indices = unspecified; // Since C++26
	277	}
	278
270	279	// [range.repeat], repeat view
271	280	template<class T>
272	281	concept integer-like-with-usable-difference-type = // exposition only
...	...	@@ -339,6 +348,41 @@ namespace std::ranges {
339	348
340	349	namespace views { inline constexpr unspecified zip = unspecified; } // C++23
341	350
	351	// [range.zip.transform], zip transform view
	352	template<move_constructible F, input_range... Views>
	353	requires (view<Views> && ...) && (sizeof...(Views) > 0) && is_object_v<F> &&
	354	regular_invocable<F&, range_reference_t<Views>...> &&
	355	can-reference<invoke_result_t<F&, range_reference_t<Views>...>>
	356	class zip_transform_view; // C++23
	357
	358	namespace views { inline constexpr unspecified zip_transform = unspecified; } // C++23
	359
	360	// [range.adjacent], adjacent view
	361	template<forward_range V, size_t N>
	362	requires view<V> && (N > 0)
	363	class adjacent_view;
	364
	365	template<class V, size_t N>
	366	constexpr bool enable_borrowed_range<adjacent_view<V, N>> =
	367	enable_borrowed_range<V>;
	368
	369	namespace views {
	370	template<size_t N>
	371	constexpr unspecified adjacent = unspecified;
	372	inline constexpr auto pairwise = adjacent<2>;
	373	}
	374
	375	// [range.adjacent.transform], adjacent transform view
	376	template<forward_range V, move_constructible F, size_t N>
	377	requires see below
	378	class adjacent_transform_view;
	379
	380	namespace views {
	381	template<size_t N>
	382	constexpr unspecified adjacent_transform = unspecified;
	383	inline constexpr auto pairwise_transform = adjacent_transform<2>;
	384	}
	385
342	386	// [range.as.rvalue]
343	387	template <view V>
344	388	requires input_range<V>
...	...	@@ -404,6 +448,7 @@ namespace std {
404	448	# include <__ranges/data.h>
405	449	# include <__ranges/drop_view.h>
406	450	# include <__ranges/drop_while_view.h>
	451	# include <__ranges/elements_of.h>
407	452	# include <__ranges/elements_view.h>
408	453	# include <__ranges/empty.h>
409	454	# include <__ranges/empty_view.h>
...	...	@@ -433,12 +478,15 @@ namespace std {
433	478	# endif
434	479
435	480	# if _LIBCPP_STD_VER >= 23
	481	# include <__ranges/adjacent_transform_view.h>
	482	# include <__ranges/adjacent_view.h>
436	483	# include <__ranges/as_rvalue_view.h>
437	484	# include <__ranges/chunk_by_view.h>
438	485	# include <__ranges/from_range.h>
439	486	# include <__ranges/join_with_view.h>
440	487	# include <__ranges/repeat_view.h>
441	488	# include <__ranges/to.h>
	489	# include <__ranges/zip_transform_view.h>
442	490	# include <__ranges/zip_view.h>
443	491	# endif
444	492

lib/libcxx/include/regex+33-31

...	...	@@ -986,7 +986,7 @@ public:
986	986	explicit regex_error(regex_constants::error_type __ecode);
987	987	_LIBCPP_HIDE_FROM_ABI regex_error(const regex_error&) _NOEXCEPT = default;
988	988	~regex_error() _NOEXCEPT override;
989		_LIBCPP_HIDE_FROM_ABI regex_constants::error_type code() const { return __code_; }
	989	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI regex_constants::error_type code() const { return __code_; }
990	990	};
991	991
992	992	template <regex_constants::error_type _Ev>
...	...	@@ -1004,7 +1004,7 @@ public:
1004	1004	typedef _CharT char_type;
1005	1005	typedef basic_string<char_type> string_type;
1006	1006	typedef locale locale_type;
1007		# if defined(__BIONIC__) \|\| defined(_NEWLIB_VERSION)
	1007	# if defined(__BIONIC__) \|\| _LIBCPP_LIBC_NEWLIB
1008	1008	// Originally bionic's ctype_base used its own ctype masks because the
1009	1009	// builtin ctype implementation wasn't in libc++ yet. Bionic's ctype mask
1010	1010	// was only 8 bits wide and already saturated, so it used a wider type here
...	...	@@ -1013,9 +1013,7 @@ public:
1013	1013	// implementation, but this was not updated to match. Since then Android has
1014	1014	// needed to maintain a stable libc++ ABI, and this can't be changed without
1015	1015	// an ABI break.
1016		// We also need this workaround for newlib since _NEWLIB_VERSION is not
1017		// defined yet inside __config, so we can't set the
1018		// _LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE macro. Additionally, newlib is
	1016	// We also need this workaround for newlib since newlib is
1019	1017	// often used for space constrained environments, so it makes sense not to
1020	1018	// duplicate the ctype table.
1021	1019	typedef uint16_t char_class_type;
...	...	@@ -2120,7 +2118,7 @@ public:
2120	2118	__ranges_.push_back(
2121	2119	std::make_pair(__traits_.transform(__b.begin(), __b.end()), __traits_.transform(__e.begin(), __e.end())));
2122	2120	} else {
2123		if (__b.size() != 1 \|\| __e.size() != 1)
	2121	if (__b.size() != 1 \|\| __e.size() != 1 \|\| char_traits<typename string_type::value_type>::lt(__e[0], __b[0]))
2124	2122	std::__throw_regex_error<regex_constants::error_range>();
2125	2123	if (__icase_) {
2126	2124	__b[0] = __traits_.translate_nocase(__b[0]);
...	...	@@ -2414,8 +2412,8 @@ public:
2414	2412	# endif // _LIBCPP_CXX03_LANG
2415	2413
2416	2414	// const operations:
2417		_LIBCPP_HIDE_FROM_ABI unsigned mark_count() const { return __marked_count_; }
2418		_LIBCPP_HIDE_FROM_ABI flag_type flags() const { return __flags_; }
	2415	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI unsigned mark_count() const { return __marked_count_; }
	2416	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI flag_type flags() const { return __flags_; }
2419	2417
2420	2418	// locale:
2421	2419	_LIBCPP_HIDE_FROM_ABI locale_type imbue(locale_type __loc) {
...	...	@@ -2423,7 +2421,7 @@ public:
2423	2421	__start_.reset();
2424	2422	return __traits_.imbue(__loc);
2425	2423	}
2426		_LIBCPP_HIDE_FROM_ABI locale_type getloc() const { return __traits_.getloc(); }
	2424	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI locale_type getloc() const { return __traits_.getloc(); }
2427	2425
2428	2426	// swap:
2429	2427	void swap(basic_regex& __r);
...	...	@@ -4208,17 +4206,17 @@ public:
4208	4206
4209	4207	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR sub_match() : matched() {}
4210	4208
4211		_LIBCPP_HIDE_FROM_ABI difference_type length() const {
	4209	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI difference_type length() const {
4212	4210	return matched ? std::distance(this->first, this->second) : 0;
4213	4211	}
4214		_LIBCPP_HIDE_FROM_ABI string_type str() const {
	4212	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type str() const {
4215	4213	return matched ? string_type(this->first, this->second) : string_type();
4216	4214	}
4217	4215	_LIBCPP_HIDE_FROM_ABI operator string_type() const { return str(); }
4218	4216
4219		_LIBCPP_HIDE_FROM_ABI int compare(const sub_match& __s) const { return str().compare(__s.str()); }
4220		_LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return str().compare(__s); }
4221		_LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return str().compare(__s); }
	4217	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int compare(const sub_match& __s) const { return str().compare(__s.str()); }
	4218	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int compare(const string_type& __s) const { return str().compare(__s); }
	4219	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int compare(const value_type* __s) const { return str().compare(__s); }
4222	4220
4223	4221	_LIBCPP_HIDE_FROM_ABI void swap(sub_match& __s) _NOEXCEPT_(__is_nothrow_swappable_v<_BidirectionalIterator>) {
4224	4222	this->pair<_BidirectionalIterator, _BidirectionalIterator>::swap(__s);
...	...	@@ -4583,49 +4581,53 @@ public:
4583	4581	_LIBCPP_HIDE_FROM_ABI bool ready() const { return __ready_; }
4584	4582
4585	4583	// size:
4586		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __matches_.size(); }
4587		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __matches_.max_size(); }
	4584	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __matches_.size(); }
	4585	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __matches_.max_size(); }
4588	4586	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return size() == 0; }
4589	4587
4590	4588	// element access:
4591		_LIBCPP_HIDE_FROM_ABI difference_type length(size_type __sub = 0) const {
	4589	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI difference_type length(size_type __sub = 0) const {
4592	4590	// If the match results are not ready, this will return `0`.
4593	4591	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::length() called when not ready");
4594	4592	return (*this)[__sub].length();
4595	4593	}
4596		_LIBCPP_HIDE_FROM_ABI difference_type position(size_type __sub = 0) const {
	4594	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI difference_type position(size_type __sub = 0) const {
4597	4595	// If the match results are not ready, this will return the result of subtracting two default-constructed iterators
4598	4596	// (which is typically a well-defined operation).
4599	4597	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::position() called when not ready");
4600	4598	return std::distance(__position_start_, (*this)[__sub].first);
4601	4599	}
4602		_LIBCPP_HIDE_FROM_ABI string_type str(size_type __sub = 0) const {
	4600	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type str(size_type __sub = 0) const {
4603	4601	// If the match results are not ready, this will return an empty string.
4604	4602	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::str() called when not ready");
4605	4603	return (*this)[__sub].str();
4606	4604	}
4607		_LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const {
	4605	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __n) const {
4608	4606	// If the match results are not ready, this call will be equivalent to calling this function with `__n >= size()`,
4609	4607	// returning an empty subrange.
4610	4608	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::operator[]() called when not ready");
4611	4609	return __n < __matches_.size() ? __matches_[__n] : __unmatched_;
4612	4610	}
4613	4611
4614		_LIBCPP_HIDE_FROM_ABI const_reference prefix() const {
	4612	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference prefix() const {
4615	4613	// If the match results are not ready, this will return a default-constructed empty `__suffix_`.
4616	4614	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::prefix() called when not ready");
4617	4615	return __prefix_;
4618	4616	}
4619		_LIBCPP_HIDE_FROM_ABI const_reference suffix() const {
	4617	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference suffix() const {
4620	4618	// If the match results are not ready, this will return a default-constructed empty `__suffix_`.
4621	4619	_LIBCPP_ASSERT_PEDANTIC(ready(), "match_results::suffix() called when not ready");
4622	4620	return __suffix_;
4623	4621	}
4624	4622
4625		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const { return empty() ? __matches_.end() : __matches_.begin(); }
4626		_LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __matches_.end(); }
4627		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const { return empty() ? __matches_.end() : __matches_.begin(); }
4628		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const { return __matches_.end(); }
	4623	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const {
	4624	return empty() ? __matches_.end() : __matches_.begin();
	4625	}
	4626	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const { return __matches_.end(); }
	4627	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const {
	4628	return empty() ? __matches_.end() : __matches_.begin();
	4629	}
	4630	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const { return __matches_.end(); }
4629	4631
4630	4632	// format:
4631	4633	template <class _OutputIter>
...	...	@@ -4641,14 +4643,14 @@ public:
4641	4643	return format(__output_iter, __fmt.data(), __fmt.data() + __fmt.size(), __flags);
4642	4644	}
4643	4645	template <class _ST, class _SA>
4644		_LIBCPP_HIDE_FROM_ABI basic_string<char_type, _ST, _SA>
	4646	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_string<char_type, _ST, _SA>
4645	4647	format(const basic_string<char_type, _ST, _SA>& __fmt,
4646	4648	regex_constants::match_flag_type __flags = regex_constants::format_default) const {
4647	4649	basic_string<char_type, _ST, _SA> __r;
4648	4650	format(std::back_inserter(__r), __fmt.data(), __fmt.data() + __fmt.size(), __flags);
4649	4651	return __r;
4650	4652	}
4651		_LIBCPP_HIDE_FROM_ABI string_type
	4653	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type
4652	4654	format(const char_type* __fmt, regex_constants::match_flag_type __flags = regex_constants::format_default) const {
4653	4655	string_type __r;
4654	4656	format(std::back_inserter(__r), __fmt, __fmt + char_traits<char_type>::length(__fmt), __flags);
...	...	@@ -4656,7 +4658,7 @@ public:
4656	4658	}
4657	4659
4658	4660	// allocator:
4659		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return __matches_.get_allocator(); }
	4661	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const { return __matches_.get_allocator(); }
4660	4662
4661	4663	// swap:
4662	4664	void swap(match_results& __m);
...	...	@@ -5377,7 +5379,7 @@ public:
5377	5379	_LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_iterator& __x) const { return !(*this == __x); }
5378	5380	# endif
5379	5381
5380		_LIBCPP_HIDE_FROM_ABI reference operator*() const { return __match_; }
	5382	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __match_; }
5381	5383	_LIBCPP_HIDE_FROM_ABI pointer operator->() const { return std::addressof(__match_); }
5382	5384
5383	5385	regex_iterator& operator++();
...	...	@@ -5558,7 +5560,7 @@ public:
5558	5560	_LIBCPP_HIDE_FROM_ABI bool operator!=(const regex_token_iterator& __x) const { return !(*this == __x); }
5559	5561	# endif
5560	5562
5561		_LIBCPP_HIDE_FROM_ABI const value_type& operator() const { return __result_; }
	5563	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const value_type& operator() const { return __result_; }
5562	5564	_LIBCPP_HIDE_FROM_ABI const value_type* operator->() const { return __result_; }
5563	5565
5564	5566	regex_token_iterator& operator++();

lib/libcxx/include/scoped_allocator+16-11

...	...	@@ -382,13 +382,17 @@ public:
382	382	// scoped_allocator_adaptor& operator=(scoped_allocator_adaptor&&) = default;
383	383	// ~scoped_allocator_adaptor() = default;
384	384
385		_LIBCPP_HIDE_FROM_ABI inner_allocator_type& inner_allocator() _NOEXCEPT { return _Base::inner_allocator(); }
386		_LIBCPP_HIDE_FROM_ABI const inner_allocator_type& inner_allocator() const _NOEXCEPT {
	385	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inner_allocator_type& inner_allocator() _NOEXCEPT {
	386	return _Base::inner_allocator();
	387	}
	388	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const inner_allocator_type& inner_allocator() const _NOEXCEPT {
387	389	return _Base::inner_allocator();
388	390	}
389	391
390		_LIBCPP_HIDE_FROM_ABI outer_allocator_type& outer_allocator() _NOEXCEPT { return _Base::outer_allocator(); }
391		_LIBCPP_HIDE_FROM_ABI const outer_allocator_type& outer_allocator() const _NOEXCEPT {
	392	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI outer_allocator_type& outer_allocator() _NOEXCEPT {
	393	return _Base::outer_allocator();
	394	}
	395	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const outer_allocator_type& outer_allocator() const _NOEXCEPT {
392	396	return _Base::outer_allocator();
393	397	}
394	398
...	...	@@ -403,7 +407,7 @@ public:
403	407	allocator_traits<outer_allocator_type>::deallocate(outer_allocator(), __p, __n);
404	408	}
405	409
406		_LIBCPP_HIDE_FROM_ABI size_type max_size() const {
	410	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const {
407	411	return allocator_traits<outer_allocator_type>::max_size(outer_allocator());
408	412	}
409	413
...	...	@@ -434,10 +438,10 @@ public:
434	438	piecewise_construct,
435	439	__transform_tuple(typename __uses_alloc_ctor< _T1, inner_allocator_type&, _Args1... >::type(),
436	440	std::move(__x),
437		typename __make_tuple_indices<sizeof...(_Args1)>::type{}),
	441	__index_sequence_for<_Args1...>()),
438	442	__transform_tuple(typename __uses_alloc_ctor< _T2, inner_allocator_type&, _Args2... >::type(),
439	443	std::move(__y),
440		typename __make_tuple_indices<sizeof...(_Args2)>::type{}));
	444	__index_sequence_for<_Args2...>()));
441	445	}
442	446
443	447	template <class _T1, class _T2>
...	...	@@ -473,7 +477,8 @@ public:
473	477	allocator_traits<typename _OM::type>::destroy(_OM()(outer_allocator()), __p);
474	478	}
475	479
476		_LIBCPP_HIDE_FROM_ABI scoped_allocator_adaptor select_on_container_copy_construction() const _NOEXCEPT {
	480	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI scoped_allocator_adaptor
	481	select_on_container_copy_construction() const _NOEXCEPT {
477	482	return _Base::select_on_container_copy_construction();
478	483	}
479	484
...	...	@@ -503,20 +508,20 @@ private:
503	508
504	509	template <class... _Args, size_t... _Idx>
505	510	_LIBCPP_HIDE_FROM_ABI tuple<_Args&&...>
506		__transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
	511	__transform_tuple(integral_constant<int, 0>, tuple<_Args...>&& __t, __index_sequence<_Idx...>) {
507	512	return std::forward_as_tuple(std::get<_Idx>(std::move(__t))...);
508	513	}
509	514
510	515	template <class... _Args, size_t... _Idx>
511	516	_LIBCPP_HIDE_FROM_ABI tuple<allocator_arg_t, inner_allocator_type&, _Args&&...>
512		__transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
	517	__transform_tuple(integral_constant<int, 1>, tuple<_Args...>&& __t, __index_sequence<_Idx...>) {
513	518	using _Tup = tuple<allocator_arg_t, inner_allocator_type&, _Args&&...>;
514	519	return _Tup(allocator_arg, inner_allocator(), std::get<_Idx>(std::move(__t))...);
515	520	}
516	521
517	522	template <class... _Args, size_t... _Idx>
518	523	_LIBCPP_HIDE_FROM_ABI tuple<_Args&&..., inner_allocator_type&>
519		__transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, __tuple_indices<_Idx...>) {
	524	__transform_tuple(integral_constant<int, 2>, tuple<_Args...>&& __t, __index_sequence<_Idx...>) {
520	525	using _Tup = tuple<_Args&&..., inner_allocator_type&>;
521	526	return _Tup(std::get<_Idx>(std::move(__t))..., inner_allocator());
522	527	}

lib/libcxx/include/semaphore+15-18

...	...	@@ -55,12 +55,11 @@ using binary_semaphore = counting_semaphore<1>; // since C++20
55	55	# include <__assert>
56	56	# include <__atomic/atomic.h>
57	57	# include <__atomic/atomic_sync.h>
	58	# include <__atomic/atomic_sync_timed.h>
58	59	# include <__atomic/memory_order.h>
59	60	# include <__chrono/time_point.h>
60	61	# include <__cstddef/ptrdiff_t.h>
61		# include <__thread/poll_with_backoff.h>
62	62	# include <__thread/support.h>
63		# include <__thread/timed_backoff_policy.h>
64	63	# include <limits>
65	64	# include <version>
66	65
...	...	@@ -90,7 +89,7 @@ class __atomic_semaphore_base {
90	89
91	90	public:
92	91	_LIBCPP_HIDE_FROM_ABI constexpr explicit __atomic_semaphore_base(ptrdiff_t __count) : __a_(__count) {}
93		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
	92	_LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
94	93	auto __old = __a_.fetch_add(__update, memory_order_release);
95	94	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
96	95	__update <= _LIBCPP_SEMAPHORE_MAX - __old, "update is greater than the expected value");
...	...	@@ -98,26 +97,26 @@ public:
98	97	__a_.notify_all();
99	98	}
100	99	}
101		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() {
	100	_LIBCPP_HIDE_FROM_ABI void acquire() {
102	101	std::__atomic_wait_unless(__a_, memory_order_relaxed, [this](ptrdiff_t& __old) {
103	102	return __try_acquire_impl(__old);
104	103	});
105	104	}
106	105	template <class _Rep, class _Period>
107		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
108		try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
	106	_LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
109	107	if (__rel_time == chrono::duration<_Rep, _Period>::zero())
110	108	return try_acquire();
111		auto const __poll_fn = [this]() { return try_acquire(); };
112		return std::__libcpp_thread_poll_with_backoff(__poll_fn, __libcpp_timed_backoff_policy(), __rel_time);
	109
	110	return std::__atomic_wait_unless_with_timeout(
	111	__a_, memory_order_relaxed, [this](ptrdiff_t& __old) { return __try_acquire_impl(__old); }, __rel_time);
113	112	}
114		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool try_acquire() {
	113	_LIBCPP_HIDE_FROM_ABI bool try_acquire() {
115	114	auto __old = __a_.load(memory_order_relaxed);
116	115	return __try_acquire_impl(__old);
117	116	}
118	117
119	118	private:
120		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool __try_acquire_impl(ptrdiff_t& __old) {
	119	_LIBCPP_HIDE_FROM_ABI bool __try_acquire_impl(ptrdiff_t& __old) {
121	120	while (true) {
122	121	if (__old == 0)
123	122	return false;
...	...	@@ -134,7 +133,7 @@ class counting_semaphore {
134	133	public:
135	134	static_assert(__least_max_value >= 0, "The least maximum value must be a positive number");
136	135
137		static constexpr ptrdiff_t max() noexcept { return __least_max_value; }
	136	[[nodiscard]] static constexpr ptrdiff_t max() noexcept { return __least_max_value; }
138	137
139	138	_LIBCPP_HIDE_FROM_ABI constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore_(__count) {
140	139	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(
...	...	@@ -151,20 +150,18 @@ public:
151	150	counting_semaphore(const counting_semaphore&) = delete;
152	151	counting_semaphore& operator=(const counting_semaphore&) = delete;
153	152
154		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
	153	_LIBCPP_HIDE_FROM_ABI void release(ptrdiff_t __update = 1) {
155	154	_LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(__update >= 0, "counting_semaphore:release called with a negative value");
156	155	__semaphore_.release(__update);
157	156	}
158		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void acquire() { __semaphore_.acquire(); }
	157	_LIBCPP_HIDE_FROM_ABI void acquire() { __semaphore_.acquire(); }
159	158	template <class _Rep, class _Period>
160		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
161		try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
	159	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_for(chrono::duration<_Rep, _Period> const& __rel_time) {
162	160	return __semaphore_.try_acquire_for(chrono::duration_cast<chrono::nanoseconds>(__rel_time));
163	161	}
164		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); }
	162	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire() { return __semaphore_.try_acquire(); }
165	163	template <class _Clock, class _Duration>
166		_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI bool
167		try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) {
	164	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool try_acquire_until(chrono::time_point<_Clock, _Duration> const& __abs_time) {
168	165	auto const __current = _Clock::now();
169	166	if (__current >= __abs_time)
170	167	return try_acquire();

lib/libcxx/include/set+220-192

...	...	@@ -518,19 +518,19 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred); // C++20
518	518	# include <__algorithm/equal.h>
519	519	# include <__algorithm/lexicographical_compare.h>
520	520	# include <__algorithm/lexicographical_compare_three_way.h>
	521	# include <__algorithm/specialized_algorithms.h>
521	522	# include <__assert>
522	523	# include <__config>
523	524	# include <__functional/is_transparent.h>
524	525	# include <__functional/operations.h>
525		# include <__fwd/set.h>
526	526	# include <__iterator/erase_if_container.h>
527	527	# include <__iterator/iterator_traits.h>
528		# include <__iterator/ranges_iterator_traits.h>
529	528	# include <__iterator/reverse_iterator.h>
530	529	# include <__memory/allocator.h>
531	530	# include <__memory/allocator_traits.h>
532	531	# include <__memory_resource/polymorphic_allocator.h>
533	532	# include <__node_handle>
	533	# include <__ranges/access.h>
534	534	# include <__ranges/concepts.h>
535	535	# include <__ranges/container_compatible_range.h>
536	536	# include <__ranges/from_range.h>
...	...	@@ -538,7 +538,6 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred); // C++20
538	538	# include <__type_traits/container_traits.h>
539	539	# include <__type_traits/enable_if.h>
540	540	# include <__type_traits/is_allocator.h>
541		# include <__type_traits/is_nothrow_assignable.h>
542	541	# include <__type_traits/is_nothrow_constructible.h>
543	542	# include <__type_traits/is_same.h>
544	543	# include <__type_traits/is_swappable.h>
...	...	@@ -570,7 +569,10 @@ _LIBCPP_PUSH_MACROS
570	569
571	570	_LIBCPP_BEGIN_NAMESPACE_STD
572	571
573		template <class _Key, class _Compare, class _Allocator>
	572	template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
	573	class multiset;
	574
	575	template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
574	576	class set {
575	577	public:
576	578	// types:
...	...	@@ -660,22 +662,20 @@ public:
660	662	: set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
661	663	# endif
662	664
663		_LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
	665	_LIBCPP_HIDE_FROM_ABI set(const set& __s) = default;
664	666
665	667	_LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
666	668
667	669	# ifndef _LIBCPP_CXX03_LANG
668		_LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
	670	_LIBCPP_HIDE_FROM_ABI set(set&& __s) = default;
669	671	# endif // _LIBCPP_CXX03_LANG
670	672
671	673	_LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
672	674
673		_LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __a) : __tree_(__s.__tree_.value_comp(), __a) {
674		insert(__s.begin(), __s.end());
675		}
	675	_LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __alloc) : __tree_(__s.__tree_, __alloc) {}
676	676
677	677	# ifndef _LIBCPP_CXX03_LANG
678		_LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __a);
	678	_LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __alloc) : __tree_(std::move(__s.__tree_), __alloc) {}
679	679
680	680	_LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
681	681	: __tree_(__comp) {
...	...	@@ -693,36 +693,38 @@ public:
693	693	# endif
694	694
695	695	_LIBCPP_HIDE_FROM_ABI set& operator=(initializer_list<value_type> __il) {
696		__tree_.__assign_unique(__il.begin(), __il.end());
	696	clear();
	697	insert(__il.begin(), __il.end());
697	698	return *this;
698	699	}
699	700
700		_LIBCPP_HIDE_FROM_ABI set& operator=(set&& __s) noexcept(is_nothrow_move_assignable<__base>::value) {
701		__tree_ = std::move(__s.__tree_);
702		return *this;
703		}
	701	_LIBCPP_HIDE_FROM_ABI set& operator=(set&& __s) = default;
704	702	# endif // _LIBCPP_CXX03_LANG
705	703
706	704	_LIBCPP_HIDE_FROM_ABI ~set() { static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
707	705
708		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
709		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
710		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
711		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
	706	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
	707	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
	708	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
	709	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
712	710
713		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
714		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
715		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
716		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	711	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
	712	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	713	return const_reverse_iterator(end());
	714	}
	715	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
	716	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	717	return const_reverse_iterator(begin());
	718	}
717	719
718		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
719		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
720		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
721		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	720	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
	721	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
	722	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
	723	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
722	724
723	725	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
724		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
725		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
	726	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
	727	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
726	728
727	729	// modifiers:
728	730	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -732,28 +734,24 @@ public:
732	734	}
733	735	template <class... _Args>
734	736	_LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __p, _Args&&... __args) {
735		return __tree_.__emplace_hint_unique(__p, std::forward<_Args>(__args)...);
	737	return __tree_.__emplace_hint_unique(__p, std::forward<_Args>(__args)...).first;
736	738	}
737	739	# endif // _LIBCPP_CXX03_LANG
738	740
739	741	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __v) { return __tree_.__emplace_unique(__v); }
740	742	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __v) {
741		return __tree_.__emplace_hint_unique(__p, __v);
	743	return __tree_.__emplace_hint_unique(__p, __v).first;
742	744	}
743	745
744	746	template <class _InputIterator>
745		_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
746		for (const_iterator __e = cend(); __f != __l; ++__f)
747		__tree_.__emplace_hint_unique(__e, *__f);
	747	_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
	748	__tree_.__insert_range_unique(__first, __last);
748	749	}
749	750
750	751	# if _LIBCPP_STD_VER >= 23
751	752	template <_ContainerCompatibleRange<value_type> _Range>
752	753	_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
753		const_iterator __end = cend();
754		for (auto&& __element : __range) {
755		__tree_.__emplace_hint_unique(__end, std::forward<decltype(__element)>(__element));
756		}
	754	__tree_.__insert_range_unique(ranges::begin(__range), ranges::end(__range));
757	755	}
758	756	# endif
759	757
...	...	@@ -763,7 +761,7 @@ public:
763	761	}
764	762
765	763	_LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __v) {
766		return __tree_.__emplace_hint_unique(__p, std::move(__v));
	764	return __tree_.__emplace_hint_unique(__p, std::move(__v)).first;
767	765	}
768	766
769	767	_LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
...	...	@@ -785,10 +783,10 @@ public:
785	783	"node_type with incompatible allocator passed to set::insert()");
786	784	return __tree_.template __node_handle_insert_unique<node_type>(__hint, std::move(__nh));
787	785	}
788		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	786	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
789	787	return __tree_.template __node_handle_extract<node_type>(__key);
790	788	}
791		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	789	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
792	790	return __tree_.template __node_handle_extract<node_type>(__it);
793	791	}
794	792	template <class _Compare2>
...	...	@@ -819,101 +817,120 @@ public:
819	817
820	818	_LIBCPP_HIDE_FROM_ABI void swap(set& __s) _NOEXCEPT_(__is_nothrow_swappable_v<__base>) { __tree_.swap(__s.__tree_); }
821	819
822		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
823		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
824		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
	820	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
	821	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
	822	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
825	823
826	824	// set operations:
827		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
828		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
	825	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
	826	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
829	827	# if _LIBCPP_STD_VER >= 14
830	828	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
831		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	829	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
832	830	return __tree_.find(__k);
833	831	}
834	832	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
835		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	833	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
836	834	return __tree_.find(__k);
837	835	}
838	836	# endif
839	837
840		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_unique(__k); }
	838	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	839	return __tree_.__count_unique(__k);
	840	}
841	841	# if _LIBCPP_STD_VER >= 14
842	842	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
843		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	843	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
844	844	return __tree_.__count_multi(__k);
845	845	}
846	846	# endif
847	847
848	848	# if _LIBCPP_STD_VER >= 20
849		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	849	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
850	850	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
851		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	851	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
852	852	return find(__k) != end();
853	853	}
854	854	# endif // _LIBCPP_STD_VER >= 20
855	855
856		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
857		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
	856	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) {
	857	return __tree_.__lower_bound_unique(__k);
	858	}
	859
	860	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const {
	861	return __tree_.__lower_bound_unique(__k);
	862	}
	863
	864	// The transparent versions of the lookup functions use the _multi version, since a non-element key is allowed to
	865	// match multiple elements.
858	866	# if _LIBCPP_STD_VER >= 14
859	867	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
860		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
861		return __tree_.lower_bound(__k);
	868	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
	869	return __tree_.__lower_bound_multi(__k);
862	870	}
863	871
864	872	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
865		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
866		return __tree_.lower_bound(__k);
	873	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
	874	return __tree_.__lower_bound_multi(__k);
867	875	}
868	876	# endif
869	877
870		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
871		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
	878	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) {
	879	return __tree_.__upper_bound_unique(__k);
	880	}
	881
	882	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const {
	883	return __tree_.__upper_bound_unique(__k);
	884	}
	885
872	886	# if _LIBCPP_STD_VER >= 14
873	887	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
874		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
875		return __tree_.upper_bound(__k);
	888	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
	889	return __tree_.__upper_bound_multi(__k);
876	890	}
877	891	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
878		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
879		return __tree_.upper_bound(__k);
	892	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
	893	return __tree_.__upper_bound_multi(__k);
880	894	}
881	895	# endif
882	896
883		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	897	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
884	898	return __tree_.__equal_range_unique(__k);
885	899	}
886		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	900	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
887	901	return __tree_.__equal_range_unique(__k);
888	902	}
889	903	# if _LIBCPP_STD_VER >= 14
890	904	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
891		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	905	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
892	906	return __tree_.__equal_range_multi(__k);
893	907	}
894	908	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
895		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	909	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
896	910	return __tree_.__equal_range_multi(__k);
897	911	}
898	912	# endif
	913
	914	template <class, class...>
	915	friend struct __specialized_algorithm;
899	916	};
900	917
901	918	# if _LIBCPP_STD_VER >= 17
902	919	template <class _InputIterator,
903		class _Compare = less<__iter_value_type<_InputIterator>>,
904		class _Allocator = allocator<__iter_value_type<_InputIterator>>,
	920	class _Compare = less<__iterator_value_type<_InputIterator>>,
	921	class _Allocator = allocator<__iterator_value_type<_InputIterator>>,
905	922	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
906		class = enable_if_t<__is_allocator<_Allocator>::value, void>,
907		class = enable_if_t<!__is_allocator<_Compare>::value, void>>
	923	class = enable_if_t<__is_allocator_v<_Allocator>>,
	924	class = enable_if_t<!__is_allocator_v<_Compare>>>
908	925	set(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
909		-> set<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
	926	-> set<__iterator_value_type<_InputIterator>, _Compare, _Allocator>;
910	927
911	928	# if _LIBCPP_STD_VER >= 23
912	929	template <ranges::input_range _Range,
913	930	class _Compare = less<ranges::range_value_t<_Range>>,
914	931	class _Allocator = allocator<ranges::range_value_t<_Range>>,
915		class = enable_if_t<__is_allocator<_Allocator>::value, void>,
916		class = enable_if_t<!__is_allocator<_Compare>::value, void>>
	932	class = enable_if_t<__is_allocator_v<_Allocator>>,
	933	class = enable_if_t<!__is_allocator_v<_Compare>>>
917	934	set(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
918	935	-> set<ranges::range_value_t<_Range>, _Compare, _Allocator>;
919	936	# endif
...	...	@@ -921,41 +938,43 @@ set(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
921	938	template <class _Key,
922	939	class _Compare = less<_Key>,
923	940	class _Allocator = allocator<_Key>,
924		class = enable_if_t<!__is_allocator<_Compare>::value, void>,
925		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	941	class = enable_if_t<!__is_allocator_v<_Compare>>,
	942	class = enable_if_t<__is_allocator_v<_Allocator>>>
926	943	set(initializer_list<_Key>, _Compare = _Compare(), _Allocator = _Allocator()) -> set<_Key, _Compare, _Allocator>;
927	944
928	945	template <class _InputIterator,
929	946	class _Allocator,
930	947	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
931		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
932		set(_InputIterator,
933		_InputIterator,
934		_Allocator) -> set<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
	948	class = enable_if_t<__is_allocator_v<_Allocator>>>
	949	set(_InputIterator, _InputIterator, _Allocator)
	950	-> set<__iterator_value_type<_InputIterator>, less<__iterator_value_type<_InputIterator>>, _Allocator>;
935	951
936	952	# if _LIBCPP_STD_VER >= 23
937		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
938		set(from_range_t,
939		_Range&&,
940		_Allocator) -> set<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
	953	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
	954	set(from_range_t, _Range&&, _Allocator)
	955	-> set<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
941	956	# endif
942	957
943		template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	958	template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
944	959	set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
945	960	# endif
946	961
947		# ifndef _LIBCPP_CXX03_LANG
	962	# if _LIBCPP_STD_VER >= 14
	963	template <class _Alg, class _Key, class _Compare, class _Allocator>
	964	struct __specialized_algorithm<_Alg, __single_range<set<_Key, _Compare, _Allocator>>> {
	965	using __set _LIBCPP_NODEBUG = set<_Key, _Compare, _Allocator>;
948	966
949		template <class _Key, class _Compare, class _Allocator>
950		set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {
951		if (__a != __s.get_allocator()) {
952		const_iterator __e = cend();
953		while (!__s.empty())
954		insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
955		}
956		}
	967	static const bool __has_algorithm =
	968	__specialized_algorithm<_Alg, __single_range<typename __set::__base>>::__has_algorithm;
957	969
958		# endif // _LIBCPP_CXX03_LANG
	970	// set's begin() and end() are identical with and without const qualification
	971	template <class... _Args>
	972	_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
	973	return __specialized_algorithm<_Alg, __single_range<typename __set::__base>>()(
	974	__set.__tree_, std::forward<_Args>(__args)...);
	975	}
	976	};
	977	# endif
959	978
960	979	template <class _Key, class _Compare, class _Allocator>
961	980	inline _LIBCPP_HIDE_FROM_ABI bool
...	...	@@ -1119,24 +1138,17 @@ public:
1119	1138	: multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
1120	1139	# endif
1121	1140
1122		_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
1123		: __tree_(__s.__tree_.value_comp(),
1124		__alloc_traits::select_on_container_copy_construction(__s.__tree_.__alloc())) {
1125		insert(__s.begin(), __s.end());
1126		}
	1141	_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s) = default;
1127	1142
1128	1143	_LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
1129	1144
1130	1145	# ifndef _LIBCPP_CXX03_LANG
1131		_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
	1146	_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) = default;
1132	1147
1133		_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
	1148	_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {}
1134	1149	# endif // _LIBCPP_CXX03_LANG
1135	1150	_LIBCPP_HIDE_FROM_ABI explicit multiset(const allocator_type& __a) : __tree_(__a) {}
1136		_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a)
1137		: __tree_(__s.__tree_.value_comp(), __a) {
1138		insert(__s.begin(), __s.end());
1139		}
	1151	_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a) : __tree_(__s.__tree_, __a) {}
1140	1152
1141	1153	# ifndef _LIBCPP_CXX03_LANG
1142	1154	_LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
...	...	@@ -1156,38 +1168,40 @@ public:
1156	1168	# endif
1157	1169
1158	1170	_LIBCPP_HIDE_FROM_ABI multiset& operator=(initializer_list<value_type> __il) {
1159		__tree_.__assign_multi(__il.begin(), __il.end());
	1171	clear();
	1172	insert(__il.begin(), __il.end());
1160	1173	return *this;
1161	1174	}
1162	1175
1163		_LIBCPP_HIDE_FROM_ABI multiset& operator=(multiset&& __s) _NOEXCEPT_(is_nothrow_move_assignable<__base>::value) {
1164		__tree_ = std::move(__s.__tree_);
1165		return *this;
1166		}
	1176	_LIBCPP_HIDE_FROM_ABI multiset& operator=(multiset&& __s) = default;
1167	1177	# endif // _LIBCPP_CXX03_LANG
1168	1178
1169	1179	_LIBCPP_HIDE_FROM_ABI ~multiset() {
1170	1180	static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), "");
1171	1181	}
1172	1182
1173		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
1174		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
1175		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
1176		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
	1183	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __tree_.begin(); }
	1184	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __tree_.begin(); }
	1185	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __tree_.end(); }
	1186	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __tree_.end(); }
1177	1187
1178		_LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
1179		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
1180		_LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
1181		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
	1188	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
	1189	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	1190	return const_reverse_iterator(end());
	1191	}
	1192	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
	1193	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	1194	return const_reverse_iterator(begin());
	1195	}
1182	1196
1183		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
1184		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
1185		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
1186		_LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	1197	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return begin(); }
	1198	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return end(); }
	1199	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return rbegin(); }
	1200	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
1187	1201
1188	1202	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __tree_.size() == 0; }
1189		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
1190		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
	1203	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __tree_.size(); }
	1204	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __tree_.max_size(); }
1191	1205
1192	1206	// modifiers:
1193	1207	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -1207,18 +1221,14 @@ public:
1207	1221	}
1208	1222
1209	1223	template <class _InputIterator>
1210		_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
1211		for (const_iterator __e = cend(); __f != __l; ++__f)
1212		__tree_.__emplace_hint_multi(__e, *__f);
	1224	_LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
	1225	__tree_.__insert_range_multi(__first, __last);
1213	1226	}
1214	1227
1215	1228	# if _LIBCPP_STD_VER >= 23
1216	1229	template <_ContainerCompatibleRange<value_type> _Range>
1217	1230	_LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
1218		const_iterator __end = cend();
1219		for (auto&& __element : __range) {
1220		__tree_.__emplace_hint_multi(__end, std::forward<decltype(__element)>(__element));
1221		}
	1231	__tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
1222	1232	}
1223	1233	# endif
1224	1234
...	...	@@ -1248,10 +1258,10 @@ public:
1248	1258	"node_type with incompatible allocator passed to multiset::insert()");
1249	1259	return __tree_.template __node_handle_insert_multi<node_type>(__hint, std::move(__nh));
1250	1260	}
1251		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
1252	1262	return __tree_.template __node_handle_extract<node_type>(__key);
1253	1263	}
1254		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	1264	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
1255	1265	return __tree_.template __node_handle_extract<node_type>(__it);
1256	1266	}
1257	1267	template <class _Compare2>
...	...	@@ -1284,101 +1294,118 @@ public:
1284	1294	__tree_.swap(__s.__tree_);
1285	1295	}
1286	1296
1287		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
1288		_LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
1289		_LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
	1297	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT { return __tree_.__alloc(); }
	1298	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __tree_.value_comp(); }
	1299	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __tree_.value_comp(); }
1290	1300
1291	1301	// set operations:
1292		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
1293		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
	1302	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __tree_.find(__k); }
	1303	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __tree_.find(__k); }
1294	1304	# if _LIBCPP_STD_VER >= 14
1295	1305	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1296		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1306	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
1297	1307	return __tree_.find(__k);
1298	1308	}
1299	1309	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1300		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1310	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
1301	1311	return __tree_.find(__k);
1302	1312	}
1303	1313	# endif
1304	1314
1305		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __tree_.__count_multi(__k); }
	1315	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1316	return __tree_.__count_multi(__k);
	1317	}
1306	1318	# if _LIBCPP_STD_VER >= 14
1307	1319	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1308		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1320	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
1309	1321	return __tree_.__count_multi(__k);
1310	1322	}
1311	1323	# endif
1312	1324
1313	1325	# if _LIBCPP_STD_VER >= 20
1314		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1326	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
1315	1327	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1316		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1328	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
1317	1329	return find(__k) != end();
1318	1330	}
1319	1331	# endif // _LIBCPP_STD_VER >= 20
1320	1332
1321		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) { return __tree_.lower_bound(__k); }
1322		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const { return __tree_.lower_bound(__k); }
	1333	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __k) {
	1334	return __tree_.__lower_bound_multi(__k);
	1335	}
	1336
	1337	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __k) const {
	1338	return __tree_.__lower_bound_multi(__k);
	1339	}
	1340
1323	1341	# if _LIBCPP_STD_VER >= 14
1324	1342	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1325		_LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
1326		return __tree_.lower_bound(__k);
	1343	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _K2& __k) {
	1344	return __tree_.__lower_bound_multi(__k);
1327	1345	}
1328	1346
1329	1347	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1330		_LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
1331		return __tree_.lower_bound(__k);
	1348	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _K2& __k) const {
	1349	return __tree_.__lower_bound_multi(__k);
1332	1350	}
1333	1351	# endif
1334	1352
1335		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) { return __tree_.upper_bound(__k); }
1336		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const { return __tree_.upper_bound(__k); }
	1353	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __k) {
	1354	return __tree_.__upper_bound_multi(__k);
	1355	}
	1356
	1357	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __k) const {
	1358	return __tree_.__upper_bound_multi(__k);
	1359	}
	1360
1337	1361	# if _LIBCPP_STD_VER >= 14
1338	1362	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1339		_LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
1340		return __tree_.upper_bound(__k);
	1363	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _K2& __k) {
	1364	return __tree_.__upper_bound_multi(__k);
1341	1365	}
1342	1366	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1343		_LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
1344		return __tree_.upper_bound(__k);
	1367	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _K2& __k) const {
	1368	return __tree_.__upper_bound_multi(__k);
1345	1369	}
1346	1370	# endif
1347	1371
1348		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1372	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
1349	1373	return __tree_.__equal_range_multi(__k);
1350	1374	}
1351		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1375	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
1352	1376	return __tree_.__equal_range_multi(__k);
1353	1377	}
1354	1378	# if _LIBCPP_STD_VER >= 14
1355	1379	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1356		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1380	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
1357	1381	return __tree_.__equal_range_multi(__k);
1358	1382	}
1359	1383	template <typename _K2, enable_if_t<__is_transparent_v<_Compare, _K2>, int> = 0>
1360		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1384	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
1361	1385	return __tree_.__equal_range_multi(__k);
1362	1386	}
1363	1387	# endif
	1388
	1389	template <class, class...>
	1390	friend struct __specialized_algorithm;
1364	1391	};
1365	1392
1366	1393	# if _LIBCPP_STD_VER >= 17
1367	1394	template <class _InputIterator,
1368		class _Compare = less<__iter_value_type<_InputIterator>>,
1369		class _Allocator = allocator<__iter_value_type<_InputIterator>>,
	1395	class _Compare = less<__iterator_value_type<_InputIterator>>,
	1396	class _Allocator = allocator<__iterator_value_type<_InputIterator>>,
1370	1397	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1371		class = enable_if_t<__is_allocator<_Allocator>::value, void>,
1372		class = enable_if_t<!__is_allocator<_Compare>::value, void>>
	1398	class = enable_if_t<__is_allocator_v<_Allocator>>,
	1399	class = enable_if_t<!__is_allocator_v<_Compare>>>
1373	1400	multiset(_InputIterator, _InputIterator, _Compare = _Compare(), _Allocator = _Allocator())
1374		-> multiset<__iter_value_type<_InputIterator>, _Compare, _Allocator>;
	1401	-> multiset<__iterator_value_type<_InputIterator>, _Compare, _Allocator>;
1375	1402
1376	1403	# if _LIBCPP_STD_VER >= 23
1377	1404	template <ranges::input_range _Range,
1378	1405	class _Compare = less<ranges::range_value_t<_Range>>,
1379	1406	class _Allocator = allocator<ranges::range_value_t<_Range>>,
1380		class = enable_if_t<__is_allocator<_Allocator>::value, void>,
1381		class = enable_if_t<!__is_allocator<_Compare>::value, void>>
	1407	class = enable_if_t<__is_allocator_v<_Allocator>>,
	1408	class = enable_if_t<!__is_allocator_v<_Compare>>>
1382	1409	multiset(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator())
1383	1410	-> multiset<ranges::range_value_t<_Range>, _Compare, _Allocator>;
1384	1411	# endif
...	...	@@ -1386,43 +1413,44 @@ multiset(from_range_t, _Range&&, _Compare = _Compare(), _Allocator = _Allocator(
1386	1413	template <class _Key,
1387	1414	class _Compare = less<_Key>,
1388	1415	class _Allocator = allocator<_Key>,
1389		class = enable_if_t<__is_allocator<_Allocator>::value, void>,
1390		class = enable_if_t<!__is_allocator<_Compare>::value, void>>
1391		multiset(initializer_list<_Key>,
1392		_Compare = _Compare(),
1393		_Allocator = _Allocator()) -> multiset<_Key, _Compare, _Allocator>;
	1416	class = enable_if_t<__is_allocator_v<_Allocator>>,
	1417	class = enable_if_t<!__is_allocator_v<_Compare>>>
	1418	multiset(initializer_list<_Key>, _Compare = _Compare(), _Allocator = _Allocator())
	1419	-> multiset<_Key, _Compare, _Allocator>;
1394	1420
1395	1421	template <class _InputIterator,
1396	1422	class _Allocator,
1397	1423	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value, void>,
1398		class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1424	class = enable_if_t<__is_allocator_v<_Allocator>>>
1399	1425	multiset(_InputIterator, _InputIterator, _Allocator)
1400		-> multiset<__iter_value_type<_InputIterator>, less<__iter_value_type<_InputIterator>>, _Allocator>;
	1426	-> multiset<__iterator_value_type<_InputIterator>, less<__iterator_value_type<_InputIterator>>, _Allocator>;
1401	1427
1402	1428	# if _LIBCPP_STD_VER >= 23
1403		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
1404		multiset(from_range_t,
1405		_Range&&,
1406		_Allocator) -> multiset<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
	1429	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
	1430	multiset(from_range_t, _Range&&, _Allocator)
	1431	-> multiset<ranges::range_value_t<_Range>, less<ranges::range_value_t<_Range>>, _Allocator>;
1407	1432	# endif
1408	1433
1409		template <class _Key, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value, void>>
	1434	template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1410	1435	multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
1411	1436	# endif
1412	1437
1413		# ifndef _LIBCPP_CXX03_LANG
	1438	# if _LIBCPP_STD_VER >= 14
	1439	template <class _Alg, class _Key, class _Compare, class _Allocator>
	1440	struct __specialized_algorithm<_Alg, __single_range<multiset<_Key, _Compare, _Allocator>>> {
	1441	using __set _LIBCPP_NODEBUG = multiset<_Key, _Compare, _Allocator>;
1414	1442
1415		template <class _Key, class _Compare, class _Allocator>
1416		multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_type& __a)
1417		: __tree_(std::move(__s.__tree_), __a) {
1418		if (__a != __s.get_allocator()) {
1419		const_iterator __e = cend();
1420		while (!__s.empty())
1421		insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
1422		}
1423		}
	1443	static const bool __has_algorithm =
	1444	__specialized_algorithm<_Alg, __single_range<typename __set::__base>>::__has_algorithm;
1424	1445
1425		# endif // _LIBCPP_CXX03_LANG
	1446	// set's begin() and end() are identical with and without const qualification
	1447	template <class... _Args>
	1448	_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
	1449	return __specialized_algorithm<_Alg, __single_range<typename __set::__base>>()(
	1450	__set.__tree_, std::forward<_Args>(__args)...);
	1451	}
	1452	};
	1453	# endif
1426	1454
1427	1455	template <class _Key, class _Compare, class _Allocator>
1428	1456	inline _LIBCPP_HIDE_FROM_ABI bool

lib/libcxx/include/shared_mutex+3-8

...	...	@@ -138,6 +138,7 @@ template <class Mutex>
138	138	# include <__mutex/tag_types.h>
139	139	# include <__mutex/unique_lock.h>
140	140	# include <__system_error/throw_system_error.h>
	141	# include <__utility/move.h>
141	142	# include <__utility/swap.h>
142	143	# include <cerrno>
143	144	# include <version>
...	...	@@ -340,14 +341,8 @@ public:
340	341	}
341	342
342	343	_LIBCPP_HIDE_FROM_ABI shared_lock& operator=(shared_lock&& __u) _NOEXCEPT {
343		if (__owns_)
344		__m_->unlock_shared();
345		__m_ = nullptr;
346		__owns_ = false;
347		__m_ = __u.__m_;
348		__owns_ = __u.__owns_;
349		__u.__m_ = nullptr;
350		__u.__owns_ = false;
	344	if (this != std::addressof(__u))
	345	shared_lock(std::move(__u)).swap(*this);
351	346	return *this;
352	347	}
353	348

lib/libcxx/include/span+53-37

...	...	@@ -310,30 +310,32 @@ public:
310	310	}
311	311
312	312	template <size_t _Count>
313		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
	313	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
314	314	static_assert(_Count <= _Extent, "span<T, N>::first<Count>(): Count out of range");
315	315	return span<element_type, _Count>{data(), _Count};
316	316	}
317	317
318	318	template <size_t _Count>
319		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
	319	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
320	320	static_assert(_Count <= _Extent, "span<T, N>::last<Count>(): Count out of range");
321	321	return span<element_type, _Count>{data() + size() - _Count, _Count};
322	322	}
323	323
324		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> first(size_type __count) const noexcept {
	324	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
	325	first(size_type __count) const noexcept {
325	326	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T, N>::first(count): count out of range");
326	327	return {data(), __count};
327	328	}
328	329
329		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> last(size_type __count) const noexcept {
	330	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
	331	last(size_type __count) const noexcept {
330	332	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T, N>::last(count): count out of range");
331	333	return {data() + size() - __count, __count};
332	334	}
333	335
334	336	template <size_t _Offset, size_t _Count = dynamic_extent>
335		_LIBCPP_HIDE_FROM_ABI constexpr auto
336		subspan() const noexcept -> span<element_type, _Count != dynamic_extent ? _Count : _Extent - _Offset> {
	337	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto subspan() const noexcept
	338	-> span<element_type, _Count != dynamic_extent ? _Count : _Extent - _Offset> {
337	339	static_assert(_Offset <= _Extent, "span<T, N>::subspan<Offset, Count>(): Offset out of range");
338	340	static_assert(_Count == dynamic_extent \|\| _Count <= _Extent - _Offset,
339	341	"span<T, N>::subspan<Offset, Count>(): Offset + Count out of range");
...	...	@@ -342,7 +344,7 @@ public:
342	344	return _ReturnType{data() + _Offset, _Count == dynamic_extent ? size() - _Offset : _Count};
343	345	}
344	346
345		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
	347	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
346	348	subspan(size_type __offset, size_type __count = dynamic_extent) const noexcept {
347	349	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__offset <= size(), "span<T, N>::subspan(offset, count): offset out of range");
348	350	if (__count == dynamic_extent)
...	...	@@ -352,52 +354,58 @@ public:
352	354	return {data() + __offset, __count};
353	355	}
354	356
355		_LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return _Extent; }
356		_LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept { return _Extent * sizeof(element_type); }
	357	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return _Extent; }
	358	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept {
	359	return _Extent * sizeof(element_type);
	360	}
357	361	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const noexcept { return _Extent == 0; }
358	362
359		_LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
	363	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
360	364	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < size(), "span<T, N>::operator[](index): index out of range");
361	365	return __data_[__idx];
362	366	}
363	367
364	368	# if _LIBCPP_STD_VER >= 26
365		_LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
	369	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
366	370	if (__index >= size())
367	371	std::__throw_out_of_range("span");
368	372	return __data_[__index];
369	373	}
370	374	# endif
371	375
372		_LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
	376	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
373	377	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T, N>::front() on empty span");
374	378	return __data_[0];
375	379	}
376	380
377		_LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
	381	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
378	382	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T, N>::back() on empty span");
379	383	return __data_[size() - 1];
380	384	}
381	385
382		_LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
	386	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
383	387
384	388	// [span.iter], span iterator support
385		_LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
	389	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
386	390	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
387	391	return std::__make_bounded_iter(data(), data(), data() + size());
388	392	# else
389	393	return iterator(data());
390	394	# endif
391	395	}
392		_LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
	396	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
393	397	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
394	398	return std::__make_bounded_iter(data() + size(), data(), data() + size());
395	399	# else
396	400	return iterator(data() + size());
397	401	# endif
398	402	}
399		_LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
400		_LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
	403	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept {
	404	return reverse_iterator(end());
	405	}
	406	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept {
	407	return reverse_iterator(begin());
	408	}
401	409
402	410	_LIBCPP_HIDE_FROM_ABI span<const byte, _Extent * sizeof(element_type)> __as_bytes() const noexcept {
403	411	return span<const byte, _Extent * sizeof(element_type)>{reinterpret_cast<const byte*>(data()), size_bytes()};
...	...	@@ -478,36 +486,38 @@ public:
478	486	: __data_{__other.data()}, __size_{__other.size()} {}
479	487
480	488	template <size_t _Count>
481		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
	489	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> first() const noexcept {
482	490	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count <= size(), "span<T>::first<Count>(): Count out of range");
483	491	return span<element_type, _Count>{data(), _Count};
484	492	}
485	493
486	494	template <size_t _Count>
487		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
	495	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> last() const noexcept {
488	496	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count <= size(), "span<T>::last<Count>(): Count out of range");
489	497	return span<element_type, _Count>{data() + size() - _Count, _Count};
490	498	}
491	499
492		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> first(size_type __count) const noexcept {
	500	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
	501	first(size_type __count) const noexcept {
493	502	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T>::first(count): count out of range");
494	503	return {data(), __count};
495	504	}
496	505
497		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent> last(size_type __count) const noexcept {
	506	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, dynamic_extent>
	507	last(size_type __count) const noexcept {
498	508	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__count <= size(), "span<T>::last(count): count out of range");
499	509	return {data() + size() - __count, __count};
500	510	}
501	511
502	512	template <size_t _Offset, size_t _Count = dynamic_extent>
503		_LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> subspan() const noexcept {
	513	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr span<element_type, _Count> subspan() const noexcept {
504	514	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Offset <= size(), "span<T>::subspan<Offset, Count>(): Offset out of range");
505	515	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(_Count == dynamic_extent \|\| _Count <= size() - _Offset,
506	516	"span<T>::subspan<Offset, Count>(): Offset + Count out of range");
507	517	return span<element_type, _Count>{data() + _Offset, _Count == dynamic_extent ? size() - _Offset : _Count};
508	518	}
509	519
510		constexpr span<element_type, dynamic_extent> _LIBCPP_HIDE_FROM_ABI
	520	[[nodiscard]] constexpr span<element_type, dynamic_extent> _LIBCPP_HIDE_FROM_ABI
511	521	subspan(size_type __offset, size_type __count = dynamic_extent) const noexcept {
512	522	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__offset <= size(), "span<T>::subspan(offset, count): offset out of range");
513	523	if (__count == dynamic_extent)
...	...	@@ -517,52 +527,58 @@ public:
517	527	return {data() + __offset, __count};
518	528	}
519	529
520		_LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return __size_; }
521		_LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept { return __size_ * sizeof(element_type); }
	530	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_type size() const noexcept { return __size_; }
	531	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_type size_bytes() const noexcept {
	532	return __size_ * sizeof(element_type);
	533	}
522	534	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool empty() const noexcept { return __size_ == 0; }
523	535
524		_LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
	536	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference operator[](size_type __idx) const noexcept {
525	537	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < size(), "span<T>::operator[](index): index out of range");
526	538	return __data_[__idx];
527	539	}
528	540
529	541	# if _LIBCPP_STD_VER >= 26
530		_LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
	542	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference at(size_type __index) const {
531	543	if (__index >= size())
532	544	std::__throw_out_of_range("span");
533	545	return __data_[__index];
534	546	}
535	547	# endif
536	548
537		_LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
	549	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference front() const noexcept {
538	550	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T>::front() on empty span");
539	551	return __data_[0];
540	552	}
541	553
542		_LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
	554	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reference back() const noexcept {
543	555	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "span<T>::back() on empty span");
544	556	return __data_[size() - 1];
545	557	}
546	558
547		_LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
	559	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr pointer data() const noexcept { return __data_; }
548	560
549	561	// [span.iter], span iterator support
550		_LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
	562	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() const noexcept {
551	563	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
552	564	return std::__make_bounded_iter(data(), data(), data() + size());
553	565	# else
554	566	return iterator(data());
555	567	# endif
556	568	}
557		_LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
	569	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr iterator end() const noexcept {
558	570	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
559	571	return std::__make_bounded_iter(data() + size(), data(), data() + size());
560	572	# else
561	573	return iterator(data() + size());
562	574	# endif
563	575	}
564		_LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept { return reverse_iterator(end()); }
565		_LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept { return reverse_iterator(begin()); }
	576	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rbegin() const noexcept {
	577	return reverse_iterator(end());
	578	}
	579	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr reverse_iterator rend() const noexcept {
	580	return reverse_iterator(begin());
	581	}
566	582
567	583	_LIBCPP_HIDE_FROM_ABI span<const byte, dynamic_extent> __as_bytes() const noexcept {
568	584	return {reinterpret_cast<const byte*>(data()), size_bytes()};
...	...	@@ -585,13 +601,13 @@ inline constexpr bool ranges::enable_view<span<_ElementType, _Extent>> = true;
585	601
586	602	// as_bytes & as_writable_bytes
587	603	template <class _Tp, size_t _Extent>
588		_LIBCPP_HIDE_FROM_ABI auto as_bytes(span<_Tp, _Extent> __s) noexcept {
	604	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto as_bytes(span<_Tp, _Extent> __s) noexcept {
589	605	return __s.__as_bytes();
590	606	}
591	607
592	608	template <class _Tp, size_t _Extent>
593	609	requires(!is_const_v<_Tp>)
594		_LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept {
	610	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept {
595	611	return __s.__as_writable_bytes();
596	612	}
597	613

lib/libcxx/include/sstream+34-28

...	...	@@ -461,15 +461,15 @@ public:
461	461	// [stringbuf.members] Member functions:
462	462
463	463	# if _LIBCPP_STD_VER >= 20
464		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
	464	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
465	465	# endif
466	466
467	467	# if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_BUILDING_LIBRARY)
468		string_type str() const;
	468	[[__nodiscard__]] string_type str() const;
469	469	# else
470		_LIBCPP_HIDE_FROM_ABI string_type str() const& { return str(__str_.get_allocator()); }
	470	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() const& { return str(__str_.get_allocator()); }
471	471
472		_LIBCPP_HIDE_FROM_ABI string_type str() && {
	472	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() && {
473	473	const basic_string_view<_CharT, _Traits> __view = view();
474	474	typename string_type::size_type __pos = __view.empty() ? 0 : __view.data() - __str_.data();
475	475	// In C++23, this is just string_type(std::move(__str_), __pos, __view.size(), __str_.get_allocator());
...	...	@@ -484,12 +484,12 @@ public:
484	484
485	485	# if _LIBCPP_STD_VER >= 20
486	486	template <class _SAlloc>
487		requires __is_allocator<_SAlloc>::value
488		_LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
	487	requires __is_allocator_v<_SAlloc>
	488	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
489	489	return basic_string<_CharT, _Traits, _SAlloc>(view(), __sa);
490	490	}
491	491
492		_LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept;
	492	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept;
493	493	# endif // _LIBCPP_STD_VER >= 20
494	494
495	495	void str(const string_type& __s) {
...	...	@@ -949,26 +949,28 @@ public:
949	949	}
950	950
951	951	// [istringstream.members] Member functions:
952		_LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
	952	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
953	953	return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
954	954	}
955	955
956	956	# if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_BUILDING_LIBRARY)
957		_LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
	957	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
958	958	# else
959		_LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
	959	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
960	960
961		_LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
	961	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
962	962	# endif
963	963
964	964	# if _LIBCPP_STD_VER >= 20
965	965	template <class _SAlloc>
966		requires __is_allocator<_SAlloc>::value
967		_LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
	966	requires __is_allocator_v<_SAlloc>
	967	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
968	968	return __sb_.str(__sa);
969	969	}
970	970
971		_LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
	971	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept {
	972	return __sb_.view();
	973	}
972	974	# endif // _LIBCPP_STD_VER >= 20
973	975
974	976	_LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
...	...	@@ -1087,26 +1089,28 @@ public:
1087	1089	}
1088	1090
1089	1091	// [ostringstream.members] Member functions:
1090		_LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
	1092	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
1091	1093	return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
1092	1094	}
1093	1095
1094	1096	# if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_BUILDING_LIBRARY)
1095		_LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
	1097	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
1096	1098	# else
1097		_LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
	1099	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
1098	1100
1099		_LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
	1101	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
1100	1102	# endif
1101	1103
1102	1104	# if _LIBCPP_STD_VER >= 20
1103	1105	template <class _SAlloc>
1104		requires __is_allocator<_SAlloc>::value
1105		_LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
	1106	requires __is_allocator_v<_SAlloc>
	1107	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
1106	1108	return __sb_.str(__sa);
1107	1109	}
1108	1110
1109		_LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
	1111	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept {
	1112	return __sb_.view();
	1113	}
1110	1114	# endif // _LIBCPP_STD_VER >= 20
1111	1115
1112	1116	_LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }
...	...	@@ -1227,26 +1231,28 @@ public:
1227	1231	}
1228	1232
1229	1233	// [stringstream.members] Member functions:
1230		_LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
	1234	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI basic_stringbuf<char_type, traits_type, allocator_type>* rdbuf() const {
1231	1235	return const_cast<basic_stringbuf<char_type, traits_type, allocator_type>*>(std::addressof(__sb_));
1232	1236	}
1233	1237
1234	1238	# if _LIBCPP_STD_VER <= 17 \|\| defined(_LIBCPP_BUILDING_LIBRARY)
1235		_LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
	1239	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI string_type str() const { return __sb_.str(); }
1236	1240	# else
1237		_LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
	1241	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() const& { return __sb_.str(); }
1238	1242
1239		_LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
	1243	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI string_type str() && { return std::move(__sb_).str(); }
1240	1244	# endif
1241	1245
1242	1246	# if _LIBCPP_STD_VER >= 20
1243	1247	template <class _SAlloc>
1244		requires __is_allocator<_SAlloc>::value
1245		_LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
	1248	requires __is_allocator_v<_SAlloc>
	1249	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string<char_type, traits_type, _SAlloc> str(const _SAlloc& __sa) const {
1246	1250	return __sb_.str(__sa);
1247	1251	}
1248	1252
1249		_LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept { return __sb_.view(); }
	1253	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI basic_string_view<char_type, traits_type> view() const noexcept {
	1254	return __sb_.view();
	1255	}
1250	1256	# endif // _LIBCPP_STD_VER >= 20
1251	1257
1252	1258	_LIBCPP_HIDE_FROM_ABI void str(const string_type& __s) { __sb_.str(__s); }

lib/libcxx/include/stack+13-15

...	...	@@ -235,9 +235,9 @@ public:
235	235	# endif
236	236
237	237	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const { return c.empty(); }
238		_LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
239		_LIBCPP_HIDE_FROM_ABI reference top() { return c.back(); }
240		_LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.back(); }
	238	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const { return c.size(); }
	239	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI reference top() { return c.back(); }
	240	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_reference top() const { return c.back(); }
241	241
242	242	_LIBCPP_HIDE_FROM_ABI void push(const value_type& __v) { c.push_back(__v); }
243	243	# ifndef _LIBCPP_CXX03_LANG
...	...	@@ -294,19 +294,19 @@ public:
294	294	};
295	295
296	296	# if _LIBCPP_STD_VER >= 17
297		template <class _Container, class = enable_if_t<!__is_allocator<_Container>::value> >
	297	template <class _Container, class = enable_if_t<!__is_allocator_v<_Container>>>
298	298	stack(_Container) -> stack<typename _Container::value_type, _Container>;
299	299
300	300	template <class _Container,
301	301	class _Alloc,
302		class = enable_if_t<!__is_allocator<_Container>::value>,
	302	class = enable_if_t<!__is_allocator_v<_Container>>,
303	303	class = enable_if_t<uses_allocator<_Container, _Alloc>::value> >
304	304	stack(_Container, _Alloc) -> stack<typename _Container::value_type, _Container>;
305	305	# endif
306	306
307	307	# if _LIBCPP_STD_VER >= 23
308	308	template <class _InputIterator, __enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0>
309		stack(_InputIterator, _InputIterator) -> stack<__iter_value_type<_InputIterator>>;
	309	stack(_InputIterator, _InputIterator) -> stack<__iterator_value_type<_InputIterator>>;
310	310
311	311	template <ranges::input_range _Range>
312	312	stack(from_range_t, _Range&&) -> stack<ranges::range_value_t<_Range>>;
...	...	@@ -314,15 +314,13 @@ stack(from_range_t, _Range&&) -> stack<ranges::range_value_t<_Range>>;
314	314	template <class _InputIterator,
315	315	class _Alloc,
316	316	__enable_if_t<__has_input_iterator_category<_InputIterator>::value, int> = 0,
317		__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
318		stack(_InputIterator,
319		_InputIterator,
320		_Alloc) -> stack<__iter_value_type<_InputIterator>, deque<__iter_value_type<_InputIterator>, _Alloc>>;
321
322		template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
323		stack(from_range_t,
324		_Range&&,
325		_Alloc) -> stack<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
	317	__enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
	318	stack(_InputIterator, _InputIterator, _Alloc)
	319	-> stack<__iterator_value_type<_InputIterator>, deque<__iterator_value_type<_InputIterator>, _Alloc>>;
	320
	321	template <ranges::input_range _Range, class _Alloc, __enable_if_t<__is_allocator_v<_Alloc>, int> = 0>
	322	stack(from_range_t, _Range&&, _Alloc)
	323	-> stack<ranges::range_value_t<_Range>, deque<ranges::range_value_t<_Range>, _Alloc>>;
326	324
327	325	# endif
328	326

lib/libcxx/include/stddef.h+10-10

...	...	@@ -25,24 +25,24 @@ Types:
25	25	*/
26	26
27	27	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
28		# include <__cxx03/stddef.h>
	28	# include <__cxx03/__config>
29	29	#else
30	30	# include <__config>
	31	#endif
31	32
32		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
33		# pragma GCC system_header
34		# endif
	33	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	34	# pragma GCC system_header
	35	#endif
35	36
36	37	// Note: This include is outside of header guards because we sometimes get included multiple times
37	38	// with different defines and the underlying <stddef.h> will know how to deal with that.
38		# include_next <stddef.h>
	39	#include_next <stddef.h>
39	40
40		# ifndef _LIBCPP_STDDEF_H
41		# define _LIBCPP_STDDEF_H
	41	#ifndef _LIBCPP_STDDEF_H
	42	# define _LIBCPP_STDDEF_H
42	43
43		# ifdef __cplusplus
	44	# ifdef __cplusplus
44	45	typedef decltype(nullptr) nullptr_t;
45		# endif
46		# endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	46	# endif
47	47
48	48	#endif // _LIBCPP_STDDEF_H

lib/libcxx/include/stdexcept+2-2

...	...	@@ -91,7 +91,7 @@ public:
91	91
92	92	~logic_error() _NOEXCEPT override;
93	93
94		const char* what() const _NOEXCEPT override;
	94	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
95	95	# else
96	96
97	97	public:
...	...	@@ -115,7 +115,7 @@ public:
115	115
116	116	~runtime_error() _NOEXCEPT override;
117	117
118		const char* what() const _NOEXCEPT override;
	118	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
119	119	# else
120	120
121	121	public:

lib/libcxx/include/stdio.h+20-21

...	...	@@ -88,35 +88,34 @@ void perror(const char* s);
88	88	*/
89	89
90	90	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
91		# include <__cxx03/stdio.h>
	91	# include <__cxx03/__config>
92	92	#else
93	93	# include <__config>
	94	#endif
94	95
95		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
96		# pragma GCC system_header
97		# endif
	96	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	97	# pragma GCC system_header
	98	#endif
98	99
99	100	// The inclusion of the system's <stdio.h> is intentionally done once outside of any include
100	101	// guards because some code expects to be able to include the underlying system header multiple
101	102	// times to get different definitions based on the macros that are set before inclusion.
102		# if __has_include_next(<stdio.h>)
103		# include_next <stdio.h>
104		# endif
	103	#if __has_include_next(<stdio.h>)
	104	# include_next <stdio.h>
	105	#endif
105	106
106		# ifndef _LIBCPP_STDIO_H
107		# define _LIBCPP_STDIO_H
	107	#ifndef _LIBCPP_STDIO_H
	108	# define _LIBCPP_STDIO_H
108	109
109		# ifdef __cplusplus
	110	# ifdef __cplusplus
110	111
111		# undef getc
112		# undef putc
113		# undef clearerr
114		# undef feof
115		# undef ferror
116		# undef putchar
117		# undef getchar
	112	# undef getc
	113	# undef putc
	114	# undef clearerr
	115	# undef feof
	116	# undef ferror
	117	# undef putchar
	118	# undef getchar
118	119
119		# endif // __cplusplus
120		# endif // _LIBCPP_STDIO_H
121
122		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	120	# endif // __cplusplus
	121	#endif // _LIBCPP_STDIO_H

lib/libcxx/include/streambuf+59-19

...	...	@@ -119,6 +119,7 @@ protected:
119	119	# include <__locale>
120	120	# include <__type_traits/is_same.h>
121	121	# include <__utility/is_valid_range.h>
	122	# include <__utility/scope_guard.h>
122	123	# include <climits>
123	124	# include <ios>
124	125	# include <iosfwd>
...	...	@@ -149,47 +150,56 @@ public:
149	150	virtual ~basic_streambuf() {}
150	151
151	152	// 27.6.2.2.1 locales:
152		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale pubimbue(const locale& __loc) {
	153	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 locale pubimbue(const locale& __loc) {
153	154	imbue(__loc);
154	155	locale __r = __loc_;
155	156	__loc_ = __loc;
156	157	return __r;
157	158	}
158	159
159		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 locale getloc() const { return __loc_; }
	160	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 locale getloc() const { return __loc_; }
160	161
161	162	// 27.6.2.2.2 buffer and positioning:
162		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 basic_streambuf* pubsetbuf(char_type* __s, streamsize __n) {
	163	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 basic_streambuf* pubsetbuf(char_type* __s, streamsize __n) {
163	164	return setbuf(__s, __n);
164	165	}
165	166
166		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type
	167	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 pos_type
167	168	pubseekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __which = ios_base::in \| ios_base::out) {
168	169	return seekoff(__off, __way, __which);
169	170	}
170	171
171		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 pos_type
	172	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 pos_type
172	173	pubseekpos(pos_type __sp, ios_base::openmode __which = ios_base::in \| ios_base::out) {
173	174	return seekpos(__sp, __which);
174	175	}
175	176
176		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int pubsync() { return sync(); }
	177	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int pubsync() { return sync(); }
177	178
178	179	// Get and put areas:
179	180	// 27.6.2.2.3 Get area:
180		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize in_avail() {
	181	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 streamsize in_avail() {
	182	__check_invariants();
	183	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	184
181	185	if (gptr() < egptr())
182	186	return static_cast<streamsize>(egptr() - gptr());
183	187	return showmanyc();
184	188	}
185	189
186		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type snextc() {
	190	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type snextc() {
	191	__check_invariants();
	192	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	193
187	194	if (sbumpc() == traits_type::eof())
188	195	return traits_type::eof();
189	196	return sgetc();
190	197	}
191	198
192		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sbumpc() {
	199	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type sbumpc() {
	200	__check_invariants();
	201	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	202
193	203	if (gptr() == egptr())
194	204	return uflow();
195	205	int_type __c = traits_type::to_int_type(*gptr());
...	...	@@ -197,23 +207,32 @@ public:
197	207	return __c;
198	208	}
199	209
200		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sgetc() {
	210	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type sgetc() {
	211	__check_invariants();
	212	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	213
201	214	if (gptr() == egptr())
202	215	return underflow();
203	216	return traits_type::to_int_type(*gptr());
204	217	}
205	218
206		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sgetn(char_type* __s, streamsize __n) { return xsgetn(__s, __n); }
	219	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 streamsize sgetn(char_type* __s, streamsize __n) { return xsgetn(__s, __n); }
207	220
208	221	// 27.6.2.2.4 Putback:
209		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputbackc(char_type __c) {
	222	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type sputbackc(char_type __c) {
	223	__check_invariants();
	224	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	225
210	226	if (eback() == gptr() \|\| !traits_type::eq(__c, *(gptr() - 1)))
211	227	return pbackfail(traits_type::to_int_type(__c));
212	228	this->gbump(-1);
213	229	return traits_type::to_int_type(*gptr());
214	230	}
215	231
216		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sungetc() {
	232	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type sungetc() {
	233	__check_invariants();
	234	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	235
217	236	if (eback() == gptr())
218	237	return pbackfail();
219	238	this->gbump(-1);
...	...	@@ -221,7 +240,10 @@ public:
221	240	}
222	241
223	242	// 27.6.2.2.5 Put area:
224		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputc(char_type __c) {
	243	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 int_type sputc(char_type __c) {
	244	__check_invariants();
	245	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	246
225	247	if (pptr() == epptr())
226	248	return overflow(traits_type::to_int_type(__c));
227	249	*pptr() = __c;
...	...	@@ -229,7 +251,7 @@ public:
229	251	return traits_type::to_int_type(__c);
230	252	}
231	253
232		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize sputn(const char_type* __s, streamsize __n) {
	254	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 streamsize sputn(const char_type* __s, streamsize __n) {
233	255	return xsputn(__s, __n);
234	256	}
235	257
...	...	@@ -270,12 +292,12 @@ protected:
270	292	_LIBCPP_HIDE_FROM_ABI char_type* gptr() const { return __ninp_; }
271	293	_LIBCPP_HIDE_FROM_ABI char_type* egptr() const { return __einp_; }
272	294
273		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void gbump(int __n) { __ninp_ += __n; }
	295	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void gbump(int __n) { __ninp_ += __n; }
274	296
275	297	// gbump takes an int, so it might not be able to represent the offset we want to add.
276	298	_LIBCPP_HIDE_FROM_ABI void __gbump_ptrdiff(ptrdiff_t __n) { __ninp_ += __n; }
277	299
278		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setg(char_type* __gbeg, char_type* __gnext, char_type* __gend) {
	300	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void setg(char_type* __gbeg, char_type* __gnext, char_type* __gend) {
279	301	_LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gbeg, __gnext), "[gbeg, gnext) must be a valid range");
280	302	_LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gbeg, __gend), "[gbeg, gend) must be a valid range");
281	303	_LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__gnext, __gend), "[gnext, gend) must be a valid range");
...	...	@@ -289,11 +311,11 @@ protected:
289	311	_LIBCPP_HIDE_FROM_ABI char_type* pptr() const { return __nout_; }
290	312	_LIBCPP_HIDE_FROM_ABI char_type* epptr() const { return __eout_; }
291	313
292		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void pbump(int __n) { __nout_ += __n; }
	314	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void pbump(int __n) { __nout_ += __n; }
293	315
294	316	_LIBCPP_HIDE_FROM_ABI void __pbump(streamsize __n) { __nout_ += __n; }
295	317
296		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 void setp(char_type* __pbeg, char_type* __pend) {
	318	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 void setp(char_type* __pbeg, char_type* __pend) {
297	319	_LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__pbeg, __pend), "[pbeg, pend) must be a valid range");
298	320	__bout_ = __nout_ = __pbeg;
299	321	__eout_ = __pend;
...	...	@@ -317,6 +339,9 @@ protected:
317	339	virtual streamsize showmanyc() { return 0; }
318	340
319	341	virtual streamsize xsgetn(char_type* __s, streamsize __n) {
	342	__check_invariants();
	343	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	344
320	345	int_type __c;
321	346	streamsize __i = 0;
322	347	while (__i < __n) {
...	...	@@ -338,6 +363,9 @@ protected:
338	363
339	364	virtual int_type underflow() { return traits_type::eof(); }
340	365	virtual int_type uflow() {
	366	__check_invariants();
	367	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	368
341	369	if (underflow() == traits_type::eof())
342	370	return traits_type::eof();
343	371	int_type __c = traits_type::to_int_type(*gptr());
...	...	@@ -350,6 +378,9 @@ protected:
350	378
351	379	// 27.6.2.4.5 Put area:
352	380	virtual streamsize xsputn(const char_type* __s, streamsize __n) {
	381	__check_invariants();
	382	auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
	383
353	384	streamsize __i = 0;
354	385	while (__i < __n) {
355	386	if (pptr() >= epptr()) {
...	...	@@ -370,6 +401,15 @@ protected:
370	401
371	402	virtual int_type overflow(int_type = traits_type::eof()) { return traits_type::eof(); }
372	403
	404	// This function checks some invariants of the class (it isn't exhaustive).
	405	_LIBCPP_HIDE_FROM_ABI void __check_invariants() const {
	406	_LIBCPP_ASSERT_INTERNAL(pbase() <= pptr(), "this is an invariant of the class");
	407	_LIBCPP_ASSERT_INTERNAL(pptr() <= epptr(), "this is an invariant of the class");
	408
	409	_LIBCPP_ASSERT_INTERNAL(eback() <= gptr(), "this is an invariant of the class");
	410	_LIBCPP_ASSERT_INTERNAL(gptr() <= egptr(), "this is an invariant of the class");
	411	}
	412
373	413	private:
374	414	locale __loc_;
375	415	char_type* __binp_ = nullptr;

lib/libcxx/include/string+574-573

...	...	@@ -280,6 +280,8 @@ public:
280	280	basic_string substr(size_type pos = 0, size_type n = npos) const; // constexpr in C++20, removed in C++23
281	281	basic_string substr(size_type pos = 0, size_type n = npos) const&; // since C++23
282	282	constexpr basic_string substr(size_type pos = 0, size_type n = npos) &&; // since C++23
	283	constexpr basic_string_view<charT, traits> subview(size_type pos = 0,
	284	size_type n = npos) const; // since C++26
283	285	void swap(basic_string& str)
284	286	noexcept(allocator_traits<allocator_type>::propagate_on_container_swap::value \|\|
285	287	allocator_traits<allocator_type>::is_always_equal::value); // C++17, constexpr since C++20
...	...	@@ -598,9 +600,9 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
598	600	# include <__debug_utils/sanitizers.h>
599	601	# include <__format/enable_insertable.h>
600	602	# include <__functional/hash.h>
	603	# include <__functional/is_transparent.h>
601	604	# include <__functional/unary_function.h>
602	605	# include <__fwd/string.h>
603		# include <__ios/fpos.h>
604	606	# include <__iterator/bounded_iter.h>
605	607	# include <__iterator/distance.h>
606	608	# include <__iterator/iterator_traits.h>
...	...	@@ -620,7 +622,6 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
620	622	# include <__ranges/concepts.h>
621	623	# include <__ranges/container_compatible_range.h>
622	624	# include <__ranges/from_range.h>
623		# include <__ranges/size.h>
624	625	# include <__string/char_traits.h>
625	626	# include <__string/extern_template_lists.h>
626	627	# include <__type_traits/conditional.h>
...	...	@@ -628,24 +629,23 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
628	629	# include <__type_traits/is_allocator.h>
629	630	# include <__type_traits/is_array.h>
630	631	# include <__type_traits/is_convertible.h>
	632	# include <__type_traits/is_generic_transparent_comparator.h>
631	633	# include <__type_traits/is_nothrow_assignable.h>
632	634	# include <__type_traits/is_nothrow_constructible.h>
633		# include <__type_traits/is_replaceable.h>
634	635	# include <__type_traits/is_same.h>
635	636	# include <__type_traits/is_standard_layout.h>
636	637	# include <__type_traits/is_trivially_constructible.h>
637	638	# include <__type_traits/is_trivially_copyable.h>
638	639	# include <__type_traits/is_trivially_relocatable.h>
639	640	# include <__type_traits/remove_cvref.h>
640		# include <__type_traits/void_t.h>
641		# include <__utility/auto_cast.h>
642		# include <__utility/declval.h>
	641	# include <__utility/default_three_way_comparator.h>
	642	# include <__utility/exception_guard.h>
643	643	# include <__utility/forward.h>
644	644	# include <__utility/is_pointer_in_range.h>
645	645	# include <__utility/move.h>
	646	# include <__utility/no_destroy.h>
646	647	# include <__utility/scope_guard.h>
647	648	# include <__utility/swap.h>
648		# include <__utility/unreachable.h>
649	649	# include <climits>
650	650	# include <cstdio> // EOF
651	651	# include <cstring>
...	...	@@ -700,18 +700,18 @@ __concatenate_strings(const _Allocator& __alloc,
700	700	__type_identity_t<basic_string_view<_CharT, _Traits> > __str2);
701	701
702	702	template <class _Iter>
703		struct __string_is_trivial_iterator : public false_type {};
	703	inline const bool __string_is_trivial_iterator_v = false;
704	704
705	705	template <class _Tp>
706		struct __string_is_trivial_iterator<_Tp*> : public is_arithmetic<_Tp> {};
	706	inline const bool __string_is_trivial_iterator_v<_Tp*> = is_arithmetic<_Tp>::value;
707	707
708	708	template <class _Iter>
709		struct __string_is_trivial_iterator<__wrap_iter<_Iter> > : public __string_is_trivial_iterator<_Iter> {};
	709	inline const bool __string_is_trivial_iterator_v<__wrap_iter<_Iter> > = __string_is_trivial_iterator_v<_Iter>;
710	710
711	711	template <class _CharT, class _Traits, class _Tp>
712		struct __can_be_converted_to_string_view
713		: public _BoolConstant< is_convertible<const _Tp&, basic_string_view<_CharT, _Traits> >::value &&
714		!is_convertible<const _Tp&, const _CharT*>::value > {};
	712	inline const bool __can_be_converted_to_string_view_v =
	713	is_convertible<const _Tp&, basic_string_view<_CharT, _Traits> >::value &&
	714	!is_convertible<const _Tp&, const _CharT*>::value;
715	715
716	716	struct __uninitialized_size_tag {};
717	717	struct __init_with_sentinel_tag {};
...	...	@@ -756,20 +756,13 @@ public:
756	756	// external memory. In such cases, the destructor is responsible for unpoisoning
757	757	// the memory to avoid triggering false positives.
758	758	// Therefore it's crucial to ensure the destructor is called.
759		//
760		// However, it is replaceable since implementing move-assignment as a destroy + move-construct
761		// will maintain the right ASAN state.
762		using __trivially_relocatable = void;
	759	using __trivially_relocatable _LIBCPP_NODEBUG = void;
763	760	# else
764	761	using __trivially_relocatable _LIBCPP_NODEBUG = __conditional_t<
765	762	__libcpp_is_trivially_relocatable<allocator_type>::value && __libcpp_is_trivially_relocatable<pointer>::value,
766	763	basic_string,
767	764	void>;
768	765	# endif
769		using __replaceable _LIBCPP_NODEBUG =
770		__conditional_t<__is_replaceable_v<pointer> && __container_allocator_is_replaceable<__alloc_traits>::value,
771		basic_string,
772		void>;
773	766
774	767	# if __has_feature(address_sanitizer) && _LIBCPP_INSTRUMENTED_WITH_ASAN
775	768	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __asan_volatile_wrapper(pointer const& __ptr) const {
...	...	@@ -819,12 +812,21 @@ public:
819	812	using reverse_iterator = std::reverse_iterator<iterator>;
820	813	using const_reverse_iterator = std::reverse_iterator<const_iterator>;
821	814
	815	using __alloc_result _LIBCPP_NODEBUG = __allocation_result<pointer, size_type>;
	816
822	817	private:
823	818	static_assert(CHAR_BIT == 8, "This implementation assumes that one byte contains 8 bits");
824	819
825	820	# ifdef _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
826	821
827	822	struct __long {
	823	__long() = default;
	824
	825	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __long(__alloc_result __alloc, size_type __size)
	826	: __data_(__alloc.ptr), __size_(__size), __cap_(__alloc.count / __endian_factor), __is_long_(true) {
	827	_LIBCPP_ASSERT_INTERNAL(!__fits_in_sso(__alloc.count), "Long capacity should always be larger than the SSO");
	828	}
	829
828	830	pointer __data_;
829	831	size_type __size_;
830	832	size_type __cap_ : sizeof(size_type) * CHAR_BIT - 1;
...	...	@@ -872,6 +874,13 @@ private:
872	874	// some platforms bit fields have a default size rather than the actual
873	875	// size used, e.g., it is 4 bytes on AIX. See D128285 for details.
874	876	struct __long {
	877	__long() = default;
	878
	879	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __long(__alloc_result __alloc, size_type __size)
	880	: __is_long_(true), __cap_(__alloc.count / __endian_factor), __size_(__size), __data_(__alloc.ptr) {
	881	_LIBCPP_ASSERT_INTERNAL(!__fits_in_sso(__alloc.count), "Long capacity should always be larger than the SSO");
	882	}
	883
875	884	struct _LIBCPP_PACKED {
876	885	size_type __is_long_ : 1;
877	886	size_type __cap_ : sizeof(size_type) * CHAR_BIT - 1;
...	...	@@ -898,6 +907,11 @@ private:
898	907	union __rep {
899	908	__short __s;
900	909	__long __l;
	910
	911	__rep() = default;
	912	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __rep(__short __r) : __s(__r) {}
	913	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __rep(__long __r) : __l(__r) {}
	914	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __rep(__uninitialized_tag) {}
901	915	};
902	916
903	917	_LIBCPP_COMPRESSED_PAIR(__rep, __rep_, allocator_type, __alloc_);
...	...	@@ -917,20 +931,7 @@ private:
917	931	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(
918	932	__uninitialized_size_tag, size_type __size, const allocator_type& __a)
919	933	: __alloc_(__a) {
920		if (__size > max_size())
921		this->__throw_length_error();
922		if (__fits_in_sso(__size)) {
923		__rep_ = __rep();
924		__set_short_size(__size);
925		} else {
926		auto __capacity = __recommend(__size) + 1;
927		auto __allocation = __alloc_traits::allocate(__alloc_, __capacity);
928		__begin_lifetime(__allocation, __capacity);
929		__set_long_cap(__capacity);
930		__set_long_pointer(__allocation);
931		__set_long_size(__size);
932		}
933		__annotate_new(__size);
	934	__init_internal_buffer(__size);
934	935	}
935	936
936	937	template <class _Iter, class _Sent>
...	...	@@ -966,7 +967,7 @@ private:
966	967	std::__wrap_iter<const_pointer>(__get_pointer() + size()));
967	968	# else
968	969	return const_iterator(__p);
969		# endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
	970	# endif // _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING
970	971	}
971	972
972	973	public:
...	...	@@ -1057,13 +1058,13 @@ public:
1057	1058	}
1058	1059	# endif // _LIBCPP_CXX03_LANG
1059	1060
1060		template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
	1061	template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0>
1061	1062	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s) {
1062	1063	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "basic_string(const char*) detected nullptr");
1063	1064	__init(__s, traits_type::length(__s));
1064	1065	}
1065	1066
1066		template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
	1067	template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0>
1067	1068	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
1068	1069	basic_string(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, const _Allocator& __a)
1069	1070	: __alloc_(__a) {
...	...	@@ -1075,13 +1076,15 @@ public:
1075	1076	basic_string(nullptr_t) = delete;
1076	1077	# endif
1077	1078
1078		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s, size_type __n) {
	1079	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const _CharT* __s, size_type __n)
	1080	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1079	1081	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "basic_string(const char*, n) detected nullptr");
1080	1082	__init(__s, __n);
1081	1083	}
1082	1084
1083	1085	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
1084	1086	basic_string(const _CharT* __s, size_type __n, const _Allocator& __a)
	1087	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero")
1085	1088	: __alloc_(__a) {
1086	1089	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "basic_string(const char*, n, allocator) detected nullptr");
1087	1090	__init(__s, __n);
...	...	@@ -1110,7 +1113,7 @@ public:
1110	1113	}
1111	1114	# endif
1112	1115
1113		template <__enable_if_t<__is_allocator<_Allocator>::value, int> = 0>
	1116	template <__enable_if_t<__is_allocator_v<_Allocator>, int> = 0>
1114	1117	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(size_type __n, _CharT __c, const _Allocator& __a)
1115	1118	: __alloc_(__a) {
1116	1119	__init(__n, __c);
...	...	@@ -1135,7 +1138,7 @@ public:
1135	1138	}
1136	1139
1137	1140	template <class _Tp,
1138		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1141	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1139	1142	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1140	1143	int> = 0>
1141	1144	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
...	...	@@ -1147,7 +1150,7 @@ public:
1147	1150	}
1148	1151
1149	1152	template <class _Tp,
1150		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1153	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1151	1154	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1152	1155	int> = 0>
1153	1156	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t) {
...	...	@@ -1156,7 +1159,7 @@ public:
1156	1159	}
1157	1160
1158	1161	template <class _Tp,
1159		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1162	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1160	1163	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1161	1164	int> = 0>
1162	1165	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t, const allocator_type& __a)
...	...	@@ -1201,11 +1204,10 @@ public:
1201	1204	}
1202	1205	# endif // _LIBCPP_CXX03_LANG
1203	1206
1204		inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() {
1205		__annotate_delete();
1206		if (__is_long())
1207		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
1208		}
	1207	// TODO(boomanaiden154): Once we mark this in destructors as dead on return,
	1208	// we can use a normal call to __reset_internal_buffer and remove the extra
	1209	// __rep constructor.
	1210	inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() { __reset_internal_buffer(__rep(__uninitialized_tag())); }
1209	1211
1210	1212	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 operator __self_view() const _NOEXCEPT {
1211	1213	return __self_view(typename __self_view::__assume_valid(), data(), size());
...	...	@@ -1215,7 +1217,7 @@ public:
1215	1217	operator=(const basic_string& __str);
1216	1218
1217	1219	template <class _Tp,
1218		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1220	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1219	1221	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1220	1222	int> = 0>
1221	1223	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const _Tp& __t) {
...	...	@@ -1243,45 +1245,55 @@ public:
1243	1245	# endif
1244	1246	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(value_type __c);
1245	1247
1246		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT {
	1248	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() _NOEXCEPT {
1247	1249	return __make_iterator(__get_pointer());
1248	1250	}
1249		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT {
	1251	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator begin() const _NOEXCEPT {
1250	1252	return __make_const_iterator(__get_pointer());
1251	1253	}
1252		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT {
	1254	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() _NOEXCEPT {
1253	1255	return __make_iterator(__get_pointer() + size());
1254	1256	}
1255		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
	1257	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator end() const _NOEXCEPT {
1256	1258	return __make_const_iterator(__get_pointer() + size());
1257	1259	}
1258	1260
1259		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
	1261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rbegin() _NOEXCEPT {
1260	1262	return reverse_iterator(end());
1261	1263	}
1262		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rbegin() const _NOEXCEPT {
	1264	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator
	1265	rbegin() const _NOEXCEPT {
1263	1266	return const_reverse_iterator(end());
1264	1267	}
1265		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
	1268	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reverse_iterator rend() _NOEXCEPT {
1266	1269	return reverse_iterator(begin());
1267	1270	}
1268		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
	1271	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator rend() const _NOEXCEPT {
1269	1272	return const_reverse_iterator(begin());
1270	1273	}
1271	1274
1272		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT { return begin(); }
1273		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT { return end(); }
1274		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crbegin() const _NOEXCEPT {
	1275	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cbegin() const _NOEXCEPT {
	1276	return begin();
	1277	}
	1278	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_iterator cend() const _NOEXCEPT {
	1279	return end();
	1280	}
	1281	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator
	1282	crbegin() const _NOEXCEPT {
1275	1283	return rbegin();
1276	1284	}
1277		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT { return rend(); }
	1285	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reverse_iterator crend() const _NOEXCEPT {
	1286	return rend();
	1287	}
1278	1288
1279		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT {
	1289	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type size() const _NOEXCEPT {
1280	1290	return __is_long() ? __get_long_size() : __get_short_size();
1281	1291	}
1282		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type length() const _NOEXCEPT { return size(); }
	1292	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type length() const _NOEXCEPT {
	1293	return size();
	1294	}
1283	1295
1284		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT {
	1296	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type max_size() const _NOEXCEPT {
1285	1297	if (size_type __m = __alloc_traits::max_size(__alloc_); __m <= std::numeric_limits<size_type>::max() / 2) {
1286	1298	size_type __res = __m - __alignment;
1287	1299
...	...	@@ -1299,7 +1311,7 @@ public:
1299	1311	}
1300	1312	}
1301	1313
1302		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
	1314	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type capacity() const _NOEXCEPT {
1303	1315	return (__is_long() ? __get_long_cap() : static_cast<size_type>(__min_cap)) - 1;
1304	1316	}
1305	1317
...	...	@@ -1311,13 +1323,19 @@ public:
1311	1323	# if _LIBCPP_STD_VER >= 23
1312	1324	template <class _Op>
1313	1325	_LIBCPP_HIDE_FROM_ABI constexpr void resize_and_overwrite(size_type __n, _Op __op) {
1314		__resize_default_init(__n);
1315		__erase_to_end(std::move(__op)(data(), _LIBCPP_AUTO_CAST(__n)));
	1326	using __result_type = decltype(std::move(__op)(data(), auto(__n)));
	1327	static_assert(__integer_like<__result_type>, "Operation return type must be integer-like");
	1328	size_type __sz = size();
	1329	size_type __cap = capacity();
	1330	if (__n > __cap)
	1331	__grow_by_without_replace(__cap, __n - __cap, __sz, __sz, 0);
	1332	__annotate_delete();
	1333	__set_size(__n);
	1334	__annotate_new(__n);
	1335	__erase_to_end(std::move(__op)(data(), auto(__n)));
1316	1336	}
1317	1337	# endif
1318	1338
1319		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __resize_default_init(size_type __n);
1320
1321	1339	# if _LIBCPP_STD_VER < 26 \|\| defined(_LIBCPP_ENABLE_CXX26_REMOVED_STRING_RESERVE)
1322	1340	_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void reserve() _NOEXCEPT { shrink_to_fit(); }
1323	1341	# endif
...	...	@@ -1328,7 +1346,8 @@ public:
1328	1346	return size() == 0;
1329	1347	}
1330	1348
1331		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference operator[](size_type __pos) const _NOEXCEPT {
	1349	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference
	1350	operator[](size_type __pos) const _NOEXCEPT {
1332	1351	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos <= size(), "string index out of bounds");
1333	1352	if (__builtin_constant_p(__pos) && !__fits_in_sso(__pos)) {
1334	1353	return *(__get_long_pointer() + __pos);
...	...	@@ -1336,7 +1355,8 @@ public:
1336	1355	return *(data() + __pos);
1337	1356	}
1338	1357
1339		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference operator[](size_type __pos) _NOEXCEPT {
	1358	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference
	1359	operator[](size_type __pos) _NOEXCEPT {
1340	1360	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos <= size(), "string index out of bounds");
1341	1361	if (__builtin_constant_p(__pos) && !__fits_in_sso(__pos)) {
1342	1362	return *(__get_long_pointer() + __pos);
...	...	@@ -1344,15 +1364,15 @@ public:
1344	1364	return *(__get_pointer() + __pos);
1345	1365	}
1346	1366
1347		_LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference at(size_type __n) const;
1348		_LIBCPP_CONSTEXPR_SINCE_CXX20 reference at(size_type __n);
	1367	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference at(size_type __n) const;
	1368	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 reference at(size_type __n);
1349	1369
1350	1370	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const basic_string& __str) {
1351	1371	return append(__str);
1352	1372	}
1353	1373
1354	1374	template <class _Tp,
1355		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1375	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1356	1376	!is_same<__remove_cvref_t<_Tp>, basic_string >::value,
1357	1377	int> = 0>
1358	1378	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const _Tp& __t) {
...	...	@@ -1381,7 +1401,7 @@ public:
1381	1401	}
1382	1402
1383	1403	template <class _Tp,
1384		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1404	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1385	1405	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1386	1406	int> = 0>
1387	1407	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const _Tp& __t) {
...	...	@@ -1392,7 +1412,7 @@ public:
1392	1412	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const basic_string& __str, size_type __pos, size_type __n = npos);
1393	1413
1394	1414	template <class _Tp,
1395		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1415	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1396	1416	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1397	1417	int> = 0>
1398	1418	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
...	...	@@ -1404,12 +1424,11 @@ public:
1404	1424	return append(__sv.data() + __pos, std::min(__n, __sz - __pos));
1405	1425	}
1406	1426
1407		_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const value_type* __s, size_type __n);
	1427	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const value_type* __s, size_type __n)
	1428	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero");
1408	1429	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s);
1409	1430	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(size_type __n, value_type __c);
1410	1431
1411		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __append_default_init(size_type __n);
1412
1413	1432	template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
1414	1433	_LIBCPP_HIDE_FROM_ABI _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1415	1434	append(_InputIterator __first, _InputIterator __last) {
...	...	@@ -1424,20 +1443,21 @@ public:
1424	1443	size_type __sz = size();
1425	1444	size_type __cap = capacity();
1426	1445	size_type __n = static_cast<size_type>(std::distance(__first, __last));
1427		if (__n) {
1428		if (__string_is_trivial_iterator<_ForwardIterator>::value && !__addr_in_range(*__first)) {
1429		if (__cap - __sz < __n)
1430		__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
1431		__annotate_increase(__n);
1432		auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
1433		traits_type::assign(*__end, value_type());
1434		__set_size(__sz + __n);
1435		} else {
1436		const basic_string __temp(__first, __last, __alloc_);
1437		append(__temp.data(), __temp.size());
1438		}
	1446	if (__n == 0)
	1447	return *this;
	1448
	1449	if (__string_is_trivial_iterator_v<_ForwardIterator> && !__addr_in_range(*__first)) {
	1450	if (__cap - __sz < __n)
	1451	__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
	1452	__annotate_increase(__n);
	1453	auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
	1454	traits_type::assign(*__end, value_type());
	1455	__set_size(__sz + __n);
	1456	return *this;
	1457	} else {
	1458	const basic_string __temp(__first, __last, __alloc_);
	1459	return append(__temp.data(), __temp.size());
1439	1460	}
1440		return *this;
1441	1461	}
1442	1462
1443	1463	# if _LIBCPP_STD_VER >= 23
...	...	@@ -1457,27 +1477,27 @@ public:
1457	1477	_LIBCPP_CONSTEXPR_SINCE_CXX20 void push_back(value_type __c);
1458	1478	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void pop_back();
1459	1479
1460		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() _NOEXCEPT {
	1480	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference front() _NOEXCEPT {
1461	1481	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::front(): string is empty");
1462	1482	return *__get_pointer();
1463	1483	}
1464	1484
1465		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const _NOEXCEPT {
	1485	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference front() const _NOEXCEPT {
1466	1486	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::front(): string is empty");
1467	1487	return *data();
1468	1488	}
1469	1489
1470		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() _NOEXCEPT {
	1490	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference back() _NOEXCEPT {
1471	1491	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::back(): string is empty");
1472	1492	return *(__get_pointer() + size() - 1);
1473	1493	}
1474	1494
1475		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const _NOEXCEPT {
	1495	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const_reference back() const _NOEXCEPT {
1476	1496	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string::back(): string is empty");
1477	1497	return *(data() + size() - 1);
1478	1498	}
1479	1499
1480		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
	1500	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
1481	1501	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const _Tp& __t) {
1482	1502	__self_view __sv = __t;
1483	1503	return assign(__sv.data(), __sv.size());
...	...	@@ -1519,7 +1539,7 @@ public:
1519	1539	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const basic_string& __str, size_type __pos, size_type __n = npos);
1520	1540
1521	1541	template <class _Tp,
1522		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1542	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1523	1543	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1524	1544	int> = 0>
1525	1545	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
...	...	@@ -1531,8 +1551,9 @@ public:
1531	1551	return assign(__sv.data() + __pos, std::min(__n, __sz - __pos));
1532	1552	}
1533	1553
1534		_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* __s, size_type __n);
1535		_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* __s);
	1554	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* __s, size_type __n)
	1555	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero");
	1556	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s);
1536	1557	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& assign(size_type __n, value_type __c);
1537	1558
1538	1559	template <class _InputIterator, __enable_if_t<__has_exactly_input_iterator_category<_InputIterator>::value, int> = 0>
...	...	@@ -1545,7 +1566,7 @@ public:
1545	1566	template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> = 0>
1546	1567	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1547	1568	assign(_ForwardIterator __first, _ForwardIterator __last) {
1548		if (__string_is_trivial_iterator<_ForwardIterator>::value) {
	1569	if (__string_is_trivial_iterator_v<_ForwardIterator>) {
1549	1570	size_type __n = static_cast<size_type>(std::distance(__first, __last));
1550	1571	__assign_trivial(__first, __last, __n);
1551	1572	} else {
...	...	@@ -1558,7 +1579,7 @@ public:
1558	1579	# if _LIBCPP_STD_VER >= 23
1559	1580	template <_ContainerCompatibleRange<_CharT> _Range>
1560	1581	_LIBCPP_HIDE_FROM_ABI constexpr basic_string& assign_range(_Range&& __range) {
1561		if constexpr (__string_is_trivial_iterator<ranges::iterator_t<_Range>>::value &&
	1582	if constexpr (__string_is_trivial_iterator_v<ranges::iterator_t<_Range>> &&
1562	1583	(ranges::forward_range<_Range> \|\| ranges::sized_range<_Range>)) {
1563	1584	size_type __n = static_cast<size_type>(ranges::distance(__range));
1564	1585	__assign_trivial(ranges::begin(__range), ranges::end(__range), __n);
...	...	@@ -1582,14 +1603,14 @@ public:
1582	1603	return insert(__pos1, __str.data(), __str.size());
1583	1604	}
1584	1605
1585		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
	1606	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
1586	1607	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos1, const _Tp& __t) {
1587	1608	__self_view __sv = __t;
1588	1609	return insert(__pos1, __sv.data(), __sv.size());
1589	1610	}
1590	1611
1591	1612	template <class _Tp,
1592		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1613	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1593	1614	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1594	1615	int> = 0>
1595	1616	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
...	...	@@ -1603,7 +1624,8 @@ public:
1603	1624
1604	1625	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1605	1626	insert(size_type __pos1, const basic_string& __str, size_type __pos2, size_type __n = npos);
1606		_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, const value_type* __s, size_type __n);
	1627	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, const value_type* __s, size_type __n)
	1628	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero");
1607	1629	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s);
1608	1630	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& insert(size_type __pos, size_type __n, value_type __c);
1609	1631	_LIBCPP_CONSTEXPR_SINCE_CXX20 iterator insert(const_iterator __pos, value_type __c);
...	...	@@ -1659,7 +1681,7 @@ public:
1659	1681	return replace(__pos1, __n1, __str.data(), __str.size());
1660	1682	}
1661	1683
1662		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
	1684	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
1663	1685	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1664	1686	replace(size_type __pos1, size_type __n1, const _Tp& __t) {
1665	1687	__self_view __sv = __t;
...	...	@@ -1670,7 +1692,7 @@ public:
1670	1692	replace(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos);
1671	1693
1672	1694	template <class _Tp,
1673		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	1695	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
1674	1696	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
1675	1697	int> = 0>
1676	1698	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
...	...	@@ -1683,8 +1705,10 @@ public:
1683	1705	}
1684	1706
1685	1707	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1686		replace(size_type __pos, size_type __n1, const value_type* __s, size_type __n2);
1687		_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& replace(size_type __pos, size_type __n1, const value_type* __s);
	1708	replace(size_type __pos, size_type __n1, const value_type* __s, size_type __n2)
	1709	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n2 != 0 && __s == nullptr, " if n2 is not zero");
	1710	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
	1711	replace(size_type __pos, size_type __n1, const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s);
1688	1712	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& replace(size_type __pos, size_type __n1, size_type __n2, value_type __c);
1689	1713
1690	1714	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
...	...	@@ -1693,7 +1717,7 @@ public:
1693	1717	static_cast<size_type>(__i1 - begin()), static_cast<size_type>(__i2 - __i1), __str.data(), __str.size());
1694	1718	}
1695	1719
1696		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
	1720	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
1697	1721	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
1698	1722	replace(const_iterator __i1, const_iterator __i2, const _Tp& __t) {
1699	1723	__self_view __sv = __t;
...	...	@@ -1741,19 +1765,24 @@ public:
1741	1765	_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type copy(value_type* __s, size_type __n, size_type __pos = 0) const;
1742	1766
1743	1767	# if _LIBCPP_STD_VER <= 20
1744		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string
1745		substr(size_type __pos = 0, size_type __n = npos) const {
	1768	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI
	1769	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string substr(size_type __pos = 0, size_type __n = npos) const {
1746	1770	return basic_string(*this, __pos, __n);
1747	1771	}
1748	1772	# else
1749		_LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) const& {
	1773	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) const& {
1750	1774	return basic_string(*this, __pos, __n);
1751	1775	}
1752	1776
1753		_LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) && {
	1777	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr basic_string substr(size_type __pos = 0, size_type __n = npos) && {
1754	1778	return basic_string(std::move(*this), __pos, __n);
1755	1779	}
1756	1780	# endif
	1781	# if _LIBCPP_STD_VER >= 26
	1782	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __self_view subview(size_type __pos = 0, size_type __n = npos) const {
	1783	return __self_view(*this).subview(__pos, __n);
	1784	}
	1785	# endif
1757	1786
1758	1787	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(basic_string& __str)
1759	1788	# if _LIBCPP_STD_VER >= 14
...	...	@@ -1765,225 +1794,238 @@ public:
1765	1794	// [string.ops]
1766	1795	// ------------
1767	1796
1768		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* c_str() const _NOEXCEPT { return data(); }
1769		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* data() const _NOEXCEPT {
	1797	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* c_str() const _NOEXCEPT {
	1798	return data();
	1799	}
	1800
	1801	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const value_type* data() const _NOEXCEPT {
1770	1802	return std::__to_address(__get_pointer());
1771	1803	}
1772	1804	# if _LIBCPP_STD_VER >= 17
1773		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 value_type* data() _NOEXCEPT {
	1805	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 value_type* data() _NOEXCEPT {
1774	1806	return std::__to_address(__get_pointer());
1775	1807	}
1776	1808	# endif
1777	1809
1778		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
	1810	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator_type get_allocator() const _NOEXCEPT {
1779	1811	return __alloc_;
1780	1812	}
1781	1813
1782	1814	// find
1783	1815
1784		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1816	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1785	1817	find(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
1786	1818	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __str.data(), __pos, __str.size());
1787	1819	}
1788	1820
1789		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1790		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1821	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1822	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1791	1823	find(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
1792	1824	__self_view __sv = __t;
1793	1825	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
1794	1826	}
1795	1827
1796		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	1828	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1829	find(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	1830	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1797	1831	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::find(): received nullptr");
1798	1832	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1799	1833	}
1800	1834
1801		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1835	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1802	1836	find(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
1803	1837	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find(): received nullptr");
1804	1838	return std::__str_find<value_type, size_type, traits_type, npos>(
1805	1839	data(), size(), __s, __pos, traits_type::length(__s));
1806	1840	}
1807	1841
1808		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(value_type __c, size_type __pos = 0) const _NOEXCEPT {
	1842	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type find(value_type __c, size_type __pos = 0) const _NOEXCEPT {
1809	1843	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
1810	1844	}
1811	1845
1812	1846	// rfind
1813	1847
1814		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1848	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1815	1849	rfind(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
1816	1850	return std::__str_rfind<value_type, size_type, traits_type, npos>(
1817	1851	data(), size(), __str.data(), __pos, __str.size());
1818	1852	}
1819	1853
1820		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1821		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1854	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1855	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1822	1856	rfind(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
1823	1857	__self_view __sv = __t;
1824	1858	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __sv.data(), __pos, __sv.size());
1825	1859	}
1826	1860
1827		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	1861	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1862	rfind(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	1863	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1828	1864	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::rfind(): received nullptr");
1829	1865	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1830	1866	}
1831	1867
1832		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1868	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1833	1869	rfind(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
1834	1870	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::rfind(): received nullptr");
1835	1871	return std::__str_rfind<value_type, size_type, traits_type, npos>(
1836	1872	data(), size(), __s, __pos, traits_type::length(__s));
1837	1873	}
1838	1874
1839		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type rfind(value_type __c, size_type __pos = npos) const _NOEXCEPT {
	1875	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1876	rfind(value_type __c, size_type __pos = npos) const _NOEXCEPT {
1840	1877	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
1841	1878	}
1842	1879
1843	1880	// find_first_of
1844	1881
1845		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1882	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1846	1883	find_first_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
1847	1884	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
1848	1885	data(), size(), __str.data(), __pos, __str.size());
1849	1886	}
1850	1887
1851		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1852		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1888	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1889	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1853	1890	find_first_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
1854	1891	__self_view __sv = __t;
1855	1892	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
1856	1893	data(), size(), __sv.data(), __pos, __sv.size());
1857	1894	}
1858	1895
1859		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1860		find_first_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	1896	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1897	find_first_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	1898	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1861	1899	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::find_first_of(): received nullptr");
1862	1900	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1863	1901	}
1864	1902
1865		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1903	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1866	1904	find_first_of(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
1867	1905	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_of(): received nullptr");
1868	1906	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
1869	1907	data(), size(), __s, __pos, traits_type::length(__s));
1870	1908	}
1871	1909
1872		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1910	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1873	1911	find_first_of(value_type __c, size_type __pos = 0) const _NOEXCEPT {
1874	1912	return find(__c, __pos);
1875	1913	}
1876	1914
1877	1915	// find_last_of
1878	1916
1879		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1917	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1880	1918	find_last_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
1881	1919	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
1882	1920	data(), size(), __str.data(), __pos, __str.size());
1883	1921	}
1884	1922
1885		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1886		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1923	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1924	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1887	1925	find_last_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
1888	1926	__self_view __sv = __t;
1889	1927	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
1890	1928	data(), size(), __sv.data(), __pos, __sv.size());
1891	1929	}
1892	1930
1893		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1894		find_last_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	1931	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1932	find_last_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	1933	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1895	1934	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::find_last_of(): received nullptr");
1896	1935	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1897	1936	}
1898	1937
1899		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1938	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1900	1939	find_last_of(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
1901	1940	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_of(): received nullptr");
1902	1941	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
1903	1942	data(), size(), __s, __pos, traits_type::length(__s));
1904	1943	}
1905	1944
1906		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1945	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1907	1946	find_last_of(value_type __c, size_type __pos = npos) const _NOEXCEPT {
1908	1947	return rfind(__c, __pos);
1909	1948	}
1910	1949
1911	1950	// find_first_not_of
1912	1951
1913		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1952	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1914	1953	find_first_not_of(const basic_string& __str, size_type __pos = 0) const _NOEXCEPT {
1915	1954	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
1916	1955	data(), size(), __str.data(), __pos, __str.size());
1917	1956	}
1918	1957
1919		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1920		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1958	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1959	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1921	1960	find_first_not_of(const _Tp& __t, size_type __pos = 0) const _NOEXCEPT {
1922	1961	__self_view __sv = __t;
1923	1962	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
1924	1963	data(), size(), __sv.data(), __pos, __sv.size());
1925	1964	}
1926	1965
1927		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1928		find_first_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	1966	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1967	find_first_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	1968	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1929	1969	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::find_first_not_of(): received nullptr");
1930	1970	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1931	1971	}
1932	1972
1933		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1973	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1934	1974	find_first_not_of(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
1935	1975	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_first_not_of(): received nullptr");
1936	1976	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
1937	1977	data(), size(), __s, __pos, traits_type::length(__s));
1938	1978	}
1939	1979
1940		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1980	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1941	1981	find_first_not_of(value_type __c, size_type __pos = 0) const _NOEXCEPT {
1942	1982	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
1943	1983	}
1944	1984
1945	1985	// find_last_not_of
1946	1986
1947		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1987	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1948	1988	find_last_not_of(const basic_string& __str, size_type __pos = npos) const _NOEXCEPT {
1949	1989	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
1950	1990	data(), size(), __str.data(), __pos, __str.size());
1951	1991	}
1952	1992
1953		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1954		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	1993	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	1994	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1955	1995	find_last_not_of(const _Tp& __t, size_type __pos = npos) const _NOEXCEPT {
1956	1996	__self_view __sv = __t;
1957	1997	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
1958	1998	data(), size(), __sv.data(), __pos, __sv.size());
1959	1999	}
1960	2000
1961		_LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1962		find_last_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	2001	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	2002	find_last_not_of(const value_type* __s, size_type __pos, size_type __n) const _NOEXCEPT
	2003	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
1963	2004	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::find_last_not_of(): received nullptr");
1964	2005	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
1965	2006	}
1966	2007
1967		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	2008	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1968	2009	find_last_not_of(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
1969	2010	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::find_last_not_of(): received nullptr");
1970	2011	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
1971	2012	data(), size(), __s, __pos, traits_type::length(__s));
1972	2013	}
1973	2014
1974		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	2015	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
1975	2016	find_last_not_of(value_type __c, size_type __pos = npos) const _NOEXCEPT {
1976	2017	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
1977	2018	}
1978	2019
1979	2020	// compare
1980	2021
1981		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const basic_string& __str) const _NOEXCEPT {
	2022	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2023	compare(const basic_string& __str) const _NOEXCEPT {
1982	2024	return compare(__self_view(__str));
1983	2025	}
1984	2026
1985		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
1986		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const _Tp& __t) const _NOEXCEPT {
	2027	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	2028	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const _Tp& __t) const _NOEXCEPT {
1987	2029	__self_view __sv = __t;
1988	2030	size_t __lhs_sz = size();
1989	2031	size_t __rhs_sz = __sv.size();
...	...	@@ -1997,73 +2039,77 @@ public:
1997	2039	return 0;
1998	2040	}
1999	2041
2000		template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value, int> = 0>
2001		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2042	template <class _Tp, __enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp>, int> = 0>
	2043	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
2002	2044	compare(size_type __pos1, size_type __n1, const _Tp& __t) const {
2003	2045	__self_view __sv = __t;
2004	2046	return compare(__pos1, __n1, __sv.data(), __sv.size());
2005	2047	}
2006	2048
2007		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2049	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
2008	2050	compare(size_type __pos1, size_type __n1, const basic_string& __str) const {
2009	2051	return compare(__pos1, __n1, __str.data(), __str.size());
2010	2052	}
2011	2053
2012		_LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2054	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 int
2013	2055	compare(size_type __pos1, size_type __n1, const basic_string& __str, size_type __pos2, size_type __n2 = npos) const {
2014	2056	return compare(__pos1, __n1, __self_view(__str), __pos2, __n2);
2015	2057	}
2016	2058
2017	2059	template <class _Tp,
2018		__enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
	2060	__enable_if_t<__can_be_converted_to_string_view_v<_CharT, _Traits, _Tp> &&
2019	2061	!is_same<__remove_cvref_t<_Tp>, basic_string>::value,
2020	2062	int> = 0>
2021		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2063	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
2022	2064	compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const {
2023	2065	__self_view __sv = __t;
2024	2066	return __self_view(*this).substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
2025	2067	}
2026	2068
2027		_LIBCPP_CONSTEXPR_SINCE_CXX20 int compare(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const _NOEXCEPT {
	2069	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2070	compare(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const _NOEXCEPT {
2028	2071	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
2029	2072	return compare(0, npos, __s, traits_type::length(__s));
2030	2073	}
2031	2074
2032		_LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2075	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 int
2033	2076	compare(size_type __pos1, size_type __n1, const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const {
2034	2077	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string::compare(): received nullptr");
2035	2078	return compare(__pos1, __n1, __s, traits_type::length(__s));
2036	2079	}
2037	2080
2038		_LIBCPP_CONSTEXPR_SINCE_CXX20 int
2039		compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const;
	2081	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 int
	2082	compare(size_type __pos1, size_type __n1, const value_type* __s, size_type __n2) const
	2083	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n2 != 0 && __s == nullptr, " if n2 is not zero");
2040	2084
2041	2085	// starts_with
2042	2086
2043	2087	# if _LIBCPP_STD_VER >= 20
2044		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(__self_view __sv) const noexcept {
	2088	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(__self_view __sv) const noexcept {
2045	2089	return __self_view(typename __self_view::__assume_valid(), data(), size()).starts_with(__sv);
2046	2090	}
2047	2091
2048		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
	2092	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
2049	2093	return !empty() && _Traits::eq(front(), __c);
2050	2094	}
2051	2095
2052		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
	2096	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool
	2097	starts_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
2053	2098	return starts_with(__self_view(__s));
2054	2099	}
2055	2100
2056	2101	// ends_with
2057	2102
2058		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(__self_view __sv) const noexcept {
	2103	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(__self_view __sv) const noexcept {
2059	2104	return __self_view(typename __self_view::__assume_valid(), data(), size()).ends_with(__sv);
2060	2105	}
2061	2106
2062		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
	2107	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
2063	2108	return !empty() && _Traits::eq(back(), __c);
2064	2109	}
2065	2110
2066		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
	2111	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool
	2112	ends_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
2067	2113	return ends_with(__self_view(__s));
2068	2114	}
2069	2115	# endif
...	...	@@ -2071,15 +2117,16 @@ public:
2071	2117	// contains
2072	2118
2073	2119	# if _LIBCPP_STD_VER >= 23
2074		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(__self_view __sv) const noexcept {
	2120	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool contains(__self_view __sv) const noexcept {
2075	2121	return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__sv);
2076	2122	}
2077	2123
2078		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept {
	2124	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept {
2079	2125	return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__c);
2080	2126	}
2081	2127
2082		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const {
	2128	[[__nodiscard__]] constexpr _LIBCPP_HIDE_FROM_ABI bool
	2129	contains(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const {
2083	2130	return __self_view(typename __self_view::__assume_valid(), data(), size()).contains(__s);
2084	2131	}
2085	2132	# endif
...	...	@@ -2095,18 +2142,6 @@ private:
2095	2142	return __rep_.__s.__is_long_;
2096	2143	}
2097	2144
2098		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __begin_lifetime(pointer __begin, size_type __n) {
2099		# if _LIBCPP_STD_VER >= 20
2100		if (__libcpp_is_constant_evaluated()) {
2101		for (size_type __i = 0; __i != __n; ++__i)
2102		std::construct_at(std::addressof(__begin[__i]));
2103		}
2104		# else
2105		(void)__begin;
2106		(void)__n;
2107		# endif // _LIBCPP_STD_VER >= 20
2108		}
2109
2110	2145	_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI static bool __fits_in_sso(size_type __sz) { return __sz < __min_cap; }
2111	2146
2112	2147	template <class _Iterator, class _Sentinel>
...	...	@@ -2165,6 +2200,9 @@ private:
2165	2200	_LIBCPP_CONSTEXPR_SINCE_CXX20 iterator
2166	2201	__insert_with_size(const_iterator __pos, _Iterator __first, _Sentinel __last, size_type __n);
2167	2202
	2203	// internal buffer accessors
	2204	// -------------------------
	2205
2168	2206	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS void
2169	2207	__set_short_size(size_type __s) _NOEXCEPT {
2170	2208	_LIBCPP_ASSERT_INTERNAL(__s < __min_cap, "__s should never be greater than or equal to the short string capacity");
...	...	@@ -2194,21 +2232,11 @@ private:
2194	2232	__set_short_size(__s);
2195	2233	}
2196	2234
2197		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_long_cap(size_type __s) _NOEXCEPT {
2198		_LIBCPP_ASSERT_INTERNAL(!__fits_in_sso(__s), "Long capacity should always be larger than the SSO");
2199		__rep_.__l.__cap_ = __s / __endian_factor;
2200		__rep_.__l.__is_long_ = true;
2201		}
2202
2203	2235	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __get_long_cap() const _NOEXCEPT {
2204	2236	_LIBCPP_ASSERT_INTERNAL(__rep_.__l.__is_long_, "String has to be long when trying to get the long capacity");
2205	2237	return __rep_.__l.__cap_ * __endian_factor;
2206	2238	}
2207	2239
2208		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __set_long_pointer(pointer __p) _NOEXCEPT {
2209		__rep_.__l.__data_ = __p;
2210		}
2211
2212	2240	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __get_long_pointer() _NOEXCEPT {
2213	2241	_LIBCPP_ASSERT_INTERNAL(__rep_.__l.__is_long_, "String has to be long when trying to get the long pointer");
2214	2242	return _LIBCPP_ASAN_VOLATILE_WRAPPER(__rep_.__l.__data_);
...	...	@@ -2236,6 +2264,58 @@ private:
2236	2264	return __is_long() ? __get_long_pointer() : __get_short_pointer();
2237	2265	}
2238	2266
	2267	// Internal buffer management
	2268	// --------------------------
	2269	//
	2270	// These functions are only responsible for managing the buffer itself, not the value inside the buffer. As such,
	2271	// none of these facilities ensure that there is a null terminator at `data()[size()]`.
	2272
	2273	// Allocate a buffer of __capacity size with __alloc and return it
	2274	_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 __long
	2275	__allocate_long_buffer(_Allocator& __alloc, size_type __capacity) {
	2276	_LIBCPP_ASSERT_INTERNAL(!__fits_in_sso(__capacity),
	2277	"Trying to allocate long buffer for a capacity what would fit into the small buffer");
	2278	auto __buffer = std::__allocate_at_least(__alloc, __align_allocation_size(__capacity));
	2279
	2280	if (__libcpp_is_constant_evaluated()) {
	2281	for (size_type __i = 0; __i != __buffer.count; ++__i)
	2282	std::__construct_at(std::addressof(__buffer.ptr[__i]));
	2283	}
	2284
	2285	return __long(__buffer, __capacity);
	2286	}
	2287
	2288	// Replace the current buffer with __new_rep. Deallocate the old long buffer if it exists.
	2289	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __reset_internal_buffer(__rep __new_rep = __short()) {
	2290	__annotate_delete();
	2291	if (__is_long())
	2292	__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
	2293	__rep_ = __new_rep;
	2294	}
	2295
	2296	// Initialize the internal buffer to hold __size elements
	2297	// The elements and null terminator have to be set by the caller
	2298	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pointer __init_internal_buffer(size_type __size) {
	2299	if (__libcpp_is_constant_evaluated())
	2300	__rep_ = __rep();
	2301
	2302	if (__size > max_size())
	2303	__throw_length_error();
	2304
	2305	if (__fits_in_sso(__size)) {
	2306	__set_short_size(__size);
	2307	__annotate_new(__size);
	2308	return __get_short_pointer();
	2309	} else {
	2310	__rep_.__l = __allocate_long_buffer(__alloc_, __size);
	2311	__annotate_new(__size);
	2312	return __get_long_pointer();
	2313	}
	2314	}
	2315
	2316	// ASan annotation helpers
	2317	// -----------------------
	2318
2239	2319	// The following functions are no-ops outside of AddressSanitizer mode.
2240	2320	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
2241	2321	__annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
...	...	@@ -2243,7 +2323,7 @@ private:
2243	2323	(void)__new_mid;
2244	2324	# if _LIBCPP_INSTRUMENTED_WITH_ASAN
2245	2325	# if defined(__APPLE__)
2246		// TODO: remove after addressing issue #96099 (https://github.com/llvm/llvm-project/issues/96099)
	2326	// TODO: remove after addressing issue #96099 (https://llvm.org/PR96099)
2247	2327	if (!__is_long())
2248	2328	return;
2249	2329	# endif
...	...	@@ -2287,19 +2367,36 @@ private:
2287	2367	return (__s + (__a - 1)) & ~(__a - 1);
2288	2368	}
2289	2369	enum { __alignment = 8 };
2290		static _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type __recommend(size_type __s) _NOEXCEPT {
2291		if (__s < __min_cap) {
2292		return static_cast<size_type>(__min_cap) - 1;
2293		}
	2370
	2371	// This makes sure that we're using a capacity with some extra alignment, since allocators almost always over-align
	2372	// the allocations anyways, improving memory usage. More importantly, this ensures that the lowest bit is never set
	2373	// if __endian_factor == 2, allowing us to store whether we're in the long string inside the lowest bit.
	2374	_LIBCPP_HIDE_FROM_ABI static _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	2375	__align_allocation_size(size_type __size) _NOEXCEPT {
	2376	_LIBCPP_ASSERT_INTERNAL(
	2377	!__fits_in_sso(__size), "Trying to align allocation of a size which would fit into the SSO");
2294	2378	const size_type __boundary = sizeof(value_type) < __alignment ? __alignment / sizeof(value_type) : __endian_factor;
2295		size_type __guess = __align_it<__boundary>(__s + 1) - 1;
2296		if (__guess == __min_cap)
	2379	size_type __guess = __align_it<__boundary>(__size + 1);
	2380	if (__guess == __min_cap + 1)
2297	2381	__guess += __endian_factor;
2298	2382
2299		_LIBCPP_ASSERT_INTERNAL(__guess >= __s, "recommendation is below the requested size");
	2383	_LIBCPP_ASSERT_INTERNAL(__guess >= __size, "aligned allocation size is below the requested size");
2300	2384	return __guess;
2301	2385	}
2302	2386
	2387	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 size_type
	2388	__get_amortized_growth_capacity(size_type __required_capacity) {
	2389	size_type __max_size = max_size();
	2390	if (__required_capacity > __max_size)
	2391	__throw_length_error();
	2392	size_type __current_cap = capacity();
	2393	_LIBCPP_ASSERT_INTERNAL(
	2394	__current_cap < __required_capacity, "Trying to grow string even though there is enough capacity already?");
	2395	if (__current_cap > __max_size / 2 - __alignment)
	2396	return __max_size;
	2397	return std::max(__required_capacity, 2 * __current_cap);
	2398	}
	2399
2303	2400	inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(const value_type* __s, size_type __sz);
2304	2401	inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void __init(size_type __n, value_type __c);
2305	2402
...	...	@@ -2366,7 +2463,8 @@ private:
2366	2463
2367	2464	// __erase_external_with_move is invoked for erase() invocations where
2368	2465	// `n ~= npos`, likely requiring memory moves on the string data.
2369		_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void __erase_external_with_move(size_type __pos, size_type __n);
	2466	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
	2467	__erase_external_with_move(size_type __pos, size_type __n) _NOEXCEPT;
2370	2468
2371	2469	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __copy_assign_alloc(const basic_string& __str) {
2372	2470	__copy_assign_alloc(
...	...	@@ -2378,24 +2476,14 @@ private:
2378	2476	__alloc_ = __str.__alloc_;
2379	2477	else {
2380	2478	if (!__str.__is_long()) {
2381		if (__is_long()) {
2382		__annotate_delete();
2383		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
2384		__rep_ = __rep();
2385		}
	2479	__reset_internal_buffer();
2386	2480	__alloc_ = __str.__alloc_;
2387	2481	} else {
2388	2482	__annotate_delete();
2389		auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
2390		allocator_type __a = __str.__alloc_;
2391		auto __allocation = std::__allocate_at_least(__a, __str.__get_long_cap());
2392		__begin_lifetime(__allocation.ptr, __allocation.count);
2393		if (__is_long())
2394		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
2395		__alloc_ = std::move(__a);
2396		__set_long_pointer(__allocation.ptr);
2397		__set_long_cap(__allocation.count);
2398		__set_long_size(__str.__get_long_size());
	2483	auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
	2484	auto __alloc = __str.__alloc_;
	2485	__reset_internal_buffer(__allocate_long_buffer(__alloc, __str.size()));
	2486	__alloc_ = std::move(__alloc);
2399	2487	}
2400	2488	}
2401	2489	}
...	...	@@ -2507,26 +2595,55 @@ _LIBCPP_STRING_V1_EXTERN_TEMPLATE_LIST(_LIBCPP_DECLARE, wchar_t)
2507	2595	# endif
2508	2596	# undef _LIBCPP_DECLARE
2509	2597
	2598	# if _LIBCPP_STD_VER <= 17 \|\| !__has_builtin(__builtin_lt_synthesizes_from_spaceship)
	2599	template <class _CharT, class _Traits, class _Alloc>
	2600	struct __default_three_way_comparator<basic_string<_CharT, _Traits, _Alloc>, basic_string<_CharT, _Traits, _Alloc> > {
	2601	using __string_t _LIBCPP_NODEBUG = basic_string<_CharT, _Traits, _Alloc>;
	2602
	2603	_LIBCPP_HIDE_FROM_ABI static int operator()(const __string_t& __lhs, const __string_t& __rhs) {
	2604	auto __min_len = std::min(__lhs.size(), __rhs.size());
	2605	auto __ret = _Traits::compare(__lhs.data(), __rhs.data(), __min_len);
	2606	if (__ret == 0)
	2607	return __lhs.size() == __rhs.size() ? 0 : __lhs.size() < __rhs.size() ? -1 : 1;
	2608	return __ret;
	2609	}
	2610	};
	2611	# endif
	2612
	2613	template <class _Comparator, class _CharT, class _Traits, class _Alloc>
	2614	inline const bool __is_transparently_comparable_v<_Comparator,
	2615	basic_string<_CharT, _Traits, _Alloc>,
	2616	const _CharT*,
	2617	__enable_if_t<__is_generic_transparent_comparator_v<_Comparator> > > =
	2618	true;
	2619
	2620	template <class _Comparator, class _CharT, class _Traits, class _Alloc, size_t _Np>
	2621	inline const bool __is_transparently_comparable_v<_Comparator,
	2622	basic_string<_CharT, _Traits, _Alloc>,
	2623	_CharT[_Np],
	2624	__enable_if_t<__is_generic_transparent_comparator_v<_Comparator> > > =
	2625	true;
	2626
2510	2627	# if _LIBCPP_STD_VER >= 17
2511	2628	template <class _InputIterator,
2512		class _CharT = __iter_value_type<_InputIterator>,
	2629	class _CharT = __iterator_value_type<_InputIterator>,
2513	2630	class _Allocator = allocator<_CharT>,
2514	2631	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
2515		class = enable_if_t<__is_allocator<_Allocator>::value> >
	2632	class = enable_if_t<__is_allocator_v<_Allocator>>>
2516	2633	basic_string(_InputIterator, _InputIterator, _Allocator = _Allocator())
2517	2634	-> basic_string<_CharT, char_traits<_CharT>, _Allocator>;
2518	2635
2519	2636	template <class _CharT,
2520	2637	class _Traits,
2521	2638	class _Allocator = allocator<_CharT>,
2522		class = enable_if_t<__is_allocator<_Allocator>::value> >
	2639	class = enable_if_t<__is_allocator_v<_Allocator>>>
2523	2640	explicit basic_string(basic_string_view<_CharT, _Traits>, const _Allocator& = _Allocator())
2524	2641	-> basic_string<_CharT, _Traits, _Allocator>;
2525	2642
2526	2643	template <class _CharT,
2527	2644	class _Traits,
2528	2645	class _Allocator = allocator<_CharT>,
2529		class = enable_if_t<__is_allocator<_Allocator>::value>,
	2646	class = enable_if_t<__is_allocator_v<_Allocator>>,
2530	2647	class _Sz = typename allocator_traits<_Allocator>::size_type >
2531	2648	basic_string(basic_string_view<_CharT, _Traits>, _Sz, _Sz, const _Allocator& = _Allocator())
2532	2649	-> basic_string<_CharT, _Traits, _Allocator>;
...	...	@@ -2535,7 +2652,7 @@ basic_string(basic_string_view<_CharT, _Traits>, _Sz, _Sz, const _Allocator& = _
2535	2652	# if _LIBCPP_STD_VER >= 23
2536	2653	template <ranges::input_range _Range,
2537	2654	class _Allocator = allocator<ranges::range_value_t<_Range>>,
2538		class = enable_if_t<__is_allocator<_Allocator>::value> >
	2655	class = enable_if_t<__is_allocator_v<_Allocator>>>
2539	2656	basic_string(from_range_t, _Range&&, _Allocator = _Allocator())
2540	2657	-> basic_string<ranges::range_value_t<_Range>, char_traits<ranges::range_value_t<_Range>>, _Allocator>;
2541	2658	# endif
...	...	@@ -2543,73 +2660,23 @@ basic_string(from_range_t, _Range&&, _Allocator = _Allocator())
2543	2660	template <class _CharT, class _Traits, class _Allocator>
2544	2661	_LIBCPP_CONSTEXPR_SINCE_CXX20 void
2545	2662	basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_type __sz) {
2546		if (__libcpp_is_constant_evaluated())
2547		__rep_ = __rep();
2548		if (__sz > max_size())
2549		this->__throw_length_error();
2550		pointer __p;
2551		if (__fits_in_sso(__sz)) {
2552		__set_short_size(__sz);
2553		__p = __get_short_pointer();
2554		} else {
2555		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
2556		__p = __allocation.ptr;
2557		__begin_lifetime(__p, __allocation.count);
2558		__set_long_pointer(__p);
2559		__set_long_cap(__allocation.count);
2560		__set_long_size(__sz);
2561		}
	2663	pointer __p = __init_internal_buffer(__sz);
2562	2664	traits_type::copy(std::__to_address(__p), __s, __sz);
2563	2665	traits_type::assign(__p[__sz], value_type());
2564		__annotate_new(__sz);
2565	2666	}
2566	2667
2567	2668	template <class _CharT, class _Traits, class _Allocator>
2568	2669	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
2569	2670	basic_string<_CharT, _Traits, _Allocator>::__init_copy_ctor_external(const value_type* __s, size_type __sz) {
2570		if (__libcpp_is_constant_evaluated())
2571		__rep_ = __rep();
2572
2573		pointer __p;
2574		if (__fits_in_sso(__sz)) {
2575		__p = __get_short_pointer();
2576		__set_short_size(__sz);
2577		} else {
2578		if (__sz > max_size())
2579		this->__throw_length_error();
2580		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
2581		__p = __allocation.ptr;
2582		__begin_lifetime(__p, __allocation.count);
2583		__set_long_pointer(__p);
2584		__set_long_cap(__allocation.count);
2585		__set_long_size(__sz);
2586		}
	2671	pointer __p = __init_internal_buffer(__sz);
2587	2672	traits_type::copy(std::__to_address(__p), __s, __sz + 1);
2588		__annotate_new(__sz);
2589	2673	}
2590	2674
2591	2675	template <class _CharT, class _Traits, class _Allocator>
2592	2676	_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__init(size_type __n, value_type __c) {
2593		if (__libcpp_is_constant_evaluated())
2594		__rep_ = __rep();
2595
2596		if (__n > max_size())
2597		this->__throw_length_error();
2598		pointer __p;
2599		if (__fits_in_sso(__n)) {
2600		__set_short_size(__n);
2601		__p = __get_short_pointer();
2602		} else {
2603		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__n) + 1);
2604		__p = __allocation.ptr;
2605		__begin_lifetime(__p, __allocation.count);
2606		__set_long_pointer(__p);
2607		__set_long_cap(__allocation.count);
2608		__set_long_size(__n);
2609		}
	2677	pointer __p = __init_internal_buffer(__n);
2610	2678	traits_type::assign(std::__to_address(__p), __n, __c);
2611	2679	traits_type::assign(__p[__n], value_type());
2612		__annotate_new(__n);
2613	2680	}
2614	2681
2615	2682	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -2626,19 +2693,10 @@ basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator _
2626	2693	__rep_ = __rep();
2627	2694	__annotate_new(0);
2628	2695
2629		# if _LIBCPP_HAS_EXCEPTIONS
2630		try {
2631		# endif // _LIBCPP_HAS_EXCEPTIONS
2632		for (; __first != __last; ++__first)
2633		push_back(*__first);
2634		# if _LIBCPP_HAS_EXCEPTIONS
2635		} catch (...) {
2636		__annotate_delete();
2637		if (__is_long())
2638		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
2639		throw;
2640		}
2641		# endif // _LIBCPP_HAS_EXCEPTIONS
	2696	auto __guard = std::__make_exception_guard([this] { __reset_internal_buffer(); });
	2697	for (; __first != __last; ++__first)
	2698	push_back(*__first);
	2699	__guard.__complete();
2642	2700	}
2643	2701
2644	2702	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -2653,39 +2711,12 @@ template <class _CharT, class _Traits, class _Allocator>
2653	2711	template <class _InputIterator, class _Sentinel>
2654	2712	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
2655	2713	basic_string<_CharT, _Traits, _Allocator>::__init_with_size(_InputIterator __first, _Sentinel __last, size_type __sz) {
2656		if (__libcpp_is_constant_evaluated())
2657		__rep_ = __rep();
	2714	pointer __p = __init_internal_buffer(__sz);
2658	2715
2659		if (__sz > max_size())
2660		this->__throw_length_error();
2661
2662		pointer __p;
2663		if (__fits_in_sso(__sz)) {
2664		__set_short_size(__sz);
2665		__p = __get_short_pointer();
2666
2667		} else {
2668		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__sz) + 1);
2669		__p = __allocation.ptr;
2670		__begin_lifetime(__p, __allocation.count);
2671		__set_long_pointer(__p);
2672		__set_long_cap(__allocation.count);
2673		__set_long_size(__sz);
2674		}
2675
2676		# if _LIBCPP_HAS_EXCEPTIONS
2677		try {
2678		# endif // _LIBCPP_HAS_EXCEPTIONS
2679		auto __end = __copy_non_overlapping_range(std::move(__first), std::move(__last), std::__to_address(__p));
2680		traits_type::assign(*__end, value_type());
2681		# if _LIBCPP_HAS_EXCEPTIONS
2682		} catch (...) {
2683		if (__is_long())
2684		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
2685		throw;
2686		}
2687		# endif // _LIBCPP_HAS_EXCEPTIONS
2688		__annotate_new(__sz);
	2716	auto __guard = std::__make_exception_guard([this] { __reset_internal_buffer(); });
	2717	auto __end = __copy_non_overlapping_range(std::move(__first), std::move(__last), std::__to_address(__p));
	2718	traits_type::assign(*__end, value_type());
	2719	__guard.__complete();
2689	2720	}
2690	2721
2691	2722	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -2697,32 +2728,22 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::__
2697	2728	size_type __n_del,
2698	2729	size_type __n_add,
2699	2730	const value_type* __p_new_stuff) {
2700		size_type __ms = max_size();
2701		if (__delta_cap > __ms - __old_cap)
2702		__throw_length_error();
	2731	__long __buffer = __allocate_long_buffer(__alloc_, __get_amortized_growth_capacity(__old_cap + __delta_cap));
2703	2732	pointer __old_p = __get_pointer();
2704		size_type __cap =
2705		__old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms;
2706	2733	__annotate_delete();
2707		auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
2708		auto __allocation = std::__allocate_at_least(__alloc_, __cap + 1);
2709		pointer __p = __allocation.ptr;
2710		__begin_lifetime(__p, __allocation.count);
	2734	auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
2711	2735	if (__n_copy != 0)
2712		traits_type::copy(std::__to_address(__p), std::__to_address(__old_p), __n_copy);
	2736	traits_type::copy(std::__to_address(__buffer.__data_), std::__to_address(__old_p), __n_copy);
2713	2737	if (__n_add != 0)
2714		traits_type::copy(std::__to_address(__p) + __n_copy, __p_new_stuff, __n_add);
	2738	traits_type::copy(std::__to_address(__buffer.__data_) + __n_copy, __p_new_stuff, __n_add);
2715	2739	size_type __sec_cp_sz = __old_sz - __n_del - __n_copy;
2716	2740	if (__sec_cp_sz != 0)
2717		traits_type::copy(
2718		std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
2719		if (__old_cap + 1 != __min_cap)
2720		__alloc_traits::deallocate(__alloc_, __old_p, __old_cap + 1);
2721		__set_long_pointer(__p);
2722		__set_long_cap(__allocation.count);
2723		__old_sz = __n_copy + __n_add + __sec_cp_sz;
2724		__set_long_size(__old_sz);
2725		traits_type::assign(__p[__old_sz], value_type());
	2741	traits_type::copy(std::__to_address(__buffer.__data_) + __n_copy + __n_add,
	2742	std::__to_address(__old_p) + __n_copy + __n_del,
	2743	__sec_cp_sz);
	2744	__buffer.__size_ = __n_copy + __n_add + __sec_cp_sz;
	2745	traits_type::assign(__buffer.__data_[__buffer.__size_], value_type());
	2746	__reset_internal_buffer(__buffer);
2726	2747	}
2727	2748
2728	2749	// __grow_by is deprecated because it does not set the size. It may not update the size when the size is changed, and it
...	...	@@ -2740,25 +2761,20 @@ _LIBCPP_DEPRECATED_("use __grow_by_without_replace") basic_string<_CharT, _Trait
2740	2761	size_type __n_copy,
2741	2762	size_type __n_del,
2742	2763	size_type __n_add) {
2743		size_type __ms = max_size();
2744		if (__delta_cap > __ms - __old_cap)
2745		this->__throw_length_error();
	2764	__long __buffer = __allocate_long_buffer(__alloc_, __get_amortized_growth_capacity(__old_cap + __delta_cap));
2746	2765	pointer __old_p = __get_pointer();
2747		size_type __cap =
2748		__old_cap < __ms / 2 - __alignment ? __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) : __ms;
2749		auto __allocation = std::__allocate_at_least(__alloc_, __cap + 1);
2750		pointer __p = __allocation.ptr;
2751		__begin_lifetime(__p, __allocation.count);
2752	2766	if (__n_copy != 0)
2753		traits_type::copy(std::__to_address(__p), std::__to_address(__old_p), __n_copy);
	2767	traits_type::copy(std::__to_address(__buffer.__data_), std::__to_address(__old_p), __n_copy);
2754	2768	size_type __sec_cp_sz = __old_sz - __n_del - __n_copy;
2755	2769	if (__sec_cp_sz != 0)
2756		traits_type::copy(
2757		std::__to_address(__p) + __n_copy + __n_add, std::__to_address(__old_p) + __n_copy + __n_del, __sec_cp_sz);
2758		if (__old_cap + 1 != __min_cap)
2759		__alloc_traits::deallocate(__alloc_, __old_p, __old_cap + 1);
2760		__set_long_pointer(__p);
2761		__set_long_cap(__allocation.count);
	2770	traits_type::copy(std::__to_address(__buffer.__data_) + __n_copy + __n_add,
	2771	std::__to_address(__old_p) + __n_copy + __n_del,
	2772	__sec_cp_sz);
	2773
	2774	// This is -1 to make sure the caller sets the size properly, since old versions of this function didn't set the size
	2775	// at all.
	2776	__buffer.__size_ = -1;
	2777	__reset_internal_buffer(__buffer);
2762	2778	}
2763	2779
2764	2780	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -2775,6 +2791,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_without_replace(
2775	2791	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
2776	2792	__grow_by(__old_cap, __delta_cap, __old_sz, __n_copy, __n_del, __n_add);
2777	2793	_LIBCPP_SUPPRESS_DEPRECATED_POP
	2794	// Due to the ABI of __grow_by we have to set the size after calling it.
2778	2795	__set_long_size(__old_sz - __n_del + __n_add);
2779	2796	}
2780	2797
...	...	@@ -2786,24 +2803,23 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string<_CharT, _Traits, _Al
2786	2803	basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias(const value_type* __s, size_type __n) {
2787	2804	const auto __cap = __is_short ? static_cast<size_type>(__min_cap) : __get_long_cap();
2788	2805	const auto __size = __is_short ? __get_short_size() : __get_long_size();
2789		if (__n < __cap) {
2790		if (__n > __size)
2791		__annotate_increase(__n - __size);
2792		pointer __p;
2793		if (__is_short) {
2794		__p = __get_short_pointer();
2795		__set_short_size(__n);
2796		} else {
2797		__p = __get_long_pointer();
2798		__set_long_size(__n);
2799		}
2800		traits_type::copy(std::__to_address(__p), __s, __n);
2801		traits_type::assign(__p[__n], value_type());
2802		if (__size > __n)
2803		__annotate_shrink(__size);
2804		} else {
	2806	if (__n >= __cap) {
2805	2807	__grow_by_and_replace(__cap - 1, __n - __cap + 1, __size, 0, __size, __n, __s);
	2808	return *this;
2806	2809	}
	2810
	2811	__annotate_delete();
	2812	auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
	2813	pointer __p;
	2814	if (__is_short) {
	2815	__p = __get_short_pointer();
	2816	__set_short_size(__n);
	2817	} else {
	2818	__p = __get_long_pointer();
	2819	__set_long_size(__n);
	2820	}
	2821	traits_type::copy(std::__to_address(__p), __s, __n);
	2822	traits_type::assign(__p[__n], value_type());
2807	2823	return *this;
2808	2824	}
2809	2825
...	...	@@ -2910,7 +2926,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr
2910	2926	{
2911	2927	__annotate_delete();
2912	2928	if (__is_long()) {
2913		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
	2929	__reset_internal_buffer();
2914	2930	# if _LIBCPP_STD_VER <= 14
2915	2931	if (!is_nothrow_move_assignable<allocator_type>::value) {
2916	2932	__set_short_size(0);
...	...	@@ -2961,7 +2977,7 @@ template <class _Iterator, class _Sentinel>
2961	2977	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
2962	2978	basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _Sentinel __last, size_type __n) {
2963	2979	_LIBCPP_ASSERT_INTERNAL(
2964		__string_is_trivial_iterator<_Iterator>::value, "The iterator type given to `__assign_trivial` must be trivial");
	2980	__string_is_trivial_iterator_v<_Iterator>, "The iterator type given to `__assign_trivial` must be trivial");
2965	2981
2966	2982	size_type __old_size = size();
2967	2983	size_type __cap = capacity();
...	...	@@ -3017,52 +3033,40 @@ basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_ty
3017	3033	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string::append received nullptr");
3018	3034	size_type __cap = capacity();
3019	3035	size_type __sz = size();
3020		if (__cap - __sz >= __n) {
3021		if (__n) {
3022		__annotate_increase(__n);
3023		value_type* __p = std::__to_address(__get_pointer());
3024		traits_type::copy(__p + __sz, __s, __n);
3025		__sz += __n;
3026		__set_size(__sz);
3027		traits_type::assign(__p[__sz], value_type());
3028		}
3029		} else
	3036	if (__cap - __sz < __n) {
3030	3037	__grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __sz, 0, __n, __s);
	3038	return *this;
	3039	}
	3040
	3041	if (__n == 0)
	3042	return *this;
	3043
	3044	__annotate_increase(__n);
	3045	value_type* __p = std::__to_address(__get_pointer());
	3046	traits_type::copy(__p + __sz, __s, __n);
	3047	__sz += __n;
	3048	__set_size(__sz);
	3049	traits_type::assign(__p[__sz], value_type());
3031	3050	return *this;
3032	3051	}
3033	3052
3034	3053	template <class _CharT, class _Traits, class _Allocator>
3035	3054	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
3036	3055	basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c) {
3037		if (__n) {
3038		size_type __cap = capacity();
3039		size_type __sz = size();
3040		if (__cap - __sz < __n)
3041		__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
3042		__annotate_increase(__n);
3043		pointer __p = __get_pointer();
3044		traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
3045		__sz += __n;
3046		__set_size(__sz);
3047		traits_type::assign(__p[__sz], value_type());
3048		}
3049		return *this;
3050		}
	3056	if (__n == 0)
	3057	return *this;
3051	3058
3052		template <class _CharT, class _Traits, class _Allocator>
3053		_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
3054		basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n) {
3055		if (__n) {
3056		size_type __cap = capacity();
3057		size_type __sz = size();
3058		if (__cap - __sz < __n)
3059		__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
3060		__annotate_increase(__n);
3061		pointer __p = __get_pointer();
3062		__sz += __n;
3063		__set_size(__sz);
3064		traits_type::assign(__p[__sz], value_type());
3065		}
	3059	size_type __cap = capacity();
	3060	size_type __sz = size();
	3061	if (__cap - __sz < __n)
	3062	__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
	3063	__annotate_increase(__n);
	3064	pointer __p = __get_pointer();
	3065	traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
	3066	__sz += __n;
	3067	__set_size(__sz);
	3068	traits_type::assign(__p[__sz], value_type());
	3069	return *this;
3066	3070	}
3067	3071
3068	3072	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -3120,23 +3124,27 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_t
3120	3124	if (__pos > __sz)
3121	3125	this->__throw_out_of_range();
3122	3126	size_type __cap = capacity();
3123		if (__cap - __sz >= __n) {
3124		if (__n) {
3125		__annotate_increase(__n);
3126		value_type* __p = std::__to_address(__get_pointer());
3127		size_type __n_move = __sz - __pos;
3128		if (__n_move != 0) {
3129		if (std::__is_pointer_in_range(__p + __pos, __p + __sz, __s))
3130		__s += __n;
3131		traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
3132		}
3133		traits_type::move(__p + __pos, __s, __n);
3134		__sz += __n;
3135		__set_size(__sz);
3136		traits_type::assign(__p[__sz], value_type());
3137		}
3138		} else
	3127
	3128	if (__cap - __sz < __n) {
3139	3129	__grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n, __s);
	3130	return *this;
	3131	}
	3132
	3133	if (__n == 0)
	3134	return *this;
	3135
	3136	__annotate_increase(__n);
	3137	value_type* __p = std::__to_address(__get_pointer());
	3138	size_type __n_move = __sz - __pos;
	3139	if (__n_move != 0) {
	3140	if (std::__is_pointer_in_range(__p + __pos, __p + __sz, __s))
	3141	__s += __n;
	3142	traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
	3143	}
	3144	traits_type::move(__p + __pos, __s, __n);
	3145	__sz += __n;
	3146	__set_size(__sz);
	3147	traits_type::assign(__p[__sz], value_type());
3140	3148	return *this;
3141	3149	}
3142	3150
...	...	@@ -3146,24 +3154,26 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, size_type __n
3146	3154	size_type __sz = size();
3147	3155	if (__pos > __sz)
3148	3156	this->__throw_out_of_range();
3149		if (__n) {
3150		size_type __cap = capacity();
3151		value_type* __p;
3152		if (__cap - __sz >= __n) {
3153		__annotate_increase(__n);
3154		__p = std::__to_address(__get_pointer());
3155		size_type __n_move = __sz - __pos;
3156		if (__n_move != 0)
3157		traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
3158		} else {
3159		__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n);
3160		__p = std::__to_address(__get_long_pointer());
3161		}
3162		traits_type::assign(__p + __pos, __n, __c);
3163		__sz += __n;
3164		__set_size(__sz);
3165		traits_type::assign(__p[__sz], value_type());
	3157
	3158	if (__n == 0)
	3159	return *this;
	3160
	3161	size_type __cap = capacity();
	3162	value_type* __p;
	3163	if (__cap - __sz >= __n) {
	3164	__annotate_increase(__n);
	3165	__p = std::__to_address(__get_pointer());
	3166	size_type __n_move = __sz - __pos;
	3167	if (__n_move != 0)
	3168	traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
	3169	} else {
	3170	__grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n);
	3171	__p = std::__to_address(__get_long_pointer());
3166	3172	}
	3173	traits_type::assign(__p + __pos, __n, __c);
	3174	__sz += __n;
	3175	__set_size(__sz);
	3176	traits_type::assign(__p[__sz], value_type());
3167	3177	return *this;
3168	3178	}
3169	3179
...	...	@@ -3176,7 +3186,7 @@ basic_string<_CharT, _Traits, _Allocator>::__insert_with_size(
3176	3186	if (__n == 0)
3177	3187	return begin() + __ip;
3178	3188
3179		if (__string_is_trivial_iterator<_Iterator>::value && !__addr_in_range(*__first)) {
	3189	if (__string_is_trivial_iterator_v<_Iterator> && !__addr_in_range(*__first)) {
3180	3190	return __insert_from_safe_copy(__n, __ip, std::move(__first), std::move(__last));
3181	3191	} else {
3182	3192	const basic_string __temp(__init_with_sentinel_tag(), std::move(__first), std::move(__last), __alloc_);
...	...	@@ -3237,38 +3247,38 @@ basic_string<_CharT, _Traits, _Allocator>::replace(
3237	3247	this->__throw_out_of_range();
3238	3248	__n1 = std::min(__n1, __sz - __pos);
3239	3249	size_type __cap = capacity();
3240		if (__cap - __sz + __n1 >= __n2) {
3241		value_type* __p = std::__to_address(__get_pointer());
3242		if (__n1 != __n2) {
3243		if (__n2 > __n1)
3244		__annotate_increase(__n2 - __n1);
3245		size_type __n_move = __sz - __pos - __n1;
3246		if (__n_move != 0) {
3247		if (__n1 > __n2) {
3248		traits_type::move(__p + __pos, __s, __n2);
3249		traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
3250		return __null_terminate_at(__p, __sz + (__n2 - __n1));
3251		}
3252		if (std::__is_pointer_in_range(__p + __pos + 1, __p + __sz, __s)) {
3253		if (__p + __pos + __n1 <= __s)
3254		__s += __n2 - __n1;
3255		else // __p + __pos < __s < __p + __pos + __n1
3256		{
3257		traits_type::move(__p + __pos, __s, __n1);
3258		__pos += __n1;
3259		__s += __n2;
3260		__n2 -= __n1;
3261		__n1 = 0;
3262		}
3263		}
	3250	if (__cap - __sz + __n1 < __n2) {
	3251	__grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
	3252	return *this;
	3253	}
	3254
	3255	value_type* __p = std::__to_address(__get_pointer());
	3256	if (__n1 != __n2) {
	3257	if (__n2 > __n1)
	3258	__annotate_increase(__n2 - __n1);
	3259	size_type __n_move = __sz - __pos - __n1;
	3260	if (__n_move != 0) {
	3261	if (__n1 > __n2) {
	3262	traits_type::move(__p + __pos, __s, __n2);
3264	3263	traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
	3264	return __null_terminate_at(__p, __sz + (__n2 - __n1));
	3265	}
	3266	if (std::__is_pointer_in_range(__p + __pos + 1, __p + __sz, __s)) {
	3267	if (__p + __pos + __n1 <= __s) {
	3268	__s += __n2 - __n1;
	3269	} else { // __p + __pos < __s < __p + __pos + __n1
	3270	traits_type::move(__p + __pos, __s, __n1);
	3271	__pos += __n1;
	3272	__s += __n2;
	3273	__n2 -= __n1;
	3274	__n1 = 0;
	3275	}
3265	3276	}
	3277	traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
3266	3278	}
3267		traits_type::move(__p + __pos, __s, __n2);
3268		return __null_terminate_at(__p, __sz + (__n2 - __n1));
3269		} else
3270		__grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
3271		return *this;
	3279	}
	3280	traits_type::move(__p + __pos, __s, __n2);
	3281	return __null_terminate_at(__p, __sz + (__n2 - __n1));
3272	3282	}
3273	3283
3274	3284	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -3320,16 +3330,17 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
3320	3330	// Does not check __pos against size()
3321	3331	template <class _CharT, class _Traits, class _Allocator>
3322	3332	_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
3323		basic_string<_CharT, _Traits, _Allocator>::__erase_external_with_move(size_type __pos, size_type __n) {
3324		if (__n) {
3325		size_type __sz = size();
3326		value_type* __p = std::__to_address(__get_pointer());
3327		__n = std::min(__n, __sz - __pos);
3328		size_type __n_move = __sz - __pos - __n;
3329		if (__n_move != 0)
3330		traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
3331		__null_terminate_at(__p, __sz - __n);
3332		}
	3333	basic_string<_CharT, _Traits, _Allocator>::__erase_external_with_move(size_type __pos, size_type __n) _NOEXCEPT {
	3334	if (__n == 0)
	3335	return;
	3336
	3337	size_type __sz = size();
	3338	value_type* __p = std::__to_address(__get_pointer());
	3339	__n = std::min(__n, __sz - __pos);
	3340	size_type __n_move = __sz - __pos - __n;
	3341	if (__n_move != 0)
	3342	traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
	3343	__null_terminate_at(__p, __sz - __n);
3333	3344	}
3334	3345
3335	3346	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -3396,16 +3407,6 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::re
3396	3407	__erase_to_end(__n);
3397	3408	}
3398	3409
3399		template <class _CharT, class _Traits, class _Allocator>
3400		_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
3401		basic_string<_CharT, _Traits, _Allocator>::__resize_default_init(size_type __n) {
3402		size_type __sz = size();
3403		if (__n > __sz) {
3404		__append_default_init(__n - __sz);
3405		} else
3406		__erase_to_end(__n);
3407		}
3408
3409	3410	template <class _CharT, class _Traits, class _Allocator>
3410	3411	_LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::reserve(size_type __requested_capacity) {
3411	3412	if (__requested_capacity > max_size())
...	...	@@ -3418,31 +3419,23 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::re
3418	3419	return;
3419	3420
3420	3421	__annotation_guard __g(*this);
3421		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__requested_capacity) + 1);
3422		auto __size = size();
3423		__begin_lifetime(__allocation.ptr, __allocation.count);
3424		traits_type::copy(std::__to_address(__allocation.ptr), data(), __size + 1);
3425		if (__is_long())
3426		__alloc_traits::deallocate(__alloc_, __get_long_pointer(), __get_long_cap());
3427		__set_long_cap(__allocation.count);
3428		__set_long_size(__size);
3429		__set_long_pointer(__allocation.ptr);
	3422	__long __buffer = __allocate_long_buffer(__alloc_, __requested_capacity);
	3423	__buffer.__size_ = size();
	3424	traits_type::copy(std::__to_address(__buffer.__data_), data(), __buffer.__size_ + 1);
	3425	__reset_internal_buffer(__buffer);
3430	3426	}
3431	3427
3432	3428	template <class _CharT, class _Traits, class _Allocator>
3433	3429	inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocator>::shrink_to_fit() _NOEXCEPT {
3434		size_type __target_capacity = __recommend(size());
3435		if (__target_capacity == capacity())
	3430	if (!__is_long())
3436	3431	return;
3437	3432
3438		_LIBCPP_ASSERT_INTERNAL(__is_long(), "Trying to shrink small string");
3439
3440		// We're a long string and we're shrinking into the small buffer.
3441	3433	const auto __ptr = __get_long_pointer();
3442	3434	const auto __size = __get_long_size();
3443	3435	const auto __cap = __get_long_cap();
3444	3436
3445		if (__fits_in_sso(__target_capacity)) {
	3437	// We're a long string and we're shrinking into the small buffer.
	3438	if (__fits_in_sso(__size)) {
3446	3439	__annotation_guard __g(*this);
3447	3440	__set_short_size(__size);
3448	3441	traits_type::copy(std::__to_address(__get_short_pointer()), std::__to_address(__ptr), __size + 1);
...	...	@@ -3450,25 +3443,25 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20 void basic_string<_CharT, _Traits, _Allocat
3450	3443	return;
3451	3444	}
3452	3445
	3446	if (__align_allocation_size(__size) == __cap)
	3447	return;
	3448
3453	3449	# if _LIBCPP_HAS_EXCEPTIONS
3454	3450	try {
3455	3451	# endif // _LIBCPP_HAS_EXCEPTIONS
3456	3452	__annotation_guard __g(*this);
3457		auto __allocation = std::__allocate_at_least(__alloc_, __target_capacity + 1);
	3453	__long __buffer = __allocate_long_buffer(__alloc_, __size);
3458	3454
3459	3455	// The Standard mandates shrink_to_fit() does not increase the capacity.
3460	3456	// With equal capacity keep the existing buffer. This avoids extra work
3461	3457	// due to swapping the elements.
3462		if (__allocation.count - 1 >= capacity()) {
3463		__alloc_traits::deallocate(__alloc_, __allocation.ptr, __allocation.count);
	3458	if (__buffer.__cap_ * __endian_factor - 1 >= capacity()) {
	3459	__alloc_traits::deallocate(__alloc_, __buffer.__data_, __buffer.__cap_ * __endian_factor);
3464	3460	return;
3465	3461	}
3466	3462
3467		__begin_lifetime(__allocation.ptr, __allocation.count);
3468		traits_type::copy(std::__to_address(__allocation.ptr), std::__to_address(__ptr), __size + 1);
3469		__alloc_traits::deallocate(__alloc_, __ptr, __cap);
3470		__set_long_cap(__allocation.count);
3471		__set_long_pointer(__allocation.ptr);
	3463	traits_type::copy(std::__to_address(__buffer.__data_), std::__to_address(__get_long_pointer()), __size + 1);
	3464	__reset_internal_buffer(__buffer);
3472	3465	# if _LIBCPP_HAS_EXCEPTIONS
3473	3466	} catch (...) {
3474	3467	return;
...	...	@@ -3574,7 +3567,8 @@ operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
3574	3567
3575	3568	template <class _CharT, class _Traits, class _Allocator>
3576	3569	inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool
3577		operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs, const _CharT* __rhs) _NOEXCEPT {
	3570	operator==(const basic_string<_CharT, _Traits, _Allocator>& __lhs,
	3571	const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __rhs) _NOEXCEPT {
3578	3572	_LIBCPP_ASSERT_NON_NULL(__rhs != nullptr, "operator==(basic_string, char*): received nullptr");
3579	3573
3580	3574	using _String = basic_string<_CharT, _Traits, _Allocator>;
...	...	@@ -3858,46 +3852,52 @@ swap(basic_string<_CharT, _Traits, _Allocator>& __lhs, basic_string<_CharT, _Tra
3858	3852	__lhs.swap(__rhs);
3859	3853	}
3860	3854
3861		_LIBCPP_EXPORTED_FROM_ABI int stoi(const string& __str, size_t* __idx = nullptr, int __base = 10);
3862		_LIBCPP_EXPORTED_FROM_ABI long stol(const string& __str, size_t* __idx = nullptr, int __base = 10);
3863		_LIBCPP_EXPORTED_FROM_ABI unsigned long stoul(const string& __str, size_t* __idx = nullptr, int __base = 10);
3864		_LIBCPP_EXPORTED_FROM_ABI long long stoll(const string& __str, size_t* __idx = nullptr, int __base = 10);
3865		_LIBCPP_EXPORTED_FROM_ABI unsigned long long stoull(const string& __str, size_t* __idx = nullptr, int __base = 10);
3866
3867		_LIBCPP_EXPORTED_FROM_ABI float stof(const string& __str, size_t* __idx = nullptr);
3868		_LIBCPP_EXPORTED_FROM_ABI double stod(const string& __str, size_t* __idx = nullptr);
3869		_LIBCPP_EXPORTED_FROM_ABI long double stold(const string& __str, size_t* __idx = nullptr);
3870
3871		_LIBCPP_EXPORTED_FROM_ABI string to_string(int __val);
3872		_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned __val);
3873		_LIBCPP_EXPORTED_FROM_ABI string to_string(long __val);
3874		_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long __val);
3875		_LIBCPP_EXPORTED_FROM_ABI string to_string(long long __val);
3876		_LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long long __val);
3877		_LIBCPP_EXPORTED_FROM_ABI string to_string(float __val);
3878		_LIBCPP_EXPORTED_FROM_ABI string to_string(double __val);
3879		_LIBCPP_EXPORTED_FROM_ABI string to_string(long double __val);
	3855	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI int stoi(const string& __str, size_t* __idx = nullptr, int __base = 10);
	3856	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI unsigned long
	3857	stoul(const string& __str, size_t* __idx = nullptr, int __base = 10);
	3858	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long stol(const string& __str, size_t* __idx = nullptr, int __base = 10);
	3859	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long long
	3860	stoll(const string& __str, size_t* __idx = nullptr, int __base = 10);
	3861	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI unsigned long long
	3862	stoull(const string& __str, size_t* __idx = nullptr, int __base = 10);
	3863
	3864	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI float stof(const string& __str, size_t* __idx = nullptr);
	3865	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI double stod(const string& __str, size_t* __idx = nullptr);
	3866	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long double stold(const string& __str, size_t* __idx = nullptr);
	3867
	3868	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(int __val);
	3869	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned __val);
	3870	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(long __val);
	3871	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long __val);
	3872	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(long long __val);
	3873	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(unsigned long long __val);
	3874	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(float __val);
	3875	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(double __val);
	3876	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI string to_string(long double __val);
3880	3877
3881	3878	# if _LIBCPP_HAS_WIDE_CHARACTERS
3882		_LIBCPP_EXPORTED_FROM_ABI int stoi(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
3883		_LIBCPP_EXPORTED_FROM_ABI long stol(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
3884		_LIBCPP_EXPORTED_FROM_ABI unsigned long stoul(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
3885		_LIBCPP_EXPORTED_FROM_ABI long long stoll(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
3886		_LIBCPP_EXPORTED_FROM_ABI unsigned long long stoull(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
3887
3888		_LIBCPP_EXPORTED_FROM_ABI float stof(const wstring& __str, size_t* __idx = nullptr);
3889		_LIBCPP_EXPORTED_FROM_ABI double stod(const wstring& __str, size_t* __idx = nullptr);
3890		_LIBCPP_EXPORTED_FROM_ABI long double stold(const wstring& __str, size_t* __idx = nullptr);
3891
3892		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(int __val);
3893		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned __val);
3894		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long __val);
3895		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long __val);
3896		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long long __val);
3897		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long long __val);
3898		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(float __val);
3899		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(double __val);
3900		_LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long double __val);
	3879	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI int stoi(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
	3880	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long stol(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
	3881	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI unsigned long
	3882	stoul(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
	3883	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long long
	3884	stoll(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
	3885	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI unsigned long long
	3886	stoull(const wstring& __str, size_t* __idx = nullptr, int __base = 10);
	3887
	3888	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI float stof(const wstring& __str, size_t* __idx = nullptr);
	3889	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI double stod(const wstring& __str, size_t* __idx = nullptr);
	3890	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI long double stold(const wstring& __str, size_t* __idx = nullptr);
	3891
	3892	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(int __val);
	3893	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned __val);
	3894	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long __val);
	3895	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long __val);
	3896	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long long __val);
	3897	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(unsigned long long __val);
	3898	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(float __val);
	3899	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(double __val);
	3900	[[__nodiscard__]] _LIBCPP_EXPORTED_FROM_ABI wstring to_wstring(long double __val);
3901	3901	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
3902	3902
3903	3903	template <class _CharT, class _Traits, class _Allocator>
...	...	@@ -3906,7 +3906,7 @@ _LIBCPP_TEMPLATE_DATA_VIS const typename basic_string<_CharT, _Traits, _Allocato
3906	3906
3907	3907	template <class _CharT, class _Allocator>
3908	3908	struct __string_hash : public __unary_function<basic_string<_CharT, char_traits<_CharT>, _Allocator>, size_t> {
3909		_LIBCPP_HIDE_FROM_ABI size_t
	3909	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t
3910	3910	operator()(const basic_string<_CharT, char_traits<_CharT>, _Allocator>& __val) const _NOEXCEPT {
3911	3911	return std::__do_string_hash(__val.data(), __val.data() + __val.size());
3912	3912	}
...	...	@@ -3977,30 +3977,31 @@ erase_if(basic_string<_CharT, _Traits, _Allocator>& __str, _Predicate __pred) {
3977	3977	// Literal suffixes for basic_string [basic.string.literals]
3978	3978	inline namespace literals {
3979	3979	inline namespace string_literals {
3980		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char>
	3980	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char>
3981	3981	operator""s(const char* __str, size_t __len) {
3982	3982	return basic_string<char>(__str, __len);
3983	3983	}
3984	3984
3985	3985	# if _LIBCPP_HAS_WIDE_CHARACTERS
3986		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<wchar_t>
3987		operator""s(const wchar_t* __str, size_t __len) {
	3986	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	3987	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<wchar_t> operator""s(const wchar_t* __str, size_t __len) {
3988	3988	return basic_string<wchar_t>(__str, __len);
3989	3989	}
3990	3990	# endif
3991	3991
3992	3992	# if _LIBCPP_HAS_CHAR8_T
3993		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string<char8_t> operator""s(const char8_t* __str, size_t __len) {
	3993	[[__nodiscard__]] inline
	3994	_LIBCPP_HIDE_FROM_ABI constexpr basic_string<char8_t> operator""s(const char8_t* __str, size_t __len) {
3994	3995	return basic_string<char8_t>(__str, __len);
3995	3996	}
3996	3997	# endif
3997	3998
3998		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char16_t>
3999		operator""s(const char16_t* __str, size_t __len) {
	3999	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	4000	_LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char16_t> operator""s(const char16_t* __str, size_t __len) {
4000	4001	return basic_string<char16_t>(__str, __len);
4001	4002	}
4002	4003
4003		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char32_t>
	4004	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<char32_t>
4004	4005	operator""s(const char32_t* __str, size_t __len) {
4005	4006	return basic_string<char32_t>(__str, __len);
4006	4007	}

lib/libcxx/include/string_view+113-89

...	...	@@ -130,6 +130,8 @@ namespace std {
130	130	size_type copy(charT* s, size_type n, size_type pos = 0) const; // constexpr in C++20
131	131
132	132	constexpr basic_string_view substr(size_type pos = 0, size_type n = npos) const;
	133	constexpr basic_string_view subview(size_type pos = 0,
	134	size_type n = npos) const; // freestanding-deleted, since C++26
133	135	constexpr int compare(basic_string_view s) const noexcept;
134	136	constexpr int compare(size_type pos1, size_type n1, basic_string_view s) const;
135	137	constexpr int compare(size_type pos1, size_type n1,
...	...	@@ -318,9 +320,9 @@ public:
318	320	_LIBCPP_HIDE_FROM_ABI basic_string_view& operator=(const basic_string_view&) _NOEXCEPT = default;
319	321
320	322	_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s, size_type __len) _NOEXCEPT
321		: __data_(__s),
322		__size_(__len) {
323		# if _LIBCPP_STD_VER >= 14
	323	_LIBCPP_DIAGNOSE_NULLPTR_IF(__len != 0 && __s == nullptr, " if len is not zero")
	324	: __data_(__s), __size_(__len) {
	325	# if !defined(_LIBCPP_CXX03_LANG) && (!defined(_LIBCPP_COMPILER_GCC) \|\| _LIBCPP_STD_VER >= 14)
324	326	// Allocations must fit in `ptrdiff_t` for pointer arithmetic to work. If `__len` exceeds it, the input
325	327	// range could not have been valid. Most likely the caller underflowed some arithmetic and inadvertently
326	328	// passed in a negative length.
...	...	@@ -352,7 +354,7 @@ public:
352	354	: __data_(ranges::data(__r)), __size_(ranges::size(__r)) {}
353	355	# endif // _LIBCPP_STD_VER >= 23
354	356
355		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s)
	357	_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s)
356	358	: __data_(__s), __size_(std::__char_traits_length_checked<_Traits>(__s)) {}
357	359
358	360	# if _LIBCPP_STD_VER >= 23
...	...	@@ -360,11 +362,11 @@ public:
360	362	# endif
361	363
362	364	// [string.view.iterators], iterators
363		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return cbegin(); }
	365	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return cbegin(); }
364	366
365		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return cend(); }
	367	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return cend(); }
366	368
367		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
	369	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
368	370	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
369	371	return std::__make_bounded_iter(data(), data(), data() + size());
370	372	# else
...	...	@@ -372,7 +374,7 @@ public:
372	374	# endif
373	375	}
374	376
375		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
	377	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT {
376	378	# ifdef _LIBCPP_ABI_BOUNDED_ITERATORS
377	379	return std::__make_bounded_iter(data() + size(), data(), data() + size());
378	380	# else
...	...	@@ -380,51 +382,54 @@ public:
380	382	# endif
381	383	}
382	384
383		_LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
	385	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	386	rbegin() const _NOEXCEPT {
384	387	return const_reverse_iterator(cend());
385	388	}
386	389
387		_LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
	390	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
388	391	return const_reverse_iterator(cbegin());
389	392	}
390	393
391		_LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
	394	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator
	395	crbegin() const _NOEXCEPT {
392	396	return const_reverse_iterator(cend());
393	397	}
394	398
395		_LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
	399	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX17 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
396	400	return const_reverse_iterator(cbegin());
397	401	}
398	402
399	403	// [string.view.capacity], capacity
400		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
	404	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
401	405
402		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type length() const _NOEXCEPT { return __size_; }
	406	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type length() const _NOEXCEPT { return __size_; }
403	407
404		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
	408	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
405	409	return numeric_limits<size_type>::max() / sizeof(value_type);
406	410	}
407	411
408	412	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool empty() const _NOEXCEPT { return __size_ == 0; }
409	413
410	414	// [string.view.access], element access
411		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference operator[](size_type __pos) const _NOEXCEPT {
	415	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference
	416	operator[](size_type __pos) const _NOEXCEPT {
412	417	return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__pos < size(), "string_view[] index out of bounds"), __data_[__pos];
413	418	}
414	419
415		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __pos) const {
	420	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference at(size_type __pos) const {
416	421	return __pos >= size() ? (__throw_out_of_range("string_view::at"), __data_[0]) : __data_[__pos];
417	422	}
418	423
419		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
	424	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference front() const _NOEXCEPT {
420	425	return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string_view::front(): string is empty"), __data_[0];
421	426	}
422	427
423		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
	428	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_reference back() const _NOEXCEPT {
424	429	return _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "string_view::back(): string is empty"), __data_[__size_ - 1];
425	430	}
426	431
427		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_pointer data() const _NOEXCEPT { return __data_; }
	432	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI const_pointer data() const _NOEXCEPT { return __data_; }
428	433
429	434	// [string.view.modifiers], modifiers:
430	435	_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI void remove_prefix(size_type __n) _NOEXCEPT {
...	...	@@ -457,15 +462,23 @@ public:
457	462	return __rlen;
458	463	}
459	464
460		_LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view substr(size_type __pos = 0, size_type __n = npos) const {
	465	[[__nodiscard__]] _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view
	466	substr(size_type __pos = 0, size_type __n = npos) const {
461	467	// Use the `__assume_valid` form of the constructor to avoid an unnecessary check. Any substring of a view is a
462	468	// valid view. In particular, `size()` is known to be smaller than `numeric_limits<difference_type>::max()`, so the
463		// new size is also smaller. See also https://github.com/llvm/llvm-project/issues/91634.
	469	// new size is also smaller. See also https://llvm.org/PR91634.
464	470	return __pos > size() ? (__throw_out_of_range("string_view::substr"), basic_string_view())
465	471	: basic_string_view(__assume_valid(), data() + __pos, std::min(__n, size() - __pos));
466	472	}
467	473
468		_LIBCPP_CONSTEXPR_SINCE_CXX14 int compare(basic_string_view __sv) const _NOEXCEPT {
	474	# if _LIBCPP_STD_VER >= 26
	475	[[nodiscard]]
	476	_LIBCPP_HIDE_FROM_ABI constexpr basic_string_view subview(size_type __pos = 0, size_type __n = npos) const {
	477	return substr(__pos, __n);
	478	}
	479	# endif
	480
	481	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 int compare(basic_string_view __sv) const _NOEXCEPT {
469	482	size_type __rlen = std::min(size(), __sv.size());
470	483	int __retval = _Traits::compare(data(), __sv.data(), __rlen);
471	484	if (__retval == 0) // first __rlen chars matched
...	...	@@ -473,217 +486,226 @@ public:
473	486	return __retval;
474	487	}
475	488
476		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
	489	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
477	490	compare(size_type __pos1, size_type __n1, basic_string_view __sv) const {
478	491	return substr(__pos1, __n1).compare(__sv);
479	492	}
480	493
481		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
	494	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
482	495	compare(size_type __pos1, size_type __n1, basic_string_view __sv, size_type __pos2, size_type __n2) const {
483	496	return substr(__pos1, __n1).compare(__sv.substr(__pos2, __n2));
484	497	}
485	498
486		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int compare(const _CharT* __s) const _NOEXCEPT {
	499	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
	500	compare(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s) const _NOEXCEPT {
487	501	return compare(basic_string_view(__s));
488	502	}
489	503
490		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
491		compare(size_type __pos1, size_type __n1, const _CharT* __s) const {
	504	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
	505	compare(size_type __pos1, size_type __n1, const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s) const {
492	506	return substr(__pos1, __n1).compare(basic_string_view(__s));
493	507	}
494	508
495		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
496		compare(size_type __pos1, size_type __n1, const _CharT* __s, size_type __n2) const {
	509	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI int
	510	compare(size_type __pos1, size_type __n1, const _CharT* __s, size_type __n2) const
	511	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n2 != 0 && __s == nullptr, " if n2 is not zero") {
497	512	return substr(__pos1, __n1).compare(basic_string_view(__s, __n2));
498	513	}
499	514
500	515	// find
501		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	516	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
502	517	find(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
503		_LIBCPP_ASSERT_NON_NULL(__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find(): received nullptr");
504	518	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
505	519	}
506	520
507		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type find(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
	521	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	522	find(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
508	523	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
509	524	}
510	525
511		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
512		find(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	526	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	527	find(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	528	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
513	529	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::find(): received nullptr");
514	530	return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
515	531	}
516	532
517		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
518		find(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
	533	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	534	find(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
519	535	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find(): received nullptr");
520	536	return std::__str_find<value_type, size_type, traits_type, npos>(
521	537	data(), size(), __s, __pos, traits_type::length(__s));
522	538	}
523	539
524	540	// rfind
525		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	541	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
526	542	rfind(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
527		_LIBCPP_ASSERT_NON_NULL(__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find(): received nullptr");
528	543	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
529	544	}
530	545
531		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	546	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
532	547	rfind(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
533	548	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
534	549	}
535	550
536		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
537		rfind(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	551	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	552	rfind(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	553	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
538	554	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::rfind(): received nullptr");
539	555	return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
540	556	}
541	557
542		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
543		rfind(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
	558	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	559	rfind(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
544	560	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::rfind(): received nullptr");
545	561	return std::__str_rfind<value_type, size_type, traits_type, npos>(
546	562	data(), size(), __s, __pos, traits_type::length(__s));
547	563	}
548	564
549	565	// find_first_of
550		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	566	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
551	567	find_first_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
552		_LIBCPP_ASSERT_NON_NULL(__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find_first_of(): received nullptr");
553	568	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
554	569	data(), size(), __s.data(), __pos, __s.size());
555	570	}
556	571
557		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	572	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
558	573	find_first_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
559	574	return find(__c, __pos);
560	575	}
561	576
562		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
563		find_first_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	577	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	578	find_first_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	579	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
564	580	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::find_first_of(): received nullptr");
565	581	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
566	582	}
567	583
568		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
569		find_first_of(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
	584	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	585	find_first_of(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
570	586	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_first_of(): received nullptr");
571	587	return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
572	588	data(), size(), __s, __pos, traits_type::length(__s));
573	589	}
574	590
575	591	// find_last_of
576		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	592	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
577	593	find_last_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
578		_LIBCPP_ASSERT_NON_NULL(__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find_last_of(): received nullptr");
579	594	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
580	595	data(), size(), __s.data(), __pos, __s.size());
581	596	}
582	597
583		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	598	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
584	599	find_last_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
585	600	return rfind(__c, __pos);
586	601	}
587	602
588		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
589		find_last_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	603	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	604	find_last_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	605	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
590	606	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::find_last_of(): received nullptr");
591	607	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
592	608	}
593	609
594		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
595		find_last_of(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
	610	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	611	find_last_of(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
596	612	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_last_of(): received nullptr");
597	613	return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
598	614	data(), size(), __s, __pos, traits_type::length(__s));
599	615	}
600	616
601	617	// find_first_not_of
602		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	618	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
603	619	find_first_not_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
604		_LIBCPP_ASSERT_NON_NULL(
605		__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find_first_not_of(): received nullptr");
606	620	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
607	621	data(), size(), __s.data(), __pos, __s.size());
608	622	}
609	623
610		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	624	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
611	625	find_first_not_of(_CharT __c, size_type __pos = 0) const _NOEXCEPT {
612	626	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
613	627	}
614	628
615		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
616		find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	629	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	630	find_first_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	631	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
617	632	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::find_first_not_of(): received nullptr");
618	633	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
619	634	}
620	635
621		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
622		find_first_not_of(const _CharT* __s, size_type __pos = 0) const _NOEXCEPT {
	636	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	637	find_first_not_of(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = 0) const _NOEXCEPT {
623	638	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_first_not_of(): received nullptr");
624	639	return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
625	640	data(), size(), __s, __pos, traits_type::length(__s));
626	641	}
627	642
628	643	// find_last_not_of
629		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	644	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
630	645	find_last_not_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
631		_LIBCPP_ASSERT_NON_NULL(
632		__s.size() == 0 \|\| __s.data() != nullptr, "string_view::find_last_not_of(): received nullptr");
633	646	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
634	647	data(), size(), __s.data(), __pos, __s.size());
635	648	}
636	649
637		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	650	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
638	651	find_last_not_of(_CharT __c, size_type __pos = npos) const _NOEXCEPT {
639	652	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __c, __pos);
640	653	}
641	654
642		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
643		find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT {
	655	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	656	find_last_not_of(const _CharT* __s, size_type __pos, size_type __n) const _NOEXCEPT
	657	_LIBCPP_DIAGNOSE_NULLPTR_IF(__n != 0 && __s == nullptr, " if n is not zero") {
644	658	_LIBCPP_ASSERT_NON_NULL(__n == 0 \|\| __s != nullptr, "string_view::find_last_not_of(): received nullptr");
645	659	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(data(), size(), __s, __pos, __n);
646	660	}
647	661
648		_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
649		find_last_not_of(const _CharT* __s, size_type __pos = npos) const _NOEXCEPT {
	662	[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
	663	find_last_not_of(const _CharT* _LIBCPP_DIAGNOSE_NULLPTR __s, size_type __pos = npos) const _NOEXCEPT {
650	664	_LIBCPP_ASSERT_NON_NULL(__s != nullptr, "string_view::find_last_not_of(): received nullptr");
651	665	return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
652	666	data(), size(), __s, __pos, traits_type::length(__s));
653	667	}
654	668
655	669	# if _LIBCPP_STD_VER >= 20
656		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(basic_string_view __s) const noexcept {
	670	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(basic_string_view __s) const noexcept {
657	671	return size() >= __s.size() && compare(0, __s.size(), __s) == 0;
658	672	}
659	673
660		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
	674	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(value_type __c) const noexcept {
661	675	return !empty() && _Traits::eq(front(), __c);
662	676	}
663	677
664		constexpr _LIBCPP_HIDE_FROM_ABI bool starts_with(const value_type* __s) const noexcept {
	678	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool
	679	starts_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
665	680	return starts_with(basic_string_view(__s));
666	681	}
667	682
668		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(basic_string_view __s) const noexcept {
	683	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(basic_string_view __s) const noexcept {
669	684	return size() >= __s.size() && compare(size() - __s.size(), npos, __s) == 0;
670	685	}
671	686
672		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
	687	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(value_type __c) const noexcept {
673	688	return !empty() && _Traits::eq(back(), __c);
674	689	}
675	690
676		constexpr _LIBCPP_HIDE_FROM_ABI bool ends_with(const value_type* __s) const noexcept {
	691	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool
	692	ends_with(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const noexcept {
677	693	return ends_with(basic_string_view(__s));
678	694	}
679	695	# endif
680	696
681	697	# if _LIBCPP_STD_VER >= 23
682		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(basic_string_view __sv) const noexcept { return find(__sv) != npos; }
	698	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool contains(basic_string_view __sv) const noexcept {
	699	return find(__sv) != npos;
	700	}
683	701
684		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept { return find(__c) != npos; }
	702	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool contains(value_type __c) const noexcept {
	703	return find(__c) != npos;
	704	}
685	705
686		constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* __s) const { return find(__s) != npos; }
	706	[[nodiscard]] constexpr _LIBCPP_HIDE_FROM_ABI bool contains(const value_type* _LIBCPP_DIAGNOSE_NULLPTR __s) const {
	707	return find(__s) != npos;
	708	}
687	709	# endif
688	710
689	711	private:
...	...	@@ -886,7 +908,8 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, basic_string_view<_CharT, _Trai
886	908	// [string.view.hash]
887	909	template <class _CharT>
888	910	struct __string_view_hash : public __unary_function<basic_string_view<_CharT, char_traits<_CharT> >, size_t> {
889		_LIBCPP_HIDE_FROM_ABI size_t operator()(const basic_string_view<_CharT, char_traits<_CharT> > __val) const _NOEXCEPT {
	911	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t
	912	operator()(const basic_string_view<_CharT, char_traits<_CharT> > __val) const _NOEXCEPT {
890	913	return std::__do_string_hash(__val.data(), __val.data() + __val.size());
891	914	}
892	915	};
...	...	@@ -913,30 +936,31 @@ struct hash<basic_string_view<wchar_t, char_traits<wchar_t> > > : __string_view_
913	936	# if _LIBCPP_STD_VER >= 14
914	937	inline namespace literals {
915	938	inline namespace string_view_literals {
916		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char> operator""sv(const char* __str, size_t __len) noexcept {
	939	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char>
	940	operator""sv(const char* __str, size_t __len) noexcept {
917	941	return basic_string_view<char>(__str, __len);
918	942	}
919	943
920	944	# if _LIBCPP_HAS_WIDE_CHARACTERS
921		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<wchar_t>
	945	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<wchar_t>
922	946	operator""sv(const wchar_t* __str, size_t __len) noexcept {
923	947	return basic_string_view<wchar_t>(__str, __len);
924	948	}
925	949	# endif
926	950
927	951	# if _LIBCPP_HAS_CHAR8_T
928		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char8_t>
	952	[[nodiscard]] inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char8_t>
929	953	operator""sv(const char8_t* __str, size_t __len) noexcept {
930	954	return basic_string_view<char8_t>(__str, __len);
931	955	}
932	956	# endif
933	957
934		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char16_t>
	958	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char16_t>
935	959	operator""sv(const char16_t* __str, size_t __len) noexcept {
936	960	return basic_string_view<char16_t>(__str, __len);
937	961	}
938	962
939		inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char32_t>
	963	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr basic_string_view<char32_t>
940	964	operator""sv(const char32_t* __str, size_t __len) noexcept {
941	965	return basic_string_view<char32_t>(__str, __len);
942	966	}

lib/libcxx/include/strstream+10-10

...	...	@@ -179,8 +179,8 @@ public:
179	179	void swap(strstreambuf& __rhs);
180	180
181	181	void freeze(bool __freezefl = true);
182		char* str();
183		int pcount() const;
	182	[[__nodiscard__]] char* str();
	183	[[__nodiscard__]] int pcount() const;
184	184
185	185	protected:
186	186	int_type overflow(int_type __c = EOF) override;
...	...	@@ -264,8 +264,8 @@ public:
264	264	__sb_.swap(__rhs.__sb_);
265	265	}
266	266
267		_LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
268		_LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
	267	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
	268	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
269	269
270	270	private:
271	271	strstreambuf __sb_;
...	...	@@ -297,10 +297,10 @@ public:
297	297	__sb_.swap(__rhs.__sb_);
298	298	}
299	299
300		_LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
	300	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
301	301	_LIBCPP_HIDE_FROM_ABI void freeze(bool __freezefl = true) { __sb_.freeze(__freezefl); }
302		_LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
303		_LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
	302	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
	303	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
304	304
305	305	private:
306	306	strstreambuf __sb_; // exposition only
...	...	@@ -340,10 +340,10 @@ public:
340	340	}
341	341
342	342	// Members:
343		_LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
	343	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI strstreambuf* rdbuf() const { return const_cast<strstreambuf*>(&__sb_); }
344	344	_LIBCPP_HIDE_FROM_ABI void freeze(bool __freezefl = true) { __sb_.freeze(__freezefl); }
345		_LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
346		_LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
	345	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI int pcount() const { return __sb_.pcount(); }
	346	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI char* str() { return __sb_.str(); }
347	347
348	348	private:
349	349	strstreambuf __sb_; // exposition only

lib/libcxx/include/syncstream+4-4

...	...	@@ -317,9 +317,9 @@ public:
317	317
318	318	_LIBCPP_HIDE_FROM_ABI bool emit() { return emit(false); }
319	319
320		_LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __wrapped_; }
	320	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __wrapped_; }
321	321
322		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
	322	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const noexcept { return __str_.get_allocator(); }
323	323
324	324	_LIBCPP_HIDE_FROM_ABI void set_emit_on_sync(bool __b) noexcept { __emit_on_sync_ = __b; }
325	325
...	...	@@ -496,9 +496,9 @@ public:
496	496	}
497	497	}
498	498
499		_LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __sb_.get_wrapped(); }
	499	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI streambuf_type* get_wrapped() const noexcept { return __sb_.get_wrapped(); }
500	500
501		_LIBCPP_HIDE_FROM_ABI syncbuf_type* rdbuf() const noexcept {
	501	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI syncbuf_type* rdbuf() const noexcept {
502	502	return const_cast<syncbuf_type*>(std::addressof(__sb_));
503	503	}
504	504

lib/libcxx/include/tgmath.h+12-12

...	...	@@ -18,22 +18,22 @@
18	18	*/
19	19
20	20	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
21		# include <__cxx03/tgmath.h>
	21	# include <__cxx03/__config>
22	22	#else
23	23	# include <__config>
	24	#endif
24	25
25		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
26		# pragma GCC system_header
27		# endif
	26	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	27	# pragma GCC system_header
	28	#endif
28	29
29		# ifdef __cplusplus
30		# include <cmath>
31		# include <complex>
32		# else
33		# if __has_include_next(<tgmath.h>)
34		# include_next <tgmath.h>
35		# endif
	30	#ifdef __cplusplus
	31	# include <cmath>
	32	# include <complex>
	33	#else
	34	# if __has_include_next(<tgmath.h>)
	35	# include_next <tgmath.h>
36	36	# endif
37		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	37	#endif
38	38
39	39	#endif // _LIBCPP_TGMATH_H

lib/libcxx/include/tuple+242-212

...	...	@@ -106,6 +106,11 @@ public:
106	106
107	107	void swap(tuple&) noexcept(AND(swap(declval<T&>(), declval<T&>())...)); // constexpr in C++20
108	108	constexpr void swap(const tuple&) const noexcept(see-below); // C++23
	109
	110	template<tuple-like UTuple>
	111	friend constexpr bool operator==(const tuple& t, const UTuple& u); // C++23
	112	template<tuple-like UTuple>
	113	friend constexpr auto operator<=>(const tuple& t, const UTuple& u); // C++23
109	114	};
110	115
111	116
...	...	@@ -220,19 +225,16 @@ template <class... Types>
220	225	# include <__config>
221	226	# include <__cstddef/size_t.h>
222	227	# include <__fwd/array.h>
	228	# include <__fwd/get.h>
223	229	# include <__fwd/pair.h>
224	230	# include <__fwd/tuple.h>
225	231	# include <__memory/allocator_arg_t.h>
226	232	# include <__memory/uses_allocator.h>
227	233	# include <__tuple/find_index.h>
228	234	# include <__tuple/ignore.h>
229		# include <__tuple/make_tuple_types.h>
230		# include <__tuple/sfinae_helpers.h>
231	235	# include <__tuple/tuple_element.h>
232		# include <__tuple/tuple_indices.h>
233		# include <__tuple/tuple_like_ext.h>
	236	# include <__tuple/tuple_like.h>
234	237	# include <__tuple/tuple_size.h>
235		# include <__tuple/tuple_types.h>
236	238	# include <__type_traits/common_reference.h>
237	239	# include <__type_traits/common_type.h>
238	240	# include <__type_traits/conditional.h>
...	...	@@ -241,7 +243,6 @@ template <class... Types>
241	243	# include <__type_traits/disjunction.h>
242	244	# include <__type_traits/enable_if.h>
243	245	# include <__type_traits/invoke.h>
244		# include <__type_traits/is_arithmetic.h>
245	246	# include <__type_traits/is_assignable.h>
246	247	# include <__type_traits/is_constructible.h>
247	248	# include <__type_traits/is_convertible.h>
...	...	@@ -251,7 +252,6 @@ template <class... Types>
251	252	# include <__type_traits/is_nothrow_assignable.h>
252	253	# include <__type_traits/is_nothrow_constructible.h>
253	254	# include <__type_traits/is_reference.h>
254		# include <__type_traits/is_replaceable.h>
255	255	# include <__type_traits/is_same.h>
256	256	# include <__type_traits/is_swappable.h>
257	257	# include <__type_traits/is_trivially_relocatable.h>
...	...	@@ -263,12 +263,12 @@ template <class... Types>
263	263	# include <__type_traits/remove_cv.h>
264	264	# include <__type_traits/remove_cvref.h>
265	265	# include <__type_traits/remove_reference.h>
	266	# include <__type_traits/type_list.h>
266	267	# include <__type_traits/unwrap_ref.h>
267	268	# include <__utility/declval.h>
268	269	# include <__utility/forward.h>
269	270	# include <__utility/integer_sequence.h>
270	271	# include <__utility/move.h>
271		# include <__utility/piecewise_construct.h>
272	272	# include <__utility/swap.h>
273	273	# include <version>
274	274
...	...	@@ -288,9 +288,65 @@ _LIBCPP_BEGIN_NAMESPACE_STD
288	288
289	289	# ifndef _LIBCPP_CXX03_LANG
290	290
	291	template <size_t _Ip, class _Tp, class _Up>
	292	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool __tuple_compare_equal(const _Tp& __x, const _Up& __y) {
	293	if constexpr (_Ip == 0)
	294	return true;
	295	else
	296	return std::__tuple_compare_equal<_Ip - 1>(__x, __y) && std::get<_Ip - 1>(__x) == std::get<_Ip - 1>(__y);
	297	}
	298
	299	# if _LIBCPP_STD_VER >= 26
	300	template <class _Tp, class _Up, class _IndexSeq = make_index_sequence<tuple_size_v<_Tp>>>
	301	inline constexpr bool __can_tuple_compare_equal = false;
	302
	303	// TODO(LLVM 23): Remove `tuple_size_v<_Tp> == tuple_size_v<_Up>` here once once LLVM-20 support ends
	304	// because the resolution of CWG2369 landed in LLVM-21.
	305	template <class _Tp, class _Up, size_t... _Is>
	306	requires(tuple_size_v<_Tp> == tuple_size_v<_Up>)
	307	inline constexpr bool __can_tuple_compare_equal<_Tp, _Up, index_sequence<_Is...>> =
	308	__all<requires(const tuple_element_t<_Is, _Tp>& __t, const tuple_element_t<_Is, _Up>& __u) {
	309	{ __t == __u } -> __boolean_testable;
	310	}...>::value;
	311	# endif // _LIBCPP_STD_VER >= 26
	312
	313	# if _LIBCPP_STD_VER >= 20
	314	template <class _Ret, class _Tp, class _Up, size_t... _Is>
	315	_LIBCPP_HIDE_FROM_ABI constexpr _Ret __tuple_compare_three_way(const _Tp& __x, const _Up& __y, index_sequence<_Is...>) {
	316	_Ret __result = strong_ordering::equal;
	317	static_cast<void>(
	318	((__result = std::__synth_three_way(std::get<_Is>(__x), std::get<_Is>(__y)), __result != 0) \|\| ...));
	319	return __result;
	320	}
	321	# endif // _LIBCPP_STD_VER >= 20
	322
	323	# if _LIBCPP_STD_VER >= 23
	324	template <class _Tp>
	325	concept __tuple_like_no_tuple = __tuple_like<_Tp> && !__is_tuple_v<_Tp>;
	326
	327	template <class _Tp, class _Up, class _IndexSeq>
	328	struct __tuple_common_comparison_category_impl {};
	329
	330	// TODO(LLVM 23): Remove `tuple_size_v<_Tp> == tuple_size_v<_Up>` here once once LLVM-20 support ends
	331	// because the resolution of CWG2369 landed in LLVM-21.
	332	template <class _Tp, class _Up, size_t... _Is>
	333	requires(tuple_size_v<_Tp> == tuple_size_v<_Up>) && requires {
	334	typename common_comparison_category_t<
	335	__synth_three_way_result<tuple_element_t<_Is, _Tp>, tuple_element_t<_Is, _Up>>...>;
	336	}
	337	struct __tuple_common_comparison_category_impl<_Tp, _Up, index_sequence<_Is...>> {
	338	using type _LIBCPP_NODEBUG =
	339	common_comparison_category_t<__synth_three_way_result<tuple_element_t<_Is, _Tp>, tuple_element_t<_Is, _Up>>...>;
	340	};
	341
	342	template <__tuple_like _Tp, __tuple_like _Up>
	343	using __tuple_common_comparison_category _LIBCPP_NODEBUG =
	344	__tuple_common_comparison_category_impl<_Tp, _Up, make_index_sequence<tuple_size_v<_Tp>>>::type;
	345	# endif // _LIBCPP_STD_VER >= 23
	346
291	347	// __tuple_leaf
292	348
293		template <size_t _Ip, class _Hp, bool = is_empty<_Hp>::value && !__libcpp_is_final<_Hp>::value >
	349	template <size_t _Ip, class _Hp, bool = is_empty<_Hp>::value && !__is_final_v<_Hp> >
294	350	class __tuple_leaf;
295	351
296	352	template <size_t _Ip, class _Hp, bool _Ep>
...	...	@@ -444,62 +500,52 @@ public:
444	500	template <class... _Tp>
445	501	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __swallow(_Tp&&...) _NOEXCEPT {}
446	502
447		template <class _Tp>
448		struct __all_default_constructible;
449
450		template <class... _Tp>
451		struct __all_default_constructible<__tuple_types<_Tp...>> : __all<is_default_constructible<_Tp>::value...> {};
452
453	503	// __tuple_impl
454	504
455	505	template <class _Indx, class... _Tp>
456	506	struct __tuple_impl;
457	507
	508	struct __forward_args {};
	509	struct __value_init {};
	510	struct __from_tuple {};
	511
458	512	template <size_t... _Indx, class... _Tp>
459	513	struct _LIBCPP_DECLSPEC_EMPTY_BASES
460		__tuple_impl<__tuple_indices<_Indx...>, _Tp...> : public __tuple_leaf<_Indx, _Tp>... {
	514	__tuple_impl<__index_sequence<_Indx...>, _Tp...> : public __tuple_leaf<_Indx, _Tp>... {
461	515	_LIBCPP_HIDE_FROM_ABI constexpr __tuple_impl() noexcept(
462	516	__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
463	517
464		template <size_t... _Uf, class... _Tf, size_t... _Ul, class... _Tl, class... _Up>
465		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(
466		__tuple_indices<_Uf...>,
467		__tuple_types<_Tf...>,
468		__tuple_indices<_Ul...>,
469		__tuple_types<_Tl...>,
470		_Up&&... __u) noexcept(__all<is_nothrow_constructible<_Tf, _Up>::value...>::value &&
471		__all<is_nothrow_default_constructible<_Tl>::value...>::value)
472		: __tuple_leaf<_Uf, _Tf>(std::forward<_Up>(__u))..., __tuple_leaf<_Ul, _Tl>()... {}
473
474		template <class _Alloc, size_t... _Uf, class... _Tf, size_t... _Ul, class... _Tl, class... _Up>
	518	template <class... _Args>
	519	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(__forward_args, _Args&&... __args)
	520	: __tuple_leaf<_Indx, _Tp>(std::forward<_Args>(__args))... {}
	521
	522	template <class _Alloc>
	523	_LIBCPP_HIDE_FROM_ABI
	524	_LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(allocator_arg_t, const _Alloc& __alloc, __value_init)
	525	: __tuple_leaf<_Indx, _Tp>(__uses_alloc_ctor<_Tp, _Alloc>(), __alloc)... {}
	526
	527	template <class _Alloc, class... _Args>
475	528	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit __tuple_impl(
476		allocator_arg_t,
477		const _Alloc& __a,
478		__tuple_indices<_Uf...>,
479		__tuple_types<_Tf...>,
480		__tuple_indices<_Ul...>,
481		__tuple_types<_Tl...>,
482		_Up&&... __u)
483		: __tuple_leaf<_Uf, _Tf>(__uses_alloc_ctor<_Tf, _Alloc, _Up>(), __a, std::forward<_Up>(__u))...,
484		__tuple_leaf<_Ul, _Tl>(__uses_alloc_ctor<_Tl, _Alloc>(), __a)... {}
485
486		template <class _Tuple, __enable_if_t<__tuple_constructible<_Tuple, tuple<_Tp...> >::value, int> = 0>
487		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(_Tuple&& __t) noexcept(
488		(__all<is_nothrow_constructible<
489		_Tp,
490		typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>::value...>::value))
	529	allocator_arg_t, const _Alloc& __alloc, __forward_args, _Args&&... __args)
	530	: __tuple_leaf<_Indx, _Tp>(__uses_alloc_ctor<_Tp, _Alloc, _Args>(), __alloc, std::forward<_Args>(__args))... {}
	531
	532	template <class _Tuple>
	533	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(__from_tuple, _Tuple&& __t) noexcept(
	534	(__all<is_nothrow_constructible<_Tp, __copy_cvref_t<_Tuple, __tuple_element_t<_Indx, __remove_cvref_t<_Tuple>>>>::
	535	value...>::value))
491	536	: __tuple_leaf<_Indx, _Tp>(
492		std::forward<typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(
	537	std::forward<__copy_cvref_t<_Tuple, __tuple_element_t<_Indx, __remove_cvref_t<_Tuple>>>>(
493	538	std::get<_Indx>(__t)))... {}
494	539
495		template <class _Alloc, class _Tuple, __enable_if_t<__tuple_constructible<_Tuple, tuple<_Tp...> >::value, int> = 0>
496		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
	540	template <class _Alloc, class _Tuple>
	541	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	542	__tuple_impl(allocator_arg_t, const _Alloc& __a, __from_tuple, _Tuple&& __t)
497	543	: __tuple_leaf<_Indx, _Tp>(
498	544	__uses_alloc_ctor<_Tp,
499	545	_Alloc,
500		typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(),
	546	__copy_cvref_t<_Tuple, __tuple_element_t<_Indx, __remove_cvref_t<_Tuple>>>>(),
501	547	__a,
502		std::forward<typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>(
	548	std::forward<__copy_cvref_t<_Tuple, __tuple_element_t<_Indx, __remove_cvref_t<_Tuple>>>>(
503	549	std::get<_Indx>(__t)))... {}
504	550
505	551	__tuple_impl(const __tuple_impl&) = default;
...	...	@@ -518,19 +564,19 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES
518	564
519	565	template <class _Dest, class _Source, size_t... _Np>
520	566	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
521		__memberwise_copy_assign(_Dest& __dest, _Source const& __source, __tuple_indices<_Np...>) {
	567	__memberwise_copy_assign(_Dest& __dest, _Source const& __source, __index_sequence<_Np...>) {
522	568	std::__swallow(((std::get<_Np>(__dest) = std::get<_Np>(__source)), void(), 0)...);
523	569	}
524	570
525	571	template <class _Dest, class _Source, class... _Up, size_t... _Np>
526	572	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
527		__memberwise_forward_assign(_Dest& __dest, _Source&& __source, __tuple_types<_Up...>, __tuple_indices<_Np...>) {
	573	__memberwise_forward_assign(_Dest& __dest, _Source&& __source, __type_list<_Up...>, __index_sequence<_Np...>) {
528	574	std::__swallow(((std::get<_Np>(__dest) = std::forward<_Up>(std::get<_Np>(__source))), void(), 0)...);
529	575	}
530	576
531	577	template <class... _Tp>
532	578	class _LIBCPP_NO_SPECIALIZATIONS tuple {
533		typedef __tuple_impl<typename __make_tuple_indices<sizeof...(_Tp)>::type, _Tp...> _BaseT;
	579	typedef __tuple_impl<__index_sequence_for<_Tp...>, _Tp...> _BaseT;
534	580
535	581	_BaseT __base_;
536	582
...	...	@@ -549,7 +595,6 @@ class _LIBCPP_NO_SPECIALIZATIONS tuple {
549	595	public:
550	596	using __trivially_relocatable _LIBCPP_NODEBUG =
551	597	__conditional_t<_And<__libcpp_is_trivially_relocatable<_Tp>...>::value, tuple, void>;
552		using __replaceable _LIBCPP_NODEBUG = __conditional_t<_And<__is_replaceable<_Tp>...>::value, tuple, void>;
553	598
554	599	// [tuple.cnstr]
555	600
...	...	@@ -566,12 +611,7 @@ public:
566	611	__enable_if_t< _And< _IsDefault<_Tp>... >::value, int> = 0>
567	612	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, _IsImpDefault<_Tp>...> >::value)
568	613	tuple(allocator_arg_t, _Alloc const& __a)
569		: __base_(allocator_arg_t(),
570		__a,
571		__tuple_indices<>(),
572		__tuple_types<>(),
573		typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
574		__tuple_types<_Tp...>()) {}
	614	: __base_(allocator_arg_t(), __a, __value_init{}) {}
575	615
576	616	// tuple(const T&...) constructors (including allocator_arg_t variants)
577	617	template <template <class...> class _And = _And,
...	...	@@ -579,11 +619,7 @@ public:
579	619	_LIBCPP_HIDE_FROM_ABI
580	620	_LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value)
581	621	tuple(const _Tp&... __t) noexcept(_And<is_nothrow_copy_constructible<_Tp>...>::value)
582		: __base_(typename __make_tuple_indices<sizeof...(_Tp)>::type(),
583		typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
584		typename __make_tuple_indices<0>::type(),
585		typename __make_tuple_types<tuple, 0>::type(),
586		__t...) {}
	622	: __base_(__forward_args{}, __t...) {}
587	623
588	624	template <class _Alloc,
589	625	template <class...> class _And = _And,
...	...	@@ -591,13 +627,7 @@ public:
591	627	_LIBCPP_HIDE_FROM_ABI
592	628	_LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<const _Tp&, _Tp>...> >::value)
593	629	tuple(allocator_arg_t, const _Alloc& __a, const _Tp&... __t)
594		: __base_(allocator_arg_t(),
595		__a,
596		typename __make_tuple_indices<sizeof...(_Tp)>::type(),
597		typename __make_tuple_types<tuple, sizeof...(_Tp)>::type(),
598		typename __make_tuple_indices<0>::type(),
599		typename __make_tuple_types<tuple, 0>::type(),
600		__t...) {}
	630	: __base_(allocator_arg_t(), __a, __forward_args{}, __t...) {}
601	631
602	632	// tuple(U&& ...) constructors (including allocator_arg_t variants)
603	633	template <class... _Up>
...	...	@@ -616,11 +646,7 @@ public:
616	646	int> = 0>
617	647	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
618	648	tuple(_Up&&... __u) noexcept(_And<is_nothrow_constructible<_Tp, _Up>...>::value)
619		: __base_(typename __make_tuple_indices<sizeof...(_Up)>::type(),
620		typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
621		typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
622		typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
623		std::forward<_Up>(__u)...) {}
	649	: __base_(__forward_args{}, std::forward<_Up>(__u)...) {}
624	650
625	651	template <class _Alloc,
626	652	class... _Up,
...	...	@@ -628,13 +654,7 @@ public:
628	654	int> = 0>
629	655	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
630	656	tuple(allocator_arg_t, const _Alloc& __a, _Up&&... __u)
631		: __base_(allocator_arg_t(),
632		__a,
633		typename __make_tuple_indices<sizeof...(_Up)>::type(),
634		typename __make_tuple_types<tuple, sizeof...(_Up)>::type(),
635		typename __make_tuple_indices<sizeof...(_Tp), sizeof...(_Up)>::type(),
636		typename __make_tuple_types<tuple, sizeof...(_Tp), sizeof...(_Up)>::type(),
637		std::forward<_Up>(__u)...) {}
	657	: __base_(allocator_arg_t(), __a, __forward_args{}, std::forward<_Up>(__u)...) {}
638	658
639	659	// Copy and move constructors (including the allocator_arg_t variants)
640	660	tuple(const tuple&) = default;
...	...	@@ -644,13 +664,13 @@ public:
644	664	template <class...> class _And = _And,
645	665	__enable_if_t< _And<is_copy_constructible<_Tp>...>::value, int> = 0>
646	666	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, const tuple& __t)
647		: __base_(allocator_arg_t(), __alloc, __t) {}
	667	: __base_(allocator_arg_t(), __alloc, __from_tuple(), __t) {}
648	668
649	669	template <class _Alloc,
650	670	template <class...> class _And = _And,
651	671	__enable_if_t< _And<is_move_constructible<_Tp>...>::value, int> = 0>
652	672	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, tuple&& __t)
653		: __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
	673	: __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__t)) {}
654	674
655	675	// tuple(const tuple<U...>&) constructors (including allocator_arg_t variants)
656	676
...	...	@@ -683,7 +703,7 @@ public:
683	703	_LIBCPP_HIDE_FROM_ABI
684	704	_LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
685	705	tuple(const tuple<_Up...>& __t) noexcept(_And<is_nothrow_constructible<_Tp, const _Up&>...>::value)
686		: __base_(__t) {}
	706	: __base_(__from_tuple(), __t) {}
687	707
688	708	template <class... _Up,
689	709	class _Alloc,
...	...	@@ -691,33 +711,33 @@ public:
691	711	_LIBCPP_HIDE_FROM_ABI
692	712	_LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
693	713	tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
694		: __base_(allocator_arg_t(), __a, __t) {}
	714	: __base_(allocator_arg_t(), __a, __from_tuple(), __t) {}
695	715
696	716	# if _LIBCPP_STD_VER >= 23
697	717	// tuple(tuple<U...>&) constructors (including allocator_arg_t variants)
698	718
699	719	template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
700	720	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value) tuple(tuple<_Up...>& __t)
701		: __base_(__t) {}
	721	: __base_(__from_tuple(), __t) {}
702	722
703	723	template <class _Alloc, class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
704	724	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value)
705	725	tuple(allocator_arg_t, const _Alloc& __alloc, tuple<_Up...>& __t)
706		: __base_(allocator_arg_t(), __alloc, __t) {}
	726	: __base_(allocator_arg_t(), __alloc, __from_tuple(), __t) {}
707	727	# endif // _LIBCPP_STD_VER >= 23
708	728
709	729	// tuple(tuple<U...>&&) constructors (including allocator_arg_t variants)
710	730	template <class... _Up, __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
711	731	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
712	732	tuple(tuple<_Up...>&& __t) noexcept(_And<is_nothrow_constructible<_Tp, _Up>...>::value)
713		: __base_(std::move(__t)) {}
	733	: __base_(__from_tuple(), std::move(__t)) {}
714	734
715	735	template <class _Alloc,
716	736	class... _Up,
717	737	__enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
718	738	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
719	739	tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
720		: __base_(allocator_arg_t(), __a, std::move(__t)) {}
	740	: __base_(allocator_arg_t(), __a, __from_tuple(), std::move(__t)) {}
721	741
722	742	# if _LIBCPP_STD_VER >= 23
723	743	// tuple(const tuple<U...>&&) constructors (including allocator_arg_t variants)
...	...	@@ -725,14 +745,14 @@ public:
725	745	template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
726	746	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
727	747	tuple(const tuple<_Up...>&& __t)
728		: __base_(std::move(__t)) {}
	748	: __base_(__from_tuple(), std::move(__t)) {}
729	749
730	750	template <class _Alloc,
731	751	class... _Up,
732	752	enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
733	753	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
734	754	tuple(allocator_arg_t, const _Alloc& __alloc, const tuple<_Up...>&& __t)
735		: __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
	755	: __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__t)) {}
736	756	# endif // _LIBCPP_STD_VER >= 23
737	757
738	758	// tuple(const pair<U1, U2>&) constructors (including allocator_arg_t variants)
...	...	@@ -767,7 +787,7 @@ public:
767	787	_LIBCPP_HIDE_FROM_ABI
768	788	_LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
769	789	tuple(const pair<_Up1, _Up2>& __p) noexcept(_NothrowConstructibleFromPair<const pair<_Up1, _Up2>&>::value)
770		: __base_(__p) {}
	790	: __base_(__from_tuple(), __p) {}
771	791
772	792	template <class _Alloc,
773	793	class _Up1,
...	...	@@ -777,7 +797,7 @@ public:
777	797	_LIBCPP_HIDE_FROM_ABI
778	798	_LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
779	799	tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
780		: __base_(allocator_arg_t(), __a, __p) {}
	800	: __base_(allocator_arg_t(), __a, __from_tuple(), __p) {}
781	801
782	802	# if _LIBCPP_STD_VER >= 23
783	803	// tuple(pair<U1, U2>&) constructors (including allocator_arg_t variants)
...	...	@@ -785,7 +805,7 @@ public:
785	805	template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<pair<_U1, _U2>&>::value>* = nullptr>
786	806	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
787	807	tuple(pair<_U1, _U2>& __p)
788		: __base_(__p) {}
	808	: __base_(__from_tuple(), __p) {}
789	809
790	810	template <class _Alloc,
791	811	class _U1,
...	...	@@ -793,7 +813,7 @@ public:
793	813	enable_if_t< _EnableCtorFromPair<std::pair<_U1, _U2>&>::value>* = nullptr>
794	814	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
795	815	tuple(allocator_arg_t, const _Alloc& __alloc, pair<_U1, _U2>& __p)
796		: __base_(allocator_arg_t(), __alloc, __p) {}
	816	: __base_(allocator_arg_t(), __alloc, __from_tuple(), __p) {}
797	817	# endif
798	818
799	819	// tuple(pair<U1, U2>&&) constructors (including allocator_arg_t variants)
...	...	@@ -805,7 +825,7 @@ public:
805	825	_LIBCPP_HIDE_FROM_ABI
806	826	_LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
807	827	tuple(pair<_Up1, _Up2>&& __p) noexcept(_NothrowConstructibleFromPair<pair<_Up1, _Up2>&&>::value)
808		: __base_(std::move(__p)) {}
	828	: __base_(__from_tuple(), std::move(__p)) {}
809	829
810	830	template <class _Alloc,
811	831	class _Up1,
...	...	@@ -815,7 +835,7 @@ public:
815	835	_LIBCPP_HIDE_FROM_ABI
816	836	_LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
817	837	tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
818		: __base_(allocator_arg_t(), __a, std::move(__p)) {}
	838	: __base_(allocator_arg_t(), __a, __from_tuple(), std::move(__p)) {}
819	839
820	840	# if _LIBCPP_STD_VER >= 23
821	841	// tuple(const pair<U1, U2>&&) constructors (including allocator_arg_t variants)
...	...	@@ -823,7 +843,7 @@ public:
823	843	template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
824	844	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
825	845	tuple(const pair<_U1, _U2>&& __p)
826		: __base_(std::move(__p)) {}
	846	: __base_(__from_tuple(), std::move(__p)) {}
827	847
828	848	template <class _Alloc,
829	849	class _U1,
...	...	@@ -831,14 +851,14 @@ public:
831	851	enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
832	852	_LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
833	853	tuple(allocator_arg_t, const _Alloc& __alloc, const pair<_U1, _U2>&& __p)
834		: __base_(allocator_arg_t(), __alloc, std::move(__p)) {}
	854	: __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__p)) {}
835	855	# endif // _LIBCPP_STD_VER >= 23
836	856
837	857	// [tuple.assign]
838	858	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
839	859	operator=(_If<_And<is_copy_assignable<_Tp>...>::value, tuple, __nat> const& __tuple) noexcept(
840	860	_And<is_nothrow_copy_assignable<_Tp>...>::value) {
841		std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
	861	std::__memberwise_copy_assign(*this, __tuple, __index_sequence_for<_Tp...>());
842	862	return *this;
843	863	}
844	864
...	...	@@ -846,15 +866,14 @@ public:
846	866	_LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple const& __tuple) const
847	867	requires(_And<is_copy_assignable<const _Tp>...>::value)
848	868	{
849		std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
	869	std::__memberwise_copy_assign(*this, __tuple, __index_sequence_for<_Tp...>());
850	870	return *this;
851	871	}
852	872
853	873	_LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple&& __tuple) const
854	874	requires(_And<is_assignable<const _Tp&, _Tp>...>::value)
855	875	{
856		std::__memberwise_forward_assign(
857		*this, std::move(__tuple), __tuple_types<_Tp...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
	876	std::__memberwise_forward_assign(*this, std::move(__tuple), __type_list<_Tp...>(), __index_sequence_for<_Tp...>());
858	877	return *this;
859	878	}
860	879	# endif // _LIBCPP_STD_VER >= 23
...	...	@@ -862,8 +881,7 @@ public:
862	881	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
863	882	operator=(_If<_And<is_move_assignable<_Tp>...>::value, tuple, __nat>&& __tuple) noexcept(
864	883	_And<is_nothrow_move_assignable<_Tp>...>::value) {
865		std::__memberwise_forward_assign(
866		*this, std::move(__tuple), __tuple_types<_Tp...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
	884	std::__memberwise_forward_assign(*this, std::move(__tuple), __type_list<_Tp...>(), __index_sequence_for<_Tp...>());
867	885	return *this;
868	886	}
869	887
...	...	@@ -873,7 +891,7 @@ public:
873	891	int> = 0>
874	892	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
875	893	operator=(tuple<_Up...> const& __tuple) noexcept(_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value) {
876		std::__memberwise_copy_assign(*this, __tuple, typename __make_tuple_indices<sizeof...(_Tp)>::type());
	894	std::__memberwise_copy_assign(*this, __tuple, __index_sequence_for<_Tp...>());
877	895	return *this;
878	896	}
879	897
...	...	@@ -882,8 +900,7 @@ public:
882	900	int> = 0>
883	901	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
884	902	operator=(tuple<_Up...>&& __tuple) noexcept(_And<is_nothrow_assignable<_Tp&, _Up>...>::value) {
885		std::__memberwise_forward_assign(
886		*this, std::move(__tuple), __tuple_types<_Up...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
	903	std::__memberwise_forward_assign(*this, std::move(__tuple), __type_list<_Up...>(), __index_sequence_for<_Tp...>());
887	904	return *this;
888	905	}
889	906
...	...	@@ -892,7 +909,7 @@ public:
892	909	enable_if_t< _And<_BoolConstant<sizeof...(_Tp) == sizeof...(_UTypes)>,
893	910	is_assignable<const _Tp&, const _UTypes&>...>::value>* = nullptr>
894	911	_LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(const tuple<_UTypes...>& __u) const {
895		std::__memberwise_copy_assign(*this, __u, typename __make_tuple_indices<sizeof...(_Tp)>::type());
	912	std::__memberwise_copy_assign(*this, __u, index_sequence_for<_Tp...>());
896	913	return *this;
897	914	}
898	915
...	...	@@ -900,8 +917,7 @@ public:
900	917	enable_if_t< _And<_BoolConstant<sizeof...(_Tp) == sizeof...(_UTypes)>,
901	918	is_assignable<const _Tp&, _UTypes>...>::value>* = nullptr>
902	919	_LIBCPP_HIDE_FROM_ABI constexpr const tuple& operator=(tuple<_UTypes...>&& __u) const {
903		std::__memberwise_forward_assign(
904		*this, __u, __tuple_types<_UTypes...>(), typename __make_tuple_indices<sizeof...(_Tp)>::type());
	920	std::__memberwise_forward_assign(*this, __u, __type_list<_UTypes...>(), index_sequence_for<_Tp...>());
905	921	return *this;
906	922	}
907	923	# endif // _LIBCPP_STD_VER >= 23
...	...	@@ -967,7 +983,7 @@ public:
967	983	__enable_if_t< _And< _BoolConstant<_Np == sizeof...(_Tp)>, is_assignable<_Tp&, _Up const&>... >::value, int> = 0>
968	984	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
969	985	operator=(array<_Up, _Np> const& __array) noexcept(_And<is_nothrow_assignable<_Tp&, _Up const&>...>::value) {
970		std::__memberwise_copy_assign(*this, __array, typename __make_tuple_indices<sizeof...(_Tp)>::type());
	986	std::__memberwise_copy_assign(*this, __array, __index_sequence_for<_Tp...>());
971	987	return *this;
972	988	}
973	989
...	...	@@ -979,10 +995,7 @@ public:
979	995	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple&
980	996	operator=(array<_Up, _Np>&& __array) noexcept(_And<is_nothrow_assignable<_Tp&, _Up>...>::value) {
981	997	std::__memberwise_forward_assign(
982		*this,
983		std::move(__array),
984		__tuple_types<_If<true, _Up, _Tp>...>(),
985		typename __make_tuple_indices<sizeof...(_Tp)>::type());
	998	*this, std::move(__array), __type_list<_If<true, _Up, _Tp>...>(), __index_sequence_for<_Tp...>());
986	999	return *this;
987	1000	}
988	1001
...	...	@@ -997,7 +1010,24 @@ public:
997	1010	noexcept(__all<is_nothrow_swappable_v<const _Tp&>...>::value) {
998	1011	__base_.swap(__t.__base_);
999	1012	}
1000		# endif // _LIBCPP_STD_VER >= 23
	1013
	1014	template <__tuple_like_no_tuple _UTuple>
	1015	# if _LIBCPP_STD_VER >= 26
	1016	requires __can_tuple_compare_equal<tuple, _UTuple> && (sizeof...(_Tp) == tuple_size_v<_UTuple>)
	1017	# endif // _LIBCPP_STD_VER >= 26
	1018	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const tuple& __x, const _UTuple& __y) {
	1019	static_assert(sizeof...(_Tp) == tuple_size_v<_UTuple>, "Can't compare tuple-like values of different sizes");
	1020	return std::__tuple_compare_equal<sizeof...(_Tp)>(__x, __y);
	1021	}
	1022
	1023	template <__tuple_like_no_tuple _UTuple>
	1024	requires(sizeof...(_Tp) == tuple_size_v<_UTuple>)
	1025	_LIBCPP_HIDE_FROM_ABI friend constexpr __tuple_common_comparison_category<tuple, _UTuple>
	1026	operator<=>(const tuple& __x, const _UTuple& __y) {
	1027	return std::__tuple_compare_three_way<__tuple_common_comparison_category<tuple, _UTuple>>(
	1028	__x, __y, index_sequence_for<_Tp...>{});
	1029	}
	1030	# endif // _LIBCPP_STD_VER >= 23
1001	1031	};
1002	1032
1003	1033	_LIBCPP_DIAGNOSTIC_PUSH
...	...	@@ -1019,6 +1049,21 @@ public:
1019	1049	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void swap(tuple&) _NOEXCEPT {}
1020	1050	# if _LIBCPP_STD_VER >= 23
1021	1051	_LIBCPP_HIDE_FROM_ABI constexpr void swap(const tuple&) const noexcept {}
	1052
	1053	template <__tuple_like_no_tuple _UTuple>
	1054	# if _LIBCPP_STD_VER >= 26
	1055	requires(tuple_size_v<_UTuple> == 0)
	1056	# endif // _LIBCPP_STD_VER >= 26
	1057	_LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const tuple&, const _UTuple&) {
	1058	static_assert(tuple_size_v<_UTuple> == 0, "Can't compare tuple-like values of different sizes");
	1059	return true;
	1060	}
	1061
	1062	template <__tuple_like_no_tuple _UTuple>
	1063	requires(tuple_size_v<_UTuple> == 0)
	1064	_LIBCPP_HIDE_FROM_ABI friend constexpr strong_ordering operator<=>(const tuple&, const _UTuple&) {
	1065	return strong_ordering::equal;
	1066	}
1022	1067	# endif
1023	1068	};
1024	1069	_LIBCPP_DIAGNOSTIC_POP
...	...	@@ -1068,28 +1113,32 @@ swap(const tuple<_Tp...>& __lhs,
1068	1113	// get
1069	1114
1070	1115	template <size_t _Ip, class... _Tp>
1071		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&
	1116	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	1117	typename tuple_element<_Ip, tuple<_Tp...> >::type&
1072	1118	get(tuple<_Tp...>& __t) _NOEXCEPT {
1073	1119	using type _LIBCPP_NODEBUG = typename tuple_element<_Ip, tuple<_Tp...> >::type;
1074	1120	return static_cast<__tuple_leaf<_Ip, type>&>(__t.__base_).get();
1075	1121	}
1076	1122
1077	1123	template <size_t _Ip, class... _Tp>
1078		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&
	1124	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	1125	_LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&
1079	1126	get(const tuple<_Tp...>& __t) _NOEXCEPT {
1080	1127	using type _LIBCPP_NODEBUG = typename tuple_element<_Ip, tuple<_Tp...> >::type;
1081	1128	return static_cast<const __tuple_leaf<_Ip, type>&>(__t.__base_).get();
1082	1129	}
1083	1130
1084	1131	template <size_t _Ip, class... _Tp>
1085		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename tuple_element<_Ip, tuple<_Tp...> >::type&&
	1132	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
	1133	typename tuple_element<_Ip, tuple<_Tp...> >::type&&
1086	1134	get(tuple<_Tp...>&& __t) _NOEXCEPT {
1087	1135	using type _LIBCPP_NODEBUG = typename tuple_element<_Ip, tuple<_Tp...> >::type;
1088	1136	return static_cast<type&&>(static_cast<__tuple_leaf<_Ip, type>&&>(__t.__base_).get());
1089	1137	}
1090	1138
1091	1139	template <size_t _Ip, class... _Tp>
1092		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&&
	1140	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
	1141	_LIBCPP_CONSTEXPR_SINCE_CXX14 const typename tuple_element<_Ip, tuple<_Tp...> >::type&&
1093	1142	get(const tuple<_Tp...>&& __t) _NOEXCEPT {
1094	1143	using type _LIBCPP_NODEBUG = typename tuple_element<_Ip, tuple<_Tp...> >::type;
1095	1144	return static_cast<const type&&>(static_cast<const __tuple_leaf<_Ip, type>&&>(__t.__base_).get());
...	...	@@ -1098,22 +1147,22 @@ get(const tuple<_Tp...>&& __t) _NOEXCEPT {
1098	1147	# if _LIBCPP_STD_VER >= 14
1099	1148
1100	1149	template <class _T1, class... _Args>
1101		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(tuple<_Args...>& __tup) noexcept {
	1150	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1& get(tuple<_Args...>& __tup) noexcept {
1102	1151	return std::get<__find_exactly_one_t<_T1, _Args...>::value>(__tup);
1103	1152	}
1104	1153
1105	1154	template <class _T1, class... _Args>
1106		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(tuple<_Args...> const& __tup) noexcept {
	1155	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const& get(tuple<_Args...> const& __tup) noexcept {
1107	1156	return std::get<__find_exactly_one_t<_T1, _Args...>::value>(__tup);
1108	1157	}
1109	1158
1110	1159	template <class _T1, class... _Args>
1111		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(tuple<_Args...>&& __tup) noexcept {
	1160	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1&& get(tuple<_Args...>&& __tup) noexcept {
1112	1161	return std::get<__find_exactly_one_t<_T1, _Args...>::value>(std::move(__tup));
1113	1162	}
1114	1163
1115	1164	template <class _T1, class... _Args>
1116		inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(tuple<_Args...> const&& __tup) noexcept {
	1165	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(tuple<_Args...> const&& __tup) noexcept {
1117	1166	return std::get<__find_exactly_one_t<_T1, _Args...>::value>(std::move(__tup));
1118	1167	}
1119	1168
...	...	@@ -1122,37 +1171,22 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr _T1 const&& get(tuple<_Args...> const&& _
1122	1171	// tie
1123	1172
1124	1173	template <class... _Tp>
1125		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&...> tie(_Tp&... __t) _NOEXCEPT {
	1174	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&...> tie(_Tp&... __t) _NOEXCEPT {
1126	1175	return tuple<_Tp&...>(__t...);
1127	1176	}
1128	1177
1129	1178	template <class... _Tp>
1130		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<__unwrap_ref_decay_t<_Tp>...>
	1179	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<__unwrap_ref_decay_t<_Tp>...>
1131	1180	make_tuple(_Tp&&... __t) {
1132	1181	return tuple<__unwrap_ref_decay_t<_Tp>...>(std::forward<_Tp>(__t)...);
1133	1182	}
1134	1183
1135	1184	template <class... _Tp>
1136		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&&...> forward_as_tuple(_Tp&&... __t) _NOEXCEPT {
	1185	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<_Tp&&...>
	1186	forward_as_tuple(_Tp&&... __t) _NOEXCEPT {
1137	1187	return tuple<_Tp&&...>(std::forward<_Tp>(__t)...);
1138	1188	}
1139	1189
1140		template <size_t _Ip>
1141		struct __tuple_equal {
1142		template <class _Tp, class _Up>
1143		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp& __x, const _Up& __y) {
1144		return __tuple_equal<_Ip - 1>()(__x, __y) && std::get<_Ip - 1>(__x) == std::get<_Ip - 1>(__y);
1145		}
1146		};
1147
1148		template <>
1149		struct __tuple_equal<0> {
1150		template <class _Tp, class _Up>
1151		_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool operator()(const _Tp&, const _Up&) {
1152		return true;
1153		}
1154		};
1155
1156	1190	template <class... _Tp, class... _Up>
1157	1191	# if _LIBCPP_STD_VER >= 26
1158	1192	requires(__all<requires(const _Tp& __t, const _Up& __u) {
...	...	@@ -1162,27 +1196,19 @@ template <class... _Tp, class... _Up>
1162	1196	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
1163	1197	operator==(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
1164	1198	static_assert(sizeof...(_Tp) == sizeof...(_Up), "Can't compare tuples of different sizes");
1165		return __tuple_equal<sizeof...(_Tp)>()(__x, __y);
	1199	return std::__tuple_compare_equal<sizeof...(_Tp)>(__x, __y);
1166	1200	}
1167	1201
1168	1202	# if _LIBCPP_STD_VER >= 20
1169	1203
1170	1204	// operator<=>
1171	1205
1172		template <class... _Tp, class... _Up, size_t... _Is>
1173		_LIBCPP_HIDE_FROM_ABI constexpr auto
1174		__tuple_compare_three_way(const tuple<_Tp...>& __x, const tuple<_Up...>& __y, index_sequence<_Is...>) {
1175		common_comparison_category_t<__synth_three_way_result<_Tp, _Up>...> __result = strong_ordering::equal;
1176		static_cast<void>(
1177		((__result = std::__synth_three_way(std::get<_Is>(__x), std::get<_Is>(__y)), __result != 0) \|\| ...));
1178		return __result;
1179		}
1180
1181	1206	template <class... _Tp, class... _Up>
1182	1207	requires(sizeof...(_Tp) == sizeof...(_Up))
1183	1208	_LIBCPP_HIDE_FROM_ABI constexpr common_comparison_category_t<__synth_three_way_result<_Tp, _Up>...>
1184	1209	operator<=>(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
1185		return std::__tuple_compare_three_way(__x, __y, index_sequence_for<_Tp...>{});
	1210	return std::__tuple_compare_three_way<common_comparison_category_t<__synth_three_way_result<_Tp, _Up>...>>(
	1211	__x, __y, index_sequence_for<_Tp...>{});
1186	1212	}
1187	1213
1188	1214	# else // _LIBCPP_STD_VER >= 20
...	...	@@ -1243,65 +1269,59 @@ operator<=(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
1243	1269
1244	1270	// tuple_cat
1245	1271
1246		template <class _Tp, class _Up>
1247		struct __tuple_cat_type;
	1272	template <class... _Tuples>
	1273	struct __tuple_cat_return_impl;
1248	1274
1249		template <class... _Ttypes, class... _Utypes>
1250		struct __tuple_cat_type<tuple<_Ttypes...>, __tuple_types<_Utypes...> > {
1251		using type _LIBCPP_NODEBUG = tuple<_Ttypes..., _Utypes...>;
	1275	template <class... _Types>
	1276	struct __tuple_cat_return_impl<tuple<_Types...>> {
	1277	using type _LIBCPP_NODEBUG = tuple<_Types...>;
1252	1278	};
1253	1279
1254		template <class _ResultTuple, bool _Is_Tuple0TupleLike, class... _Tuples>
1255		struct __tuple_cat_return_1 {};
	1280	template <class... _Types0, class... _Types1, class... _Tuples>
	1281	struct __tuple_cat_return_impl<tuple<_Types0...>, tuple<_Types1...>, _Tuples...>
	1282	: __tuple_cat_return_impl<tuple<_Types0..., _Types1...>, _Tuples...> {};
1256	1283
1257		template <class... _Types, class _Tuple0>
1258		struct __tuple_cat_return_1<tuple<_Types...>, true, _Tuple0> {
1259		using type _LIBCPP_NODEBUG =
1260		typename __tuple_cat_type< tuple<_Types...>,
1261		typename __make_tuple_types<__remove_cvref_t<_Tuple0> >::type >::type;
1262		};
1263
1264		template <class... _Types, class _Tuple0, class _Tuple1, class... _Tuples>
1265		struct __tuple_cat_return_1<tuple<_Types...>, true, _Tuple0, _Tuple1, _Tuples...>
1266		: public __tuple_cat_return_1<
1267		typename __tuple_cat_type< tuple<_Types...>,
1268		typename __make_tuple_types<__remove_cvref_t<_Tuple0> >::type >::type,
1269		__tuple_like_ext<__libcpp_remove_reference_t<_Tuple1> >::value,
1270		_Tuple1,
1271		_Tuples...> {};
	1284	template <class... _Types0, class _Tp, class _Up, class... _Tuples>
	1285	struct __tuple_cat_return_impl<tuple<_Types0...>, pair<_Tp, _Up>, _Tuples...>
	1286	: __tuple_cat_return_impl<tuple<_Types0..., _Tp, _Up>, _Tuples...> {};
1272	1287
1273		template <class... _Tuples>
1274		struct __tuple_cat_return;
	1288	template <class, class, class>
	1289	struct __tuple_cat_array;
1275	1290
1276		template <class _Tuple0, class... _Tuples>
1277		struct __tuple_cat_return<_Tuple0, _Tuples...>
1278		: public __tuple_cat_return_1<tuple<>,
1279		__tuple_like_ext<__libcpp_remove_reference_t<_Tuple0> >::value,
1280		_Tuple0,
1281		_Tuples...> {};
	1291	template <class... _Types, class _ValueT, size_t... _Indices>
	1292	struct __tuple_cat_array<tuple<_Types...>, _ValueT, __index_sequence<_Indices...>> {
	1293	template <size_t>
	1294	using __value_type _LIBCPP_NODEBUG = _ValueT;
1282	1295
1283		template <>
1284		struct __tuple_cat_return<> {
1285		using type _LIBCPP_NODEBUG = tuple<>;
	1296	using type _LIBCPP_NODEBUG = tuple<_Types..., __value_type<_Indices>...>;
1286	1297	};
1287	1298
1288		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<> tuple_cat() { return tuple<>(); }
	1299	template <class... _Types, class _ValueT, size_t _Np, class... _Tuples>
	1300	struct __tuple_cat_return_impl<tuple<_Types...>, array<_ValueT, _Np>, _Tuples...>
	1301	: __tuple_cat_return_impl<typename __tuple_cat_array<tuple<_Types...>, _ValueT, __make_index_sequence<_Np>>::type,
	1302	_Tuples...> {};
	1303
	1304	template <class... _Tuples>
	1305	using __tuple_cat_return_t _LIBCPP_NODEBUG =
	1306	typename __tuple_cat_return_impl<tuple<>, __remove_cvref_t<_Tuples>...>::type;
	1307
	1308	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 tuple<> tuple_cat() { return tuple<>(); }
1289	1309
1290	1310	template <class _Rp, class _Indices, class _Tuple0, class... _Tuples>
1291	1311	struct __tuple_cat_return_ref_imp;
1292	1312
1293	1313	template <class... _Types, size_t... _I0, class _Tuple0>
1294		struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0> {
	1314	struct __tuple_cat_return_ref_imp<tuple<_Types...>, __index_sequence<_I0...>, _Tuple0> {
1295	1315	using _T0 _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tuple0>;
1296	1316	typedef tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_I0, _T0>::type>&&...> type;
1297	1317	};
1298	1318
1299	1319	template <class... _Types, size_t... _I0, class _Tuple0, class _Tuple1, class... _Tuples>
1300		struct __tuple_cat_return_ref_imp<tuple<_Types...>, __tuple_indices<_I0...>, _Tuple0, _Tuple1, _Tuples...>
	1320	struct __tuple_cat_return_ref_imp<tuple<_Types...>, __index_sequence<_I0...>, _Tuple0, _Tuple1, _Tuples...>
1301	1321	: public __tuple_cat_return_ref_imp<
1302	1322	tuple<_Types...,
1303	1323	__copy_cvref_t<_Tuple0, typename tuple_element<_I0, __libcpp_remove_reference_t<_Tuple0>>::type>&&...>,
1304		typename __make_tuple_indices<tuple_size<__libcpp_remove_reference_t<_Tuple1> >::value>::type,
	1324	__make_index_sequence<tuple_size<__libcpp_remove_reference_t<_Tuple1> >::value>,
1305	1325	_Tuple1,
1306	1326	_Tuples...> {};
1307	1327
...	...	@@ -1309,7 +1329,7 @@ template <class _Tuple0, class... _Tuples>
1309	1329	struct __tuple_cat_return_ref
1310	1330	: public __tuple_cat_return_ref_imp<
1311	1331	tuple<>,
1312		typename __make_tuple_indices< tuple_size<__libcpp_remove_reference_t<_Tuple0> >::value >::type,
	1332	__make_index_sequence< tuple_size<__libcpp_remove_reference_t<_Tuple0> >::value >,
1313	1333	_Tuple0,
1314	1334	_Tuples...> {};
1315	1335
...	...	@@ -1317,7 +1337,7 @@ template <class _Types, class _I0, class _J0>
1317	1337	struct __tuple_cat;
1318	1338
1319	1339	template <class... _Types, size_t... _I0, size_t... _J0>
1320		struct __tuple_cat<tuple<_Types...>, __tuple_indices<_I0...>, __tuple_indices<_J0...> > {
	1340	struct __tuple_cat<tuple<_Types...>, __index_sequence<_I0...>, __index_sequence<_J0...>> {
1321	1341	template <class _Tuple0>
1322	1342	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
1323	1343	typename __tuple_cat_return_ref<tuple<_Types...>&&, _Tuple0&&>::type
...	...	@@ -1335,8 +1355,8 @@ struct __tuple_cat<tuple<_Types...>, __tuple_indices<_I0...>, __tuple_indices<_J
1335	1355	using _T0 _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tuple0>;
1336	1356	using _T1 _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tuple1>;
1337	1357	return __tuple_cat<tuple<_Types..., __copy_cvref_t<_Tuple0, typename tuple_element<_J0, _T0>::type>&&...>,
1338		typename __make_tuple_indices<sizeof...(_Types) + tuple_size<_T0>::value>::type,
1339		typename __make_tuple_indices<tuple_size<_T1>::value>::type>()(
	1358	__make_index_sequence<sizeof...(_Types) + tuple_size<_T0>::value>,
	1359	__make_index_sequence<tuple_size<_T1>::value>>()(
1340	1360	std::forward_as_tuple(
1341	1361	std::forward<_Types>(std::get<_I0>(__t))..., std::get<_J0>(std::forward<_Tuple0>(__t0))...),
1342	1362	std::forward<_Tuple1>(__t1),
...	...	@@ -1346,21 +1366,21 @@ struct __tuple_cat<tuple<_Types...>, __tuple_indices<_I0...>, __tuple_indices<_J
1346	1366
1347	1367	template <class _TupleDst, class _TupleSrc, size_t... _Indices>
1348	1368	inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _TupleDst
1349		__tuple_cat_select_element_wise(_TupleSrc&& __src, __tuple_indices<_Indices...>) {
	1369	__tuple_cat_select_element_wise(_TupleSrc&& __src, __index_sequence<_Indices...>) {
1350	1370	static_assert(tuple_size<_TupleDst>::value == tuple_size<_TupleSrc>::value,
1351	1371	"misuse of __tuple_cat_select_element_wise with tuples of different sizes");
1352	1372	return _TupleDst(std::get<_Indices>(std::forward<_TupleSrc>(__src))...);
1353	1373	}
1354	1374
1355	1375	template <class _Tuple0, class... _Tuples>
1356		inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename __tuple_cat_return<_Tuple0, _Tuples...>::type
	1376	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_cat_return_t<_Tuple0, _Tuples...>
1357	1377	tuple_cat(_Tuple0&& __t0, _Tuples&&... __tpls) {
1358	1378	using _T0 _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tuple0>;
1359		using _TRet _LIBCPP_NODEBUG = typename __tuple_cat_return<_Tuple0, _Tuples...>::type;
1360		using _T0Indices _LIBCPP_NODEBUG = typename __make_tuple_indices<tuple_size<_T0>::value>::type;
1361		using _TRetIndices _LIBCPP_NODEBUG = typename __make_tuple_indices<tuple_size<_TRet>::value>::type;
	1379	using _TRet _LIBCPP_NODEBUG = __tuple_cat_return_t<_Tuple0, _Tuples...>;
	1380	using _T0Indices _LIBCPP_NODEBUG = __make_index_sequence<tuple_size<_T0>::value>;
	1381	using _TRetIndices _LIBCPP_NODEBUG = __make_index_sequence<tuple_size<_TRet>::value>;
1362	1382	return std::__tuple_cat_select_element_wise<_TRet>(
1363		__tuple_cat<tuple<>, __tuple_indices<>, _T0Indices>()(
	1383	__tuple_cat<tuple<>, __index_sequence<>, _T0Indices>()(
1364	1384	tuple<>(), std::forward<_Tuple0>(__t0), std::forward<_Tuples>(__tpls)...),
1365	1385	_TRetIndices());
1366	1386	}
...	...	@@ -1376,7 +1396,7 @@ struct uses_allocator<tuple<_Tp...>, _Alloc> : true_type {};
1376	1396	// clang-format off
1377	1397	template <class _Fn, class _Tuple, size_t... _Id>
1378	1398	inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto)
1379		__apply_tuple_impl(_Fn&& __f, _Tuple&& __t, __tuple_indices<_Id...>)
	1399	__apply_tuple_impl(_Fn&& __f, _Tuple&& __t, index_sequence<_Id...>)
1380	1400	_LIBCPP_NOEXCEPT_RETURN(std::__invoke(std::forward<_Fn>(__f), std::get<_Id>(std::forward<_Tuple>(__t))...))
1381	1401
1382	1402	template <class _Fn, class _Tuple>
...	...	@@ -1384,28 +1404,29 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) apply(_Fn&& __f, _Tuple&&
1384	1404	_LIBCPP_NOEXCEPT_RETURN(std::__apply_tuple_impl(
1385	1405	std::forward<_Fn>(__f),
1386	1406	std::forward<_Tuple>(__t),
1387		typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type{}))
	1407	make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>()))
1388	1408
1389	1409	#if _LIBCPP_STD_VER >= 20
1390	1410	template <class _Tp, class _Tuple, size_t... _Idx>
1391		inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>)
	1411	inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, index_sequence<_Idx...>)
1392	1412	noexcept(noexcept(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...)))
1393	1413	requires is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...> {
1394	1414	return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...);
1395	1415	}
1396	1416	#else
1397	1417	template <class _Tp, class _Tuple, size_t... _Idx>
1398		inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>,
	1418	inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, index_sequence<_Idx...>,
1399	1419	enable_if_t<is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...>> * = nullptr)
1400	1420	_LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))
1401	1421	#endif // _LIBCPP_STD_VER >= 20
	1422	#undef _LIBCPP_NOEXCEPT_RETURN
1402	1423
1403	1424	template <class _Tp, class _Tuple,
1404		class _Seq = typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type, class = void>
	1425	class _Seq = make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>, class = void>
1405	1426	inline constexpr bool __can_make_from_tuple = false;
1406	1427
1407	1428	template <class _Tp, class _Tuple, size_t... _Idx>
1408		inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, __tuple_indices<_Idx...>,
	1429	inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, index_sequence<_Idx...>,
1409	1430	enable_if_t<is_constructible_v<_Tp, decltype(std::get<_Idx>(std::declval<_Tuple>()))...>>> = true;
1410	1431
1411	1432	// Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9,
...	...	@@ -1418,10 +1439,19 @@ template <class _Tp, class _Tuple>
1418	1439	#else
1419	1440	template <class _Tp, class _Tuple, class = enable_if_t<__can_make_from_tuple<_Tp, _Tuple>>> // strengthen
1420	1441	#endif // _LIBCPP_STD_VER >= 20
1421		inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t)
1422		_LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>(
1423		std::forward<_Tuple>(__t), typename __make_tuple_indices<tuple_size_v<remove_reference_t<_Tuple>>>::type{}))
1424		# undef _LIBCPP_NOEXCEPT_RETURN
	1442
	1443	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t)
	1444	noexcept(noexcept(std::__make_from_tuple_impl<_Tp>(std::forward<_Tuple>(__t),
	1445	make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>()))) {
	1446	#if _LIBCPP_STD_VER >= 23
	1447	if constexpr (tuple_size_v<remove_reference_t<_Tuple>> == 1) {
	1448	static_assert(!std::reference_constructs_from_temporary_v<_Tp, decltype(std::get<0>(std::declval<_Tuple>()))>,
	1449	"Attempted construction of reference element binds to a temporary whose lifetime has ended");
	1450	}
	1451	#endif // _LIBCPP_STD_VER >= 23
	1452	return std::__make_from_tuple_impl<_Tp>(
	1453	std::forward<_Tuple>(__t), make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>());
	1454	}
1425	1455
1426	1456	# endif // _LIBCPP_STD_VER >= 17
1427	1457

lib/libcxx/include/type_traits+8-2

...	...	@@ -454,6 +454,10 @@ namespace std
454	454	template<class B> inline constexpr bool negation_v
455	455	= negation<B>::value; // since C++17
456	456
	457	// [meta.const.eval], constant evaluation context
	458	constexpr bool is_constant_evaluated() noexcept; // C++20
	459	template<class T>
	460	consteval bool is_within_lifetime(const T*) noexcept; // C++26
457	461	}
458	462
459	463	*/
...	...	@@ -546,9 +550,7 @@ namespace std
546	550
547	551	# if _LIBCPP_STD_VER >= 20
548	552	# include <__type_traits/common_reference.h>
549		# include <__type_traits/is_bounded_array.h>
550	553	# include <__type_traits/is_constant_evaluated.h>
551		# include <__type_traits/is_unbounded_array.h>
552	554	# include <__type_traits/type_identity.h>
553	555	# include <__type_traits/unwrap_ref.h>
554	556	# endif
...	...	@@ -559,6 +561,10 @@ namespace std
559	561	# include <__type_traits/reference_converts_from_temporary.h>
560	562	# endif
561	563
	564	# if _LIBCPP_STD_VER >= 26
	565	# include <__type_traits/is_within_lifetime.h>
	566	# endif
	567
562	568	# include <version>
563	569
564	570	# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)

lib/libcxx/include/typeindex+5-3

...	...	@@ -86,8 +86,8 @@ public:
86	86	}
87	87	# endif
88	88
89		_LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __t_->hash_code(); }
90		_LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __t_->name(); }
	89	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __t_->hash_code(); }
	90	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __t_->name(); }
91	91	};
92	92
93	93	template <class _Tp>
...	...	@@ -95,7 +95,9 @@ struct hash;
95	95
96	96	template <>
97	97	struct hash<type_index> : public __unary_function<type_index, size_t> {
98		_LIBCPP_HIDE_FROM_ABI size_t operator()(type_index __index) const _NOEXCEPT { return __index.hash_code(); }
	98	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t operator()(type_index __index) const _NOEXCEPT {
	99	return __index.hash_code();
	100	}
99	101	};
100	102
101	103	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/include/typeinfo+92-89

...	...	@@ -95,11 +95,13 @@ class _LIBCPP_EXPORTED_FROM_ABI type_info {
95	95	public:
96	96	virtual ~type_info();
97	97
98		const char* name() const _NOEXCEPT;
	98	[[__nodiscard__]] const char* name() const _NOEXCEPT;
99	99
100		_LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT { return __compare(__arg) < 0; }
	100	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT {
	101	return __compare(__arg) < 0;
	102	}
101	103
102		size_t hash_code() const _NOEXCEPT;
	104	[[__nodiscard__]] size_t hash_code() const _NOEXCEPT;
103	105
104	106	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const type_info& __arg) const _NOEXCEPT {
105	107	// When evaluated in a constant expression, both type infos simply can't come
...	...	@@ -186,99 +188,99 @@ public:
186	188	# endif
187	189	# endif
188	190
189		struct __type_info_implementations {
190		struct __string_impl_base {
191		typedef const char* __type_name_t;
192		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static const char*
193		__type_name_to_string(__type_name_t __v) _NOEXCEPT {
194		return __v;
195		}
196		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static __type_name_t
197		__string_to_type_name(const char* __v) _NOEXCEPT {
198		return __v;
199		}
200		};
	191	namespace __type_info_implementations {
	192	struct __string_impl_base {
	193	typedef const char* __type_name_t;
	194	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static const char*
	195	__type_name_to_string(__type_name_t __v) _NOEXCEPT {
	196	return __v;
	197	}
	198	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE _LIBCPP_CONSTEXPR static __type_name_t
	199	__string_to_type_name(const char* __v) _NOEXCEPT {
	200	return __v;
	201	}
	202	};
201	203
202		struct __unique_impl : __string_impl_base {
203		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
204		return reinterpret_cast<size_t>(__v);
205		}
206		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
207		return __lhs == __rhs;
208		}
209		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
210		return __lhs < __rhs;
211		}
212		};
213
214		struct __non_unique_impl : __string_impl_base {
215		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __ptr) _NOEXCEPT {
216		size_t __hash = 5381;
217		while (unsigned char __c = static_cast<unsigned char>(*__ptr++))
218		__hash = (__hash * 33) ^ __c;
219		return __hash;
220		}
221		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
222		return __lhs == __rhs \|\| __builtin_strcmp(__lhs, __rhs) == 0;
223		}
224		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
225		return __builtin_strcmp(__lhs, __rhs) < 0;
226		}
227		};
	204	struct __unique_impl : __string_impl_base {
	205	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
	206	return reinterpret_cast<size_t>(__v);
	207	}
	208	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	209	return __lhs == __rhs;
	210	}
	211	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	212	return __lhs < __rhs;
	213	}
	214	};
228	215
229		struct __non_unique_arm_rtti_bit_impl {
230		typedef uintptr_t __type_name_t;
	216	struct __non_unique_impl : __string_impl_base {
	217	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __ptr) _NOEXCEPT {
	218	size_t __hash = 5381;
	219	while (unsigned char __c = static_cast<unsigned char>(*__ptr++))
	220	__hash = (__hash * 33) ^ __c;
	221	return __hash;
	222	}
	223	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	224	return __lhs == __rhs \|\| __builtin_strcmp(__lhs, __rhs) == 0;
	225	}
	226	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	227	return __builtin_strcmp(__lhs, __rhs) < 0;
	228	}
	229	};
231	230
232		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static const char* __type_name_to_string(__type_name_t __v) _NOEXCEPT {
233		return reinterpret_cast<const char*>(__v & ~__non_unique_rtti_bit::value);
234		}
235		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static __type_name_t __string_to_type_name(const char* __v) _NOEXCEPT {
236		return reinterpret_cast<__type_name_t>(__v);
237		}
	231	struct __non_unique_arm_rtti_bit_impl {
	232	typedef uintptr_t __type_name_t;
238	233
239		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
240		if (__is_type_name_unique(__v))
241		return __v;
242		return __non_unique_impl::__hash(__type_name_to_string(__v));
243		}
244		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
245		if (__lhs == __rhs)
246		return true;
247		if (__is_type_name_unique(__lhs) \|\| __is_type_name_unique(__rhs))
248		// Either both are unique and have a different address, or one of them
249		// is unique and the other one isn't. In both cases they are unequal.
250		return false;
251		return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) == 0;
252		}
253		_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
254		if (__is_type_name_unique(__lhs) \|\| __is_type_name_unique(__rhs))
255		return __lhs < __rhs;
256		return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) < 0;
257		}
	234	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static const char* __type_name_to_string(__type_name_t __v) _NOEXCEPT {
	235	return reinterpret_cast<const char*>(__v & ~__non_unique_rtti_bit::value);
	236	}
	237	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static __type_name_t __string_to_type_name(const char* __v) _NOEXCEPT {
	238	return reinterpret_cast<__type_name_t>(__v);
	239	}
258	240
259		private:
260		// The unique bit is the top bit. It is expected that __type_name_t is 64 bits when
261		// this implementation is actually used.
262		typedef integral_constant<__type_name_t, (1ULL << ((__CHAR_BIT__ * sizeof(__type_name_t)) - 1))>
263		__non_unique_rtti_bit;
	241	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static size_t __hash(__type_name_t __v) _NOEXCEPT {
	242	if (__is_type_name_unique(__v))
	243	return __v;
	244	return __non_unique_impl::__hash(__type_name_to_string(__v));
	245	}
	246	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __eq(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	247	if (__lhs == __rhs)
	248	return true;
	249	if (__is_type_name_unique(__lhs) \|\| __is_type_name_unique(__rhs))
	250	// Either both are unique and have a different address, or one of them
	251	// is unique and the other one isn't. In both cases they are unequal.
	252	return false;
	253	return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) == 0;
	254	}
	255	_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE static bool __lt(__type_name_t __lhs, __type_name_t __rhs) _NOEXCEPT {
	256	if (__is_type_name_unique(__lhs) \|\| __is_type_name_unique(__rhs))
	257	return __lhs < __rhs;
	258	return __builtin_strcmp(__type_name_to_string(__lhs), __type_name_to_string(__rhs)) < 0;
	259	}
264	260
265		_LIBCPP_HIDE_FROM_ABI static bool __is_type_name_unique(__type_name_t __lhs) _NOEXCEPT {
266		return !(__lhs & __non_unique_rtti_bit::value);
267		}
268		};
	261	private:
	262	// The unique bit is the top bit. It is expected that __type_name_t is 64 bits when
	263	// this implementation is actually used.
	264	typedef integral_constant<__type_name_t, (1ULL << ((__CHAR_BIT__ * sizeof(__type_name_t)) - 1))>
	265	__non_unique_rtti_bit;
	266
	267	_LIBCPP_HIDE_FROM_ABI static bool __is_type_name_unique(__type_name_t __lhs) _NOEXCEPT {
	268	return !(__lhs & __non_unique_rtti_bit::value);
	269	}
	270	};
269	271
270		typedef
	272	typedef
271	273	# if _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 1
272		__unique_impl
	274	__unique_impl
273	275	# elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 2
274		__non_unique_impl
	276	__non_unique_impl
275	277	# elif _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION == 3
276		__non_unique_arm_rtti_bit_impl
	278	__non_unique_arm_rtti_bit_impl
277	279	# else
278	280	# error invalid configuration for _LIBCPP_TYPEINFO_COMPARISON_IMPLEMENTATION
279	281	# endif
280		__impl;
281		};
	282	__impl;
	283	} // namespace __type_info_implementations
282	284
283	285	# if __has_cpp_attribute(_Clang::__ptrauth_vtable_pointer__)
284	286	# if __has_feature(ptrauth_type_info_vtable_pointer_discrimination)
...	...	@@ -306,14 +308,15 @@ protected:
306	308
307	309	public:
308	310	virtual ~type_info();
	311	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT {
	312	return __impl::__type_name_to_string(__type_name);
	313	}
309	314
310		_LIBCPP_HIDE_FROM_ABI const char* name() const _NOEXCEPT { return __impl::__type_name_to_string(__type_name); }
311
312		_LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT {
	315	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool before(const type_info& __arg) const _NOEXCEPT {
313	316	return __impl::__lt(__type_name, __arg.__type_name);
314	317	}
315	318
316		_LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __impl::__hash(__type_name); }
	319	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t hash_code() const _NOEXCEPT { return __impl::__hash(__type_name); }
317	320
318	321	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool operator==(const type_info& __arg) const _NOEXCEPT {
319	322	// When evaluated in a constant expression, both type infos simply can't come
...	...	@@ -336,7 +339,7 @@ public:
336	339	_LIBCPP_HIDE_FROM_ABI bad_cast(const bad_cast&) _NOEXCEPT = default;
337	340	_LIBCPP_HIDE_FROM_ABI bad_cast& operator=(const bad_cast&) _NOEXCEPT = default;
338	341	~bad_cast() _NOEXCEPT override;
339		const char* what() const _NOEXCEPT override;
	342	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
340	343	};
341	344
342	345	class _LIBCPP_EXPORTED_FROM_ABI bad_typeid : public exception {
...	...	@@ -345,7 +348,7 @@ public:
345	348	_LIBCPP_HIDE_FROM_ABI bad_typeid(const bad_typeid&) _NOEXCEPT = default;
346	349	_LIBCPP_HIDE_FROM_ABI bad_typeid& operator=(const bad_typeid&) _NOEXCEPT = default;
347	350	~bad_typeid() _NOEXCEPT override;
348		const char* what() const _NOEXCEPT override;
	351	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
349	352	};
350	353
351	354	} // namespace std

lib/libcxx/include/unordered_map+179-304

...	...	@@ -600,6 +600,7 @@ template <class Key, class T, class Hash, class Pred, class Alloc>
600	600	# include <__memory/addressof.h>
601	601	# include <__memory/allocator.h>
602	602	# include <__memory/allocator_traits.h>
	603	# include <__memory/compressed_pair.h>
603	604	# include <__memory/pointer_traits.h>
604	605	# include <__memory/unique_ptr.h>
605	606	# include <__memory_resource/polymorphic_allocator.h>
...	...	@@ -643,34 +644,9 @@ _LIBCPP_PUSH_MACROS
643	644
644	645	_LIBCPP_BEGIN_NAMESPACE_STD
645	646
646		template <class _Key,
647		class _Cp,
648		class _Hash,
649		class _Pred,
650		bool = is_empty<_Hash>::value && !__libcpp_is_final<_Hash>::value>
651		class __unordered_map_hasher : private _Hash {
652		public:
653		_LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value) : _Hash() {}
654		_LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
655		: _Hash(__h) {}
656		_LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return *this; }
657		_LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
658		_LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return static_cast<const _Hash&>(*this)(__x); }
659		# if _LIBCPP_STD_VER >= 20
660		template <typename _K2>
661		_LIBCPP_HIDE_FROM_ABI size_t operator()(const _K2& __x) const {
662		return static_cast<const _Hash&>(*this)(__x);
663		}
664		# endif
665		_LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_hasher& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Hash>) {
666		using std::swap;
667		swap(static_cast<_Hash&>(*this), static_cast<_Hash&>(__y));
668		}
669		};
670
671	647	template <class _Key, class _Cp, class _Hash, class _Pred>
672		class __unordered_map_hasher<_Key, _Cp, _Hash, _Pred, false> {
673		_Hash __hash_;
	648	class __unordered_map_hasher {
	649	_LIBCPP_COMPRESSED_ELEMENT(_Hash, __hash_);
674	650
675	651	public:
676	652	_LIBCPP_HIDE_FROM_ABI __unordered_map_hasher() _NOEXCEPT_(is_nothrow_default_constructible<_Hash>::value)
...	...	@@ -692,60 +668,16 @@ public:
692	668	}
693	669	};
694	670
695		template <class _Key, class _Cp, class _Hash, class _Pred, bool __b>
	671	template <class _Key, class _Cp, class _Hash, class _Pred>
696	672	inline _LIBCPP_HIDE_FROM_ABI void
697		swap(__unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __x,
698		__unordered_map_hasher<_Key, _Cp, _Hash, _Pred, __b>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
	673	swap(__unordered_map_hasher<_Key, _Cp, _Hash, _Pred>& __x, __unordered_map_hasher<_Key, _Cp, _Hash, _Pred>& __y)
	674	_NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
699	675	__x.swap(__y);
700	676	}
701	677
702		template <class _Key,
703		class _Cp,
704		class _Pred,
705		class _Hash,
706		bool = is_empty<_Pred>::value && !__libcpp_is_final<_Pred>::value>
707		class __unordered_map_equal : private _Pred {
708		public:
709		_LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value) : _Pred() {}
710		_LIBCPP_HIDE_FROM_ABI __unordered_map_equal(const _Pred& __p) _NOEXCEPT_(is_nothrow_copy_constructible<_Pred>::value)
711		: _Pred(__p) {}
712		_LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return *this; }
713		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
714		return static_cast<const _Pred&>(*this)(__x.first, __y.first);
715		}
716		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
717		return static_cast<const _Pred&>(*this)(__x.first, __y);
718		}
719		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
720		return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
721		}
722		# if _LIBCPP_STD_VER >= 20
723		template <typename _K2>
724		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
725		return static_cast<const _Pred&>(*this)(__x.first, __y);
726		}
727		template <typename _K2>
728		_LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
729		return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
730		}
731		template <typename _K2>
732		_LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _K2& __y) const {
733		return static_cast<const _Pred&>(*this)(__x, __y);
734		}
735		template <typename _K2>
736		_LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Key& __y) const {
737		return static_cast<const _Pred&>(*this)(__x, __y);
738		}
739		# endif
740		_LIBCPP_HIDE_FROM_ABI void swap(__unordered_map_equal& __y) _NOEXCEPT_(__is_nothrow_swappable_v<_Pred>) {
741		using std::swap;
742		swap(static_cast<_Pred&>(*this), static_cast<_Pred&>(__y));
743		}
744		};
745
746	678	template <class _Key, class _Cp, class _Pred, class _Hash>
747		class __unordered_map_equal<_Key, _Cp, _Pred, _Hash, false> {
748		_Pred __pred_;
	679	class __unordered_map_equal {
	680	_LIBCPP_COMPRESSED_ELEMENT(_Pred, __pred_);
749	681
750	682	public:
751	683	_LIBCPP_HIDE_FROM_ABI __unordered_map_equal() _NOEXCEPT_(is_nothrow_default_constructible<_Pred>::value)
...	...	@@ -780,9 +712,9 @@ public:
780	712	}
781	713	};
782	714
783		template <class _Key, class _Cp, class _Pred, class _Hash, bool __b>
	715	template <class _Key, class _Cp, class _Pred, class _Hash>
784	716	inline _LIBCPP_HIDE_FROM_ABI void
785		swap(__unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __x, __unordered_map_equal<_Key, _Cp, _Pred, _Hash, __b>& __y)
	717	swap(__unordered_map_equal<_Key, _Cp, _Pred, _Hash>& __x, __unordered_map_equal<_Key, _Cp, _Pred, _Hash>& __y)
786	718	_NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
787	719	__x.swap(__y);
788	720	}
...	...	@@ -844,10 +776,10 @@ class __hash_map_iterator {
844	776
845	777	public:
846	778	typedef forward_iterator_tag iterator_category;
847		typedef typename _NodeTypes::__map_value_type value_type;
848		typedef typename _NodeTypes::difference_type difference_type;
	779	using value_type = typename _HashIterator::value_type;
	780	using difference_type = ptrdiff_t;
849	781	typedef value_type& reference;
850		typedef typename _NodeTypes::__map_value_type_pointer pointer;
	782	using pointer = typename _HashIterator::pointer;
851	783
852	784	_LIBCPP_HIDE_FROM_ABI __hash_map_iterator() _NOEXCEPT {}
853	785
...	...	@@ -895,10 +827,10 @@ class __hash_map_const_iterator {
895	827
896	828	public:
897	829	typedef forward_iterator_tag iterator_category;
898		typedef typename _NodeTypes::__map_value_type value_type;
899		typedef typename _NodeTypes::difference_type difference_type;
	830	using value_type = typename _HashIterator::value_type;
	831	using difference_type = ptrdiff_t;
900	832	typedef const value_type& reference;
901		typedef typename _NodeTypes::__const_map_value_type_pointer pointer;
	833	using pointer = typename _HashIterator::pointer;
902	834
903	835	_LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator() _NOEXCEPT {}
904	836
...	...	@@ -972,9 +904,6 @@ private:
972	904
973	905	__table __table_;
974	906
975		typedef typename __table::_NodeTypes _NodeTypes;
976		typedef typename __table::__node_pointer __node_pointer;
977		typedef typename __table::__node_const_pointer __node_const_pointer;
978	907	typedef typename __table::__node_traits __node_traits;
979	908	typedef typename __table::__node_allocator __node_allocator;
980	909	typedef typename __table::__node __node;
...	...	@@ -1046,10 +975,10 @@ public:
1046	975	# endif
1047	976
1048	977	_LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
1049		_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
	978	_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u) = default;
1050	979	_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
1051	980	# ifndef _LIBCPP_CXX03_LANG
1052		_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
	981	_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) = default;
1053	982	_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
1054	983	_LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
1055	984	_LIBCPP_HIDE_FROM_ABI
...	...	@@ -1099,41 +1028,26 @@ public:
1099	1028	static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
1100	1029	}
1101	1030
1102		_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
1103		# ifndef _LIBCPP_CXX03_LANG
1104		__table_ = __u.__table_;
1105		# else
1106		if (this != std::addressof(__u)) {
1107		__table_.clear();
1108		__table_.hash_function() = __u.__table_.hash_function();
1109		__table_.key_eq() = __u.__table_.key_eq();
1110		__table_.max_load_factor() = __u.__table_.max_load_factor();
1111		__table_.__copy_assign_alloc(__u.__table_);
1112		insert(__u.begin(), __u.end());
1113		}
1114		# endif
1115		return *this;
1116		}
	1031	_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) = default;
1117	1032	# ifndef _LIBCPP_CXX03_LANG
1118		_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
1119		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
	1033	_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u) = default;
1120	1034	_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
1121	1035	# endif // _LIBCPP_CXX03_LANG
1122	1036
1123		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	1037	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
1124	1038	return allocator_type(__table_.__node_alloc());
1125	1039	}
1126	1040
1127	1041	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
1128		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
1129		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
	1042	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
	1043	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
1130	1044
1131		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
1132		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
1133		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
1134		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
1135		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
1136		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
	1045	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
	1046	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
	1047	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
	1048	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
	1049	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
	1050	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
1137	1051
1138	1052	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return __table_.__emplace_unique(__x); }
1139	1053
...	...	@@ -1187,14 +1101,13 @@ public:
1187	1101	# if _LIBCPP_STD_VER >= 17
1188	1102	template <class... _Args>
1189	1103	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args) {
1190		return __table_.__emplace_unique_key_args(
1191		__k, piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple(std::forward<_Args>(__args)...));
	1104	return __table_.__emplace_unique(
	1105	piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple(std::forward<_Args>(__args)...));
1192	1106	}
1193	1107
1194	1108	template <class... _Args>
1195	1109	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> try_emplace(key_type&& __k, _Args&&... __args) {
1196		return __table_.__emplace_unique_key_args(
1197		__k,
	1110	return __table_.__emplace_unique(
1198	1111	piecewise_construct,
1199	1112	std::forward_as_tuple(std::move(__k)),
1200	1113	std::forward_as_tuple(std::forward<_Args>(__args)...));
...	...	@@ -1212,7 +1125,7 @@ public:
1212	1125
1213	1126	template <class _Vp>
1214	1127	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(const key_type& __k, _Vp&& __v) {
1215		pair<iterator, bool> __res = __table_.__emplace_unique_key_args(__k, __k, std::forward<_Vp>(__v));
	1128	pair<iterator, bool> __res = __table_.__emplace_unique(__k, std::forward<_Vp>(__v));
1216	1129	if (!__res.second) {
1217	1130	__res.first->second = std::forward<_Vp>(__v);
1218	1131	}
...	...	@@ -1221,7 +1134,7 @@ public:
1221	1134
1222	1135	template <class _Vp>
1223	1136	_LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert_or_assign(key_type&& __k, _Vp&& __v) {
1224		pair<iterator, bool> __res = __table_.__emplace_unique_key_args(__k, std::move(__k), std::forward<_Vp>(__v));
	1137	pair<iterator, bool> __res = __table_.__emplace_unique(std::move(__k), std::forward<_Vp>(__v));
1225	1138	if (!__res.second) {
1226	1139	__res.first->second = std::forward<_Vp>(__v);
1227	1140	}
...	...	@@ -1258,10 +1171,10 @@ public:
1258	1171	"node_type with incompatible allocator passed to unordered_map::insert()");
1259	1172	return __table_.template __node_handle_insert_unique<node_type>(__hint.__i_, std::move(__nh));
1260	1173	}
1261		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1174	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
1262	1175	return __table_.template __node_handle_extract<node_type>(__key);
1263	1176	}
1264		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	1177	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
1265	1178	return __table_.template __node_handle_extract<node_type>(__it.__i_);
1266	1179	}
1267	1180
...	...	@@ -1295,52 +1208,56 @@ public:
1295	1208	__table_.swap(__u.__table_);
1296	1209	}
1297	1210
1298		_LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function().hash_function(); }
1299		_LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
	1211	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI hasher hash_function() const {
	1212	return __table_.hash_function().hash_function();
	1213	}
	1214	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
1300	1215
1301		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
1302		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
	1216	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
	1217	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
1303	1218	# if _LIBCPP_STD_VER >= 20
1304	1219	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1305		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1220	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
1306	1221	return __table_.find(__k);
1307	1222	}
1308	1223	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1309		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1224	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
1310	1225	return __table_.find(__k);
1311	1226	}
1312	1227	# endif // _LIBCPP_STD_VER >= 20
1313	1228
1314		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
	1229	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1230	return __table_.__count_unique(__k);
	1231	}
1315	1232	# if _LIBCPP_STD_VER >= 20
1316	1233	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1317		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1234	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
1318	1235	return __table_.__count_unique(__k);
1319	1236	}
1320	1237	# endif // _LIBCPP_STD_VER >= 20
1321	1238
1322	1239	# if _LIBCPP_STD_VER >= 20
1323		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1240	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
1324	1241
1325	1242	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1326		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1243	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
1327	1244	return find(__k) != end();
1328	1245	}
1329	1246	# endif // _LIBCPP_STD_VER >= 20
1330	1247
1331		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1248	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
1332	1249	return __table_.__equal_range_unique(__k);
1333	1250	}
1334		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1251	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
1335	1252	return __table_.__equal_range_unique(__k);
1336	1253	}
1337	1254	# if _LIBCPP_STD_VER >= 20
1338	1255	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1339		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1256	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
1340	1257	return __table_.__equal_range_unique(__k);
1341	1258	}
1342	1259	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1343		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1260	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
1344	1261	return __table_.__equal_range_unique(__k);
1345	1262	}
1346	1263	# endif // _LIBCPP_STD_VER >= 20
...	...	@@ -1350,24 +1267,32 @@ public:
1350	1267	_LIBCPP_HIDE_FROM_ABI mapped_type& operator[](key_type&& __k);
1351	1268	# endif
1352	1269
1353		_LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
1354		_LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
	1270	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI mapped_type& at(const key_type& __k);
	1271	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const mapped_type& at(const key_type& __k) const;
1355	1272
1356		_LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
1357		_LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
	1273	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
	1274	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT {
	1275	return __table_.max_bucket_count();
	1276	}
1358	1277
1359		_LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
1360		_LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
	1278	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const {
	1279	return __table_.bucket_size(__n);
	1280	}
	1281	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
1361	1282
1362		_LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
1363		_LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
1364		_LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
1365		_LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
1366		_LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
1367		_LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
	1283	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
	1284	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
	1285	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const {
	1286	return __table_.cbegin(__n);
	1287	}
	1288	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
	1289	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const {
	1290	return __table_.cbegin(__n);
	1291	}
	1292	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
1368	1293
1369		_LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
1370		_LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
	1294	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
	1295	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
1371	1296	_LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
1372	1297	_LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_unique(__n); }
1373	1298	_LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
...	...	@@ -1384,10 +1309,10 @@ template <class _InputIterator,
1384	1309	class _Pred = equal_to<__iter_key_type<_InputIterator>>,
1385	1310	class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
1386	1311	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1387		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1312	class = enable_if_t<!__is_allocator_v<_Hash>>,
1388	1313	class = enable_if_t<!is_integral<_Hash>::value>,
1389		class = enable_if_t<!__is_allocator<_Pred>::value>,
1390		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1314	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1315	class = enable_if_t<__is_allocator_v<_Allocator>>>
1391	1316	unordered_map(_InputIterator,
1392	1317	_InputIterator,
1393	1318	typename allocator_traits<_Allocator>::size_type = 0,
...	...	@@ -1401,10 +1326,10 @@ template <ranges::input_range _Range,
1401	1326	class _Hash = hash<__range_key_type<_Range>>,
1402	1327	class _Pred = equal_to<__range_key_type<_Range>>,
1403	1328	class _Allocator = allocator<__range_to_alloc_type<_Range>>,
1404		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1329	class = enable_if_t<!__is_allocator_v<_Hash>>,
1405	1330	class = enable_if_t<!is_integral<_Hash>::value>,
1406		class = enable_if_t<!__is_allocator<_Pred>::value>,
1407		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1331	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1332	class = enable_if_t<__is_allocator_v<_Allocator>>>
1408	1333	unordered_map(from_range_t,
1409	1334	_Range&&,
1410	1335	typename allocator_traits<_Allocator>::size_type = 0,
...	...	@@ -1419,10 +1344,10 @@ template <class _Key,
1419	1344	class _Hash = hash<remove_const_t<_Key>>,
1420	1345	class _Pred = equal_to<remove_const_t<_Key>>,
1421	1346	class _Allocator = allocator<pair<const _Key, _Tp>>,
1422		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1347	class = enable_if_t<!__is_allocator_v<_Hash>>,
1423	1348	class = enable_if_t<!is_integral<_Hash>::value>,
1424		class = enable_if_t<!__is_allocator<_Pred>::value>,
1425		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1349	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1350	class = enable_if_t<__is_allocator_v<_Allocator>>>
1426	1351	unordered_map(initializer_list<pair<_Key, _Tp>>,
1427	1352	typename allocator_traits<_Allocator>::size_type = 0,
1428	1353	_Hash = _Hash(),
...	...	@@ -1432,7 +1357,7 @@ unordered_map(initializer_list<pair<_Key, _Tp>>,
1432	1357	template <class _InputIterator,
1433	1358	class _Allocator,
1434	1359	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1435		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1360	class = enable_if_t<__is_allocator_v<_Allocator>>>
1436	1361	unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
1437	1362	-> unordered_map<__iter_key_type<_InputIterator>,
1438	1363	__iter_mapped_type<_InputIterator>,
...	...	@@ -1443,7 +1368,7 @@ unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocat
1443	1368	template <class _InputIterator,
1444	1369	class _Allocator,
1445	1370	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1446		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1371	class = enable_if_t<__is_allocator_v<_Allocator>>>
1447	1372	unordered_map(_InputIterator, _InputIterator, _Allocator)
1448	1373	-> unordered_map<__iter_key_type<_InputIterator>,
1449	1374	__iter_mapped_type<_InputIterator>,
...	...	@@ -1455,9 +1380,9 @@ template <class _InputIterator,
1455	1380	class _Hash,
1456	1381	class _Allocator,
1457	1382	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1458		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1383	class = enable_if_t<!__is_allocator_v<_Hash>>,
1459	1384	class = enable_if_t<!is_integral<_Hash>::value>,
1460		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1385	class = enable_if_t<__is_allocator_v<_Allocator>>>
1461	1386	unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1462	1387	-> unordered_map<__iter_key_type<_InputIterator>,
1463	1388	__iter_mapped_type<_InputIterator>,
...	...	@@ -1467,7 +1392,7 @@ unordered_map(_InputIterator, _InputIterator, typename allocator_traits<_Allocat
1467	1392
1468	1393	# if _LIBCPP_STD_VER >= 23
1469	1394
1470		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1395	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1471	1396	unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
1472	1397	-> unordered_map<__range_key_type<_Range>,
1473	1398	__range_mapped_type<_Range>,
...	...	@@ -1475,7 +1400,7 @@ unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::siz
1475	1400	equal_to<__range_key_type<_Range>>,
1476	1401	_Allocator>;
1477	1402
1478		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1403	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1479	1404	unordered_map(from_range_t, _Range&&, _Allocator)
1480	1405	-> unordered_map<__range_key_type<_Range>,
1481	1406	__range_mapped_type<_Range>,
...	...	@@ -1486,9 +1411,9 @@ unordered_map(from_range_t, _Range&&, _Allocator)
1486	1411	template <ranges::input_range _Range,
1487	1412	class _Hash,
1488	1413	class _Allocator,
1489		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1414	class = enable_if_t<!__is_allocator_v<_Hash>>,
1490	1415	class = enable_if_t<!is_integral<_Hash>::value>,
1491		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1416	class = enable_if_t<__is_allocator_v<_Allocator>>>
1492	1417	unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1493	1418	-> unordered_map<__range_key_type<_Range>,
1494	1419	__range_mapped_type<_Range>,
...	...	@@ -1498,11 +1423,11 @@ unordered_map(from_range_t, _Range&&, typename allocator_traits<_Allocator>::siz
1498	1423
1499	1424	# endif
1500	1425
1501		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1426	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1502	1427	unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
1503	1428	-> unordered_map<remove_const_t<_Key>, _Tp, hash<remove_const_t<_Key>>, equal_to<remove_const_t<_Key>>, _Allocator>;
1504	1429
1505		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1430	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1506	1431	unordered_map(initializer_list<pair<_Key, _Tp>>, _Allocator)
1507	1432	-> unordered_map<remove_const_t<_Key>, _Tp, hash<remove_const_t<_Key>>, equal_to<remove_const_t<_Key>>, _Allocator>;
1508	1433
...	...	@@ -1510,9 +1435,9 @@ template <class _Key,
1510	1435	class _Tp,
1511	1436	class _Hash,
1512	1437	class _Allocator,
1513		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1438	class = enable_if_t<!__is_allocator_v<_Hash>>,
1514	1439	class = enable_if_t<!is_integral<_Hash>::value>,
1515		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1440	class = enable_if_t<__is_allocator_v<_Allocator>>>
1516	1441	unordered_map(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1517	1442	-> unordered_map<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
1518	1443	# endif
...	...	@@ -1563,12 +1488,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
1563	1488	insert(__first, __last);
1564	1489	}
1565	1490
1566		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1567		unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u) : __table_(__u.__table_) {
1568		__table_.__rehash_unique(__u.bucket_count());
1569		insert(__u.begin(), __u.end());
1570		}
1571
1572	1491	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1573	1492	unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u, const allocator_type& __a)
1574	1493	: __table_(__u.__table_, typename __table::allocator_type(__a)) {
...	...	@@ -1578,11 +1497,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_ma
1578	1497
1579	1498	# ifndef _LIBCPP_CXX03_LANG
1580	1499
1581		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1582		inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
1583		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
1584		: __table_(std::move(__u.__table_)) {}
1585
1586	1500	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1587	1501	unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u, const allocator_type& __a)
1588	1502	: __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
...	...	@@ -1618,14 +1532,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
1618	1532	insert(__il.begin(), __il.end());
1619	1533	}
1620	1534
1621		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1622		inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
1623		unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_map&& __u)
1624		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
1625		__table_ = std::move(__u.__table_);
1626		return *this;
1627		}
1628
1629	1535	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1630	1536	inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
1631	1537	unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
...	...	@@ -1646,16 +1552,13 @@ inline void unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::insert(_InputIterato
1646	1552
1647	1553	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1648	1554	_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
1649		return __table_
1650		.__emplace_unique_key_args(__k, piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
	1555	return __table_.__emplace_unique(piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
1651	1556	.first->second;
1652	1557	}
1653	1558
1654	1559	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1655	1560	_Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](key_type&& __k) {
1656		return __table_
1657		.__emplace_unique_key_args(
1658		__k, piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
	1561	return __table_.__emplace_unique(piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
1659	1562	.first->second;
1660	1563	}
1661	1564	# else // _LIBCPP_CXX03_LANG
...	...	@@ -1781,12 +1684,8 @@ private:
1781	1684
1782	1685	__table __table_;
1783	1686
1784		typedef typename __table::_NodeTypes _NodeTypes;
1785	1687	typedef typename __table::__node_traits __node_traits;
1786		typedef typename __table::__node_allocator __node_allocator;
1787	1688	typedef typename __table::__node __node;
1788		typedef __hash_map_node_destructor<__node_allocator> _Dp;
1789		typedef unique_ptr<__node, _Dp> __node_holder;
1790	1689	typedef allocator_traits<allocator_type> __alloc_traits;
1791	1690	static_assert(is_same<typename __node_traits::size_type, typename __alloc_traits::size_type>::value,
1792	1691	"Allocator uses different size_type for different types");
...	...	@@ -1852,11 +1751,10 @@ public:
1852	1751	# endif
1853	1752
1854	1753	_LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
1855		_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
	1754	_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u) = default;
1856	1755	_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
1857	1756	# ifndef _LIBCPP_CXX03_LANG
1858		_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
1859		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
	1757	_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u) = default;
1860	1758	_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
1861	1759	_LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il);
1862	1760	_LIBCPP_HIDE_FROM_ABI unordered_multimap(
...	...	@@ -1906,41 +1804,26 @@ public:
1906	1804	static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
1907	1805	}
1908	1806
1909		_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) {
1910		# ifndef _LIBCPP_CXX03_LANG
1911		__table_ = __u.__table_;
1912		# else
1913		if (this != std::addressof(__u)) {
1914		__table_.clear();
1915		__table_.hash_function() = __u.__table_.hash_function();
1916		__table_.key_eq() = __u.__table_.key_eq();
1917		__table_.max_load_factor() = __u.__table_.max_load_factor();
1918		__table_.__copy_assign_alloc(__u.__table_);
1919		insert(__u.begin(), __u.end());
1920		}
1921		# endif
1922		return *this;
1923		}
	1807	_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) = default;
1924	1808	# ifndef _LIBCPP_CXX03_LANG
1925		_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
1926		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
	1809	_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u) = default;
1927	1810	_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
1928	1811	# endif // _LIBCPP_CXX03_LANG
1929	1812
1930		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	1813	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
1931	1814	return allocator_type(__table_.__node_alloc());
1932	1815	}
1933	1816
1934	1817	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
1935		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
1936		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
	1818	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
	1819	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
1937	1820
1938		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
1939		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
1940		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
1941		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
1942		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
1943		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
	1821	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
	1822	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
	1823	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
	1824	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
	1825	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
	1826	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
1944	1827
1945	1828	_LIBCPP_HIDE_FROM_ABI iterator insert(const value_type& __x) { return __table_.__emplace_multi(__x); }
1946	1829
...	...	@@ -2008,10 +1891,10 @@ public:
2008	1891	"node_type with incompatible allocator passed to unordered_multimap::insert()");
2009	1892	return __table_.template __node_handle_insert_multi<node_type>(__hint.__i_, std::move(__nh));
2010	1893	}
2011		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1894	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
2012	1895	return __table_.template __node_handle_extract<node_type>(__key);
2013	1896	}
2014		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	1897	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
2015	1898	return __table_.template __node_handle_extract<node_type>(__it.__i_);
2016	1899	}
2017	1900
...	...	@@ -2045,71 +1928,83 @@ public:
2045	1928	__table_.swap(__u.__table_);
2046	1929	}
2047	1930
2048		_LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function().hash_function(); }
2049		_LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
	1931	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI hasher hash_function() const {
	1932	return __table_.hash_function().hash_function();
	1933	}
	1934	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq().key_eq(); }
2050	1935
2051		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
2052		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
	1936	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
	1937	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
2053	1938	# if _LIBCPP_STD_VER >= 20
2054	1939	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2055		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1940	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
2056	1941	return __table_.find(__k);
2057	1942	}
2058	1943	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2059		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1944	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
2060	1945	return __table_.find(__k);
2061	1946	}
2062	1947	# endif // _LIBCPP_STD_VER >= 20
2063	1948
2064		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
	1949	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1950	return __table_.__count_multi(__k);
	1951	}
2065	1952	# if _LIBCPP_STD_VER >= 20
2066	1953	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2067		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1954	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
2068	1955	return __table_.__count_multi(__k);
2069	1956	}
2070	1957	# endif // _LIBCPP_STD_VER >= 20
2071	1958
2072	1959	# if _LIBCPP_STD_VER >= 20
2073		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1960	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
2074	1961
2075	1962	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2076		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1963	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
2077	1964	return find(__k) != end();
2078	1965	}
2079	1966	# endif // _LIBCPP_STD_VER >= 20
2080	1967
2081		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1968	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
2082	1969	return __table_.__equal_range_multi(__k);
2083	1970	}
2084		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1971	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
2085	1972	return __table_.__equal_range_multi(__k);
2086	1973	}
2087	1974	# if _LIBCPP_STD_VER >= 20
2088	1975	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2089		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1976	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
2090	1977	return __table_.__equal_range_multi(__k);
2091	1978	}
2092	1979	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
2093		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1980	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
2094	1981	return __table_.__equal_range_multi(__k);
2095	1982	}
2096	1983	# endif // _LIBCPP_STD_VER >= 20
2097	1984
2098		_LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
2099		_LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
	1985	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
	1986	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT {
	1987	return __table_.max_bucket_count();
	1988	}
2100	1989
2101		_LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
2102		_LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
	1990	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const {
	1991	return __table_.bucket_size(__n);
	1992	}
	1993	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
2103	1994
2104		_LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
2105		_LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
2106		_LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
2107		_LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
2108		_LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
2109		_LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
	1995	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
	1996	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
	1997	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const {
	1998	return __table_.cbegin(__n);
	1999	}
	2000	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
	2001	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const {
	2002	return __table_.cbegin(__n);
	2003	}
	2004	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
2110	2005
2111		_LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
2112		_LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
	2006	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
	2007	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
2113	2008	_LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
2114	2009	_LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_multi(__n); }
2115	2010	_LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
...	...	@@ -2121,10 +2016,10 @@ template <class _InputIterator,
2121	2016	class _Pred = equal_to<__iter_key_type<_InputIterator>>,
2122	2017	class _Allocator = allocator<__iter_to_alloc_type<_InputIterator>>,
2123	2018	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
2124		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2019	class = enable_if_t<!__is_allocator_v<_Hash>>,
2125	2020	class = enable_if_t<!is_integral<_Hash>::value>,
2126		class = enable_if_t<!__is_allocator<_Pred>::value>,
2127		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2021	class = enable_if_t<!__is_allocator_v<_Pred>>,
	2022	class = enable_if_t<__is_allocator_v<_Allocator>>>
2128	2023	unordered_multimap(_InputIterator,
2129	2024	_InputIterator,
2130	2025	typename allocator_traits<_Allocator>::size_type = 0,
...	...	@@ -2142,10 +2037,10 @@ template <ranges::input_range _Range,
2142	2037	class _Hash = hash<__range_key_type<_Range>>,
2143	2038	class _Pred = equal_to<__range_key_type<_Range>>,
2144	2039	class _Allocator = allocator<__range_to_alloc_type<_Range>>,
2145		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2040	class = enable_if_t<!__is_allocator_v<_Hash>>,
2146	2041	class = enable_if_t<!is_integral<_Hash>::value>,
2147		class = enable_if_t<!__is_allocator<_Pred>::value>,
2148		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2042	class = enable_if_t<!__is_allocator_v<_Pred>>,
	2043	class = enable_if_t<__is_allocator_v<_Allocator>>>
2149	2044	unordered_multimap(from_range_t,
2150	2045	_Range&&,
2151	2046	typename allocator_traits<_Allocator>::size_type = 0,
...	...	@@ -2160,21 +2055,21 @@ template <class _Key,
2160	2055	class _Hash = hash<remove_const_t<_Key>>,
2161	2056	class _Pred = equal_to<remove_const_t<_Key>>,
2162	2057	class _Allocator = allocator<pair<const _Key, _Tp>>,
2163		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2058	class = enable_if_t<!__is_allocator_v<_Hash>>,
2164	2059	class = enable_if_t<!is_integral<_Hash>::value>,
2165		class = enable_if_t<!__is_allocator<_Pred>::value>,
2166		class = enable_if_t<__is_allocator<_Allocator>::value>>
2167		unordered_multimap(
2168		initializer_list<pair<_Key, _Tp>>,
2169		typename allocator_traits<_Allocator>::size_type = 0,
2170		_Hash = _Hash(),
2171		_Pred = _Pred(),
2172		_Allocator = _Allocator()) -> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, _Pred, _Allocator>;
	2060	class = enable_if_t<!__is_allocator_v<_Pred>>,
	2061	class = enable_if_t<__is_allocator_v<_Allocator>>>
	2062	unordered_multimap(initializer_list<pair<_Key, _Tp>>,
	2063	typename allocator_traits<_Allocator>::size_type = 0,
	2064	_Hash = _Hash(),
	2065	_Pred = _Pred(),
	2066	_Allocator = _Allocator())
	2067	-> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, _Pred, _Allocator>;
2173	2068
2174	2069	template <class _InputIterator,
2175	2070	class _Allocator,
2176	2071	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
2177		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2072	class = enable_if_t<__is_allocator_v<_Allocator>>>
2178	2073	unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
2179	2074	-> unordered_multimap<__iter_key_type<_InputIterator>,
2180	2075	__iter_mapped_type<_InputIterator>,
...	...	@@ -2185,7 +2080,7 @@ unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Al
2185	2080	template <class _InputIterator,
2186	2081	class _Allocator,
2187	2082	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
2188		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2083	class = enable_if_t<__is_allocator_v<_Allocator>>>
2189	2084	unordered_multimap(_InputIterator, _InputIterator, _Allocator)
2190	2085	-> unordered_multimap<__iter_key_type<_InputIterator>,
2191	2086	__iter_mapped_type<_InputIterator>,
...	...	@@ -2197,9 +2092,9 @@ template <class _InputIterator,
2197	2092	class _Hash,
2198	2093	class _Allocator,
2199	2094	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
2200		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2095	class = enable_if_t<!__is_allocator_v<_Hash>>,
2201	2096	class = enable_if_t<!is_integral<_Hash>::value>,
2202		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2097	class = enable_if_t<__is_allocator_v<_Allocator>>>
2203	2098	unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
2204	2099	-> unordered_multimap<__iter_key_type<_InputIterator>,
2205	2100	__iter_mapped_type<_InputIterator>,
...	...	@@ -2209,7 +2104,7 @@ unordered_multimap(_InputIterator, _InputIterator, typename allocator_traits<_Al
2209	2104
2210	2105	# if _LIBCPP_STD_VER >= 23
2211	2106
2212		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	2107	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
2213	2108	unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
2214	2109	-> unordered_multimap<__range_key_type<_Range>,
2215	2110	__range_mapped_type<_Range>,
...	...	@@ -2217,7 +2112,7 @@ unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>
2217	2112	equal_to<__range_key_type<_Range>>,
2218	2113	_Allocator>;
2219	2114
2220		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	2115	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
2221	2116	unordered_multimap(from_range_t, _Range&&, _Allocator)
2222	2117	-> unordered_multimap<__range_key_type<_Range>,
2223	2118	__range_mapped_type<_Range>,
...	...	@@ -2228,9 +2123,9 @@ unordered_multimap(from_range_t, _Range&&, _Allocator)
2228	2123	template <ranges::input_range _Range,
2229	2124	class _Hash,
2230	2125	class _Allocator,
2231		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2126	class = enable_if_t<!__is_allocator_v<_Hash>>,
2232	2127	class = enable_if_t<!is_integral<_Hash>::value>,
2233		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2128	class = enable_if_t<__is_allocator_v<_Allocator>>>
2234	2129	unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
2235	2130	-> unordered_multimap<__range_key_type<_Range>,
2236	2131	__range_mapped_type<_Range>,
...	...	@@ -2240,7 +2135,7 @@ unordered_multimap(from_range_t, _Range&&, typename allocator_traits<_Allocator>
2240	2135
2241	2136	# endif
2242	2137
2243		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	2138	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
2244	2139	unordered_multimap(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Allocator)
2245	2140	-> unordered_multimap<remove_const_t<_Key>,
2246	2141	_Tp,
...	...	@@ -2248,7 +2143,7 @@ unordered_multimap(initializer_list<pair<_Key, _Tp>>, typename allocator_traits<
2248	2143	equal_to<remove_const_t<_Key>>,
2249	2144	_Allocator>;
2250	2145
2251		template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	2146	template <class _Key, class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
2252	2147	unordered_multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
2253	2148	-> unordered_multimap<remove_const_t<_Key>,
2254	2149	_Tp,
...	...	@@ -2260,9 +2155,9 @@ template <class _Key,
2260	2155	class _Tp,
2261	2156	class _Hash,
2262	2157	class _Allocator,
2263		class = enable_if_t<!__is_allocator<_Hash>::value>,
	2158	class = enable_if_t<!__is_allocator_v<_Hash>>,
2264	2159	class = enable_if_t<!is_integral<_Hash>::value>,
2265		class = enable_if_t<__is_allocator<_Allocator>::value>>
	2160	class = enable_if_t<__is_allocator_v<_Allocator>>>
2266	2161	unordered_multimap(
2267	2162	initializer_list<pair<_Key, _Tp>>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
2268	2163	-> unordered_multimap<remove_const_t<_Key>, _Tp, _Hash, equal_to<remove_const_t<_Key>>, _Allocator>;
...	...	@@ -2315,13 +2210,6 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2315	2210	inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const allocator_type& __a)
2316	2211	: __table_(typename __table::allocator_type(__a)) {}
2317	2212
2318		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2319		unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const unordered_multimap& __u)
2320		: __table_(__u.__table_) {
2321		__table_.__rehash_multi(__u.bucket_count());
2322		insert(__u.begin(), __u.end());
2323		}
2324
2325	2213	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2326	2214	unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
2327	2215	const unordered_multimap& __u, const allocator_type& __a)
...	...	@@ -2332,11 +2220,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
2332	2220
2333	2221	# ifndef _LIBCPP_CXX03_LANG
2334	2222
2335		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2336		inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
2337		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
2338		: __table_(std::move(__u.__table_)) {}
2339
2340	2223	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2341	2224	unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
2342	2225	unordered_multimap&& __u, const allocator_type& __a)
...	...	@@ -2373,14 +2256,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
2373	2256	insert(__il.begin(), __il.end());
2374	2257	}
2375	2258
2376		template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2377		inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
2378		unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_multimap&& __u)
2379		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
2380		__table_ = std::move(__u.__table_);
2381		return *this;
2382		}
2383
2384	2259	template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2385	2260	inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
2386	2261	unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {

lib/libcxx/include/unordered_set+168-195

...	...	@@ -544,8 +544,6 @@ template <class Value, class Hash, class Pred, class Alloc>
544	544	# include <__iterator/distance.h>
545	545	# include <__iterator/erase_if_container.h>
546	546	# include <__iterator/iterator_traits.h>
547		# include <__iterator/ranges_iterator_traits.h>
548		# include <__memory/addressof.h>
549	547	# include <__memory/allocator.h>
550	548	# include <__memory/allocator_traits.h>
551	549	# include <__memory_resource/polymorphic_allocator.h>
...	...	@@ -558,7 +556,6 @@ template <class Value, class Hash, class Pred, class Alloc>
558	556	# include <__type_traits/invoke.h>
559	557	# include <__type_traits/is_allocator.h>
560	558	# include <__type_traits/is_integral.h>
561		# include <__type_traits/is_nothrow_assignable.h>
562	559	# include <__type_traits/is_nothrow_constructible.h>
563	560	# include <__type_traits/is_same.h>
564	561	# include <__type_traits/is_swappable.h>
...	...	@@ -703,10 +700,10 @@ public:
703	700	# endif
704	701
705	702	_LIBCPP_HIDE_FROM_ABI explicit unordered_set(const allocator_type& __a);
706		_LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u);
	703	_LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u) = default;
707	704	_LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u, const allocator_type& __a);
708	705	# ifndef _LIBCPP_CXX03_LANG
709		_LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
	706	_LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) = default;
710	707	_LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u, const allocator_type& __a);
711	708	_LIBCPP_HIDE_FROM_ABI unordered_set(initializer_list<value_type> __il);
712	709	_LIBCPP_HIDE_FROM_ABI
...	...	@@ -733,30 +730,26 @@ public:
733	730	static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
734	731	}
735	732
736		_LIBCPP_HIDE_FROM_ABI unordered_set& operator=(const unordered_set& __u) {
737		__table_ = __u.__table_;
738		return *this;
739		}
	733	_LIBCPP_HIDE_FROM_ABI unordered_set& operator=(const unordered_set& __u) = default;
740	734	# ifndef _LIBCPP_CXX03_LANG
741		_LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u)
742		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
	735	_LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u) = default;
743	736	_LIBCPP_HIDE_FROM_ABI unordered_set& operator=(initializer_list<value_type> __il);
744	737	# endif // _LIBCPP_CXX03_LANG
745	738
746		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	739	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
747	740	return allocator_type(__table_.__node_alloc());
748	741	}
749	742
750	743	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
751		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
752		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
	744	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
	745	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
753	746
754		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
755		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
756		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
757		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
758		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
759		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
	747	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
	748	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
	749	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
	750	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
	751	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
	752	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
760	753
761	754	# ifndef _LIBCPP_CXX03_LANG
762	755	template <class... _Args>
...	...	@@ -808,10 +801,10 @@ public:
808	801	"node_type with incompatible allocator passed to unordered_set::insert()");
809	802	return __table_.template __node_handle_insert_unique<node_type>(__h, std::move(__nh));
810	803	}
811		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	804	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
812	805	return __table_.template __node_handle_extract<node_type>(__key);
813	806	}
814		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
	807	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __it) {
815	808	return __table_.template __node_handle_extract<node_type>(__it);
816	809	}
817	810
...	...	@@ -845,71 +838,81 @@ public:
845	838	__table_.swap(__u.__table_);
846	839	}
847	840
848		_LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
849		_LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
	841	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
	842	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
850	843
851		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
852		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
	844	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
	845	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
853	846	# if _LIBCPP_STD_VER >= 20
854	847	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
855		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	848	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
856	849	return __table_.find(__k);
857	850	}
858	851	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
859		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	852	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
860	853	return __table_.find(__k);
861	854	}
862	855	# endif // _LIBCPP_STD_VER >= 20
863	856
864		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_unique(__k); }
	857	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	858	return __table_.__count_unique(__k);
	859	}
865	860	# if _LIBCPP_STD_VER >= 20
866	861	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
867		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	862	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
868	863	return __table_.__count_unique(__k);
869	864	}
870	865	# endif // _LIBCPP_STD_VER >= 20
871	866
872	867	# if _LIBCPP_STD_VER >= 20
873		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	868	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
874	869
875	870	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
876		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	871	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
877	872	return find(__k) != end();
878	873	}
879	874	# endif // _LIBCPP_STD_VER >= 20
880	875
881		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	876	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
882	877	return __table_.__equal_range_unique(__k);
883	878	}
884		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	879	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
885	880	return __table_.__equal_range_unique(__k);
886	881	}
887	882	# if _LIBCPP_STD_VER >= 20
888	883	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
889		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	884	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
890	885	return __table_.__equal_range_unique(__k);
891	886	}
892	887	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
893		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	888	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
894	889	return __table_.__equal_range_unique(__k);
895	890	}
896	891	# endif // _LIBCPP_STD_VER >= 20
897	892
898		_LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
899		_LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
	893	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
	894	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT {
	895	return __table_.max_bucket_count();
	896	}
900	897
901		_LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
902		_LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
	898	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const {
	899	return __table_.bucket_size(__n);
	900	}
	901	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
903	902
904		_LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
905		_LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
906		_LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
907		_LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
908		_LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
909		_LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
	903	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
	904	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
	905	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const {
	906	return __table_.cbegin(__n);
	907	}
	908	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
	909	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const {
	910	return __table_.cbegin(__n);
	911	}
	912	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
910	913
911		_LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
912		_LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
	914	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
	915	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
913	916	_LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
914	917	_LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_unique(__n); }
915	918	_LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_unique(__n); }
...	...	@@ -917,47 +920,48 @@ public:
917	920
918	921	# if _LIBCPP_STD_VER >= 17
919	922	template <class _InputIterator,
920		class _Hash = hash<__iter_value_type<_InputIterator>>,
921		class _Pred = equal_to<__iter_value_type<_InputIterator>>,
922		class _Allocator = allocator<__iter_value_type<_InputIterator>>,
	923	class _Hash = hash<__iterator_value_type<_InputIterator>>,
	924	class _Pred = equal_to<__iterator_value_type<_InputIterator>>,
	925	class _Allocator = allocator<__iterator_value_type<_InputIterator>>,
923	926	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
924		class = enable_if_t<!__is_allocator<_Hash>::value>,
	927	class = enable_if_t<!__is_allocator_v<_Hash>>,
925	928	class = enable_if_t<!is_integral<_Hash>::value>,
926		class = enable_if_t<!__is_allocator<_Pred>::value>,
927		class = enable_if_t<__is_allocator<_Allocator>::value>>
	929	class = enable_if_t<!__is_allocator_v<_Pred>>,
	930	class = enable_if_t<__is_allocator_v<_Allocator>>>
928	931	unordered_set(_InputIterator,
929	932	_InputIterator,
930	933	typename allocator_traits<_Allocator>::size_type = 0,
931	934	_Hash = _Hash(),
932	935	_Pred = _Pred(),
933		_Allocator = _Allocator()) -> unordered_set<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
	936	_Allocator = _Allocator())
	937	-> unordered_set<__iterator_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
934	938
935	939	# if _LIBCPP_STD_VER >= 23
936	940	template <ranges::input_range _Range,
937	941	class _Hash = hash<ranges::range_value_t<_Range>>,
938	942	class _Pred = equal_to<ranges::range_value_t<_Range>>,
939	943	class _Allocator = allocator<ranges::range_value_t<_Range>>,
940		class = enable_if_t<!__is_allocator<_Hash>::value>,
	944	class = enable_if_t<!__is_allocator_v<_Hash>>,
941	945	class = enable_if_t<!is_integral<_Hash>::value>,
942		class = enable_if_t<!__is_allocator<_Pred>::value>,
943		class = enable_if_t<__is_allocator<_Allocator>::value>>
944		unordered_set(
945		from_range_t,
946		_Range&&,
947		typename allocator_traits<_Allocator>::size_type = 0,
948		_Hash = _Hash(),
949		_Pred = _Pred(),
950		_Allocator = _Allocator()) -> unordered_set<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
	946	class = enable_if_t<!__is_allocator_v<_Pred>>,
	947	class = enable_if_t<__is_allocator_v<_Allocator>>>
	948	unordered_set(from_range_t,
	949	_Range&&,
	950	typename allocator_traits<_Allocator>::size_type = 0,
	951	_Hash = _Hash(),
	952	_Pred = _Pred(),
	953	_Allocator = _Allocator())
	954	-> unordered_set<ranges::range_value_t<_Range>, _Hash, _Pred, _Allocator>; // C++23
951	955	# endif
952	956
953	957	template <class _Tp,
954	958	class _Hash = hash<_Tp>,
955	959	class _Pred = equal_to<_Tp>,
956	960	class _Allocator = allocator<_Tp>,
957		class = enable_if_t<!__is_allocator<_Hash>::value>,
	961	class = enable_if_t<!__is_allocator_v<_Hash>>,
958	962	class = enable_if_t<!is_integral<_Hash>::value>,
959		class = enable_if_t<!__is_allocator<_Pred>::value>,
960		class = enable_if_t<__is_allocator<_Allocator>::value>>
	963	class = enable_if_t<!__is_allocator_v<_Pred>>,
	964	class = enable_if_t<__is_allocator_v<_Allocator>>>
961	965	unordered_set(initializer_list<_Tp>,
962	966	typename allocator_traits<_Allocator>::size_type = 0,
963	967	_Hash = _Hash(),
...	...	@@ -967,33 +971,36 @@ unordered_set(initializer_list<_Tp>,
967	971	template <class _InputIterator,
968	972	class _Allocator,
969	973	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
970		class = enable_if_t<__is_allocator<_Allocator>::value>>
	974	class = enable_if_t<__is_allocator_v<_Allocator>>>
971	975	unordered_set(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
972		-> unordered_set<__iter_value_type<_InputIterator>,
973		hash<__iter_value_type<_InputIterator>>,
974		equal_to<__iter_value_type<_InputIterator>>,
	976	-> unordered_set<__iterator_value_type<_InputIterator>,
	977	hash<__iterator_value_type<_InputIterator>>,
	978	equal_to<__iterator_value_type<_InputIterator>>,
975	979	_Allocator>;
976	980
977	981	template <class _InputIterator,
978	982	class _Hash,
979	983	class _Allocator,
980	984	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
981		class = enable_if_t<!__is_allocator<_Hash>::value>,
	985	class = enable_if_t<!__is_allocator_v<_Hash>>,
982	986	class = enable_if_t<!is_integral<_Hash>::value>,
983		class = enable_if_t<__is_allocator<_Allocator>::value>>
	987	class = enable_if_t<__is_allocator_v<_Allocator>>>
984	988	unordered_set(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
985		-> unordered_set<__iter_value_type<_InputIterator>, _Hash, equal_to<__iter_value_type<_InputIterator>>, _Allocator>;
	989	-> unordered_set<__iterator_value_type<_InputIterator>,
	990	_Hash,
	991	equal_to<__iterator_value_type<_InputIterator>>,
	992	_Allocator>;
986	993
987	994	# if _LIBCPP_STD_VER >= 23
988	995
989		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	996	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
990	997	unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
991	998	-> unordered_set<ranges::range_value_t<_Range>,
992	999	hash<ranges::range_value_t<_Range>>,
993	1000	equal_to<ranges::range_value_t<_Range>>,
994	1001	_Allocator>;
995	1002
996		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1003	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
997	1004	unordered_set(from_range_t, _Range&&, _Allocator)
998	1005	-> unordered_set<ranges::range_value_t<_Range>,
999	1006	hash<ranges::range_value_t<_Range>>,
...	...	@@ -1003,24 +1010,24 @@ unordered_set(from_range_t, _Range&&, _Allocator)
1003	1010	template <ranges::input_range _Range,
1004	1011	class _Hash,
1005	1012	class _Allocator,
1006		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1013	class = enable_if_t<!__is_allocator_v<_Hash>>,
1007	1014	class = enable_if_t<!is_integral<_Hash>::value>,
1008		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1015	class = enable_if_t<__is_allocator_v<_Allocator>>>
1009	1016	unordered_set(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1010	1017	-> unordered_set<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
1011	1018
1012	1019	# endif
1013	1020
1014		template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1021	template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1015	1022	unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
1016	1023	-> unordered_set<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
1017	1024
1018	1025	template <class _Tp,
1019	1026	class _Hash,
1020	1027	class _Allocator,
1021		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1028	class = enable_if_t<!__is_allocator_v<_Hash>>,
1022	1029	class = enable_if_t<!is_integral<_Hash>::value>,
1023		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1030	class = enable_if_t<__is_allocator_v<_Allocator>>>
1024	1031	unordered_set(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1025	1032	-> unordered_set<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
1026	1033	# endif
...	...	@@ -1070,12 +1077,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
1070	1077	template <class _Value, class _Hash, class _Pred, class _Alloc>
1071	1078	inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const allocator_type& __a) : __table_(__a) {}
1072	1079
1073		template <class _Value, class _Hash, class _Pred, class _Alloc>
1074		unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u) : __table_(__u.__table_) {
1075		__table_.__rehash_unique(__u.bucket_count());
1076		insert(__u.begin(), __u.end());
1077		}
1078
1079	1080	template <class _Value, class _Hash, class _Pred, class _Alloc>
1080	1081	unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u, const allocator_type& __a)
1081	1082	: __table_(__u.__table_, __a) {
...	...	@@ -1085,11 +1086,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set&
1085	1086
1086	1087	# ifndef _LIBCPP_CXX03_LANG
1087	1088
1088		template <class _Value, class _Hash, class _Pred, class _Alloc>
1089		inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u)
1090		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
1091		: __table_(std::move(__u.__table_)) {}
1092
1093	1089	template <class _Value, class _Hash, class _Pred, class _Alloc>
1094	1090	unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u, const allocator_type& __a)
1095	1091	: __table_(std::move(__u.__table_), __a) {
...	...	@@ -1125,14 +1121,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
1125	1121	insert(__il.begin(), __il.end());
1126	1122	}
1127	1123
1128		template <class _Value, class _Hash, class _Pred, class _Alloc>
1129		inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
1130		unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_set&& __u)
1131		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
1132		__table_ = std::move(__u.__table_);
1133		return *this;
1134		}
1135
1136	1124	template <class _Value, class _Hash, class _Pred, class _Alloc>
1137	1125	inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
1138	1126	unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
...	...	@@ -1308,11 +1296,10 @@ public:
1308	1296	# endif
1309	1297
1310	1298	_LIBCPP_HIDE_FROM_ABI explicit unordered_multiset(const allocator_type& __a);
1311		_LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u);
	1299	_LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u) = default;
1312	1300	_LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u, const allocator_type& __a);
1313	1301	# ifndef _LIBCPP_CXX03_LANG
1314		_LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u)
1315		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
	1302	_LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u) = default;
1316	1303	_LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u, const allocator_type& __a);
1317	1304	_LIBCPP_HIDE_FROM_ABI unordered_multiset(initializer_list<value_type> __il);
1318	1305	_LIBCPP_HIDE_FROM_ABI unordered_multiset(
...	...	@@ -1339,30 +1326,26 @@ public:
1339	1326	static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
1340	1327	}
1341	1328
1342		_LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(const unordered_multiset& __u) {
1343		__table_ = __u.__table_;
1344		return *this;
1345		}
	1329	_LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(const unordered_multiset& __u) = default;
1346	1330	# ifndef _LIBCPP_CXX03_LANG
1347		_LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u)
1348		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
	1331	_LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u) = default;
1349	1332	_LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(initializer_list<value_type> __il);
1350	1333	# endif // _LIBCPP_CXX03_LANG
1351	1334
1352		_LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
	1335	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT {
1353	1336	return allocator_type(__table_.__node_alloc());
1354	1337	}
1355	1338
1356	1339	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __table_.size() == 0; }
1357		_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
1358		_LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
	1340	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __table_.size(); }
	1341	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT { return __table_.max_size(); }
1359	1342
1360		_LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
1361		_LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
1362		_LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
1363		_LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
1364		_LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
1365		_LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
	1343	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __table_.begin(); }
	1344	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __table_.end(); }
	1345	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __table_.begin(); }
	1346	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __table_.end(); }
	1347	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __table_.begin(); }
	1348	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __table_.end(); }
1366	1349
1367	1350	# ifndef _LIBCPP_CXX03_LANG
1368	1351	template <class... _Args>
...	...	@@ -1410,10 +1393,10 @@ public:
1410	1393	"node_type with incompatible allocator passed to unordered_multiset::insert()");
1411	1394	return __table_.template __node_handle_insert_multi<node_type>(__hint, std::move(__nh));
1412	1395	}
1413		_LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __position) {
	1396	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(const_iterator __position) {
1414	1397	return __table_.template __node_handle_extract<node_type>(__position);
1415	1398	}
1416		_LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
	1399	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI node_type extract(key_type const& __key) {
1417	1400	return __table_.template __node_handle_extract<node_type>(__key);
1418	1401	}
1419	1402
...	...	@@ -1454,71 +1437,81 @@ public:
1454	1437	__table_.swap(__u.__table_);
1455	1438	}
1456	1439
1457		_LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
1458		_LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
	1440	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI hasher hash_function() const { return __table_.hash_function(); }
	1441	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI key_equal key_eq() const { return __table_.key_eq(); }
1459	1442
1460		_LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
1461		_LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
	1443	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __k) { return __table_.find(__k); }
	1444	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __k) const { return __table_.find(__k); }
1462	1445	# if _LIBCPP_STD_VER >= 20
1463	1446	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1464		_LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
	1447	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI iterator find(const _K2& __k) {
1465	1448	return __table_.find(__k);
1466	1449	}
1467	1450	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1468		_LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
	1451	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_iterator find(const _K2& __k) const {
1469	1452	return __table_.find(__k);
1470	1453	}
1471	1454	# endif // _LIBCPP_STD_VER >= 20
1472	1455
1473		_LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const { return __table_.__count_multi(__k); }
	1456	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __k) const {
	1457	return __table_.__count_multi(__k);
	1458	}
1474	1459	# if _LIBCPP_STD_VER >= 20
1475	1460	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1476		_LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
	1461	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type count(const _K2& __k) const {
1477	1462	return __table_.__count_multi(__k);
1478	1463	}
1479	1464	# endif // _LIBCPP_STD_VER >= 20
1480	1465
1481	1466	# if _LIBCPP_STD_VER >= 20
1482		_LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
	1467	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __k) const { return find(__k) != end(); }
1483	1468
1484	1469	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1485		_LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
	1470	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool contains(const _K2& __k) const {
1486	1471	return find(__k) != end();
1487	1472	}
1488	1473	# endif // _LIBCPP_STD_VER >= 20
1489	1474
1490		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
	1475	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __k) {
1491	1476	return __table_.__equal_range_multi(__k);
1492	1477	}
1493		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
	1478	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __k) const {
1494	1479	return __table_.__equal_range_multi(__k);
1495	1480	}
1496	1481	# if _LIBCPP_STD_VER >= 20
1497	1482	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1498		_LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
	1483	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _K2& __k) {
1499	1484	return __table_.__equal_range_multi(__k);
1500	1485	}
1501	1486	template <class _K2, enable_if_t<__is_transparent_v<hasher, _K2> && __is_transparent_v<key_equal, _K2>>* = nullptr>
1502		_LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
	1487	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _K2& __k) const {
1503	1488	return __table_.__equal_range_multi(__k);
1504	1489	}
1505	1490	# endif // _LIBCPP_STD_VER >= 20
1506	1491
1507		_LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
1508		_LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT { return __table_.max_bucket_count(); }
	1492	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_count() const _NOEXCEPT { return __table_.bucket_count(); }
	1493	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type max_bucket_count() const _NOEXCEPT {
	1494	return __table_.max_bucket_count();
	1495	}
1509	1496
1510		_LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const { return __table_.bucket_size(__n); }
1511		_LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
	1497	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket_size(size_type __n) const {
	1498	return __table_.bucket_size(__n);
	1499	}
	1500	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_type bucket(const key_type& __k) const { return __table_.bucket(__k); }
1512	1501
1513		_LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
1514		_LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
1515		_LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const { return __table_.cbegin(__n); }
1516		_LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
1517		_LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const { return __table_.cbegin(__n); }
1518		_LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
	1502	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator begin(size_type __n) { return __table_.begin(__n); }
	1503	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI local_iterator end(size_type __n) { return __table_.end(__n); }
	1504	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator begin(size_type __n) const {
	1505	return __table_.cbegin(__n);
	1506	}
	1507	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator end(size_type __n) const { return __table_.cend(__n); }
	1508	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cbegin(size_type __n) const {
	1509	return __table_.cbegin(__n);
	1510	}
	1511	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const_local_iterator cend(size_type __n) const { return __table_.cend(__n); }
1519	1512
1520		_LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
1521		_LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
	1513	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float load_factor() const _NOEXCEPT { return __table_.load_factor(); }
	1514	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI float max_load_factor() const _NOEXCEPT { return __table_.max_load_factor(); }
1522	1515	_LIBCPP_HIDE_FROM_ABI void max_load_factor(float __mlf) { __table_.max_load_factor(__mlf); }
1523	1516	_LIBCPP_HIDE_FROM_ABI void rehash(size_type __n) { __table_.__rehash_multi(__n); }
1524	1517	_LIBCPP_HIDE_FROM_ABI void reserve(size_type __n) { __table_.__reserve_multi(__n); }
...	...	@@ -1526,31 +1519,31 @@ public:
1526	1519
1527	1520	# if _LIBCPP_STD_VER >= 17
1528	1521	template <class _InputIterator,
1529		class _Hash = hash<__iter_value_type<_InputIterator>>,
1530		class _Pred = equal_to<__iter_value_type<_InputIterator>>,
1531		class _Allocator = allocator<__iter_value_type<_InputIterator>>,
	1522	class _Hash = hash<__iterator_value_type<_InputIterator>>,
	1523	class _Pred = equal_to<__iterator_value_type<_InputIterator>>,
	1524	class _Allocator = allocator<__iterator_value_type<_InputIterator>>,
1532	1525	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1533		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1526	class = enable_if_t<!__is_allocator_v<_Hash>>,
1534	1527	class = enable_if_t<!is_integral<_Hash>::value>,
1535		class = enable_if_t<!__is_allocator<_Pred>::value>,
1536		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1528	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1529	class = enable_if_t<__is_allocator_v<_Allocator>>>
1537	1530	unordered_multiset(
1538	1531	_InputIterator,
1539	1532	_InputIterator,
1540	1533	typename allocator_traits<_Allocator>::size_type = 0,
1541	1534	_Hash = _Hash(),
1542	1535	_Pred = _Pred(),
1543		_Allocator = _Allocator()) -> unordered_multiset<__iter_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
	1536	_Allocator = _Allocator()) -> unordered_multiset<__iterator_value_type<_InputIterator>, _Hash, _Pred, _Allocator>;
1544	1537
1545	1538	# if _LIBCPP_STD_VER >= 23
1546	1539	template <ranges::input_range _Range,
1547	1540	class _Hash = hash<ranges::range_value_t<_Range>>,
1548	1541	class _Pred = equal_to<ranges::range_value_t<_Range>>,
1549	1542	class _Allocator = allocator<ranges::range_value_t<_Range>>,
1550		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1543	class = enable_if_t<!__is_allocator_v<_Hash>>,
1551	1544	class = enable_if_t<!is_integral<_Hash>::value>,
1552		class = enable_if_t<!__is_allocator<_Pred>::value>,
1553		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1545	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1546	class = enable_if_t<__is_allocator_v<_Allocator>>>
1554	1547	unordered_multiset(
1555	1548	from_range_t,
1556	1549	_Range&&,
...	...	@@ -1564,10 +1557,10 @@ template <class _Tp,
1564	1557	class _Hash = hash<_Tp>,
1565	1558	class _Pred = equal_to<_Tp>,
1566	1559	class _Allocator = allocator<_Tp>,
1567		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1560	class = enable_if_t<!__is_allocator_v<_Hash>>,
1568	1561	class = enable_if_t<!is_integral<_Hash>::value>,
1569		class = enable_if_t<!__is_allocator<_Pred>::value>,
1570		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1562	class = enable_if_t<!__is_allocator_v<_Pred>>,
	1563	class = enable_if_t<__is_allocator_v<_Allocator>>>
1571	1564	unordered_multiset(initializer_list<_Tp>,
1572	1565	typename allocator_traits<_Allocator>::size_type = 0,
1573	1566	_Hash = _Hash(),
...	...	@@ -1577,36 +1570,36 @@ unordered_multiset(initializer_list<_Tp>,
1577	1570	template <class _InputIterator,
1578	1571	class _Allocator,
1579	1572	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1580		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1573	class = enable_if_t<__is_allocator_v<_Allocator>>>
1581	1574	unordered_multiset(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Allocator)
1582		-> unordered_multiset<__iter_value_type<_InputIterator>,
1583		hash<__iter_value_type<_InputIterator>>,
1584		equal_to<__iter_value_type<_InputIterator>>,
	1575	-> unordered_multiset<__iterator_value_type<_InputIterator>,
	1576	hash<__iterator_value_type<_InputIterator>>,
	1577	equal_to<__iterator_value_type<_InputIterator>>,
1585	1578	_Allocator>;
1586	1579
1587	1580	template <class _InputIterator,
1588	1581	class _Hash,
1589	1582	class _Allocator,
1590	1583	class = enable_if_t<__has_input_iterator_category<_InputIterator>::value>,
1591		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1584	class = enable_if_t<!__is_allocator_v<_Hash>>,
1592	1585	class = enable_if_t<!is_integral<_Hash>::value>,
1593		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1586	class = enable_if_t<__is_allocator_v<_Allocator>>>
1594	1587	unordered_multiset(_InputIterator, _InputIterator, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1595		-> unordered_multiset<__iter_value_type<_InputIterator>,
	1588	-> unordered_multiset<__iterator_value_type<_InputIterator>,
1596	1589	_Hash,
1597		equal_to<__iter_value_type<_InputIterator>>,
	1590	equal_to<__iterator_value_type<_InputIterator>>,
1598	1591	_Allocator>;
1599	1592
1600	1593	# if _LIBCPP_STD_VER >= 23
1601	1594
1602		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1595	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1603	1596	unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Allocator)
1604	1597	-> unordered_multiset<ranges::range_value_t<_Range>,
1605	1598	hash<ranges::range_value_t<_Range>>,
1606	1599	equal_to<ranges::range_value_t<_Range>>,
1607	1600	_Allocator>;
1608	1601
1609		template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1602	template <ranges::input_range _Range, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1610	1603	unordered_multiset(from_range_t, _Range&&, _Allocator)
1611	1604	-> unordered_multiset<ranges::range_value_t<_Range>,
1612	1605	hash<ranges::range_value_t<_Range>>,
...	...	@@ -1616,24 +1609,24 @@ unordered_multiset(from_range_t, _Range&&, _Allocator)
1616	1609	template <ranges::input_range _Range,
1617	1610	class _Hash,
1618	1611	class _Allocator,
1619		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1612	class = enable_if_t<!__is_allocator_v<_Hash>>,
1620	1613	class = enable_if_t<!is_integral<_Hash>::value>,
1621		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1614	class = enable_if_t<__is_allocator_v<_Allocator>>>
1622	1615	unordered_multiset(from_range_t, _Range&&, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1623	1616	-> unordered_multiset<ranges::range_value_t<_Range>, _Hash, equal_to<ranges::range_value_t<_Range>>, _Allocator>;
1624	1617
1625	1618	# endif
1626	1619
1627		template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator<_Allocator>::value>>
	1620	template <class _Tp, class _Allocator, class = enable_if_t<__is_allocator_v<_Allocator>>>
1628	1621	unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Allocator)
1629	1622	-> unordered_multiset<_Tp, hash<_Tp>, equal_to<_Tp>, _Allocator>;
1630	1623
1631	1624	template <class _Tp,
1632	1625	class _Hash,
1633	1626	class _Allocator,
1634		class = enable_if_t<!__is_allocator<_Hash>::value>,
	1627	class = enable_if_t<!__is_allocator_v<_Hash>>,
1635	1628	class = enable_if_t<!is_integral<_Hash>::value>,
1636		class = enable_if_t<__is_allocator<_Allocator>::value>>
	1629	class = enable_if_t<__is_allocator_v<_Allocator>>>
1637	1630	unordered_multiset(initializer_list<_Tp>, typename allocator_traits<_Allocator>::size_type, _Hash, _Allocator)
1638	1631	-> unordered_multiset<_Tp, _Hash, equal_to<_Tp>, _Allocator>;
1639	1632	# endif
...	...	@@ -1685,13 +1678,6 @@ template <class _Value, class _Hash, class _Pred, class _Alloc>
1685	1678	inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const allocator_type& __a)
1686	1679	: __table_(__a) {}
1687	1680
1688		template <class _Value, class _Hash, class _Pred, class _Alloc>
1689		unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const unordered_multiset& __u)
1690		: __table_(__u.__table_) {
1691		__table_.__rehash_multi(__u.bucket_count());
1692		insert(__u.begin(), __u.end());
1693		}
1694
1695	1681	template <class _Value, class _Hash, class _Pred, class _Alloc>
1696	1682	unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
1697	1683	const unordered_multiset& __u, const allocator_type& __a)
...	...	@@ -1702,11 +1688,6 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
1702	1688
1703	1689	# ifndef _LIBCPP_CXX03_LANG
1704	1690
1705		template <class _Value, class _Hash, class _Pred, class _Alloc>
1706		inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(unordered_multiset&& __u)
1707		_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
1708		: __table_(std::move(__u.__table_)) {}
1709
1710	1691	template <class _Value, class _Hash, class _Pred, class _Alloc>
1711	1692	unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
1712	1693	unordered_multiset&& __u, const allocator_type& __a)
...	...	@@ -1743,14 +1724,6 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
1743	1724	insert(__il.begin(), __il.end());
1744	1725	}
1745	1726
1746		template <class _Value, class _Hash, class _Pred, class _Alloc>
1747		inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
1748		unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_multiset&& __u)
1749		_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
1750		__table_ = std::move(__u.__table_);
1751		return *this;
1752		}
1753
1754	1727	template <class _Value, class _Hash, class _Pred, class _Alloc>
1755	1728	inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
1756	1729	unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {

lib/libcxx/include/utility+12

...	...	@@ -216,6 +216,18 @@ template<size_t N>
216	216	template<class... T>
217	217	using index_sequence_for = make_index_sequence<sizeof...(T)>;
218	218
	219	template<class T, T... Values> // C++26
	220	struct tuple_size<integer_sequence<T, Values...>>;
	221
	222	template<size_t I, class T, T... Values> // C++26
	223	struct tuple_element<I, integer_sequence<T, Values...>>;
	224
	225	template<size_t I, class T, T... Values> // C++26
	226	struct tuple_element<I, const integer_sequence<T, Values...>>;
	227
	228	template<size_t I, class T, T... Values> // C++26
	229	constexpr T get(integer_sequence<T, Values...>) noexcept;
	230
219	231	template<class T, class U=T>
220	232	constexpr T exchange(T& obj, U&& new_value) // constexpr in C++17, noexcept in C++23
221	233	noexcept(is_nothrow_move_constructible<T>::value && is_nothrow_assignable<T&, U>::value);

lib/libcxx/include/valarray+106-161

...	...	@@ -362,6 +362,7 @@ template <class T> unspecified2 end(const valarray<T>& v);
362	362	# include <__memory/uninitialized_algorithms.h>
363	363	# include <__type_traits/decay.h>
364	364	# include <__type_traits/remove_reference.h>
	365	# include <__utility/exception_guard.h>
365	366	# include <__utility/move.h>
366	367	# include <__utility/swap.h>
367	368	# include <cmath>
...	...	@@ -395,9 +396,9 @@ public:
395	396	_LIBCPP_HIDE_FROM_ABI slice(size_t __start, size_t __size, size_t __stride)
396	397	: __start_(__start), __size_(__size), __stride_(__stride) {}
397	398
398		_LIBCPP_HIDE_FROM_ABI size_t start() const { return __start_; }
399		_LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
400		_LIBCPP_HIDE_FROM_ABI size_t stride() const { return __stride_; }
	399	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t start() const { return __start_; }
	400	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t size() const { return __size_; }
	401	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t stride() const { return __stride_; }
401	402
402	403	# if _LIBCPP_STD_VER >= 20
403	404
...	...	@@ -792,7 +793,7 @@ private:
792	793	public:
793	794	// construct/destroy:
794	795	_LIBCPP_HIDE_FROM_ABI valarray() : __begin_(nullptr), __end_(nullptr) {}
795		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 explicit valarray(size_t __n);
	796	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 explicit valarray(size_t __n);
796	797	_LIBCPP_HIDE_FROM_ABI valarray(const value_type& __x, size_t __n);
797	798	valarray(const value_type* __p, size_t __n);
798	799	valarray(const valarray& __v);
...	...	@@ -804,7 +805,7 @@ public:
804	805	valarray(const gslice_array<value_type>& __ga);
805	806	valarray(const mask_array<value_type>& __ma);
806	807	valarray(const indirect_array<value_type>& __ia);
807		inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 ~valarray();
	808	inline _LIBCPP_HIDE_FROM_ABI_SINCE_LLVM8 ~valarray();
808	809
809	810	// assignment:
810	811	valarray& operator=(const valarray& __v);
...	...	@@ -821,36 +822,41 @@ public:
821	822	_LIBCPP_HIDE_FROM_ABI valarray& operator=(const __val_expr<_ValExpr>& __v);
822	823
823	824	// element access:
824		_LIBCPP_HIDE_FROM_ABI const value_type& operator[](size_t __i) const {
	825	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI const value_type& operator[](size_t __i) const {
825	826	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < size(), "valarray::operator[] index out of bounds");
826	827	return __begin_[__i];
827	828	}
828	829
829		_LIBCPP_HIDE_FROM_ABI value_type& operator[](size_t __i) {
	830	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_type& operator[](size_t __i) {
830	831	_LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__i < size(), "valarray::operator[] index out of bounds");
831	832	return __begin_[__i];
832	833	}
833	834
834	835	// subset operations:
835		_LIBCPP_HIDE_FROM_ABI __val_expr<__slice_expr<const valarray&> > operator[](slice __s) const;
836		_LIBCPP_HIDE_FROM_ABI slice_array<value_type> operator[](slice __s);
837		_LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const gslice& __gs) const;
838		_LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](const gslice& __gs);
	836	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI __val_expr<__slice_expr<const valarray&> > operator[](slice __s) const;
	837	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI slice_array<value_type> operator[](slice __s);
	838	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> >
	839	operator[](const gslice& __gs) const;
	840	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](const gslice& __gs);
839	841	# ifndef _LIBCPP_CXX03_LANG
840		_LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](gslice&& __gs) const;
841		_LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](gslice&& __gs);
	842	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](gslice&& __gs) const;
	843	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI gslice_array<value_type> operator[](gslice&& __gs);
842	844	# endif // _LIBCPP_CXX03_LANG
	845	[[__nodiscard__]]
843	846	_LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](const valarray<bool>& __vb) const;
844		_LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](const valarray<bool>& __vb);
	847	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](const valarray<bool>& __vb);
845	848	# ifndef _LIBCPP_CXX03_LANG
	849	[[__nodiscard__]]
846	850	_LIBCPP_HIDE_FROM_ABI __val_expr<__mask_expr<const valarray&> > operator[](valarray<bool>&& __vb) const;
847		_LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](valarray<bool>&& __vb);
	851	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI mask_array<value_type> operator[](valarray<bool>&& __vb);
848	852	# endif // _LIBCPP_CXX03_LANG
	853	[[__nodiscard__]]
849	854	_LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](const valarray<size_t>& __vs) const;
850		_LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](const valarray<size_t>& __vs);
	855	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](const valarray<size_t>& __vs);
851	856	# ifndef _LIBCPP_CXX03_LANG
	857	[[__nodiscard__]]
852	858	_LIBCPP_HIDE_FROM_ABI __val_expr<__indirect_expr<const valarray&> > operator[](valarray<size_t>&& __vs) const;
853		_LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](valarray<size_t>&& __vs);
	859	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI indirect_array<value_type> operator[](valarray<size_t>&& __vs);
854	860	# endif // _LIBCPP_CXX03_LANG
855	861
856	862	// unary operators:
...	...	@@ -904,16 +910,16 @@ public:
904	910	// member functions:
905	911	_LIBCPP_HIDE_FROM_ABI void swap(valarray& __v) _NOEXCEPT;
906	912
907		_LIBCPP_HIDE_FROM_ABI size_t size() const { return static_cast<size_t>(__end_ - __begin_); }
	913	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t size() const { return static_cast<size_t>(__end_ - __begin_); }
908	914
909		_LIBCPP_HIDE_FROM_ABI value_type sum() const;
910		_LIBCPP_HIDE_FROM_ABI value_type min() const;
911		_LIBCPP_HIDE_FROM_ABI value_type max() const;
	915	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_type sum() const;
	916	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_type min() const;
	917	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI value_type max() const;
912	918
913		valarray shift(int __i) const;
914		valarray cshift(int __i) const;
915		valarray apply(value_type __f(value_type)) const;
916		valarray apply(value_type __f(const value_type&)) const;
	919	[[__nodiscard__]] valarray shift(int __i) const;
	920	[[__nodiscard__]] valarray cshift(int __i) const;
	921	[[__nodiscard__]] valarray apply(value_type __f(value_type)) const;
	922	[[__nodiscard__]] valarray apply(value_type __f(const value_type&)) const;
917	923	void resize(size_t __n, value_type __x = value_type());
918	924
919	925	private:
...	...	@@ -1248,11 +1254,11 @@ public:
1248	1254
1249	1255	# endif // _LIBCPP_CXX03_LANG
1250	1256
1251		_LIBCPP_HIDE_FROM_ABI size_t start() const { return __1d_.size() ? __1d_[0] : 0; }
	1257	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI size_t start() const { return __1d_.size() ? __1d_[0] : 0; }
1252	1258
1253		_LIBCPP_HIDE_FROM_ABI valarray<size_t> size() const { return __size_; }
	1259	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI valarray<size_t> size() const { return __size_; }
1254	1260
1255		_LIBCPP_HIDE_FROM_ABI valarray<size_t> stride() const { return __stride_; }
	1261	[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI valarray<size_t> stride() const { return __stride_; }
1256	1262
1257	1263	private:
1258	1264	void __init(size_t __start);
...	...	@@ -1992,17 +1998,10 @@ template <class _Tp>
1992	1998	inline valarray<_Tp>::valarray(size_t __n) : __begin_(nullptr), __end_(nullptr) {
1993	1999	if (__n) {
1994	2000	__begin_ = __end_ = allocator<value_type>().allocate(__n);
1995		# if _LIBCPP_HAS_EXCEPTIONS
1996		try {
1997		# endif // _LIBCPP_HAS_EXCEPTIONS
1998		for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
1999		::new ((void*)__end_) value_type();
2000		# if _LIBCPP_HAS_EXCEPTIONS
2001		} catch (...) {
2002		__clear(__n);
2003		throw;
2004		}
2005		# endif // _LIBCPP_HAS_EXCEPTIONS
	2001	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2002	for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
	2003	::new ((void*)__end_) value_type();
	2004	__guard.__complete();
2006	2005	}
2007	2006	}
2008	2007
...	...	@@ -2015,17 +2014,10 @@ template <class _Tp>
2015	2014	valarray<_Tp>::valarray(const value_type* __p, size_t __n) : __begin_(nullptr), __end_(nullptr) {
2016	2015	if (__n) {
2017	2016	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2018		# if _LIBCPP_HAS_EXCEPTIONS
2019		try {
2020		# endif // _LIBCPP_HAS_EXCEPTIONS
2021		for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
2022		::new ((void)__end_) value_type(__p);
2023		# if _LIBCPP_HAS_EXCEPTIONS
2024		} catch (...) {
2025		__clear(__n);
2026		throw;
2027		}
2028		# endif // _LIBCPP_HAS_EXCEPTIONS
	2017	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2018	for (size_t __n_left = __n; __n_left; ++__end_, ++__p, --__n_left)
	2019	::new ((void)__end_) value_type(__p);
	2020	__guard.__complete();
2029	2021	}
2030	2022	}
2031	2023
...	...	@@ -2033,17 +2025,10 @@ template <class _Tp>
2033	2025	valarray<_Tp>::valarray(const valarray& __v) : __begin_(nullptr), __end_(nullptr) {
2034	2026	if (__v.size()) {
2035	2027	__begin_ = __end_ = allocator<value_type>().allocate(__v.size());
2036		# if _LIBCPP_HAS_EXCEPTIONS
2037		try {
2038		# endif // _LIBCPP_HAS_EXCEPTIONS
2039		for (value_type* __p = __v.__begin_; __p != __v.__end_; ++__end_, ++__p)
2040		::new ((void)__end_) value_type(__p);
2041		# if _LIBCPP_HAS_EXCEPTIONS
2042		} catch (...) {
2043		__clear(__v.size());
2044		throw;
2045		}
2046		# endif // _LIBCPP_HAS_EXCEPTIONS
	2028	auto __guard = std::__make_exception_guard([&] { __clear(__v.size()); });
	2029	for (value_type* __p = __v.__begin_; __p != __v.__end_; ++__end_, ++__p)
	2030	::new ((void)__end_) value_type(__p);
	2031	__guard.__complete();
2047	2032	}
2048	2033	}
2049	2034
...	...	@@ -2059,18 +2044,11 @@ valarray<_Tp>::valarray(initializer_list<value_type> __il) : __begin_(nullptr),
2059	2044	const size_t __n = __il.size();
2060	2045	if (__n) {
2061	2046	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2062		# if _LIBCPP_HAS_EXCEPTIONS
2063		try {
2064		# endif // _LIBCPP_HAS_EXCEPTIONS
2065		size_t __n_left = __n;
2066		for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
2067		::new ((void)__end_) value_type(__p);
2068		# if _LIBCPP_HAS_EXCEPTIONS
2069		} catch (...) {
2070		__clear(__n);
2071		throw;
2072		}
2073		# endif // _LIBCPP_HAS_EXCEPTIONS
	2047	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2048	size_t __n_left = __n;
	2049	for (const value_type* __p = __il.begin(); __n_left; ++__end_, ++__p, --__n_left)
	2050	::new ((void)__end_) value_type(__p);
	2051	__guard.__complete();
2074	2052	}
2075	2053	}
2076	2054
...	...	@@ -2081,18 +2059,11 @@ valarray<_Tp>::valarray(const slice_array<value_type>& __sa) : __begin_(nullptr)
2081	2059	const size_t __n = __sa.__size_;
2082	2060	if (__n) {
2083	2061	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2084		# if _LIBCPP_HAS_EXCEPTIONS
2085		try {
2086		# endif // _LIBCPP_HAS_EXCEPTIONS
2087		size_t __n_left = __n;
2088		for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
2089		::new ((void)__end_) value_type(__p);
2090		# if _LIBCPP_HAS_EXCEPTIONS
2091		} catch (...) {
2092		__clear(__n);
2093		throw;
2094		}
2095		# endif // _LIBCPP_HAS_EXCEPTIONS
	2062	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2063	size_t __n_left = __n;
	2064	for (const value_type* __p = __sa.__vp_; __n_left; ++__end_, __p += __sa.__stride_, --__n_left)
	2065	::new ((void)__end_) value_type(__p);
	2066	__guard.__complete();
2096	2067	}
2097	2068	}
2098	2069
...	...	@@ -2101,19 +2072,12 @@ valarray<_Tp>::valarray(const gslice_array<value_type>& __ga) : __begin_(nullptr
2101	2072	const size_t __n = __ga.__1d_.size();
2102	2073	if (__n) {
2103	2074	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2104		# if _LIBCPP_HAS_EXCEPTIONS
2105		try {
2106		# endif // _LIBCPP_HAS_EXCEPTIONS
2107		typedef const size_t* _Ip;
2108		const value_type* __s = __ga.__vp_;
2109		for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__end_)
2110		::new ((void)__end_) value_type(__s[__i]);
2111		# if _LIBCPP_HAS_EXCEPTIONS
2112		} catch (...) {
2113		__clear(__n);
2114		throw;
2115		}
2116		# endif // _LIBCPP_HAS_EXCEPTIONS
	2075	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2076	typedef const size_t* _Ip;
	2077	const value_type* __s = __ga.__vp_;
	2078	for (_Ip __i = __ga.__1d_.__begin_, __e = __ga.__1d_.__end_; __i != __e; ++__i, ++__end_)
	2079	::new ((void)__end_) value_type(__s[__i]);
	2080	__guard.__complete();
2117	2081	}
2118	2082	}
2119	2083
...	...	@@ -2122,19 +2086,12 @@ valarray<_Tp>::valarray(const mask_array<value_type>& __ma) : __begin_(nullptr),
2122	2086	const size_t __n = __ma.__1d_.size();
2123	2087	if (__n) {
2124	2088	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2125		# if _LIBCPP_HAS_EXCEPTIONS
2126		try {
2127		# endif // _LIBCPP_HAS_EXCEPTIONS
2128		typedef const size_t* _Ip;
2129		const value_type* __s = __ma.__vp_;
2130		for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__end_)
2131		::new ((void)__end_) value_type(__s[__i]);
2132		# if _LIBCPP_HAS_EXCEPTIONS
2133		} catch (...) {
2134		__clear(__n);
2135		throw;
2136		}
2137		# endif // _LIBCPP_HAS_EXCEPTIONS
	2089	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2090	typedef const size_t* _Ip;
	2091	const value_type* __s = __ma.__vp_;
	2092	for (_Ip __i = __ma.__1d_.__begin_, __e = __ma.__1d_.__end_; __i != __e; ++__i, ++__end_)
	2093	::new ((void)__end_) value_type(__s[__i]);
	2094	__guard.__complete();
2138	2095	}
2139	2096	}
2140	2097
...	...	@@ -2143,19 +2100,12 @@ valarray<_Tp>::valarray(const indirect_array<value_type>& __ia) : __begin_(nullp
2143	2100	const size_t __n = __ia.__1d_.size();
2144	2101	if (__n) {
2145	2102	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2146		# if _LIBCPP_HAS_EXCEPTIONS
2147		try {
2148		# endif // _LIBCPP_HAS_EXCEPTIONS
2149		typedef const size_t* _Ip;
2150		const value_type* __s = __ia.__vp_;
2151		for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__end_)
2152		::new ((void)__end_) value_type(__s[__i]);
2153		# if _LIBCPP_HAS_EXCEPTIONS
2154		} catch (...) {
2155		__clear(__n);
2156		throw;
2157		}
2158		# endif // _LIBCPP_HAS_EXCEPTIONS
	2103	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2104	typedef const size_t* _Ip;
	2105	const value_type* __s = __ia.__vp_;
	2106	for (_Ip __i = __ia.__1d_.__begin_, __e = __ia.__1d_.__end_; __i != __e; ++__i, ++__end_)
	2107	::new ((void)__end_) value_type(__s[__i]);
	2108	__guard.__complete();
2159	2109	}
2160	2110	}
2161	2111
...	...	@@ -2644,17 +2594,10 @@ void valarray<_Tp>::resize(size_t __n, value_type __x) {
2644	2594	__clear(size());
2645	2595	if (__n) {
2646	2596	__begin_ = __end_ = allocator<value_type>().allocate(__n);
2647		# if _LIBCPP_HAS_EXCEPTIONS
2648		try {
2649		# endif // _LIBCPP_HAS_EXCEPTIONS
2650		for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
2651		::new ((void*)__end_) value_type(__x);
2652		# if _LIBCPP_HAS_EXCEPTIONS
2653		} catch (...) {
2654		__clear(__n);
2655		throw;
2656		}
2657		# endif // _LIBCPP_HAS_EXCEPTIONS
	2597	auto __guard = std::__make_exception_guard([&] { __clear(__n); });
	2598	for (size_t __n_left = __n; __n_left; --__n_left, ++__end_)
	2599	::new ((void*)__end_) value_type(__x);
	2600	__guard.__complete();
2658	2601	}
2659	2602	}
2660	2603
...	...	@@ -3168,7 +3111,7 @@ operator>=(const typename _Expr::value_type& __x, const _Expr& __y) {
3168	3111	}
3169	3112
3170	3113	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3171		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__abs_expr<typename _Expr::value_type>, _Expr> >
	3114	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__abs_expr<typename _Expr::value_type>, _Expr> >
3172	3115	abs(const _Expr& __x) {
3173	3116	typedef typename _Expr::value_type value_type;
3174	3117	typedef _UnaryOp<__abs_expr<value_type>, _Expr> _Op;
...	...	@@ -3176,7 +3119,7 @@ abs(const _Expr& __x) {
3176	3119	}
3177	3120
3178	3121	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3179		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__acos_expr<typename _Expr::value_type>, _Expr> >
	3122	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__acos_expr<typename _Expr::value_type>, _Expr> >
3180	3123	acos(const _Expr& __x) {
3181	3124	typedef typename _Expr::value_type value_type;
3182	3125	typedef _UnaryOp<__acos_expr<value_type>, _Expr> _Op;
...	...	@@ -3184,7 +3127,7 @@ acos(const _Expr& __x) {
3184	3127	}
3185	3128
3186	3129	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3187		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__asin_expr<typename _Expr::value_type>, _Expr> >
	3130	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__asin_expr<typename _Expr::value_type>, _Expr> >
3188	3131	asin(const _Expr& __x) {
3189	3132	typedef typename _Expr::value_type value_type;
3190	3133	typedef _UnaryOp<__asin_expr<value_type>, _Expr> _Op;
...	...	@@ -3192,7 +3135,7 @@ asin(const _Expr& __x) {
3192	3135	}
3193	3136
3194	3137	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3195		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__atan_expr<typename _Expr::value_type>, _Expr> >
	3138	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__atan_expr<typename _Expr::value_type>, _Expr> >
3196	3139	atan(const _Expr& __x) {
3197	3140	typedef typename _Expr::value_type value_type;
3198	3141	typedef _UnaryOp<__atan_expr<value_type>, _Expr> _Op;
...	...	@@ -3202,15 +3145,16 @@ atan(const _Expr& __x) {
3202	3145	template <class _Expr1,
3203	3146	class _Expr2,
3204	3147	__enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
3205		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__atan2_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
3206		atan2(const _Expr1& __x, const _Expr2& __y) {
	3148	[[__nodiscard__]] inline
	3149	_LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__atan2_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
	3150	atan2(const _Expr1& __x, const _Expr2& __y) {
3207	3151	typedef typename _Expr1::value_type value_type;
3208	3152	typedef _BinaryOp<__atan2_expr<value_type>, _Expr1, _Expr2> _Op;
3209	3153	return __val_expr<_Op>(_Op(__atan2_expr<value_type>(), __x, __y));
3210	3154	}
3211	3155
3212	3156	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3213		inline _LIBCPP_HIDE_FROM_ABI
	3157	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
3214	3158	__val_expr<_BinaryOp<__atan2_expr<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
3215	3159	atan2(const _Expr& __x, const typename _Expr::value_type& __y) {
3216	3160	typedef typename _Expr::value_type value_type;
...	...	@@ -3219,7 +3163,7 @@ atan2(const _Expr& __x, const typename _Expr::value_type& __y) {
3219	3163	}
3220	3164
3221	3165	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3222		inline _LIBCPP_HIDE_FROM_ABI
	3166	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
3223	3167	__val_expr<_BinaryOp<__atan2_expr<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
3224	3168	atan2(const typename _Expr::value_type& __x, const _Expr& __y) {
3225	3169	typedef typename _Expr::value_type value_type;
...	...	@@ -3228,7 +3172,7 @@ atan2(const typename _Expr::value_type& __x, const _Expr& __y) {
3228	3172	}
3229	3173
3230	3174	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3231		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cos_expr<typename _Expr::value_type>, _Expr> >
	3175	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cos_expr<typename _Expr::value_type>, _Expr> >
3232	3176	cos(const _Expr& __x) {
3233	3177	typedef typename _Expr::value_type value_type;
3234	3178	typedef _UnaryOp<__cos_expr<value_type>, _Expr> _Op;
...	...	@@ -3236,7 +3180,7 @@ cos(const _Expr& __x) {
3236	3180	}
3237	3181
3238	3182	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3239		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cosh_expr<typename _Expr::value_type>, _Expr> >
	3183	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__cosh_expr<typename _Expr::value_type>, _Expr> >
3240	3184	cosh(const _Expr& __x) {
3241	3185	typedef typename _Expr::value_type value_type;
3242	3186	typedef _UnaryOp<__cosh_expr<value_type>, _Expr> _Op;
...	...	@@ -3244,7 +3188,7 @@ cosh(const _Expr& __x) {
3244	3188	}
3245	3189
3246	3190	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3247		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__exp_expr<typename _Expr::value_type>, _Expr> >
	3191	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__exp_expr<typename _Expr::value_type>, _Expr> >
3248	3192	exp(const _Expr& __x) {
3249	3193	typedef typename _Expr::value_type value_type;
3250	3194	typedef _UnaryOp<__exp_expr<value_type>, _Expr> _Op;
...	...	@@ -3252,7 +3196,7 @@ exp(const _Expr& __x) {
3252	3196	}
3253	3197
3254	3198	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3255		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log_expr<typename _Expr::value_type>, _Expr> >
	3199	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log_expr<typename _Expr::value_type>, _Expr> >
3256	3200	log(const _Expr& __x) {
3257	3201	typedef typename _Expr::value_type value_type;
3258	3202	typedef _UnaryOp<__log_expr<value_type>, _Expr> _Op;
...	...	@@ -3260,7 +3204,7 @@ log(const _Expr& __x) {
3260	3204	}
3261	3205
3262	3206	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3263		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log10_expr<typename _Expr::value_type>, _Expr> >
	3207	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__log10_expr<typename _Expr::value_type>, _Expr> >
3264	3208	log10(const _Expr& __x) {
3265	3209	typedef typename _Expr::value_type value_type;
3266	3210	typedef _UnaryOp<__log10_expr<value_type>, _Expr> _Op;
...	...	@@ -3270,15 +3214,16 @@ log10(const _Expr& __x) {
3270	3214	template <class _Expr1,
3271	3215	class _Expr2,
3272	3216	__enable_if_t<__is_val_expr<_Expr1>::value && __is_val_expr<_Expr2>::value, int> = 0>
3273		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__pow_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
3274		pow(const _Expr1& __x, const _Expr2& __y) {
	3217	[[__nodiscard__]] inline
	3218	_LIBCPP_HIDE_FROM_ABI __val_expr<_BinaryOp<__pow_expr<typename _Expr1::value_type>, _Expr1, _Expr2> >
	3219	pow(const _Expr1& __x, const _Expr2& __y) {
3275	3220	typedef typename _Expr1::value_type value_type;
3276	3221	typedef _BinaryOp<__pow_expr<value_type>, _Expr1, _Expr2> _Op;
3277	3222	return __val_expr<_Op>(_Op(__pow_expr<value_type>(), __x, __y));
3278	3223	}
3279	3224
3280	3225	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3281		inline _LIBCPP_HIDE_FROM_ABI
	3226	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
3282	3227	__val_expr<_BinaryOp<__pow_expr<typename _Expr::value_type>, _Expr, __scalar_expr<typename _Expr::value_type> > >
3283	3228	pow(const _Expr& __x, const typename _Expr::value_type& __y) {
3284	3229	typedef typename _Expr::value_type value_type;
...	...	@@ -3287,7 +3232,7 @@ pow(const _Expr& __x, const typename _Expr::value_type& __y) {
3287	3232	}
3288	3233
3289	3234	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3290		inline _LIBCPP_HIDE_FROM_ABI
	3235	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI
3291	3236	__val_expr<_BinaryOp<__pow_expr<typename _Expr::value_type>, __scalar_expr<typename _Expr::value_type>, _Expr> >
3292	3237	pow(const typename _Expr::value_type& __x, const _Expr& __y) {
3293	3238	typedef typename _Expr::value_type value_type;
...	...	@@ -3296,7 +3241,7 @@ pow(const typename _Expr::value_type& __x, const _Expr& __y) {
3296	3241	}
3297	3242
3298	3243	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3299		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sin_expr<typename _Expr::value_type>, _Expr> >
	3244	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sin_expr<typename _Expr::value_type>, _Expr> >
3300	3245	sin(const _Expr& __x) {
3301	3246	typedef typename _Expr::value_type value_type;
3302	3247	typedef _UnaryOp<__sin_expr<value_type>, _Expr> _Op;
...	...	@@ -3304,7 +3249,7 @@ sin(const _Expr& __x) {
3304	3249	}
3305	3250
3306	3251	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3307		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sinh_expr<typename _Expr::value_type>, _Expr> >
	3252	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sinh_expr<typename _Expr::value_type>, _Expr> >
3308	3253	sinh(const _Expr& __x) {
3309	3254	typedef typename _Expr::value_type value_type;
3310	3255	typedef _UnaryOp<__sinh_expr<value_type>, _Expr> _Op;
...	...	@@ -3312,7 +3257,7 @@ sinh(const _Expr& __x) {
3312	3257	}
3313	3258
3314	3259	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3315		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sqrt_expr<typename _Expr::value_type>, _Expr> >
	3260	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__sqrt_expr<typename _Expr::value_type>, _Expr> >
3316	3261	sqrt(const _Expr& __x) {
3317	3262	typedef typename _Expr::value_type value_type;
3318	3263	typedef _UnaryOp<__sqrt_expr<value_type>, _Expr> _Op;
...	...	@@ -3320,7 +3265,7 @@ sqrt(const _Expr& __x) {
3320	3265	}
3321	3266
3322	3267	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3323		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tan_expr<typename _Expr::value_type>, _Expr> >
	3268	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tan_expr<typename _Expr::value_type>, _Expr> >
3324	3269	tan(const _Expr& __x) {
3325	3270	typedef typename _Expr::value_type value_type;
3326	3271	typedef _UnaryOp<__tan_expr<value_type>, _Expr> _Op;
...	...	@@ -3328,7 +3273,7 @@ tan(const _Expr& __x) {
3328	3273	}
3329	3274
3330	3275	template <class _Expr, __enable_if_t<__is_val_expr<_Expr>::value, int> = 0>
3331		inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tanh_expr<typename _Expr::value_type>, _Expr> >
	3276	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI __val_expr<_UnaryOp<__tanh_expr<typename _Expr::value_type>, _Expr> >
3332	3277	tanh(const _Expr& __x) {
3333	3278	typedef typename _Expr::value_type value_type;
3334	3279	typedef _UnaryOp<__tanh_expr<value_type>, _Expr> _Op;
...	...	@@ -3336,22 +3281,22 @@ tanh(const _Expr& __x) {
3336	3281	}
3337	3282
3338	3283	template <class _Tp>
3339		inline _LIBCPP_HIDE_FROM_ABI _Tp* begin(valarray<_Tp>& __v) {
	3284	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _Tp* begin(valarray<_Tp>& __v) {
3340	3285	return __v.__begin_;
3341	3286	}
3342	3287
3343	3288	template <class _Tp>
3344		inline _LIBCPP_HIDE_FROM_ABI const _Tp* begin(const valarray<_Tp>& __v) {
	3289	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI const _Tp* begin(const valarray<_Tp>& __v) {
3345	3290	return __v.__begin_;
3346	3291	}
3347	3292
3348	3293	template <class _Tp>
3349		inline _LIBCPP_HIDE_FROM_ABI _Tp* end(valarray<_Tp>& __v) {
	3294	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _Tp* end(valarray<_Tp>& __v) {
3350	3295	return __v.__end_;
3351	3296	}
3352	3297
3353	3298	template <class _Tp>
3354		inline _LIBCPP_HIDE_FROM_ABI const _Tp* end(const valarray<_Tp>& __v) {
	3299	[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI const _Tp* end(const valarray<_Tp>& __v) {
3355	3300	return __v.__end_;
3356	3301	}
3357	3302

lib/libcxx/include/variant+25-33

...	...	@@ -247,7 +247,6 @@ namespace std {
247	247	# include <__type_traits/is_nothrow_assignable.h>
248	248	# include <__type_traits/is_nothrow_constructible.h>
249	249	# include <__type_traits/is_reference.h>
250		# include <__type_traits/is_replaceable.h>
251	250	# include <__type_traits/is_same.h>
252	251	# include <__type_traits/is_swappable.h>
253	252	# include <__type_traits/is_trivially_assignable.h>
...	...	@@ -289,7 +288,7 @@ _LIBCPP_BEGIN_UNVERSIONED_NAMESPACE_STD
289	288
290	289	class _LIBCPP_EXPORTED_FROM_ABI bad_variant_access : public exception {
291	290	public:
292		const char* what() const _NOEXCEPT override;
	291	[[__nodiscard__]] const char* what() const _NOEXCEPT override;
293	292	};
294	293
295	294	_LIBCPP_END_UNVERSIONED_NAMESPACE_STD
...	...	@@ -918,18 +917,10 @@ protected:
918	917	_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __assign_alt(__alt<_Ip, _Tp>& __a, _Arg&& __arg) {
919	918	if (this->index() == _Ip) {
920	919	__a.__value = std::forward<_Arg>(__arg);
	920	} else if constexpr (is_nothrow_constructible_v<_Tp, _Arg> \|\| !is_nothrow_move_constructible_v<_Tp>) {
	921	this->__emplace<_Ip>(std::forward<_Arg>(__arg));
921	922	} else {
922		struct {
923		_LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(true_type) const {
924		__this->__emplace<_Ip>(std::forward<_Arg>(__arg));
925		}
926		_LIBCPP_HIDDEN _LIBCPP_CONSTEXPR_SINCE_CXX20 void operator()(false_type) const {
927		__this->__emplace<_Ip>(_Tp(std::forward<_Arg>(__arg)));
928		}
929		__assignment* __this;
930		_Arg&& __arg;
931		} __impl{this, std::forward<_Arg>(__arg)};
932		__impl(bool_constant < is_nothrow_constructible_v<_Tp, _Arg> \|\| !is_nothrow_move_constructible_v < _Tp >> {});
	923	this->__emplace<_Ip>(_Tp(std::forward<_Arg>(__arg)));
933	924	}
934	925	}
935	926
...	...	@@ -1127,14 +1118,14 @@ template <class _IdxSeq>
1127	1118	struct __make_overloads_imp;
1128	1119
1129	1120	template <size_t... _Idx>
1130		struct __make_overloads_imp<__tuple_indices<_Idx...> > {
	1121	struct __make_overloads_imp<index_sequence<_Idx...> > {
1131	1122	template <class... _Types>
1132	1123	using _Apply _LIBCPP_NODEBUG = __all_overloads<__overload<_Types, _Idx>...>;
1133	1124	};
1134	1125
1135	1126	template <class... _Types>
1136	1127	using _MakeOverloads _LIBCPP_NODEBUG =
1137		typename __make_overloads_imp< __make_indices_imp<sizeof...(_Types), 0> >::template _Apply<_Types...>;
	1128	typename __make_overloads_imp<make_index_sequence<sizeof...(_Types)>>::template _Apply<_Types...>;
1138	1129
1139	1130	template <class _Tp, class... _Types>
1140	1131	using __best_match_t _LIBCPP_NODEBUG = typename invoke_result_t<_MakeOverloads<_Types...>, _Tp, _Tp>::type;
...	...	@@ -1172,7 +1163,6 @@ class _LIBCPP_DECLSPEC_EMPTY_BASES _LIBCPP_NO_SPECIALIZATIONS variant
1172	1163	public:
1173	1164	using __trivially_relocatable _LIBCPP_NODEBUG =
1174	1165	conditional_t<_And<__libcpp_is_trivially_relocatable<_Types>...>::value, variant, void>;
1175		using __replaceable _LIBCPP_NODEBUG = conditional_t<_And<__is_replaceable<_Types>...>::value, variant, void>;
1176	1166
1177	1167	template <bool _Dummy = true,
1178	1168	enable_if_t<__dependent_type<is_default_constructible<__first_type>, _Dummy>::value, int> = 0>
...	...	@@ -1284,11 +1274,11 @@ public:
1284	1274	return __impl_.template __emplace<_Ip>(__il, std::forward<_Args>(__args)...);
1285	1275	}
1286	1276
1287		_LIBCPP_HIDE_FROM_ABI constexpr bool valueless_by_exception() const noexcept {
	1277	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool valueless_by_exception() const noexcept {
1288	1278	return __impl_.valueless_by_exception();
1289	1279	}
1290	1280
1291		_LIBCPP_HIDE_FROM_ABI constexpr size_t index() const noexcept { return __impl_.index(); }
	1281	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_t index() const noexcept { return __impl_.index(); }
1292	1282
1293	1283	template < bool _Dummy = true,
1294	1284	enable_if_t< __all<(__dependent_type<is_move_constructible<_Types>, _Dummy>::value &&
...	...	@@ -1299,7 +1289,7 @@ public:
1299	1289	__impl_.__swap(__that.__impl_);
1300	1290	}
1301	1291
1302		# if _LIBCPP_STD_VER >= 26 && _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER
	1292	# if _LIBCPP_STD_VER >= 26
1303	1293	// Helper class to implement [variant.visit]/10
1304	1294	// Constraints: The call to visit does not use an explicit template-argument-list
1305	1295	// that begins with a type template-argument.
...	...	@@ -1331,7 +1321,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __holds_alternative(const variant<_Types...
1331	1321	}
1332	1322
1333	1323	template <class _Tp, class... _Types>
1334		_LIBCPP_HIDE_FROM_ABI constexpr bool holds_alternative(const variant<_Types...>& __v) noexcept {
	1324	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool holds_alternative(const variant<_Types...>& __v) noexcept {
1335	1325	return std::__holds_alternative<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
1336	1326	}
1337	1327
...	...	@@ -1345,21 +1335,23 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto&& __generic_get(_Vp&& __v) {
1345	1335	}
1346	1336
1347	1337	template <size_t _Ip, class... _Types>
1348		_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>& get(variant<_Types...>& __v) {
	1338	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>&
	1339	get(variant<_Types...>& __v) {
1349	1340	static_assert(_Ip < sizeof...(_Types));
1350	1341	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
1351	1342	return std::__generic_get<_Ip>(__v);
1352	1343	}
1353	1344
1354	1345	template <size_t _Ip, class... _Types>
1355		_LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>&& get(variant<_Types...>&& __v) {
	1346	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr variant_alternative_t<_Ip, variant<_Types...>>&&
	1347	get(variant<_Types...>&& __v) {
1356	1348	static_assert(_Ip < sizeof...(_Types));
1357	1349	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
1358	1350	return std::__generic_get<_Ip>(std::move(__v));
1359	1351	}
1360	1352
1361	1353	template <size_t _Ip, class... _Types>
1362		_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
	1354	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&
1363	1355	get(const variant<_Types...>& __v) {
1364	1356	static_assert(_Ip < sizeof...(_Types));
1365	1357	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
...	...	@@ -1367,7 +1359,7 @@ get(const variant<_Types...>& __v) {
1367	1359	}
1368	1360
1369	1361	template <size_t _Ip, class... _Types>
1370		_LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
	1362	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const variant_alternative_t<_Ip, variant<_Types...>>&&
1371	1363	get(const variant<_Types...>&& __v) {
1372	1364	static_assert(_Ip < sizeof...(_Types));
1373	1365	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
...	...	@@ -1375,25 +1367,25 @@ get(const variant<_Types...>&& __v) {
1375	1367	}
1376	1368
1377	1369	template <class _Tp, class... _Types>
1378		_LIBCPP_HIDE_FROM_ABI constexpr _Tp& get(variant<_Types...>& __v) {
	1370	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp& get(variant<_Types...>& __v) {
1379	1371	static_assert(!is_void_v<_Tp>);
1380	1372	return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
1381	1373	}
1382	1374
1383	1375	template <class _Tp, class... _Types>
1384		_LIBCPP_HIDE_FROM_ABI constexpr _Tp&& get(variant<_Types...>&& __v) {
	1376	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp&& get(variant<_Types...>&& __v) {
1385	1377	static_assert(!is_void_v<_Tp>);
1386	1378	return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
1387	1379	}
1388	1380
1389	1381	template <class _Tp, class... _Types>
1390		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp& get(const variant<_Types...>& __v) {
	1382	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp& get(const variant<_Types...>& __v) {
1391	1383	static_assert(!is_void_v<_Tp>);
1392	1384	return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
1393	1385	}
1394	1386
1395	1387	template <class _Tp, class... _Types>
1396		_LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const variant<_Types...>&& __v) {
	1388	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&& get(const variant<_Types...>&& __v) {
1397	1389	static_assert(!is_void_v<_Tp>);
1398	1390	return std::get<__find_exactly_one_t<_Tp, _Types...>::value>(std::move(__v));
1399	1391	}
...	...	@@ -1405,7 +1397,7 @@ _LIBCPP_HIDE_FROM_ABI constexpr auto* __generic_get_if(_Vp* __v) noexcept {
1405	1397	}
1406	1398
1407	1399	template <size_t _Ip, class... _Types>
1408		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<variant_alternative_t<_Ip, variant<_Types...>>>
	1400	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<variant_alternative_t<_Ip, variant<_Types...>>>
1409	1401	get_if(variant<_Types...>* __v) noexcept {
1410	1402	static_assert(_Ip < sizeof...(_Types));
1411	1403	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
...	...	@@ -1413,7 +1405,7 @@ get_if(variant<_Types...>* __v) noexcept {
1413	1405	}
1414	1406
1415	1407	template <size_t _Ip, class... _Types>
1416		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const variant_alternative_t<_Ip, variant<_Types...>>>
	1408	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const variant_alternative_t<_Ip, variant<_Types...>>>
1417	1409	get_if(const variant<_Types...>* __v) noexcept {
1418	1410	static_assert(_Ip < sizeof...(_Types));
1419	1411	static_assert(!is_void_v<variant_alternative_t<_Ip, variant<_Types...>>>);
...	...	@@ -1421,13 +1413,13 @@ get_if(const variant<_Types...>* __v) noexcept {
1421	1413	}
1422	1414
1423	1415	template <class _Tp, class... _Types>
1424		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp> get_if(variant<_Types...>* __v) noexcept {
	1416	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<_Tp> get_if(variant<_Types...>* __v) noexcept {
1425	1417	static_assert(!is_void_v<_Tp>);
1426	1418	return std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
1427	1419	}
1428	1420
1429	1421	template <class _Tp, class... _Types>
1430		_LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const _Tp> get_if(const variant<_Types...>* __v) noexcept {
	1422	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<const _Tp> get_if(const variant<_Types...>* __v) noexcept {
1431	1423	static_assert(!is_void_v<_Tp>);
1432	1424	return std::get_if<__find_exactly_one_t<_Tp, _Types...>::value>(__v);
1433	1425	}
...	...	@@ -1608,7 +1600,7 @@ struct hash< __enable_hash_helper<variant<_Types...>, remove_const_t<_Types>...>
1608	1600	using result_type _LIBCPP_DEPRECATED_IN_CXX17 = size_t;
1609	1601	# endif
1610	1602
1611		_LIBCPP_HIDE_FROM_ABI size_t operator()(const variant<_Types...>& __v) const {
	1603	[[nodiscard]] _LIBCPP_HIDE_FROM_ABI size_t operator()(const variant<_Types...>& __v) const {
1612	1604	using __variant_detail::__visitation::__variant;
1613	1605	size_t __res =
1614	1606	__v.valueless_by_exception()

lib/libcxx/include/version+41-24

...	...	@@ -37,7 +37,8 @@ __cpp_lib_atomic_float 201711L <atomic>
37	37	__cpp_lib_atomic_is_always_lock_free 201603L <atomic>
38	38	__cpp_lib_atomic_lock_free_type_aliases 201907L <atomic>
39	39	__cpp_lib_atomic_min_max 202403L <atomic>
40		__cpp_lib_atomic_ref 201806L <atomic>
	40	__cpp_lib_atomic_ref 202411L <atomic>
	41	201806L // C++20
41	42	__cpp_lib_atomic_shared_ptr 201711L <atomic>
42	43	__cpp_lib_atomic_value_initialization 201911L <atomic> <memory>
43	44	__cpp_lib_atomic_wait 201907L <atomic>
...	...	@@ -62,7 +63,7 @@ __cpp_lib_clamp 201603L <algorithm>
62	63	__cpp_lib_common_reference 202302L <type_traits>
63	64	__cpp_lib_common_reference_wrapper 202302L <functional>
64	65	__cpp_lib_complex_udls 201309L <complex>
65		__cpp_lib_concepts 202002L <concepts>
	66	__cpp_lib_concepts 202207L <concepts>
66	67	__cpp_lib_constexpr_algorithms 202306L <algorithm> <utility>
67	68	201806L // C++20
68	69	__cpp_lib_constexpr_bitset 202207L <bitset>
...	...	@@ -70,6 +71,8 @@ __cpp_lib_constexpr_charconv 202207L <charconv>
70	71	__cpp_lib_constexpr_cmath 202202L <cmath> <cstdlib>
71	72	__cpp_lib_constexpr_complex 201711L <complex>
72	73	__cpp_lib_constexpr_dynamic_alloc 201907L <memory>
	74	__cpp_lib_constexpr_flat_map 202502L <flat_map>
	75	__cpp_lib_constexpr_flat_set 202502L <flat_set>
73	76	__cpp_lib_constexpr_forward_list 202502L <forward_list>
74	77	__cpp_lib_constexpr_functional 201907L <functional>
75	78	__cpp_lib_constexpr_iterator 201811L <iterator>
...	...	@@ -108,8 +111,8 @@ __cpp_lib_execution 201902L <execution>
108	111	201603L // C++17
109	112	__cpp_lib_expected 202211L <expected>
110	113	__cpp_lib_filesystem 201703L <filesystem>
111		__cpp_lib_flat_map 202207L <flat_map>
112		__cpp_lib_flat_set 202207L <flat_set>
	114	__cpp_lib_flat_map 202511L <flat_map>
	115	__cpp_lib_flat_set 202511L <flat_set>
113	116	__cpp_lib_format 202110L <format>
114	117	__cpp_lib_format_path 202403L <filesystem>
115	118	__cpp_lib_format_ranges 202207L <format>
...	...	@@ -138,7 +141,8 @@ __cpp_lib_incomplete_container_elements 201505L <forward_list> <
138	141	__cpp_lib_inplace_vector 202406L <inplace_vector>
139	142	__cpp_lib_int_pow2 202002L <bit>
140	143	__cpp_lib_integer_comparison_functions 202002L <utility>
141		__cpp_lib_integer_sequence 201304L <utility>
	144	__cpp_lib_integer_sequence 202511L <utility>
	145	201304L // C++14
142	146	__cpp_lib_integral_constant_callable 201304L <type_traits>
143	147	__cpp_lib_interpolate 201902L <cmath> <numeric>
144	148	__cpp_lib_invoke 201411L <functional>
...	...	@@ -184,7 +188,8 @@ __cpp_lib_nonmember_container_access 201411L <array> <deque>
184	188	__cpp_lib_not_fn 202306L <functional>
185	189	201603L // C++17
186	190	__cpp_lib_null_iterators 201304L <iterator>
187		__cpp_lib_optional 202110L <optional>
	191	__cpp_lib_optional 202506L <optional>
	192	202110L // C++23
188	193	202106L // C++20
189	194	201606L // C++17
190	195	__cpp_lib_optional_range_support 202406L <optional>
...	...	@@ -205,6 +210,7 @@ __cpp_lib_ranges_chunk_by 202202L <ranges>
205	210	__cpp_lib_ranges_concat 202403L <ranges>
206	211	__cpp_lib_ranges_contains 202207L <algorithm>
207	212	__cpp_lib_ranges_find_last 202207L <algorithm>
	213	__cpp_lib_ranges_indices 202506L <ranges>
208	214	__cpp_lib_ranges_iota 202202L <numeric>
209	215	__cpp_lib_ranges_join_with 202202L <ranges>
210	216	__cpp_lib_ranges_repeat 202207L <ranges>
...	...	@@ -245,6 +251,7 @@ __cpp_lib_starts_ends_with 201711L <string> <string
245	251	__cpp_lib_stdatomic_h 202011L <stdatomic.h>
246	252	__cpp_lib_string_contains 202011L <string> <string_view>
247	253	__cpp_lib_string_resize_and_overwrite 202110L <string>
	254	__cpp_lib_string_subview 202506L <string> <string_view>
248	255	__cpp_lib_string_udls 201304L <string>
249	256	__cpp_lib_string_view 202403L <string> <string_view>
250	257	201803L // C++20
...	...	@@ -333,13 +340,11 @@ __cpp_lib_void_t 201411L <type_traits>
333	340	# define __cpp_lib_clamp 201603L
334	341	# define __cpp_lib_enable_shared_from_this 201603L
335	342	// # define __cpp_lib_execution 201603L
336		# if _LIBCPP_HAS_FILESYSTEM && _LIBCPP_AVAILABILITY_HAS_FILESYSTEM_LIBRARY
	343	# if _LIBCPP_HAS_FILESYSTEM
337	344	# define __cpp_lib_filesystem 201703L
338	345	# endif
339	346	# define __cpp_lib_gcd_lcm 201606L
340		# if defined(__GCC_DESTRUCTIVE_SIZE) && defined(__GCC_CONSTRUCTIVE_SIZE)
341		# define __cpp_lib_hardware_interference_size 201703L
342		# endif
	347	# define __cpp_lib_hardware_interference_size 201703L
343	348	# define __cpp_lib_has_unique_object_representations 201606L
344	349	# define __cpp_lib_hypot 201603L
345	350	# define __cpp_lib_incomplete_container_elements 201505L
...	...	@@ -391,10 +396,8 @@ __cpp_lib_void_t 201411L <type_traits>
391	396	# define __cpp_lib_atomic_ref 201806L
392	397	// # define __cpp_lib_atomic_shared_ptr 201711L
393	398	# define __cpp_lib_atomic_value_initialization 201911L
394		# if _LIBCPP_AVAILABILITY_HAS_SYNC
395		# define __cpp_lib_atomic_wait 201907L
396		# endif
397		# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC
	399	# define __cpp_lib_atomic_wait 201907L
	400	# if _LIBCPP_HAS_THREADS
398	401	# define __cpp_lib_barrier 201907L
399	402	# endif
400	403	# define __cpp_lib_bind_front 201907L
...	...	@@ -406,7 +409,7 @@ __cpp_lib_void_t 201411L <type_traits>
406	409	# endif
407	410	# define __cpp_lib_common_reference 202302L
408	411	# define __cpp_lib_common_reference_wrapper 202302L
409		# define __cpp_lib_concepts 202002L
	412	# define __cpp_lib_concepts 202207L
410	413	# define __cpp_lib_constexpr_algorithms 201806L
411	414	# define __cpp_lib_constexpr_complex 201711L
412	415	# define __cpp_lib_constexpr_dynamic_alloc 201907L
...	...	@@ -439,10 +442,10 @@ __cpp_lib_void_t 201411L <type_traits>
439	442	// # define __cpp_lib_is_layout_compatible 201907L
440	443	# define __cpp_lib_is_nothrow_convertible 201806L
441	444	// # define __cpp_lib_is_pointer_interconvertible 201907L
442		# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC
	445	# if _LIBCPP_HAS_THREADS
443	446	# define __cpp_lib_jthread 201911L
444	447	# endif
445		# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC
	448	# if _LIBCPP_HAS_THREADS
446	449	# define __cpp_lib_latch 201907L
447	450	# endif
448	451	# define __cpp_lib_list_remove_return_type 201806L
...	...	@@ -455,7 +458,7 @@ __cpp_lib_void_t 201411L <type_traits>
455	458	# endif
456	459	# define __cpp_lib_ranges 202110L
457	460	# define __cpp_lib_remove_cvref 201711L
458		# if _LIBCPP_HAS_THREADS && _LIBCPP_AVAILABILITY_HAS_SYNC
	461	# if _LIBCPP_HAS_THREADS
459	462	# define __cpp_lib_semaphore 201907L
460	463	# endif
461	464	# undef __cpp_lib_shared_ptr_arrays
...	...	@@ -494,8 +497,8 @@ __cpp_lib_void_t 201411L <type_traits>
494	497	# define __cpp_lib_constexpr_typeinfo 202106L
495	498	# define __cpp_lib_containers_ranges 202202L
496	499	# define __cpp_lib_expected 202211L
497		# define __cpp_lib_flat_map 202207L
498		# define __cpp_lib_flat_set 202207L
	500	# define __cpp_lib_flat_map 202511L
	501	# define __cpp_lib_flat_set 202511L
499	502	# define __cpp_lib_format_ranges 202207L
500	503	// # define __cpp_lib_formatters 202302L
501	504	# define __cpp_lib_forward_like 202207L
...	...	@@ -528,7 +531,7 @@ __cpp_lib_void_t 201411L <type_traits>
528	531	// # define __cpp_lib_ranges_slide 202202L
529	532	# define __cpp_lib_ranges_starts_ends_with 202106L
530	533	# define __cpp_lib_ranges_to_container 202202L
531		// # define __cpp_lib_ranges_zip 202110L
	534	# define __cpp_lib_ranges_zip 202110L
532	535	// # define __cpp_lib_reference_from_temporary 202202L
533	536	// # define __cpp_lib_spanstream 202106L
534	537	// # define __cpp_lib_stacktrace 202011L
...	...	@@ -544,18 +547,22 @@ __cpp_lib_void_t 201411L <type_traits>
544	547	# define __cpp_lib_aligned_accessor 202411L
545	548	// # define __cpp_lib_associative_heterogeneous_insertion 202306L
546	549	// # define __cpp_lib_atomic_min_max 202403L
	550	# undef __cpp_lib_atomic_ref
	551	# define __cpp_lib_atomic_ref 202411L
547	552	# undef __cpp_lib_bind_front
548	553	# define __cpp_lib_bind_front 202306L
549	554	# define __cpp_lib_bitset 202306L
550	555	# undef __cpp_lib_constexpr_algorithms
551	556	# define __cpp_lib_constexpr_algorithms 202306L
	557	# define __cpp_lib_constexpr_flat_map 202502L
	558	# define __cpp_lib_constexpr_flat_set 202502L
552	559	# define __cpp_lib_constexpr_forward_list 202502L
553	560	# define __cpp_lib_constexpr_list 202502L
554	561	# if !defined(_LIBCPP_ABI_VCRUNTIME)
555	562	# define __cpp_lib_constexpr_new 202406L
556	563	# endif
557	564	# define __cpp_lib_constexpr_queue 202502L
558		// # define __cpp_lib_constrained_equality 202411L
	565	# define __cpp_lib_constrained_equality 202411L
559	566	// # define __cpp_lib_copyable_function 202306L
560	567	// # define __cpp_lib_debugging 202311L
561	568	// # define __cpp_lib_default_template_type_for_algorithm_values 202403L
...	...	@@ -575,21 +582,30 @@ __cpp_lib_void_t 201411L <type_traits>
575	582	// # define __cpp_lib_generate_random 202403L
576	583	// # define __cpp_lib_hazard_pointer 202306L
577	584	// # define __cpp_lib_inplace_vector 202406L
	585	# undef __cpp_lib_integer_sequence
	586	# define __cpp_lib_integer_sequence 202511L
578	587	# define __cpp_lib_is_sufficiently_aligned 202411L
579	588	# if __has_builtin(__builtin_is_virtual_base_of)
580	589	# define __cpp_lib_is_virtual_base_of 202406L
581	590	# endif
582		// # define __cpp_lib_is_within_lifetime 202306L
	591	# if __has_builtin(__builtin_is_within_lifetime)
	592	# define __cpp_lib_is_within_lifetime 202306L
	593	# endif
583	594	// # define __cpp_lib_linalg 202311L
584	595	# undef __cpp_lib_mdspan
585	596	# define __cpp_lib_mdspan 202406L
586	597	# undef __cpp_lib_not_fn
587	598	# define __cpp_lib_not_fn 202306L
588		// # define __cpp_lib_optional_range_support 202406L
	599	# undef __cpp_lib_optional
	600	# define __cpp_lib_optional 202506L
	601	# if _LIBCPP_HAS_EXPERIMENTAL_OPTIONAL_ITERATOR
	602	# define __cpp_lib_optional_range_support 202406L
	603	# endif
589	604	# undef __cpp_lib_out_ptr
590	605	# define __cpp_lib_out_ptr 202311L
591	606	// # define __cpp_lib_philox_engine 202406L
592	607	// # define __cpp_lib_ranges_concat 202403L
	608	# define __cpp_lib_ranges_indices 202506L
593	609	# define __cpp_lib_ratio 202306L
594	610	// # define __cpp_lib_rcu 202306L
595	611	# define __cpp_lib_reference_wrapper 202403L
...	...	@@ -599,6 +615,7 @@ __cpp_lib_void_t 201411L <type_traits>
599	615	# define __cpp_lib_span_at 202311L
600	616	# define __cpp_lib_span_initializer_list 202311L
601	617	# define __cpp_lib_sstream_from_string_view 202306L
	618	# define __cpp_lib_string_subview 202506L
602	619	# undef __cpp_lib_string_view
603	620	# define __cpp_lib_string_view 202403L
604	621	// # define __cpp_lib_submdspan 202306L

lib/libcxx/include/wctype.h+29-29

...	...	@@ -45,13 +45,14 @@ wctrans_t wctrans(const char* property);
45	45	*/
46	46
47	47	#if defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
48		# include <__cxx03/wctype.h>
	48	# include <__cxx03/__config>
49	49	#else
50	50	# include <__config>
	51	#endif
51	52
52		# if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
53		# pragma GCC system_header
54		# endif
	53	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	54	# pragma GCC system_header
	55	#endif
55	56
56	57	// TODO:
57	58	// In the future, we should unconditionally include_next <wctype.h> here and instead
...	...	@@ -62,33 +63,32 @@ wctrans_t wctrans(const char* property);
62	63	// nothing (with using_if_exists), and if we include another header that defines one
63	64	// of these declarations (e.g. <wchar.h>), the second `using ::wint_t` with using_if_exists
64	65	// will fail because it does not refer to the same declaration.
65		# if __has_include_next(<wctype.h>)
66		# include_next <wctype.h>
67		# define _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
68		# endif
	66	#if __has_include_next(<wctype.h>)
	67	# include_next <wctype.h>
	68	# define _LIBCPP_INCLUDED_C_LIBRARY_WCTYPE_H
	69	#endif
69	70
70		# ifdef __cplusplus
	71	#ifdef __cplusplus
71	72
72		# undef iswalnum
73		# undef iswalpha
74		# undef iswblank
75		# undef iswcntrl
76		# undef iswdigit
77		# undef iswgraph
78		# undef iswlower
79		# undef iswprint
80		# undef iswpunct
81		# undef iswspace
82		# undef iswupper
83		# undef iswxdigit
84		# undef iswctype
85		# undef wctype
86		# undef towlower
87		# undef towupper
88		# undef towctrans
89		# undef wctrans
	73	# undef iswalnum
	74	# undef iswalpha
	75	# undef iswblank
	76	# undef iswcntrl
	77	# undef iswdigit
	78	# undef iswgraph
	79	# undef iswlower
	80	# undef iswprint
	81	# undef iswpunct
	82	# undef iswspace
	83	# undef iswupper
	84	# undef iswxdigit
	85	# undef iswctype
	86	# undef wctype
	87	# undef towlower
	88	# undef towupper
	89	# undef towctrans
	90	# undef wctrans
90	91
91		# endif // __cplusplus
92		#endif // defined(__cplusplus) && __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
	92	#endif // __cplusplus
93	93
94	94	#endif // _LIBCPP_WCTYPE_H

lib/libcxx/src/any.cpp+4

...	...	@@ -14,6 +14,8 @@ const char* bad_any_cast::what() const noexcept { return "bad any cast"; }
14	14
15	15	#include <__config>
16	16
	17	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 7
	18
17	19	// Preserve std::experimental::any_bad_cast for ABI compatibility
18	20	// Even though it no longer exists in a header file
19	21	_LIBCPP_BEGIN_NAMESPACE_LFTS
...	...	@@ -25,4 +27,6 @@ public:
25	27
26	28	const char* bad_any_cast::what() const noexcept { return "bad any cast"; }
27	29
	30	#endif
	31
28	32	_LIBCPP_END_NAMESPACE_LFTS

lib/libcxx/src/atomic.cpp+365-80

...	...	@@ -9,8 +9,13 @@
9	9	#include <__thread/timed_backoff_policy.h>
10	10	#include <atomic>
11	11	#include <climits>
	12	#include <cstddef>
	13	#include <cstdint>
	14	#include <cstring>
12	15	#include <functional>
	16	#include <new>
13	17	#include <thread>
	18	#include <type_traits>
14	19
15	20	#include "include/apple_availability.h"
16	21
...	...	@@ -41,6 +46,11 @@
41	46	// OpenBSD has no indirect syscalls
42	47	# define _LIBCPP_FUTEX(...) futex(__VA_ARGS__)
43	48
	49	#elif defined(_WIN32)
	50
	51	# include <memory>
	52	# include <windows.h>
	53
44	54	#else // <- Add other operating systems here
45	55
46	56	// Baseline needs no new headers
...	...	@@ -49,17 +59,34 @@
49	59
50	60	#endif
51	61
	62	_LIBCPP_PUSH_MACROS
	63	#include <__undef_macros>
	64
52	65	_LIBCPP_BEGIN_NAMESPACE_STD
53	66
	67	struct NoTimeout {};
	68
54	69	#ifdef __linux__
55	70
56		static void
57		__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
58		static constexpr timespec __timeout = {2, 0};
59		_LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, __val, &__timeout, 0, 0);
	71	template <std::size_t _Size, class MaybeTimeout>
	72	static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
	73	static_assert(_Size == 4, "Can only wait on 4 bytes value");
	74	alignas(__cxx_contention_t) char buffer[_Size];
	75	std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
	76	static constexpr timespec __default_timeout = {2, 0};
	77	timespec __timeout;
	78	if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
	79	__timeout = __default_timeout;
	80	} else {
	81	__timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
	82	__timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
	83	}
	84	_LIBCPP_FUTEX(__ptr, FUTEX_WAIT_PRIVATE, reinterpret_cast<__cxx_contention_t const>(&buffer), &__timeout, 0, 0);
60	85	}
61	86
62		static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
	87	template <std::size_t _Size>
	88	static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
	89	static_assert(_Size == 4, "Can only wake up on 4 bytes value");
63	90	_LIBCPP_FUTEX(__ptr, FUTEX_WAKE_PRIVATE, __notify_one ? 1 : INT_MAX, 0, 0, 0);
64	91	}
65	92
...	...	@@ -70,19 +97,41 @@ extern "C" int __ulock_wait(
70	97	extern "C" int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
71	98
72	99	// https://github.com/apple/darwin-xnu/blob/2ff845c2e033bd0ff64b5b6aa6063a1f8f65aa32/bsd/sys/ulock.h#L82
	100	# define UL_COMPARE_AND_WAIT 1
73	101	# define UL_COMPARE_AND_WAIT64 5
74	102	# define ULF_WAKE_ALL 0x00000100
75	103
76		static void
77		__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
78		static_assert(sizeof(__cxx_atomic_contention_t) == 8, "Waiting on 8 bytes value");
79		__ulock_wait(UL_COMPARE_AND_WAIT64, const_cast<__cxx_atomic_contention_t*>(__ptr), __val, 0);
	104	template <std::size_t _Size, class MaybeTimeout>
	105	static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
	106	static_assert(_Size == 8 \|\| _Size == 4, "Can only wait on 8 bytes or 4 bytes value");
	107	auto __timeout_us = [&] {
	108	if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
	109	return uint32_t(0);
	110	} else {
	111	return std::max(static_cast<uint32_t>(maybe_timeout_ns / 1000), uint32_t(1));
	112	}
	113	}();
	114	if constexpr (_Size == 4) {
	115	alignas(uint32_t) char buffer[_Size];
	116	std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
	117	__ulock_wait(
	118	UL_COMPARE_AND_WAIT, const_cast<void>(__ptr), reinterpret_cast<uint32_t const*>(&buffer), __timeout_us);
	119	} else {
	120	alignas(uint64_t) char buffer[_Size];
	121	std::memcpy(&buffer, const_cast<const void*>(__val), _Size);
	122	__ulock_wait(
	123	UL_COMPARE_AND_WAIT64, const_cast<void>(__ptr), reinterpret_cast<uint64_t const*>(&buffer), __timeout_us);
	124	}
80	125	}
81	126
82		static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
83		static_assert(sizeof(__cxx_atomic_contention_t) == 8, "Waking up on 8 bytes value");
84		__ulock_wake(
85		UL_COMPARE_AND_WAIT64 \| (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<__cxx_atomic_contention_t*>(__ptr), 0);
	127	template <std::size_t _Size>
	128	static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
	129	static_assert(_Size == 8 \|\| _Size == 4, "Can only wake up on 8 bytes or 4 bytes value");
	130
	131	if constexpr (_Size == 4)
	132	__ulock_wake(UL_COMPARE_AND_WAIT \| (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
	133	else
	134	__ulock_wake(UL_COMPARE_AND_WAIT64 \| (__notify_one ? 0 : ULF_WAKE_ALL), const_cast<void*>(__ptr), 0);
86	135	}
87	136
88	137	#elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
...	...	@@ -92,119 +141,355 @@ static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const vo
92	141	* limit its use to architectures where long and int64_t are synonyms.
93	142	*/
94	143
95		static void
96		__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
97		_umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAIT, __val, nullptr, nullptr);
	144	template <std::size_t _Size, class MaybeTimeout>
	145	static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
	146	static_assert(_Size == 8, "Can only wait on 8 bytes value");
	147	// At the moment, FreeBSD is stuck on stable ABI, which only supports platform wait with __cxx_contention_t
	148	// It is safe to reinterpret_cast the val as it is ever going to be passed a __cxx_contention_t under this ABI
	149	// If in the future FreeBSD decides to experiment unstable ABI to support more types, this cast will no longer be
	150	// safe.
	151	__cxx_contention_t value = reinterpret_cast<const __cxx_contention_t>(__val);
	152	if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
	153	_umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAIT, value, nullptr, nullptr);
	154	} else {
	155	timespec timeout{};
	156	timeout.tv_sec = maybe_timeout_ns / 1'000'000'000;
	157	timeout.tv_nsec = maybe_timeout_ns % 1'000'000'000;
	158
	159	_umtx_op(const_cast<void*>(__ptr),
	160	UMTX_OP_WAIT,
	161	value,
	162	reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(timeout))),
	163	&timeout);
	164	}
	165	}
	166
	167	template <std::size_t _Size>
	168	static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
	169	static_assert(_Size == 8, "Can only wake up on 8 bytes value");
	170	_umtx_op(const_cast<void*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
	171	}
	172
	173	#elif defined(_WIN32)
	174
	175	static void* win32_get_synch_api_function(const char* function_name) {
	176	// Attempt to load the API set. Note that as per the Microsoft STL implementation, we assume this API is already
	177	// loaded and accessible. While this isn't explicitly guaranteed by publicly available Win32 API documentation, it is
	178	// true in practice, and may be guaranteed by internal documentation not released publicly. In any case the fact that
	179	// the Microsoft STL made this assumption is reasonable basis to say that we can too. The alternative to this would be
	180	// to use LoadLibrary, but then leak the module handle. We can't call FreeLibrary, as this would have to be triggered
	181	// by a global static destructor, which would hang off DllMain, and calling FreeLibrary from DllMain is explicitly
	182	// mentioned as not being allowed:
	183	// https://learn.microsoft.com/en-us/windows/win32/dlls/dllmain
	184	// Given the range of bad options here, we have chosen to mirror what Microsoft did, as it seems fair to assume that
	185	// Microsoft will guarantee compatibility for us, as we are exposed to the same conditions as all existing Windows
	186	// apps using the Microsoft STL VS2015/2017/2019/2022 runtimes, where Windows 7 support has not been excluded at
	187	// compile time.
	188	static auto module_handle = GetModuleHandleW(L"api-ms-win-core-synch-l1-2-0.dll");
	189	if (module_handle == nullptr) {
	190	return nullptr;
	191	}
	192
	193	// Attempt to locate the function in the API and return the result to the caller. Note that the NULL return from this
	194	// method is documented as being interchangeable with nullptr.
	195	// https://devblogs.microsoft.com/oldnewthing/20180307-00/?p=98175
	196	return reinterpret_cast<void*>(GetProcAddress(module_handle, function_name));
98	197	}
99	198
100		static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile* __ptr, bool __notify_one) {
101		_umtx_op(const_cast<__cxx_atomic_contention_t*>(__ptr), UMTX_OP_WAKE, __notify_one ? 1 : INT_MAX, nullptr, nullptr);
	199	template <std::size_t _Size, class MaybeTimeout>
	200	static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
	201	static_assert(_Size == 8, "Can only wait on 8 bytes value");
	202	// WaitOnAddress was added in Windows 8 (build 9200)
	203	static auto wait_on_address =
	204	reinterpret_cast<BOOL(WINAPI)(void, PVOID, SIZE_T, DWORD)>(win32_get_synch_api_function("WaitOnAddress"));
	205	if (wait_on_address != nullptr) {
	206	auto timeout_ms = [&]() -> DWORD {
	207	if constexpr (is_same_v<MaybeTimeout, NoTimeout>) {
	208	return INFINITE;
	209	} else {
	210	uint64_t ms = maybe_timeout_ns / 1'000'000;
	211	if (ms == 0 && maybe_timeout_ns > 100'000)
	212	// Round up to 1ms if requested between 100us - 1ms
	213	return 1;
	214
	215	return static_cast<DWORD>(std::min(static_cast<uint64_t>(INFINITE), ms));
	216	}
	217	}();
	218	wait_on_address(const_cast<void>(__ptr), const_cast<void>(__val), _Size, timeout_ms);
	219	} else {
	220	std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
	221	if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
	222	timeout = std::chrono::nanoseconds(maybe_timeout_ns);
	223	}
	224	__libcpp_thread_poll_with_backoff(
	225	[=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
	226	__libcpp_timed_backoff_policy(),
	227	timeout);
	228	}
	229	}
	230
	231	template <std::size_t _Size>
	232	static void __platform_wake_by_address(void const* __ptr, bool __notify_one) {
	233	static_assert(_Size == 8, "Can only wake up on 8 bytes value");
	234	if (__notify_one) {
	235	// WakeByAddressSingle was added in Windows 8 (build 9200)
	236	static auto wake_by_address_single =
	237	reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressSingle"));
	238	if (wake_by_address_single != nullptr) {
	239	wake_by_address_single(const_cast<void*>(__ptr));
	240	} else {
	241	// The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
	242	// there's nothing to do here.
	243	}
	244	} else {
	245	// WakeByAddressAll was added in Windows 8 (build 9200)
	246	static auto wake_by_address_all =
	247	reinterpret_cast<void(WINAPI*)(PVOID)>(win32_get_synch_api_function("WakeByAddressAll"));
	248	if (wake_by_address_all != nullptr) {
	249	wake_by_address_all(const_cast<void*>(__ptr));
	250	} else {
	251	// The fallback implementation of waking does nothing, as the fallback wait implementation just does polling, so
	252	// there's nothing to do here.
	253	}
	254	}
102	255	}
103	256
104	257	#else // <- Add other operating systems here
105	258
106	259	// Baseline is just a timed backoff
107	260
108		static void
109		__libcpp_platform_wait_on_address(__cxx_atomic_contention_t const volatile* __ptr, __cxx_contention_t __val) {
	261	template <std::size_t _Size, class MaybeTimeout>
	262	static void __platform_wait_on_address(void const* __ptr, void const* __val, MaybeTimeout maybe_timeout_ns) {
	263	std::chrono::nanoseconds timeout = std::chrono::nanoseconds(0);
	264	if constexpr (!is_same_v<MaybeTimeout, NoTimeout>) {
	265	timeout = std::chrono::nanoseconds(maybe_timeout_ns);
	266	}
110	267	__libcpp_thread_poll_with_backoff(
111		[=]() -> bool { return !__cxx_nonatomic_compare_equal(__cxx_atomic_load(__ptr, memory_order_relaxed), __val); },
112		__libcpp_timed_backoff_policy());
	268	[=]() -> bool { return std::memcmp(const_cast<const void*>(__ptr), __val, _Size) != 0; },
	269	__libcpp_timed_backoff_policy(),
	270	timeout);
113	271	}
114	272
115		static void __libcpp_platform_wake_by_address(__cxx_atomic_contention_t const volatile*, bool) {}
	273	template <std::size_t _Size>
	274	static void __platform_wake_by_address(void const*, bool) {}
116	275
117	276	#endif // __linux__
118	277
119		static constexpr size_t __libcpp_contention_table_size = (1 << 8); /* < there's no magic in this number */
120
121		struct alignas(64) /* aim to avoid false sharing */ __libcpp_contention_table_entry {
122		__cxx_atomic_contention_t __contention_state;
123		__cxx_atomic_contention_t __platform_state;
124		inline constexpr __libcpp_contention_table_entry() : __contention_state(0), __platform_state(0) {}
125		};
126
127		static __libcpp_contention_table_entry __libcpp_contention_table[__libcpp_contention_table_size];
128
129		static hash<void const volatile*> __libcpp_contention_hasher;
130
131		static __libcpp_contention_table_entry* __libcpp_contention_state(void const volatile* p) {
132		return &__libcpp_contention_table[__libcpp_contention_hasher(p) & (__libcpp_contention_table_size - 1)];
133		}
	278	// =============================
	279	// Local hidden helper functions
	280	// =============================
134	281
135	282	/* Given an atomic to track contention and an atomic to actually wait on, which may be
136	283	the same atomic, we try to detect contention to avoid spuriously calling the platform. */
137	284
138		static void __libcpp_contention_notify(__cxx_atomic_contention_t volatile* __contention_state,
139		__cxx_atomic_contention_t const volatile* __platform_state,
140		bool __notify_one) {
141		if (0 != __cxx_atomic_load(__contention_state, memory_order_seq_cst))
	285	template <std::size_t _Size>
	286	static void
	287	__contention_notify(__cxx_atomic_contention_t* __waiter_count, void const* __address_to_notify, bool __notify_one) {
	288	if (0 != __cxx_atomic_load(__waiter_count, memory_order_seq_cst))
142	289	// We only call 'wake' if we consumed a contention bit here.
143		__libcpp_platform_wake_by_address(__platform_state, __notify_one);
144		}
145		static __cxx_contention_t
146		__libcpp_contention_monitor_for_wait(__cxx_atomic_contention_t volatile* /__contention_state/,
147		__cxx_atomic_contention_t const volatile* __platform_state) {
148		// We will monitor this value.
149		return __cxx_atomic_load(__platform_state, memory_order_acquire);
150		}
151		static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __contention_state,
152		__cxx_atomic_contention_t const volatile* __platform_state,
153		__cxx_contention_t __old_value) {
154		__cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_relaxed);
155		// https://github.com/llvm/llvm-project/issues/109290
	290	__platform_wake_by_address<_Size>(__address_to_notify, __notify_one);
	291	}
	292
	293	template <std::size_t _Size, class MaybeTimeout>
	294	static void __contention_wait(__cxx_atomic_contention_t* __waiter_count,
	295	void const* __address_to_wait,
	296	void const* __old_value,
	297	MaybeTimeout maybe_timeout_ns) {
	298	__cxx_atomic_fetch_add(__waiter_count, __cxx_contention_t(1), memory_order_relaxed);
	299	// https://llvm.org/PR109290
156	300	// There are no platform guarantees of a memory barrier in the platform wait implementation
157	301	__cxx_atomic_thread_fence(memory_order_seq_cst);
158	302	// We sleep as long as the monitored value hasn't changed.
159		__libcpp_platform_wait_on_address(__platform_state, __old_value);
160		__cxx_atomic_fetch_sub(__contention_state, __cxx_contention_t(1), memory_order_release);
	303	__platform_wait_on_address<_Size>(__address_to_wait, __old_value, maybe_timeout_ns);
	304	__cxx_atomic_fetch_sub(__waiter_count, __cxx_contention_t(1), memory_order_release);
	305	}
	306
	307	static constexpr size_t __contention_table_size = (1 << 8); /* < there's no magic in this number */
	308
	309	static constexpr hash<void const*> __contention_hasher;
	310
	311	// Waiter count table for all atomics with the correct size that use itself as the wait/notify address.
	312
	313	struct alignas(
	314	std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_native {
	315	__cxx_atomic_contention_t __waiter_count;
	316	constexpr __contention_state_native() : __waiter_count(0) {}
	317	};
	318
	319	static __contention_state_native __contention_table_native[__contention_table_size];
	320
	321	static __cxx_atomic_contention_t* __get_native_waiter_count(void const* p) {
	322	return &__contention_table_native[__contention_hasher(p) & (__contention_table_size - 1)].__waiter_count;
	323	}
	324
	325	// Global contention table for all atomics with the wrong size that use the global table's atomic as wait/notify
	326	// address.
	327
	328	struct alignas(
	329	std::hardware_constructive_interference_size) /* aim to avoid false sharing */ __contention_state_global {
	330	__cxx_atomic_contention_t __waiter_count;
	331	__cxx_atomic_contention_t __platform_state;
	332	constexpr __contention_state_global() : __waiter_count(0), __platform_state(0) {}
	333	};
	334
	335	static __contention_state_global __contention_table_global[__contention_table_size];
	336
	337	static __contention_state_global* __get_global_contention_state(void const* p) {
	338	return &__contention_table_global[__contention_hasher(p) & (__contention_table_size - 1)];
161	339	}
162	340
163	341	/* When the incoming atomic is the wrong size for the platform wait size, need to
164	342	launder the value sequence through an atomic from our table. */
165	343
166		static void __libcpp_atomic_notify(void const volatile* __location) {
167		auto const __entry = __libcpp_contention_state(__location);
	344	static void __atomic_notify_global_table(void const* __location) {
	345	auto const __entry = __get_global_contention_state(__location);
168	346	// The value sequence laundering happens on the next line below.
169	347	__cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_seq_cst);
170		__libcpp_contention_notify(
171		&__entry->__contention_state,
172		&__entry->__platform_state,
173		false /* when laundering, we can't handle notify_one */);
	348	__contention_notify<sizeof(__cxx_atomic_contention_t)>(
	349	&__entry->__waiter_count, &__entry->__platform_state, false /* when laundering, we can't handle notify_one */);
	350	}
	351
	352	// =============================
	353	// New dylib exported symbols
	354	// =============================
	355
	356	// global
	357	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __atomic_monitor_global(void const* __location) noexcept {
	358	auto const __entry = __get_global_contention_state(__location);
	359	return __cxx_atomic_load(&__entry->__platform_state, memory_order_acquire);
	360	}
	361
	362	_LIBCPP_EXPORTED_FROM_ABI void
	363	__atomic_wait_global_table(void const* __location, __cxx_contention_t __old_value) noexcept {
	364	auto const __entry = __get_global_contention_state(__location);
	365	__contention_wait<sizeof(__cxx_atomic_contention_t)>(
	366	&__entry->__waiter_count, &__entry->__platform_state, &__old_value, NoTimeout{});
	367	}
	368
	369	_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_global_table_with_timeout(
	370	void const* __location, __cxx_contention_t __old_value, uint64_t __timeout_ns) _NOEXCEPT {
	371	auto const __entry = __get_global_contention_state(__location);
	372	__contention_wait<sizeof(__cxx_atomic_contention_t)>(
	373	&__entry->__waiter_count, &__entry->__platform_state, &__old_value, __timeout_ns);
	374	}
	375
	376	_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_global_table(void const* __location) noexcept {
	377	__atomic_notify_global_table(__location);
	378	}
	379
	380	_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_global_table(void const* __location) noexcept {
	381	__atomic_notify_global_table(__location);
	382	}
	383
	384	// native
	385
	386	template <std::size_t _Size>
	387	_LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native(void const* __address, void const* __old_value) noexcept {
	388	__contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, NoTimeout{});
174	389	}
	390
	391	template <std::size_t _Size>
	392	_LIBCPP_EXPORTED_FROM_ABI void
	393	__atomic_wait_native_with_timeout(void const* __address, void const* __old_value, uint64_t __timeout_ns) noexcept {
	394	__contention_wait<_Size>(__get_native_waiter_count(__address), __address, __old_value, __timeout_ns);
	395	}
	396
	397	template <std::size_t _Size>
	398	_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(void const* __location) noexcept {
	399	__contention_notify<_Size>(__get_native_waiter_count(__location), __location, true);
	400	}
	401
	402	template <std::size_t _Size>
	403	_LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(void const* __location) noexcept {
	404	__contention_notify<_Size>(__get_native_waiter_count(__location), __location, false);
	405	}
	406
	407	// ==================================================
	408	// Instantiation of the templates with supported size
	409	// ==================================================
	410
	411	#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
	412
	413	# define _INSTANTIATE(_SIZE) \
	414	template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native<_SIZE>(void const, void const) noexcept; \
	415	template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<_SIZE>( \
	416	void const, void const, uint64_t) noexcept; \
	417	template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native<_SIZE>(void const*) noexcept; \
	418	template _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native<_SIZE>(void const*) noexcept;
	419
	420	_LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_INSTANTIATE)
	421
	422	# undef _INSTANTIATE
	423
	424	#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
	425
	426	template _LIBCPP_EXPORTED_FROM_ABI void
	427	__atomic_wait_native<sizeof(__cxx_contention_t)>(void const* __address, void const* __old_value) noexcept;
	428
	429	template _LIBCPP_EXPORTED_FROM_ABI void __atomic_wait_native_with_timeout<sizeof(__cxx_contention_t)>(
	430	void const* __address, void const* __old_value, uint64_t) noexcept;
	431
	432	template _LIBCPP_EXPORTED_FROM_ABI void
	433	__atomic_notify_one_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
	434
	435	template _LIBCPP_EXPORTED_FROM_ABI void
	436	__atomic_notify_all_native<sizeof(__cxx_contention_t)>(void const* __location) noexcept;
	437
	438	#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
	439
	440	// =============================================================
	441	// Old dylib exported symbols, for backwards compatibility
	442	// =============================================================
	443	_LIBCPP_DIAGNOSTIC_PUSH
	444	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
	445
175	446	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(void const volatile* __location) noexcept {
176		__libcpp_atomic_notify(__location);
	447	__atomic_notify_global_table(const_cast<void const*>(__location));
177	448	}
	449
178	450	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(void const volatile* __location) noexcept {
179		__libcpp_atomic_notify(__location);
	451	__atomic_notify_global_table(const_cast<void const*>(__location));
180	452	}
	453
181	454	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t __libcpp_atomic_monitor(void const volatile* __location) noexcept {
182		auto const __entry = __libcpp_contention_state(__location);
183		return __libcpp_contention_monitor_for_wait(&__entry->__contention_state, &__entry->__platform_state);
	455	auto const __entry = __get_global_contention_state(const_cast<void const*>(__location));
	456	return __cxx_atomic_load(&__entry->__platform_state, memory_order_acquire);
184	457	}
	458
185	459	_LIBCPP_EXPORTED_FROM_ABI void
186	460	__libcpp_atomic_wait(void const volatile* __location, __cxx_contention_t __old_value) noexcept {
187		auto const __entry = __libcpp_contention_state(__location);
188		__libcpp_contention_wait(&__entry->__contention_state, &__entry->__platform_state, __old_value);
	461	auto const __entry = __get_global_contention_state(const_cast<void const*>(__location));
	462	__contention_wait<sizeof(__cxx_atomic_contention_t)>(
	463	&__entry->__waiter_count, &__entry->__platform_state, &__old_value, NoTimeout{});
189	464	}
190	465
191		/* When the incoming atomic happens to be the platform wait size, we still need to use the
192		table for the contention detection, but we can use the atomic directly for the wait. */
193
194	466	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_one(__cxx_atomic_contention_t const volatile* __location) noexcept {
195		__libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, true);
	467	auto __location_cast = const_cast<const void>(static_cast<const volatile void>(__location));
	468	__contention_notify<sizeof(__cxx_atomic_contention_t)>(
	469	__get_native_waiter_count(__location_cast), __location_cast, true);
196	470	}
	471
197	472	_LIBCPP_EXPORTED_FROM_ABI void __cxx_atomic_notify_all(__cxx_atomic_contention_t const volatile* __location) noexcept {
198		__libcpp_contention_notify(&__libcpp_contention_state(__location)->__contention_state, __location, false);
199		}
200		// This function is never used, but still exported for ABI compatibility.
201		_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
202		__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
203		return __libcpp_contention_monitor_for_wait(&__libcpp_contention_state(__location)->__contention_state, __location);
	473	auto __location_cast = const_cast<const void>(static_cast<const volatile void>(__location));
	474	__contention_notify<sizeof(__cxx_atomic_contention_t)>(
	475	__get_native_waiter_count(__location_cast), __location_cast, false);
204	476	}
	477
205	478	_LIBCPP_EXPORTED_FROM_ABI void
206	479	__libcpp_atomic_wait(__cxx_atomic_contention_t const volatile* __location, __cxx_contention_t __old_value) noexcept {
207		__libcpp_contention_wait(&__libcpp_contention_state(__location)->__contention_state, __location, __old_value);
	480	auto __location_cast = const_cast<const void>(static_cast<const volatile void>(__location));
	481	__contention_wait<sizeof(__cxx_atomic_contention_t)>(
	482	__get_native_waiter_count(__location_cast), __location_cast, &__old_value, NoTimeout{});
208	483	}
209	484
	485	// this function is even unused in the old ABI
	486	_LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
	487	__libcpp_atomic_monitor(__cxx_atomic_contention_t const volatile* __location) noexcept {
	488	return __cxx_atomic_load(__location, memory_order_acquire);
	489	}
	490
	491	_LIBCPP_DIAGNOSTIC_POP
	492
210	493	_LIBCPP_END_NAMESPACE_STD
	494
	495	_LIBCPP_POP_MACROS

lib/libcxx/src/barrier.cpp+4-3

...	...	@@ -60,11 +60,12 @@ public:
60	60	_LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base* __construct_barrier_algorithm_base(ptrdiff_t& __expected) {
61	61	return new __barrier_algorithm_base(__expected);
62	62	}
63		_LIBCPP_EXPORTED_FROM_ABI bool
64		__arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept {
	63	_LIBCPP_EXPORTED_FROM_ABI bool __arrive_barrier_algorithm_base(
	64	_LIBCPP_NOESCAPE __barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept {
65	65	return __barrier->__arrive(__old_phase);
66	66	}
67		_LIBCPP_EXPORTED_FROM_ABI void __destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) noexcept {
	67	_LIBCPP_EXPORTED_FROM_ABI void
	68	__destroy_barrier_algorithm_base(_LIBCPP_NOESCAPE __barrier_algorithm_base* __barrier) noexcept {
68	69	delete __barrier;
69	70	}
70	71

lib/libcxx/src/charconv.cpp+6-3

...	...	@@ -14,17 +14,20 @@
14	14
15	15	_LIBCPP_BEGIN_NAMESPACE_STD
16	16
17		#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
	17	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 15
18	18
19	19	namespace __itoa {
20	20
	21	_LIBCPP_DIAGNOSTIC_PUSH
	22	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
	23	// These functions exist for ABI compatibility, so we don't ever want a declaration prior to the definition.
21	24	_LIBCPP_EXPORTED_FROM_ABI char* __u32toa(uint32_t value, char* buffer) noexcept { return __base_10_u32(buffer, value); }
22
23	25	_LIBCPP_EXPORTED_FROM_ABI char* __u64toa(uint64_t value, char* buffer) noexcept { return __base_10_u64(buffer, value); }
	26	_LIBCPP_DIAGNOSTIC_POP
24	27
25	28	} // namespace __itoa
26	29
27		#endif // _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
	30	#endif // _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 15
28	31
29	32	// The original version of floating-point to_chars was written by Microsoft and
30	33	// contributed with the following license.

lib/libcxx/src/condition_variable_destructor.cpp+1-1

...	...	@@ -14,7 +14,7 @@
14	14	#include <__config>
15	15	#include <__thread/support.h>
16	16
17		#if _LIBCPP_ABI_VERSION == 1 \|\| !_LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
	17	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 9 \|\| !_LIBCPP_HAS_TRIVIAL_CONDVAR_DESTRUCTION
18	18	# define NEEDS_CONDVAR_DESTRUCTOR
19	19	#endif
20	20

lib/libcxx/src/error_category.cpp+3-1

...	...	@@ -8,7 +8,9 @@
8	8
9	9	#include <__config>
10	10
11		#ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
	11	// This has technically been removed in LLVM 3.4
	12	#if !defined(_LIBCPP_OBJECT_FORMAT_COFF) && !defined(_LIBCPP_OBJECT_FORMAT_XCOFF) && \
	13	_LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 4
12	14	# define _LIBCPP_ERROR_CATEGORY_DEFINE_LEGACY_INLINE_FUNCTIONS
13	15	#endif
14	16

lib/libcxx/src/exception.cpp+2-10

...	...	@@ -9,20 +9,12 @@
9	9	#define _LIBCPP_ENABLE_CXX20_REMOVED_UNCAUGHT_EXCEPTION
10	10	#define _LIBCPP_DISABLE_DEPRECATION_WARNINGS
11	11
12		#include <exception>
13		#include <new>
14		#include <typeinfo>
15
16		#if defined(LIBCXXRT) \|\| defined(LIBCXX_BUILDING_LIBCXXABI)
17		# include <cxxabi.h>
18		using namespace __cxxabiv1;
19		# define HAVE_DEPENDENT_EH_ABI 1
20		#endif
	12	#include <__config>
21	13
22	14	#if defined(_LIBCPP_ABI_MICROSOFT)
23	15	# include "support/runtime/exception_msvc.ipp"
24	16	# include "support/runtime/exception_pointer_msvc.ipp"
25		#elif defined(_LIBCPPABI_VERSION)
	17	#elif defined(LIBCXX_BUILDING_LIBCXXABI)
26	18	# include "support/runtime/exception_libcxxabi.ipp"
27	19	# include "support/runtime/exception_pointer_cxxabi.ipp"
28	20	#elif defined(LIBCXXRT)

lib/libcxx/src/experimental/time_zone.cpp+1-1

...	...	@@ -720,7 +720,7 @@ __get_sys_info(sys_seconds __time,
720	720	// Iff the "offsets" are the same '__current.__end' is replaced with
721	721	// '__next.__end', which effectively merges the two objects in one object. The
722	722	// function returns true if a merge occurred.
723		[[nodiscard]] bool __merge_continuation(sys_info& __current, const sys_info& __next) {
	723	[[nodiscard]] static bool __merge_continuation(sys_info& __current, const sys_info& __next) {
724	724	if (__current.end != __next.begin)
725	725	return false;
726	726

lib/libcxx/src/experimental/tzdb.cpp+4-1

...	...	@@ -49,6 +49,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
49	49
50	50	namespace chrono {
51	51
	52	_LIBCPP_DIAGNOSTIC_PUSH
	53	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
52	54	// This function is weak so it can be overriden in the tests. The
53	55	// declaration is in the test header test/support/test_tzdb.h
54	56	_LIBCPP_WEAK string_view __libcpp_tzdb_directory() {
...	...	@@ -59,6 +61,7 @@ _LIBCPP_WEAK string_view __libcpp_tzdb_directory() {
59	61	abort();
60	62	#endif
61	63	}
	64	_LIBCPP_DIAGNOSTIC_POP
62	65
63	66	//===----------------------------------------------------------------------===//
64	67	// Details
...	...	@@ -767,7 +770,7 @@ void __init_tzdb(tzdb& __tzdb, __tz::__rules_storage_type& __rules) {
767	770	// On Linux systems it seems /etc/timezone is deprecated and being phased out.
768	771	// This file is used when /etc/localtime does not exist, or when it exists but
769	772	// is not a symlink. For more information and links see
770		// https://github.com/llvm/llvm-project/issues/105634
	773	// https://llvm.org/PR105634
771	774
772	775	string __name = chrono::__current_zone_environment();
773	776

lib/libcxx/src/filesystem/error.h+4-22

...	...	@@ -128,17 +128,8 @@ struct ErrorHandler {
128	128	T report(const error_code& ec, const char* msg, ...) const {
129	129	va_list ap;
130	130	va_start(ap, msg);
131		#if _LIBCPP_HAS_EXCEPTIONS
132		try {
133		#endif // _LIBCPP_HAS_EXCEPTIONS
134		report_impl(ec, msg, ap);
135		#if _LIBCPP_HAS_EXCEPTIONS
136		} catch (...) {
137		va_end(ap);
138		throw;
139		}
140		#endif // _LIBCPP_HAS_EXCEPTIONS
141		va_end(ap);
	131	__scope_guard guard([&] { va_end(ap); });
	132	report_impl(ec, msg, ap);
142	133	return error_value<T>();
143	134	}
144	135
...	...	@@ -148,17 +139,8 @@ struct ErrorHandler {
148	139	T report(errc const& err, const char* msg, ...) const {
149	140	va_list ap;
150	141	va_start(ap, msg);
151		#if _LIBCPP_HAS_EXCEPTIONS
152		try {
153		#endif // _LIBCPP_HAS_EXCEPTIONS
154		report_impl(make_error_code(err), msg, ap);
155		#if _LIBCPP_HAS_EXCEPTIONS
156		} catch (...) {
157		va_end(ap);
158		throw;
159		}
160		#endif // _LIBCPP_HAS_EXCEPTIONS
161		va_end(ap);
	142	__scope_guard guard([&] { va_end(ap); });
	143	report_impl(make_error_code(err), msg, ap);
162	144	return error_value<T>();
163	145	}
164	146

lib/libcxx/src/filesystem/format_string.h+9-18

...	...	@@ -11,6 +11,7 @@
11	11
12	12	#include <__assert>
13	13	#include <__config>
	14	#include <__utility/scope_guard.h>
14	15	#include <array>
15	16	#include <cstdarg>
16	17	#include <cstddef>
...	...	@@ -34,20 +35,19 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 0) string vformat_string(const ch
34	35
35	36	va_list apcopy;
36	37	va_copy(apcopy, ap);
37		int ret = ::vsnprintf(buf.data(), buf.size(), msg, apcopy);
	38	int size = ::vsnprintf(buf.data(), buf.size(), msg, apcopy);
38	39	va_end(apcopy);
39	40
40	41	string result;
41		if (static_cast<size_t>(ret) < buf.size()) {
42		result.assign(buf.data(), static_cast<size_t>(ret));
	42	if (static_cast<size_t>(size) < buf.size()) {
	43	result.assign(buf.data(), static_cast<size_t>(size));
43	44	} else {
44	45	// we did not provide a long enough buffer on our first attempt. The
45	46	// return value is the number of bytes (excluding the null byte) that are
46	47	// needed for formatting.
47		size_t size_with_null = static_cast<size_t>(ret) + 1;
48		result.__resize_default_init(size_with_null - 1);
49		ret = ::vsnprintf(&result[0], size_with_null, msg, ap);
50		_LIBCPP_ASSERT_INTERNAL(static_cast<size_t>(ret) == (size_with_null - 1), "TODO");
	48	result.resize_and_overwrite(size, [&](char* res, size_t n) { return ::vsnprintf(res, n, msg, ap); });
	49	_LIBCPP_ASSERT_INTERNAL(static_cast<size_t>(size) == result.size(),
	50	"vsnprintf did not result in the same number of characters as the first attempt?");
51	51	}
52	52	return result;
53	53	}
...	...	@@ -56,17 +56,8 @@ inline _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2) string format_string(const cha
56	56	string ret;
57	57	va_list ap;
58	58	va_start(ap, msg);
59		#if _LIBCPP_HAS_EXCEPTIONS
60		try {
61		#endif // _LIBCPP_HAS_EXCEPTIONS
62		ret = detail::vformat_string(msg, ap);
63		#if _LIBCPP_HAS_EXCEPTIONS
64		} catch (...) {
65		va_end(ap);
66		throw;
67		}
68		#endif // _LIBCPP_HAS_EXCEPTIONS
69		va_end(ap);
	59	__scope_guard guard([&] { va_end(ap); });
	60	ret = detail::vformat_string(msg, ap);
70	61	return ret;
71	62	}
72	63

lib/libcxx/src/filesystem/int128_builtins.cpp+2

...	...	@@ -16,6 +16,8 @@
16	16	#include <__config>
17	17	#include <climits>
18	18
	19	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes") // See the FIXME above
	20
19	21	#if _LIBCPP_HAS_INT128
20	22
21	23	extern "C" __attribute__((no_sanitize("undefined"))) _LIBCPP_EXPORTED_FROM_ABI __int128_t

lib/libcxx/src/filesystem/operations.cpp+2-9

...	...	@@ -41,17 +41,10 @@
41	41	#include <time.h>
42	42
43	43	// since Linux 4.5 and FreeBSD 13, but the Linux libc wrapper is only provided by glibc >= 2.27 and musl
44		#if defined(__linux__)
45		# if defined(_LIBCPP_GLIBC_PREREQ)
46		# if _LIBCPP_GLIBC_PREREQ(2, 27)
47		# define _LIBCPP_FILESYSTEM_USE_COPY_FILE_RANGE
48		# endif
49		# elif _LIBCPP_HAS_MUSL_LIBC
50		# define _LIBCPP_FILESYSTEM_USE_COPY_FILE_RANGE
51		# endif
52		#elif defined(__FreeBSD__)
	44	#if _LIBCPP_GLIBC_PREREQ(2, 27) \|\| _LIBCPP_HAS_MUSL_LIBC \|\| defined(__FreeBSD__)
53	45	# define _LIBCPP_FILESYSTEM_USE_COPY_FILE_RANGE
54	46	#endif
	47
55	48	#if __has_include(<sys/sendfile.h>)
56	49	# include <sys/sendfile.h>
57	50	# define _LIBCPP_FILESYSTEM_USE_SENDFILE

lib/libcxx/src/filesystem/path.cpp+3-1

...	...	@@ -292,7 +292,9 @@ path path::lexically_relative(const path& base) const {
292	292	// return a path constructed with 'n' dot-dot elements, followed by the
293	293	// elements of '*this' after the mismatch.
294	294	path Result;
295		// FIXME: Reserve enough room in Result that it won't have to re-allocate.
	295	constexpr size_t ElemSize = 2; // ".."
	296	constexpr size_t SeparatorSize = 1; // separator is always a single char
	297	Result.__reserve(ElemCount * (ElemSize + SeparatorSize) + SeparatorSize + PP.Path.size());
296	298	while (ElemCount--)
297	299	Result /= PATHSTR("..");
298	300	for (; PP; ++PP)

lib/libcxx/src/include/aligned_alloc.h created+65

...	...	@@ -0,0 +1,65 @@
	1	//===----------------------------------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef _LIBCPP_SRC_ALIGNED_ALLOC_H
	10	#define _LIBCPP_SRC_ALIGNED_ALLOC_H
	11
	12	#include <__config>
	13	#include <cstdlib>
	14
	15	#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
	16	# pragma GCC system_header
	17	#endif
	18
	19	_LIBCPP_BEGIN_NAMESPACE_STD
	20
	21	#if _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
	22
	23	// Low-level helpers to call the aligned allocation and deallocation functions
	24	// on the target platform. This is used to implement libc++'s own memory
	25	// allocation routines -- if you need to allocate memory inside the library,
	26	// chances are that you want to use `__libcpp_allocate` instead.
	27	//
	28	// Returns the allocated memory, or `nullptr` on failure.
	29	inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_aligned_alloc(std::size_t __alignment, std::size_t __size) {
	30	# if defined(_LIBCPP_MSVCRT_LIKE)
	31	return ::_aligned_malloc(__size, __alignment);
	32
	33	// Android only provides aligned_alloc when targeting API 28 or higher.
	34	# elif !defined(__ANDROID__) \|\| __ANDROID_API__ >= 28
	35	// aligned_alloc() requires that __size is a multiple of __alignment,
	36	// but for C++ [new.delete.general], only states "if the value of an
	37	// alignment argument passed to any of these functions is not a valid
	38	// alignment value, the behavior is undefined".
	39	// To handle calls such as ::operator new(1, std::align_val_t(128)), we
	40	// round __size up to the next multiple of __alignment.
	41	size_t __rounded_size = (__size + __alignment - 1) & ~(__alignment - 1);
	42	// Rounding up could have wrapped around to zero, so we have to add another
	43	// max() ternary to the actual call site to avoid succeeded in that case.
	44	return ::aligned_alloc(__alignment, __size > __rounded_size ? __size : __rounded_size);
	45	# else
	46	void* __result = nullptr;
	47	(void)::posix_memalign(&__result, __alignment, __size);
	48	// If posix_memalign fails, __result is unmodified so we still return `nullptr`.
	49	return __result;
	50	# endif
	51	}
	52
	53	inline _LIBCPP_HIDE_FROM_ABI void __libcpp_aligned_free(void* __ptr) {
	54	# if defined(_LIBCPP_MSVCRT_LIKE)
	55	::_aligned_free(__ptr);
	56	# else
	57	::free(__ptr);
	58	# endif
	59	}
	60
	61	#endif // _LIBCPP_HAS_LIBRARY_ALIGNED_ALLOCATION
	62
	63	_LIBCPP_END_NAMESPACE_STD
	64
	65	#endif // _LIBCPP_SRC_ALIGNED_ALLOC_H

lib/libcxx/src/include/config_elast.h+1-1

...	...	@@ -23,7 +23,7 @@
23	23	# define _LIBCPP_ELAST ELAST
24	24	#elif defined(__LLVM_LIBC__)
25	25	// No _LIBCPP_ELAST needed for LLVM libc
26		#elif defined(_NEWLIB_VERSION)
	26	#elif _LIBCPP_LIBC_NEWLIB
27	27	# define _LIBCPP_ELAST __ELASTERROR
28	28	#elif defined(__NuttX__)
29	29	// No _LIBCPP_ELAST needed on NuttX

lib/libcxx/src/include/from_chars_floating_point.h+11-7

...	...	@@ -9,11 +9,6 @@
9	9	#ifndef _LIBCPP_SRC_INCLUDE_FROM_CHARS_FLOATING_POINT_H
10	10	#define _LIBCPP_SRC_INCLUDE_FROM_CHARS_FLOATING_POINT_H
11	11
12		// These headers are in the shared LLVM-libc header library.
13		#include "shared/fp_bits.h"
14		#include "shared/str_to_float.h"
15		#include "shared/str_to_integer.h"
16
17	12	#include <__assert>
18	13	#include <__config>
19	14	#include <cctype>
...	...	@@ -21,6 +16,15 @@
21	16	#include <concepts>
22	17	#include <limits>
23	18
	19	// Make sure we use libc++'s assertion machinery within the shared code we use
	20	// from LLVM libc.
	21	#define LIBC_ASSERT(cond) _LIBCPP_ASSERT((cond), _LIBCPP_TOSTRING(cond))
	22
	23	// These headers are in the shared LLVM-libc header library.
	24	#include "shared/fp_bits.h"
	25	#include "shared/str_to_float.h"
	26	#include "shared/str_to_integer.h"
	27
24	28	// Included for the _Floating_type_traits class
25	29	#include "to_chars_floating_point.h"
26	30
...	...	@@ -193,7 +197,7 @@ struct __exponent_result {
193	197	// __offset, 0, false. This allows using the results unconditionally, the
194	198	// __present is important for the scientific notation, where the value is
195	199	// mandatory.
196		__exponent_result __parse_exponent(const char* __input, size_t __n, size_t __offset, char __marker) {
	200	static __exponent_result __parse_exponent(const char* __input, size_t __n, size_t __offset, char __marker) {
197	201	if (__offset + 1 < __n && // an exponent always needs at least one digit.
198	202	std::tolower(__input[__offset]) == __marker && //
199	203	!std::isspace(__input[__offset + 1]) // leading whitespace is not allowed.
...	...	@@ -213,7 +217,7 @@ __exponent_result __parse_exponent(const char* __input, size_t __n, size_t __off
213	217	}
214	218
215	219	// Here we do this operation as int64 to avoid overflow.
216		int32_t __merge_exponents(int64_t __fractional, int64_t __exponent, int __max_biased_exponent) {
	220	static int32_t __merge_exponents(int64_t __fractional, int64_t __exponent, int __max_biased_exponent) {
217	221	int64_t __sum = __fractional + __exponent;
218	222
219	223	if (__sum > __max_biased_exponent)

lib/libcxx/src/include/overridable_function.h+8-10

...	...	@@ -29,12 +29,12 @@
29	29	// This is a low-level utility which does not work on all platforms, since it needs
30	30	// to make assumptions about the object file format in use. Furthermore, it requires
31	31	// the "base definition" of the function (the one we want to check whether it has been
32		// overridden) to be defined using the _LIBCPP_OVERRIDABLE_FUNCTION macro.
	32	// overridden) to be defined using the OVERRIDABLE_FUNCTION macro.
33	33	//
34	34	// This currently works with Mach-O files (used on Darwin) and with ELF files (used on Linux
35	35	// and others). On platforms where we know how to implement this detection, the macro
36	36	// _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION is defined to 1, and it is defined to 0 on
37		// other platforms. The _LIBCPP_OVERRIDABLE_FUNCTION macro is defined to perform a normal
	37	// other platforms. The OVERRIDABLE_FUNCTION macro is defined to perform a normal
38	38	// function definition on unsupported platforms so that it can be used to define functions
39	39	// regardless of whether detection is actually supported.
40	40	//
...	...	@@ -44,7 +44,7 @@
44	44	// Let's say we want to check whether a weak function `f` has been overridden by the user.
45	45	// The general mechanism works by placing `f`'s definition (in the libc++ built library)
46	46	// inside a special section, which we do using the `__section__` attribute via the
47		// _LIBCPP_OVERRIDABLE_FUNCTION macro.
	47	// OVERRIDABLE_FUNCTION macro.
48	48	//
49	49	// Then, when comes the time to check whether the function has been overridden, we take
50	50	// the address of the function and we check whether it falls inside the special function
...	...	@@ -66,11 +66,10 @@
66	66	#if defined(_LIBCPP_OBJECT_FORMAT_MACHO)
67	67
68	68	# define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
69		# define _LIBCPP_OVERRIDABLE_FUNCTION(type, name, arglist) \
70		__attribute__((__section__("__TEXT,__lcxx_override,regular,pure_instructions"))) _LIBCPP_WEAK type name arglist
	69	# define OVERRIDABLE_FUNCTION \
	70	__attribute__((__section__("__TEXT,__lcxx_override,regular,pure_instructions"))) _LIBCPP_WEAK
71	71
72		_LIBCPP_BEGIN_NAMESPACE_STD
73		template <typename T, T* _Func>
	72	_LIBCPP_BEGIN_NAMESPACE_STD template <typename T, T* _Func>
74	73	_LIBCPP_HIDE_FROM_ABI inline bool __is_function_overridden() noexcept {
75	74	// Declare two dummy bytes and give them these special `__asm` values. These values are
76	75	// defined by the linker, which means that referring to `&__lcxx_override_start` will
...	...	@@ -100,8 +99,7 @@ _LIBCPP_END_NAMESPACE_STD
100	99	#elif defined(_LIBCPP_OBJECT_FORMAT_ELF) && !defined(__NVPTX__)
101	100
102	101	# define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 1
103		# define _LIBCPP_OVERRIDABLE_FUNCTION(type, name, arglist) \
104		__attribute__((__section__("__lcxx_override"))) _LIBCPP_WEAK type name arglist
	102	# define OVERRIDABLE_FUNCTION __attribute__((__section__("__lcxx_override"))) _LIBCPP_WEAK
105	103
106	104	// This is very similar to what we do for Mach-O above. The ELF linker will implicitly define
107	105	// variables with those names corresponding to the start and the end of the section.
...	...	@@ -129,7 +127,7 @@ _LIBCPP_END_NAMESPACE_STD
129	127	#else
130	128
131	129	# define _LIBCPP_CAN_DETECT_OVERRIDDEN_FUNCTION 0
132		# define _LIBCPP_OVERRIDABLE_FUNCTION(type, name, arglist) _LIBCPP_WEAK type name arglist
	130	# define OVERRIDABLE_FUNCTION _LIBCPP_WEAK
133	131
134	132	#endif
135	133

lib/libcxx/src/iostream.cpp+56-37

...	...	@@ -16,24 +16,40 @@
16	16
17	17	_LIBCPP_BEGIN_NAMESPACE_STD
18	18
	19	// This file implements the various stream objects provided inside <iostream>. We're doing some ODR violations in here,
	20	// so this quite fragile. Specifically, the size of the stream objects (i.e. cout, cin etc.) needs to stay the same.
	21	// For that reason, we have `stream` and `stream_data` separated into two objects. The public `stream` objects only
	22	// contain the actual stream, while the private `stream_data` objects contains the `basic_streambuf` we're using as well
	23	// as the mbstate_t. `stream_data` objects are only accessible within the library, so they aren't ABI sensitive and we
	24	// can change them as we want.
	25
	26	template <class StreamT>
	27	union stream {
	28	constexpr stream() {}
	29	stream(const stream&) = delete;
	30	stream& operator=(const stream&) = delete;
	31	constexpr ~stream() {}
	32
	33	StreamT value;
	34	};
	35
19	36	template <class StreamT, class BufferT>
20	37	union stream_data {
21	38	constexpr stream_data() {}
22	39	constexpr ~stream_data() {}
23	40	struct {
24		// The stream has to be the first element, since that's referenced by the stream declarations in <iostream>
25		StreamT stream;
26	41	BufferT buffer;
27	42	mbstate_t mb;
28	43	};
29
30		void init(FILE* stdstream) {
31		mb = {};
32		std::construct_at(&buffer, stdstream, &mb);
33		std::construct_at(&stream, &buffer);
34		}
35	44	};
36	45
	46	template <class StreamT, class BufferT>
	47	void init_stream(FILE* stdstream, stream<StreamT>& stream, stream_data<StreamT, BufferT>& data) {
	48	data.mb = {};
	49	std::construct_at(&data.buffer, stdstream, &data.mb);
	50	std::construct_at(&stream.value, &data.buffer);
	51	}
	52
37	53	#define CHAR_MANGLING_char "D"
38	54	#define CHAR_MANGLING_wchar_t "_W"
39	55	#define CHAR_MANGLING(CharT) CHAR_MANGLING_##CharT
...	...	@@ -46,25 +62,28 @@ union stream_data {
46	62
47	63	#ifdef _LIBCPP_ABI_MICROSOFT
48	64	# define STREAM(StreamT, BufferT, CharT, var) \
49		STRING_DATA_CONSTINIT stream_data<StreamT<CharT>, BufferT<CharT>> var __asm__( \
	65	STRING_DATA_CONSTINIT stream_data<StreamT<CharT>, BufferT<CharT>> var##_data; \
	66	_LIBCPP_EXPORTED_FROM_ABI STRING_DATA_CONSTINIT stream<StreamT<CharT>> var __asm__( \
50	67	"?" #var "@" ABI_NAMESPACE_STR "@std@@3V?$" #StreamT \
51	68	"@" CHAR_MANGLING(CharT) "U?$char_traits@" CHAR_MANGLING(CharT) "@" ABI_NAMESPACE_STR "@std@@@12@A")
52	69	#else
53		# define STREAM(StreamT, BufferT, CharT, var) STRING_DATA_CONSTINIT stream_data<StreamT<CharT>, BufferT<CharT>> var
	70	# define STREAM(StreamT, BufferT, CharT, var) \
	71	STRING_DATA_CONSTINIT stream_data<StreamT<CharT>, BufferT<CharT>> var##_data; \
	72	_LIBCPP_EXPORTED_FROM_ABI STRING_DATA_CONSTINIT stream<StreamT<CharT>> var
54	73	#endif
55	74
56	75	// These definitions and the declarations in <iostream> technically cause ODR violations, since they have different
57	76	// types (stream_data and {i,o}stream respectively). This means that <iostream> should never be included in this TU.
58	77
59		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_istream, __stdinbuf, char, cin);
60		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, cout);
61		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, cerr);
62		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, char, clog);
	78	STREAM(basic_istream, __stdinbuf, char, cin);
	79	STREAM(basic_ostream, __stdoutbuf, char, cout);
	80	STREAM(basic_ostream, __stdoutbuf, char, cerr);
	81	STREAM(basic_ostream, __stdoutbuf, char, clog);
63	82	#if _LIBCPP_HAS_WIDE_CHARACTERS
64		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_istream, __stdinbuf, wchar_t, wcin);
65		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wcout);
66		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wcerr);
67		_LIBCPP_EXPORTED_FROM_ABI STREAM(basic_ostream, __stdoutbuf, wchar_t, wclog);
	83	STREAM(basic_istream, __stdinbuf, wchar_t, wcin);
	84	STREAM(basic_ostream, __stdoutbuf, wchar_t, wcout);
	85	STREAM(basic_ostream, __stdoutbuf, wchar_t, wcerr);
	86	STREAM(basic_ostream, __stdoutbuf, wchar_t, wclog);
68	87	#endif // _LIBCPP_HAS_WIDE_CHARACTERS
69	88
70	89	// Pretend we're inside a system header so the compiler doesn't flag the use of the init_priority
...	...	@@ -98,34 +117,34 @@ public:
98	117	DoIOSInit::DoIOSInit() {
99	118	force_locale_initialization();
100	119
101		cin.init(stdin);
102		cout.init(stdout);
103		cerr.init(stderr);
104		clog.init(stderr);
	120	init_stream(stdin, cin, cin_data);
	121	init_stream(stdout, cout, cout_data);
	122	init_stream(stderr, cerr, cerr_data);
	123	init_stream(stderr, clog, clog_data);
105	124
106		cin.stream.tie(&cout.stream);
107		std::unitbuf(cerr.stream);
108		cerr.stream.tie(&cout.stream);
	125	cin.value.tie(&cout.value);
	126	std::unitbuf(cerr.value);
	127	cerr.value.tie(&cout.value);
109	128
110	129	#if _LIBCPP_HAS_WIDE_CHARACTERS
111		wcin.init(stdin);
112		wcout.init(stdout);
113		wcerr.init(stderr);
114		wclog.init(stderr);
115
116		wcin.stream.tie(&wcout.stream);
117		std::unitbuf(wcerr.stream);
118		wcerr.stream.tie(&wcout.stream);
	130	init_stream(stdin, wcin, wcin_data);
	131	init_stream(stdout, wcout, wcout_data);
	132	init_stream(stderr, wcerr, wcerr_data);
	133	init_stream(stderr, wclog, wclog_data);
	134
	135	wcin.value.tie(&wcout.value);
	136	std::unitbuf(wcerr.value);
	137	wcerr.value.tie(&wcout.value);
119	138	#endif
120	139	}
121	140
122	141	DoIOSInit::~DoIOSInit() {
123		cout.stream.flush();
124		clog.stream.flush();
	142	cout.value.flush();
	143	clog.value.flush();
125	144
126	145	#if _LIBCPP_HAS_WIDE_CHARACTERS
127		wcout.stream.flush();
128		wclog.stream.flush();
	146	wcout.value.flush();
	147	wclog.value.flush();
129	148	#endif
130	149	}
131	150

lib/libcxx/src/locale.cpp+213-196

...	...	@@ -60,9 +60,8 @@ struct __libcpp_unique_locale {
60	60
61	61	__locale::__locale_t __loc_;
62	62
63		private:
64		__libcpp_unique_locale(__libcpp_unique_locale const&);
65		__libcpp_unique_locale& operator=(__libcpp_unique_locale const&);
	63	__libcpp_unique_locale(__libcpp_unique_locale const&) = delete;
	64	__libcpp_unique_locale& operator=(__libcpp_unique_locale const&) = delete;
66	65	};
67	66
68	67	#ifdef __cloc_defined
...	...	@@ -88,16 +87,6 @@ T& make(Args... args) {
88	87	return *obj;
89	88	}
90	89
91		template <typename T, size_t N>
92		inline constexpr size_t countof(const T (&)[N]) {
93		return N;
94		}
95
96		template <typename T>
97		inline constexpr size_t countof(const T* const begin, const T* const end) {
98		return static_cast<size_t>(end - begin);
99		}
100
101	90	string build_name(const string& other, const string& one, locale::category c) {
102	91	if (other == "" \|\| one == "")
103	92	return "*";
...	...	@@ -215,63 +204,58 @@ locale::__imp::__imp(size_t refs) : facet(refs), facets_(N), name_("C") {
215	204	}
216	205
217	206	locale::__imp::__imp(const string& name, size_t refs) : facet(refs), facets_(N), name_(name) {
218		#if _LIBCPP_HAS_EXCEPTIONS
219		try {
220		#endif // _LIBCPP_HAS_EXCEPTIONS
221		facets_ = locale::classic().__locale_->facets_;
	207	__exception_guard guard([&] {
222	208	for (unsigned i = 0; i < facets_.size(); ++i)
223	209	if (facets_[i])
224		facets_[i]->__add_shared();
225		install(new collate_byname<char>(name_));
	210	facets_[i]->__release_shared();
	211	});
	212	facets_ = locale::classic().__locale_->facets_;
	213	for (unsigned i = 0; i < facets_.size(); ++i)
	214	if (facets_[i])
	215	facets_[i]->__add_shared();
	216	install(new collate_byname<char>(name_));
226	217	#if _LIBCPP_HAS_WIDE_CHARACTERS
227		install(new collate_byname<wchar_t>(name_));
	218	install(new collate_byname<wchar_t>(name_));
228	219	#endif
229		install(new ctype_byname<char>(name_));
	220	install(new ctype_byname<char>(name_));
230	221	#if _LIBCPP_HAS_WIDE_CHARACTERS
231		install(new ctype_byname<wchar_t>(name_));
	222	install(new ctype_byname<wchar_t>(name_));
232	223	#endif
233		install(new codecvt_byname<char, char, mbstate_t>(name_));
	224	install(new codecvt_byname<char, char, mbstate_t>(name_));
234	225	#if _LIBCPP_HAS_WIDE_CHARACTERS
235		install(new codecvt_byname<wchar_t, char, mbstate_t>(name_));
	226	install(new codecvt_byname<wchar_t, char, mbstate_t>(name_));
236	227	#endif
237		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
238		install(new codecvt_byname<char16_t, char, mbstate_t>(name_));
239		install(new codecvt_byname<char32_t, char, mbstate_t>(name_));
240		_LIBCPP_SUPPRESS_DEPRECATED_POP
	228	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
	229	install(new codecvt_byname<char16_t, char, mbstate_t>(name_));
	230	install(new codecvt_byname<char32_t, char, mbstate_t>(name_));
	231	_LIBCPP_SUPPRESS_DEPRECATED_POP
241	232	#if _LIBCPP_HAS_CHAR8_T
242		install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name_));
243		install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name_));
	233	install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name_));
	234	install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name_));
244	235	#endif
245		install(new numpunct_byname<char>(name_));
	236	install(new numpunct_byname<char>(name_));
246	237	#if _LIBCPP_HAS_WIDE_CHARACTERS
247		install(new numpunct_byname<wchar_t>(name_));
	238	install(new numpunct_byname<wchar_t>(name_));
248	239	#endif
249		install(new moneypunct_byname<char, false>(name_));
250		install(new moneypunct_byname<char, true>(name_));
	240	install(new moneypunct_byname<char, false>(name_));
	241	install(new moneypunct_byname<char, true>(name_));
251	242	#if _LIBCPP_HAS_WIDE_CHARACTERS
252		install(new moneypunct_byname<wchar_t, false>(name_));
253		install(new moneypunct_byname<wchar_t, true>(name_));
	243	install(new moneypunct_byname<wchar_t, false>(name_));
	244	install(new moneypunct_byname<wchar_t, true>(name_));
254	245	#endif
255		install(new time_get_byname<char>(name_));
	246	install(new time_get_byname<char>(name_));
256	247	#if _LIBCPP_HAS_WIDE_CHARACTERS
257		install(new time_get_byname<wchar_t>(name_));
	248	install(new time_get_byname<wchar_t>(name_));
258	249	#endif
259		install(new time_put_byname<char>(name_));
	250	install(new time_put_byname<char>(name_));
260	251	#if _LIBCPP_HAS_WIDE_CHARACTERS
261		install(new time_put_byname<wchar_t>(name_));
	252	install(new time_put_byname<wchar_t>(name_));
262	253	#endif
263		install(new messages_byname<char>(name_));
	254	install(new messages_byname<char>(name_));
264	255	#if _LIBCPP_HAS_WIDE_CHARACTERS
265		install(new messages_byname<wchar_t>(name_));
	256	install(new messages_byname<wchar_t>(name_));
266	257	#endif
267		#if _LIBCPP_HAS_EXCEPTIONS
268		} catch (...) {
269		for (unsigned i = 0; i < facets_.size(); ++i)
270		if (facets_[i])
271		facets_[i]->__release_shared();
272		throw;
273		}
274		#endif // _LIBCPP_HAS_EXCEPTIONS
	258	guard.__complete();
275	259	}
276	260
277	261	locale::__imp::__imp(const __imp& other) : facets_(max<size_t>(N, other.facets_.size())), name_(other.name_) {
...	...	@@ -287,71 +271,66 @@ locale::__imp::__imp(const __imp& other, const string& name, locale::category c)
287	271	for (unsigned i = 0; i < facets_.size(); ++i)
288	272	if (facets_[i])
289	273	facets_[i]->__add_shared();
290		#if _LIBCPP_HAS_EXCEPTIONS
291		try {
292		#endif // _LIBCPP_HAS_EXCEPTIONS
293		if (c & locale::collate) {
294		install(new collate_byname<char>(name));
	274	__exception_guard guard([&] {
	275	for (unsigned i = 0; i < facets_.size(); ++i)
	276	if (facets_[i])
	277	facets_[i]->__release_shared();
	278	});
	279	if (c & locale::collate) {
	280	install(new collate_byname<char>(name));
295	281	#if _LIBCPP_HAS_WIDE_CHARACTERS
296		install(new collate_byname<wchar_t>(name));
	282	install(new collate_byname<wchar_t>(name));
297	283	#endif
298		}
299		if (c & locale::ctype) {
300		install(new ctype_byname<char>(name));
	284	}
	285	if (c & locale::ctype) {
	286	install(new ctype_byname<char>(name));
301	287	#if _LIBCPP_HAS_WIDE_CHARACTERS
302		install(new ctype_byname<wchar_t>(name));
	288	install(new ctype_byname<wchar_t>(name));
303	289	#endif
304		install(new codecvt_byname<char, char, mbstate_t>(name));
	290	install(new codecvt_byname<char, char, mbstate_t>(name));
305	291	#if _LIBCPP_HAS_WIDE_CHARACTERS
306		install(new codecvt_byname<wchar_t, char, mbstate_t>(name));
	292	install(new codecvt_byname<wchar_t, char, mbstate_t>(name));
307	293	#endif
308		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
309		install(new codecvt_byname<char16_t, char, mbstate_t>(name));
310		install(new codecvt_byname<char32_t, char, mbstate_t>(name));
311		_LIBCPP_SUPPRESS_DEPRECATED_POP
	294	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
	295	install(new codecvt_byname<char16_t, char, mbstate_t>(name));
	296	install(new codecvt_byname<char32_t, char, mbstate_t>(name));
	297	_LIBCPP_SUPPRESS_DEPRECATED_POP
312	298	#if _LIBCPP_HAS_CHAR8_T
313		install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name));
314		install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name));
	299	install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name));
	300	install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name));
315	301	#endif
316		}
317		if (c & locale::monetary) {
318		install(new moneypunct_byname<char, false>(name));
319		install(new moneypunct_byname<char, true>(name));
	302	}
	303	if (c & locale::monetary) {
	304	install(new moneypunct_byname<char, false>(name));
	305	install(new moneypunct_byname<char, true>(name));
320	306	#if _LIBCPP_HAS_WIDE_CHARACTERS
321		install(new moneypunct_byname<wchar_t, false>(name));
322		install(new moneypunct_byname<wchar_t, true>(name));
	307	install(new moneypunct_byname<wchar_t, false>(name));
	308	install(new moneypunct_byname<wchar_t, true>(name));
323	309	#endif
324		}
325		if (c & locale::numeric) {
326		install(new numpunct_byname<char>(name));
	310	}
	311	if (c & locale::numeric) {
	312	install(new numpunct_byname<char>(name));
327	313	#if _LIBCPP_HAS_WIDE_CHARACTERS
328		install(new numpunct_byname<wchar_t>(name));
	314	install(new numpunct_byname<wchar_t>(name));
329	315	#endif
330		}
331		if (c & locale::time) {
332		install(new time_get_byname<char>(name));
	316	}
	317	if (c & locale::time) {
	318	install(new time_get_byname<char>(name));
333	319	#if _LIBCPP_HAS_WIDE_CHARACTERS
334		install(new time_get_byname<wchar_t>(name));
	320	install(new time_get_byname<wchar_t>(name));
335	321	#endif
336		install(new time_put_byname<char>(name));
	322	install(new time_put_byname<char>(name));
337	323	#if _LIBCPP_HAS_WIDE_CHARACTERS
338		install(new time_put_byname<wchar_t>(name));
	324	install(new time_put_byname<wchar_t>(name));
339	325	#endif
340		}
341		if (c & locale::messages) {
342		install(new messages_byname<char>(name));
	326	}
	327	if (c & locale::messages) {
	328	install(new messages_byname<char>(name));
343	329	#if _LIBCPP_HAS_WIDE_CHARACTERS
344		install(new messages_byname<wchar_t>(name));
	330	install(new messages_byname<wchar_t>(name));
345	331	#endif
346		}
347		#if _LIBCPP_HAS_EXCEPTIONS
348		} catch (...) {
349		for (unsigned i = 0; i < facets_.size(); ++i)
350		if (facets_[i])
351		facets_[i]->__release_shared();
352		throw;
353	332	}
354		#endif // _LIBCPP_HAS_EXCEPTIONS
	333	guard.__complete();
355	334	}
356	335
357	336	template <class F>
...	...	@@ -366,87 +345,83 @@ locale::__imp::__imp(const __imp& other, const __imp& one, locale::category c)
366	345	for (unsigned i = 0; i < facets_.size(); ++i)
367	346	if (facets_[i])
368	347	facets_[i]->__add_shared();
369		#if _LIBCPP_HAS_EXCEPTIONS
370		try {
371		#endif // _LIBCPP_HAS_EXCEPTIONS
372		if (c & locale::collate) {
373		install_from<std::collate<char> >(one);
	348	__exception_guard guard([&] {
	349	for (unsigned i = 0; i < facets_.size(); ++i)
	350	if (facets_[i])
	351	facets_[i]->__release_shared();
	352	});
	353
	354	if (c & locale::collate) {
	355	install_from<std::collate<char> >(one);
374	356	#if _LIBCPP_HAS_WIDE_CHARACTERS
375		install_from<std::collate<wchar_t> >(one);
	357	install_from<std::collate<wchar_t> >(one);
376	358	#endif
377		}
378		if (c & locale::ctype) {
379		install_from<std::ctype<char> >(one);
	359	}
	360	if (c & locale::ctype) {
	361	install_from<std::ctype<char> >(one);
380	362	#if _LIBCPP_HAS_WIDE_CHARACTERS
381		install_from<std::ctype<wchar_t> >(one);
	363	install_from<std::ctype<wchar_t> >(one);
382	364	#endif
383		install_from<std::codecvt<char, char, mbstate_t> >(one);
384		_LIBCPP_SUPPRESS_DEPRECATED_PUSH
385		install_from<std::codecvt<char16_t, char, mbstate_t> >(one);
386		install_from<std::codecvt<char32_t, char, mbstate_t> >(one);
387		_LIBCPP_SUPPRESS_DEPRECATED_POP
	365	install_from<std::codecvt<char, char, mbstate_t> >(one);
	366	_LIBCPP_SUPPRESS_DEPRECATED_PUSH
	367	install_from<std::codecvt<char16_t, char, mbstate_t> >(one);
	368	install_from<std::codecvt<char32_t, char, mbstate_t> >(one);
	369	_LIBCPP_SUPPRESS_DEPRECATED_POP
388	370	#if _LIBCPP_HAS_CHAR8_T
389		install_from<std::codecvt<char16_t, char8_t, mbstate_t> >(one);
390		install_from<std::codecvt<char32_t, char8_t, mbstate_t> >(one);
	371	install_from<std::codecvt<char16_t, char8_t, mbstate_t> >(one);
	372	install_from<std::codecvt<char32_t, char8_t, mbstate_t> >(one);
391	373	#endif
392	374	#if _LIBCPP_HAS_WIDE_CHARACTERS
393		install_from<std::codecvt<wchar_t, char, mbstate_t> >(one);
	375	install_from<std::codecvt<wchar_t, char, mbstate_t> >(one);
394	376	#endif
395		}
396		if (c & locale::monetary) {
397		install_from<moneypunct<char, false> >(one);
398		install_from<moneypunct<char, true> >(one);
	377	}
	378	if (c & locale::monetary) {
	379	install_from<moneypunct<char, false> >(one);
	380	install_from<moneypunct<char, true> >(one);
399	381	#if _LIBCPP_HAS_WIDE_CHARACTERS
400		install_from<moneypunct<wchar_t, false> >(one);
401		install_from<moneypunct<wchar_t, true> >(one);
	382	install_from<moneypunct<wchar_t, false> >(one);
	383	install_from<moneypunct<wchar_t, true> >(one);
402	384	#endif
403		install_from<money_get<char> >(one);
	385	install_from<money_get<char> >(one);
404	386	#if _LIBCPP_HAS_WIDE_CHARACTERS
405		install_from<money_get<wchar_t> >(one);
	387	install_from<money_get<wchar_t> >(one);
406	388	#endif
407		install_from<money_put<char> >(one);
	389	install_from<money_put<char> >(one);
408	390	#if _LIBCPP_HAS_WIDE_CHARACTERS
409		install_from<money_put<wchar_t> >(one);
	391	install_from<money_put<wchar_t> >(one);
410	392	#endif
411		}
412		if (c & locale::numeric) {
413		install_from<numpunct<char> >(one);
	393	}
	394	if (c & locale::numeric) {
	395	install_from<numpunct<char> >(one);
414	396	#if _LIBCPP_HAS_WIDE_CHARACTERS
415		install_from<numpunct<wchar_t> >(one);
	397	install_from<numpunct<wchar_t> >(one);
416	398	#endif
417		install_from<num_get<char> >(one);
	399	install_from<num_get<char> >(one);
418	400	#if _LIBCPP_HAS_WIDE_CHARACTERS
419		install_from<num_get<wchar_t> >(one);
	401	install_from<num_get<wchar_t> >(one);
420	402	#endif
421		install_from<num_put<char> >(one);
	403	install_from<num_put<char> >(one);
422	404	#if _LIBCPP_HAS_WIDE_CHARACTERS
423		install_from<num_put<wchar_t> >(one);
	405	install_from<num_put<wchar_t> >(one);
424	406	#endif
425		}
426		if (c & locale::time) {
427		install_from<time_get<char> >(one);
	407	}
	408	if (c & locale::time) {
	409	install_from<time_get<char> >(one);
428	410	#if _LIBCPP_HAS_WIDE_CHARACTERS
429		install_from<time_get<wchar_t> >(one);
	411	install_from<time_get<wchar_t> >(one);
430	412	#endif
431		install_from<time_put<char> >(one);
	413	install_from<time_put<char> >(one);
432	414	#if _LIBCPP_HAS_WIDE_CHARACTERS
433		install_from<time_put<wchar_t> >(one);
	415	install_from<time_put<wchar_t> >(one);
434	416	#endif
435		}
436		if (c & locale::messages) {
437		install_from<std::messages<char> >(one);
	417	}
	418	if (c & locale::messages) {
	419	install_from<std::messages<char> >(one);
438	420	#if _LIBCPP_HAS_WIDE_CHARACTERS
439		install_from<std::messages<wchar_t> >(one);
	421	install_from<std::messages<wchar_t> >(one);
440	422	#endif
441		}
442		#if _LIBCPP_HAS_EXCEPTIONS
443		} catch (...) {
444		for (unsigned i = 0; i < facets_.size(); ++i)
445		if (facets_[i])
446		facets_[i]->__release_shared();
447		throw;
448	423	}
449		#endif // _LIBCPP_HAS_EXCEPTIONS
	424	guard.__complete();
450	425	}
451	426
452	427	locale::__imp::__imp(const __imp& other, facet* f, long id)
...	...	@@ -933,7 +908,7 @@ const ctype<char>::mask* ctype<char>::classic_table() noexcept {
933	908	return __pctype_func();
934	909	# elif defined(__EMSCRIPTEN__)
935	910	return *__ctype_b_loc();
936		# elif defined(_NEWLIB_VERSION)
	911	# elif _LIBCPP_LIBC_NEWLIB
937	912	// Newlib has a 257-entry table in ctype_.c, where (char)0 starts at [1].
938	913	return _ctype_ + 1;
939	914	# elif defined(_AIX)
...	...	@@ -3961,13 +3936,7 @@ wstring numpunct<wchar_t>::do_falsename() const { return L"false"; }
3961	3936
3962	3937	// numpunct_byname<char>
3963	3938
3964		numpunct_byname<char>::numpunct_byname(const char* nm, size_t refs) : numpunct<char>(refs) { __init(nm); }
3965
3966		numpunct_byname<char>::numpunct_byname(const string& nm, size_t refs) : numpunct<char>(refs) { __init(nm.c_str()); }
3967
3968		numpunct_byname<char>::~numpunct_byname() {}
3969
3970		void numpunct_byname<char>::__init(const char* nm) {
	3939	numpunct_byname<char>::numpunct_byname(const char* nm, size_t refs) : numpunct<char>(refs) {
3971	3940	typedef numpunct<char> base;
3972	3941	if (strcmp(nm, "C") != 0) {
3973	3942	__libcpp_unique_locale loc(nm);
...	...	@@ -3988,18 +3957,14 @@ void numpunct_byname<char>::__init(const char* nm) {
3988	3957	}
3989	3958	}
3990	3959
3991		// numpunct_byname<wchar_t>
3992
3993		#if _LIBCPP_HAS_WIDE_CHARACTERS
3994		numpunct_byname<wchar_t>::numpunct_byname(const char* nm, size_t refs) : numpunct<wchar_t>(refs) { __init(nm); }
	3960	numpunct_byname<char>::numpunct_byname(const string& nm, size_t refs) : numpunct_byname<char>(nm.c_str(), refs) {}
3995	3961
3996		numpunct_byname<wchar_t>::numpunct_byname(const string& nm, size_t refs) : numpunct<wchar_t>(refs) {
3997		__init(nm.c_str());
3998		}
	3962	numpunct_byname<char>::~numpunct_byname() {}
3999	3963
4000		numpunct_byname<wchar_t>::~numpunct_byname() {}
	3964	// numpunct_byname<wchar_t>
4001	3965
4002		void numpunct_byname<wchar_t>::__init(const char* nm) {
	3966	#if _LIBCPP_HAS_WIDE_CHARACTERS
	3967	numpunct_byname<wchar_t>::numpunct_byname(const char* nm, size_t refs) : numpunct<wchar_t>(refs) {
4003	3968	if (strcmp(nm, "C") != 0) {
4004	3969	__libcpp_unique_locale loc(nm);
4005	3970	if (!loc)
...	...	@@ -4016,6 +3981,10 @@ void numpunct_byname<wchar_t>::__init(const char* nm) {
4016	3981	// localization for truename and falsename is not available
4017	3982	}
4018	3983	}
	3984
	3985	numpunct_byname<wchar_t>::numpunct_byname(const string& nm, size_t refs) : numpunct_byname<wchar_t>(nm.c_str(), refs) {}
	3986
	3987	numpunct_byname<wchar_t>::~numpunct_byname() {}
4019	3988	#endif // _LIBCPP_HAS_WIDE_CHARACTERS
4020	3989
4021	3990	// num_get helpers
...	...	@@ -4383,7 +4352,7 @@ string __time_get_storage<char>::__analyze(char fmt, const ctype<char>& ct) {
4383	4352	char f[3] = {0};
4384	4353	f[0] = '%';
4385	4354	f[1] = fmt;
4386		size_t n = __locale::__strftime(buf, countof(buf), f, &t, __loc_);
	4355	size_t n = __locale::__strftime(buf, std::size(buf), f, &t, __loc_);
4387	4356	char* bb = buf;
4388	4357	char* be = buf + n;
4389	4358	string result;
...	...	@@ -4514,12 +4483,12 @@ wstring __time_get_storage<wchar_t>::__analyze(char fmt, const ctype<wchar_t>& c
4514	4483	char f[3] = {0};
4515	4484	f[0] = '%';
4516	4485	f[1] = fmt;
4517		__locale::__strftime(buf, countof(buf), f, &t, __loc_);
	4486	__locale::__strftime(buf, std::size(buf), f, &t, __loc_);
4518	4487	wchar_t wbuf[100];
4519	4488	wchar_t* wbb = wbuf;
4520	4489	mbstate_t mb = {0};
4521	4490	const char* bb = buf;
4522		size_t j = __locale::__mbsrtowcs(wbb, &bb, countof(wbuf), &mb, __loc_);
	4491	size_t j = __locale::__mbsrtowcs(wbb, &bb, std::size(wbuf), &mb, __loc_);
4523	4492	if (j == size_t(-1))
4524	4493	std::__throw_runtime_error("locale not supported");
4525	4494	wchar_t* wbe = wbb + j;
...	...	@@ -4640,25 +4609,25 @@ void __time_get_storage<char>::init(const ctype<char>& ct) {
4640	4609	// __weeks_
4641	4610	for (int i = 0; i < 7; ++i) {
4642	4611	t.tm_wday = i;
4643		__locale::__strftime(buf, countof(buf), "%A", &t, __loc_);
	4612	__locale::__strftime(buf, std::size(buf), "%A", &t, __loc_);
4644	4613	__weeks_[i] = buf;
4645		__locale::__strftime(buf, countof(buf), "%a", &t, __loc_);
	4614	__locale::__strftime(buf, std::size(buf), "%a", &t, __loc_);
4646	4615	__weeks_[i + 7] = buf;
4647	4616	}
4648	4617	// __months_
4649	4618	for (int i = 0; i < 12; ++i) {
4650	4619	t.tm_mon = i;
4651		__locale::__strftime(buf, countof(buf), "%B", &t, __loc_);
	4620	__locale::__strftime(buf, std::size(buf), "%B", &t, __loc_);
4652	4621	__months_[i] = buf;
4653		__locale::__strftime(buf, countof(buf), "%b", &t, __loc_);
	4622	__locale::__strftime(buf, std::size(buf), "%b", &t, __loc_);
4654	4623	__months_[i + 12] = buf;
4655	4624	}
4656	4625	// __am_pm_
4657	4626	t.tm_hour = 1;
4658		__locale::__strftime(buf, countof(buf), "%p", &t, __loc_);
	4627	__locale::__strftime(buf, std::size(buf), "%p", &t, __loc_);
4659	4628	__am_pm_[0] = buf;
4660	4629	t.tm_hour = 13;
4661		__locale::__strftime(buf, countof(buf), "%p", &t, __loc_);
	4630	__locale::__strftime(buf, std::size(buf), "%p", &t, __loc_);
4662	4631	__am_pm_[1] = buf;
4663	4632	__c_ = __analyze('c', ct);
4664	4633	__r_ = __analyze('r', ct);
...	...	@@ -4677,18 +4646,18 @@ void __time_get_storage<wchar_t>::init(const ctype<wchar_t>& ct) {
4677	4646	// __weeks_
4678	4647	for (int i = 0; i < 7; ++i) {
4679	4648	t.tm_wday = i;
4680		__locale::__strftime(buf, countof(buf), "%A", &t, __loc_);
	4649	__locale::__strftime(buf, std::size(buf), "%A", &t, __loc_);
4681	4650	mb = mbstate_t();
4682	4651	const char* bb = buf;
4683		size_t j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4652	size_t j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4684	4653	if (j == size_t(-1) \|\| j == 0)
4685	4654	std::__throw_runtime_error("locale not supported");
4686	4655	wbe = wbuf + j;
4687	4656	__weeks_[i].assign(wbuf, wbe);
4688		__locale::__strftime(buf, countof(buf), "%a", &t, __loc_);
	4657	__locale::__strftime(buf, std::size(buf), "%a", &t, __loc_);
4689	4658	mb = mbstate_t();
4690	4659	bb = buf;
4691		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4660	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4692	4661	if (j == size_t(-1) \|\| j == 0)
4693	4662	std::__throw_runtime_error("locale not supported");
4694	4663	wbe = wbuf + j;
...	...	@@ -4697,18 +4666,18 @@ void __time_get_storage<wchar_t>::init(const ctype<wchar_t>& ct) {
4697	4666	// __months_
4698	4667	for (int i = 0; i < 12; ++i) {
4699	4668	t.tm_mon = i;
4700		__locale::__strftime(buf, countof(buf), "%B", &t, __loc_);
	4669	__locale::__strftime(buf, std::size(buf), "%B", &t, __loc_);
4701	4670	mb = mbstate_t();
4702	4671	const char* bb = buf;
4703		size_t j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4672	size_t j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4704	4673	if (j == size_t(-1) \|\| j == 0)
4705	4674	std::__throw_runtime_error("locale not supported");
4706	4675	wbe = wbuf + j;
4707	4676	__months_[i].assign(wbuf, wbe);
4708		__locale::__strftime(buf, countof(buf), "%b", &t, __loc_);
	4677	__locale::__strftime(buf, std::size(buf), "%b", &t, __loc_);
4709	4678	mb = mbstate_t();
4710	4679	bb = buf;
4711		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4680	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4712	4681	if (j == size_t(-1) \|\| j == 0)
4713	4682	std::__throw_runtime_error("locale not supported");
4714	4683	wbe = wbuf + j;
...	...	@@ -4716,19 +4685,19 @@ void __time_get_storage<wchar_t>::init(const ctype<wchar_t>& ct) {
4716	4685	}
4717	4686	// __am_pm_
4718	4687	t.tm_hour = 1;
4719		__locale::__strftime(buf, countof(buf), "%p", &t, __loc_);
	4688	__locale::__strftime(buf, std::size(buf), "%p", &t, __loc_);
4720	4689	mb = mbstate_t();
4721	4690	const char* bb = buf;
4722		size_t j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4691	size_t j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4723	4692	if (j == size_t(-1))
4724	4693	std::__throw_runtime_error("locale not supported");
4725	4694	wbe = wbuf + j;
4726	4695	__am_pm_[0].assign(wbuf, wbe);
4727	4696	t.tm_hour = 13;
4728		__locale::__strftime(buf, countof(buf), "%p", &t, __loc_);
	4697	__locale::__strftime(buf, std::size(buf), "%p", &t, __loc_);
4729	4698	mb = mbstate_t();
4730	4699	bb = buf;
4731		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, __loc_);
	4700	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, __loc_);
4732	4701	if (j == size_t(-1))
4733	4702	std::__throw_runtime_error("locale not supported");
4734	4703	wbe = wbuf + j;
...	...	@@ -4957,7 +4926,7 @@ void __time_put::__do_put(char* __nb, char& __ne, const tm __tm, char __fmt, c
4957	4926	char fmt[] = {'%', __fmt, __mod, 0};
4958	4927	if (__mod != 0)
4959	4928	swap(fmt[1], fmt[2]);
4960		size_t n = __locale::__strftime(__nb, countof(__nb, __ne), fmt, __tm, __loc_);
	4929	size_t n = __locale::__strftime(__nb, std::distance(__nb, __ne), fmt, __tm, __loc_);
4961	4930	__ne = __nb + n;
4962	4931	}
4963	4932
...	...	@@ -4968,7 +4937,7 @@ void __time_put::__do_put(wchar_t* __wb, wchar_t& __we, const tm __tm, char __
4968	4937	__do_put(__nar, __ne, __tm, __fmt, __mod);
4969	4938	mbstate_t mb = {0};
4970	4939	const char* __nb = __nar;
4971		size_t j = __locale::__mbsrtowcs(__wb, &__nb, countof(__wb, __we), &mb, __loc_);
	4940	size_t j = __locale::__mbsrtowcs(__wb, &__nb, std::distance(__wb, __we), &mb, __loc_);
4972	4941	if (j == size_t(-1))
4973	4942	std::__throw_runtime_error("locale not supported");
4974	4943	__we = __wb + j;
...	...	@@ -5443,7 +5412,7 @@ void moneypunct_byname<wchar_t, false>::init(const char* nm) {
5443	5412	wchar_t wbuf[100];
5444	5413	mbstate_t mb = {0};
5445	5414	const char* bb = lc->currency_symbol;
5446		size_t j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5415	size_t j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5447	5416	if (j == size_t(-1))
5448	5417	std::__throw_runtime_error("locale not supported");
5449	5418	wchar_t* wbe = wbuf + j;
...	...	@@ -5457,7 +5426,7 @@ void moneypunct_byname<wchar_t, false>::init(const char* nm) {
5457	5426	else {
5458	5427	mb = mbstate_t();
5459	5428	bb = lc->positive_sign;
5460		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5429	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5461	5430	if (j == size_t(-1))
5462	5431	std::__throw_runtime_error("locale not supported");
5463	5432	wbe = wbuf + j;
...	...	@@ -5468,7 +5437,7 @@ void moneypunct_byname<wchar_t, false>::init(const char* nm) {
5468	5437	else {
5469	5438	mb = mbstate_t();
5470	5439	bb = lc->negative_sign;
5471		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5440	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5472	5441	if (j == size_t(-1))
5473	5442	std::__throw_runtime_error("locale not supported");
5474	5443	wbe = wbuf + j;
...	...	@@ -5498,7 +5467,7 @@ void moneypunct_byname<wchar_t, true>::init(const char* nm) {
5498	5467	wchar_t wbuf[100];
5499	5468	mbstate_t mb = {0};
5500	5469	const char* bb = lc->int_curr_symbol;
5501		size_t j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5470	size_t j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5502	5471	if (j == size_t(-1))
5503	5472	std::__throw_runtime_error("locale not supported");
5504	5473	wchar_t* wbe = wbuf + j;
...	...	@@ -5516,7 +5485,7 @@ void moneypunct_byname<wchar_t, true>::init(const char* nm) {
5516	5485	else {
5517	5486	mb = mbstate_t();
5518	5487	bb = lc->positive_sign;
5519		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5488	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5520	5489	if (j == size_t(-1))
5521	5490	std::__throw_runtime_error("locale not supported");
5522	5491	wbe = wbuf + j;
...	...	@@ -5531,7 +5500,7 @@ void moneypunct_byname<wchar_t, true>::init(const char* nm) {
5531	5500	else {
5532	5501	mb = mbstate_t();
5533	5502	bb = lc->negative_sign;
5534		j = __locale::__mbsrtowcs(wbuf, &bb, countof(wbuf), &mb, loc.get());
	5503	j = __locale::__mbsrtowcs(wbuf, &bb, std::size(wbuf), &mb, loc.get());
5535	5504	if (j == size_t(-1))
5536	5505	std::__throw_runtime_error("locale not supported");
5537	5506	wbe = wbuf + j;
...	...	@@ -5571,6 +5540,54 @@ string __num_get<_CharT>::__stage2_int_prep(ios_base& __iob, _CharT* __atoms, _C
5571	5540	return __np.grouping();
5572	5541	}
5573	5542
	5543	template <class _CharT>
	5544	int __num_get<_CharT>::__stage2_int_loop(
	5545	_CharT __ct,
	5546	int __base,
	5547	char* __a,
	5548	char*& __a_end,
	5549	unsigned& __dc,
	5550	_CharT __thousands_sep,
	5551	const string& __grouping,
	5552	unsigned* __g,
	5553	unsigned*& __g_end,
	5554	_CharT* __atoms) {
	5555	if (__a_end == __a && (__ct == __atoms[24] \|\| __ct == __atoms[25])) {
	5556	*__a_end++ = __ct == __atoms[24] ? '+' : '-';
	5557	__dc = 0;
	5558	return 0;
	5559	}
	5560	if (__grouping.size() != 0 && __ct == __thousands_sep) {
	5561	if (__g_end - __g < __num_get_buf_sz) {
	5562	*__g_end++ = __dc;
	5563	__dc = 0;
	5564	}
	5565	return 0;
	5566	}
	5567	ptrdiff_t __f = __atoms_offset(__atoms, __ct);
	5568	if (__f >= 24)
	5569	return -1;
	5570	switch (__base) {
	5571	case 8:
	5572	case 10:
	5573	if (__f >= __base)
	5574	return -1;
	5575	break;
	5576	case 16:
	5577	if (__f < 22)
	5578	break;
	5579	if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
	5580	__dc = 0;
	5581	*__a_end++ = __src[__f];
	5582	return 0;
	5583	}
	5584	return -1;
	5585	}
	5586	*__a_end++ = __src[__f];
	5587	++__dc;
	5588	return 0;
	5589	}
	5590
5574	5591	template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS collate<char>;
5575	5592	_LIBCPP_IF_WIDE_CHARACTERS(template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS collate<wchar_t>;)
5576	5593

lib/libcxx/src/memory.cpp+12-14

...	...	@@ -7,7 +7,8 @@
7	7	//===----------------------------------------------------------------------===//
8	8
9	9	#include <__config>
10		#ifdef _LIBCPP_DEPRECATED_ABI_LEGACY_LIBRARY_DEFINITIONS_FOR_INLINE_FUNCTIONS
	10	#if !defined(_LIBCPP_OBJECT_FORMAT_COFF) && !defined(_LIBCPP_OBJECT_FORMAT_XCOFF) && \
	11	_LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 5
11	12	# define _LIBCPP_SHARED_PTR_DEFINE_LEGACY_INLINE_FUNCTIONS
12	13	#endif
13	14
...	...	@@ -132,19 +133,16 @@ __sp_mut& __get_sp_mut(const void* p) {
132	133
133	134	#endif // _LIBCPP_HAS_THREADS
134	135
135		void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
136		void* r = nullptr;
137		if (size <= space) {
138		char* p1 = static_cast<char*>(ptr);
139		char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
140		size_t d = static_cast<size_t>(p2 - p1);
141		if (d <= space - size) {
142		r = p2;
143		ptr = r;
144		space -= d;
145		}
146		}
147		return r;
	136	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 21
	137
	138	_LIBCPP_DIAGNOSTIC_PUSH
	139	_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wmissing-prototypes")
	140	// This function only exists for ABI compatibility and we therefore don't provide a declaration in the headers
	141	_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
	142	return __align_inline::align(alignment, size, ptr, space);
148	143	}
	144	_LIBCPP_DIAGNOSTIC_POP
	145
	146	#endif
149	147
150	148	_LIBCPP_END_NAMESPACE_STD

lib/libcxx/src/mutex_destructor.cpp+1-1

...	...	@@ -19,7 +19,7 @@
19	19	#include <__config>
20	20	#include <__thread/support.h>
21	21
22		#if _LIBCPP_ABI_VERSION == 1 \|\| !_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
	22	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 9 \|\| !_LIBCPP_HAS_TRIVIAL_MUTEX_DESTRUCTION
23	23	# define NEEDS_MUTEX_DESTRUCTOR
24	24	#endif
25	25

lib/libcxx/src/new.cpp+5-5

...	...	@@ -6,9 +6,9 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8
	9	#include "include/aligned_alloc.h"
9	10	#include "include/overridable_function.h"
10	11	#include <__assert>
11		#include <__memory/aligned_alloc.h>
12	12	#include <cstddef>
13	13	#include <cstdlib>
14	14	#include <new>
...	...	@@ -43,7 +43,7 @@ static void* operator_new_impl(std::size_t size) {
43	43	return p;
44	44	}
45	45
46		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new, (std::size_t size)) _THROW_BAD_ALLOC {
	46	OVERRIDABLE_FUNCTION void* operator new(std::size_t size) _THROW_BAD_ALLOC {
47	47	void* p = operator_new_impl(size);
48	48	if (p == nullptr)
49	49	__throw_bad_alloc_shim();
...	...	@@ -74,7 +74,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
74	74	# endif
75	75	}
76	76
77		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new[], (size_t size)) _THROW_BAD_ALLOC { return ::operator new(size); }
	77	OVERRIDABLE_FUNCTION void* operator new[](size_t size) _THROW_BAD_ALLOC { return ::operator new(size); }
78	78
79	79	_LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
80	80	# if !_LIBCPP_HAS_EXCEPTIONS
...	...	@@ -134,7 +134,7 @@ static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignm
134	134	return p;
135	135	}
136	136
137		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new, (std::size_t size, std::align_val_t alignment)) _THROW_BAD_ALLOC {
	137	OVERRIDABLE_FUNCTION void* operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
138	138	void* p = operator_new_aligned_impl(size, alignment);
139	139	if (p == nullptr)
140	140	__throw_bad_alloc_shim();
...	...	@@ -165,7 +165,7 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
165	165	# endif
166	166	}
167	167
168		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new[], (size_t size, std::align_val_t alignment)) _THROW_BAD_ALLOC {
	168	OVERRIDABLE_FUNCTION void* operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
169	169	return ::operator new(size, alignment);
170	170	}
171	171

lib/libcxx/src/optional.cpp+4

...	...	@@ -19,6 +19,8 @@ const char* bad_optional_access::what() const noexcept { return "bad_optional_ac
19	19
20	20	#include <__config>
21	21
	22	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 7
	23
22	24	// Preserve std::experimental::bad_optional_access for ABI compatibility
23	25	// Even though it no longer exists in a header file
24	26	_LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL
...	...	@@ -34,3 +36,5 @@ public:
34	36	bad_optional_access::~bad_optional_access() noexcept = default;
35	37
36	38	_LIBCPP_END_NAMESPACE_EXPERIMENTAL
	39
	40	#endif // _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 7

lib/libcxx/src/print.cpp+9-1

...	...	@@ -22,6 +22,14 @@
22	22	# include <windows.h>
23	23	#elif __has_include(<unistd.h>)
24	24	# include <unistd.h>
	25	# if defined(_NEWLIB_VERSION)
	26	# if defined(_POSIX_C_SOURCE) && __has_include(<stdio.h>)
	27	# include <stdio.h>
	28	# define HAS_FILENO_AND_ISATTY
	29	# endif
	30	# else
	31	# define HAS_FILENO_AND_ISATTY
	32	# endif
25	33	#endif
26	34
27	35	_LIBCPP_BEGIN_NAMESPACE_STD
...	...	@@ -56,7 +64,7 @@ __write_to_windows_console([[maybe_unused]] FILE* __stream, [[maybe_unused]] wst
56	64	}
57	65	# endif // _LIBCPP_HAS_WIDE_CHARACTERS
58	66
59		#elif __has_include(<unistd.h>) // !_LIBCPP_WIN32API
	67	#elif defined(HAS_FILENO_AND_ISATTY) // !_LIBCPP_WIN32API
60	68
61	69	_LIBCPP_EXPORTED_FROM_ABI bool __is_posix_terminal(FILE* __stream) { return isatty(fileno(__stream)); }
62	70	#endif

lib/libcxx/src/random.cpp-26

...	...	@@ -31,8 +31,6 @@
31	31	# include <linux/random.h>
32	32	# include <sys/ioctl.h>
33	33	# endif
34		#elif defined(_LIBCPP_USING_NACL_RANDOM)
35		# include <nacl/nacl_random.h>
36	34	#elif defined(_LIBCPP_USING_FUCHSIA_CPRNG)
37	35	# include <zircon/syscalls.h>
38	36	#endif
...	...	@@ -93,30 +91,6 @@ unsigned random_device::operator()() {
93	91	return r;
94	92	}
95	93
96		#elif defined(_LIBCPP_USING_NACL_RANDOM)
97
98		random_device::random_device(const string& __token) {
99		if (__token != "/dev/urandom")
100		std::__throw_system_error(ENOENT, ("random device not supported " + __token).c_str());
101		int error = nacl_secure_random_init();
102		if (error)
103		std::__throw_system_error(error, ("random device failed to open " + __token).c_str());
104		}
105
106		random_device::~random_device() {}
107
108		unsigned random_device::operator()() {
109		unsigned r;
110		size_t n = sizeof(r);
111		size_t bytes_written;
112		int error = nacl_secure_random(&r, n, &bytes_written);
113		if (error != 0)
114		std::__throw_system_error(error, "random_device failed getting bytes");
115		else if (bytes_written != n)
116		std::__throw_runtime_error("random_device failed to obtain enough bytes");
117		return r;
118		}
119
120	94	#elif defined(_LIBCPP_USING_WIN32_RANDOM)
121	95
122	96	random_device::random_device(const string& __token) {

lib/libcxx/src/string.cpp+10-25

...	...	@@ -20,7 +20,7 @@
20	20
21	21	_LIBCPP_BEGIN_NAMESPACE_STD
22	22
23		#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
	23	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 14
24	24
25	25	template <bool>
26	26	struct __basic_string_common;
...	...	@@ -35,45 +35,30 @@ struct __basic_string_common<true> {
35	35	void __basic_string_common<true>::__throw_length_error() const { std::__throw_length_error("basic_string"); }
36	36	void __basic_string_common<true>::__throw_out_of_range() const { std::__throw_out_of_range("basic_string"); }
37	37
38		#endif // _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
	38	#endif // _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 14
39	39
40	40	// Define legacy ABI functions
41	41	// ---------------------------
42	42
43		#ifndef _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
	43	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 21
44	44
	45	// This initializes the string with [__s, __s + __sz), but capacity() == __reserve. Assumes that __reserve >= __sz.
45	46	template <class _CharT, class _Traits, class _Allocator>
46	47	void basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_type __sz, size_type __reserve) {
47		if (__libcpp_is_constant_evaluated())
48		__rep_ = __rep();
49		if (__reserve > max_size())
50		__throw_length_error();
51		pointer __p;
52		if (__fits_in_sso(__reserve)) {
53		__set_short_size(__sz);
54		__p = __get_short_pointer();
55		} else {
56		auto __allocation = std::__allocate_at_least(__alloc_, __recommend(__reserve) + 1);
57		__p = __allocation.ptr;
58		__begin_lifetime(__p, __allocation.count);
59		__set_long_pointer(__p);
60		__set_long_cap(__allocation.count);
61		__set_long_size(__sz);
62		}
	48	pointer __p = __init_internal_buffer(__reserve);
	49	__annotate_delete();
	50	__set_size(__sz);
63	51	traits_type::copy(std::__to_address(__p), __s, __sz);
64	52	traits_type::assign(__p[__sz], value_type());
65	53	__annotate_new(__sz);
66	54	}
67	55
68		# define STRING_LEGACY_API(CharT) \
69		template _LIBCPP_EXPORTED_FROM_ABI void basic_string<CharT>::__init(const value_type*, size_type, size_type)
70
71		STRING_LEGACY_API(char);
	56	template _LIBCPP_EXPORTED_FROM_ABI void basic_string<char>::__init(const value_type*, size_type, size_type);
72	57	# if _LIBCPP_HAS_WIDE_CHARACTERS
73		STRING_LEGACY_API(wchar_t);
	58	template _LIBCPP_EXPORTED_FROM_ABI void basic_string<wchar_t>::__init(const value_type*, size_type, size_type);
74	59	# endif
75	60
76		#endif // _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION
	61	#endif // _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 21
77	62
78	63	#define _LIBCPP_EXTERN_TEMPLATE_DEFINE(...) template _LIBCPP_EXPORTED_FROM_ABI __VA_ARGS__;
79	64	#ifdef _LIBCPP_ABI_STRING_OPTIMIZED_EXTERNAL_INSTANTIATION

lib/libcxx/src/support/runtime/exception_fallback.ipp+2

...	...	@@ -8,6 +8,8 @@
8	8	//===----------------------------------------------------------------------===//
9	9
10	10	#include <__verbose_abort>
	11	#include <exception>
	12	#include "include/atomic_support.h"
11	13
12	14	namespace std {
13	15

lib/libcxx/src/support/runtime/exception_glibcxx.ipp+3

...	...	@@ -11,6 +11,9 @@
11	11	# error header can only be used when targeting libstdc++ or libsupc++
12	12	#endif
13	13
	14	#include <exception>
	15	#include <new>
	16
14	17	namespace std {
15	18
16	19	bad_alloc::bad_alloc() noexcept {}

lib/libcxx/src/support/runtime/exception_libcxxabi.ipp+6-2

...	...	@@ -7,6 +7,10 @@
7	7	//
8	8	//===----------------------------------------------------------------------===//
9	9
	10	#include <exception>
	11
	12	#include <cxxabi.h>
	13
10	14	#ifndef _LIBCPPABI_VERSION
11	15	# error this header can only be used with libc++abi
12	16	#endif
...	...	@@ -17,9 +21,9 @@ bool uncaught_exception() noexcept { return uncaught_exceptions() > 0; }
17	21
18	22	int uncaught_exceptions() noexcept {
19	23	#if _LIBCPPABI_VERSION > 1001
20		return __cxa_uncaught_exceptions();
	24	return abi::__cxa_uncaught_exceptions();
21	25	#else
22		return __cxa_uncaught_exception() ? 1 : 0;
	26	return abi::__cxa_uncaught_exception() ? 1 : 0;
23	27	#endif
24	28	}
25	29

lib/libcxx/src/support/runtime/exception_libcxxrt.ipp+2

...	...	@@ -11,6 +11,8 @@
11	11	# error this header may only be used when targeting libcxxrt
12	12	#endif
13	13
	14	#include <exception>
	15
14	16	namespace std {
15	17
16	18	bad_exception::~bad_exception() noexcept {}

lib/libcxx/src/support/runtime/exception_msvc.ipp+2

...	...	@@ -12,6 +12,8 @@
12	12	#endif
13	13
14	14	#include <__verbose_abort>
	15	#include <exception>
	16	#include <new>
15	17
16	18	extern "C" {
17	19	typedef void(__cdecl* terminate_handler)();

lib/libcxx/src/support/runtime/exception_pointer_cxxabi.ipp+9-10

...	...	@@ -7,22 +7,21 @@
7	7	//
8	8	//===----------------------------------------------------------------------===//
9	9
10		#ifndef HAVE_DEPENDENT_EH_ABI
11		# error this header may only be used with libc++abi or libcxxrt
12		#endif
	10	#include <cxxabi.h>
	11	#include <exception>
13	12
14	13	namespace std {
15	14
16		exception_ptr::~exception_ptr() noexcept { __cxa_decrement_exception_refcount(__ptr_); }
	15	exception_ptr::~exception_ptr() noexcept { abi::__cxa_decrement_exception_refcount(__ptr_); }
17	16
18	17	exception_ptr::exception_ptr(const exception_ptr& other) noexcept : __ptr_(other.__ptr_) {
19		__cxa_increment_exception_refcount(__ptr_);
	18	abi::__cxa_increment_exception_refcount(__ptr_);
20	19	}
21	20
22	21	exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
23	22	if (__ptr_ != other.__ptr_) {
24		__cxa_increment_exception_refcount(other.__ptr_);
25		__cxa_decrement_exception_refcount(__ptr_);
	23	abi::__cxa_increment_exception_refcount(other.__ptr_);
	24	abi::__cxa_decrement_exception_refcount(__ptr_);
26	25	__ptr_ = other.__ptr_;
27	26	}
28	27	return *this;
...	...	@@ -31,7 +30,7 @@ exception_ptr& exception_ptr::operator=(const exception_ptr& other) noexcept {
31	30	exception_ptr exception_ptr::__from_native_exception_pointer(void* __e) noexcept {
32	31	exception_ptr ptr;
33	32	ptr.__ptr_ = __e;
34		__cxa_increment_exception_refcount(ptr.__ptr_);
	33	abi::__cxa_increment_exception_refcount(ptr.__ptr_);
35	34
36	35	return ptr;
37	36	}
...	...	@@ -51,12 +50,12 @@ exception_ptr current_exception() noexcept {
51	50	// this whole function would be just:
52	51	// return exception_ptr(__cxa_current_primary_exception());
53	52	exception_ptr ptr;
54		ptr.__ptr_ = __cxa_current_primary_exception();
	53	ptr.__ptr_ = abi::__cxa_current_primary_exception();
55	54	return ptr;
56	55	}
57	56
58	57	void rethrow_exception(exception_ptr p) {
59		__cxa_rethrow_primary_exception(p.__ptr_);
	58	abi::__cxa_rethrow_primary_exception(p.__ptr_);
60	59	// if p.__ptr_ is NULL, above returns so we terminate
61	60	terminate();
62	61	}

lib/libcxx/src/support/runtime/exception_pointer_glibcxx.ipp+2

...	...	@@ -16,6 +16,8 @@
16	16	// stable ABI), and its rethrow_exception(std::__exception_ptr::exception_ptr)
17	17	// function.
18	18
	19	#include <exception>
	20
19	21	namespace std {
20	22
21	23	namespace __exception_ptr {

lib/libcxx/src/support/runtime/exception_pointer_msvc.ipp+1

...	...	@@ -7,6 +7,7 @@
7	7	//
8	8	//===----------------------------------------------------------------------===//
9	9
	10	#include <exception>
10	11	#include <stdio.h>
11	12	#include <stdlib.h>
12	13

lib/libcxx/src/support/runtime/exception_pointer_unimplemented.ipp+1

...	...	@@ -8,6 +8,7 @@
8	8	//===----------------------------------------------------------------------===//
9	9
10	10	#include <__verbose_abort>
	11	#include <exception>
11	12
12	13	namespace std {
13	14

lib/libcxx/src/support/win32/locale_win32.cpp+1-1

...	...	@@ -144,7 +144,7 @@ int __snprintf(char* ret, size_t n, __locale_t loc, const char* format, ...) {
144	144	// Like sprintf, but when return value >= 0 it returns
145	145	// a pointer to a malloc'd string in *sptr.
146	146	// If return >= 0, use free to delete *sptr.
147		int __libcpp_vasprintf(char** sptr, const char* __restrict format, va_list ap) {
	147	static int __libcpp_vasprintf(char** sptr, const char* __restrict format, va_list ap) {
148	148	*sptr = nullptr;
149	149	// Query the count required.
150	150	va_list ap_copy;

lib/libcxx/src/system_error.cpp+2

...	...	@@ -95,6 +95,8 @@ std::optional<errc> __win_err_to_errc(int err) {
95	95	return errc::no_lock_available;
96	96	case ERROR_NEGATIVE_SEEK:
97	97	return errc::invalid_argument;
	98	case ERROR_NETNAME_DELETED:
	99	return errc::no_such_file_or_directory;
98	100	case ERROR_NOACCESS:
99	101	return errc::permission_denied;
100	102	case ERROR_NOT_ENOUGH_MEMORY:

lib/libcxx/src/thread.cpp+1-3

...	...	@@ -74,9 +74,7 @@ unsigned thread::hardware_concurrency() noexcept {
74	74	return 0;
75	75	return static_cast<unsigned>(result);
76	76	#elif defined(_LIBCPP_WIN32API)
77		SYSTEM_INFO info;
78		GetSystemInfo(&info);
79		return info.dwNumberOfProcessors;
	77	return static_cast<unsigned>(GetActiveProcessorCount(ALL_PROCESSOR_GROUPS));
80	78	#else // defined(CTL_HW) && defined(HW_NCPU)
81	79	// TODO: grovel through /proc or check cpuid on x86 and similar
82	80	// instructions on other architectures.

lib/libcxx/src/valarray.cpp+1-2

...	...	@@ -10,8 +10,7 @@
10	10
11	11	_LIBCPP_BEGIN_NAMESPACE_STD
12	12
13		// These two symbols are part of the v1 ABI but not part of the >=v2 ABI.
14		#if _LIBCPP_ABI_VERSION == 1
	13	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 9
15	14	template _LIBCPP_EXPORTED_FROM_ABI valarray<size_t>::valarray(size_t);
16	15	template _LIBCPP_EXPORTED_FROM_ABI valarray<size_t>::~valarray();
17	16	#endif

lib/libcxx/src/vector.cpp+2-2

...	...	@@ -10,7 +10,7 @@
10	10
11	11	_LIBCPP_BEGIN_NAMESPACE_STD
12	12
13		#ifndef _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
	13	#if _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 15
14	14
15	15	template <bool>
16	16	struct __vector_base_common;
...	...	@@ -25,6 +25,6 @@ void __vector_base_common<true>::__throw_length_error() const { std::__throw_len
25	25
26	26	void __vector_base_common<true>::__throw_out_of_range() const { std::__throw_out_of_range("vector"); }
27	27
28		#endif // _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
	28	#endif // _LIBCPP_AVAILABILITY_MINIMUM_HEADER_VERSION < 15
29	29
30	30	_LIBCPP_END_NAMESPACE_STD

lib/libcxxabi/include/__cxxabi_config.h+42-23

...	...	@@ -14,10 +14,6 @@
14	14	#define _LIBCXXABI_ARM_EHABI
15	15	#endif
16	16
17		#if !defined(__has_attribute)
18		#define __has_attribute(_attribute_) 0
19		#endif
20
21	17	#if defined(__clang__)
22	18	# define _LIBCXXABI_COMPILER_CLANG
23	19	# ifndef __apple_build_version__
...	...	@@ -25,10 +21,6 @@
25	21	# endif
26	22	#elif defined(__GNUC__)
27	23	# define _LIBCXXABI_COMPILER_GCC
28		#elif defined(_MSC_VER)
29		# define _LIBCXXABI_COMPILER_MSVC
30		#elif defined(__IBMCPP__)
31		# define _LIBCXXABI_COMPILER_IBM
32	24	#endif
33	25
34	26	#if defined(_WIN32)
...	...	@@ -66,17 +58,7 @@
66	58	#endif
67	59	#endif
68	60
69		#if defined(_LIBCXXABI_COMPILER_MSVC)
70		#define _LIBCXXABI_WEAK
71		#else
72	61	#define _LIBCXXABI_WEAK __attribute__((__weak__))
73		#endif
74
75		#if defined(__clang__)
76		#define _LIBCXXABI_COMPILER_CLANG
77		#elif defined(__GNUC__)
78		#define _LIBCXXABI_COMPILER_GCC
79		#endif
80	62
81	63	#if __has_attribute(__no_sanitize__) && defined(_LIBCXXABI_COMPILER_CLANG)
82	64	#define _LIBCXXABI_NO_CFI __attribute__((__no_sanitize__("cfi")))
...	...	@@ -89,11 +71,7 @@
89	71	# define _LIBCXXABI_GUARD_ABI_ARM
90	72	#endif
91	73
92		#if defined(_LIBCXXABI_COMPILER_CLANG)
93		# if !__has_feature(cxx_exceptions)
94		# define _LIBCXXABI_NO_EXCEPTIONS
95		# endif
96		#elif defined(_LIBCXXABI_COMPILER_GCC) && !defined(__EXCEPTIONS)
	74	#if !defined(__cpp_exceptions) \|\| __cpp_exceptions < 199711L
97	75	# define _LIBCXXABI_NO_EXCEPTIONS
98	76	#endif
99	77
...	...	@@ -103,6 +81,47 @@
103	81	#define _LIBCXXABI_DTOR_FUNC
104	82	#endif
105	83
	84	#if __has_include(<ptrauth.h>)
	85	# include <ptrauth.h>
	86	#endif
	87
	88	#if __has_feature(ptrauth_calls)
	89
	90	// ptrauth_string_discriminator("__cxa_exception::actionRecord") == 0xFC91
	91	# define __ptrauth_cxxabi_action_record __ptrauth(ptrauth_key_process_dependent_data, 1, 0xFC91)
	92
	93	// ptrauth_string_discriminator("__cxa_exception::languageSpecificData") == 0xE8EE
	94	# define __ptrauth_cxxabi_lsd __ptrauth(ptrauth_key_process_dependent_data, 1, 0xE8EE)
	95
	96	// ptrauth_string_discriminator("__cxa_exception::catchTemp") == 0xFA58
	97	# define __ptrauth_cxxabi_catch_temp_disc 0xFA58
	98	# define __ptrauth_cxxabi_catch_temp_key ptrauth_key_process_dependent_data
	99	# define __ptrauth_cxxabi_catch_temp __ptrauth(__ptrauth_cxxabi_catch_temp_key, 1, __ptrauth_cxxabi_catch_temp_disc)
	100
	101	// ptrauth_string_discriminator("__cxa_exception::adjustedPtr") == 0x99E4
	102	# define __ptrauth_cxxabi_adjusted_ptr __ptrauth(ptrauth_key_process_dependent_data, 1, 0x99E4)
	103
	104	// ptrauth_string_discriminator("__cxa_exception::unexpectedHandler") == 0x99A9
	105	# define __ptrauth_cxxabi_unexpected_handler __ptrauth(ptrauth_key_function_pointer, 1, 0x99A9)
	106
	107	// ptrauth_string_discriminator("__cxa_exception::terminateHandler") == 0x0886)
	108	# define __ptrauth_cxxabi_terminate_handler __ptrauth(ptrauth_key_function_pointer, 1, 0x886)
	109
	110	// ptrauth_string_discriminator("__cxa_exception::exceptionDestructor") == 0xC088
	111	# define __ptrauth_cxxabi_exception_destructor __ptrauth(ptrauth_key_function_pointer, 1, 0xC088)
	112
	113	#else
	114
	115	# define __ptrauth_cxxabi_action_record
	116	# define __ptrauth_cxxabi_lsd
	117	# define __ptrauth_cxxabi_catch_temp
	118	# define __ptrauth_cxxabi_adjusted_ptr
	119	# define __ptrauth_cxxabi_unexpected_handler
	120	# define __ptrauth_cxxabi_terminate_handler
	121	# define __ptrauth_cxxabi_exception_destructor
	122
	123	#endif
	124
106	125	#if __cplusplus < 201103L
107	126	# define _LIBCXXABI_NOEXCEPT throw()
108	127	#else

lib/libcxxabi/src/cxa_exception.cpp+4-2

...	...	@@ -192,7 +192,9 @@ void *__cxa_allocate_exception(size_t thrown_size) throw() {
192	192	std::terminate();
193	193	__cxa_exception *exception_header =
194	194	static_cast<__cxa_exception >((void )(raw_buffer + header_offset));
195		::memset(exception_header, 0, actual_size);
	195	// We warn on memset to a non-trivially castable type. We might want to
	196	// change that diagnostic to not fire on a trivially obvious zero fill.
	197	::memset(static_cast<void*>(exception_header), 0, actual_size);
196	198	return thrown_object_from_cxa_exception(exception_header);
197	199	}
198	200
...	...	@@ -226,7 +228,7 @@ __cxa_exception* __cxa_init_primary_exception(void* object, std::type_info* tinf
226	228	}
227	229
228	230	// This function shall allocate a __cxa_dependent_exception and
229		// return a pointer to it. (Really to the object, not past its' end).
	231	// return a pointer to it. (Really to the object, not past its end).
230	232	// Otherwise, it will work like __cxa_allocate_exception.
231	233	void * __cxa_allocate_dependent_exception () {
232	234	size_t actual_size = sizeof(__cxa_dependent_exception);

lib/libcxxabi/src/cxa_exception.h+16-14

...	...	@@ -47,10 +47,10 @@ struct _LIBCXXABI_HIDDEN __cxa_exception {
47	47	// In Wasm, a destructor returns its argument
48	48	void (_LIBCXXABI_DTOR_FUNC exceptionDestructor)(void *);
49	49	#else
50		void (_LIBCXXABI_DTOR_FUNC exceptionDestructor)(void );
	50	void (_LIBCXXABI_DTOR_FUNC __ptrauth_cxxabi_exception_destructor exceptionDestructor)(void );
51	51	#endif
52		std::unexpected_handler unexpectedHandler;
53		std::terminate_handler terminateHandler;
	52	std::unexpected_handler __ptrauth_cxxabi_unexpected_handler unexpectedHandler;
	53	std::terminate_handler __ptrauth_cxxabi_terminate_handler terminateHandler;
54	54
55	55	__cxa_exception *nextException;
56	56
...	...	@@ -61,10 +61,10 @@ struct _LIBCXXABI_HIDDEN __cxa_exception {
61	61	int propagationCount;
62	62	#else
63	63	int handlerSwitchValue;
64		const unsigned char *actionRecord;
65		const unsigned char *languageSpecificData;
66		void *catchTemp;
67		void *adjustedPtr;
	64	const unsigned char *__ptrauth_cxxabi_action_record actionRecord;
	65	const unsigned char *__ptrauth_cxxabi_lsd languageSpecificData;
	66	void *__ptrauth_cxxabi_catch_temp catchTemp;
	67	void *__ptrauth_cxxabi_adjusted_ptr adjustedPtr;
68	68	#endif
69	69
70	70	#if !defined(__LP64__) && !defined(_WIN64) && !defined(_LIBCXXABI_ARM_EHABI)
...	...	@@ -79,6 +79,8 @@ struct _LIBCXXABI_HIDDEN __cxa_exception {
79	79	// http://sourcery.mentor.com/archives/cxx-abi-dev/msg01924.html
80	80	// The layout of this structure MUST match the layout of __cxa_exception, with
81	81	// primaryException instead of referenceCount.
	82	// The pointer authentication schemas specified here must also match those of
	83	// the corresponding members in __cxa_exception.
82	84	struct _LIBCXXABI_HIDDEN __cxa_dependent_exception {
83	85	#if defined(__LP64__) \|\| defined(_WIN64) \|\| defined(_LIBCXXABI_ARM_EHABI)
84	86	void* reserve; // padding.
...	...	@@ -86,9 +88,9 @@ struct _LIBCXXABI_HIDDEN __cxa_dependent_exception {
86	88	#endif
87	89
88	90	std::type_info *exceptionType;
89		void (_LIBCXXABI_DTOR_FUNC exceptionDestructor)(void );
90		std::unexpected_handler unexpectedHandler;
91		std::terminate_handler terminateHandler;
	91	void (_LIBCXXABI_DTOR_FUNC __ptrauth_cxxabi_exception_destructor exceptionDestructor)(void );
	92	std::unexpected_handler __ptrauth_cxxabi_unexpected_handler unexpectedHandler;
	93	std::terminate_handler __ptrauth_cxxabi_terminate_handler terminateHandler;
92	94
93	95	__cxa_exception *nextException;
94	96
...	...	@@ -99,10 +101,10 @@ struct _LIBCXXABI_HIDDEN __cxa_dependent_exception {
99	101	int propagationCount;
100	102	#else
101	103	int handlerSwitchValue;
102		const unsigned char *actionRecord;
103		const unsigned char *languageSpecificData;
104		void * catchTemp;
105		void *adjustedPtr;
	104	const unsigned char *__ptrauth_cxxabi_action_record actionRecord;
	105	const unsigned char *__ptrauth_cxxabi_lsd languageSpecificData;
	106	void *__ptrauth_cxxabi_catch_temp catchTemp;
	107	void *__ptrauth_cxxabi_adjusted_ptr adjustedPtr;
106	108	#endif
107	109
108	110	#if !defined(__LP64__) && !defined(_WIN64) && !defined(_LIBCXXABI_ARM_EHABI)

lib/libcxxabi/src/cxa_personality.cpp+125-12

...	...	@@ -20,7 +20,52 @@
20	20	#include "cxa_exception.h"
21	21	#include "cxa_handlers.h"
22	22	#include "private_typeinfo.h"
23		#include "unwind.h"
	23
	24	#if __has_feature(ptrauth_calls)
	25
	26	// CXXABI depends on defintions in libunwind as pointer auth couples the
	27	// definitions
	28	# include "libunwind.h"
	29
	30	// The actual value of the discriminators listed below is not important.
	31	// The derivation of the constants is only being included for the purpose
	32	// of maintaining a record of how they were originally produced.
	33
	34	// ptrauth_string_discriminator("scan_results::languageSpecificData") == 0xE50D)
	35	# define __ptrauth_scan_results_lsd __ptrauth(ptrauth_key_process_dependent_code, 1, 0xE50D)
	36
	37	// ptrauth_string_discriminator("scan_results::actionRecord") == 0x9823
	38	# define __ptrauth_scan_results_action_record __ptrauth(ptrauth_key_process_dependent_code, 1, 0x9823)
	39
	40	// scan result is broken up as we have a manual re-sign that requires each component
	41	# define __ptrauth_scan_results_landingpad_key ptrauth_key_process_dependent_code
	42	// ptrauth_string_discriminator("scan_results::landingPad") == 0xD27C
	43	# define __ptrauth_scan_results_landingpad_disc 0xD27C
	44	# define __ptrauth_scan_results_landingpad \
	45	__ptrauth(__ptrauth_scan_results_landingpad_key, 1, __ptrauth_scan_results_landingpad_disc)
	46
	47	// `__ptrauth_restricted_intptr` is a feature of apple clang that predates
	48	// support for direct application of `__ptrauth` to integer types. This
	49	// guard is necessary to support compilation with those compiler.
	50	# if __has_extension(ptrauth_restricted_intptr_qualifier)
	51	# define __ptrauth_scan_results_landingpad_intptr \
	52	__ptrauth_restricted_intptr(__ptrauth_scan_results_landingpad_key, 1, __ptrauth_scan_results_landingpad_disc)
	53	# else
	54	# define __ptrauth_scan_results_landingpad_intptr \
	55	__ptrauth(__ptrauth_scan_results_landingpad_key, 1, __ptrauth_scan_results_landingpad_disc)
	56	# endif
	57
	58	#else
	59	# define __ptrauth_scan_results_lsd
	60	# define __ptrauth_scan_results_action_record
	61	# define __ptrauth_scan_results_landingpad
	62	# define __ptrauth_scan_results_landingpad_intptr
	63	#endif
	64
	65	// The functions defined in this file are magic functions called only by the compiler.
	66	#ifdef __clang__
	67	# pragma clang diagnostic ignored "-Wmissing-prototypes"
	68	#endif
24	69
25	70	// TODO: This is a temporary workaround for libc++abi to recognize that it's being
26	71	// built against LLVM's libunwind. LLVM's libunwind started reporting _LIBUNWIND_VERSION
...	...	@@ -527,12 +572,17 @@ get_thrown_object_ptr(_Unwind_Exception* unwind_exception)
527	572	namespace
528	573	{
529	574
	575	typedef const uint8_t *__ptrauth_scan_results_lsd lsd_ptr_t;
	576	typedef const uint8_t *__ptrauth_scan_results_action_record action_ptr_t;
	577	typedef uintptr_t __ptrauth_scan_results_landingpad_intptr landing_pad_t;
	578	typedef void *__ptrauth_scan_results_landingpad landing_pad_ptr_t;
	579
530	580	struct scan_results
531	581	{
532	582	int64_t ttypeIndex; // > 0 catch handler, < 0 exception spec handler, == 0 a cleanup
533		const uint8_t* actionRecord; // Currently unused. Retained to ease future maintenance.
534		const uint8_t* languageSpecificData; // Needed only for __cxa_call_unexpected
535		uintptr_t landingPad; // null -> nothing found, else something found
	583	action_ptr_t actionRecord; // Currently unused. Retained to ease future maintenance.
	584	lsd_ptr_t languageSpecificData; // Needed only for __cxa_call_unexpected
	585	landing_pad_t landingPad; // null -> nothing found, else something found
536	586	void* adjustedPtr; // Used in cxa_exception.cpp
537	587	_Unwind_Reason_Code reason; // One of _URC_FATAL_PHASE1_ERROR,
538	588	// _URC_FATAL_PHASE2_ERROR,
...	...	@@ -557,7 +607,23 @@ set_registers(_Unwind_Exception* unwind_exception, _Unwind_Context* context,
557	607	reinterpret_cast<uintptr_t>(unwind_exception));
558	608	_Unwind_SetGR(context, __builtin_eh_return_data_regno(1),
559	609	static_cast<uintptr_t>(results.ttypeIndex));
	610	#if __has_feature(ptrauth_calls)
	611	auto stackPointer = _Unwind_GetGR(context, UNW_REG_SP);
	612	// We manually re-sign the IP as the __ptrauth qualifiers cannot
	613	// express the required relationship with the destination address
	614	const auto existingDiscriminator =
	615	ptrauth_blend_discriminator(&results.landingPad,
	616	__ptrauth_scan_results_landingpad_disc);
	617	unw_word_t newIP /* opaque __ptrauth(ptrauth_key_return_address, stackPointer, 0) */ =
	618	(unw_word_t)ptrauth_auth_and_resign((void const*)&results.landingPad,
	619	__ptrauth_scan_results_landingpad_key,
	620	existingDiscriminator,
	621	ptrauth_key_return_address,
	622	stackPointer);
	623	_Unwind_SetIP(context, newIP);
	624	#else
560	625	_Unwind_SetIP(context, results.landingPad);
	626	#endif
561	627	}
562	628
563	629	/*
...	...	@@ -691,12 +757,12 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions,
691	757	// The call sites are ordered in increasing value of start
692	758	uintptr_t start = readEncodedPointer(&callSitePtr, callSiteEncoding);
693	759	uintptr_t length = readEncodedPointer(&callSitePtr, callSiteEncoding);
694		uintptr_t landingPad = readEncodedPointer(&callSitePtr, callSiteEncoding);
	760	landing_pad_t landingPad = readEncodedPointer(&callSitePtr, callSiteEncoding);
695	761	uintptr_t actionEntry = readULEB128(&callSitePtr);
696	762	if ((start <= ipOffset) && (ipOffset < (start + length)))
697	763	#else // __USING_SJLJ_EXCEPTIONS__ \|\| __WASM_EXCEPTIONS__
698	764	// ip is 1-based index into this table
699		uintptr_t landingPad = readULEB128(&callSitePtr);
	765	landing_pad_t landingPad = readULEB128(&callSitePtr);
700	766	uintptr_t actionEntry = readULEB128(&callSitePtr);
701	767	if (--ip == 0)
702	768	#endif // __USING_SJLJ_EXCEPTIONS__ \|\| __WASM_EXCEPTIONS__
...	...	@@ -903,6 +969,57 @@ _UA_CLEANUP_PHASE
903	969	*/
904	970
905	971	#if !defined(_LIBCXXABI_ARM_EHABI)
	972
	973	// We use these helper functions to work around the behavior of casting between
	974	// integers (even those that are authenticated) and authenticated pointers.
	975	// Because the schemas being used are address discriminated we cannot use a
	976	// trivial value union to coerce the types so instead we perform the re-signing
	977	// manually.
	978	using __cxa_catch_temp_type = decltype(__cxa_exception::catchTemp);
	979	static inline void set_landing_pad(scan_results& results,
	980	const __cxa_catch_temp_type& source) {
	981	#if __has_feature(ptrauth_calls)
	982	const uintptr_t sourceDiscriminator =
	983	ptrauth_blend_discriminator(&source, __ptrauth_cxxabi_catch_temp_disc);
	984	const uintptr_t targetDiscriminator =
	985	ptrauth_blend_discriminator(&results.landingPad,
	986	__ptrauth_scan_results_landingpad_disc);
	987	uintptr_t reauthenticatedLandingPad =
	988	(uintptr_t)ptrauth_auth_and_resign(reinterpret_cast<void const*>(&source),
	989	__ptrauth_cxxabi_catch_temp_key,
	990	sourceDiscriminator,
	991	__ptrauth_scan_results_landingpad_key,
	992	targetDiscriminator);
	993	memmove(reinterpret_cast<void *>(&results.landingPad),
	994	reinterpret_cast<void *>(&reauthenticatedLandingPad),
	995	sizeof(reauthenticatedLandingPad));
	996	#else
	997	results.landingPad = reinterpret_cast<landing_pad_t>(source);
	998	#endif
	999	}
	1000
	1001	static inline void get_landing_pad(__cxa_catch_temp_type &dest,
	1002	const scan_results &results) {
	1003	#if __has_feature(ptrauth_calls)
	1004	const uintptr_t sourceDiscriminator =
	1005	ptrauth_blend_discriminator(&results.landingPad,
	1006	__ptrauth_scan_results_landingpad_disc);
	1007	const uintptr_t targetDiscriminator =
	1008	ptrauth_blend_discriminator(&dest, __ptrauth_cxxabi_catch_temp_disc);
	1009	uintptr_t reauthenticatedPointer =
	1010	(uintptr_t)ptrauth_auth_and_resign(reinterpret_cast<void const*>(&results.landingPad),
	1011	__ptrauth_scan_results_landingpad_key,
	1012	sourceDiscriminator,
	1013	__ptrauth_cxxabi_catch_temp_key,
	1014	targetDiscriminator);
	1015	memmove(reinterpret_cast<void *>(&dest),
	1016	reinterpret_cast<void *>(&reauthenticatedPointer),
	1017	sizeof(reauthenticatedPointer));
	1018	#else
	1019	dest = reinterpret_cast<__cxa_catch_temp_type>(results.landingPad);
	1020	#endif
	1021	}
	1022
906	1023	#ifdef __WASM_EXCEPTIONS__
907	1024	_Unwind_Reason_Code __gxx_personality_wasm0
908	1025	#elif defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)
...	...	@@ -935,8 +1052,7 @@ __gxx_personality_v0
935	1052	results.ttypeIndex = exception_header->handlerSwitchValue;
936	1053	results.actionRecord = exception_header->actionRecord;
937	1054	results.languageSpecificData = exception_header->languageSpecificData;
938		results.landingPad =
939		reinterpret_cast<uintptr_t>(exception_header->catchTemp);
	1055	set_landing_pad(results, exception_header->catchTemp);
940	1056	results.adjustedPtr = exception_header->adjustedPtr;
941	1057
942	1058	// Jump to the handler.
...	...	@@ -970,7 +1086,7 @@ __gxx_personality_v0
970	1086	exc->handlerSwitchValue = static_cast<int>(results.ttypeIndex);
971	1087	exc->actionRecord = results.actionRecord;
972	1088	exc->languageSpecificData = results.languageSpecificData;
973		exc->catchTemp = reinterpret_cast<void*>(results.landingPad);
	1089	get_landing_pad(exc->catchTemp, results);
974	1090	exc->adjustedPtr = results.adjustedPtr;
975	1091	#ifdef __WASM_EXCEPTIONS__
976	1092	// Wasm only uses a single phase (_UA_SEARCH_PHASE), so save the
...	...	@@ -1009,9 +1125,6 @@ __gxx_personality_seh0(PEXCEPTION_RECORD ms_exc, void *this_frame,
1009	1125
1010	1126	#else
1011	1127
1012		extern "C" _Unwind_Reason_Code __gnu_unwind_frame(_Unwind_Exception*,
1013		_Unwind_Context*);
1014
1015	1128	// Helper function to unwind one frame.
1016	1129	// ARM EHABI 7.3 and 7.4: If the personality function returns _URC_CONTINUE_UNWIND, the
1017	1130	// personality routine should update the virtual register set (VRS) according to the

lib/libcxxabi/src/cxa_thread_atexit.cpp+2-1

...	...	@@ -106,6 +106,7 @@ namespace {
106	106
107	107	#endif // HAVE___CXA_THREAD_ATEXIT_IMPL
108	108
	109	#if defined(__linux__) \|\| defined(__Fuchsia__)
109	110	extern "C" {
110	111
111	112	_LIBCXXABI_FUNC_VIS int __cxa_thread_atexit(Dtor dtor, void* obj, void* dso_symbol) throw() {
...	...	@@ -140,6 +141,6 @@ extern "C" {
140	141	}
141	142	#endif // HAVE___CXA_THREAD_ATEXIT_IMPL
142	143	}
143
144	144	} // extern "C"
	145	#endif // defined(__linux__) \|\| defined(__Fuchsia__)
145	146	} // namespace __cxxabiv1

lib/libcxxabi/src/demangle/DemangleConfig.h+4

...	...	@@ -115,4 +115,8 @@
115	115	#define DEMANGLE_NAMESPACE_BEGIN namespace { namespace itanium_demangle {
116	116	#define DEMANGLE_NAMESPACE_END } }
117	117
	118	// The DEMANGLE_ABI macro resolves to nothing when building libc++abi. Only
	119	// the llvm copy defines DEMANGLE_ABI as a visibility attribute.
	120	#define DEMANGLE_ABI
	121
118	122	#endif // LIBCXXABI_DEMANGLE_DEMANGLE_CONFIG_H

lib/libcxxabi/src/demangle/ItaniumDemangle.h+9-6

...	...	@@ -1366,7 +1366,7 @@ public:
1366	1366	template <typename Fn> void match(Fn F) const { F(Name, Params, Requires); }
1367	1367
1368	1368	void printLeft(OutputBuffer &OB) const override {
1369		ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
	1369	ScopedOverride<bool> LT(OB.TemplateTracker.InsideTemplate, true);
1370	1370	OB += "template<";
1371	1371	Params.printWithComma(OB);
1372	1372	OB += "> typename ";
...	...	@@ -1550,7 +1550,7 @@ public:
1550	1550	NodeArray getParams() { return Params; }
1551	1551
1552	1552	void printLeft(OutputBuffer &OB) const override {
1553		ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
	1553	ScopedOverride<bool> LT(OB.TemplateTracker.InsideTemplate, true);
1554	1554	OB += "<";
1555	1555	Params.printWithComma(OB);
1556	1556	OB += ">";
...	...	@@ -1824,7 +1824,7 @@ public:
1824	1824
1825	1825	void printDeclarator(OutputBuffer &OB) const {
1826	1826	if (!TemplateParams.empty()) {
1827		ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
	1827	ScopedOverride<bool> LT(OB.TemplateTracker.InsideTemplate, true);
1828	1828	OB += "<";
1829	1829	TemplateParams.printWithComma(OB);
1830	1830	OB += ">";
...	...	@@ -1885,7 +1885,9 @@ public:
1885	1885	}
1886	1886
1887	1887	void printLeft(OutputBuffer &OB) const override {
1888		bool ParenAll = OB.isGtInsideTemplateArgs() &&
	1888	// If we're printing a '<' inside of a template argument, and we haven't
	1889	// yet parenthesized the expression, do so now.
	1890	bool ParenAll = !OB.isInParensInTemplateArgs() &&
1889	1891	(InfixOperator == ">" \|\| InfixOperator == ">>");
1890	1892	if (ParenAll)
1891	1893	OB.printOpen();
...	...	@@ -2061,7 +2063,7 @@ public:
2061	2063	void printLeft(OutputBuffer &OB) const override {
2062	2064	OB += CastKind;
2063	2065	{
2064		ScopedOverride<unsigned> LT(OB.GtIsGt, 0);
	2066	ScopedOverride<bool> LT(OB.TemplateTracker.InsideTemplate, true);
2065	2067	OB += "<";
2066	2068	OB.printLeft(*To);
2067	2069	OB += ">";
...	...	@@ -3049,7 +3051,8 @@ template <typename Derived, typename Alloc> struct AbstractManglingParser {
3049	3051	Node *parse(bool ParseParams = true);
3050	3052	};
3051	3053
3052		const char* parse_discriminator(const char* first, const char* last);
	3054	DEMANGLE_ABI const char parse_discriminator(const char first,
	3055	const char *last);
3053	3056
3054	3057	// <name> ::= <nested-name> // N
3055	3058	// ::= <local-name> # See Scope Encoding below // Z

lib/libcxxabi/src/demangle/Utility.h+21-7

...	...	@@ -81,7 +81,7 @@ public:
81	81	OutputBuffer(const OutputBuffer &) = delete;
82	82	OutputBuffer &operator=(const OutputBuffer &) = delete;
83	83
84		virtual ~OutputBuffer() {}
	84	virtual ~OutputBuffer() = default;
85	85
86	86	operator std::string_view() const {
87	87	return std::string_view(Buffer, CurrentPosition);
...	...	@@ -104,18 +104,32 @@ public:
104	104	unsigned CurrentPackIndex = std::numeric_limits<unsigned>::max();
105	105	unsigned CurrentPackMax = std::numeric_limits<unsigned>::max();
106	106
107		/// When zero, we're printing template args and '>' needs to be parenthesized.
108		/// Use a counter so we can simply increment inside parentheses.
109		unsigned GtIsGt = 1;
	107	struct {
	108	/// The depth of '(' and ')' inside the currently printed template
	109	/// arguments.
	110	unsigned ParenDepth = 0;
110	111
111		bool isGtInsideTemplateArgs() const { return GtIsGt == 0; }
	112	/// True if we're currently printing a template argument.
	113	bool InsideTemplate = false;
	114	} TemplateTracker;
	115
	116	/// Returns true if we're currently between a '(' and ')' when printing
	117	/// template args.
	118	bool isInParensInTemplateArgs() const {
	119	return TemplateTracker.ParenDepth > 0;
	120	}
	121
	122	/// Returns true if we're printing template args.
	123	bool isInsideTemplateArgs() const { return TemplateTracker.InsideTemplate; }
112	124
113	125	void printOpen(char Open = '(') {
114		GtIsGt++;
	126	if (isInsideTemplateArgs())
	127	TemplateTracker.ParenDepth++;
115	128	*this += Open;
116	129	}
117	130	void printClose(char Close = ')') {
118		GtIsGt--;
	131	if (isInsideTemplateArgs())
	132	TemplateTracker.ParenDepth--;
119	133	*this += Close;
120	134	}
121	135

lib/libcxxabi/src/fallback_malloc.cpp+1-1

...	...	@@ -16,7 +16,7 @@
16	16	#endif
17	17	#endif
18	18
19		#include <__memory/aligned_alloc.h>
	19	#include "include/aligned_alloc.h" // from libc++
20	20	#include <__assert>
21	21	#include <stdlib.h> // for malloc, calloc, free
22	22	#include <string.h> // for memset

lib/libcxxabi/src/private_typeinfo.cpp+6

...	...	@@ -831,6 +831,10 @@ bool __pointer_to_member_type_info::can_catch_nested(
831	831	#pragma clang diagnostic ignored "-Wmissing-field-initializers"
832	832	#endif
833	833
	834	#pragma GCC diagnostic push
	835	// __dynamic_cast is called by the compiler, so there is no prototype
	836	#pragma GCC diagnostic ignored "-Wmissing-prototypes"
	837
834	838	// __dynamic_cast
835	839
836	840	// static_ptr: pointer to an object of type static_type; nonnull, and since the
...	...	@@ -953,6 +957,8 @@ __dynamic_cast(const void static_ptr, const __class_type_info static_type,
953	957	return const_cast<void*>(dst_ptr);
954	958	}
955	959
	960	#pragma GCC diagnostic pop
	961
956	962	#ifdef __clang__
957	963	#pragma clang diagnostic pop
958	964	#endif

lib/libcxxabi/src/stdlib_new_delete.cpp+5-5

...	...	@@ -8,8 +8,8 @@
8	8
9	9	#include "__cxxabi_config.h"
10	10	#include "abort_message.h"
	11	#include "include/aligned_alloc.h" // from libc++
11	12	#include "include/overridable_function.h" // from libc++
12		#include <__memory/aligned_alloc.h>
13	13	#include <cstddef>
14	14	#include <cstdlib>
15	15	#include <new>
...	...	@@ -63,7 +63,7 @@ static void* operator_new_impl(std::size_t size) {
63	63	return p;
64	64	}
65	65
66		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new, (std::size_t size)) _THROW_BAD_ALLOC {
	66	OVERRIDABLE_FUNCTION void* operator new(std::size_t size) _THROW_BAD_ALLOC {
67	67	void* p = operator_new_impl(size);
68	68	if (p == nullptr)
69	69	__throw_bad_alloc_shim();
...	...	@@ -94,7 +94,7 @@ _LIBCPP_WEAK void* operator new(size_t size, const std::nothrow_t&) noexcept {
94	94	#endif
95	95	}
96	96
97		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new[], (size_t size)) _THROW_BAD_ALLOC { return ::operator new(size); }
	97	OVERRIDABLE_FUNCTION void* operator new[](size_t size) _THROW_BAD_ALLOC { return ::operator new(size); }
98	98
99	99	_LIBCPP_WEAK void* operator new[](size_t size, const std::nothrow_t&) noexcept {
100	100	#if !_LIBCPP_HAS_EXCEPTIONS
...	...	@@ -154,7 +154,7 @@ static void* operator_new_aligned_impl(std::size_t size, std::align_val_t alignm
154	154	return p;
155	155	}
156	156
157		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new, (std::size_t size, std::align_val_t alignment)) _THROW_BAD_ALLOC {
	157	OVERRIDABLE_FUNCTION void* operator new(std::size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
158	158	void* p = operator_new_aligned_impl(size, alignment);
159	159	if (p == nullptr)
160	160	__throw_bad_alloc_shim();
...	...	@@ -185,7 +185,7 @@ _LIBCPP_WEAK void* operator new(size_t size, std::align_val_t alignment, const s
185	185	# endif
186	186	}
187	187
188		_LIBCPP_OVERRIDABLE_FUNCTION(void*, operator new[], (size_t size, std::align_val_t alignment)) _THROW_BAD_ALLOC {
	188	OVERRIDABLE_FUNCTION void* operator new[](size_t size, std::align_val_t alignment) _THROW_BAD_ALLOC {
189	189	return ::operator new(size, alignment);
190	190	}
191	191

lib/libtsan/LICENSE.TXT created+311

...	...	@@ -0,0 +1,311 @@
	1	==============================================================================
	2	The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
	3	==============================================================================
	4
	5	Apache License
	6	Version 2.0, January 2004
	7	http://www.apache.org/licenses/
	8
	9	TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
	10
	11	1. Definitions.
	12
	13	"License" shall mean the terms and conditions for use, reproduction,
	14	and distribution as defined by Sections 1 through 9 of this document.
	15
	16	"Licensor" shall mean the copyright owner or entity authorized by
	17	the copyright owner that is granting the License.
	18
	19	"Legal Entity" shall mean the union of the acting entity and all
	20	other entities that control, are controlled by, or are under common
	21	control with that entity. For the purposes of this definition,
	22	"control" means (i) the power, direct or indirect, to cause the
	23	direction or management of such entity, whether by contract or
	24	otherwise, or (ii) ownership of fifty percent (50%) or more of the
	25	outstanding shares, or (iii) beneficial ownership of such entity.
	26
	27	"You" (or "Your") shall mean an individual or Legal Entity
	28	exercising permissions granted by this License.
	29
	30	"Source" form shall mean the preferred form for making modifications,
	31	including but not limited to software source code, documentation
	32	source, and configuration files.
	33
	34	"Object" form shall mean any form resulting from mechanical
	35	transformation or translation of a Source form, including but
	36	not limited to compiled object code, generated documentation,
	37	and conversions to other media types.
	38
	39	"Work" shall mean the work of authorship, whether in Source or
	40	Object form, made available under the License, as indicated by a
	41	copyright notice that is included in or attached to the work
	42	(an example is provided in the Appendix below).
	43
	44	"Derivative Works" shall mean any work, whether in Source or Object
	45	form, that is based on (or derived from) the Work and for which the
	46	editorial revisions, annotations, elaborations, or other modifications
	47	represent, as a whole, an original work of authorship. For the purposes
	48	of this License, Derivative Works shall not include works that remain
	49	separable from, or merely link (or bind by name) to the interfaces of,
	50	the Work and Derivative Works thereof.
	51
	52	"Contribution" shall mean any work of authorship, including
	53	the original version of the Work and any modifications or additions
	54	to that Work or Derivative Works thereof, that is intentionally
	55	submitted to Licensor for inclusion in the Work by the copyright owner
	56	or by an individual or Legal Entity authorized to submit on behalf of
	57	the copyright owner. For the purposes of this definition, "submitted"
	58	means any form of electronic, verbal, or written communication sent
	59	to the Licensor or its representatives, including but not limited to
	60	communication on electronic mailing lists, source code control systems,
	61	and issue tracking systems that are managed by, or on behalf of, the
	62	Licensor for the purpose of discussing and improving the Work, but
	63	excluding communication that is conspicuously marked or otherwise
	64	designated in writing by the copyright owner as "Not a Contribution."
	65
	66	"Contributor" shall mean Licensor and any individual or Legal Entity
	67	on behalf of whom a Contribution has been received by Licensor and
	68	subsequently incorporated within the Work.
	69
	70	2. Grant of Copyright License. Subject to the terms and conditions of
	71	this License, each Contributor hereby grants to You a perpetual,
	72	worldwide, non-exclusive, no-charge, royalty-free, irrevocable
	73	copyright license to reproduce, prepare Derivative Works of,
	74	publicly display, publicly perform, sublicense, and distribute the
	75	Work and such Derivative Works in Source or Object form.
	76
	77	3. Grant of Patent License. Subject to the terms and conditions of
	78	this License, each Contributor hereby grants to You a perpetual,
	79	worldwide, non-exclusive, no-charge, royalty-free, irrevocable
	80	(except as stated in this section) patent license to make, have made,
	81	use, offer to sell, sell, import, and otherwise transfer the Work,
	82	where such license applies only to those patent claims licensable
	83	by such Contributor that are necessarily infringed by their
	84	Contribution(s) alone or by combination of their Contribution(s)
	85	with the Work to which such Contribution(s) was submitted. If You
	86	institute patent litigation against any entity (including a
	87	cross-claim or counterclaim in a lawsuit) alleging that the Work
	88	or a Contribution incorporated within the Work constitutes direct
	89	or contributory patent infringement, then any patent licenses
	90	granted to You under this License for that Work shall terminate
	91	as of the date such litigation is filed.
	92
	93	4. Redistribution. You may reproduce and distribute copies of the
	94	Work or Derivative Works thereof in any medium, with or without
	95	modifications, and in Source or Object form, provided that You
	96	meet the following conditions:
	97
	98	(a) You must give any other recipients of the Work or
	99	Derivative Works a copy of this License; and
	100
	101	(b) You must cause any modified files to carry prominent notices
	102	stating that You changed the files; and
	103
	104	(c) You must retain, in the Source form of any Derivative Works
	105	that You distribute, all copyright, patent, trademark, and
	106	attribution notices from the Source form of the Work,
	107	excluding those notices that do not pertain to any part of
	108	the Derivative Works; and
	109
	110	(d) If the Work includes a "NOTICE" text file as part of its
	111	distribution, then any Derivative Works that You distribute must
	112	include a readable copy of the attribution notices contained
	113	within such NOTICE file, excluding those notices that do not
	114	pertain to any part of the Derivative Works, in at least one
	115	of the following places: within a NOTICE text file distributed
	116	as part of the Derivative Works; within the Source form or
	117	documentation, if provided along with the Derivative Works; or,
	118	within a display generated by the Derivative Works, if and
	119	wherever such third-party notices normally appear. The contents
	120	of the NOTICE file are for informational purposes only and
	121	do not modify the License. You may add Your own attribution
	122	notices within Derivative Works that You distribute, alongside
	123	or as an addendum to the NOTICE text from the Work, provided
	124	that such additional attribution notices cannot be construed
	125	as modifying the License.
	126
	127	You may add Your own copyright statement to Your modifications and
	128	may provide additional or different license terms and conditions
	129	for use, reproduction, or distribution of Your modifications, or
	130	for any such Derivative Works as a whole, provided Your use,
	131	reproduction, and distribution of the Work otherwise complies with
	132	the conditions stated in this License.
	133
	134	5. Submission of Contributions. Unless You explicitly state otherwise,
	135	any Contribution intentionally submitted for inclusion in the Work
	136	by You to the Licensor shall be under the terms and conditions of
	137	this License, without any additional terms or conditions.
	138	Notwithstanding the above, nothing herein shall supersede or modify
	139	the terms of any separate license agreement you may have executed
	140	with Licensor regarding such Contributions.
	141
	142	6. Trademarks. This License does not grant permission to use the trade
	143	names, trademarks, service marks, or product names of the Licensor,
	144	except as required for reasonable and customary use in describing the
	145	origin of the Work and reproducing the content of the NOTICE file.
	146
	147	7. Disclaimer of Warranty. Unless required by applicable law or
	148	agreed to in writing, Licensor provides the Work (and each
	149	Contributor provides its Contributions) on an "AS IS" BASIS,
	150	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
	151	implied, including, without limitation, any warranties or conditions
	152	of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
	153	PARTICULAR PURPOSE. You are solely responsible for determining the
	154	appropriateness of using or redistributing the Work and assume any
	155	risks associated with Your exercise of permissions under this License.
	156
	157	8. Limitation of Liability. In no event and under no legal theory,
	158	whether in tort (including negligence), contract, or otherwise,
	159	unless required by applicable law (such as deliberate and grossly
	160	negligent acts) or agreed to in writing, shall any Contributor be
	161	liable to You for damages, including any direct, indirect, special,
	162	incidental, or consequential damages of any character arising as a
	163	result of this License or out of the use or inability to use the
	164	Work (including but not limited to damages for loss of goodwill,
	165	work stoppage, computer failure or malfunction, or any and all
	166	other commercial damages or losses), even if such Contributor
	167	has been advised of the possibility of such damages.
	168
	169	9. Accepting Warranty or Additional Liability. While redistributing
	170	the Work or Derivative Works thereof, You may choose to offer,
	171	and charge a fee for, acceptance of support, warranty, indemnity,
	172	or other liability obligations and/or rights consistent with this
	173	License. However, in accepting such obligations, You may act only
	174	on Your own behalf and on Your sole responsibility, not on behalf
	175	of any other Contributor, and only if You agree to indemnify,
	176	defend, and hold each Contributor harmless for any liability
	177	incurred by, or claims asserted against, such Contributor by reason
	178	of your accepting any such warranty or additional liability.
	179
	180	END OF TERMS AND CONDITIONS
	181
	182	APPENDIX: How to apply the Apache License to your work.
	183
	184	To apply the Apache License to your work, attach the following
	185	boilerplate notice, with the fields enclosed by brackets "[]"
	186	replaced with your own identifying information. (Don't include
	187	the brackets!) The text should be enclosed in the appropriate
	188	comment syntax for the file format. We also recommend that a
	189	file or class name and description of purpose be included on the
	190	same "printed page" as the copyright notice for easier
	191	identification within third-party archives.
	192
	193	Copyright [yyyy] [name of copyright owner]
	194
	195	Licensed under the Apache License, Version 2.0 (the "License");
	196	you may not use this file except in compliance with the License.
	197	You may obtain a copy of the License at
	198
	199	http://www.apache.org/licenses/LICENSE-2.0
	200
	201	Unless required by applicable law or agreed to in writing, software
	202	distributed under the License is distributed on an "AS IS" BASIS,
	203	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	204	See the License for the specific language governing permissions and
	205	limitations under the License.
	206
	207
	208	---- LLVM Exceptions to the Apache 2.0 License ----
	209
	210	As an exception, if, as a result of your compiling your source code, portions
	211	of this Software are embedded into an Object form of such source code, you
	212	may redistribute such embedded portions in such Object form without complying
	213	with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
	214
	215	In addition, if you combine or link compiled forms of this Software with
	216	software that is licensed under the GPLv2 ("Combined Software") and if a
	217	court of competent jurisdiction determines that the patent provision (Section
	218	3), the indemnity provision (Section 9) or other Section of the License
	219	conflicts with the conditions of the GPLv2, you may retroactively and
	220	prospectively choose to deem waived or otherwise exclude such Section(s) of
	221	the License, but only in their entirety and only with respect to the Combined
	222	Software.
	223
	224	==============================================================================
	225	Software from third parties included in the LLVM Project:
	226	==============================================================================
	227	The LLVM Project contains third party software which is under different license
	228	terms. All such code will be identified clearly using at least one of two
	229	mechanisms:
	230	1) It will be in a separate directory tree with its own `LICENSE.txt` or
	231	`LICENSE` file at the top containing the specific license and restrictions
	232	which apply to that software, or
	233	2) It will contain specific license and restriction terms at the top of every
	234	file.
	235
	236	==============================================================================
	237	Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
	238	==============================================================================
	239
	240	The compiler_rt library is dual licensed under both the University of Illinois
	241	"BSD-Like" license and the MIT license. As a user of this code you may choose
	242	to use it under either license. As a contributor, you agree to allow your code
	243	to be used under both.
	244
	245	Full text of the relevant licenses is included below.
	246
	247	==============================================================================
	248
	249	University of Illinois/NCSA
	250	Open Source License
	251
	252	Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT
	253
	254	All rights reserved.
	255
	256	Developed by:
	257
	258	LLVM Team
	259
	260	University of Illinois at Urbana-Champaign
	261
	262	http://llvm.org
	263
	264	Permission is hereby granted, free of charge, to any person obtaining a copy of
	265	this software and associated documentation files (the "Software"), to deal with
	266	the Software without restriction, including without limitation the rights to
	267	use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
	268	of the Software, and to permit persons to whom the Software is furnished to do
	269	so, subject to the following conditions:
	270
	271	* Redistributions of source code must retain the above copyright notice,
	272	this list of conditions and the following disclaimers.
	273
	274	* Redistributions in binary form must reproduce the above copyright notice,
	275	this list of conditions and the following disclaimers in the
	276	documentation and/or other materials provided with the distribution.
	277
	278	* Neither the names of the LLVM Team, University of Illinois at
	279	Urbana-Champaign, nor the names of its contributors may be used to
	280	endorse or promote products derived from this Software without specific
	281	prior written permission.
	282
	283	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	284	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
	285	FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	286	CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	287	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	288	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
	289	SOFTWARE.
	290
	291	==============================================================================
	292
	293	Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT
	294
	295	Permission is hereby granted, free of charge, to any person obtaining a copy
	296	of this software and associated documentation files (the "Software"), to deal
	297	in the Software without restriction, including without limitation the rights
	298	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	299	copies of the Software, and to permit persons to whom the Software is
	300	furnished to do so, subject to the following conditions:
	301
	302	The above copyright notice and this permission notice shall be included in
	303	all copies or substantial portions of the Software.
	304
	305	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	306	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	307	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	308	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	309	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	310	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	311	THE SOFTWARE.

lib/libtsan/builtins/assembly.h+41-8

...	...	@@ -14,7 +14,7 @@
14	14	#ifndef COMPILERRT_ASSEMBLY_H
15	15	#define COMPILERRT_ASSEMBLY_H
16	16
17		#if defined(__linux__) && defined(__CET__)
	17	#ifdef __CET__
18	18	#if __has_include(<cet.h>)
19	19	#include <cet.h>
20	20	#endif
...	...	@@ -71,19 +71,35 @@
71	71
72	72	#endif
73	73
	74	#if defined(__aarch64__) && defined(__ELF__) && \
	75	defined(COMPILER_RT_EXECUTE_ONLY_CODE)
	76	// The assembler always creates an implicit '.text' section with default flags
	77	// (SHF_ALLOC \| SHF_EXECINSTR), which is incompatible with the execute-only
	78	// '.text' section we want to create here because of the missing
	79	// SHF_AARCH64_PURECODE section flag. To solve this, we use 'unique,0' to
	80	// differentiate the two sections. The output will therefore have two separate
	81	// sections named '.text', where code will be placed into the execute-only
	82	// '.text' section, and the implicitly-created one will be empty.
	83	#define TEXT_SECTION \
	84	.section .text,"axy",@progbits,unique,0
	85	#else
	86	#define TEXT_SECTION \
	87	.text
	88	#endif
	89
74	90	#if defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__arm64ec__)
75	91	#define FUNC_ALIGN \
76		.text SEPARATOR \
77	92	.balign 16 SEPARATOR
78	93	#else
79	94	#define FUNC_ALIGN
80	95	#endif
81	96
82		// BTI and PAC gnu property note
	97	// BTI, PAC, and GCS gnu property note
83	98	#define NT_GNU_PROPERTY_TYPE_0 5
84	99	#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
85	100	#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
86	101	#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
	102	#define GNU_PROPERTY_AARCH64_FEATURE_1_GCS 4
87	103
88	104	#if defined(__ARM_FEATURE_BTI_DEFAULT)
89	105	#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
...	...	@@ -97,6 +113,12 @@
97	113	#define PAC_FLAG 0
98	114	#endif
99	115
	116	#if defined(__ARM_FEATURE_GCS_DEFAULT)
	117	#define GCS_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_GCS
	118	#else
	119	#define GCS_FLAG 0
	120	#endif
	121
100	122	#define GNU_PROPERTY(type, value) \
101	123	.pushsection .note.gnu.property, "a" SEPARATOR \
102	124	.p2align 3 SEPARATOR \
...	...	@@ -118,11 +140,12 @@
118	140	#define BTI_J
119	141	#endif
120	142
121		#if (BTI_FLAG \| PAC_FLAG) != 0
122		#define GNU_PROPERTY_BTI_PAC \
123		GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG \| PAC_FLAG)
	143	#if (BTI_FLAG \| PAC_FLAG \| GCS_FLAG) != 0
	144	#define GNU_PROPERTY_BTI_PAC_GCS \
	145	GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, \
	146	BTI_FLAG \| PAC_FLAG \| GCS_FLAG)
124	147	#else
125		#define GNU_PROPERTY_BTI_PAC
	148	#define GNU_PROPERTY_BTI_PAC_GCS
126	149	#endif
127	150
128	151	#if defined(__clang__) \|\| defined(__GCC_HAVE_DWARF2_CFI_ASM)
...	...	@@ -247,6 +270,7 @@
247	270	#endif
248	271
249	272	#define DEFINE_COMPILERRT_FUNCTION(name) \
	273	TEXT_SECTION SEPARATOR \
250	274	DEFINE_CODE_STATE \
251	275	FILE_LEVEL_DIRECTIVE SEPARATOR \
252	276	.globl FUNC_SYMBOL(SYMBOL_NAME(name)) SEPARATOR \
...	...	@@ -256,6 +280,7 @@
256	280	FUNC_SYMBOL(SYMBOL_NAME(name)):
257	281
258	282	#define DEFINE_COMPILERRT_THUMB_FUNCTION(name) \
	283	TEXT_SECTION SEPARATOR \
259	284	DEFINE_CODE_STATE \
260	285	FILE_LEVEL_DIRECTIVE SEPARATOR \
261	286	.globl FUNC_SYMBOL(SYMBOL_NAME(name)) SEPARATOR \
...	...	@@ -265,6 +290,7 @@
265	290	FUNC_SYMBOL(SYMBOL_NAME(name)):
266	291
267	292	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \
	293	TEXT_SECTION SEPARATOR \
268	294	DEFINE_CODE_STATE \
269	295	FILE_LEVEL_DIRECTIVE SEPARATOR \
270	296	.globl FUNC_SYMBOL(SYMBOL_NAME(name)) SEPARATOR \
...	...	@@ -274,6 +300,7 @@
274	300	FUNC_SYMBOL(SYMBOL_NAME(name)):
275	301
276	302	#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \
	303	TEXT_SECTION SEPARATOR \
277	304	DEFINE_CODE_STATE \
278	305	.globl FUNC_SYMBOL(name) SEPARATOR \
279	306	SYMBOL_IS_FUNC(name) SEPARATOR \
...	...	@@ -282,6 +309,7 @@
282	309	FUNC_SYMBOL(name):
283	310
284	311	#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name) \
	312	TEXT_SECTION SEPARATOR \
285	313	DEFINE_CODE_STATE \
286	314	FUNC_ALIGN \
287	315	.globl FUNC_SYMBOL(name) SEPARATOR \
...	...	@@ -296,7 +324,7 @@
296	324	.globl FUNC_SYMBOL(SYMBOL_NAME(name)) SEPARATOR \
297	325	SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
298	326	DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
299		.set FUNC_SYMBOL(SYMBOL_NAME(name)), FUNC_SYMBOL(target) SEPARATOR
	327	.set FUNC_SYMBOL(SYMBOL_NAME(name)), FUNC_SYMBOL(SYMBOL_NAME(target)) SEPARATOR
300	328
301	329	#if defined(__ARM_EABI__)
302	330	#define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \
...	...	@@ -329,4 +357,9 @@
329	357	#endif
330	358	#endif
331	359
	360	#if defined(__ASSEMBLER__) && (defined(__i386__) \|\| defined(__amd64__)) && \
	361	!defined(__arm64ec__)
	362	.att_syntax
	363	#endif
	364
332	365	#endif // COMPILERRT_ASSEMBLY_H

lib/libtsan/interception/interception_win.cpp+4

...	...	@@ -646,6 +646,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
646	646	case 0xC033: // 33 C0 : xor eax, eax
647	647	case 0xC933: // 33 C9 : xor ecx, ecx
648	648	case 0xD233: // 33 D2 : xor edx, edx
	649	case 0xFF33: // 33 FF : xor edi, edi
649	650	case 0x9066: // 66 90 : xchg %ax,%ax (Two-byte NOP)
650	651	case 0xDB84: // 84 DB : test bl,bl
651	652	case 0xC084: // 84 C0 : test al,al
...	...	@@ -764,6 +765,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
764	765
765	766	switch (0x00FFFFFF & (u32 )address) {
766	767	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
	768	case 0x02b70f: // 0f b7 02 : movzx eax, WORD PTR [rdx]
767	769	case 0xc00b4d: // 4d 0b c0 : or r8, r8
768	770	case 0xc03345: // 45 33 c0 : xor r8d, r8d
769	771	case 0xc08548: // 48 85 c0 : test rax, rax
...	...	@@ -799,6 +801,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
799	801	case 0xc9854d: // 4d 85 c9 : test r9, r9
800	802	case 0xc98b4c: // 4c 8b c9 : mov r9, rcx
801	803	case 0xd12948: // 48 29 d1 : sub rcx, rdx
	804	case 0xc22b4c: // 4c 2b c2 : sub r8, rdx
802	805	case 0xca2b48: // 48 2b ca : sub rcx, rdx
803	806	case 0xca3b48: // 48 3b ca : cmp rcx, rdx
804	807	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
...	...	@@ -813,6 +816,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
813	816	case 0xd9f748: // 48 f7 d9 : neg rcx
814	817	case 0xc03145: // 45 31 c0 : xor r8d,r8d
815	818	case 0xc93145: // 45 31 c9 : xor r9d,r9d
	819	case 0xd23345: // 45 33 d2 : xor r10d, r10d
816	820	case 0xdb3345: // 45 33 db : xor r11d, r11d
817	821	case 0xc08445: // 45 84 c0 : test r8b,r8b
818	822	case 0xd28445: // 45 84 d2 : test r10b,r10b

lib/libtsan/sanitizer_common/sanitizer_allocator_primary32.h+1

...	...	@@ -288,6 +288,7 @@ class SizeClassAllocator32 {
288	288	uptr ComputeRegionId(uptr mem) const {
289	289	if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
290	290	mem &= (kSpaceSize - 1);
	291	mem -= kSpaceBeg;
291	292	const uptr res = mem >> kRegionSizeLog;
292	293	CHECK_LT(res, kNumPossibleRegions);
293	294	return res;

lib/libtsan/sanitizer_common/sanitizer_allocator_primary64.h+18

...	...	@@ -113,6 +113,24 @@ class SizeClassAllocator64 {
113	113	// ~(uptr)0.
114	114	void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
115	115	uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
	116
	117	uptr MaxAddr = GetMaxUserVirtualAddress();
	118	// VReport does not call the sanitizer allocator.
	119	VReport(3, "Max user virtual address: 0x%zx\n", MaxAddr);
	120	VReport(3, "Total space size for primary allocator: 0x%zx\n",
	121	TotalSpaceSize);
	122	// TODO: revise the check if we ever configure sanitizers to deliberately
	123	// map beyond the 2**48 barrier (note that Linux pretends the VMA is
	124	// limited to 48-bit for backwards compatibility, but allows apps to
	125	// explicitly specify an address beyond that).
	126	if (heap_start + TotalSpaceSize >= MaxAddr) {
	127	// We can't easily adjust the requested heap size, because kSpaceSize is
	128	// const (for optimization) and used throughout the code.
	129	VReport(0, "Error: heap size %zx exceeds max user virtual address %zx\n",
	130	TotalSpaceSize, MaxAddr);
	131	VReport(
	132	0, "Try using a kernel that allows a larger virtual address space\n");
	133	}
116	134	PremappedHeap = heap_start != 0;
117	135	if (PremappedHeap) {
118	136	CHECK(!kUsingConstantSpaceBeg);

lib/libtsan/sanitizer_common/sanitizer_common.h+15-2

...	...	@@ -78,8 +78,8 @@ uptr GetMmapGranularity();
78	78	uptr GetMaxVirtualAddress();
79	79	uptr GetMaxUserVirtualAddress();
80	80	// Threads
81		tid_t GetTid();
82		int TgKill(pid_t pid, tid_t tid, int sig);
	81	ThreadID GetTid();
	82	int TgKill(pid_t pid, ThreadID tid, int sig);
83	83	uptr GetThreadSelf();
84	84	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
85	85	uptr *stack_bottom);
...	...	@@ -390,6 +390,9 @@ void ReportDeadlySignal(const SignalContext &sig, u32 tid,
390	390	void SetAlternateSignalStack();
391	391	void UnsetAlternateSignalStack();
392	392
	393	bool IsSignalHandlerFromSanitizer(int signum);
	394	bool SetSignalHandlerFromSanitizer(int signum, bool new_state);
	395
393	396	// Construct a one-line string:
394	397	// SUMMARY: SanitizerToolName: error_message
395	398	// and pass it to __sanitizer_report_error_summary.
...	...	@@ -484,6 +487,13 @@ inline uptr Log2(uptr x) {
484	487	return LeastSignificantSetBitIndex(x);
485	488	}
486	489
	490	inline bool IntervalsAreSeparate(uptr start1, uptr end1, uptr start2,
	491	uptr end2) {
	492	CHECK_LE(start1, end1);
	493	CHECK_LE(start2, end2);
	494	return (end1 < start2) \|\| (end2 < start1);
	495	}
	496
487	497	// Don't use std::min, std::max or std::swap, to minimize dependency
488	498	// on libstdc++.
489	499	template <class T>
...	...	@@ -734,6 +744,7 @@ enum ModuleArch {
734	744	kModuleArchARMV7S,
735	745	kModuleArchARMV7K,
736	746	kModuleArchARM64,
	747	kModuleArchARM64E,
737	748	kModuleArchLoongArch64,
738	749	kModuleArchRISCV64,
739	750	kModuleArchHexagon
...	...	@@ -807,6 +818,8 @@ inline const char *ModuleArchToString(ModuleArch arch) {
807	818	return "armv7k";
808	819	case kModuleArchARM64:
809	820	return "arm64";
	821	case kModuleArchARM64E:
	822	return "arm64e";
810	823	case kModuleArchLoongArch64:
811	824	return "loongarch64";
812	825	case kModuleArchRISCV64:

lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc+30-4

...	...	@@ -1285,8 +1285,34 @@ INTERCEPTOR(int, puts, char *s) {
1285	1285	#endif
1286	1286
1287	1287	#if SANITIZER_INTERCEPT_PRCTL
1288		INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1289		unsigned long arg4, unsigned long arg5) {
	1288
	1289	# if defined(__aarch64__)
	1290	// https://llvm.org/docs/PointerAuth.html
	1291	// AArch64 is currently the only architecture with full PAC support.
	1292	// Avoid adding PAC instructions to prevent crashes caused by
	1293	// prctl(PR_PAC_RESET_KEYS, ...). Since PR_PAC_RESET_KEYS resets the
	1294	// authentication key, using the old key afterward will lead to a crash.
	1295
	1296	# if defined(__ARM_FEATURE_BTI_DEFAULT)
	1297	# define BRANCH_PROTECTION_ATTRIBUTE \
	1298	__attribute__((target("branch-protection=bti")))
	1299	# else
	1300	# define BRANCH_PROTECTION_ATTRIBUTE \
	1301	__attribute__((target("branch-protection=none")))
	1302	# endif
	1303
	1304	# define PRCTL_INTERCEPTOR(ret_type, func, ...) \
	1305	DEFINE_REAL(ret_type, func, __VA_ARGS__) \
	1306	DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
	1307	extern "C" INTERCEPTOR_ATTRIBUTE BRANCH_PROTECTION_ATTRIBUTE ret_type \
	1308	WRAP(func)(__VA_ARGS__)
	1309
	1310	# else
	1311	# define PRCTL_INTERCEPTOR INTERCEPTOR
	1312	# endif
	1313
	1314	PRCTL_INTERCEPTOR(int, prctl, int option, unsigned long arg2,
	1315	unsigned long arg3, unsigned long arg4, unsigned long arg5) {
1290	1316	void *ctx;
1291	1317	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
1292	1318	static const int PR_SET_NAME = 15;
...	...	@@ -1300,7 +1326,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1300	1326	static const int PR_SET_SECCOMP = 22;
1301	1327	static const int SECCOMP_MODE_FILTER = 2;
1302	1328	# endif
1303		if (option == PR_SET_VMA && arg2 == 0UL) {
	1329	if (option == PR_SET_VMA && arg2 == 0UL && arg5 != 0UL) {
1304	1330	char name = (char )arg5;
1305	1331	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1306	1332	}
...	...	@@ -1326,7 +1352,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1326	1352	}
1327	1353	return res;
1328	1354	}
1329		#define INIT_PRCTL COMMON_INTERCEPT_FUNCTION(prctl)
	1355	# define INIT_PRCTL COMMON_INTERCEPT_FUNCTION(prctl)
1330	1356	#else
1331	1357	#define INIT_PRCTL
1332	1358	#endif // SANITIZER_INTERCEPT_PRCTL

lib/libtsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc+4

...	...	@@ -344,12 +344,16 @@ static void ioctl_table_fill() {
344	344	_(SOUND_PCM_WRITE_CHANNELS, WRITE, sizeof(int));
345	345	_(SOUND_PCM_WRITE_FILTER, WRITE, sizeof(int));
346	346	_(TCFLSH, NONE, 0);
	347	# if SANITIZER_TERMIOS_IOCTL_CONSTANTS
347	348	_(TCGETS, WRITE, struct_termios_sz);
	349	# endif
348	350	_(TCSBRK, NONE, 0);
349	351	_(TCSBRKP, NONE, 0);
	352	# if SANITIZER_TERMIOS_IOCTL_CONSTANTS
350	353	_(TCSETS, READ, struct_termios_sz);
351	354	_(TCSETSF, READ, struct_termios_sz);
352	355	_(TCSETSW, READ, struct_termios_sz);
	356	# endif
353	357	_(TCXONC, NONE, 0);
354	358	_(TIOCGLCKTRMIOS, WRITE, struct_termios_sz);
355	359	_(TIOCGSOFTCAR, WRITE, sizeof(int));

lib/libtsan/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S+2-1

...	...	@@ -5,6 +5,7 @@
5	5
6	6	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
7	7
	8	TEXT_SECTION
8	9	.comm _ZN14__interception10real_vforkE,8,8
9	10	.globl ASM_WRAPPER_NAME(vfork)
10	11	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
...	...	@@ -43,6 +44,6 @@ ASM_SIZE(vfork)
43	44	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
44	45	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
45	46
46		GNU_PROPERTY_BTI_PAC
	47	GNU_PROPERTY_BTI_PAC_GCS
47	48
48	49	#endif

lib/libtsan/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S+2

...	...	@@ -2,6 +2,8 @@
2	2
3	3	#include "sanitizer_common/sanitizer_asm.h"
4	4
	5	.att_syntax
	6
5	7	.comm _ZN14__interception10real_vforkE,4,4
6	8	.globl ASM_WRAPPER_NAME(vfork)
7	9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))

lib/libtsan/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S+2

...	...	@@ -2,6 +2,8 @@
2	2
3	3	#include "sanitizer_common/sanitizer_asm.h"
4	4
	5	.att_syntax
	6
5	7	.comm _ZN14__interception10real_vforkE,8,8
6	8	.globl ASM_WRAPPER_NAME(vfork)
7	9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))

lib/libtsan/sanitizer_common/sanitizer_common_syscalls.inc+18

...	...	@@ -143,6 +143,12 @@ struct sanitizer_kernel_sockaddr {
143	143	char sa_data[14];
144	144	};
145	145
	146	struct sanitizer_kernel_open_how {
	147	u64 flags;
	148	u64 mode;
	149	u64 resolve;
	150	};
	151
146	152	// Real sigset size is always passed as a syscall argument.
147	153	// Declare it "void" to catch sizeof(kernel_sigset_t).
148	154	typedef void kernel_sigset_t;
...	...	@@ -2843,6 +2849,18 @@ PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
2843	2849	POST_SYSCALL(openat)
2844	2850	(long res, long dfd, const void *filename, long flags, long mode) {}
2845	2851
	2852	PRE_SYSCALL(openat2)(long dfd, const void* filename,
	2853	const sanitizer_kernel_open_how* how, uptr howlen) {
	2854	if (filename)
	2855	PRE_READ(filename, __sanitizer::internal_strlen((const char*)filename) + 1);
	2856
	2857	if (how)
	2858	PRE_READ(how, howlen);
	2859	}
	2860
	2861	POST_SYSCALL(openat2)(long res, long dfd, const void* filename,
	2862	const sanitizer_kernel_open_how* how, uptr howlen) {}
	2863
2846	2864	PRE_SYSCALL(newfstatat)
2847	2865	(long dfd, const void filename, void statbuf, long flag) {
2848	2866	if (filename)

lib/libtsan/sanitizer_common/sanitizer_coverage_interface.inc deleted-43

...	...	@@ -1,43 +0,0 @@
1		//===-- sanitizer_coverage_interface.inc ----------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// Sanitizer Coverage interface list.
9		//===----------------------------------------------------------------------===//
10		INTERFACE_FUNCTION(__sanitizer_cov_dump)
11		INTERFACE_FUNCTION(__sanitizer_cov_reset)
12		INTERFACE_FUNCTION(__sanitizer_dump_coverage)
13		INTERFACE_FUNCTION(__sanitizer_dump_trace_pc_guard_coverage)
14		INTERFACE_WEAK_FUNCTION(__sancov_default_options)
15		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp)
16		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp1)
17		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp2)
18		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp4)
19		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_cmp8)
20		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp1)
21		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp2)
22		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp4)
23		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_const_cmp8)
24		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div4)
25		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_div8)
26		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
27		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
28		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
29		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
30		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1)
31		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2)
32		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4)
33		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8)
34		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16)
35		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1)
36		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2)
37		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4)
38		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8)
39		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16)
40		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
41		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
42		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)
43		INTERFACE_WEAK_FUNCTION(__sanitizer_cov_pcs_init)

lib/libtsan/sanitizer_common/sanitizer_file.cpp+40-12

...	...	@@ -36,9 +36,17 @@ void RawWrite(const char *buffer) {
36	36
37	37	void ReportFile::ReopenIfNecessary() {
38	38	mu->CheckLocked();
39		if (fd == kStdoutFd \|\| fd == kStderrFd) return;
40
41	39	uptr pid = internal_getpid();
	40	if (fallbackToStderrActive && fd_pid != pid) {
	41	// If fallbackToStderrActive is set then we fellback to stderr. If this is a
	42	// new process, mark fd as invalid so we attempt to open again.
	43	CHECK_EQ(fd, kStderrFd);
	44	fd = kInvalidFd;
	45	fallbackToStderrActive = false;
	46	}
	47	if (fd == kStdoutFd \|\| fd == kStderrFd)
	48	return;
	49
42	50	// If in tracer, use the parent's file.
43	51	if (pid == stoptheworld_tracer_pid)
44	52	pid = stoptheworld_tracer_ppid;
...	...	@@ -48,8 +56,7 @@ void ReportFile::ReopenIfNecessary() {
48	56	// process, close it now.
49	57	if (fd_pid == pid)
50	58	return;
51		else
52		CloseFile(fd);
	59	CloseFile(fd);
53	60	}
54	61
55	62	const char *exe_name = GetProcessName();
...	...	@@ -65,18 +72,24 @@ void ReportFile::ReopenIfNecessary() {
65	72	error_t err;
66	73	fd = OpenFile(full_path, WrOnly, &err);
67	74	if (fd == kInvalidFd) {
68		const char *ErrorMsgPrefix = "ERROR: Can't open file: ";
	75	bool fallback = common_flags()->log_fallback_to_stderr;
	76	const char *ErrorMsgPrefix =
	77	fallback ? "WARNING: Can't open file, falling back to stderr: "
	78	: "ERROR: Can't open file: ";
69	79	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
70	80	WriteToFile(kStderrFd, full_path, internal_strlen(full_path));
71	81	char errmsg[100];
72	82	internal_snprintf(errmsg, sizeof(errmsg), " (reason: %d)\n", err);
73	83	WriteToFile(kStderrFd, errmsg, internal_strlen(errmsg));
74		Die();
	84	if (!fallback)
	85	Die();
	86	fallbackToStderrActive = true;
	87	fd = kStderrFd;
75	88	}
76	89	fd_pid = pid;
77	90	}
78	91
79		static void RecursiveCreateParentDirs(char *path) {
	92	static void RecursiveCreateParentDirs(char *path, fd_t &fd) {
80	93	if (path[0] == '\0')
81	94	return;
82	95	for (int i = 1; path[i] != '\0'; ++i) {
...	...	@@ -85,12 +98,19 @@ static void RecursiveCreateParentDirs(char *path) {
85	98	continue;
86	99	path[i] = '\0';
87	100	if (!DirExists(path) && !CreateDir(path)) {
88		const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
	101	bool fallback = common_flags()->log_fallback_to_stderr;
	102	const char *ErrorMsgPrefix =
	103	fallback ? "WARNING: Can't create directory, falling back to stderr: "
	104	: "ERROR: Can't create directory: ";
89	105	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
90	106	WriteToFile(kStderrFd, path, internal_strlen(path));
91	107	const char *ErrorMsgSuffix = "\n";
92	108	WriteToFile(kStderrFd, ErrorMsgSuffix, internal_strlen(ErrorMsgSuffix));
93		Die();
	109	if (!fallback)
	110	Die();
	111	path[i] = save;
	112	fd = kStderrFd;
	113	return;
94	114	}
95	115	path[i] = save;
96	116	}
...	...	@@ -108,6 +128,9 @@ static void ParseAndSetPath(const char pattern, char dest,
108	128	CHECK(dest);
109	129	CHECK_GE(dest_size, 1);
110	130	dest[0] = '\0';
	131	// Return empty string if empty string was passed
	132	if (internal_strlen(pattern) == 0)
	133	return;
111	134	uptr next_substr_start_idx = 0;
112	135	for (uptr i = 0; i < internal_strlen(pattern) - 1; i++) {
113	136	if (pattern[i] != '%')
...	...	@@ -161,12 +184,17 @@ void ReportFile::SetReportPath(const char *path) {
161	184	if (path) {
162	185	uptr len = internal_strlen(path);
163	186	if (len > sizeof(path_prefix) - 100) {
164		const char *message = "ERROR: Path is too long: ";
	187	bool fallback = common_flags()->log_fallback_to_stderr;
	188	const char *message =
	189	fallback ? "WARNING: Path is too long, falling back to stderr: "
	190	: "ERROR: Path is too long: ";
165	191	WriteToFile(kStderrFd, message, internal_strlen(message));
166	192	WriteToFile(kStderrFd, path, 8);
167	193	message = "...\n";
168	194	WriteToFile(kStderrFd, message, internal_strlen(message));
169		Die();
	195	if (!fallback)
	196	Die();
	197	path = "stderr";
170	198	}
171	199	}
172	200
...	...	@@ -180,7 +208,7 @@ void ReportFile::SetReportPath(const char *path) {
180	208	fd = kStdoutFd;
181	209	} else {
182	210	ParseAndSetPath(path, path_prefix, kMaxPathLength);
183		RecursiveCreateParentDirs(path_prefix);
	211	RecursiveCreateParentDirs(path_prefix, fd);
184	212	}
185	213	}
186	214

lib/libtsan/sanitizer_common/sanitizer_file.h+3

...	...	@@ -43,6 +43,9 @@ struct ReportFile {
43	43	// PID of the process that opened fd. If a fork() occurs,
44	44	// the PID of child will be different from fd_pid.
45	45	uptr fd_pid;
	46	// Set to true if the last attempt to open the logfile failed, perhaps due to
	47	// permission errors
	48	bool fallbackToStderrActive = false;
46	49
47	50	private:
48	51	void ReopenIfNecessary();

lib/libtsan/sanitizer_common/sanitizer_flags.inc+7

...	...	@@ -65,6 +65,8 @@ COMMON_FLAG(
65	65	bool, log_to_syslog, (bool)SANITIZER_ANDROID \|\| (bool)SANITIZER_APPLE,
66	66	"Write all sanitizer output to syslog in addition to other means of "
67	67	"logging.")
	68	COMMON_FLAG(bool, log_fallback_to_stderr, false,
	69	"When set, fallback to stderr if we are unable to open log path.")
68	70	COMMON_FLAG(
69	71	int, verbosity, 0,
70	72	"Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
...	...	@@ -111,6 +113,11 @@ COMMON_FLAG(HandleSignalMode, handle_sigfpe, kHandleSignalYes,
111	113	COMMON_FLAG(bool, allow_user_segv_handler, true,
112	114	"Deprecated. True has no effect, use handle_sigbus=1. If false, "
113	115	"handle_=1 will be upgraded to handle_=2.")
	116	COMMON_FLAG(bool, cloak_sanitizer_signal_handlers, false,
	117	"If set, signal/sigaction will pretend that sanitizers did not "
	118	"preinstall any signal handlers. If the user subsequently installs "
	119	"a signal handler, this will disable cloaking for the respective "
	120	"signal.")
114	121	COMMON_FLAG(bool, use_sigaltstack, true,
115	122	"If set, uses alternate stack for signal handling.")
116	123	COMMON_FLAG(bool, detect_deadlocks, true,

lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp+30-3

...	...	@@ -14,6 +14,7 @@
14	14	#include "sanitizer_fuchsia.h"
15	15	#if SANITIZER_FUCHSIA
16	16
	17	# include <limits.h>
17	18	# include <pthread.h>
18	19	# include <stdlib.h>
19	20	# include <unistd.h>
...	...	@@ -68,7 +69,7 @@ int internal_dlinfo(void handle, int request, void p) { UNIMPLEMENTED(); }
68	69
69	70	uptr GetThreadSelf() { return reinterpret_cast<uptr>(thrd_current()); }
70	71
71		tid_t GetTid() { return GetThreadSelf(); }
	72	ThreadID GetTid() { return GetThreadSelf(); }
72	73
73	74	void Abort() { abort(); }
74	75
...	...	@@ -117,11 +118,37 @@ uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
117	118
118	119	sanitizer_shadow_bounds_t ShadowBounds;
119	120
	121	// Any sanitizer that utilizes shadow should explicitly call whenever it's
	122	// appropriate for that sanitizer to reference shadow bounds. For ASan, this is
	123	// done in `InitializeShadowMemory` and for HWASan, this is done in
	124	// `InitShadow`.
120	125	void InitShadowBounds() { ShadowBounds = __sanitizer_shadow_bounds(); }
121	126
	127	// TODO(leonardchan): It's not immediately clear from a user perspective if
	128	// `GetMaxUserVirtualAddress` should be called exatly once on runtime startup
	129	// or can be called multiple times. Currently it looks like most instances of
	130	// `GetMaxUserVirtualAddress` are meant to be called once, but if someone
	131	// decides to call this multiple times in the future, we should have a separate
	132	// function that's ok to call multiple times. Ideally we would just invoke this
	133	// syscall once. Also for Fuchsia, this syscall technically gets invoked twice
	134	// since `__sanitizer_shadow_bounds` also invokes this syscall under the hood.
122	135	uptr GetMaxUserVirtualAddress() {
123		InitShadowBounds();
124		return ShadowBounds.memory_limit - 1;
	136	zx_info_vmar_t info;
	137	zx_status_t status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR,
	138	&info, sizeof(info), NULL, NULL);
	139	CHECK_EQ(status, ZX_OK);
	140
	141	// Find the top of the accessible address space.
	142	uintptr_t top = info.base + info.len;
	143
	144	// Round it up to a power-of-two size. There may be some pages at
	145	// the top that can't actually be mapped, but for purposes of the
	146	// the shadow, we'll pretend they could be.
	147	int bit = (sizeof(uintptr_t) * CHAR_BIT) - __builtin_clzl(top);
	148	if (top != (uintptr_t)1 << bit)
	149	top = (uintptr_t)1 << (bit + 1);
	150
	151	return top - 1;
125	152	}
126	153
127	154	uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }

lib/libtsan/sanitizer_common/sanitizer_haiku.cpp+2-2

...	...	@@ -231,12 +231,12 @@ uptr internal_execve(const char filename, char const argv[],
231	231	}
232	232
233	233	# if 0
234		tid_t GetTid() {
	234	ThreadID GetTid() {
235	235	DEFINE__REAL(int, _lwp_self);
236	236	return _REAL(_lwp_self);
237	237	}
238	238
239		int TgKill(pid_t pid, tid_t tid, int sig) {
	239	int TgKill(pid_t pid, ThreadID tid, int sig) {
240	240	DEFINE__REAL(int, _lwp_kill, int a, int b);
241	241	(void)pid;
242	242	return _REAL(_lwp_kill, tid, sig);

lib/libtsan/sanitizer_common/sanitizer_internal_defs.h+1-1

...	...	@@ -209,7 +209,7 @@ typedef long ssize;
209	209	typedef sptr ssize;
210	210	#endif
211	211
212		typedef u64 tid_t;
	212	typedef u64 ThreadID;
213	213
214	214	// ----------- ATTENTION -------------
215	215	// This header should NOT include any other headers to avoid portability issues.

lib/libtsan/sanitizer_common/sanitizer_libc.cpp+8

...	...	@@ -190,6 +190,14 @@ uptr internal_strlcat(char dst, const char src, uptr maxlen) {
190	190	return dstlen + srclen;
191	191	}
192	192
	193	char* internal_strcat(char* dst, const char* src) {
	194	uptr len = internal_strlen(dst);
	195	uptr i;
	196	for (i = 0; src[i]; i++) dst[len + i] = src[i];
	197	dst[len + i] = 0;
	198	return dst;
	199	}
	200
193	201	char internal_strncat(char dst, const char *src, uptr n) {
194	202	uptr len = internal_strlen(dst);
195	203	uptr i;

lib/libtsan/sanitizer_common/sanitizer_libc.h+1

...	...	@@ -59,6 +59,7 @@ char internal_strdup(const char s);
59	59	uptr internal_strlen(const char *s);
60	60	uptr internal_strlcat(char dst, const char src, uptr maxlen);
61	61	char internal_strncat(char dst, const char *src, uptr n);
	62	char* internal_strcat(char* dst, const char* src);
62	63	int internal_strncmp(const char s1, const char s2, uptr n);
63	64	uptr internal_strlcpy(char dst, const char src, uptr maxlen);
64	65	char internal_strncpy(char dst, const char *src, uptr n);

lib/libtsan/sanitizer_common/sanitizer_linux.cpp+15-7

...	...	@@ -635,7 +635,7 @@ bool DirExists(const char *path) {
635	635	}
636	636
637	637	# if !SANITIZER_NETBSD
638		tid_t GetTid() {
	638	ThreadID GetTid() {
639	639	# if SANITIZER_FREEBSD
640	640	long Tid;
641	641	thr_self(&Tid);
...	...	@@ -649,7 +649,7 @@ tid_t GetTid() {
649	649	# endif
650	650	}
651	651
652		int TgKill(pid_t pid, tid_t tid, int sig) {
	652	int TgKill(pid_t pid, ThreadID tid, int sig) {
653	653	# if SANITIZER_LINUX
654	654	return internal_syscall(SYSCALL(tgkill), pid, tid, sig);
655	655	# elif SANITIZER_FREEBSD
...	...	@@ -1091,7 +1091,7 @@ ThreadLister::ThreadLister(pid_t pid) : buffer_(4096) {
1091	1091	}
1092	1092
1093	1093	ThreadLister::Result ThreadLister::ListThreads(
1094		InternalMmapVector<tid_t> *threads) {
	1094	InternalMmapVector<ThreadID> *threads) {
1095	1095	int descriptor = internal_open(task_path_.data(), O_RDONLY \| O_DIRECTORY);
1096	1096	if (internal_iserror(descriptor)) {
1097	1097	Report("Can't open %s for reading.\n", task_path_.data());
...	...	@@ -1146,7 +1146,7 @@ ThreadLister::Result ThreadLister::ListThreads(
1146	1146	}
1147	1147	}
1148	1148
1149		const char *ThreadLister::LoadStatus(tid_t tid) {
	1149	const char *ThreadLister::LoadStatus(ThreadID tid) {
1150	1150	status_path_.clear();
1151	1151	status_path_.AppendF("%s/%llu/status", task_path_.data(), tid);
1152	1152	auto cleanup = at_scope_exit([&] {
...	...	@@ -1159,7 +1159,7 @@ const char *ThreadLister::LoadStatus(tid_t tid) {
1159	1159	return buffer_.data();
1160	1160	}
1161	1161
1162		bool ThreadLister::IsAlive(tid_t tid) {
	1162	bool ThreadLister::IsAlive(ThreadID tid) {
1163	1163	// /proc/%d/task/%d/status uses same call to detect alive threads as
1164	1164	// proc_task_readdir. See task_state implementation in Linux.
1165	1165	static const char kPrefix[] = "\nPPid:";
...	...	@@ -1289,7 +1289,7 @@ uptr GetPageSize() {
1289	1289
1290	1290	uptr ReadBinaryName(/out/ char *buf, uptr buf_len) {
1291	1291	# if SANITIZER_HAIKU
1292		int cookie = 0;
	1292	int32 cookie = 0;
1293	1293	image_info info;
1294	1294	const char *argv0 = "<UNKNOWN>";
1295	1295	while (get_next_image_info(B_CURRENT_TEAM, &cookie, &info) == B_OK) {
...	...	@@ -1989,7 +1989,10 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1989	1989	# elif SANITIZER_NETBSD
1990	1990	uptr err = ucontext->uc_mcontext.__gregs[_REG_ERR];
1991	1991	# elif SANITIZER_HAIKU
1992		uptr err = ucontext->uc_mcontext.r13;
	1992	uptr err = 0; // FIXME: ucontext->uc_mcontext.r13;
	1993	// The err register was added on the main branch and not
	1994	// available with the current release. To be reverted later.
	1995	// https://github.com/haiku/haiku/commit/11adda21aa4e6b24f71a496868a44d7607bc3764
1993	1996	# elif SANITIZER_SOLARIS && defined(__i386__)
1994	1997	const int Err = 13;
1995	1998	uptr err = ucontext->uc_mcontext.gregs[Err];
...	...	@@ -2619,6 +2622,11 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2619	2622	*pc = ucontext->uc_mcontext.mc_eip;
2620	2623	*bp = ucontext->uc_mcontext.mc_ebp;
2621	2624	*sp = ucontext->uc_mcontext.mc_esp;
	2625	# elif SANITIZER_HAIKU
	2626	ucontext_t ucontext = (ucontext_t )context;
	2627	*pc = ucontext->uc_mcontext.eip;
	2628	*bp = ucontext->uc_mcontext.ebp;
	2629	*sp = ucontext->uc_mcontext.esp;
2622	2630	# else
2623	2631	ucontext_t ucontext = (ucontext_t )context;
2624	2632	# if SANITIZER_SOLARIS

lib/libtsan/sanitizer_common/sanitizer_linux.h+3-3

...	...	@@ -108,11 +108,11 @@ class ThreadLister {
108	108	Incomplete,
109	109	Ok,
110	110	};
111		Result ListThreads(InternalMmapVector<tid_t> *threads);
112		const char *LoadStatus(tid_t tid);
	111	Result ListThreads(InternalMmapVector<ThreadID> *threads);
	112	const char *LoadStatus(ThreadID tid);
113	113
114	114	private:
115		bool IsAlive(tid_t tid);
	115	bool IsAlive(ThreadID tid);
116	116
117	117	InternalScopedString task_path_;
118	118	InternalScopedString status_path_;

lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp+1

...	...	@@ -29,6 +29,7 @@
29	29	# include "sanitizer_solaris.h"
30	30
31	31	# if SANITIZER_HAIKU
	32	# define _GNU_SOURCE
32	33	# define _DEFAULT_SOURCE
33	34	# endif
34	35

lib/libtsan/sanitizer_common/sanitizer_mac.cpp+255-104

...	...	@@ -22,6 +22,11 @@
22	22	# endif
23	23	# include <stdio.h>
24	24
	25	// Start searching for available memory region past PAGEZERO, which is
	26	// 4KB on 32-bit and 4GB on 64-bit.
	27	# define GAP_SEARCH_START_ADDRESS \
	28	((SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000)
	29
25	30	# include "sanitizer_common.h"
26	31	# include "sanitizer_file.h"
27	32	# include "sanitizer_flags.h"
...	...	@@ -58,9 +63,11 @@ extern char ***_NSGetArgv(void);
58	63	# include <dlfcn.h> // for dladdr()
59	64	# include <errno.h>
60	65	# include <fcntl.h>
	66	# include <inttypes.h>
61	67	# include <libkern/OSAtomic.h>
62	68	# include <mach-o/dyld.h>
63	69	# include <mach/mach.h>
	70	# include <mach/mach_error.h>
64	71	# include <mach/mach_time.h>
65	72	# include <mach/vm_statistics.h>
66	73	# include <malloc/malloc.h>
...	...	@@ -96,8 +103,16 @@ extern "C" {
96	103	natural_t *nesting_depth,
97	104	vm_region_recurse_info_t info,
98	105	mach_msg_type_number_t *infoCnt);
	106
	107	extern const void* _dyld_get_shared_cache_range(size_t* length);
99	108	}
100	109
	110	# if !SANITIZER_GO
	111	// Weak symbol no-op when TSan is not linked
	112	SANITIZER_WEAK_ATTRIBUTE extern void __tsan_set_in_internal_write_call(
	113	bool value) {}
	114	# endif
	115
101	116	namespace __sanitizer {
102	117
103	118	#include "sanitizer_syscall_generic.inc"
...	...	@@ -168,7 +183,15 @@ uptr internal_read(fd_t fd, void *buf, uptr count) {
168	183	}
169	184
170	185	uptr internal_write(fd_t fd, const void *buf, uptr count) {
	186	# if SANITIZER_GO
171	187	return write(fd, buf, count);
	188	# else
	189	// We need to disable interceptors when writing in TSan
	190	__tsan_set_in_internal_write_call(true);
	191	uptr res = write(fd, buf, count);
	192	__tsan_set_in_internal_write_call(false);
	193	return res;
	194	# endif
172	195	}
173	196
174	197	uptr internal_stat(const char path, void buf) {
...	...	@@ -258,53 +281,43 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
258	281	(size_t)newlen);
259	282	}
260	283
261		static fd_t internal_spawn_impl(const char argv[], const char envp[],
262		pid_t *pid) {
263		fd_t primary_fd = kInvalidFd;
264		fd_t secondary_fd = kInvalidFd;
	284	bool internal_spawn(const char* argv[], const char* envp[], pid_t* pid,
	285	fd_t fd_stdin, fd_t fd_stdout) {
	286	// NOTE: Caller ensures that fd_stdin and fd_stdout are not 0, 1, or 2, since
	287	// this can break communication.
	288	//
	289	// NOTE: Caller is responsible for closing fd_stdin after the process has
	290	// died.
265	291
	292	int res;
266	293	auto fd_closer = at_scope_exit([&] {
267		internal_close(primary_fd);
268		internal_close(secondary_fd);
	294	// NOTE: We intentionally do not close fd_stdin since this can
	295	// cause us to receive a fatal SIGPIPE if the process dies.
	296	internal_close(fd_stdout);
269	297	});
270	298
271		// We need a new pseudoterminal to avoid buffering problems. The 'atos' tool
272		// in particular detects when it's talking to a pipe and forgets to flush the
273		// output stream after sending a response.
274		primary_fd = posix_openpt(O_RDWR);
275		if (primary_fd == kInvalidFd)
276		return kInvalidFd;
277
278		int res = grantpt(primary_fd) \|\| unlockpt(primary_fd);
279		if (res != 0) return kInvalidFd;
280
281		// Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.
282		char secondary_pty_name[128];
283		res = ioctl(primary_fd, TIOCPTYGNAME, secondary_pty_name);
284		if (res == -1) return kInvalidFd;
285
286		secondary_fd = internal_open(secondary_pty_name, O_RDWR);
287		if (secondary_fd == kInvalidFd)
288		return kInvalidFd;
289
290	299	// File descriptor actions
291	300	posix_spawn_file_actions_t acts;
292	301	res = posix_spawn_file_actions_init(&acts);
293		if (res != 0) return kInvalidFd;
	302	if (res != 0)
	303	return false;
294	304
295	305	auto acts_cleanup = at_scope_exit([&] {
296	306	posix_spawn_file_actions_destroy(&acts);
297	307	});
298	308
299		res = posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDIN_FILENO) \|\|
300		posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDOUT_FILENO) \|\|
301		posix_spawn_file_actions_addclose(&acts, secondary_fd);
302		if (res != 0) return kInvalidFd;
	309	res = posix_spawn_file_actions_adddup2(&acts, fd_stdin, STDIN_FILENO) \|\|
	310	posix_spawn_file_actions_adddup2(&acts, fd_stdout, STDOUT_FILENO) \|\|
	311	posix_spawn_file_actions_addclose(&acts, fd_stdin) \|\|
	312	posix_spawn_file_actions_addclose(&acts, fd_stdout);
	313	if (res != 0)
	314	return false;
303	315
304	316	// Spawn attributes
305	317	posix_spawnattr_t attrs;
306	318	res = posix_spawnattr_init(&attrs);
307		if (res != 0) return kInvalidFd;
	319	if (res != 0)
	320	return false;
308	321
309	322	auto attrs_cleanup = at_scope_exit([&] {
310	323	posix_spawnattr_destroy(&attrs);
...	...	@@ -313,50 +326,17 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],
313	326	// In the spawned process, close all file descriptors that are not explicitly
314	327	// described by the file actions object. This is Darwin-specific extension.
315	328	res = posix_spawnattr_setflags(&attrs, POSIX_SPAWN_CLOEXEC_DEFAULT);
316		if (res != 0) return kInvalidFd;
	329	if (res != 0)
	330	return false;
317	331
318	332	// posix_spawn
319	333	char argv_casted = const_cast<char >(argv);
320	334	char envp_casted = const_cast<char >(envp);
321	335	res = posix_spawn(pid, argv[0], &acts, &attrs, argv_casted, envp_casted);
322		if (res != 0) return kInvalidFd;
323
324		// Disable echo in the new terminal, disable CR.
325		struct termios termflags;
326		tcgetattr(primary_fd, &termflags);
327		termflags.c_oflag &= ~ONLCR;
328		termflags.c_lflag &= ~ECHO;
329		tcsetattr(primary_fd, TCSANOW, &termflags);
330
331		// On success, do not close primary_fd on scope exit.
332		fd_t fd = primary_fd;
333		primary_fd = kInvalidFd;
334
335		return fd;
336		}
337
338		fd_t internal_spawn(const char argv[], const char envp[], pid_t *pid) {
339		// The client program may close its stdin and/or stdout and/or stderr thus
340		// allowing open/posix_openpt to reuse file descriptors 0, 1 or 2. In this
341		// case the communication is broken if either the parent or the child tries to
342		// close or duplicate these descriptors. We temporarily reserve these
343		// descriptors here to prevent this.
344		fd_t low_fds[3];
345		size_t count = 0;
346
347		for (; count < 3; count++) {
348		low_fds[count] = posix_openpt(O_RDWR);
349		if (low_fds[count] >= STDERR_FILENO)
350		break;
351		}
352
353		fd_t fd = internal_spawn_impl(argv, envp, pid);
354
355		for (; count > 0; count--) {
356		internal_close(low_fds[count]);
357		}
	336	if (res != 0)
	337	return false;
358	338
359		return fd;
	339	return true;
360	340	}
361	341
362	342	uptr internal_rename(const char oldpath, const char newpath) {
...	...	@@ -394,8 +374,8 @@ bool DirExists(const char *path) {
394	374	return S_ISDIR(st.st_mode);
395	375	}
396	376
397		tid_t GetTid() {
398		tid_t tid;
	377	ThreadID GetTid() {
	378	ThreadID tid;
399	379	pthread_threadid_np(nullptr, &tid);
400	380	return tid;
401	381	}
...	...	@@ -769,11 +749,17 @@ void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
769	749	static Mutex syslog_lock;
770	750	# endif
771	751
	752	# if SANITIZER_DRIVERKIT
	753	# define SANITIZER_OS_LOG os_log
	754	# else
	755	# define SANITIZER_OS_LOG os_log_error
	756	# endif
	757
772	758	void WriteOneLineToSyslog(const char *s) {
773	759	#if !SANITIZER_GO
774	760	syslog_lock.CheckLocked();
775	761	if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {
776		os_log_error(OS_LOG_DEFAULT, "%{public}s", s);
	762	SANITIZER_OS_LOG(OS_LOG_DEFAULT, "%{public}s", s);
777	763	} else {
778	764	#pragma clang diagnostic push
779	765	// as_log is deprecated.
...	...	@@ -837,22 +823,22 @@ void LogMessageOnPrintf(const char *str) {
837	823
838	824	void LogFullErrorReport(const char *buffer) {
839	825	# if !SANITIZER_GO
840		// Log with os_log_error. This will make it into the crash log.
	826	// When logging with os_log_error this will make it into the crash log.
841	827	if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
842	828	sizeof("AddressSanitizer") - 1) == 0)
843		os_log_error(OS_LOG_DEFAULT, "Address Sanitizer reported a failure.");
	829	SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Address Sanitizer reported a failure.");
844	830	else if (internal_strncmp(SanitizerToolName, "UndefinedBehaviorSanitizer",
845	831	sizeof("UndefinedBehaviorSanitizer") - 1) == 0)
846		os_log_error(OS_LOG_DEFAULT,
847		"Undefined Behavior Sanitizer reported a failure.");
	832	SANITIZER_OS_LOG(OS_LOG_DEFAULT,
	833	"Undefined Behavior Sanitizer reported a failure.");
848	834	else if (internal_strncmp(SanitizerToolName, "ThreadSanitizer",
849	835	sizeof("ThreadSanitizer") - 1) == 0)
850		os_log_error(OS_LOG_DEFAULT, "Thread Sanitizer reported a failure.");
	836	SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Thread Sanitizer reported a failure.");
851	837	else
852		os_log_error(OS_LOG_DEFAULT, "Sanitizer tool reported a failure.");
	838	SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Sanitizer tool reported a failure.");
853	839
854	840	if (common_flags()->log_to_syslog)
855		os_log_error(OS_LOG_DEFAULT, "Consult syslog for more information.");
	841	SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Consult syslog for more information.");
856	842
857	843	// Log to syslog.
858	844	// The logging on OS X may call pthread_create so we need the threading
...	...	@@ -933,7 +919,17 @@ static void DisableMmapExcGuardExceptions() {
933	919	RTLD_DEFAULT, "task_set_exc_guard_behavior");
934	920	if (set_behavior == nullptr) return;
935	921	const task_exc_guard_behavior_t task_exc_guard_none = 0;
936		set_behavior(mach_task_self(), task_exc_guard_none);
	922	kern_return_t res = set_behavior(mach_task_self(), task_exc_guard_none);
	923	if (res != KERN_SUCCESS) {
	924	Report(
	925	"WARN: task_set_exc_guard_behavior returned %d (%s), "
	926	"mmap may fail unexpectedly.\n",
	927	res, mach_error_string(res));
	928	if (res == KERN_DENIED)
	929	Report(
	930	"HINT: Check that task_set_exc_guard_behavior is allowed by "
	931	"sandbox.\n");
	932	}
937	933	}
938	934
939	935	static void VerifyInterceptorsWorking();
...	...	@@ -1100,6 +1096,67 @@ static void StripEnv() {
1100	1096	}
1101	1097	#endif // SANITIZER_GO
1102	1098
	1099	// Prints out a consolidated memory map: contiguous regions
	1100	// are merged together.
	1101	static void PrintVmmap() {
	1102	const mach_vm_address_t max_vm_address = GetMaxVirtualAddress() + 1;
	1103	mach_vm_address_t address = GAP_SEARCH_START_ADDRESS;
	1104	kern_return_t kr = KERN_SUCCESS;
	1105
	1106	Report("Memory map:\n");
	1107	mach_vm_address_t last = 0;
	1108	mach_vm_address_t lastsz = 0;
	1109
	1110	while (1) {
	1111	mach_vm_size_t vmsize = 0;
	1112	natural_t depth = 0;
	1113	vm_region_submap_short_info_data_64_t vminfo;
	1114	mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
	1115	kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
	1116	(vm_region_info_t)&vminfo, &count);
	1117
	1118	if (kr == KERN_DENIED) {
	1119	Report(
	1120	"ERROR: mach_vm_region_recurse got KERN_DENIED when printing memory "
	1121	"map.\n");
	1122	Report(
	1123	"HINT: Check whether mach_vm_region_recurse is allowed by "
	1124	"sandbox.\n");
	1125	}
	1126
	1127	if (kr == KERN_SUCCESS && address < max_vm_address) {
	1128	if (last + lastsz == address) {
	1129	// This region is contiguous with the last; merge together.
	1130	lastsz += vmsize;
	1131	} else {
	1132	if (lastsz)
	1133	Printf("\|\| `[%p, %p]` \|\| size=0x%016" PRIx64 " \|\|\n", (void*)last,
	1134	(void*)(last + lastsz), lastsz);
	1135
	1136	last = address;
	1137	lastsz = vmsize;
	1138	}
	1139	address += vmsize;
	1140	} else {
	1141	// We've reached the end of the memory map. Print the last remaining
	1142	// region, if there is one.
	1143	if (lastsz)
	1144	Printf("\|\| `[%p, %p]` \|\| size=0x%016" PRIx64 " \|\|\n", (void*)last,
	1145	(void*)(last + lastsz), lastsz);
	1146
	1147	break;
	1148	}
	1149	}
	1150	}
	1151
	1152	static void ReportShadowAllocFail(uptr shadow_size_bytes, uptr alignment) {
	1153	Report(
	1154	"FATAL: Failed to allocate shadow memory. Tried to allocate %p bytes "
	1155	"(alignment=%p).\n",
	1156	(void)shadow_size_bytes, (void)alignment);
	1157	PrintVmmap();
	1158	}
	1159
1103	1160	char **GetArgv() {
1104	1161	return *_NSGetArgv();
1105	1162	}
...	...	@@ -1207,10 +1264,11 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1207	1264	if (new_max_vm < max_occupied_addr) {
1208	1265	Report("Unable to find a memory range for dynamic shadow.\n");
1209	1266	Report(
1210		"space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
1211		"new_max_vm = %p\n",
1212		(void )space_size, (void )largest_gap_found,
1213		(void )max_occupied_addr, (void )new_max_vm);
	1267	"\tspace_size = %p\n\tlargest_gap_found = %p\n\tmax_occupied_addr "
	1268	"= %p\n\tnew_max_vm = %p\n",
	1269	(void)space_size, (void)largest_gap_found, (void*)max_occupied_addr,
	1270	(void*)new_max_vm);
	1271	ReportShadowAllocFail(shadow_size_bytes, alignment);
1214	1272	CHECK(0 && "cannot place shadow");
1215	1273	}
1216	1274	RestrictMemoryToMaxAddress(new_max_vm);
...	...	@@ -1221,6 +1279,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1221	1279	nullptr, nullptr);
1222	1280	if (shadow_start == 0) {
1223	1281	Report("Unable to find a memory range after restricting VM.\n");
	1282	ReportShadowAllocFail(shadow_size_bytes, alignment);
1224	1283	CHECK(0 && "cannot place shadow after restricting vm");
1225	1284	}
1226	1285	}
...	...	@@ -1229,6 +1288,25 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1229	1288	return shadow_start;
1230	1289	}
1231	1290
	1291	// Returns a list of ranges which must be covered by shadow memory,
	1292	// and cannot overlap with any fixed mappings made by a sanitizer.
	1293	// This can ensure that the sanitizer runtime does not map over
	1294	// platform-reserved regions.
	1295	void GetAppReservedRanges(InternalMmapVector<ReservedRange>& ranges) {
	1296	ranges.clear();
	1297
	1298	# if SANITIZER_OSX
	1299	// On macOS, the first 512GB are platform-reserved (some of which
	1300	// may also be available to applications).
	1301	ranges.push_back({0x1000UL, 0x8000000000UL});
	1302	# endif
	1303
	1304	VReport(2, "App ranges:\n");
	1305	for (auto& [range_start, range_end] : ranges) {
	1306	VReport(2, " [%p, %p]\n", range_start, range_end);
	1307	}
	1308	}
	1309
1232	1310	uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1233	1311	uptr num_aliases, uptr ring_buffer_size) {
1234	1312	CHECK(false && "HWASan aliasing is unimplemented on Mac");
...	...	@@ -1236,40 +1314,61 @@ uptr MapDynamicShadowAndAliases(uptr shadow_size, uptr alias_size,
1236	1314	}
1237	1315
1238	1316	uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1239		uptr *largest_gap_found,
1240		uptr *max_occupied_addr) {
1241		typedef vm_region_submap_short_info_data_64_t RegionInfo;
1242		enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
1243		// Start searching for available memory region past PAGEZERO, which is
1244		// 4KB on 32-bit and 4GB on 64-bit.
1245		mach_vm_address_t start_address =
1246		(SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
1247
	1317	uptr* largest_gap_found,
	1318	uptr* max_occupied_addr) {
1248	1319	const mach_vm_address_t max_vm_address = GetMaxVirtualAddress() + 1;
1249		mach_vm_address_t address = start_address;
1250		mach_vm_address_t free_begin = start_address;
	1320	mach_vm_address_t address = GAP_SEARCH_START_ADDRESS;
	1321	mach_vm_address_t free_begin = GAP_SEARCH_START_ADDRESS;
	1322
	1323	// Restrict the search to be after any reserved ranges
	1324	InternalMmapVector<ReservedRange> app_ranges;
	1325	GetAppReservedRanges(app_ranges);
	1326
	1327	for (auto& [range_start, range_end] : app_ranges) {
	1328	address = Max(address, (mach_vm_address_t)range_end);
	1329	free_begin = Max(free_begin, (mach_vm_address_t)range_end);
	1330	}
	1331
1251	1332	kern_return_t kr = KERN_SUCCESS;
1252	1333	if (largest_gap_found) *largest_gap_found = 0;
1253	1334	if (max_occupied_addr) *max_occupied_addr = 0;
1254	1335	while (kr == KERN_SUCCESS) {
1255	1336	mach_vm_size_t vmsize = 0;
1256	1337	natural_t depth = 0;
1257		RegionInfo vminfo;
1258		mach_msg_type_number_t count = kRegionInfoSize;
	1338	vm_region_submap_short_info_data_64_t vminfo;
	1339	mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
1259	1340	kr = mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
1260	1341	(vm_region_info_t)&vminfo, &count);
1261	1342
1262		// There are cases where going beyond the processes' max vm does
1263		// not return KERN_INVALID_ADDRESS so we check for going beyond that
1264		// max address as well.
1265		if (kr == KERN_INVALID_ADDRESS \|\| address > max_vm_address) {
	1343	if (kr == KERN_SUCCESS) {
	1344	// There are cases where going beyond the processes' max vm does
	1345	// not return KERN_INVALID_ADDRESS so we check for going beyond that
	1346	// max address as well.
	1347	if (address > max_vm_address) {
	1348	address = max_vm_address;
	1349	kr = -1; // break after this iteration.
	1350	}
	1351
	1352	if (max_occupied_addr)
	1353	*max_occupied_addr = address + vmsize;
	1354	} else if (kr == KERN_INVALID_ADDRESS) {
1266	1355	// No more regions beyond "address", consider the gap at the end of VM.
1267	1356	address = max_vm_address;
1268		vmsize = 0;
1269		kr = -1; // break after this iteration.
	1357
	1358	// We will break after this iteration anyway since kr != KERN_SUCCESS
	1359	} else if (kr == KERN_DENIED) {
	1360	Report("ERROR: Unable to find a memory range for dynamic shadow.\n");
	1361	Report("HINT: Ensure mach_vm_region_recurse is allowed under sandbox.\n");
	1362	Die();
1270	1363	} else {
1271		if (max_occupied_addr) *max_occupied_addr = address + vmsize;
	1364	Report(
	1365	"WARNING: mach_vm_region_recurse returned unexpected code %d (%s)\n",
	1366	kr, mach_error_string(kr));
	1367	DCHECK(false && "mach_vm_region_recurse returned unexpected code");
	1368	break; // address is not valid unless KERN_SUCCESS, therefore we must not
	1369	// use it.
1272	1370	}
	1371
1273	1372	if (free_begin != address) {
1274	1373	// We found a free region [free_begin..address-1].
1275	1374	uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
...	...	@@ -1292,6 +1391,58 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1292	1391	return 0;
1293	1392	}
1294	1393
	1394	// This function (when used during initialization when there is
	1395	// only a single thread), can be used to verify that a range
	1396	// of memory hasn't already been mapped, and won't be mapped
	1397	// later in the shared cache.
	1398	//
	1399	// If the syscall mach_vm_region_recurse fails (due to sandbox),
	1400	// we assume that the memory is not mapped so that execution can continue.
	1401	//
	1402	// NOTE: range_end is inclusive
	1403	//
	1404	// WARNING: This function must NOT allocate memory, since it is
	1405	// used in InitializeShadowMemory between where we search for
	1406	// space for shadow and where we actually allocate it.
	1407	bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
	1408	mach_vm_size_t vmsize = 0;
	1409	natural_t depth = 0;
	1410	vm_region_submap_short_info_data_64_t vminfo;
	1411	mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
	1412	mach_vm_address_t address = range_start;
	1413
	1414	// First, check if the range is already mapped.
	1415	kern_return_t kr =
	1416	mach_vm_region_recurse(mach_task_self(), &address, &vmsize, &depth,
	1417	(vm_region_info_t)&vminfo, &count);
	1418
	1419	if (kr == KERN_DENIED) {
	1420	Report(
	1421	"WARN: mach_vm_region_recurse returned KERN_DENIED when checking "
	1422	"whether an address is mapped.\n");
	1423	Report("HINT: Is mach_vm_region_recurse allowed by sandbox?\n");
	1424	}
	1425
	1426	if (kr == KERN_SUCCESS && !IntervalsAreSeparate(address, address + vmsize - 1,
	1427	range_start, range_end)) {
	1428	// Overlaps with already-mapped memory
	1429	return false;
	1430	}
	1431
	1432	size_t cacheLength;
	1433	uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength);
	1434
	1435	if (cacheStart &&
	1436	!IntervalsAreSeparate(cacheStart, cacheStart + cacheLength - 1,
	1437	range_start, range_end)) {
	1438	// Overlaps with shared cache region
	1439	return false;
	1440	}
	1441
	1442	// We believe this address is available.
	1443	return true;
	1444	}
	1445
1295	1446	// FIXME implement on this platform.
1296	1447	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
1297	1448

lib/libtsan/sanitizer_common/sanitizer_mac.h+5

...	...	@@ -58,8 +58,13 @@ struct DarwinKernelVersion : VersionBase<DarwinKernelVersion> {
58	58	DarwinKernelVersion(u16 major, u16 minor) : VersionBase(major, minor) {}
59	59	};
60	60
	61	struct ReservedRange {
	62	uptr beg, end;
	63	};
	64
61	65	MacosVersion GetMacosAlignedVersion();
62	66	DarwinKernelVersion GetDarwinKernelVersion();
	67	void GetAppReservedRanges(InternalMmapVector<ReservedRange>& ranges);
63	68
64	69	char **GetEnviron();
65	70

lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp+2-2

...	...	@@ -229,12 +229,12 @@ uptr internal_execve(const char filename, char const argv[],
229	229	return _sys_execve(filename, argv, envp);
230	230	}
231	231
232		tid_t GetTid() {
	232	ThreadID GetTid() {
233	233	DEFINE__REAL(int, _lwp_self);
234	234	return _REAL(_lwp_self);
235	235	}
236	236
237		int TgKill(pid_t pid, tid_t tid, int sig) {
	237	int TgKill(pid_t pid, ThreadID tid, int sig) {
238	238	DEFINE__REAL(int, _lwp_kill, int a, int b);
239	239	(void)pid;
240	240	return _REAL(_lwp_kill, tid, sig);

lib/libtsan/sanitizer_common/sanitizer_platform.h+27-1

...	...	@@ -319,7 +319,11 @@
319	319	#endif
320	320
321	321	// The first address that can be returned by mmap.
322		#define SANITIZER_MMAP_BEGIN 0
	322	#if SANITIZER_AIX && SANITIZER_WORDSIZE == 64
	323	# define SANITIZER_MMAP_BEGIN 0x0a00'0000'0000'0000ULL
	324	#else
	325	# define SANITIZER_MMAP_BEGIN 0
	326	#endif
323	327
324	328	// The range of addresses which can be returned my mmap.
325	329	// FIXME: this value should be different on different platforms. Larger values
...	...	@@ -482,4 +486,26 @@
482	486	# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0
483	487	#endif
484	488
	489	#if SANITIZER_LINUX
	490	# if SANITIZER_GLIBC
	491	// Workaround for
	492	// glibc/commit/3d3572f59059e2b19b8541ea648a6172136ec42e
	493	// Linux: Keep termios ioctl constants strictly internal
	494	# if __GLIBC_PREREQ(2, 41)
	495	# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 0
	496	# else
	497	# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 1
	498	# endif
	499	# else
	500	# define SANITIZER_TERMIOS_IOCTL_CONSTANTS 1
	501	# endif
	502	#endif
	503
	504	#if SANITIZER_APPLE && SANITIZER_WORDSIZE == 64
	505	// MTE uses the lower half of the top byte.
	506	# define STRIP_MTE_TAG(addr) ((addr) & ~((uptr)0x0f << 56))
	507	#else
	508	# define STRIP_MTE_TAG(addr) (addr)
	509	#endif
	510
485	511	#endif // SANITIZER_PLATFORM_H

lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h+5-4

...	...	@@ -167,7 +167,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
167	167
168	168	#define SANITIZER_INTERCEPT_STRLEN SI_NOT_FUCHSIA
169	169	#define SANITIZER_INTERCEPT_STRNLEN (SI_NOT_MAC && SI_NOT_FUCHSIA)
170		#define SANITIZER_INTERCEPT_STRCMP (SI_NOT_FUCHSIA && SI_NOT_AIX)
	170	#define SANITIZER_INTERCEPT_STRCMP SI_NOT_FUCHSIA
171	171	#define SANITIZER_INTERCEPT_STRSTR SI_NOT_FUCHSIA
172	172	#define SANITIZER_INTERCEPT_STRCASESTR (SI_POSIX && SI_NOT_AIX)
173	173	#define SANITIZER_INTERCEPT_STRTOK SI_NOT_FUCHSIA
...	...	@@ -179,8 +179,8 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
179	179	#define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
180	180	#define SANITIZER_INTERCEPT_STRCASECMP SI_POSIX
181	181	#define SANITIZER_INTERCEPT_MEMSET 1
182		#define SANITIZER_INTERCEPT_MEMMOVE SI_NOT_AIX
183		#define SANITIZER_INTERCEPT_MEMCPY SI_NOT_AIX
	182	#define SANITIZER_INTERCEPT_MEMMOVE 1
	183	#define SANITIZER_INTERCEPT_MEMCPY 1
184	184	#define SANITIZER_INTERCEPT_MEMCMP SI_NOT_FUCHSIA
185	185	#define SANITIZER_INTERCEPT_BCMP \
186	186	SANITIZER_INTERCEPT_MEMCMP && \
...	...	@@ -551,7 +551,8 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
551	551	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
552	552	#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
553	553	#define SANITIZER_INTERCEPT_WCSLEN 1
554		#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
	554	#define SANITIZER_INTERCEPT_WCSNLEN 1
	555	#define SANITIZER_INTERCEPT_WCSCAT (SI_POSIX \|\| SI_WINDOWS)
555	556	#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
556	557	#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
557	558	#define SANITIZER_INTERCEPT_BSD_SIGNAL SI_ANDROID

lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+15-17

...	...	@@ -76,6 +76,7 @@
76	76	#include <sys/vt.h>
77	77	#include <linux/cdrom.h>
78	78	#include <linux/fd.h>
	79	#include <linux/filter.h>
79	80	#if SANITIZER_ANDROID
80	81	#include <linux/fs.h>
81	82	#endif
...	...	@@ -151,16 +152,15 @@ typedef struct user_fpregs elf_fpregset_t;
151	152	#if defined(__mips64)
152	153	# include <sys/procfs.h>
153	154	#endif
154		#include <sys/user.h>
155		#include <linux/if_eql.h>
156		#include <linux/if_plip.h>
157		#include <linux/lp.h>
158		#include <linux/mroute.h>
159		#include <linux/mroute6.h>
160		#include <linux/scc.h>
161		#include <linux/serial.h>
162		#include <sys/msg.h>
163		#include <sys/ipc.h>
	155	# include <linux/if_eql.h>
	156	# include <linux/if_plip.h>
	157	# include <linux/lp.h>
	158	# include <linux/mroute.h>
	159	# include <linux/mroute6.h>
	160	# include <linux/serial.h>
	161	# include <sys/ipc.h>
	162	# include <sys/msg.h>
	163	# include <sys/user.h>
164	164	#endif // SANITIZER_ANDROID
165	165
166	166	#include <link.h>
...	...	@@ -516,6 +516,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
516	516	unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec);
517	517	unsigned struct_synth_info_sz = sizeof(struct synth_info);
518	518	unsigned struct_vt_mode_sz = sizeof(struct vt_mode);
	519	unsigned struct_sock_fprog_sz = sizeof(struct sock_fprog);
519	520	#endif // SANITIZER_LINUX
520	521
521	522	#if SANITIZER_GLIBC
...	...	@@ -532,8 +533,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
532	533	unsigned struct_kbsentry_sz = sizeof(struct kbsentry);
533	534	unsigned struct_mtconfiginfo_sz = sizeof(struct mtconfiginfo);
534	535	unsigned struct_nr_parms_struct_sz = sizeof(struct nr_parms_struct);
535		unsigned struct_scc_modem_sz = sizeof(struct scc_modem);
536		unsigned struct_scc_stat_sz = sizeof(struct scc_stat);
537	536	unsigned struct_serial_multiport_struct_sz
538	537	= sizeof(struct serial_multiport_struct);
539	538	unsigned struct_serial_struct_sz = sizeof(struct serial_struct);
...	...	@@ -543,7 +542,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
543	542
544	543	unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
545	544	unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
546		unsigned struct_sock_fprog_sz = sizeof(struct sock_fprog);
547	545	# endif // SANITIZER_GLIBC
548	546
549	547	# if !SANITIZER_ANDROID && !SANITIZER_APPLE && !SANITIZER_HAIKU
...	...	@@ -779,16 +777,16 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
779	777	unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER;
780	778	#endif // SOUND_VERSION
781	779	unsigned IOCTL_TCFLSH = TCFLSH;
782		unsigned IOCTL_TCGETA = TCGETA;
	780	# if SANITIZER_TERMIOS_IOCTL_CONSTANTS
783	781	unsigned IOCTL_TCGETS = TCGETS;
	782	# endif
784	783	unsigned IOCTL_TCSBRK = TCSBRK;
785	784	unsigned IOCTL_TCSBRKP = TCSBRKP;
786		unsigned IOCTL_TCSETA = TCSETA;
787		unsigned IOCTL_TCSETAF = TCSETAF;
788		unsigned IOCTL_TCSETAW = TCSETAW;
	785	# if SANITIZER_TERMIOS_IOCTL_CONSTANTS
789	786	unsigned IOCTL_TCSETS = TCSETS;
790	787	unsigned IOCTL_TCSETSF = TCSETSF;
791	788	unsigned IOCTL_TCSETSW = TCSETSW;
	789	# endif
792	790	unsigned IOCTL_TCXONC = TCXONC;
793	791	unsigned IOCTL_TIOCGLCKTRMIOS = TIOCGLCKTRMIOS;
794	792	unsigned IOCTL_TIOCGSOFTCAR = TIOCGSOFTCAR;

lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h+32-6

...	...	@@ -32,6 +32,8 @@
32	32	# elif SANITIZER_GLIBC \|\| SANITIZER_ANDROID
33	33	# define SANITIZER_HAS_STAT64 1
34	34	# define SANITIZER_HAS_STATFS64 1
	35	# elif SANITIZER_HAIKU
	36	# include <stdint.h>
35	37	# endif
36	38
37	39	# if defined(__sparc__)
...	...	@@ -102,6 +104,8 @@ const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
102	104	? FIRST_32_SECOND_64(104, 128)
103	105	# if defined(_ABIN32) && _MIPS_SIM == _ABIN32
104	106	: FIRST_32_SECOND_64(176, 216);
	107	# elif SANITIZER_MUSL
	108	: FIRST_32_SECOND_64(160, 208);
105	109	# else
106	110	: FIRST_32_SECOND_64(160, 216);
107	111	# endif
...	...	@@ -476,6 +480,30 @@ struct __sanitizer_cmsghdr {
476	480	int cmsg_level;
477	481	int cmsg_type;
478	482	};
	483	# elif SANITIZER_MUSL
	484	struct __sanitizer_msghdr {
	485	void *msg_name;
	486	unsigned msg_namelen;
	487	struct __sanitizer_iovec *msg_iov;
	488	int msg_iovlen;
	489	# if SANITIZER_WORDSIZE == 64
	490	int __pad1;
	491	# endif
	492	void *msg_control;
	493	unsigned msg_controllen;
	494	# if SANITIZER_WORDSIZE == 64
	495	int __pad2;
	496	# endif
	497	int msg_flags;
	498	};
	499	struct __sanitizer_cmsghdr {
	500	unsigned cmsg_len;
	501	# if SANITIZER_WORDSIZE == 64
	502	int __pad1;
	503	# endif
	504	int cmsg_level;
	505	int cmsg_type;
	506	};
479	507	# else
480	508	// In POSIX, int msg_iovlen; socklen_t msg_controllen; socklen_t cmsg_len; but
481	509	// many implementations don't conform to the standard.
...	...	@@ -603,7 +631,7 @@ typedef unsigned long __sanitizer_sigset_t;
603	631	# elif SANITIZER_APPLE
604	632	typedef unsigned __sanitizer_sigset_t;
605	633	# elif SANITIZER_HAIKU
606		typedef unsigned long __sanitizer_sigset_t;
	634	typedef uint64_t __sanitizer_sigset_t;
607	635	# elif SANITIZER_LINUX
608	636	struct __sanitizer_sigset_t {
609	637	// The size is determined by looking at sizeof of real sigset_t on linux.
...	...	@@ -1312,16 +1340,14 @@ extern unsigned IOCTL_SNDCTL_COPR_SENDMSG;
1312	1340	extern unsigned IOCTL_SNDCTL_COPR_WCODE;
1313	1341	extern unsigned IOCTL_SNDCTL_COPR_WDATA;
1314	1342	extern unsigned IOCTL_TCFLSH;
1315		extern unsigned IOCTL_TCGETA;
1316		extern unsigned IOCTL_TCGETS;
1317	1343	extern unsigned IOCTL_TCSBRK;
1318	1344	extern unsigned IOCTL_TCSBRKP;
1319		extern unsigned IOCTL_TCSETA;
1320		extern unsigned IOCTL_TCSETAF;
1321		extern unsigned IOCTL_TCSETAW;
	1345	# if SANITIZER_TERMIOS_IOCTL_CONSTANTS
	1346	extern unsigned IOCTL_TCGETS;
1322	1347	extern unsigned IOCTL_TCSETS;
1323	1348	extern unsigned IOCTL_TCSETSF;
1324	1349	extern unsigned IOCTL_TCSETSW;
	1350	# endif
1325	1351	extern unsigned IOCTL_TCXONC;
1326	1352	extern unsigned IOCTL_TIOCGLCKTRMIOS;
1327	1353	extern unsigned IOCTL_TIOCGSOFTCAR;

lib/libtsan/sanitizer_common/sanitizer_posix.cpp+3-12

...	...	@@ -225,17 +225,9 @@ void MapWritableFileToMemory(void addr, uptr size, fd_t fd, OFF_T offset) {
225	225	return (void *)p;
226	226	}
227	227
228		static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
229		uptr start2, uptr end2) {
230		CHECK(start1 <= end1);
231		CHECK(start2 <= end2);
232		return (end1 < start2) \|\| (end2 < start1);
233		}
234
	228	# if !SANITIZER_APPLE
235	229	// FIXME: this is thread-unsafe, but should not cause problems most of the time.
236		// When the shadow is mapped only a single thread usually exists (plus maybe
237		// several worker threads on Mac, which aren't expected to map big chunks of
238		// memory).
	230	// When the shadow is mapped only a single thread usually exists
239	231	bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
240	232	MemoryMappingLayout proc_maps(/cache_enabled/true);
241	233	if (proc_maps.Error())
...	...	@@ -251,7 +243,6 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
251	243	return true;
252	244	}
253	245
254		#if !SANITIZER_APPLE
255	246	void DumpProcessMap() {
256	247	MemoryMappingLayout proc_maps(/cache_enabled/true);
257	248	const sptr kBufSize = 4095;
...	...	@@ -265,7 +256,7 @@ void DumpProcessMap() {
265	256	Report("End of process memory map.\n");
266	257	UnmapOrDie(filename, kBufSize);
267	258	}
268		#endif
	259	# endif
269	260
270	261	const char *GetPwd() {
271	262	return GetEnv("PWD");

lib/libtsan/sanitizer_common/sanitizer_posix.h+2-1

...	...	@@ -67,7 +67,8 @@ uptr internal_ptrace(int request, int pid, void addr, void data);
67	67	uptr internal_waitpid(int pid, int *status, int options);
68	68
69	69	int internal_fork();
70		fd_t internal_spawn(const char argv[], const char envp[], pid_t *pid);
	70	bool internal_spawn(const char* argv[], const char* envp[], pid_t* pid,
	71	fd_t fd_stdin, fd_t fd_stdout);
71	72
72	73	int internal_sysctl(const int name, unsigned int namelen, void oldp,
73	74	uptr oldlenp, const void newp, uptr newlen);

lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp+19

...	...	@@ -47,6 +47,8 @@ typedef void (sa_sigaction_t)(int, siginfo_t , void *);
47	47
48	48	namespace __sanitizer {
49	49
	50	[[maybe_unused]] static atomic_uint8_t signal_handler_is_from_sanitizer[64];
	51
50	52	u32 GetUid() {
51	53	return getuid();
52	54	}
...	...	@@ -210,6 +212,20 @@ void UnsetAlternateSignalStack() {
210	212	UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
211	213	}
212	214
	215	bool IsSignalHandlerFromSanitizer(int signum) {
	216	return atomic_load(&signal_handler_is_from_sanitizer[signum],
	217	memory_order_relaxed);
	218	}
	219
	220	bool SetSignalHandlerFromSanitizer(int signum, bool new_state) {
	221	if (signum < 0 \|\| static_cast<unsigned>(signum) >=
	222	ARRAY_SIZE(signal_handler_is_from_sanitizer))
	223	return false;
	224
	225	return atomic_exchange(&signal_handler_is_from_sanitizer[signum], new_state,
	226	memory_order_relaxed);
	227	}
	228
213	229	static void MaybeInstallSigaction(int signum,
214	230	SignalHandlerType handler) {
215	231	if (GetHandleSignalMode(signum) == kHandleSignalNo) return;
...	...	@@ -223,6 +239,9 @@ static void MaybeInstallSigaction(int signum,
223	239	if (common_flags()->use_sigaltstack) sigact.sa_flags \|= SA_ONSTACK;
224	240	CHECK_EQ(0, internal_sigaction(signum, &sigact, nullptr));
225	241	VReport(1, "Installed the sigaction for signal %d\n", signum);
	242
	243	if (common_flags()->cloak_sanitizer_signal_handlers)
	244	SetSignalHandlerFromSanitizer(signum, true);
226	245	}
227	246
228	247	void InstallDeadlySignalHandlers(SignalHandlerType handler) {

lib/libtsan/sanitizer_common/sanitizer_procmaps_mac.cpp+102-37

...	...	@@ -20,18 +20,21 @@
20	20	#include <mach/mach.h>
21	21
22	22	// These are not available in older macOS SDKs.
23		#ifndef CPU_SUBTYPE_X86_64_H
24		#define CPU_SUBTYPE_X86_64_H ((cpu_subtype_t)8) /* Haswell */
25		#endif
26		#ifndef CPU_SUBTYPE_ARM_V7S
27		#define CPU_SUBTYPE_ARM_V7S ((cpu_subtype_t)11) /* Swift */
28		#endif
29		#ifndef CPU_SUBTYPE_ARM_V7K
30		#define CPU_SUBTYPE_ARM_V7K ((cpu_subtype_t)12)
31		#endif
32		#ifndef CPU_TYPE_ARM64
33		#define CPU_TYPE_ARM64 (CPU_TYPE_ARM \| CPU_ARCH_ABI64)
34		#endif
	23	# ifndef CPU_SUBTYPE_X86_64_H
	24	# define CPU_SUBTYPE_X86_64_H ((cpu_subtype_t)8) /* Haswell */
	25	# endif
	26	# ifndef CPU_SUBTYPE_ARM_V7S
	27	# define CPU_SUBTYPE_ARM_V7S ((cpu_subtype_t)11) /* Swift */
	28	# endif
	29	# ifndef CPU_SUBTYPE_ARM_V7K
	30	# define CPU_SUBTYPE_ARM_V7K ((cpu_subtype_t)12)
	31	# endif
	32	# ifndef CPU_TYPE_ARM64
	33	# define CPU_TYPE_ARM64 (CPU_TYPE_ARM \| CPU_ARCH_ABI64)
	34	# endif
	35	# ifndef CPU_SUBTYPE_ARM64E
	36	# define CPU_SUBTYPE_ARM64E ((cpu_subtype_t)2)
	37	# endif
35	38
36	39	namespace __sanitizer {
37	40
...	...	@@ -42,7 +45,6 @@ struct MemoryMappedSegmentData {
42	45	const char *current_load_cmd_addr;
43	46	u32 lc_type;
44	47	uptr base_virt_addr;
45		uptr addr_mask;
46	48	};
47	49
48	50	template <typename Section>
...	...	@@ -51,12 +53,62 @@ static void NextSectionLoad(LoadedModule module, MemoryMappedSegmentData data,
51	53	const Section sc = (const Section )data->current_load_cmd_addr;
52	54	data->current_load_cmd_addr += sizeof(Section);
53	55
54		uptr sec_start = (sc->addr & data->addr_mask) + data->base_virt_addr;
	56	uptr sec_start = sc->addr + data->base_virt_addr;
55	57	uptr sec_end = sec_start + sc->size;
56	58	module->addAddressRange(sec_start, sec_end, /executable=/false, isWritable,
57	59	sc->sectname);
58	60	}
59	61
	62	static bool VerifyMemoryMapping(MemoryMappingLayout* mapping) {
	63	InternalMmapVector<LoadedModule> modules;
	64	modules.reserve(128); // matches DumpProcessMap
	65	mapping->DumpListOfModules(&modules);
	66
	67	InternalMmapVector<LoadedModule::AddressRange> segments;
	68	for (uptr i = 0; i < modules.size(); ++i) {
	69	for (auto& range : modules[i].ranges()) {
	70	if (range.beg == range.end)
	71	continue;
	72	segments.push_back(range);
	73	}
	74	}
	75
	76	// Verify that none of the segments overlap:
	77	// 1. Sort the segments by the start address
	78	// 2. Check that every segment starts after the previous one ends.
	79	Sort(segments.data(), segments.size(),
	80	[](LoadedModule::AddressRange& a, LoadedModule::AddressRange& b) {
	81	return a.beg < b.beg;
	82	});
	83
	84	// To avoid spam, we only print the report message once-per-process.
	85	static bool invalid_module_map_reported = false;
	86	bool well_formed = true;
	87
	88	for (size_t i = 1; i < segments.size(); i++) {
	89	uptr cur_start = segments[i].beg;
	90	uptr prev_end = segments[i - 1].end;
	91	if (cur_start < prev_end) {
	92	well_formed = false;
	93	VReport(2, "Overlapping mappings: %s start = %p, %s end = %p\n",
	94	segments[i].name, (void*)cur_start, segments[i - 1].name,
	95	(void*)prev_end);
	96	if (!invalid_module_map_reported) {
	97	Report(
	98	"WARN: Invalid dyld module map detected. This is most likely a bug "
	99	"in the sanitizer.\n");
	100	Report("WARN: Backtraces may be unreliable.\n");
	101	invalid_module_map_reported = true;
	102	}
	103	}
	104	}
	105
	106	for (auto& m : modules) m.clear();
	107
	108	mapping->Reset();
	109	return well_formed;
	110	}
	111
60	112	void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
61	113	// Don't iterate over sections when the caller hasn't set up the
62	114	// data pointer, when there are no sections, or when the segment
...	...	@@ -82,6 +134,7 @@ void MemoryMappedSegment::AddAddressRanges(LoadedModule *module) {
82	134
83	135	MemoryMappingLayout::MemoryMappingLayout(bool cache_enabled) {
84	136	Reset();
	137	VerifyMemoryMapping(this);
85	138	}
86	139
87	140	MemoryMappingLayout::~MemoryMappingLayout() {
...	...	@@ -123,7 +176,7 @@ void MemoryMappingLayout::Reset() {
123	176	// The dyld load address should be unchanged throughout process execution,
124	177	// and it is expensive to compute once many libraries have been loaded,
125	178	// so cache it here and do not reset.
126		static mach_header *dyld_hdr = 0;
	179	static const mach_header* dyld_hdr = 0;
127	180	static const char kDyldPath[] = "/usr/lib/dyld";
128	181	static const int kDyldImageIdx = -1;
129	182
...	...	@@ -187,17 +240,22 @@ typedef struct dyld_shared_cache_dylib_text_info
187	240
188	241	extern bool _dyld_get_shared_cache_uuid(uuid_t uuid);
189	242	extern const void _dyld_get_shared_cache_range(size_t length);
	243	extern intptr_t _dyld_get_image_slide(const struct mach_header* mh);
190	244	extern int dyld_shared_cache_iterate_text(
191	245	const uuid_t cacheUuid,
192	246	void (^callback)(const dyld_shared_cache_dylib_text_info *info));
	247	SANITIZER_WEAK_IMPORT const struct mach_header* _dyld_get_dyld_header(void);
193	248	} // extern "C"
194	249
195		static mach_header *GetDyldImageHeaderViaSharedCache() {
	250	static const mach_header* GetDyldImageHeaderViaSharedCache() {
196	251	uuid_t uuid;
197	252	bool hasCache = _dyld_get_shared_cache_uuid(uuid);
198	253	if (!hasCache)
199	254	return nullptr;
200	255
	256	if (&_dyld_get_dyld_header != nullptr)
	257	return _dyld_get_dyld_header();
	258
201	259	size_t cacheLength;
202	260	__block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength);
203	261	CHECK(cacheStart && cacheLength);
...	...	@@ -255,23 +313,21 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
255	313	layout_data->current_load_cmd_count--;
256	314	if (((const load_command *)lc)->cmd == kLCSegment) {
257	315	const SegmentCommand* sc = (const SegmentCommand *)lc;
258		uptr base_virt_addr, addr_mask;
259		if (layout_data->current_image == kDyldImageIdx) {
260		base_virt_addr = (uptr)get_dyld_hdr();
261		// vmaddr is masked with 0xfffff because on macOS versions < 10.12,
262		// it contains an absolute address rather than an offset for dyld.
263		// To make matters even more complicated, this absolute address
264		// isn't actually the absolute segment address, but the offset portion
265		// of the address is accurate when combined with the dyld base address,
266		// and the mask will give just this offset.
267		addr_mask = 0xfffff;
268		} else {
	316	if (internal_strcmp(sc->segname, "__LINKEDIT") == 0) {
	317	// The LINKEDIT sections are for internal linker use, and may alias
	318	// with the LINKEDIT section for other modules. (If we included them,
	319	// our memory map would contain overlappping sections.)
	320	return false;
	321	}
	322
	323	uptr base_virt_addr;
	324	if (layout_data->current_image == kDyldImageIdx)
	325	base_virt_addr = (uptr)_dyld_get_image_slide(get_dyld_hdr());
	326	else
269	327	base_virt_addr =
270	328	(uptr)_dyld_get_image_vmaddr_slide(layout_data->current_image);
271		addr_mask = ~0;
272		}
273	329
274		segment->start = (sc->vmaddr & addr_mask) + base_virt_addr;
	330	segment->start = sc->vmaddr + base_virt_addr;
275	331	segment->end = segment->start + sc->vmsize;
276	332	// Most callers don't need section information, so only fill this struct
277	333	// when required.
...	...	@@ -281,9 +337,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
281	337	(const char *)lc + sizeof(SegmentCommand);
282	338	seg_data->lc_type = kLCSegment;
283	339	seg_data->base_virt_addr = base_virt_addr;
284		seg_data->addr_mask = addr_mask;
285	340	internal_strncpy(seg_data->name, sc->segname,
286	341	ARRAY_SIZE(seg_data->name));
	342	seg_data->name[ARRAY_SIZE(seg_data->name) - 1] = 0;
287	343	}
288	344
289	345	// Return the initial protection.
...	...	@@ -297,6 +353,7 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
297	353	? kDyldPath
298	354	: _dyld_get_image_name(layout_data->current_image);
299	355	internal_strncpy(segment->filename, src, segment->filename_size);
	356	segment->filename[segment->filename_size - 1] = 0;
300	357	}
301	358	segment->arch = layout_data->current_arch;
302	359	internal_memcpy(segment->uuid, layout_data->current_uuid, kModuleUUIDSize);
...	...	@@ -311,18 +368,26 @@ ModuleArch ModuleArchFromCpuType(cpu_type_t cputype, cpu_subtype_t cpusubtype) {
311	368	case CPU_TYPE_I386:
312	369	return kModuleArchI386;
313	370	case CPU_TYPE_X86_64:
314		if (cpusubtype == CPU_SUBTYPE_X86_64_ALL) return kModuleArchX86_64;
315		if (cpusubtype == CPU_SUBTYPE_X86_64_H) return kModuleArchX86_64H;
	371	if (cpusubtype == CPU_SUBTYPE_X86_64_ALL)
	372	return kModuleArchX86_64;
	373	if (cpusubtype == CPU_SUBTYPE_X86_64_H)
	374	return kModuleArchX86_64H;
316	375	CHECK(0 && "Invalid subtype of x86_64");
317	376	return kModuleArchUnknown;
318	377	case CPU_TYPE_ARM:
319		if (cpusubtype == CPU_SUBTYPE_ARM_V6) return kModuleArchARMV6;
320		if (cpusubtype == CPU_SUBTYPE_ARM_V7) return kModuleArchARMV7;
321		if (cpusubtype == CPU_SUBTYPE_ARM_V7S) return kModuleArchARMV7S;
322		if (cpusubtype == CPU_SUBTYPE_ARM_V7K) return kModuleArchARMV7K;
	378	if (cpusubtype == CPU_SUBTYPE_ARM_V6)
	379	return kModuleArchARMV6;
	380	if (cpusubtype == CPU_SUBTYPE_ARM_V7)
	381	return kModuleArchARMV7;
	382	if (cpusubtype == CPU_SUBTYPE_ARM_V7S)
	383	return kModuleArchARMV7S;
	384	if (cpusubtype == CPU_SUBTYPE_ARM_V7K)
	385	return kModuleArchARMV7K;
323	386	CHECK(0 && "Invalid subtype of ARM");
324	387	return kModuleArchUnknown;
325	388	case CPU_TYPE_ARM64:
	389	if (cpusubtype == CPU_SUBTYPE_ARM64E)
	390	return kModuleArchARM64E;
326	391	return kModuleArchARM64;
327	392	default:
328	393	CHECK(0 && "Invalid CPU type");

lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h+1-1

...	...	@@ -15,7 +15,7 @@
15	15	# define SANITIZER_REDEFINE_BUILTINS_H
16	16
17	17	// The asm hack only works with GCC and Clang.
18		# if !defined(_WIN32) && !defined(_AIX)
	18	# if !defined(_WIN32) && !defined(_AIX) && !defined(__APPLE__)
19	19
20	20	asm(R"(
21	21	.set memcpy, __sanitizer_internal_memcpy

lib/libtsan/sanitizer_common/sanitizer_signal_interceptors.inc+40-2

...	...	@@ -45,6 +45,8 @@ using namespace __sanitizer;
45	45	INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
46	46	SIGNAL_INTERCEPTOR_ENTER();
47	47	if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
	48
	49	// TODO: support cloak_sanitizer_signal_handlers
48	50	SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
49	51	}
50	52	#define INIT_BSD_SIGNAL COMMON_INTERCEPT_FUNCTION(bsd_signal)
...	...	@@ -56,19 +58,55 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
56	58	INTERCEPTOR(uptr, signal, int signum, uptr handler) {
57	59	SIGNAL_INTERCEPTOR_ENTER();
58	60	if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
	61	// The user can neither view nor change the signal handler, regardless of
	62	// the cloak_sanitizer_signal_handlers setting. This differs from
	63	// sigaction().
59	64	return (uptr) nullptr;
60		SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
	65
	66	uptr ret = +[](auto signal, int signum, uptr handler) {
	67	SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
	68	}(signal, signum, handler);
	69
	70	if (ret != sig_err && SetSignalHandlerFromSanitizer(signum, false))
	71	// If the user sets a signal handler, it becomes uncloaked, even if they
	72	// reuse a sanitizer's signal handler.
	73	ret = sig_dfl;
	74
	75	return ret;
61	76	}
62	77	#define INIT_SIGNAL COMMON_INTERCEPT_FUNCTION(signal)
63	78
64	79	INTERCEPTOR(int, sigaction_symname, int signum,
65	80	const __sanitizer_sigaction act, __sanitizer_sigaction oldact) {
66	81	SIGNAL_INTERCEPTOR_ENTER();
	82
67	83	if (GetHandleSignalMode(signum) == kHandleSignalExclusive) {
68	84	if (!oldact) return 0;
69	85	act = nullptr;
	86	// If cloak_sanitizer_signal_handlers=true, the user can neither view nor
	87	// change the signal handle.
	88	// If false, the user can view but not change the signal handler. This
	89	// differs from signal().
70	90	}
71		SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact);
	91
	92	int ret = +[](int signum, const __sanitizer_sigaction* act,
	93	__sanitizer_sigaction* oldact) {
	94	SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact);
	95	}(signum, act, oldact);
	96
	97	if (act) {
	98	if (ret == 0 && SetSignalHandlerFromSanitizer(signum, false)) {
	99	// If the user sets a signal handler, it becomes uncloaked, even if they
	100	// reuse a sanitizer's signal handler.
	101
	102	if (oldact)
	103	oldact->handler = reinterpret_cast<__sanitizer_sighandler_ptr>(sig_dfl);
	104	}
	105	} else if (ret == 0 && oldact && IsSignalHandlerFromSanitizer(signum)) {
	106	oldact->handler = reinterpret_cast<__sanitizer_sighandler_ptr>(sig_dfl);
	107	}
	108
	109	return ret;
72	110	}
73	111	#define INIT_SIGACTION COMMON_INTERCEPT_FUNCTION(sigaction_symname)
74	112

lib/libtsan/sanitizer_common/sanitizer_stoptheworld.h+1-1

...	...	@@ -38,7 +38,7 @@ class SuspendedThreadsList {
38	38	}
39	39
40	40	virtual uptr ThreadCount() const { UNIMPLEMENTED(); }
41		virtual tid_t GetThreadID(uptr index) const { UNIMPLEMENTED(); }
	41	virtual ThreadID GetThreadID(uptr index) const { UNIMPLEMENTED(); }
42	42
43	43	protected:
44	44	~SuspendedThreadsList() {}

lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+83-12

...	...	@@ -94,17 +94,17 @@ class SuspendedThreadsListLinux final : public SuspendedThreadsList {
94	94	public:
95	95	SuspendedThreadsListLinux() { thread_ids_.reserve(1024); }
96	96
97		tid_t GetThreadID(uptr index) const override;
	97	ThreadID GetThreadID(uptr index) const override;
98	98	uptr ThreadCount() const override;
99		bool ContainsTid(tid_t thread_id) const;
100		void Append(tid_t tid);
	99	bool ContainsTid(ThreadID thread_id) const;
	100	void Append(ThreadID tid);
101	101
102	102	PtraceRegistersStatus GetRegistersAndSP(uptr index,
103	103	InternalMmapVector<uptr> *buffer,
104	104	uptr *sp) const override;
105	105
106	106	private:
107		InternalMmapVector<tid_t> thread_ids_;
	107	InternalMmapVector<ThreadID> thread_ids_;
108	108	};
109	109
110	110	// Structure for passing arguments into the tracer thread.
...	...	@@ -137,10 +137,10 @@ class ThreadSuspender {
137	137	private:
138	138	SuspendedThreadsListLinux suspended_threads_list_;
139	139	pid_t pid_;
140		bool SuspendThread(tid_t thread_id);
	140	bool SuspendThread(ThreadID thread_id);
141	141	};
142	142
143		bool ThreadSuspender::SuspendThread(tid_t tid) {
	143	bool ThreadSuspender::SuspendThread(ThreadID tid) {
144	144	int pterrno;
145	145	if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
146	146	&pterrno)) {
...	...	@@ -210,7 +210,7 @@ void ThreadSuspender::KillAllThreads() {
210	210	bool ThreadSuspender::SuspendAllThreads() {
211	211	ThreadLister thread_lister(pid_);
212	212	bool retry = true;
213		InternalMmapVector<tid_t> threads;
	213	InternalMmapVector<ThreadID> threads;
214	214	threads.reserve(128);
215	215	for (int i = 0; i < 30 && retry; ++i) {
216	216	retry = false;
...	...	@@ -226,7 +226,7 @@ bool ThreadSuspender::SuspendAllThreads() {
226	226	case ThreadLister::Ok:
227	227	break;
228	228	}
229		for (tid_t tid : threads) {
	229	for (ThreadID tid : threads) {
230	230	// Are we already attached to this thread?
231	231	// Currently this check takes linear time, however the number of threads
232	232	// is usually small.
...	...	@@ -403,7 +403,77 @@ struct ScopedSetTracerPID {
403	403	}
404	404	};
405	405
	406	// This detects whether ptrace is blocked (e.g., by seccomp), by forking and
	407	// then attempting ptrace.
	408	// This separate check is necessary because StopTheWorld() creates a thread
	409	// with a shared virtual address space and shared TLS, and therefore
	410	// cannot use waitpid() due to the shared errno.
	411	static void TestPTrace() {
	412	# if SANITIZER_SPARC
	413	// internal_fork() on SPARC actually calls __fork(). We can't safely fork,
	414	// because it's possible seccomp has been configured to disallow fork() but
	415	// allow clone().
	416	VReport(1, "WARNING: skipping TestPTrace() because this is SPARC\n");
	417	VReport(1,
	418	"If seccomp blocks ptrace, LeakSanitizer may hang without further "
	419	"notice\n");
	420	VReport(
	421	1,
	422	"If seccomp does not block ptrace, you can safely ignore this warning\n");
	423	# else
	424	// Heuristic: only check the first time this is called. This is not always
	425	// correct (e.g., user manually triggers leak detection, then updates
	426	// seccomp, then leak detection is triggered again).
	427	static bool checked = false;
	428	if (checked)
	429	return;
	430	checked = true;
	431
	432	// Hopefully internal_fork() is not too expensive, thanks to copy-on-write.
	433	// Besides, this is only called the first time.
	434	// Note that internal_fork() on non-SPARC Linux actually calls
	435	// SYSCALL(clone); thus, it is reasonable to use it because if seccomp kills
	436	// TestPTrace(), it would have killed StopTheWorld() anyway.
	437	int pid = internal_fork();
	438
	439	if (pid < 0) {
	440	int rverrno;
	441	if (internal_iserror(pid, &rverrno))
	442	VReport(0, "WARNING: TestPTrace() failed to fork (errno %d)\n", rverrno);
	443
	444	// We don't abort the sanitizer - it's still worth letting the sanitizer
	445	// try.
	446	return;
	447	}
	448
	449	if (pid == 0) {
	450	// Child subprocess
	451
	452	// TODO: consider checking return value of internal_ptrace, to handle
	453	// SCMP_ACT_ERRNO. However, be careful not to consume too many
	454	// resources performing a proper ptrace.
	455	internal_ptrace(PTRACE_ATTACH, 0, nullptr, nullptr);
	456	internal__exit(0);
	457	} else {
	458	int wstatus;
	459	internal_waitpid(pid, &wstatus, 0);
	460
	461	// Handle SCMP_ACT_KILL
	462	if (WIFSIGNALED(wstatus)) {
	463	VReport(0,
	464	"WARNING: ptrace appears to be blocked (is seccomp enabled?). "
	465	"LeakSanitizer may hang.\n");
	466	VReport(0, "Child exited with signal %d.\n", WTERMSIG(wstatus));
	467	// We don't abort the sanitizer - it's still worth letting the sanitizer
	468	// try.
	469	}
	470	}
	471	# endif
	472	}
	473
406	474	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
	475	TestPTrace();
	476
407	477	StopTheWorldScope in_stoptheworld;
408	478	// Prepare the arguments for TracerThread.
409	479	struct TracerThreadArgument tracer_thread_argument;
...	...	@@ -457,7 +527,8 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) {
457	527	internal_prctl(PR_SET_PTRACER, tracer_pid, 0, 0, 0);
458	528	// Allow the tracer thread to start.
459	529	tracer_thread_argument.mutex.Unlock();
460		// NOTE: errno is shared between this thread and the tracer thread.
	530	// NOTE: errno is shared between this thread and the tracer thread
	531	// (clone was called without CLONE_SETTLS / newtls).
461	532	// internal_waitpid() may call syscall() which can access/spoil errno,
462	533	// so we can't call it now. Instead we for the tracer thread to finish using
463	534	// the spin loop below. Man page for sched_yield() says "In the Linux
...	...	@@ -546,7 +617,7 @@ static constexpr uptr kExtraRegs[] = {0};
546	617	#error "Unsupported architecture"
547	618	#endif // SANITIZER_ANDROID && defined(__arm__)
548	619
549		tid_t SuspendedThreadsListLinux::GetThreadID(uptr index) const {
	620	ThreadID SuspendedThreadsListLinux::GetThreadID(uptr index) const {
550	621	CHECK_LT(index, thread_ids_.size());
551	622	return thread_ids_[index];
552	623	}
...	...	@@ -555,14 +626,14 @@ uptr SuspendedThreadsListLinux::ThreadCount() const {
555	626	return thread_ids_.size();
556	627	}
557	628
558		bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id) const {
	629	bool SuspendedThreadsListLinux::ContainsTid(ThreadID thread_id) const {
559	630	for (uptr i = 0; i < thread_ids_.size(); i++) {
560	631	if (thread_ids_[i] == thread_id) return true;
561	632	}
562	633	return false;
563	634	}
564	635
565		void SuspendedThreadsListLinux::Append(tid_t tid) {
	636	void SuspendedThreadsListLinux::Append(ThreadID tid) {
566	637	thread_ids_.push_back(tid);
567	638	}
568	639

lib/libtsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp+3-3

...	...	@@ -23,7 +23,7 @@
23	23
24	24	namespace __sanitizer {
25	25	typedef struct {
26		tid_t tid;
	26	ThreadID tid;
27	27	thread_t thread;
28	28	} SuspendedThreadInfo;
29	29
...	...	@@ -31,7 +31,7 @@ class SuspendedThreadsListMac final : public SuspendedThreadsList {
31	31	public:
32	32	SuspendedThreadsListMac() = default;
33	33
34		tid_t GetThreadID(uptr index) const override;
	34	ThreadID GetThreadID(uptr index) const override;
35	35	thread_t GetThread(uptr index) const;
36	36	uptr ThreadCount() const override;
37	37	bool ContainsThread(thread_t thread) const;
...	...	@@ -111,7 +111,7 @@ typedef x86_thread_state32_t regs_struct;
111	111	#error "Unsupported architecture"
112	112	#endif
113	113
114		tid_t SuspendedThreadsListMac::GetThreadID(uptr index) const {
	114	ThreadID SuspendedThreadsListMac::GetThreadID(uptr index) const {
115	115	CHECK_LT(index, threads_.size());
116	116	return threads_[index].tid;
117	117	}

lib/libtsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp+7-7

...	...	@@ -52,17 +52,17 @@ class SuspendedThreadsListNetBSD final : public SuspendedThreadsList {
52	52	public:
53	53	SuspendedThreadsListNetBSD() { thread_ids_.reserve(1024); }
54	54
55		tid_t GetThreadID(uptr index) const;
	55	ThreadID GetThreadID(uptr index) const;
56	56	uptr ThreadCount() const;
57		bool ContainsTid(tid_t thread_id) const;
58		void Append(tid_t tid);
	57	bool ContainsTid(ThreadID thread_id) const;
	58	void Append(ThreadID tid);
59	59
60	60	PtraceRegistersStatus GetRegistersAndSP(uptr index,
61	61	InternalMmapVector<uptr> *buffer,
62	62	uptr *sp) const;
63	63
64	64	private:
65		InternalMmapVector<tid_t> thread_ids_;
	65	InternalMmapVector<ThreadID> thread_ids_;
66	66	};
67	67
68	68	struct TracerThreadArgument {
...	...	@@ -313,7 +313,7 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) {
313	313	}
314	314	}
315	315
316		tid_t SuspendedThreadsListNetBSD::GetThreadID(uptr index) const {
	316	ThreadID SuspendedThreadsListNetBSD::GetThreadID(uptr index) const {
317	317	CHECK_LT(index, thread_ids_.size());
318	318	return thread_ids_[index];
319	319	}
...	...	@@ -322,7 +322,7 @@ uptr SuspendedThreadsListNetBSD::ThreadCount() const {
322	322	return thread_ids_.size();
323	323	}
324	324
325		bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const {
	325	bool SuspendedThreadsListNetBSD::ContainsTid(ThreadID thread_id) const {
326	326	for (uptr i = 0; i < thread_ids_.size(); i++) {
327	327	if (thread_ids_[i] == thread_id)
328	328	return true;
...	...	@@ -330,7 +330,7 @@ bool SuspendedThreadsListNetBSD::ContainsTid(tid_t thread_id) const {
330	330	return false;
331	331	}
332	332
333		void SuspendedThreadsListNetBSD::Append(tid_t tid) {
	333	void SuspendedThreadsListNetBSD::Append(ThreadID tid) {
334	334	thread_ids_.push_back(tid);
335	335	}
336	336

lib/libtsan/sanitizer_common/sanitizer_stoptheworld_win.cpp+2-2

...	...	@@ -38,7 +38,7 @@ struct SuspendedThreadsListWindows final : public SuspendedThreadsList {
38	38	InternalMmapVector<uptr> *buffer,
39	39	uptr *sp) const override;
40	40
41		tid_t GetThreadID(uptr index) const override;
	41	ThreadID GetThreadID(uptr index) const override;
42	42	uptr ThreadCount() const override;
43	43	};
44	44
...	...	@@ -68,7 +68,7 @@ PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP(
68	68	return REGISTERS_AVAILABLE;
69	69	}
70	70
71		tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const {
	71	ThreadID SuspendedThreadsListWindows::GetThreadID(uptr index) const {
72	72	CHECK_LT(index, threadIds.size());
73	73	return threadIds[index];
74	74	}

lib/libtsan/sanitizer_common/sanitizer_symbolizer_internal.h+5-1

...	...	@@ -83,7 +83,7 @@ class SymbolizerProcess {
83	83	const char SendCommand(const char command);
84	84
85	85	protected:
86		~SymbolizerProcess() {}
	86	~SymbolizerProcess();
87	87
88	88	/// The maximum number of arguments required to invoke a tool process.
89	89	static const unsigned kArgVMax = 16;
...	...	@@ -114,6 +114,10 @@ class SymbolizerProcess {
114	114	fd_t input_fd_;
115	115	fd_t output_fd_;
116	116
	117	// We hold on to the child's stdin fd (the read end of the pipe)
	118	// so that when we write to it, we don't get a SIGPIPE
	119	fd_t child_stdin_fd_;
	120
117	121	InternalMmapVector<char> buffer_;
118	122
119	123	static const uptr kMaxTimesRestarted = 5;

lib/libtsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp+11-1

...	...	@@ -476,10 +476,11 @@ const char LLVMSymbolizer::FormatAndSendCommand(const char command_prefix,
476	476	return symbolizer_process_->SendCommand(buffer_);
477	477	}
478	478
479		SymbolizerProcess::SymbolizerProcess(const char *path, bool use_posix_spawn)
	479	SymbolizerProcess::SymbolizerProcess(const char* path, bool use_posix_spawn)
480	480	: path_(path),
481	481	input_fd_(kInvalidFd),
482	482	output_fd_(kInvalidFd),
	483	child_stdin_fd_(kInvalidFd),
483	484	times_restarted_(0),
484	485	failed_to_start_(false),
485	486	reported_invalid_path_(false),
...	...	@@ -488,6 +489,11 @@ SymbolizerProcess::SymbolizerProcess(const char *path, bool use_posix_spawn)
488	489	CHECK_NE(path_[0], '\0');
489	490	}
490	491
	492	SymbolizerProcess::~SymbolizerProcess() {
	493	if (child_stdin_fd_ != kInvalidFd)
	494	CloseFile(child_stdin_fd_);
	495	}
	496
491	497	static bool IsSameModule(const char *path) {
492	498	if (const char *ProcessName = GetProcessName()) {
493	499	if (const char *SymbolizerName = StripModuleName(path)) {
...	...	@@ -533,6 +539,10 @@ bool SymbolizerProcess::Restart() {
533	539	CloseFile(input_fd_);
534	540	if (output_fd_ != kInvalidFd)
535	541	CloseFile(output_fd_);
	542	if (child_stdin_fd_ != kInvalidFd) {
	543	CloseFile(child_stdin_fd_);
	544	child_stdin_fd_ = kInvalidFd; // Don't free in destructor
	545	}
536	546	return StartSymbolizerSubprocess();
537	547	}
538	548

lib/libtsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+86-29

...	...	@@ -78,13 +78,25 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
78	78	}
79	79
80	80	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
81		return (length >= 1 && buffer[length - 1] == '\n');
	81	if (common_flags()->symbolize_inline_frames) {
	82	// When running with -i, atos sends two newlines at the end of each
	83	// address it symbolizes. This indicates the end of the set of frames
	84	// for a particular address.
	85	return length >= 2 && buffer[length - 1] == '\n' &&
	86	buffer[length - 2] == '\n';
	87	} else {
	88	// When running without -i, atos only sends a single newline at
	89	// the end of each address it symbolizes.
	90	return length >= 1 && buffer[length - 1] == '\n';
	91	}
82	92	}
83	93
84	94	void GetArgV(const char *path_to_binary,
85	95	const char *(&argv)[kArgVMax]) const override {
86	96	int i = 0;
87	97	argv[i++] = path_to_binary;
	98	if (common_flags()->symbolize_inline_frames)
	99	argv[i++] = "-i";
88	100	argv[i++] = "-p";
89	101	argv[i++] = &pid_str_[0];
90	102	if (GetMacosAlignedVersion() == MacosVersion(10, 9)) {
...	...	@@ -102,12 +114,16 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
102	114
103	115	#undef K_ATOS_ENV_VAR
104	116
105		static bool ParseCommandOutput(const char str, uptr addr, char *out_name,
106		char out_module, char out_file, uptr *line,
107		uptr *start_address) {
	117	// Parses a single frame (one line) from str, and returns the pointer to the
	118	// next character to parse (i.e. after the newline) if successful. If
	119	// it fails, returns NULL.
	120	static const char* ParseCommandOutput(const char* str, uptr addr,
	121	char out_name, char out_module,
	122	char** out_file, uptr* line,
	123	uptr* start_address) {
108	124	// Trim ending newlines.
109	125	char *trim;
110		ExtractTokenUpToDelimiter(str, "\n", &trim);
	126	str = ExtractTokenUpToDelimiter(str, "\n", &trim);
111	127
112	128	// The line from `atos` is in one of these formats:
113	129	// myfunction (in library.dylib) (sourcefile.c:17)
...	...	@@ -124,7 +140,7 @@ static bool ParseCommandOutput(const char str, uptr addr, char *out_name,
124	140	if (rest[0] == '\0') {
125	141	InternalFree(symbol_name);
126	142	InternalFree(trim);
127		return false;
	143	return NULL;
128	144	}
129	145
130	146	if (internal_strncmp(symbol_name, "0x", 2) != 0)
...	...	@@ -149,7 +165,7 @@ static bool ParseCommandOutput(const char str, uptr addr, char *out_name,
149	165	}
150	166
151	167	InternalFree(trim);
152		return true;
	168	return str;
153	169	}
154	170
155	171	AtosSymbolizer::AtosSymbolizer(const char path, LowLevelAllocator allocator)
...	...	@@ -161,31 +177,72 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
161	177	char command[32];
162	178	internal_snprintf(command, sizeof(command), "0x%zx\n", addr);
163	179	const char *buf = process_->SendCommand(command);
164		if (!buf) return false;
165		uptr line;
166		uptr start_address = AddressInfo::kUnknown;
167		if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
168		&stack->info.file, &line, &start_address)) {
169		Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr);
	180	if (!buf)
170	181	return false;
171		}
172		stack->info.line = (int)line;
173
174		if (start_address == AddressInfo::kUnknown) {
175		// Fallback to dladdr() to get function start address if atos doesn't report
176		// it.
177		Dl_info info;
178		int result = dladdr((const void *)addr, &info);
179		if (result)
180		start_address = reinterpret_cast<uptr>(info.dli_saddr);
181		}
182	182
183		// Only assign to `function_offset` if we were able to get the function's
184		// start address and we got a sensible `start_address` (dladdr doesn't always
185		// ensure that `addr >= sym_addr`).
186		if (start_address != AddressInfo::kUnknown && addr >= start_address) {
187		stack->info.function_offset = addr - start_address;
	183	SymbolizedStack* last = stack;
	184	bool top_frame = true;
	185
	186	// Parse one line of input (i.e. one frame).
	187	//
	188	// When symbolize_inline_frames=true, an empty line
	189	// (i.e. \n at the beginning of a line) indicates that the last
	190	// frame has been sent.
	191	//
	192	// When symbolize_inline_frames=false, the symbolizer will send only
	193	// one frame (without a empty line), so loop runs exactly once
	194	// and hits an early `break`.
	195	while (*buf != '\n') {
	196	uptr line;
	197	uptr start_address = AddressInfo::kUnknown;
	198
	199	SymbolizedStack* cur;
	200	if (top_frame) {
	201	cur = stack;
	202	} else {
	203	cur = SymbolizedStack::New(stack->info.address);
	204	cur->info.FillModuleInfo(stack->info.module, stack->info.module_offset,
	205	stack->info.module_arch);
	206	last->next = cur;
	207	last = cur;
	208	}
	209
	210	// Parse one line of input (i.e. one frame)
	211	// If this succeeds, buf will be updated to point to the first character
	212	// after the newline.
	213	buf = ParseCommandOutput(buf, addr, &cur->info.function, &cur->info.module,
	214	&cur->info.file, &line, &start_address);
	215
	216	// Upon failure, ParseCommandOutput returns NULL.
	217	if (!buf) {
	218	Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr);
	219	return false;
	220	}
	221	cur->info.line = (int)line;
	222
	223	if (top_frame && start_address == AddressInfo::kUnknown) {
	224	// Fallback to dladdr() to get function start address if atos doesn't
	225	// report it.
	226	Dl_info info;
	227	int result = dladdr((const void*)addr, &info);
	228	if (result)
	229	start_address = reinterpret_cast<uptr>(info.dli_saddr);
	230	}
	231
	232	// Only assign to `function_offset` if we were able to get the function's
	233	// start address and we got a sensible `start_address` (dladdr doesn't
	234	// always ensure that `addr >= sym_addr`).
	235	if (start_address != AddressInfo::kUnknown && addr >= start_address) {
	236	cur->info.function_offset = addr - start_address;
	237	}
	238
	239	// atos only sends one line when inline frames are off
	240	if (!common_flags()->symbolize_inline_frames)
	241	break;
	242
	243	top_frame = false;
188	244	}
	245
189	246	return true;
190	247	}
191	248

lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+27-16

...	...	@@ -156,30 +156,34 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
156	156	Printf("\n");
157	157	}
158	158
	159	fd_t infd[2] = {}, outfd[2] = {};
	160	if (!CreateTwoHighNumberedPipes(infd, outfd)) {
	161	Report(
	162	"WARNING: Can't create a socket pair to start "
	163	"external symbolizer (errno: %d)\n",
	164	errno);
	165	return false;
	166	}
	167
159	168	if (use_posix_spawn_) {
160	169	# if SANITIZER_APPLE
161		fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
162		if (fd == kInvalidFd) {
	170	bool success = internal_spawn(argv, const_cast<const char**>(GetEnvP()),
	171	&pid, outfd[0], infd[1]);
	172	if (!success) {
163	173	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
164	174	errno);
	175	internal_close(infd[0]);
	176	internal_close(outfd[1]);
165	177	return false;
166	178	}
167	179
168		input_fd_ = fd;
169		output_fd_ = fd;
	180	// We intentionally hold on to the read-end so that we don't get a SIGPIPE
	181	child_stdin_fd_ = outfd[0];
	182
170	183	# else // SANITIZER_APPLE
171	184	UNIMPLEMENTED();
172	185	# endif // SANITIZER_APPLE
173	186	} else {
174		fd_t infd[2] = {}, outfd[2] = {};
175		if (!CreateTwoHighNumberedPipes(infd, outfd)) {
176		Report(
177		"WARNING: Can't create a socket pair to start "
178		"external symbolizer (errno: %d)\n",
179		errno);
180		return false;
181		}
182
183	187	pid = StartSubprocess(path_, argv, GetEnvP(), /* stdin */ outfd[0],
184	188	/* stdout */ infd[1]);
185	189	if (pid < 0) {
...	...	@@ -187,11 +191,11 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
187	191	internal_close(outfd[1]);
188	192	return false;
189	193	}
190
191		input_fd_ = infd[0];
192		output_fd_ = outfd[1];
193	194	}
194	195
	196	input_fd_ = infd[0];
	197	output_fd_ = outfd[1];
	198
195	199	CHECK_GT(pid, 0);
196	200
197	201	// Check that symbolizer subprocess started successfully.
...	...	@@ -505,6 +509,13 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
505	509	}
506	510
507	511	# if SANITIZER_APPLE
	512	if (list->empty()) {
	513	Report(
	514	"WARN: No external symbolizers found. Symbols may be missing or "
	515	"unreliable.\n");
	516	Report(
	517	"HINT: Is PATH set? Does sandbox allow file-read of /usr/bin/atos?\n");
	518	}
508	519	VReport(2, "Using dladdr symbolizer.\n");
509	520	list->push_back(new (*allocator) DlAddrSymbolizer());
510	521	# endif // SANITIZER_APPLE

lib/libtsan/sanitizer_common/sanitizer_thread_registry.cpp+5-4

...	...	@@ -80,7 +80,7 @@ void ThreadContextBase::SetFinished() {
80	80	OnFinished();
81	81	}
82	82
83		void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type,
	83	void ThreadContextBase::SetStarted(ThreadID _os_id, ThreadType _thread_type,
84	84	void *arg) {
85	85	status = ThreadStatusRunning;
86	86	os_id = _os_id;
...	...	@@ -228,7 +228,8 @@ static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx,
228	228	tctx->status != ThreadStatusDead);
229	229	}
230	230
231		ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
	231	ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(
	232	ThreadID os_id) {
232	233	return FindThreadContextLocked(FindThreadContextByOsIdCallback,
233	234	(void *)os_id);
234	235	}
...	...	@@ -322,8 +323,8 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
322	323	return prev_status;
323	324	}
324	325
325		void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
326		void *arg) {
	326	void ThreadRegistry::StartThread(u32 tid, ThreadID os_id,
	327	ThreadType thread_type, void *arg) {
327	328	ThreadRegistryLock l(this);
328	329	running_threads_++;
329	330	ThreadContextBase *tctx = threads_[tid];

lib/libtsan/sanitizer_common/sanitizer_thread_registry.h+4-4

...	...	@@ -43,7 +43,7 @@ class ThreadContextBase {
43	43	const u32 tid; // Thread ID. Main thread should have tid = 0.
44	44	u64 unique_id; // Unique thread ID.
45	45	u32 reuse_count; // Number of times this tid was reused.
46		tid_t os_id; // PID (used for reporting).
	46	ThreadID os_id; // PID (used for reporting).
47	47	uptr user_id; // Some opaque user thread id (e.g. pthread_t).
48	48	char name[64]; // As annotated by user.
49	49
...	...	@@ -62,7 +62,7 @@ class ThreadContextBase {
62	62	void SetDead();
63	63	void SetJoined(void *arg);
64	64	void SetFinished();
65		void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg);
	65	void SetStarted(ThreadID _os_id, ThreadType _thread_type, void *arg);
66	66	void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
67	67	u32 _parent_tid, u32 _stack_tid, void *arg);
68	68	void Reset();
...	...	@@ -126,7 +126,7 @@ class SANITIZER_MUTEX ThreadRegistry {
126	126	// is found.
127	127	ThreadContextBase *FindThreadContextLocked(FindThreadCallback cb,
128	128	void *arg);
129		ThreadContextBase *FindThreadContextByOsIDLocked(tid_t os_id);
	129	ThreadContextBase *FindThreadContextByOsIDLocked(ThreadID os_id);
130	130
131	131	void SetThreadName(u32 tid, const char *name);
132	132	void SetThreadNameByUserId(uptr user_id, const char *name);
...	...	@@ -134,7 +134,7 @@ class SANITIZER_MUTEX ThreadRegistry {
134	134	void JoinThread(u32 tid, void *arg);
135	135	// Finishes thread and returns previous status.
136	136	ThreadStatus FinishThread(u32 tid);
137		void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
	137	void StartThread(u32 tid, ThreadID os_id, ThreadType thread_type, void *arg);
138	138	u32 ConsumeThreadUserId(uptr user_id);
139	139	void SetThreadUserId(u32 tid, uptr user_id);
140	140

lib/libtsan/sanitizer_common/sanitizer_win.cpp+1-3

...	...	@@ -108,9 +108,7 @@ int internal_dlinfo(void handle, int request, void p) {
108	108
109	109	// In contrast to POSIX, on Windows GetCurrentThreadId()
110	110	// returns a system-unique identifier.
111		tid_t GetTid() {
112		return GetCurrentThreadId();
113		}
	111	ThreadID GetTid() { return GetCurrentThreadId(); }
114	112
115	113	uptr GetThreadSelf() {
116	114	return GetTid();

lib/libtsan/tsan_debugging.cpp+2-2

...	...	@@ -165,7 +165,7 @@ int __tsan_get_report_mutex(void report, uptr idx, uptr mutex_id, void **addr,
165	165	}
166	166
167	167	SANITIZER_INTERFACE_ATTRIBUTE
168		int __tsan_get_report_thread(void report, uptr idx, int tid, tid_t *os_id,
	168	int __tsan_get_report_thread(void report, uptr idx, int tid, ThreadID *os_id,
169	169	int running, const char name, int parent_tid,
170	170	void **trace, uptr trace_size) {
171	171	const ReportDesc rep = (ReportDesc )report;
...	...	@@ -242,7 +242,7 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
242	242
243	243	SANITIZER_INTERFACE_ATTRIBUTE
244	244	int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,
245		tid_t *os_id) {
	245	ThreadID *os_id) {
246	246	MBlock *b = 0;
247	247	Allocator *a = allocator();
248	248	if (a->PointerIsMine((void *)addr)) {

lib/libtsan/tsan_flags.cpp+37

...	...	@@ -20,6 +20,43 @@
20	20	#include "tsan_rtl.h"
21	21	#include "ubsan/ubsan_flags.h"
22	22
	23	#if SANITIZER_APPLE && !SANITIZER_GO
	24	namespace __sanitizer {
	25
	26	template <>
	27	inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char *value) {
	28	if (internal_strcmp(value, "on") == 0) {
	29	*t_ = kLockDuringAllWrites;
	30	return true;
	31	}
	32	if (internal_strcmp(value, "disable_for_current_process") == 0) {
	33	*t_ = kNoLockDuringWritesCurrentProcess;
	34	return true;
	35	}
	36	if (internal_strcmp(value, "disable_for_all_processes") == 0) {
	37	*t_ = kNoLockDuringWritesAllProcesses;
	38	return true;
	39	}
	40	Printf("ERROR: Invalid value for signal handler option: '%s'\n", value);
	41	return false;
	42	}
	43
	44	template <>
	45	inline bool FlagHandler<LockDuringWriteSetting>::Format(char *buffer,
	46	uptr size) {
	47	switch (*t_) {
	48	case kLockDuringAllWrites:
	49	return FormatString(buffer, size, "on");
	50	case kNoLockDuringWritesCurrentProcess:
	51	return FormatString(buffer, size, "disable_for_current_process");
	52	case kNoLockDuringWritesAllProcesses:
	53	return FormatString(buffer, size, "disable_for_all_processes");
	54	}
	55	}
	56
	57	} // namespace __sanitizer
	58	#endif // SANITIZER_APPLE && !SANITIZER_GO
	59
23	60	namespace __tsan {
24	61
25	62	// Can be overriden in frontend.

lib/libtsan/tsan_flags.h+8

...	...	@@ -16,6 +16,14 @@
16	16	#include "sanitizer_common/sanitizer_flags.h"
17	17	#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
18	18
	19	#if SANITIZER_APPLE && !SANITIZER_GO
	20	enum LockDuringWriteSetting {
	21	kLockDuringAllWrites,
	22	kNoLockDuringWritesCurrentProcess,
	23	kNoLockDuringWritesAllProcesses,
	24	};
	25	#endif
	26
19	27	namespace __tsan {
20	28
21	29	struct Flags : DDFlags {

lib/libtsan/tsan_flags.inc+12

...	...	@@ -80,3 +80,15 @@ TSAN_FLAG(bool, shared_ptr_interceptor, true,
80	80	TSAN_FLAG(bool, print_full_thread_history, false,
81	81	"If set, prints thread creation stacks for the threads involved in "
82	82	"the report and their ancestors up to the main thread.")
	83
	84	#if SANITIZER_APPLE && !SANITIZER_GO
	85	TSAN_FLAG(LockDuringWriteSetting, lock_during_write, kLockDuringAllWrites,
	86	"Determines whether to obtain a lock while writing logs or error "
	87	"reports. "
	88	"\"on\" - [default] lock during all writes. "
	89	"\"disable_for_current_process\" - don't lock during all writes in "
	90	"the current process, but do lock for all writes in child "
	91	"processes."
	92	"\"disable_for_all_processes\" - don't lock during all writes in "
	93	"the current process and it's children processes.")
	94	#endif

lib/libtsan/tsan_interceptors.h+9-1

...	...	@@ -1,6 +1,9 @@
1	1	#ifndef TSAN_INTERCEPTORS_H
2	2	#define TSAN_INTERCEPTORS_H
3	3
	4	#if SANITIZER_APPLE && !SANITIZER_GO
	5	# include "sanitizer_common/sanitizer_mac.h"
	6	#endif
4	7	#include "sanitizer_common/sanitizer_stacktrace.h"
5	8	#include "tsan_rtl.h"
6	9
...	...	@@ -43,7 +46,12 @@ inline bool in_symbolizer() {
43	46	#endif
44	47
45	48	inline bool MustIgnoreInterceptor(ThreadState *thr) {
46		return !thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib;
	49	return !thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib
	50	#if SANITIZER_APPLE && !SANITIZER_GO
	51	\|\| (flags()->lock_during_write != kLockDuringAllWrites &&
	52	thr->in_internal_write_call)
	53	#endif
	54	;
47	55	}
48	56
49	57	} // namespace __tsan

lib/libtsan/tsan_interceptors_mac.cpp+19

...	...	@@ -281,6 +281,25 @@ TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
281	281	Acquire(thr, pc, (uptr)lock);
282	282	}
283	283
	284	// os_unfair_lock_lock_with_flags was introduced in macOS 15
	285	# if defined(__MAC_15_0) \|\| defined(__IPHONE_18_0) \|\| defined(__TVOS_18_0) \|\| \
	286	defined(__VISIONOS_2_0) \|\| defined(__WATCHOS_11_0)
	287	# pragma clang diagnostic push
	288	# pragma clang diagnostic ignored "-Wunguarded-availability-new"
	289	// We're just intercepting this - if it doesn't exist on the platform, then the
	290	// process shouldn't have called it in the first place.
	291	TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_flags, os_unfair_lock_t lock,
	292	os_unfair_lock_flags_t flags) {
	293	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {
	294	return REAL(os_unfair_lock_lock_with_flags)(lock, flags);
	295	}
	296	SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_flags, lock, flags);
	297	REAL(os_unfair_lock_lock_with_flags)(lock, flags);
	298	Acquire(thr, pc, (uptr)lock);
	299	}
	300	# pragma clang diagnostic pop
	301	# endif
	302
284	303	TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
285	304	u32 options) {
286	305	if (!cur_thread()->is_inited \|\| cur_thread()->is_dead) {

lib/libtsan/tsan_interceptors_posix.cpp+78-32

...	...	@@ -22,6 +22,7 @@
22	22	#include "sanitizer_common/sanitizer_internal_defs.h"
23	23	#include "sanitizer_common/sanitizer_libc.h"
24	24	#include "sanitizer_common/sanitizer_linux.h"
	25	#include "sanitizer_common/sanitizer_placement_new.h"
25	26	#include "sanitizer_common/sanitizer_platform_interceptors.h"
26	27	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
27	28	#include "sanitizer_common/sanitizer_platform_limits_posix.h"
...	...	@@ -30,6 +31,9 @@
30	31	#include "sanitizer_common/sanitizer_tls_get_addr.h"
31	32	#include "sanitizer_common/sanitizer_vector.h"
32	33	#include "tsan_fd.h"
	34	#if SANITIZER_APPLE && !SANITIZER_GO
	35	# include "tsan_flags.h"
	36	#endif
33	37	#include "tsan_interceptors.h"
34	38	#include "tsan_interface.h"
35	39	#include "tsan_mman.h"
...	...	@@ -78,17 +82,6 @@ struct ucontext_t {
78	82	};
79	83	#endif
80	84
81		#if defined(__x86_64__) \|\| defined(__mips__) \|\| SANITIZER_PPC64V1 \|\| \
82		defined(__s390x__)
83		#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
84		#elif defined(__aarch64__) \|\| SANITIZER_PPC64V2
85		#define PTHREAD_ABI_BASE "GLIBC_2.17"
86		#elif SANITIZER_LOONGARCH64
87		#define PTHREAD_ABI_BASE "GLIBC_2.36"
88		#elif SANITIZER_RISCV64
89		# define PTHREAD_ABI_BASE "GLIBC_2.27"
90		#endif
91
92	85	extern "C" int pthread_attr_init(void *attr);
93	86	extern "C" int pthread_attr_destroy(void *attr);
94	87	DECLARE_REAL(int, pthread_attr_getdetachstate, void , void )
...	...	@@ -340,11 +333,6 @@ void ScopedInterceptor::DisableIgnoresImpl() {
340	333	}
341	334
342	335	#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
343		#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
344		# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
345		#else
346		# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
347		#endif
348	336	#if SANITIZER_FREEBSD
349	337	# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \
350	338	INTERCEPT_FUNCTION(_pthread_##func)
...	...	@@ -1145,6 +1133,22 @@ TSAN_INTERCEPTOR(int, pthread_create,
1145	1133
1146	1134	TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {
1147	1135	SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
	1136	#if SANITIZER_ANDROID
	1137	{
	1138	// In Bionic, if the target thread has already exited when pthread_detach is
	1139	// called, pthread_detach will call pthread_join internally to clean it up.
	1140	// In that case, the thread has already been consumed by the pthread_detach
	1141	// interceptor.
	1142	Tid tid = ctx->thread_registry.FindThread(
	1143	[](ThreadContextBase* tctx, void* arg) {
	1144	return tctx->user_id == (uptr)arg;
	1145	},
	1146	th);
	1147	if (tid == kInvalidTid) {
	1148	return REAL(pthread_join)(th, ret);
	1149	}
	1150	}
	1151	#endif
1148	1152	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1149	1153	ThreadIgnoreBegin(thr, pc);
1150	1154	int res = BLOCK_REAL(pthread_join)(th, ret);
...	...	@@ -1664,6 +1668,14 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1664	1668
1665	1669	TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {
1666	1670	SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
	1671	#if SANITIZER_APPLE && !SANITIZER_GO
	1672	if (flags()->lock_during_write != kLockDuringAllWrites &&
	1673	cur_thread_init()->in_internal_write_call) {
	1674	// This is needed to make it through process launch without hanging
	1675	f();
	1676	return 0;
	1677	}
	1678	#endif
1667	1679	if (o == 0 \|\| f == 0)
1668	1680	return errno_EINVAL;
1669	1681	atomic_uint32_t *a;
...	...	@@ -2141,13 +2153,29 @@ static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) {
2141	2153	// StackTrace::GetNestInstructionPc(pc) is used because return address is
2142	2154	// expected, OutputReport() will undo this.
2143	2155	ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
2144		ThreadRegistryLock l(&ctx->thread_registry);
2145		ScopedReport rep(ReportTypeErrnoInSignal);
2146		rep.SetSigNum(sig);
2147		if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
2148		rep.AddStack(stack, true);
2149		OutputReport(thr, rep);
	2156	// Use alloca, because malloc during signal handling deadlocks
	2157	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	2158	bool suppressed;
	2159	// Take a new scope as Apple platforms require the below locks released
	2160	// before symbolizing in order to avoid a deadlock
	2161	{
	2162	ThreadRegistryLock l(&ctx->thread_registry);
	2163	new (rep) ScopedReport(ReportTypeErrnoInSignal);
	2164	rep->SetSigNum(sig);
	2165	suppressed = IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack);
	2166	if (!suppressed)
	2167	rep->AddStack(stack, true);
	2168	#if SANITIZER_APPLE
	2169	} // Close this scope to release the locks before writing report
	2170	#endif
	2171	if (!suppressed)
	2172	OutputReport(thr, *rep);
	2173
	2174	// Need to manually destroy this because we used placement new to allocate
	2175	rep->~ScopedReport();
	2176	#if !SANITIZER_APPLE
2150	2177	}
	2178	#endif
2151	2179	}
2152	2180
2153	2181	static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
...	...	@@ -2411,7 +2439,11 @@ TSAN_INTERCEPTOR(int, vfork, int fake) {
2411	2439	}
2412	2440	#endif
2413	2441
2414		#if SANITIZER_LINUX
	2442	#if SANITIZER_LINUX && !SANITIZER_ANDROID
	2443	// Bionic's pthread_create internally calls clone. When the CLONE_THREAD flag is
	2444	// set, clone does not create a new process but a new thread. This is a
	2445	// workaround for Android. Disabling the interception of clone solves the
	2446	// problem in most scenarios.
2415	2447	TSAN_INTERCEPTOR(int, clone, int (fn)(void ), void *stack, int flags,
2416	2448	void arg, int parent_tid, void tls, pid_t child_tid) {
2417	2449	SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
...	...	@@ -2888,12 +2920,12 @@ TSAN_INTERCEPTOR(void, _lwp_exit) {
2888	2920	#endif
2889	2921
2890	2922	#if SANITIZER_FREEBSD
2891		TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
	2923	TSAN_INTERCEPTOR(void, thr_exit, ThreadID *state) {
2892	2924	SCOPED_TSAN_INTERCEPTOR(thr_exit, state);
2893	2925	DestroyThreadState();
2894	2926	REAL(thr_exit(state));
2895	2927	}
2896		#define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
	2928	# define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
2897	2929	#else
2898	2930	#define TSAN_MAYBE_INTERCEPT_THR_EXIT
2899	2931	#endif
...	...	@@ -3024,12 +3056,26 @@ void InitializeInterceptors() {
3024	3056	TSAN_INTERCEPT(pthread_timedjoin_np);
3025	3057	#endif
3026	3058
3027		TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE);
3028		TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE);
3029		TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE);
3030		TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE);
3031		TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
3032		TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
	3059	// In glibc versions older than 2.36, dlsym(RTLD_NEXT, "pthread_cond_init")
	3060	// may return an outdated symbol (max(2.2,base_version)) if the port was
	3061	// introduced before 2.3.2 (when the new pthread_cond_t was introduced).
	3062	#if SANITIZER_GLIBC && !__GLIBC_PREREQ(2, 36) && \
	3063	(defined(__x86_64__) \|\| defined(__mips__) \|\| SANITIZER_PPC64V1 \|\| \
	3064	defined(__s390x__))
	3065	INTERCEPT_FUNCTION_VER(pthread_cond_init, "GLIBC_2.3.2");
	3066	INTERCEPT_FUNCTION_VER(pthread_cond_signal, "GLIBC_2.3.2");
	3067	INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
	3068	INTERCEPT_FUNCTION_VER(pthread_cond_wait, "GLIBC_2.3.2");
	3069	INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
	3070	INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2");
	3071	#else
	3072	INTERCEPT_FUNCTION(pthread_cond_init);
	3073	INTERCEPT_FUNCTION(pthread_cond_signal);
	3074	INTERCEPT_FUNCTION(pthread_cond_broadcast);
	3075	INTERCEPT_FUNCTION(pthread_cond_wait);
	3076	INTERCEPT_FUNCTION(pthread_cond_timedwait);
	3077	INTERCEPT_FUNCTION(pthread_cond_destroy);
	3078	#endif
3033	3079
3034	3080	TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT;
3035	3081
...	...	@@ -3120,7 +3166,7 @@ void InitializeInterceptors() {
3120	3166
3121	3167	TSAN_INTERCEPT(fork);
3122	3168	TSAN_INTERCEPT(vfork);
3123		#if SANITIZER_LINUX
	3169	#if SANITIZER_LINUX && !SANITIZER_ANDROID
3124	3170	TSAN_INTERCEPT(clone);
3125	3171	#endif
3126	3172	#if !SANITIZER_ANDROID

lib/libtsan/tsan_interface.h+3-3

...	...	@@ -16,7 +16,7 @@
16	16	#define TSAN_INTERFACE_H
17	17
18	18	#include <sanitizer_common/sanitizer_internal_defs.h>
19		using __sanitizer::tid_t;
	19	using __sanitizer::ThreadID;
20	20	using __sanitizer::uptr;
21	21
22	22	// This header should NOT include any other headers.
...	...	@@ -175,7 +175,7 @@ int __tsan_get_report_mutex(void report, uptr idx, uptr mutex_id, void **addr,
175	175
176	176	// Returns information about threads included in the report.
177	177	SANITIZER_INTERFACE_ATTRIBUTE
178		int __tsan_get_report_thread(void report, uptr idx, int tid, tid_t *os_id,
	178	int __tsan_get_report_thread(void report, uptr idx, int tid, ThreadID *os_id,
179	179	int running, const char name, int parent_tid,
180	180	void **trace, uptr trace_size);
181	181
...	...	@@ -192,7 +192,7 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
192	192	// Returns the allocation stack for a heap pointer.
193	193	SANITIZER_INTERFACE_ATTRIBUTE
194	194	int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,
195		tid_t *os_id);
	195	ThreadID *os_id);
196	196
197	197	#endif // SANITIZER_GO
198	198

lib/libtsan/tsan_interface_ann.cpp+23-9

...	...	@@ -437,16 +437,30 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
437	437	}
438	438
439	439	static void ReportMutexHeldWrongContext(ThreadState *thr, uptr pc) {
440		ThreadRegistryLock l(&ctx->thread_registry);
441		ScopedReport rep(ReportTypeMutexHeldWrongContext);
442		for (uptr i = 0; i < thr->mset.Size(); ++i) {
443		MutexSet::Desc desc = thr->mset.Get(i);
444		rep.AddMutex(desc.addr, desc.stack_id);
	440	// Use alloca, because malloc during signal handling deadlocks
	441	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	442	// Take a new scope as Apple platforms require the below locks released
	443	// before symbolizing in order to avoid a deadlock
	444	{
	445	ThreadRegistryLock l(&ctx->thread_registry);
	446	new (rep) ScopedReport(ReportTypeMutexHeldWrongContext);
	447	for (uptr i = 0; i < thr->mset.Size(); ++i) {
	448	MutexSet::Desc desc = thr->mset.Get(i);
	449	rep->AddMutex(desc.addr, desc.stack_id);
	450	}
	451	VarSizeStackTrace trace;
	452	ObtainCurrentStack(thr, pc, &trace);
	453	rep->AddStack(trace, true);
	454	#if SANITIZER_APPLE
	455	} // Close this scope to release the locks
	456	#endif
	457	OutputReport(thr, *rep);
	458
	459	// Need to manually destroy this because we used placement new to allocate
	460	rep->~ScopedReport();
	461	#if !SANITIZER_APPLE
445	462	}
446		VarSizeStackTrace trace;
447		ObtainCurrentStack(thr, pc, &trace);
448		rep.AddStack(trace, true);
449		OutputReport(thr, rep);
	463	#endif
450	464	}
451	465
452	466	INTERFACE_ATTRIBUTE

lib/libtsan/tsan_mman.cpp+18-4

...	...	@@ -182,10 +182,24 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
182	182	ObtainCurrentStack(thr, pc, &stack);
183	183	if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
184	184	return;
185		ThreadRegistryLock l(&ctx->thread_registry);
186		ScopedReport rep(ReportTypeSignalUnsafe);
187		rep.AddStack(stack, true);
188		OutputReport(thr, rep);
	185	// Use alloca, because malloc during signal handling deadlocks
	186	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	187	// Take a new scope as Apple platforms require the below locks released
	188	// before symbolizing in order to avoid a deadlock
	189	{
	190	ThreadRegistryLock l(&ctx->thread_registry);
	191	new (rep) ScopedReport(ReportTypeSignalUnsafe);
	192	rep->AddStack(stack, true);
	193	#if SANITIZER_APPLE
	194	} // Close this scope to release the locks
	195	#endif
	196	OutputReport(thr, *rep);
	197
	198	// Need to manually destroy this because we used placement new to allocate
	199	rep->~ScopedReport();
	200	#if !SANITIZER_APPLE
	201	}
	202	#endif
189	203	}
190	204
191	205

lib/libtsan/tsan_platform.h+44-7

...	...	@@ -681,6 +681,32 @@ struct MappingGoMips64_47 {
681	681	static const uptr kShadowAdd = 0x200000000000ull;
682	682	};
683	683
	684	/* Go on linux/riscv64 (39-bit VMA)
	685	0000 0001 0000 - 000f 0000 0000: executable and heap (60 GiB)
	686	000f 0000 0000 - 0010 0000 0000: -
	687	0010 0000 0000 - 0030 0000 0000: shadow - 128 GiB ( ~ 2 * app)
	688	0030 0000 0000 - 0038 0000 0000: metainfo - 32 GiB ( ~ 0.5 * app)
	689	0038 0000 0000 - 0040 0000 0000: -
	690	*/
	691	struct MappingGoRiscv64_39 {
	692	static const uptr kMetaShadowBeg = 0x003000000000ull;
	693	static const uptr kMetaShadowEnd = 0x003800000000ull;
	694	static const uptr kShadowBeg = 0x001000000000ull;
	695	static const uptr kShadowEnd = 0x003000000000ull;
	696	static const uptr kLoAppMemBeg = 0x000000010000ull;
	697	static const uptr kLoAppMemEnd = 0x000f00000000ull;
	698	static const uptr kMidAppMemBeg = 0;
	699	static const uptr kMidAppMemEnd = 0;
	700	static const uptr kHiAppMemBeg = 0;
	701	static const uptr kHiAppMemEnd = 0;
	702	static const uptr kHeapMemBeg = 0;
	703	static const uptr kHeapMemEnd = 0;
	704	static const uptr kVdsoBeg = 0;
	705	static const uptr kShadowMsk = 0;
	706	static const uptr kShadowXor = 0;
	707	static const uptr kShadowAdd = 0x001000000000ull;
	708	};
	709
684	710	/* Go on linux/riscv64 (48-bit VMA)
685	711	0000 0001 0000 - 00e0 0000 0000: executable and heap (896 GiB)
686	712	00e0 0000 0000 - 2000 0000 0000: -
...	...	@@ -689,13 +715,13 @@ struct MappingGoMips64_47 {
689	715	3000 0000 0000 - 3100 0000 0000: metainfo - 1 TiB ( ~ 1 * app)
690	716	3100 0000 0000 - 8000 0000 0000: -
691	717	*/
692		struct MappingGoRiscv64 {
	718	struct MappingGoRiscv64_48 {
693	719	static const uptr kMetaShadowBeg = 0x300000000000ull;
694	720	static const uptr kMetaShadowEnd = 0x310000000000ull;
695	721	static const uptr kShadowBeg = 0x200000000000ull;
696	722	static const uptr kShadowEnd = 0x240000000000ull;
697	723	static const uptr kLoAppMemBeg = 0x000000010000ull;
698		static const uptr kLoAppMemEnd = 0x000e00000000ull;
	724	static const uptr kLoAppMemEnd = 0x00e000000000ull;
699	725	static const uptr kMidAppMemBeg = 0;
700	726	static const uptr kMidAppMemEnd = 0;
701	727	static const uptr kHiAppMemBeg = 0;
...	...	@@ -756,7 +782,12 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) {
756	782	# elif defined(__loongarch_lp64)
757	783	return Func::template Apply<MappingGoLoongArch64_47>(arg);
758	784	# elif SANITIZER_RISCV64
759		return Func::template Apply<MappingGoRiscv64>(arg);
	785	switch (vmaSize) {
	786	case 39:
	787	return Func::template Apply<MappingGoRiscv64_39>(arg);
	788	case 48:
	789	return Func::template Apply<MappingGoRiscv64_48>(arg);
	790	}
760	791	# elif SANITIZER_WINDOWS
761	792	return Func::template Apply<MappingGoWindows>(arg);
762	793	# else
...	...	@@ -827,7 +858,8 @@ void ForEachMapping() {
827	858	Func::template Apply<MappingGoAarch64>();
828	859	Func::template Apply<MappingGoLoongArch64_47>();
829	860	Func::template Apply<MappingGoMips64_47>();
830		Func::template Apply<MappingGoRiscv64>();
	861	Func::template Apply<MappingGoRiscv64_39>();
	862	Func::template Apply<MappingGoRiscv64_48>();
831	863	Func::template Apply<MappingGoS390x>();
832	864	}
833	865
...	...	@@ -926,7 +958,9 @@ struct IsAppMemImpl {
926	958	};
927	959
928	960	ALWAYS_INLINE
929		bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
	961	bool IsAppMem(uptr mem) {
	962	return SelectMapping<IsAppMemImpl>(STRIP_MTE_TAG(mem));
	963	}
930	964
931	965	struct IsShadowMemImpl {
932	966	template <typename Mapping>
...	...	@@ -965,7 +999,8 @@ struct MemToShadowImpl {
965	999
966	1000	ALWAYS_INLINE
967	1001	RawShadow *MemToShadow(uptr x) {
968		return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
	1002	return reinterpret_cast<RawShadow*>(
	1003	SelectMapping<MemToShadowImpl>(STRIP_MTE_TAG(x)));
969	1004	}
970	1005
971	1006	struct MemToMetaImpl {
...	...	@@ -979,7 +1014,9 @@ struct MemToMetaImpl {
979	1014	};
980	1015
981	1016	ALWAYS_INLINE
982		u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
	1017	u32* MemToMeta(uptr x) {
	1018	return SelectMapping<MemToMetaImpl>(STRIP_MTE_TAG(x));
	1019	}
983	1020
984	1021	struct ShadowToMemImpl {
985	1022	template <typename Mapping>

lib/libtsan/tsan_platform_linux.cpp+42-14

...	...	@@ -393,9 +393,9 @@ void InitializePlatformEarly() {
393	393	Die();
394	394	}
395	395	# else
396		if (vmaSize != 48) {
	396	if (vmaSize != 39 && vmaSize != 48) {
397	397	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
398		Printf("FATAL: Found %zd - Supported 48\n", vmaSize);
	398	Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize);
399	399	Die();
400	400	}
401	401	# endif
...	...	@@ -415,7 +415,7 @@ void InitializePlatform() {
415	415	// is not compiled with -pie.
416	416	#if !SANITIZER_GO
417	417	{
418		# if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
	418	# if INIT_LONGJMP_XOR_KEY
419	419	// Initialize the xor key used in {sig}{set,long}jump.
420	420	InitializeLongjmpXorKey();
421	421	# endif
...	...	@@ -486,8 +486,20 @@ int ExtractRecvmsgFDs(void msgp, int fds, int nfd) {
486	486
487	487	// Reverse operation of libc stack pointer mangling
488	488	static uptr UnmangleLongJmpSp(uptr mangled_sp) {
489		#if defined(__x86_64__)
490		# if SANITIZER_LINUX
	489	# if SANITIZER_ANDROID && INIT_LONGJMP_XOR_KEY
	490	if (longjmp_xor_key == 0) {
	491	// bionic libc initialization process: __libc_init_globals ->
	492	// __libc_init_vdso (calls strcmp) -> __libc_init_setjmp_cookie. strcmp is
	493	// intercepted by TSan, so during TSan initialization the setjmp_cookie
	494	// remains uninitialized. On Android, longjmp_xor_key must be set on first
	495	// use.
	496	InitializeLongjmpXorKey();
	497	CHECK_NE(longjmp_xor_key, 0);
	498	}
	499	# endif
	500
	501	# if defined(__x86_64__)
	502	# if SANITIZER_LINUX
491	503	// Reverse of:
492	504	// xor %fs:0x30, %rsi
493	505	// rol $0x11, %rsi
...	...	@@ -542,13 +554,23 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
542	554	# else
543	555	# define LONG_JMP_SP_ENV_SLOT 2
544	556	# endif
545		#elif SANITIZER_LINUX
546		# ifdef __aarch64__
547		# define LONG_JMP_SP_ENV_SLOT 13
548		# elif defined(__loongarch__)
549		# define LONG_JMP_SP_ENV_SLOT 1
550		# elif defined(__mips64)
551		# define LONG_JMP_SP_ENV_SLOT 1
	557	# elif SANITIZER_ANDROID
	558	# ifdef __aarch64__
	559	# define LONG_JMP_SP_ENV_SLOT 3
	560	# elif SANITIZER_RISCV64
	561	# define LONG_JMP_SP_ENV_SLOT 3
	562	# elif defined(__x86_64__)
	563	# define LONG_JMP_SP_ENV_SLOT 6
	564	# else
	565	# error unsupported
	566	# endif
	567	# elif SANITIZER_LINUX
	568	# ifdef __aarch64__
	569	# define LONG_JMP_SP_ENV_SLOT 13
	570	# elif defined(__loongarch__)
	571	# define LONG_JMP_SP_ENV_SLOT 1
	572	# elif defined(__mips64)
	573	# define LONG_JMP_SP_ENV_SLOT 1
552	574	# elif SANITIZER_RISCV64
553	575	# define LONG_JMP_SP_ENV_SLOT 13
554	576	# elif defined(__s390x__)
...	...	@@ -556,7 +578,7 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
556	578	# else
557	579	# define LONG_JMP_SP_ENV_SLOT 6
558	580	# endif
559		#endif
	581	# endif
560	582
561	583	uptr ExtractLongJmpSp(uptr *env) {
562	584	uptr mangled_sp = env[LONG_JMP_SP_ENV_SLOT];
...	...	@@ -653,7 +675,13 @@ ThreadState *cur_thread() {
653	675	}
654	676	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, &oldset, nullptr));
655	677	}
656		return thr;
	678
	679	// Skia calls mallopt(M_THREAD_DISABLE_MEM_INIT, 1), which sets the least
	680	// significant bit of TLS_SLOT_SANITIZER to 1. Scudo allocator uses this bit
	681	// as a flag to disable memory initialization. This is a workaround to get the
	682	// correct ThreadState pointer.
	683	uptr addr = reinterpret_cast<uptr>(thr);
	684	return reinterpret_cast<ThreadState*>(addr & ~1ULL);
657	685	}
658	686
659	687	void set_cur_thread(ThreadState *thr) {

lib/libtsan/tsan_platform_mac.cpp+16-3

...	...	@@ -226,9 +226,20 @@ static void ThreadTerminateCallback(uptr thread) {
226	226	void InitializePlatformEarly() {
227	227	# if !SANITIZER_GO && SANITIZER_IOS
228	228	uptr max_vm = GetMaxUserVirtualAddress() + 1;
229		if (max_vm != HiAppMemEnd()) {
230		Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
231		(void )max_vm, (void )HiAppMemEnd());
	229	if (max_vm < HiAppMemEnd()) {
	230	Report(
	231	"ThreadSanitizer: Unsupported virtual memory layout:\n\tVM address "
	232	"limit = %p\n\tExpected %p.\n",
	233	(void)max_vm, (void)HiAppMemEnd());
	234	Die();
	235	}
	236	// In some configurations, the max_vm is expanded, but much of this space is
	237	// already mapped. TSAN will not work in this configuration.
	238	if (!MemoryRangeIsAvailable(HiAppMemEnd() - 1, HiAppMemEnd() - 1)) {
	239	Report(
	240	"ThreadSanitizer: Unsupported virtual memory layout: Address %p is "
	241	"already mapped.\n",
	242	(void*)(HiAppMemEnd() - 1));
232	243	Die();
233	244	}
234	245	#endif
...	...	@@ -248,7 +259,9 @@ void InitializePlatform() {
248	259
249	260	ThreadEventCallbacks callbacks = {
250	261	.create = ThreadCreateCallback,
	262	.start = nullptr,
251	263	.terminate = ThreadTerminateCallback,
	264	.destroy = nullptr,
252	265	};
253	266	InstallPthreadIntrospectionHook(callbacks);
254	267	#endif

lib/libtsan/tsan_report.h+14-1

...	...	@@ -12,6 +12,8 @@
12	12	#ifndef TSAN_REPORT_H
13	13	#define TSAN_REPORT_H
14	14
	15	#include "sanitizer_common/sanitizer_internal_defs.h"
	16	#include "sanitizer_common/sanitizer_stacktrace.h"
15	17	#include "sanitizer_common/sanitizer_symbolizer.h"
16	18	#include "sanitizer_common/sanitizer_thread_registry.h"
17	19	#include "sanitizer_common/sanitizer_vector.h"
...	...	@@ -56,6 +58,7 @@ struct ReportMop {
56	58	bool atomic;
57	59	uptr external_tag;
58	60	Vector<ReportMopMutex> mset;
	61	StackTrace stack_trace;
59	62	ReportStack *stack;
60	63
61	64	ReportMop();
...	...	@@ -79,25 +82,34 @@ struct ReportLocation {
79	82	int fd = 0;
80	83	bool fd_closed = false;
81	84	bool suppressable = false;
	85	StackID stack_id = 0;
82	86	ReportStack *stack = nullptr;
83	87	};
84	88
85	89	struct ReportThread {
86	90	Tid id;
87		tid_t os_id;
	91	ThreadID os_id;
88	92	bool running;
89	93	ThreadType thread_type;
90	94	char *name;
91	95	Tid parent_tid;
	96	StackID stack_id;
92	97	ReportStack *stack;
	98	bool suppressable;
93	99	};
94	100
95	101	struct ReportMutex {
96	102	int id;
97	103	uptr addr;
	104	StackID stack_id;
98	105	ReportStack *stack;
99	106	};
100	107
	108	struct AddedLocationAddr {
	109	uptr addr;
	110	usize locs_idx;
	111	};
	112
101	113	class ReportDesc {
102	114	public:
103	115	ReportType typ;
...	...	@@ -105,6 +117,7 @@ class ReportDesc {
105	117	Vector<ReportStack*> stacks;
106	118	Vector<ReportMop*> mops;
107	119	Vector<ReportLocation*> locs;
	120	Vector<AddedLocationAddr> added_location_addrs;
108	121	Vector<ReportMutex*> mutexes;
109	122	Vector<ReportThread*> threads;
110	123	Vector<Tid> unique_tids;

lib/libtsan/tsan_rtl.cpp+14

...	...	@@ -40,6 +40,13 @@ SANITIZER_WEAK_DEFAULT_IMPL
40	40	void __tsan_test_only_on_fork() {}
41	41	#endif
42	42
	43	#if SANITIZER_APPLE && !SANITIZER_GO
	44	// Override weak symbol from sanitizer_common
	45	extern void __tsan_set_in_internal_write_call(bool value) {
	46	__tsan::cur_thread_init()->in_internal_write_call = value;
	47	}
	48	#endif
	49
43	50	namespace __tsan {
44	51
45	52	#if !SANITIZER_GO
...	...	@@ -893,6 +900,13 @@ void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
893	900	ThreadIgnoreBegin(thr, pc);
894	901	ThreadIgnoreSyncBegin(thr, pc);
895	902	}
	903
	904	# if SANITIZER_APPLE && !SANITIZER_GO
	905	// This flag can have inheritance disabled - we are the child so act
	906	// accordingly
	907	if (flags()->lock_during_write == kNoLockDuringWritesCurrentProcess)
	908	flags()->lock_during_write = kLockDuringAllWrites;
	909	# endif
896	910	}
897	911	#endif
898	912

lib/libtsan/tsan_rtl.h+7-2

...	...	@@ -236,6 +236,10 @@ struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
236	236
237	237	const ReportDesc *current_report;
238	238
	239	#if SANITIZER_APPLE && !SANITIZER_GO
	240	bool in_internal_write_call;
	241	#endif
	242
239	243	explicit ThreadState(Tid tid);
240	244	};
241	245
...	...	@@ -420,6 +424,7 @@ class ScopedReportBase {
420	424	void AddSleep(StackID stack_id);
421	425	void SetCount(int count);
422	426	void SetSigNum(int sig);
	427	void SymbolizeStackElems(void);
423	428
424	429	const ReportDesc *GetReport() const;
425	430
...	...	@@ -498,7 +503,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
498	503
499	504	void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
500	505	AccessType typ);
501		bool OutputReport(ThreadState *thr, const ScopedReport &srep);
	506	bool OutputReport(ThreadState *thr, ScopedReport &srep);
502	507	bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
503	508	bool IsExpectedReport(uptr addr, uptr size);
504	509
...	...	@@ -559,7 +564,7 @@ void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
559	564	void ThreadIgnoreSyncEnd(ThreadState *thr);
560	565
561	566	Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
562		void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
	567	void ThreadStart(ThreadState *thr, Tid tid, ThreadID os_id,
563	568	ThreadType thread_type);
564	569	void ThreadFinish(ThreadState *thr);
565	570	Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);

lib/libtsan/tsan_rtl_aarch64.S+3-5

...	...	@@ -4,10 +4,8 @@
4	4	#include "sanitizer_common/sanitizer_asm.h"
5	5	#include "builtins/assembly.h"
6	6
7		#if !defined(__APPLE__)
8		.section .text
9		#else
10		.section __TEXT,__text
	7	TEXT_SECTION
	8	#if defined(__APPLE__)
11	9	.align 3
12	10	#endif
13	11
...	...	@@ -222,6 +220,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
222	220
223	221	NO_EXEC_STACK_DIRECTIVE
224	222
225		GNU_PROPERTY_BTI_PAC
	223	GNU_PROPERTY_BTI_PAC_GCS
226	224
227	225	#endif

lib/libtsan/tsan_rtl_access.cpp+8-3

...	...	@@ -419,6 +419,11 @@ NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
419	419
420	420	ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
421	421	uptr size, AccessType typ) {
	422	#if SANITIZER_APPLE && !SANITIZER_GO
	423	// Swift symbolizer can be intercepted and deadlock without this
	424	if (thr->in_symbolizer)
	425	return;
	426	#endif
422	427	RawShadow* shadow_mem = MemToShadow(addr);
423	428	UNUSED char memBuf[4][64];
424	429	DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
...	...	@@ -684,7 +689,7 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
684	689	DCHECK(IsAppMem(addr + size - 1));
685	690	}
686	691	if (!IsShadowMem(shadow_mem)) {
687		Printf("Bad shadow start addr: %p (%p)\n", shadow_mem, (void*)addr);
	692	Printf("Bad shadow start addr: %p (%p)\n", (void)shadow_mem, (void)addr);
688	693	DCHECK(IsShadowMem(shadow_mem));
689	694	}
690	695
...	...	@@ -693,12 +698,12 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
693	698	RawShadow* shadow_mem_end =
694	699	shadow_mem + rounded_size / kShadowCell * kShadowCnt;
695	700	if (!IsShadowMem(shadow_mem_end - 1)) {
696		Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end - 1,
	701	Printf("Bad shadow end addr: %p (%p)\n", (void*)(shadow_mem_end - 1),
697	702	(void*)(addr + size - 1));
698	703	Printf(
699	704	"Shadow start addr (ok): %p (%p); size: 0x%zx; rounded_size: 0x%zx; "
700	705	"kShadowMultiplier: %zx\n",
701		shadow_mem, (void*)addr, size, rounded_size, kShadowMultiplier);
	706	(void)shadow_mem, (void)addr, size, rounded_size, kShadowMultiplier);
702	707	DCHECK(IsShadowMem(shadow_mem_end - 1));
703	708	}
704	709	#endif

lib/libtsan/tsan_rtl_amd64.S+2

...	...	@@ -3,6 +3,8 @@
3	3
4	4	#include "sanitizer_common/sanitizer_asm.h"
5	5
	6	.att_syntax
	7
6	8	#if !defined(__APPLE__)
7	9	.section .text
8	10	#else

lib/libtsan/tsan_rtl_mutex.cpp+94-49

...	...	@@ -11,14 +11,15 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include <sanitizer_common/sanitizer_deadlock_detector_interface.h>
	14	#include <sanitizer_common/sanitizer_placement_new.h>
14	15	#include <sanitizer_common/sanitizer_stackdepot.h>
15	16
16		#include "tsan_rtl.h"
17	17	#include "tsan_flags.h"
18		#include "tsan_sync.h"
	18	#include "tsan_platform.h"
19	19	#include "tsan_report.h"
	20	#include "tsan_rtl.h"
20	21	#include "tsan_symbolize.h"
21		#include "tsan_platform.h"
	22	#include "tsan_sync.h"
22	23
23	24	namespace __tsan {
24	25
...	...	@@ -55,14 +56,28 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
55	56	return;
56	57	if (!ShouldReport(thr, typ))
57	58	return;
58		ThreadRegistryLock l(&ctx->thread_registry);
59		ScopedReport rep(typ);
60		rep.AddMutex(addr, creation_stack_id);
61		VarSizeStackTrace trace;
62		ObtainCurrentStack(thr, pc, &trace);
63		rep.AddStack(trace, true);
64		rep.AddLocation(addr, 1);
65		OutputReport(thr, rep);
	59	// Use alloca, because malloc during signal handling deadlocks
	60	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	61	// Take a new scope as Apple platforms require the below locks released
	62	// before symbolizing in order to avoid a deadlock
	63	{
	64	ThreadRegistryLock l(&ctx->thread_registry);
	65	new (rep) ScopedReport(typ);
	66	rep->AddMutex(addr, creation_stack_id);
	67	VarSizeStackTrace trace;
	68	ObtainCurrentStack(thr, pc, &trace);
	69	rep->AddStack(trace, true);
	70	rep->AddLocation(addr, 1);
	71	#if SANITIZER_APPLE
	72	} // Close this scope to release the locks
	73	#endif
	74	OutputReport(thr, *rep);
	75
	76	// Need to manually destroy this because we used placement new to allocate
	77	rep->~ScopedReport();
	78	#if !SANITIZER_APPLE
	79	}
	80	#endif
66	81	}
67	82
68	83	static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
...	...	@@ -528,51 +543,81 @@ void AfterSleep(ThreadState *thr, uptr pc) {
528	543	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
529	544	if (r == 0 \|\| !ShouldReport(thr, ReportTypeDeadlock))
530	545	return;
531		ThreadRegistryLock l(&ctx->thread_registry);
532		ScopedReport rep(ReportTypeDeadlock);
533		for (int i = 0; i < r->n; i++) {
534		rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
535		rep.AddUniqueTid((int)r->loop[i].thr_ctx);
536		rep.AddThread((int)r->loop[i].thr_ctx);
537		}
538		uptr dummy_pc = 0x42;
539		for (int i = 0; i < r->n; i++) {
540		for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
541		u32 stk = r->loop[i].stk[j];
542		if (stk && stk != kInvalidStackID) {
543		rep.AddStack(StackDepotGet(stk), true);
544		} else {
545		// Sometimes we fail to extract the stack trace (FIXME: investigate),
546		// but we should still produce some stack trace in the report.
547		rep.AddStack(StackTrace(&dummy_pc, 1), true);
	546	// Use alloca, because malloc during signal handling deadlocks
	547	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	548	// Take a new scope as Apple platforms require the below locks released
	549	// before symbolizing in order to avoid a deadlock
	550	{
	551	ThreadRegistryLock l(&ctx->thread_registry);
	552	new (rep) ScopedReport(ReportTypeDeadlock);
	553	for (int i = 0; i < r->n; i++) {
	554	rep->AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
	555	rep->AddUniqueTid((int)r->loop[i].thr_ctx);
	556	rep->AddThread((int)r->loop[i].thr_ctx);
	557	}
	558	uptr dummy_pc = 0x42;
	559	for (int i = 0; i < r->n; i++) {
	560	for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
	561	u32 stk = r->loop[i].stk[j];
	562	StackTrace stack;
	563	if (stk && stk != kInvalidStackID) {
	564	stack = StackDepotGet(stk);
	565	} else {
	566	// Sometimes we fail to extract the stack trace (FIXME: investigate),
	567	// but we should still produce some stack trace in the report.
	568	stack = StackTrace(&dummy_pc, 1);
	569	}
	570	rep->AddStack(stack, true);
548	571	}
549	572	}
	573	#if SANITIZER_APPLE
	574	} // Close this scope to release the locks
	575	#endif
	576	OutputReport(thr, *rep);
	577
	578	// Need to manually destroy this because we used placement new to allocate
	579	rep->~ScopedReport();
	580	#if !SANITIZER_APPLE
550	581	}
551		OutputReport(thr, rep);
	582	#endif
552	583	}
553	584
554	585	void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
555	586	FastState last_lock, StackID creation_stack_id) {
556		// We need to lock the slot during RestoreStack because it protects
557		// the slot journal.
558		Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
559		ThreadRegistryLock l0(&ctx->thread_registry);
560		Lock slots_lock(&ctx->slot_mtx);
561		ScopedReport rep(ReportTypeMutexDestroyLocked);
562		rep.AddMutex(addr, creation_stack_id);
563		VarSizeStackTrace trace;
564		ObtainCurrentStack(thr, pc, &trace);
565		rep.AddStack(trace, true);
566
567		Tid tid;
568		DynamicMutexSet mset;
569		uptr tag;
570		if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
571		0, kAccessWrite, &tid, &trace, mset, &tag))
572		return;
573		rep.AddStack(trace, true);
574		rep.AddLocation(addr, 1);
575		OutputReport(thr, rep);
	587	// Use alloca, because malloc during signal handling deadlocks
	588	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	589	// Take a new scope as Apple platforms require the below locks released
	590	// before symbolizing in order to avoid a deadlock
	591	{
	592	// We need to lock the slot during RestoreStack because it protects
	593	// the slot journal.
	594	Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
	595	ThreadRegistryLock l0(&ctx->thread_registry);
	596	Lock slots_lock(&ctx->slot_mtx);
	597	new (rep) ScopedReport(ReportTypeMutexDestroyLocked);
	598	rep->AddMutex(addr, creation_stack_id);
	599	VarSizeStackTrace trace;
	600	ObtainCurrentStack(thr, pc, &trace);
	601	rep->AddStack(trace, true);
	602
	603	Tid tid;
	604	DynamicMutexSet mset;
	605	uptr tag;
	606	if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(),
	607	addr, 0, kAccessWrite, &tid, &trace, mset, &tag))
	608	return;
	609	rep->AddStack(trace, true);
	610	rep->AddLocation(addr, 1);
	611	#if SANITIZER_APPLE
	612	} // Close this scope to release the locks
	613	#endif
	614	OutputReport(thr, *rep);
	615
	616	// Need to manually destroy this because we used placement new to allocate
	617	rep->~ScopedReport();
	618	#if !SANITIZER_APPLE
	619	}
	620	#endif
576	621	}
577	622
578	623	} // namespace __tsan

lib/libtsan/tsan_rtl_report.cpp+132-65

...	...	@@ -11,10 +11,12 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_common/sanitizer_common.h"
	14	#include "sanitizer_common/sanitizer_internal_defs.h"
14	15	#include "sanitizer_common/sanitizer_libc.h"
15	16	#include "sanitizer_common/sanitizer_placement_new.h"
16	17	#include "sanitizer_common/sanitizer_stackdepot.h"
17	18	#include "sanitizer_common/sanitizer_stacktrace.h"
	19	#include "tsan_defs.h"
18	20	#include "tsan_fd.h"
19	21	#include "tsan_flags.h"
20	22	#include "tsan_mman.h"
...	...	@@ -109,7 +111,13 @@ static ReportStack *SymbolizeStack(StackTrace trace) {
109	111	// instruction.
110	112	if ((pc & kExternalPCBit) == 0)
111	113	pc1 = StackTrace::GetPreviousInstructionPc(pc);
112		SymbolizedStack *ent = SymbolizeCode(pc1);
	114	SymbolizedStack* ent = SymbolizeCode(pc1, si == trace.size - 1);
	115	#if SANITIZER_GO
	116	if (ent == nullptr) {
	117	// Go might have 0 frames for this PC (wrapper frames aren't reported).
	118	continue;
	119	}
	120	#endif
113	121	CHECK_NE(ent, 0);
114	122	SymbolizedStack *last = ent;
115	123	while (last->next) {
...	...	@@ -187,10 +195,8 @@ void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
187	195	mop->size = size;
188	196	mop->write = !(typ & kAccessRead);
189	197	mop->atomic = typ & kAccessAtomic;
190		mop->stack = SymbolizeStack(stack);
191	198	mop->external_tag = external_tag;
192		if (mop->stack)
193		mop->stack->suppressable = true;
	199	mop->stack_trace = stack;
194	200	for (uptr i = 0; i < mset->Size(); i++) {
195	201	MutexSet::Desc d = mset->Get(i);
196	202	int id = this->AddMutex(d.addr, d.stack_id);
...	...	@@ -199,6 +205,56 @@ void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
199	205	}
200	206	}
201	207
	208	void ScopedReportBase::SymbolizeStackElems() {
	209	// symbolize memory ops
	210	for (usize i = 0, size = rep_->mops.Size(); i < size; i++) {
	211	ReportMop *mop = rep_->mops[i];
	212	mop->stack = SymbolizeStack(mop->stack_trace);
	213	if (mop->stack)
	214	mop->stack->suppressable = true;
	215	}
	216
	217	// symbolize locations
	218	for (usize i = 0, size = rep_->locs.Size(); i < size; i++) {
	219	// added locations have a NULL placeholder - don't dereference them
	220	if (ReportLocation *loc = rep_->locs[i])
	221	loc->stack = SymbolizeStackId(loc->stack_id);
	222	}
	223
	224	// symbolize any added locations
	225	for (usize i = 0, size = rep_->added_location_addrs.Size(); i < size; i++) {
	226	AddedLocationAddr *added_loc = &rep_->added_location_addrs[i];
	227	if (ReportLocation *loc = SymbolizeData(added_loc->addr)) {
	228	loc->suppressable = true;
	229	rep_->locs[added_loc->locs_idx] = loc;
	230	}
	231	}
	232
	233	// Filter out any added location placeholders that could not be symbolized
	234	usize j = 0;
	235	for (usize i = 0, size = rep_->locs.Size(); i < size; i++) {
	236	if (rep_->locs[i] != nullptr) {
	237	rep_->locs[j] = rep_->locs[i];
	238	j++;
	239	}
	240	}
	241	rep_->locs.Resize(j);
	242
	243	// symbolize threads
	244	for (usize i = 0, size = rep_->threads.Size(); i < size; i++) {
	245	ReportThread *rt = rep_->threads[i];
	246	rt->stack = SymbolizeStackId(rt->stack_id);
	247	if (rt->stack)
	248	rt->stack->suppressable = rt->suppressable;
	249	}
	250
	251	// symbolize mutexes
	252	for (usize i = 0, size = rep_->mutexes.Size(); i < size; i++) {
	253	ReportMutex *rm = rep_->mutexes[i];
	254	rm->stack = SymbolizeStackId(rm->stack_id);
	255	}
	256	}
	257
202	258	void ScopedReportBase::AddUniqueTid(Tid unique_tid) {
203	259	rep_->unique_tids.PushBack(unique_tid);
204	260	}
...	...	@@ -216,10 +272,8 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
216	272	rt->name = internal_strdup(tctx->name);
217	273	rt->parent_tid = tctx->parent_tid;
218	274	rt->thread_type = tctx->thread_type;
219		rt->stack = 0;
220		rt->stack = SymbolizeStackId(tctx->creation_stack_id);
221		if (rt->stack)
222		rt->stack->suppressable = suppressable;
	275	rt->stack_id = tctx->creation_stack_id;
	276	rt->suppressable = suppressable;
223	277	}
224	278
225	279	#if !SANITIZER_GO
...	...	@@ -270,7 +324,7 @@ int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) {
270	324	rep_->mutexes.PushBack(rm);
271	325	rm->id = rep_->mutexes.Size() - 1;
272	326	rm->addr = addr;
273		rm->stack = SymbolizeStackId(creation_stack_id);
	327	rm->stack_id = creation_stack_id;
274	328	return rm->id;
275	329	}
276	330
...	...	@@ -288,7 +342,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
288	342	loc->fd_closed = closed;
289	343	loc->fd = fd;
290	344	loc->tid = creat_tid;
291		loc->stack = SymbolizeStackId(creat_stack);
	345	loc->stack_id = creat_stack;
292	346	rep_->locs.PushBack(loc);
293	347	AddThread(creat_tid);
294	348	return;
...	...	@@ -310,7 +364,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
310	364	loc->heap_chunk_size = b->siz;
311	365	loc->external_tag = b->tag;
312	366	loc->tid = b->tid;
313		loc->stack = SymbolizeStackId(b->stk);
	367	loc->stack_id = b->stk;
314	368	rep_->locs.PushBack(loc);
315	369	AddThread(b->tid);
316	370	return;
...	...	@@ -324,11 +378,8 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
324	378	AddThread(tctx);
325	379	}
326	380	#endif
327		if (ReportLocation *loc = SymbolizeData(addr)) {
328		loc->suppressable = true;
329		rep_->locs.PushBack(loc);
330		return;
331		}
	381	rep_->added_location_addrs.PushBack({addr, rep_->locs.Size()});
	382	rep_->locs.PushBack(nullptr);
332	383	}
333	384
334	385	#if !SANITIZER_GO
...	...	@@ -628,11 +679,12 @@ static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {
628	679	return false;
629	680	}
630	681
631		bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
	682	bool OutputReport(ThreadState *thr, ScopedReport &srep) {
632	683	// These should have been checked in ShouldReport.
633	684	// It's too late to check them here, we have already taken locks.
634	685	CHECK(flags()->report_bugs);
635	686	CHECK(!thr->suppress_reports);
	687	srep.SymbolizeStackElems();
636	688	atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
637	689	const ReportDesc *rep = srep.GetReport();
638	690	CHECK_EQ(thr->current_report, nullptr);
...	...	@@ -761,65 +813,80 @@ void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
761	813	DynamicMutexSet mset1;
762	814	MutexSet *mset[kMop] = {&thr->mset, mset1};
763	815
764		// We need to lock the slot during RestoreStack because it protects
765		// the slot journal.
766		Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
767		ThreadRegistryLock l0(&ctx->thread_registry);
768		Lock slots_lock(&ctx->slot_mtx);
769		if (SpuriousRace(old))
770		return;
771		if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
772		size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
773		StoreShadow(&ctx->last_spurious_race, old.raw());
774		return;
775		}
	816	// Use alloca, because malloc during signal handling deadlocks
	817	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	818	// Take a new scope as Apple platforms require the below locks released
	819	// before symbolizing in order to avoid a deadlock
	820	{
	821	// We need to lock the slot during RestoreStack because it protects
	822	// the slot journal.
	823	Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
	824	ThreadRegistryLock l0(&ctx->thread_registry);
	825	Lock slots_lock(&ctx->slot_mtx);
	826	if (SpuriousRace(old))
	827	return;
	828	if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
	829	size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
	830	StoreShadow(&ctx->last_spurious_race, old.raw());
	831	return;
	832	}
776	833
777		if (IsFiredSuppression(ctx, rep_typ, traces[1]))
778		return;
	834	if (IsFiredSuppression(ctx, rep_typ, traces[1]))
	835	return;
779	836
780		if (HandleRacyStacks(thr, traces))
781		return;
	837	if (HandleRacyStacks(thr, traces))
	838	return;
782	839
783		// If any of the accesses has a tag, treat this as an "external" race.
784		uptr tag = kExternalTagNone;
785		for (uptr i = 0; i < kMop; i++) {
786		if (tags[i] != kExternalTagNone) {
787		rep_typ = ReportTypeExternalRace;
788		tag = tags[i];
789		break;
	840	// If any of the accesses has a tag, treat this as an "external" race.
	841	uptr tag = kExternalTagNone;
	842	for (uptr i = 0; i < kMop; i++) {
	843	if (tags[i] != kExternalTagNone) {
	844	rep_typ = ReportTypeExternalRace;
	845	tag = tags[i];
	846	break;
	847	}
790	848	}
791		}
792	849
793		ScopedReport rep(rep_typ, tag);
794		for (uptr i = 0; i < kMop; i++)
795		rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
	850	new (rep) ScopedReport(rep_typ, tag);
	851	for (uptr i = 0; i < kMop; i++)
	852	rep->AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
796	853
797		for (uptr i = 0; i < kMop; i++) {
798		ThreadContext tctx = static_cast<ThreadContext >(
799		ctx->thread_registry.GetThreadLocked(tids[i]));
800		rep.AddThread(tctx);
801		}
	854	for (uptr i = 0; i < kMop; i++) {
	855	ThreadContext tctx = static_cast<ThreadContext >(
	856	ctx->thread_registry.GetThreadLocked(tids[i]));
	857	rep->AddThread(tctx);
	858	}
802	859
803		rep.AddLocation(addr_min, addr_max - addr_min);
804
805		if (flags()->print_full_thread_history) {
806		const ReportDesc *rep_desc = rep.GetReport();
807		for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
808		Tid parent_tid = rep_desc->threads[i]->parent_tid;
809		if (parent_tid == kMainTid \|\| parent_tid == kInvalidTid)
810		continue;
811		ThreadContext parent_tctx = static_cast<ThreadContext >(
812		ctx->thread_registry.GetThreadLocked(parent_tid));
813		rep.AddThread(parent_tctx);
	860	rep->AddLocation(addr_min, addr_max - addr_min);
	861
	862	if (flags()->print_full_thread_history) {
	863	const ReportDesc *rep_desc = rep->GetReport();
	864	for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
	865	Tid parent_tid = rep_desc->threads[i]->parent_tid;
	866	if (parent_tid == kMainTid \|\| parent_tid == kInvalidTid)
	867	continue;
	868	ThreadContext parent_tctx = static_cast<ThreadContext >(
	869	ctx->thread_registry.GetThreadLocked(parent_tid));
	870	rep->AddThread(parent_tctx);
	871	}
814	872	}
815		}
816	873
817	874	#if !SANITIZER_GO
818		if (!((typ0 \| typ1) & kAccessFree) &&
819		s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
820		rep.AddSleep(thr->last_sleep_stack_id);
	875	if (!((typ0 \| typ1) & kAccessFree) &&
	876	s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
	877	rep->AddSleep(thr->last_sleep_stack_id);
	878	#endif
	879
	880	#if SANITIZER_APPLE
	881	} // Close this scope to release the locks
	882	#endif
	883	OutputReport(thr, *rep);
	884
	885	// Need to manually destroy this because we used placement new to allocate
	886	rep->~ScopedReport();
	887	#if !SANITIZER_APPLE
	888	}
821	889	#endif
822		OutputReport(thr, rep);
823	890	}
824	891
825	892	void PrintCurrentStack(ThreadState *thr, uptr pc) {

lib/libtsan/tsan_rtl_thread.cpp+31-9

...	...	@@ -88,15 +88,33 @@ void ThreadFinalize(ThreadState *thr) {
88	88	#if !SANITIZER_GO
89	89	if (!ShouldReport(thr, ReportTypeThreadLeak))
90	90	return;
91		ThreadRegistryLock l(&ctx->thread_registry);
92	91	Vector<ThreadLeak> leaks;
93		ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
94		&leaks);
	92	{
	93	ThreadRegistryLock l(&ctx->thread_registry);
	94	ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
	95	&leaks);
	96	}
	97
95	98	for (uptr i = 0; i < leaks.Size(); i++) {
96		ScopedReport rep(ReportTypeThreadLeak);
97		rep.AddThread(leaks[i].tctx, true);
98		rep.SetCount(leaks[i].count);
99		OutputReport(thr, rep);
	99	// Use alloca, because malloc during signal handling deadlocks
	100	ScopedReport rep = (ScopedReport )__builtin_alloca(sizeof(ScopedReport));
	101	// Take a new scope as Apple platforms require the below locks released
	102	// before symbolizing in order to avoid a deadlock
	103	{
	104	ThreadRegistryLock l(&ctx->thread_registry);
	105	new (rep) ScopedReport(ReportTypeThreadLeak);
	106	rep->AddThread(leaks[i].tctx, true);
	107	rep->SetCount(leaks[i].count);
	108	# if SANITIZER_APPLE
	109	} // Close this scope to release the locks
	110	# endif
	111	OutputReport(thr, *rep);
	112
	113	// Need to manually destroy this because we used placement new to allocate
	114	rep->~ScopedReport();
	115	# if !SANITIZER_APPLE
	116	}
	117	# endif
100	118	}
101	119	#endif
102	120	}
...	...	@@ -149,7 +167,7 @@ struct OnStartedArgs {
149	167	uptr tls_size;
150	168	};
151	169
152		void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
	170	void ThreadStart(ThreadState *thr, Tid tid, ThreadID os_id,
153	171	ThreadType thread_type) {
154	172	ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
155	173	if (!thr->ignore_sync) {
...	...	@@ -188,10 +206,14 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
188	206	}
189	207	#endif
190	208
191		#if !SANITIZER_GO
	209	#if !SANITIZER_GO && !SANITIZER_ANDROID
192	210	// Don't imitate stack/TLS writes for the main thread,
193	211	// because its initialization is synchronized with all
194	212	// subsequent threads anyway.
	213	// Because thr is created by MmapOrDie, the thr object
	214	// is not in tls, the pointer to the thr object is in
	215	// TLS_SLOT_SANITIZER slot. So skip this check on
	216	// Android platform.
195	217	if (tid != kMainTid) {
196	218	if (stk_addr && stk_size) {
197	219	const uptr pc = StackTrace::GetNextInstructionPc(

lib/libtsan/tsan_symbolize.cpp+1-1

...	...	@@ -79,7 +79,7 @@ static void AddFrame(void ctx, const char function_name, const char *file,
79	79	info->column = column;
80	80	}
81	81
82		SymbolizedStack *SymbolizeCode(uptr addr) {
	82	SymbolizedStack* SymbolizeCode(uptr addr, bool leaf) {
83	83	// Check if PC comes from non-native land.
84	84	if (addr & kExternalPCBit) {
85	85	SymbolizedStackBuilder ssb = {nullptr, nullptr, addr};

lib/libtsan/tsan_symbolize.h+1-1

...	...	@@ -19,7 +19,7 @@ namespace __tsan {
19	19
20	20	void EnterSymbolizer();
21	21	void ExitSymbolizer();
22		SymbolizedStack *SymbolizeCode(uptr addr);
	22	SymbolizedStack* SymbolizeCode(uptr addr, bool leaf);
23	23	ReportLocation *SymbolizeData(uptr addr);
24	24	void SymbolizeFlush();
25	25

lib/libtsan/tsan_trace.h+1-1

...	...	@@ -190,7 +190,7 @@ struct Trace {
190	190	Mutex mtx;
191	191	IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
192	192	// First node non-queued into ctx->trace_part_recycle.
193		TracePart* local_head;
	193	TracePart* local_head = nullptr;
194	194	// Final position in the last part for finished threads.
195	195	Event* final_pos = nullptr;
196	196	// Number of trace parts allocated on behalf of this trace specifically.

lib/libunwind/include/__libunwind_config.h+11-2

...	...	@@ -73,11 +73,11 @@
73	73	# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC
74	74	# elif defined(__aarch64__)
75	75	# define _LIBUNWIND_TARGET_AARCH64 1
76		# define _LIBUNWIND_CONTEXT_SIZE 66
	76	#define _LIBUNWIND_CONTEXT_SIZE 67
77	77	# if defined(__SEH__)
78	78	# define _LIBUNWIND_CURSOR_SIZE 164
79	79	# else
80		# define _LIBUNWIND_CURSOR_SIZE 78
	80	#define _LIBUNWIND_CURSOR_SIZE 79
81	81	# endif
82	82	# define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64
83	83	# elif defined(__arm__)
...	...	@@ -212,4 +212,13 @@
212	212	# define _LIBUNWIND_HIGHEST_DWARF_REGISTER 287
213	213	#endif // _LIBUNWIND_IS_NATIVE_ONLY
214	214
	215	#if defined(__has_feature)
	216	# if __has_feature(ptrauth_calls) && __has_feature(ptrauth_returns)
	217	# define _LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING 1
	218	# elif __has_feature(ptrauth_calls) != __has_feature(ptrauth_returns)
	219	# error "Either both or none of ptrauth_calls and ptrauth_returns "\
	220	"is allowed to be enabled"
	221	# endif
	222	#endif
	223
215	224	#endif // ____LIBUNWIND_CONFIG_H__

lib/libunwind/include/libunwind.h+117-11

...	...	@@ -43,6 +43,109 @@
43	43	#define LIBUNWIND_AVAIL
44	44	#endif
45	45
	46	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	47
	48	#include <ptrauth.h>
	49
	50	// `__ptrauth_restricted_intptr` is a feature of apple clang that predates
	51	// support for direct application of `__ptrauth` to integer types. This
	52	// guard is necessary to support compilation with those compiler.
	53	#if __has_extension(ptrauth_restricted_intptr_qualifier)
	54	#define __unwind_ptrauth_restricted_intptr(...) \
	55	__ptrauth_restricted_intptr(__VA_ARGS__)
	56	#else
	57	#define __unwind_ptrauth_restricted_intptr(...) \
	58	__ptrauth(__VA_ARGS__)
	59	#endif
	60
	61	// ptrauth_string_discriminator("unw_proc_info_t::handler") == 0x7405
	62	#define __ptrauth_unwind_upi_handler_disc 0x7405
	63
	64	#define __ptrauth_unwind_upi_handler \
	65	__ptrauth(ptrauth_key_function_pointer, 1, __ptrauth_unwind_upi_handler_disc)
	66
	67	#define __ptrauth_unwind_upi_handler_intptr \
	68	__unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer, 1,\
	69	__ptrauth_unwind_upi_handler_disc)
	70
	71	// ptrauth_string_discriminator("unw_proc_info_t::start_ip") == 0xCA2C
	72	#define __ptrauth_unwind_upi_startip \
	73	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_independent_code, 1, 0xCA2C)
	74
	75	// ptrauth_string_discriminator("unw_proc_info_t::end_ip") == 0xE183
	76	#define __ptrauth_unwind_upi_endip \
	77	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_independent_code, 1, 0xE183)
	78
	79	// ptrauth_string_discriminator("unw_proc_info_t::lsda") == 0x83DE
	80	#define __ptrauth_unwind_upi_lsda \
	81	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x83DE)
	82
	83	// ptrauth_string_discriminator("unw_proc_info_t::flags") == 0x79A1
	84	#define __ptrauth_unwind_upi_flags \
	85	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x79A1)
	86
	87	// ptrauth_string_discriminator("unw_proc_info_t::unwind_info") == 0xC20C
	88	#define __ptrauth_unwind_upi_info \
	89	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0xC20C)
	90
	91	// ptrauth_string_discriminator("unw_proc_info_t::extra") == 0x03DF
	92	#define __ptrauth_unwind_upi_extra \
	93	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x03DF)
	94
	95	// ptrauth_string_discriminator("Registers_arm64::link_reg_t") == 0x8301
	96	#define __ptrauth_unwind_registers_arm64_link_reg \
	97	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_code, 1, 0x8301)
	98
	99	// ptrauth_string_discriminator("UnwindInfoSections::dso_base") == 0x4FF5
	100	#define __ptrauth_unwind_uis_dso_base \
	101	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x4FF5)
	102
	103	// ptrauth_string_discriminator("UnwindInfoSections::dwarf_section") == 0x4974
	104	#define __ptrauth_unwind_uis_dwarf_section \
	105	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x4974)
	106
	107	// ptrauth_string_discriminator("UnwindInfoSections::dwarf_section_length") == 0x2A9A
	108	#define __ptrauth_unwind_uis_dwarf_section_length \
	109	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x2A9A)
	110
	111	// ptrauth_string_discriminator("UnwindInfoSections::compact_unwind_section") == 0xA27B
	112	#define __ptrauth_unwind_uis_compact_unwind_section \
	113	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0xA27B)
	114
	115	// ptrauth_string_discriminator("UnwindInfoSections::compact_unwind_section_length") == 0x5D0A
	116	#define __ptrauth_unwind_uis_compact_unwind_section_length \
	117	__unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x5D0A)
	118
	119	// ptrauth_string_discriminator("CIE_Info::personality") == 0x6A40
	120	#define __ptrauth_unwind_cie_info_personality_disc 0x6A40
	121	#define __ptrauth_unwind_cie_info_personality \
	122	__unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer, 1, \
	123	__ptrauth_unwind_cie_info_personality_disc)
	124
	125	// ptrauth_string_discriminator("personality") == 0x7EAD)
	126	#define __ptrauth_unwind_pauthtest_personality_disc 0x7EAD
	127
	128	#else
	129
	130	#define __unwind_ptrauth_restricted_intptr(...)
	131	#define __ptrauth_unwind_upi_handler
	132	#define __ptrauth_unwind_upi_handler_intptr
	133	#define __ptrauth_unwind_upi_startip
	134	#define __ptrauth_unwind_upi_endip
	135	#define __ptrauth_unwind_upi_lsda
	136	#define __ptrauth_unwind_upi_flags
	137	#define __ptrauth_unwind_upi_info
	138	#define __ptrauth_unwind_upi_extra
	139	#define __ptrauth_unwind_registers_arm64_link_reg
	140	#define __ptrauth_unwind_uis_dso_base
	141	#define __ptrauth_unwind_uis_dwarf_section
	142	#define __ptrauth_unwind_uis_dwarf_section_length
	143	#define __ptrauth_unwind_uis_compact_unwind_section
	144	#define __ptrauth_unwind_uis_compact_unwind_section_length
	145	#define __ptrauth_unwind_cie_info_personality
	146
	147	#endif
	148
46	149	#if defined(_WIN32) && defined(__SEH__)
47	150	#define LIBUNWIND_CURSOR_ALIGNMENT_ATTR __attribute__((__aligned__(16)))
48	151	#else
...	...	@@ -88,17 +191,18 @@ typedef double unw_fpreg_t;
88	191	#endif
89	192
90	193	struct unw_proc_info_t {
91		unw_word_t start_ip; /* start address of function */
92		unw_word_t end_ip; /* address after end of function */
93		unw_word_t lsda; /* address of language specific data area, */
94		/* or zero if not used */
95		unw_word_t handler; /* personality routine, or zero if not used */
96		unw_word_t gp; /* not used */
97		unw_word_t flags; /* not used */
98		uint32_t format; /* compact unwind encoding, or zero if none */
99		uint32_t unwind_info_size; /* size of DWARF unwind info, or zero if none */
100		unw_word_t unwind_info; /* address of DWARF unwind info, or zero */
101		unw_word_t extra; /* mach_header of mach-o image containing func */
	194	unw_word_t __ptrauth_unwind_upi_startip start_ip; /* start address of function */
	195	unw_word_t __ptrauth_unwind_upi_endip end_ip; /* address after end of function */
	196	unw_word_t __ptrauth_unwind_upi_lsda lsda; /* address of language specific data area, */
	197	/* or zero if not used */
	198
	199	unw_word_t __ptrauth_unwind_upi_handler_intptr handler;
	200	unw_word_t gp; /* not used */
	201	unw_word_t __ptrauth_unwind_upi_flags flags; /* not used */
	202	uint32_t format; /* compact unwind encoding, or zero if none */
	203	uint32_t unwind_info_size; /* size of DWARF unwind info, or zero if none */
	204	unw_word_t __ptrauth_unwind_upi_info unwind_info; /* address of DWARF unwind info, or zero */
	205	unw_word_t __ptrauth_unwind_upi_extra extra; /* mach_header of mach-o image containing func */
102	206	};
103	207	typedef struct unw_proc_info_t unw_proc_info_t;
104	208
...	...	@@ -130,6 +234,7 @@ extern int unw_is_fpreg(unw_cursor_t *, unw_regnum_t) LIBUNWIND_AVAIL;
130	234	extern int unw_is_signal_frame(unw_cursor_t *) LIBUNWIND_AVAIL;
131	235	extern int unw_get_proc_name(unw_cursor_t , char , size_t, unw_word_t *) LIBUNWIND_AVAIL;
132	236	//extern int unw_get_save_loc(unw_cursor_t, int, unw_save_loc_t);
	237	extern const char *unw_strerror(int) LIBUNWIND_AVAIL;
133	238
134	239	extern unw_addr_space_t unw_local_addr_space;
135	240
...	...	@@ -532,6 +637,7 @@ enum {
532	637	UNW_AARCH64_X31 = 31,
533	638	UNW_AARCH64_SP = 31,
534	639	UNW_AARCH64_PC = 32,
	640	UNW_AARCH64_VG = 46,
535	641
536	642	// reserved block
537	643	UNW_AARCH64_RA_SIGN_STATE = 34,

lib/libunwind/include/unwind_arm_ehabi.h+4-1

...	...	@@ -125,8 +125,11 @@ _Unwind_VRS_Pop(_Unwind_Context *context, _Unwind_VRS_RegClass regclass,
125	125	uint32_t discriminator,
126	126	_Unwind_VRS_DataRepresentation representation);
127	127
	128	extern _Unwind_Reason_Code __gnu_unwind_frame(_Unwind_Exception *,
	129	_Unwind_Context *);
	130
128	131	#if defined(_LIBUNWIND_UNWIND_LEVEL1_EXTERNAL_LINKAGE)
129		#define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 extern
	132	#define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 extern __inline__
130	133	#else
131	134	#define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 static __inline__
132	135	#endif

lib/libunwind/src/AddressSpace.hpp+40-19

...	...	@@ -129,22 +129,27 @@ struct UnwindInfoSections {
129	129	defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) \|\| \
130	130	defined(_LIBUNWIND_USE_DL_ITERATE_PHDR)
131	131	// No dso_base for SEH.
132		uintptr_t dso_base;
	132	uintptr_t __ptrauth_unwind_uis_dso_base
	133	dso_base = 0;
133	134	#endif
134	135	#if defined(_LIBUNWIND_USE_DL_ITERATE_PHDR)
135	136	size_t text_segment_length;
136	137	#endif
137	138	#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
138		uintptr_t dwarf_section;
139		size_t dwarf_section_length;
	139	uintptr_t __ptrauth_unwind_uis_dwarf_section
	140	dwarf_section = 0;
	141	size_t __ptrauth_unwind_uis_dwarf_section_length
	142	dwarf_section_length = 0;
140	143	#endif
141	144	#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX)
142	145	uintptr_t dwarf_index_section;
143	146	size_t dwarf_index_section_length;
144	147	#endif
145	148	#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND)
146		uintptr_t compact_unwind_section;
147		size_t compact_unwind_section_length;
	149	uintptr_t __ptrauth_unwind_uis_compact_unwind_section
	150	compact_unwind_section = 0;
	151	size_t __ptrauth_unwind_uis_compact_unwind_section_length
	152	compact_unwind_section_length = 0;
148	153	#endif
149	154	#if defined(_LIBUNWIND_ARM_EHABI)
150	155	uintptr_t arm_section;
...	...	@@ -196,11 +201,16 @@ public:
196	201	static int64_t getSLEB128(pint_t &addr, pint_t end);
197	202
198	203	pint_t getEncodedP(pint_t &addr, pint_t end, uint8_t encoding,
199		pint_t datarelBase = 0);
200		bool findFunctionName(pint_t addr, char *buf, size_t bufLen,
201		unw_word_t *offset);
202		bool findUnwindSections(pint_t targetAddr, UnwindInfoSections &info);
203		bool findOtherFDE(pint_t targetAddr, pint_t &fde);
	204	pint_t datarelBase = 0, pint_t *resultAddr = nullptr);
	205	template <typename R>
	206	bool findFunctionName(typename R::link_hardened_reg_arg_t addr, char *buf,
	207	size_t bufLen, unw_word_t *offset);
	208	template <typename R>
	209	bool findUnwindSections(typename R::link_hardened_reg_arg_t targetAddr,
	210	UnwindInfoSections &info);
	211	template <typename R>
	212	bool findOtherFDE(typename R::link_hardened_reg_arg_t targetAddr,
	213	pint_t &fde);
204	214
205	215	static LocalAddressSpace sThisAddressSpace;
206	216	};
...	...	@@ -269,7 +279,7 @@ inline int64_t LocalAddressSpace::getSLEB128(pint_t &addr, pint_t end) {
269	279
270	280	inline LocalAddressSpace::pint_t
271	281	LocalAddressSpace::getEncodedP(pint_t &addr, pint_t end, uint8_t encoding,
272		pint_t datarelBase) {
	282	pint_t datarelBase, pint_t *resultAddr) {
273	283	pint_t startAddr = addr;
274	284	const uint8_t p = (uint8_t )addr;
275	285	pint_t result;
...	...	@@ -353,8 +363,14 @@ LocalAddressSpace::getEncodedP(pint_t &addr, pint_t end, uint8_t encoding,
353	363	break;
354	364	}
355	365
356		if (encoding & DW_EH_PE_indirect)
	366	if (encoding & DW_EH_PE_indirect) {
	367	if (resultAddr)
	368	*resultAddr = result;
357	369	result = getP(result);
	370	} else {
	371	if (resultAddr)
	372	*resultAddr = startAddr;
	373	}
358	374
359	375	return result;
360	376	}
...	...	@@ -486,9 +502,9 @@ static int findUnwindSectionsByPhdr(struct dl_phdr_info *pinfo,
486	502
487	503	#endif // defined(_LIBUNWIND_USE_DL_ITERATE_PHDR)
488	504
489
490		inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
491		UnwindInfoSections &info) {
	505	template <typename R>
	506	inline bool LocalAddressSpace::findUnwindSections(
	507	typename R::link_hardened_reg_arg_t targetAddr, UnwindInfoSections &info) {
492	508	#ifdef __APPLE__
493	509	dyld_unwind_sections dyldInfo;
494	510	if (_dyld_find_unwind_sections((void *)targetAddr, &dyldInfo)) {
...	...	@@ -658,16 +674,21 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
658	674	return false;
659	675	}
660	676
661		inline bool LocalAddressSpace::findOtherFDE(pint_t targetAddr, pint_t &fde) {
	677	template <typename R>
	678	inline bool
	679	LocalAddressSpace::findOtherFDE(typename R::link_hardened_reg_arg_t targetAddr,
	680	pint_t &fde) {
662	681	// TO DO: if OS has way to dynamically register FDEs, check that.
663	682	(void)targetAddr;
664	683	(void)fde;
665	684	return false;
666	685	}
667	686
668		inline bool LocalAddressSpace::findFunctionName(pint_t addr, char *buf,
669		size_t bufLen,
670		unw_word_t *offset) {
	687	template <typename R>
	688	inline bool
	689	LocalAddressSpace::findFunctionName(typename R::link_hardened_reg_arg_t addr,
	690	char *buf, size_t bufLen,
	691	unw_word_t *offset) {
671	692	#if _LIBUNWIND_USE_DLADDR
672	693	Dl_info dyldInfo;
673	694	if (dladdr((void *)addr, &dyldInfo)) {

lib/libunwind/src/CompactUnwinder.hpp+16-5

...	...	@@ -601,11 +601,17 @@ int CompactUnwinder_arm64<A>::stepWithCompactEncodingFrameless(
601	601	savedRegisterLoc -= 8;
602	602	}
603	603
	604	// We load the link register prior to setting the new SP as the authentication
	605	// schema for LR entangles the SP of the old frame into the diversifier.
	606	Registers_arm64::reg_t linkRegister = registers.getRegister(UNW_AARCH64_LR);
	607
604	608	// subtract stack size off of sp
605	609	registers.setSP(savedRegisterLoc);
606	610
607		// set pc to be value in lr
608		registers.setIP(registers.getRegister(UNW_AARCH64_LR));
	611	// Set pc to be value in lr. This needs to be performed after the new SP has
	612	// been set, as the PC authentication schema entangles the SP of the new
	613	// frame.
	614	registers.setIP(linkRegister);
609	615
610	616	return UNW_STEP_SUCCESS;
611	617	}
...	...	@@ -614,7 +620,7 @@ template <typename A>
614	620	int CompactUnwinder_arm64<A>::stepWithCompactEncodingFrame(
615	621	compact_unwind_encoding_t encoding, uint64_t, A &addressSpace,
616	622	Registers_arm64 &registers) {
617		uint64_t savedRegisterLoc = registers.getFP() - 8;
	623	Registers_arm64::reg_t savedRegisterLoc = registers.getFP() - 8;
618	624
619	625	if (encoding & UNWIND_ARM64_FRAME_X19_X20_PAIR) {
620	626	registers.setRegister(UNW_AARCH64_X19, addressSpace.get64(savedRegisterLoc));
...	...	@@ -680,11 +686,16 @@ int CompactUnwinder_arm64<A>::stepWithCompactEncodingFrame(
680	686	savedRegisterLoc -= 8;
681	687	}
682	688
683		uint64_t fp = registers.getFP();
	689	Registers_arm64::reg_t fp = registers.getFP();
	690
684	691	// fp points to old fp
685	692	registers.setFP(addressSpace.get64(fp));
686		// old sp is fp less saved fp and lr
	693
	694	// Old sp is fp less saved fp and lr. We need to set this prior to setting
	695	// the lr as the pointer authentication schema for the lr incorporates the
	696	// sp as part of the diversifier.
687	697	registers.setSP(fp + 16);
	698
688	699	// pop return address into pc
689	700	registers.setIP(addressSpace.get64(fp + 8));
690	701

lib/libunwind/src/DwarfInstructions.hpp+24-13

...	...	@@ -22,7 +22,6 @@
22	22	#include "dwarf2.h"
23	23	#include "libunwind_ext.h"
24	24
25
26	25	namespace libunwind {
27	26
28	27
...	...	@@ -34,8 +33,10 @@ public:
34	33	typedef typename A::pint_t pint_t;
35	34	typedef typename A::sint_t sint_t;
36	35
37		static int stepWithDwarf(A &addressSpace, pint_t pc, pint_t fdeStart,
38		R &registers, bool &isSignalFrame, bool stage2);
	36	static int stepWithDwarf(A &addressSpace,
	37	typename R::link_hardened_reg_arg_t pc,
	38	pint_t fdeStart, R &registers, bool &isSignalFrame,
	39	bool stage2);
39	40
40	41	private:
41	42
...	...	@@ -64,9 +65,10 @@ private:
64	65
65	66	static pint_t getCFA(A &addressSpace, const PrologInfo &prolog,
66	67	const R &registers) {
67		if (prolog.cfaRegister != 0)
68		return (pint_t)((sint_t)registers.getRegister((int)prolog.cfaRegister) +
69		prolog.cfaRegisterOffset);
	68	if (prolog.cfaRegister != 0) {
	69	uintptr_t cfaRegister = registers.getRegister((int)prolog.cfaRegister);
	70	return (pint_t)(cfaRegister + prolog.cfaRegisterOffset);
	71	}
70	72	if (prolog.cfaExpression != 0)
71	73	return evaluateExpression((pint_t)prolog.cfaExpression, addressSpace,
72	74	registers, 0);
...	...	@@ -207,16 +209,16 @@ bool DwarfInstructions<A, R>::isReturnAddressSignedWithPC(A &addressSpace,
207	209	#endif
208	210
209	211	template <typename A, typename R>
210		int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
211		pint_t fdeStart, R &registers,
212		bool &isSignalFrame, bool stage2) {
	212	int DwarfInstructions<A, R>::stepWithDwarf(
	213	A &addressSpace, typename R::link_hardened_reg_arg_t pc, pint_t fdeStart,
	214	R &registers, bool &isSignalFrame, bool stage2) {
213	215	FDE_Info fdeInfo;
214	216	CIE_Info cieInfo;
215	217	if (CFI_Parser<A>::decodeFDE(addressSpace, fdeStart, &fdeInfo,
216	218	&cieInfo) == NULL) {
217	219	PrologInfo prolog;
218		if (CFI_Parser<A>::parseFDEInstructions(addressSpace, fdeInfo, cieInfo, pc,
219		R::getArch(), &prolog)) {
	220	if (CFI_Parser<A>::template parseFDEInstructions<R>(
	221	addressSpace, fdeInfo, cieInfo, pc, R::getArch(), &prolog)) {
220	222	// get pointer to cfa (architecture specific)
221	223	pint_t cfa = getCFA(addressSpace, prolog, registers);
222	224
...	...	@@ -264,7 +266,7 @@ int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
264	266	// by a CFI directive later on.
265	267	newRegisters.setSP(cfa);
266	268
267		pint_t returnAddress = 0;
	269	typename R::reg_t returnAddress = 0;
268	270	constexpr int lastReg = R::lastDwarfRegNum();
269	271	static_assert(static_cast<int>(CFI_Parser<A>::kMaxRegisterNumber) >=
270	272	lastReg,
...	...	@@ -300,7 +302,16 @@ int DwarfInstructions<A, R>::stepWithDwarf(A &addressSpace, pint_t pc,
300	302
301	303	isSignalFrame = cieInfo.isSignalFrame;
302	304
303		#if defined(_LIBUNWIND_TARGET_AARCH64)
	305	#if defined(_LIBUNWIND_TARGET_AARCH64) && \
	306	!defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	307	// There are two ways of return address signing: pac-ret (enabled via
	308	// -mbranch-protection=pac-ret) and ptrauth-returns (enabled as part of
	309	// Apple's arm64e or experimental pauthtest ABI on Linux). The code
	310	// below handles signed RA for pac-ret, while ptrauth-returns uses
	311	// different logic.
	312	// TODO: unify logic for both cases, see
	313	// https://github.com/llvm/llvm-project/issues/160110
	314	//
304	315	// If the target is aarch64 then the return address may have been signed
305	316	// using the v8.3 pointer authentication extensions. The original
306	317	// return address needs to be authenticated before the return address is

lib/libunwind/src/DwarfParser.hpp+60-22

...	...	@@ -23,6 +23,10 @@
23	23
24	24	#include "config.h"
25	25
	26	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	27	#include <ptrauth.h>
	28	#endif
	29
26	30	namespace libunwind {
27	31
28	32	/// CFI_Parser does basic parsing of a CFI (Call Frame Information) records.
...	...	@@ -33,6 +37,7 @@ template <typename A>
33	37	class CFI_Parser {
34	38	public:
35	39	typedef typename A::pint_t pint_t;
	40	typedef pint_t __ptrauth_unwind_cie_info_personality personality_t;
36	41
37	42	/// Information encoded in a CIE (Common Information Entry)
38	43	struct CIE_Info {
...	...	@@ -43,7 +48,7 @@ public:
43	48	uint8_t lsdaEncoding;
44	49	uint8_t personalityEncoding;
45	50	uint8_t personalityOffsetInCIE;
46		pint_t personality;
	51	personality_t personality;
47	52	uint32_t codeAlignFactor;
48	53	int dataAlignFactor;
49	54	bool isSignalFrame;
...	...	@@ -155,14 +160,17 @@ public:
155	160	}
156	161	};
157	162
158		static bool findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart,
159		size_t sectionLength, pint_t fdeHint, FDE_Info *fdeInfo,
160		CIE_Info *cieInfo);
	163	template <typename R>
	164	static bool findFDE(A &addressSpace, typename R::link_hardened_reg_arg_t pc,
	165	pint_t ehSectionStart, size_t sectionLength,
	166	pint_t fdeHint, FDE_Info fdeInfo, CIE_Info cieInfo);
161	167	static const char *decodeFDE(A &addressSpace, pint_t fdeStart,
162	168	FDE_Info fdeInfo, CIE_Info cieInfo,
163	169	bool useCIEInfo = false);
	170	template <typename R>
164	171	static bool parseFDEInstructions(A &addressSpace, const FDE_Info &fdeInfo,
165		const CIE_Info &cieInfo, pint_t upToPC,
	172	const CIE_Info &cieInfo,
	173	typename R::link_hardened_reg_arg_t upToPC,
166	174	int arch, PrologInfo *results);
167	175
168	176	static const char parseCIE(A &addressSpace, pint_t cie, CIE_Info cieInfo);
...	...	@@ -234,9 +242,12 @@ const char *CFI_Parser<A>::decodeFDE(A &addressSpace, pint_t fdeStart,
234	242
235	243	/// Scan an eh_frame section to find an FDE for a pc
236	244	template <typename A>
237		bool CFI_Parser<A>::findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart,
238		size_t sectionLength, pint_t fdeHint,
239		FDE_Info fdeInfo, CIE_Info cieInfo) {
	245	template <typename R>
	246	bool CFI_Parser<A>::findFDE(A &addressSpace,
	247	typename R::link_hardened_reg_arg_t pc,
	248	pint_t ehSectionStart, size_t sectionLength,
	249	pint_t fdeHint, FDE_Info *fdeInfo,
	250	CIE_Info *cieInfo) {
240	251	//fprintf(stderr, "findFDE(0x%llX)\n", (long long)pc);
241	252	pint_t p = (fdeHint != 0) ? fdeHint : ehSectionStart;
242	253	const pint_t ehSectionEnd = (sectionLength == SIZE_MAX)
...	...	@@ -273,7 +284,7 @@ bool CFI_Parser<A>::findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart,
273	284	pint_t pcRange = addressSpace.getEncodedP(
274	285	p, nextCFI, cieInfo->pointerEncoding & 0x0F);
275	286	// Test if pc is within the function this FDE covers.
276		if ((pcStart < pc) && (pc <= pcStart + pcRange)) {
	287	if ((pcStart <= pc) && (pc < pcStart + pcRange)) {
277	288	// parse rest of info
278	289	fdeInfo->lsda = 0;
279	290	// check for augmentation length
...	...	@@ -369,6 +380,7 @@ const char *CFI_Parser<A>::parseCIE(A &addressSpace, pint_t cie,
369	380	cieInfo->returnAddressRegister = (uint8_t)raReg;
370	381	// parse augmentation data based on augmentation string
371	382	const char *result = NULL;
	383	pint_t resultAddr = 0;
372	384	if (addressSpace.get8(strStart) == 'z') {
373	385	// parse augmentation data length
374	386	addressSpace.getULEB128(p, cieContentEnd);
...	...	@@ -377,13 +389,41 @@ const char *CFI_Parser<A>::parseCIE(A &addressSpace, pint_t cie,
377	389	case 'z':
378	390	cieInfo->fdesHaveAugmentationData = true;
379	391	break;
380		case 'P':
	392	case 'P': {
381	393	cieInfo->personalityEncoding = addressSpace.get8(p);
382	394	++p;
383	395	cieInfo->personalityOffsetInCIE = (uint8_t)(p - cie);
384		cieInfo->personality = addressSpace
385		.getEncodedP(p, cieContentEnd, cieInfo->personalityEncoding);
	396	pint_t personality = addressSpace.getEncodedP(
	397	p, cieContentEnd, cieInfo->personalityEncoding,
	398	/datarelBase=/0, &resultAddr);
	399	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	400	if (personality) {
	401	// The GOT for the personality function was signed address
	402	// authenticated. Manually re-sign with the CIE_Info::personality
	403	// schema. If we could guarantee the encoding of the personality we
	404	// could avoid this by simply giving resultAddr the correct ptrauth
	405	// schema and performing an assignment.
	406	#if defined(__arm64e__)
	407	const auto oldDiscriminator = resultAddr;
	408	#else
	409	const auto oldDiscriminator = ptrauth_blend_discriminator(
	410	(void *)resultAddr, __ptrauth_unwind_pauthtest_personality_disc);
	411	#endif
	412	const auto discriminator = ptrauth_blend_discriminator(
	413	&cieInfo->personality,
	414	__ptrauth_unwind_cie_info_personality_disc);
	415	void *signedPtr = ptrauth_auth_and_resign(
	416	(void *)personality, ptrauth_key_function_pointer,
	417	oldDiscriminator, ptrauth_key_function_pointer, discriminator);
	418	personality = (pint_t)signedPtr;
	419	}
	420	#endif
	421	// We use memmove to set the CIE personality as we have already
	422	// re-signed the pointer to the correct schema.
	423	memmove((void )&cieInfo->personality, (void )&personality,
	424	sizeof(personality));
386	425	break;
	426	}
387	427	case 'L':
388	428	cieInfo->lsdaEncoding = addressSpace.get8(p);
389	429	++p;
...	...	@@ -417,10 +457,10 @@ const char *CFI_Parser<A>::parseCIE(A &addressSpace, pint_t cie,
417	457
418	458	/// "run" the DWARF instructions and create the abstract PrologInfo for an FDE
419	459	template <typename A>
420		bool CFI_Parser<A>::parseFDEInstructions(A &addressSpace,
421		const FDE_Info &fdeInfo,
422		const CIE_Info &cieInfo, pint_t upToPC,
423		int arch, PrologInfo *results) {
	460	template <typename R>
	461	bool CFI_Parser<A>::parseFDEInstructions(
	462	A &addressSpace, const FDE_Info &fdeInfo, const CIE_Info &cieInfo,
	463	typename R::link_hardened_reg_arg_t upToPC, int arch, PrologInfo *results) {
424	464	// Alloca is used for the allocation of the rememberStack entries. It removes
425	465	// the dependency on new/malloc but the below for loop can not be refactored
426	466	// into functions. Entry could be saved during the processing of a CIE and
...	...	@@ -808,12 +848,10 @@ bool CFI_Parser<A>::parseFDEInstructions(A &addressSpace,
808	848	results->savedRegisters[UNW_AARCH64_RA_SIGN_STATE].value ^ 0x3;
809	849	results->setRegisterValue(UNW_AARCH64_RA_SIGN_STATE, value,
810	850	initialState);
811		// When calculating the value of the PC, it is assumed that the CFI
812		// instruction is placed before the signing instruction, however it is
813		// placed after. Because of this, we need to take into account the CFI
814		// instruction is one instruction call later than expected, and reduce
815		// the PC value by 4 bytes to compensate.
816		results->ptrAuthDiversifier = fdeInfo.pcStart + codeOffset - 0x4;
	851	// When using Feat_PAuthLR, the PC value needs to be captured so that
	852	// during unwinding, the correct PC value is used for re-authentication.
	853	// It is assumed that the CFI is placed before the signing instruction.
	854	results->ptrAuthDiversifier = fdeInfo.pcStart + codeOffset;
817	855	_LIBUNWIND_TRACE_DWARF(
818	856	"DW_CFA_AARCH64_negate_ra_state_with_pc(pc=0x%" PRIx64 ")\n",
819	857	static_cast<uint64_t>(results->ptrAuthDiversifier));

lib/libunwind/src/EHHeaderParser.hpp+7-4

...	...	@@ -37,8 +37,9 @@ public:
37	37
38	38	static bool decodeEHHdr(A &addressSpace, pint_t ehHdrStart, pint_t ehHdrEnd,
39	39	EHHeaderInfo &ehHdrInfo);
40		static bool findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart,
41		uint32_t sectionLength,
	40	template <typename R>
	41	static bool findFDE(A &addressSpace, typename R::link_hardened_reg_arg_t pc,
	42	pint_t ehHdrStart, uint32_t sectionLength,
42	43	typename CFI_Parser<A>::FDE_Info *fdeInfo,
43	44	typename CFI_Parser<A>::CIE_Info *cieInfo);
44	45
...	...	@@ -112,8 +113,10 @@ bool EHHeaderParser<A>::decodeTableEntry(
112	113	}
113	114
114	115	template <typename A>
115		bool EHHeaderParser<A>::findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart,
116		uint32_t sectionLength,
	116	template <typename R>
	117	bool EHHeaderParser<A>::findFDE(A &addressSpace,
	118	typename R::link_hardened_reg_arg_t pc,
	119	pint_t ehHdrStart, uint32_t sectionLength,
117	120	typename CFI_Parser<A>::FDE_Info *fdeInfo,
118	121	typename CFI_Parser<A>::CIE_Info *cieInfo) {
119	122	pint_t ehHdrEnd = ehHdrStart + sectionLength;

lib/libunwind/src/Registers.hpp+226-22

...	...	@@ -17,8 +17,13 @@
17	17
18	18	#include "config.h"
19	19	#include "libunwind.h"
	20	#include "libunwind_ext.h"
20	21	#include "shadow_stack_unwind.h"
21	22
	23	#if defined(_LIBUNWIND_HAVE_GETAUXVAL) \|\| defined(_LIBUNWIND_HAVE_ELF_AUX_INFO)
	24	#include <sys/auxv.h>
	25	#endif
	26
22	27	namespace libunwind {
23	28
24	29	// For emulating 128-bit registers
...	...	@@ -60,6 +65,10 @@ public:
60	65	Registers_x86();
61	66	Registers_x86(const void *registers);
62	67
	68	typedef uint32_t reg_t;
	69	typedef uint32_t link_reg_t;
	70	typedef const link_reg_t &link_hardened_reg_arg_t;
	71
63	72	bool validRegister(int num) const;
64	73	uint32_t getRegister(int num) const;
65	74	void setRegister(int num, uint32_t value);
...	...	@@ -278,6 +287,10 @@ public:
278	287	Registers_x86_64();
279	288	Registers_x86_64(const void *registers);
280	289
	290	typedef uint64_t reg_t;
	291	typedef uint64_t link_reg_t;
	292	typedef const link_reg_t &link_hardened_reg_arg_t;
	293
281	294	bool validRegister(int num) const;
282	295	uint64_t getRegister(int num) const;
283	296	void setRegister(int num, uint64_t value);
...	...	@@ -597,6 +610,10 @@ public:
597	610	Registers_ppc();
598	611	Registers_ppc(const void *registers);
599	612
	613	typedef uint32_t reg_t;
	614	typedef uint32_t link_reg_t;
	615	typedef const link_reg_t &link_hardened_reg_arg_t;
	616
600	617	bool validRegister(int num) const;
601	618	uint32_t getRegister(int num) const;
602	619	void setRegister(int num, uint32_t value);
...	...	@@ -1169,6 +1186,10 @@ public:
1169	1186	Registers_ppc64();
1170	1187	Registers_ppc64(const void *registers);
1171	1188
	1189	typedef uint64_t reg_t;
	1190	typedef uint64_t link_reg_t;
	1191	typedef const link_reg_t &link_hardened_reg_arg_t;
	1192
1172	1193	bool validRegister(int num) const;
1173	1194	uint64_t getRegister(int num) const;
1174	1195	void setRegister(int num, uint64_t value);
...	...	@@ -1814,7 +1835,9 @@ inline const char *Registers_ppc64::getRegisterName(int regNum) {
1814	1835	/// Registers_arm64 holds the register state of a thread in a 64-bit arm
1815	1836	/// process.
1816	1837	class _LIBUNWIND_HIDDEN Registers_arm64;
1817		extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *);
	1838	extern "C" int64_t __libunwind_Registers_arm64_za_disable();
	1839	extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *,
	1840	unsigned walkedFrames);
1818	1841
1819	1842	#if defined(_LIBUNWIND_USE_GCS)
1820	1843	extern "C" void *__libunwind_shstk_get_jump_target() {
...	...	@@ -1824,8 +1847,21 @@ extern "C" void *__libunwind_shstk_get_jump_target() {
1824	1847
1825	1848	class _LIBUNWIND_HIDDEN Registers_arm64 {
1826	1849	public:
1827		Registers_arm64();
	1850	Registers_arm64() = default;
1828	1851	Registers_arm64(const void *registers);
	1852	Registers_arm64(const Registers_arm64 &);
	1853	Registers_arm64 &operator=(const Registers_arm64 &);
	1854
	1855	typedef uint64_t reg_t;
	1856	typedef uint64_t __ptrauth_unwind_registers_arm64_link_reg link_reg_t;
	1857
	1858	// Use `link_hardened_reg_arg_t` to pass values of `link_reg_t` type as
	1859	// function arguments. We need to use a const l-value reference to keep
	1860	// signature of `__ptrauth`-qualified values of `link_reg_t` type on AArch64
	1861	// PAuth-enabled ABI intact. Passing the raw pointer by value would cause
	1862	// authentication on the caller side and make the pointer prone to
	1863	// substitution if spilled to the stack in the callee.
	1864	typedef const link_reg_t &link_hardened_reg_arg_t;
1829	1865
1830	1866	bool validRegister(int num) const;
1831	1867	uint64_t getRegister(int num) const;
...	...	@@ -1837,7 +1873,14 @@ public:
1837	1873	v128 getVectorRegister(int num) const;
1838	1874	void setVectorRegister(int num, v128 value);
1839	1875	static const char *getRegisterName(int num);
1840		void jumpto() { __libunwind_Registers_arm64_jumpto(this); }
	1876	void jumpto(unsigned walkedFrames = 0) {
	1877	zaDisable();
	1878	__libunwind_Registers_arm64_jumpto(this, walkedFrames);
	1879	}
	1880	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	1881	_LIBUNWIND_TRACE_NO_INLINE
	1882	void returnto(unsigned walkedFrames) { jumpto(walkedFrames); }
	1883	#endif
1841	1884	static constexpr int lastDwarfRegNum() {
1842	1885	return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64;
1843	1886	}
...	...	@@ -1845,27 +1888,104 @@ public:
1845	1888
1846	1889	uint64_t getSP() const { return _registers.__sp; }
1847	1890	void setSP(uint64_t value) { _registers.__sp = value; }
1848		uint64_t getIP() const { return _registers.__pc; }
1849		void setIP(uint64_t value) { _registers.__pc = value; }
1850		uint64_t getFP() const { return _registers.__fp; }
1851		void setFP(uint64_t value) { _registers.__fp = value; }
	1891	uint64_t getIP() const {
	1892	uint64_t value = _registers.__pc;
	1893	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	1894	// Note the value of the PC was signed to its address in the register state
	1895	// but everyone else expects it to be sign by the SP, so convert on return.
	1896	value = (uint64_t)ptrauth_auth_and_resign((void *)_registers.__pc,
	1897	ptrauth_key_return_address,
	1898	&_registers.__pc,
	1899	ptrauth_key_return_address,
	1900	getSP());
	1901	#endif
	1902	return value;
	1903	}
	1904	void setIP(uint64_t value) {
	1905	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	1906	// Note the value which was set should have been signed with the SP.
	1907	// We then resign with the slot we are being stored in to so that both SP
	1908	// and LR can't be spoofed at the same time.
	1909	value = (uint64_t)ptrauth_auth_and_resign((void *)value,
	1910	ptrauth_key_return_address,
	1911	getSP(),
	1912	ptrauth_key_return_address,
	1913	&_registers.__pc);
	1914	#endif
	1915	_registers.__pc = value;
	1916	}
	1917	uint64_t getFP() const { return _registers.__fp; }
	1918	void setFP(uint64_t value) { _registers.__fp = value; }
	1919
	1920	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	1921	void
	1922	loadAndAuthenticateLinkRegister(reg_t inplaceAuthedLinkRegister,
	1923	link_reg_t *referenceAuthedLinkRegister) {
	1924	// If we are in an arm64/arm64e frame, then the PC should have been signed
	1925	// with the SP
	1926	*referenceAuthedLinkRegister =
	1927	(uint64_t)ptrauth_auth_data((void *)inplaceAuthedLinkRegister,
	1928	ptrauth_key_return_address,
	1929	_registers.__sp);
	1930	}
	1931	#endif
1852	1932
1853	1933	private:
	1934	uint64_t lazyGetVG() const;
	1935
	1936	void zaDisable() const {
	1937	if (!_misc_registers.__has_sme)
	1938	return;
	1939	if (__libunwind_Registers_arm64_za_disable() != 0)
	1940	_LIBUNWIND_ABORT("SME ZA disable failed");
	1941	}
	1942
	1943	#if defined(_LIBUNWIND_HAVE_GETAUXVAL)
	1944	static bool checkHasSME() {
	1945	constexpr int hwcap2_sme = (1 << 23);
	1946	unsigned long hwcap2 = getauxval(AT_HWCAP2);
	1947	return (hwcap2 & hwcap2_sme) != 0;
	1948	}
	1949	#elif defined(_LIBUNWIND_HAVE_ELF_AUX_INFO)
	1950	static bool checkHasSME() {
	1951	constexpr int hwcap2_sme = (1 << 23);
	1952	unsigned long hwcap2 = 0;
	1953	elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
	1954	return (hwcap2 & hwcap2_sme) != 0;
	1955	}
	1956	#else
	1957	static bool checkHasSME() {
	1958	// TODO: Support other platforms.
	1959	return false;
	1960	}
	1961	#endif
	1962
1854	1963	struct GPRs {
1855		uint64_t __x[29]; // x0-x28
1856		uint64_t __fp; // Frame pointer x29
1857		uint64_t __lr; // Link register x30
1858		uint64_t __sp; // Stack pointer x31
1859		uint64_t __pc; // Program counter
1860		uint64_t __ra_sign_state; // RA sign state register
	1964	uint64_t __x[29] = {}; // x0-x28
	1965	uint64_t __fp = 0; // Frame pointer x29
	1966	uint64_t __lr = 0; // Link register x30
	1967	uint64_t __sp = 0; // Stack pointer x31
	1968	uint64_t __pc = 0; // Program counter
	1969	uint64_t __ra_sign_state = 0; // RA sign state register
	1970	};
	1971
	1972	struct Misc {
	1973	mutable uint32_t __vg = 0; // Vector Granule
	1974	bool __has_sme = checkHasSME();
1861	1975	};
1862	1976
1863		GPRs _registers;
1864		double _vectorHalfRegisters[32];
	1977	GPRs _registers = {};
1865	1978	// Currently only the lower double in 128-bit vectore registers
1866	1979	// is perserved during unwinding. We could define new register
1867	1980	// numbers (> 96) which mean whole vector registers, then this
1868	1981	// struct would need to change to contain whole vector registers.
	1982	double _vectorHalfRegisters[32] = {};
	1983
	1984	// Miscellaneous/virtual registers. These are stored below the GPRs and FPRs
	1985	// as they do not correspond to physical registers, so do not need to be
	1986	// saved/restored in UnwindRegistersRestore.S and UnwindRegistersSave.S, and
	1987	// we don't want to modify the existing offsets for GPRs and FPRs.
	1988	Misc _misc_registers;
1869	1989	};
1870	1990
1871	1991	inline Registers_arm64::Registers_arm64(const void *registers) {
...	...	@@ -1877,11 +1997,31 @@ inline Registers_arm64::Registers_arm64(const void *registers) {
1877	1997	memcpy(_vectorHalfRegisters,
1878	1998	static_cast<const uint8_t *>(registers) + sizeof(GPRs),
1879	1999	sizeof(_vectorHalfRegisters));
	2000	_misc_registers.__vg = 0;
	2001
	2002	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	2003	// We have to do some pointer authentication fixups after this copy,
	2004	// and as part of that we need to load the source pc without
	2005	// authenticating so that we maintain the signature for the resigning
	2006	// performed by setIP.
	2007	uint64_t pcRegister = 0;
	2008	memmove(&pcRegister, ((uint8_t *)&_registers) + offsetof(GPRs, __pc),
	2009	sizeof(pcRegister));
	2010	setIP(pcRegister);
	2011	#endif
1880	2012	}
1881	2013
1882		inline Registers_arm64::Registers_arm64() {
1883		memset(&_registers, 0, sizeof(_registers));
1884		memset(&_vectorHalfRegisters, 0, sizeof(_vectorHalfRegisters));
	2014	inline Registers_arm64::Registers_arm64(const Registers_arm64 &other) {
	2015	*this = other;
	2016	}
	2017
	2018	inline Registers_arm64 &
	2019	Registers_arm64::operator=(const Registers_arm64 &other) {
	2020	memmove(static_cast<void >(this), &other, sizeof(this));
	2021	// We perform this step to ensure that we correctly authenticate and re-sign
	2022	// the pc after the bitwise copy.
	2023	setIP(other.getIP());
	2024	return *this;
1885	2025	}
1886	2026
1887	2027	inline bool Registers_arm64::validRegister(int regNum) const {
...	...	@@ -1895,22 +2035,40 @@ inline bool Registers_arm64::validRegister(int regNum) const {
1895	2035	return false;
1896	2036	if (regNum == UNW_AARCH64_RA_SIGN_STATE)
1897	2037	return true;
	2038	if (regNum == UNW_AARCH64_VG)
	2039	return true;
1898	2040	if ((regNum > 32) && (regNum < 64))
1899	2041	return false;
1900	2042	return true;
1901	2043	}
1902	2044
	2045	inline uint64_t Registers_arm64::lazyGetVG() const {
	2046	if (!_misc_registers.__vg) {
	2047	#if defined(__aarch64__)
	2048	register uint64_t vg asm("x0");
	2049	asm(".inst 0x04e0e3e0" // CNTD x0
	2050	: "=r"(vg));
	2051	_misc_registers.__vg = vg;
	2052	#else
	2053	_LIBUNWIND_ABORT("arm64 VG undefined");
	2054	#endif
	2055	}
	2056	return _misc_registers.__vg;
	2057	}
	2058
1903	2059	inline uint64_t Registers_arm64::getRegister(int regNum) const {
1904	2060	if (regNum == UNW_REG_IP \|\| regNum == UNW_AARCH64_PC)
1905		return _registers.__pc;
	2061	return getIP();
1906	2062	if (regNum == UNW_REG_SP \|\| regNum == UNW_AARCH64_SP)
1907	2063	return _registers.__sp;
1908	2064	if (regNum == UNW_AARCH64_RA_SIGN_STATE)
1909	2065	return _registers.__ra_sign_state;
1910	2066	if (regNum == UNW_AARCH64_FP)
1911		return _registers.__fp;
	2067	return getFP();
1912	2068	if (regNum == UNW_AARCH64_LR)
1913	2069	return _registers.__lr;
	2070	if (regNum == UNW_AARCH64_VG)
	2071	return lazyGetVG();
1914	2072	if ((regNum >= 0) && (regNum < 29))
1915	2073	return _registers.__x[regNum];
1916	2074	_LIBUNWIND_ABORT("unsupported arm64 register");
...	...	@@ -1918,15 +2076,17 @@ inline uint64_t Registers_arm64::getRegister(int regNum) const {
1918	2076
1919	2077	inline void Registers_arm64::setRegister(int regNum, uint64_t value) {
1920	2078	if (regNum == UNW_REG_IP \|\| regNum == UNW_AARCH64_PC)
1921		_registers.__pc = value;
	2079	setIP(value);
1922	2080	else if (regNum == UNW_REG_SP \|\| regNum == UNW_AARCH64_SP)
1923	2081	_registers.__sp = value;
1924	2082	else if (regNum == UNW_AARCH64_RA_SIGN_STATE)
1925	2083	_registers.__ra_sign_state = value;
1926	2084	else if (regNum == UNW_AARCH64_FP)
1927		_registers.__fp = value;
	2085	setFP(value);
1928	2086	else if (regNum == UNW_AARCH64_LR)
1929	2087	_registers.__lr = value;
	2088	else if (regNum == UNW_AARCH64_VG)
	2089	_misc_registers.__vg = value;
1930	2090	else if ((regNum >= 0) && (regNum < 29))
1931	2091	_registers.__x[regNum] = value;
1932	2092	else
...	...	@@ -2116,6 +2276,10 @@ public:
2116	2276	Registers_arm();
2117	2277	Registers_arm(const void *registers);
2118	2278
	2279	typedef uint32_t reg_t;
	2280	typedef uint32_t link_reg_t;
	2281	typedef const link_reg_t &link_hardened_reg_arg_t;
	2282
2119	2283	bool validRegister(int num) const;
2120	2284	uint32_t getRegister(int num) const;
2121	2285	void setRegister(int num, uint32_t value);
...	...	@@ -2621,6 +2785,10 @@ public:
2621	2785	Registers_or1k();
2622	2786	Registers_or1k(const void *registers);
2623	2787
	2788	typedef uint32_t reg_t;
	2789	typedef uint32_t link_reg_t;
	2790	typedef const link_reg_t &link_hardened_reg_arg_t;
	2791
2624	2792	bool validRegister(int num) const;
2625	2793	uint32_t getRegister(int num) const;
2626	2794	void setRegister(int num, uint32_t value);
...	...	@@ -2820,6 +2988,10 @@ public:
2820	2988	Registers_mips_o32();
2821	2989	Registers_mips_o32(const void *registers);
2822	2990
	2991	typedef uint32_t reg_t;
	2992	typedef uint32_t link_reg_t;
	2993	typedef const link_reg_t &link_hardened_reg_arg_t;
	2994
2823	2995	bool validRegister(int num) const;
2824	2996	uint32_t getRegister(int num) const;
2825	2997	void setRegister(int num, uint32_t value);
...	...	@@ -3155,6 +3327,10 @@ public:
3155	3327	Registers_mips_newabi();
3156	3328	Registers_mips_newabi(const void *registers);
3157	3329
	3330	typedef uint64_t reg_t;
	3331	typedef uint64_t link_reg_t;
	3332	typedef const link_reg_t &link_hardened_reg_arg_t;
	3333
3158	3334	bool validRegister(int num) const;
3159	3335	uint64_t getRegister(int num) const;
3160	3336	void setRegister(int num, uint64_t value);
...	...	@@ -3458,6 +3634,10 @@ public:
3458	3634	Registers_sparc();
3459	3635	Registers_sparc(const void *registers);
3460	3636
	3637	typedef uint32_t reg_t;
	3638	typedef uint32_t link_reg_t;
	3639	typedef const link_reg_t &link_hardened_reg_arg_t;
	3640
3461	3641	bool validRegister(int num) const;
3462	3642	uint32_t getRegister(int num) const;
3463	3643	void setRegister(int num, uint32_t value);
...	...	@@ -3644,6 +3824,10 @@ public:
3644	3824	Registers_sparc64() = default;
3645	3825	Registers_sparc64(const void *registers);
3646	3826
	3827	typedef uint64_t reg_t;
	3828	typedef uint64_t link_reg_t;
	3829	typedef const link_reg_t &link_hardened_reg_arg_t;
	3830
3647	3831	bool validRegister(int num) const;
3648	3832	uint64_t getRegister(int num) const;
3649	3833	void setRegister(int num, uint64_t value);
...	...	@@ -3829,6 +4013,10 @@ public:
3829	4013	Registers_hexagon();
3830	4014	Registers_hexagon(const void *registers);
3831	4015
	4016	typedef uint32_t reg_t;
	4017	typedef uint32_t link_reg_t;
	4018	typedef const link_reg_t &link_hardened_reg_arg_t;
	4019
3832	4020	bool validRegister(int num) const;
3833	4021	uint32_t getRegister(int num) const;
3834	4022	void setRegister(int num, uint32_t value);
...	...	@@ -4044,6 +4232,10 @@ public:
4044	4232	Registers_riscv();
4045	4233	Registers_riscv(const void *registers);
4046	4234
	4235	typedef ::libunwind::reg_t reg_t;
	4236	typedef ::libunwind::reg_t link_reg_t;
	4237	typedef const link_reg_t &link_hardened_reg_arg_t;
	4238
4047	4239	bool validRegister(int num) const;
4048	4240	reg_t getRegister(int num) const;
4049	4241	void setRegister(int num, reg_t value);
...	...	@@ -4341,6 +4533,10 @@ public:
4341	4533	Registers_ve();
4342	4534	Registers_ve(const void *registers);
4343	4535
	4536	typedef uint64_t reg_t;
	4537	typedef uint64_t link_reg_t;
	4538	typedef const link_reg_t &link_hardened_reg_arg_t;
	4539
4344	4540	bool validRegister(int num) const;
4345	4541	uint64_t getRegister(int num) const;
4346	4542	void setRegister(int num, uint64_t value);
...	...	@@ -4784,6 +4980,10 @@ public:
4784	4980	Registers_s390x();
4785	4981	Registers_s390x(const void *registers);
4786	4982
	4983	typedef uint64_t reg_t;
	4984	typedef uint64_t link_reg_t;
	4985	typedef const link_reg_t &link_hardened_reg_arg_t;
	4986
4787	4987	bool validRegister(int num) const;
4788	4988	uint64_t getRegister(int num) const;
4789	4989	void setRegister(int num, uint64_t value);
...	...	@@ -5072,6 +5272,10 @@ public:
5072	5272	Registers_loongarch();
5073	5273	Registers_loongarch(const void *registers);
5074	5274
	5275	typedef uint64_t reg_t;
	5276	typedef uint64_t link_reg_t;
	5277	typedef const link_reg_t &link_hardened_reg_arg_t;
	5278
5075	5279	bool validRegister(int num) const;
5076	5280	uint64_t getRegister(int num) const;
5077	5281	void setRegister(int num, uint64_t value);

lib/libunwind/src/Unwind-seh.cpp+6-7

...	...	@@ -163,7 +163,7 @@ _GCC_specific_handler(PEXCEPTION_RECORD ms_exc, PVOID frame, PCONTEXT ms_ctx,
163	163	// If we were called by __libunwind_seh_personality(), indicate that
164	164	// a handler was found; otherwise, initiate phase 2 by unwinding.
165	165	if (ours && ms_exc->NumberParameters > 1)
166		return 4 /* ExceptionExecuteHandler in mingw */;
	166	return static_cast<EXCEPTION_DISPOSITION>(4);
167	167	// This should never happen in phase 2.
168	168	if (IS_UNWINDING(ms_exc->ExceptionFlags))
169	169	_LIBUNWIND_ABORT("Personality indicated exception handler in phase 2!");
...	...	@@ -182,7 +182,7 @@ _GCC_specific_handler(PEXCEPTION_RECORD ms_exc, PVOID frame, PCONTEXT ms_ctx,
182	182	// a handler was found; otherwise, it's time to initiate a collided
183	183	// unwind to the target.
184	184	if (ours && !IS_UNWINDING(ms_exc->ExceptionFlags) && ms_exc->NumberParameters > 1)
185		return 4 /* ExceptionExecuteHandler in mingw */;
	185	return static_cast<EXCEPTION_DISPOSITION>(4);
186	186	// This should never happen in phase 1.
187	187	if (!IS_UNWINDING(ms_exc->ExceptionFlags))
188	188	_LIBUNWIND_ABORT("Personality installed context during phase 1!");
...	...	@@ -259,13 +259,12 @@ __libunwind_seh_personality(int version, _Unwind_Action state,
259	259	(void )disp_ctx->LanguageHandler, (void )&ms_exc,
260	260	(void *)disp_ctx->EstablisherFrame,
261	261	(void )disp_ctx->ContextRecord, (void )disp_ctx);
262		EXCEPTION_DISPOSITION ms_act = disp_ctx->LanguageHandler(&ms_exc,
263		(PVOID)disp_ctx->EstablisherFrame,
264		disp_ctx->ContextRecord,
265		disp_ctx);
	262	int ms_act = static_cast<int>(
	263	disp_ctx->LanguageHandler(&ms_exc, (PVOID)disp_ctx->EstablisherFrame,
	264	disp_ctx->ContextRecord, disp_ctx));
266	265	_LIBUNWIND_TRACE_UNWINDING("__libunwind_seh_personality() LanguageHandler "
267	266	"returned %d",
268		(int)ms_act);
	267	ms_act);
269	268	switch (ms_act) {
270	269	case ExceptionContinueExecution: return _URC_END_OF_STACK;
271	270	case ExceptionContinueSearch: return _URC_CONTINUE_UNWIND;

lib/libunwind/src/Unwind-wasm.c+7-7

...	...	@@ -37,13 +37,13 @@ struct _Unwind_LandingPadContext {
37	37	// function
38	38	thread_local struct _Unwind_LandingPadContext __wasm_lpad_context;
39	39
40		/// Calls to this function is in landing pads in compiler-generated user code.
	40	/// Calls to this function are in landing pads in compiler-generated user code.
41	41	/// In other EH schemes, stack unwinding is done by libunwind library, which
42		/// calls the personality function for each each frame it lands. On the other
43		/// hand, WebAssembly stack unwinding process is performed by a VM, and the
44		/// personality function cannot be called from there. So the compiler inserts
45		/// a call to this function in landing pads in the user code, which in turn
46		/// calls the personality function.
	42	/// calls the personality function for each frame it lands. On the other hand,
	43	/// WebAssembly stack unwinding process is performed by a VM, and the
	44	/// personality function cannot be called from there. So the compiler inserts a
	45	/// call to this function in landing pads in the user code, which in turn calls
	46	/// the personality function.
47	47	_Unwind_Reason_Code _Unwind_CallPersonality(void *exception_ptr) {
48	48	struct _Unwind_Exception *exception_object =
49	49	(struct _Unwind_Exception *)exception_ptr;
...	...	@@ -92,7 +92,7 @@ _LIBUNWIND_EXPORT void _Unwind_SetGR(struct _Unwind_Context *context, int index,
92	92
93	93	/// Called by personality handler to get instruction pointer.
94	94	_LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) {
95		// The result will be used as an 1-based index after decrementing 1, so we
	95	// The result will be used as a 1-based index after decrementing 1, so we
96	96	// increment 2 here
97	97	uintptr_t result =
98	98	((struct _Unwind_LandingPadContext *)context)->lpad_index + 2;

lib/libunwind/src/UnwindCursor.hpp+167-49

...	...	@@ -41,7 +41,8 @@
41	41	#define _LIBUNWIND_CHECK_LINUX_SIGRETURN 1
42	42	#endif
43	43
44		#if defined(_LIBUNWIND_TARGET_HAIKU) && defined(_LIBUNWIND_TARGET_X86_64)
	44	#if defined(_LIBUNWIND_TARGET_HAIKU) && \
	45	(defined(_LIBUNWIND_TARGET_I386) \|\| defined(_LIBUNWIND_TARGET_X86_64))
45	46	#include <OS.h>
46	47	#include <signal.h>
47	48	#define _LIBUNWIND_CHECK_HAIKU_SIGRETURN 1
...	...	@@ -120,7 +121,9 @@ class _LIBUNWIND_HIDDEN DwarfFDECache {
120	121	typedef typename A::pint_t pint_t;
121	122	public:
122	123	static constexpr pint_t kSearchAll = static_cast<pint_t>(-1);
123		static pint_t findFDE(pint_t mh, pint_t pc);
	124	template <typename R>
	125	static pint_t findFDE(pint_t mh, typename R::link_hardened_reg_arg_t pc);
	126
124	127	static void add(pint_t mh, pint_t ip_start, pint_t ip_end, pint_t fde);
125	128	static void removeAllIn(pint_t mh);
126	129	static void iterateCacheEntries(void (*func)(unw_word_t ip_start,
...	...	@@ -173,7 +176,9 @@ bool DwarfFDECache<A>::_registeredForDyldUnloads = false;
173	176	#endif
174	177
175	178	template <typename A>
176		typename A::pint_t DwarfFDECache<A>::findFDE(pint_t mh, pint_t pc) {
	179	template <typename R>
	180	typename DwarfFDECache<A>::pint_t
	181	DwarfFDECache<A>::findFDE(pint_t mh, typename R::link_hardened_reg_arg_t pc) {
177	182	pint_t result = 0;
178	183	_LIBUNWIND_LOG_IF_FALSE(_lock.lock_shared());
179	184	for (entry *p = _buffer; p < _bufferUsed; ++p) {
...	...	@@ -471,7 +476,9 @@ public:
471	476	virtual void getInfo(unw_proc_info_t *) {
472	477	_LIBUNWIND_ABORT("getInfo not implemented");
473	478	}
474		virtual void jumpto() { _LIBUNWIND_ABORT("jumpto not implemented"); }
	479	_LIBUNWIND_TRACE_NO_INLINE virtual void jumpto() {
	480	_LIBUNWIND_ABORT("jumpto not implemented");
	481	}
475	482	virtual bool isSignalFrame() {
476	483	_LIBUNWIND_ABORT("isSignalFrame not implemented");
477	484	}
...	...	@@ -488,6 +495,12 @@ public:
488	495	virtual void saveVFPAsX() { _LIBUNWIND_ABORT("saveVFPAsX not implemented"); }
489	496	#endif
490	497
	498	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	499	virtual void setWalkedFrames(unsigned) {
	500	_LIBUNWIND_ABORT("setWalkedFrames not implemented");
	501	}
	502	#endif
	503
491	504	#ifdef _AIX
492	505	virtual uintptr_t getDataRelBase() {
493	506	_LIBUNWIND_ABORT("getDataRelBase not implemented");
...	...	@@ -964,7 +977,8 @@ public:
964	977	virtual void setFloatReg(int, unw_fpreg_t);
965	978	virtual int step(bool stage2 = false);
966	979	virtual void getInfo(unw_proc_info_t *);
967		virtual void jumpto();
	980	_LIBUNWIND_TRACE_NO_INLINE
	981	virtual void jumpto();
968	982	virtual bool isSignalFrame();
969	983	virtual bool getFunctionName(char buf, size_t len, unw_word_t off);
970	984	virtual void setInfoBasedOnIPRegister(bool isReturnAddress = false);
...	...	@@ -973,6 +987,10 @@ public:
973	987	virtual void saveVFPAsX();
974	988	#endif
975	989
	990	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	991	virtual void setWalkedFrames(unsigned);
	992	#endif
	993
976	994	#ifdef _AIX
977	995	virtual uintptr_t getDataRelBase();
978	996	#endif
...	...	@@ -1045,19 +1063,28 @@ private:
1045	1063	#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
1046	1064	bool getInfoFromFdeCie(const typename CFI_Parser<A>::FDE_Info &fdeInfo,
1047	1065	const typename CFI_Parser<A>::CIE_Info &cieInfo,
1048		pint_t pc, uintptr_t dso_base);
1049		bool getInfoFromDwarfSection(pint_t pc, const UnwindInfoSections &sects,
1050		uint32_t fdeSectionOffsetHint=0);
	1066	typename R::link_hardened_reg_arg_t pc,
	1067	uintptr_t dso_base);
	1068	bool getInfoFromDwarfSection(typename R::link_hardened_reg_arg_t pc,
	1069	const UnwindInfoSections &sects,
	1070	uint32_t fdeSectionOffsetHint = 0);
1051	1071	int stepWithDwarfFDE(bool stage2) {
	1072	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	1073	typename R::reg_t rawPC = this->getReg(UNW_REG_IP);
	1074	typename R::link_reg_t pc;
	1075	_registers.loadAndAuthenticateLinkRegister(rawPC, &pc);
	1076	#else
	1077	typename R::link_reg_t pc = this->getReg(UNW_REG_IP);
	1078	#endif
1052	1079	return DwarfInstructions<A, R>::stepWithDwarf(
1053		_addressSpace, (pint_t)this->getReg(UNW_REG_IP),
1054		(pint_t)_info.unwind_info, _registers, _isSignalFrame, stage2);
	1080	_addressSpace, pc, (pint_t)_info.unwind_info, _registers,
	1081	_isSignalFrame, stage2);
1055	1082	}
1056	1083	#endif
1057	1084
1058	1085	#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND)
1059		bool getInfoFromCompactEncodingSection(pint_t pc,
1060		const UnwindInfoSections &sects);
	1086	bool getInfoFromCompactEncodingSection(typename R::link_hardened_reg_arg_t pc,
	1087	const UnwindInfoSections &sects);
1061	1088	int stepWithCompactEncoding(bool stage2 = false) {
1062	1089	#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
1063	1090	if ( compactSaysUseDwarf() )
...	...	@@ -1344,9 +1371,12 @@ private:
1344	1371	bool _unwindInfoMissing;
1345	1372	bool _isSignalFrame;
1346	1373	#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) \|\| \
1347		defined(_LIBUNWIND_TARGET_HAIKU)
	1374	defined(_LIBUNWIND_CHECK_HAIKU_SIGRETURN)
1348	1375	bool _isSigReturn = false;
1349	1376	#endif
	1377	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	1378	uint32_t _walkedFrames;
	1379	#endif
1350	1380	};
1351	1381
1352	1382
...	...	@@ -1358,13 +1388,13 @@ UnwindCursor<A, R>::UnwindCursor(unw_context_t *context, A &as)
1358	1388	"UnwindCursor<> does not fit in unw_cursor_t");
1359	1389	static_assert((alignof(UnwindCursor<A, R>) <= alignof(unw_cursor_t)),
1360	1390	"UnwindCursor<> requires more alignment than unw_cursor_t");
1361		memset(&_info, 0, sizeof(_info));
	1391	memset(static_cast<void *>(&_info), 0, sizeof(_info));
1362	1392	}
1363	1393
1364	1394	template <typename A, typename R>
1365	1395	UnwindCursor<A, R>::UnwindCursor(A &as, void *)
1366	1396	: _addressSpace(as), _unwindInfoMissing(false), _isSignalFrame(false) {
1367		memset(&_info, 0, sizeof(_info));
	1397	memset(static_cast<void *>(&_info), 0, sizeof(_info));
1368	1398	// FIXME
1369	1399	// fill in _registers from thread arg
1370	1400	}
...	...	@@ -1401,7 +1431,46 @@ void UnwindCursor<A, R>::setFloatReg(int regNum, unw_fpreg_t value) {
1401	1431	}
1402	1432
1403	1433	template <typename A, typename R> void UnwindCursor<A, R>::jumpto() {
	1434	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	1435	/*
	1436
	1437	The value of `_walkedFrames` is computed in `unwind_phase2` and represents the
	1438	number of frames walked starting `unwind_phase2` to get to the landing pad.
	1439
	1440	```
	1441	// uc is initialized by __unw_getcontext in the parent frame.
	1442	// The first stack frame walked is unwind_phase2.
	1443	unsigned framesWalked = 1;
	1444	```
	1445
	1446	To that, we need to add the number of function calls in libunwind between
	1447	`unwind_phase2` & `__libunwind_Registers_arm64_jumpto` which performs the long
	1448	jump, to rebalance the execution flow.
	1449
	1450	```
	1451	frame #0: libunwind.1.dylib`__libunwind_Registers_arm64_jumpto at UnwindRegistersRestore.S:646
	1452	frame #1: libunwind.1.dylib`libunwind::Registers_arm64::returnto at Registers.hpp:2291:3
	1453	frame #2: libunwind.1.dylib`libunwind::UnwindCursor<libunwind::LocalAddressSpace, libunwind::Registers_arm64>::jumpto at UnwindCursor.hpp:1474:14
	1454	frame #3: libunwind.1.dylib`__unw_resume at libunwind.cpp:375:7
	1455	frame #4: libunwind.1.dylib`__unw_resume_with_frames_walked at libunwind.cpp:363:10
	1456	frame #5: libunwind.1.dylib`unwind_phase2 at UnwindLevel1.c:328:9
	1457	frame #6: libunwind.1.dylib`_Unwind_RaiseException at UnwindLevel1.c:480:10
	1458	frame #7: libc++abi.dylib`__cxa_throw at cxa_exception.cpp:295:5
	1459	...
	1460	```
	1461
	1462	If we look at the backtrace from `__libunwind_Registers_arm64_jumpto`, we see
	1463	there are 5 frames on the stack to reach `unwind_phase2`. However, only 4 of
	1464	them will never return, since `__libunwind_Registers_arm64_jumpto` returns
	1465	back to the landing pad, so we need to subtract 1 to the number of
	1466	`_EXTRA_LIBUNWIND_FRAMES_WALKED`.
	1467	*/
	1468
	1469	static constexpr size_t _EXTRA_LIBUNWIND_FRAMES_WALKED = 5 - 1;
	1470	_registers.returnto(_walkedFrames + _EXTRA_LIBUNWIND_FRAMES_WALKED);
	1471	#else
1404	1472	_registers.jumpto();
	1473	#endif
1405	1474	}
1406	1475
1407	1476	#ifdef __arm__
...	...	@@ -1410,6 +1479,13 @@ template <typename A, typename R> void UnwindCursor<A, R>::saveVFPAsX() {
1410	1479	}
1411	1480	#endif
1412	1481
	1482	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	1483	template <typename A, typename R>
	1484	void UnwindCursor<A, R>::setWalkedFrames(unsigned walkedFrames) {
	1485	_walkedFrames = walkedFrames;
	1486	}
	1487	#endif
	1488
1413	1489	#ifdef _AIX
1414	1490	template <typename A, typename R>
1415	1491	uintptr_t UnwindCursor<A, R>::getDataRelBase() {
...	...	@@ -1658,11 +1734,11 @@ bool UnwindCursor<A, R>::getInfoFromEHABISection(
1658	1734	template <typename A, typename R>
1659	1735	bool UnwindCursor<A, R>::getInfoFromFdeCie(
1660	1736	const typename CFI_Parser<A>::FDE_Info &fdeInfo,
1661		const typename CFI_Parser<A>::CIE_Info &cieInfo, pint_t pc,
1662		uintptr_t dso_base) {
	1737	const typename CFI_Parser<A>::CIE_Info &cieInfo,
	1738	typename R::link_hardened_reg_arg_t pc, uintptr_t dso_base) {
1663	1739	typename CFI_Parser<A>::PrologInfo prolog;
1664		if (CFI_Parser<A>::parseFDEInstructions(_addressSpace, fdeInfo, cieInfo, pc,
1665		R::getArch(), &prolog)) {
	1740	if (CFI_Parser<A>::template parseFDEInstructions<R>(
	1741	_addressSpace, fdeInfo, cieInfo, pc, R::getArch(), &prolog)) {
1666	1742	// Save off parsed FDE info
1667	1743	_info.start_ip = fdeInfo.pcStart;
1668	1744	_info.end_ip = fdeInfo.pcEnd;
...	...	@@ -1682,43 +1758,42 @@ bool UnwindCursor<A, R>::getInfoFromFdeCie(
1682	1758	}
1683	1759
1684	1760	template <typename A, typename R>
1685		bool UnwindCursor<A, R>::getInfoFromDwarfSection(pint_t pc,
1686		const UnwindInfoSections &sects,
1687		uint32_t fdeSectionOffsetHint) {
	1761	bool UnwindCursor<A, R>::getInfoFromDwarfSection(
	1762	typename R::link_hardened_reg_arg_t pc, const UnwindInfoSections &sects,
	1763	uint32_t fdeSectionOffsetHint) {
1688	1764	typename CFI_Parser<A>::FDE_Info fdeInfo;
1689	1765	typename CFI_Parser<A>::CIE_Info cieInfo;
1690	1766	bool foundFDE = false;
1691	1767	bool foundInCache = false;
1692	1768	// If compact encoding table gave offset into dwarf section, go directly there
1693	1769	if (fdeSectionOffsetHint != 0) {
1694		foundFDE = CFI_Parser<A>::findFDE(_addressSpace, pc, sects.dwarf_section,
1695		sects.dwarf_section_length,
1696		sects.dwarf_section + fdeSectionOffsetHint,
1697		&fdeInfo, &cieInfo);
	1770	foundFDE = CFI_Parser<A>::template findFDE<R>(
	1771	_addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length,
	1772	sects.dwarf_section + fdeSectionOffsetHint, &fdeInfo, &cieInfo);
1698	1773	}
1699	1774	#if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX)
1700	1775	if (!foundFDE && (sects.dwarf_index_section != 0)) {
1701		foundFDE = EHHeaderParser<A>::findFDE(
	1776	foundFDE = EHHeaderParser<A>::template findFDE<R>(
1702	1777	_addressSpace, pc, sects.dwarf_index_section,
1703	1778	(uint32_t)sects.dwarf_index_section_length, &fdeInfo, &cieInfo);
1704	1779	}
1705	1780	#endif
1706	1781	if (!foundFDE) {
1707	1782	// otherwise, search cache of previously found FDEs.
1708		pint_t cachedFDE = DwarfFDECache<A>::findFDE(sects.dso_base, pc);
	1783	pint_t cachedFDE =
	1784	DwarfFDECache<A>::template findFDE<R>(sects.dso_base, pc);
1709	1785	if (cachedFDE != 0) {
1710		foundFDE =
1711		CFI_Parser<A>::findFDE(_addressSpace, pc, sects.dwarf_section,
1712		sects.dwarf_section_length,
1713		cachedFDE, &fdeInfo, &cieInfo);
	1786	foundFDE = CFI_Parser<A>::template findFDE<R>(
	1787	_addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length,
	1788	cachedFDE, &fdeInfo, &cieInfo);
1714	1789	foundInCache = foundFDE;
1715	1790	}
1716	1791	}
1717	1792	if (!foundFDE) {
1718	1793	// Still not found, do full scan of __eh_frame section.
1719		foundFDE = CFI_Parser<A>::findFDE(_addressSpace, pc, sects.dwarf_section,
1720		sects.dwarf_section_length, 0,
1721		&fdeInfo, &cieInfo);
	1794	foundFDE = CFI_Parser<A>::template findFDE<R>(
	1795	_addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length, 0,
	1796	&fdeInfo, &cieInfo);
1722	1797	}
1723	1798	if (foundFDE) {
1724	1799	if (getInfoFromFdeCie(fdeInfo, cieInfo, pc, sects.dso_base)) {
...	...	@@ -1742,8 +1817,8 @@ bool UnwindCursor<A, R>::getInfoFromDwarfSection(pint_t pc,
1742	1817
1743	1818	#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND)
1744	1819	template <typename A, typename R>
1745		bool UnwindCursor<A, R>::getInfoFromCompactEncodingSection(pint_t pc,
1746		const UnwindInfoSections &sects) {
	1820	bool UnwindCursor<A, R>::getInfoFromCompactEncodingSection(
	1821	typename R::link_hardened_reg_arg_t pc, const UnwindInfoSections &sects) {
1747	1822	const bool log = false;
1748	1823	if (log)
1749	1824	fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX, mh=0x%llX)\n",
...	...	@@ -1974,6 +2049,16 @@ bool UnwindCursor<A, R>::getInfoFromCompactEncodingSection(pint_t pc,
1974	2049	personalityIndex * sizeof(uint32_t));
1975	2050	pint_t personalityPointer = sects.dso_base + (pint_t)personalityDelta;
1976	2051	personality = _addressSpace.getP(personalityPointer);
	2052	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	2053	// The GOT for the personality function was signed address authenticated.
	2054	// Resign it as a regular function pointer.
	2055	const auto discriminator = ptrauth_blend_discriminator(
	2056	&_info.handler, __ptrauth_unwind_upi_handler_disc);
	2057	void *signedPtr = ptrauth_auth_and_resign(
	2058	(void *)personality, ptrauth_key_function_pointer, personalityPointer,
	2059	ptrauth_key_function_pointer, discriminator);
	2060	personality = (__typeof(personality))signedPtr;
	2061	#endif
1977	2062	if (log)
1978	2063	fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX), "
1979	2064	"personalityDelta=0x%08X, personality=0x%08llX\n",
...	...	@@ -1987,7 +2072,11 @@ bool UnwindCursor<A, R>::getInfoFromCompactEncodingSection(pint_t pc,
1987	2072	_info.start_ip = funcStart;
1988	2073	_info.end_ip = funcEnd;
1989	2074	_info.lsda = lsda;
1990		_info.handler = personality;
	2075	// We use memmove to copy the personality function as we have already manually
	2076	// re-signed the pointer, and assigning directly will attempt to incorrectly
	2077	// sign the already signed value.
	2078	memmove(reinterpret_cast<void *>(&_info.handler),
	2079	reinterpret_cast<void *>(&personality), sizeof(personality));
1991	2080	_info.gp = 0;
1992	2081	_info.flags = 0;
1993	2082	_info.format = encoding;
...	...	@@ -2640,11 +2729,19 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
2640	2729	_isSigReturn = false;
2641	2730	#endif
2642	2731
2643		pint_t pc = static_cast<pint_t>(this->getReg(UNW_REG_IP));
	2732	typename R::reg_t rawPC = this->getReg(UNW_REG_IP);
	2733
2644	2734	#if defined(_LIBUNWIND_ARM_EHABI)
2645	2735	// Remove the thumb bit so the IP represents the actual instruction address.
2646	2736	// This matches the behaviour of _Unwind_GetIP on arm.
2647		pc &= (pint_t)~0x1;
	2737	rawPC &= (pint_t)~0x1;
	2738	#endif
	2739
	2740	typename R::link_reg_t pc;
	2741	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	2742	_registers.loadAndAuthenticateLinkRegister(rawPC, &pc);
	2743	#else
	2744	pc = rawPC;
2648	2745	#endif
2649	2746
2650	2747	// Exit early if at the top of the stack.
...	...	@@ -2679,7 +2776,7 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
2679	2776
2680	2777	// Ask address space object to find unwind sections for this pc.
2681	2778	UnwindInfoSections sects;
2682		if (_addressSpace.findUnwindSections(pc, sects)) {
	2779	if (_addressSpace.template findUnwindSections<R>(pc, sects)) {
2683	2780	#if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND)
2684	2781	// If there is a compact unwind encoding table, look there first.
2685	2782	if (sects.compact_unwind_section != 0) {
...	...	@@ -2735,8 +2832,8 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
2735	2832	#if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
2736	2833	// There is no static unwind info for this pc. Look to see if an FDE was
2737	2834	// dynamically registered for it.
2738		pint_t cachedFDE = DwarfFDECache<A>::findFDE(DwarfFDECache<A>::kSearchAll,
2739		pc);
	2835	pint_t cachedFDE =
	2836	DwarfFDECache<A>::template findFDE<R>(DwarfFDECache<A>::kSearchAll, pc);
2740	2837	if (cachedFDE != 0) {
2741	2838	typename CFI_Parser<A>::FDE_Info fdeInfo;
2742	2839	typename CFI_Parser<A>::CIE_Info cieInfo;
...	...	@@ -2748,7 +2845,7 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
2748	2845	// Lastly, ask AddressSpace object about platform specific ways to locate
2749	2846	// other FDEs.
2750	2847	pint_t fde;
2751		if (_addressSpace.findOtherFDE(pc, fde)) {
	2848	if (_addressSpace.template findOtherFDE<R>(pc, fde)) {
2752	2849	typename CFI_Parser<A>::FDE_Info fdeInfo;
2753	2850	typename CFI_Parser<A>::CIE_Info cieInfo;
2754	2851	if (!CFI_Parser<A>::decodeFDE(_addressSpace, fde, &fdeInfo, &cieInfo)) {
...	...	@@ -2772,6 +2869,21 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
2772	2869
2773	2870	#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) && \
2774	2871	defined(_LIBUNWIND_TARGET_AARCH64)
	2872
	2873	/*
	2874	* The linux sigreturn restorer stub will always have the form:
	2875	*
	2876	* d2801168 movz x8, #0x8b
	2877	* d4000001 svc #0x0
	2878	*/
	2879	#if defined(__AARCH64EB__)
	2880	#define MOVZ_X8_8B 0x681180d2
	2881	#define SVC_0 0x010000d4
	2882	#else
	2883	#define MOVZ_X8_8B 0xd2801168
	2884	#define SVC_0 0xd4000001
	2885	#endif
	2886
2775	2887	template <typename A, typename R>
2776	2888	bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_arm64 &) {
2777	2889	// Look for the sigreturn trampoline. The trampoline's body is two
...	...	@@ -2796,7 +2908,7 @@ bool UnwindCursor<A, R>::setInfoForSigReturn(Registers_arm64 &) {
2796	2908	return false;
2797	2909	auto instructions = reinterpret_cast<const uint32_t >(pc);
2798	2910	// Look for instructions: mov x8, #0x8b; svc #0x0
2799		if (instructions[0] != 0xd2801168 \|\| instructions[1] != 0xd4000001)
	2911	if (instructions[0] != MOVZ_X8_8B \|\| instructions[1] != SVC_0)
2800	2912	return false;
2801	2913
2802	2914	_info = {};
...	...	@@ -3188,16 +3300,22 @@ template <typename A, typename R> int UnwindCursor<A, R>::step(bool stage2) {
3188	3300	template <typename A, typename R>
3189	3301	void UnwindCursor<A, R>::getInfo(unw_proc_info_t *info) {
3190	3302	if (_unwindInfoMissing)
3191		memset(info, 0, sizeof(*info));
	3303	memset(static_cast<void >(info), 0, sizeof(info));
3192	3304	else
3193	3305	*info = _info;
3194	3306	}
3195	3307
3196	3308	template <typename A, typename R>
3197	3309	bool UnwindCursor<A, R>::getFunctionName(char *buf, size_t bufLen,
3198		unw_word_t *offset) {
3199		return _addressSpace.findFunctionName((pint_t)this->getReg(UNW_REG_IP),
3200		buf, bufLen, offset);
	3310	unw_word_t *offset) {
	3311	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	3312	typename R::reg_t rawPC = this->getReg(UNW_REG_IP);
	3313	typename R::link_reg_t pc;
	3314	_registers.loadAndAuthenticateLinkRegister(rawPC, &pc);
	3315	#else
	3316	typename R::link_reg_t pc = this->getReg(UNW_REG_IP);
	3317	#endif
	3318	return _addressSpace.template findFunctionName<R>(pc, buf, bufLen, offset);
3201	3319	}
3202	3320
3203	3321	#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)

lib/libunwind/src/UnwindLevel1.c+46-17

...	...	@@ -48,16 +48,15 @@
48	48	// avoided when invoking the `jumpto()` function. To do this, we use inline
49	49	// assemblies to "goto" the `jumpto()` for these architectures.
50	50	#if !defined(_LIBUNWIND_USE_CET) && !defined(_LIBUNWIND_USE_GCS)
51		#define __unw_phase2_resume(cursor, fn) \
	51	#define __unw_phase2_resume(cursor, payload) \
52	52	do { \
53		(void)fn; \
54		__unw_resume((cursor)); \
	53	__unw_resume_with_frames_walked((cursor), (payload)); \
55	54	} while (0)
56	55	#elif defined(_LIBUNWIND_TARGET_I386)
57	56	#define __shstk_step_size (4)
58		#define __unw_phase2_resume(cursor, fn) \
	57	#define __unw_phase2_resume(cursor, payload) \
59	58	do { \
60		_LIBUNWIND_POP_SHSTK_SSP((fn)); \
	59	_LIBUNWIND_POP_SHSTK_SSP((payload)); \
61	60	void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \
62	61	void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \
63	62	__asm__ volatile("push %%edi\n\t" \
...	...	@@ -67,9 +66,9 @@
67	66	} while (0)
68	67	#elif defined(_LIBUNWIND_TARGET_X86_64)
69	68	#define __shstk_step_size (8)
70		#define __unw_phase2_resume(cursor, fn) \
	69	#define __unw_phase2_resume(cursor, payload) \
71	70	do { \
72		_LIBUNWIND_POP_SHSTK_SSP((fn)); \
	71	_LIBUNWIND_POP_SHSTK_SSP((payload)); \
73	72	void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \
74	73	void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \
75	74	__asm__ volatile("jmpq *%%rdx\n\t" ::"D"(shstkRegContext), \
...	...	@@ -77,19 +76,37 @@
77	76	} while (0)
78	77	#elif defined(_LIBUNWIND_TARGET_AARCH64)
79	78	#define __shstk_step_size (8)
80		#define __unw_phase2_resume(cursor, fn) \
	79	#define __unw_phase2_resume(cursor, payload) \
81	80	do { \
82		_LIBUNWIND_POP_SHSTK_SSP((fn)); \
	81	_LIBUNWIND_POP_SHSTK_SSP((payload)); \
83	82	void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \
84	83	void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \
85	84	__asm__ volatile("mov x0, %0\n\t" \
	85	"mov x1, #0\n\t" \
86	86	"br %1\n\t" \
87	87	: \
88	88	: "r"(shstkRegContext), "r"(shstkJumpAddress) \
89		: "x0"); \
	89	: "x0", "x1"); \
90	90	} while (0)
91	91	#endif
92	92
	93	// We need this helper function as the semantics of casting between integers and
	94	// function pointers mean that we end up with a function pointer without the
	95	// correct signature. Instead we assign to an integer with a matching schema,
	96	// and then memmove the result into a variable of the correct type. This memmove
	97	// is possible as `_Unwind_Personality_Fn` is a standard function pointer, and
	98	// as such is not address diversified.
	99	static _Unwind_Personality_Fn get_handler_function(unw_proc_info_t *frameInfo) {
	100	uintptr_t __unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer,
	101	0,
	102	ptrauth_function_pointer_type_discriminator(_Unwind_Personality_Fn))
	103	reauthenticatedIntegerHandler = frameInfo->handler;
	104	_Unwind_Personality_Fn handler;
	105	memmove(&handler, (void *)&reauthenticatedIntegerHandler,
	106	sizeof(_Unwind_Personality_Fn));
	107	return handler;
	108	}
	109
93	110	static _Unwind_Reason_Code
94	111	unwind_phase1(unw_context_t uc, unw_cursor_t cursor, _Unwind_Exception *exception_object) {
95	112	__unw_init_local(cursor, uc);
...	...	@@ -147,8 +164,7 @@ unwind_phase1(unw_context_t uc, unw_cursor_t cursor, _Unwind_Exception *except
147	164	// If there is a personality routine, ask it if it will want to stop at
148	165	// this frame.
149	166	if (frameInfo.handler != 0) {
150		_Unwind_Personality_Fn p =
151		(_Unwind_Personality_Fn)(uintptr_t)(frameInfo.handler);
	167	_Unwind_Personality_Fn p = get_handler_function(&frameInfo);
152	168	_LIBUNWIND_TRACE_UNWINDING(
153	169	"unwind_phase1(ex_obj=%p): calling personality function %p",
154	170	(void )exception_object, (void )(uintptr_t)p);
...	...	@@ -184,11 +200,12 @@ unwind_phase1(unw_context_t uc, unw_cursor_t cursor, _Unwind_Exception *except
184	200	}
185	201	return _URC_NO_REASON;
186	202	}
187		extern int __unw_step_stage2(unw_cursor_t *);
188	203
189	204	#if defined(_LIBUNWIND_USE_GCS)
190	205	// Enable the GCS target feature to permit gcspop instructions to be used.
191	206	__attribute__((target("+gcs")))
	207	#else
	208	_LIBUNWIND_TRACE_NO_INLINE
192	209	#endif
193	210	static _Unwind_Reason_Code
194	211	unwind_phase2(unw_context_t uc, unw_cursor_t cursor,
...	...	@@ -276,8 +293,7 @@ unwind_phase2(unw_context_t uc, unw_cursor_t cursor,
276	293	++framesWalked;
277	294	// If there is a personality routine, tell it we are unwinding.
278	295	if (frameInfo.handler != 0) {
279		_Unwind_Personality_Fn p =
280		(_Unwind_Personality_Fn)(uintptr_t)(frameInfo.handler);
	296	_Unwind_Personality_Fn p = get_handler_function(&frameInfo);
281	297	_Unwind_Action action = _UA_CLEANUP_PHASE;
282	298	if (sp == exception_object->private_2) {
283	299	// Tell personality this was the frame it marked in phase 1.
...	...	@@ -334,6 +350,8 @@ unwind_phase2(unw_context_t uc, unw_cursor_t cursor,
334	350	#if defined(_LIBUNWIND_USE_GCS)
335	351	// Enable the GCS target feature to permit gcspop instructions to be used.
336	352	__attribute__((target("+gcs")))
	353	#else
	354	_LIBUNWIND_TRACE_NO_INLINE
337	355	#endif
338	356	static _Unwind_Reason_Code
339	357	unwind_phase2_forced(unw_context_t uc, unw_cursor_t cursor,
...	...	@@ -394,8 +412,7 @@ unwind_phase2_forced(unw_context_t uc, unw_cursor_t cursor,
394	412	++framesWalked;
395	413	// If there is a personality routine, tell it we are unwinding.
396	414	if (frameInfo.handler != 0) {
397		_Unwind_Personality_Fn p =
398		(_Unwind_Personality_Fn)(intptr_t)(frameInfo.handler);
	415	_Unwind_Personality_Fn p = get_handler_function(&frameInfo);
399	416	_LIBUNWIND_TRACE_UNWINDING(
400	417	"unwind_phase2_forced(ex_obj=%p): calling personality function %p",
401	418	(void )exception_object, (void )(uintptr_t)p);
...	...	@@ -597,6 +614,18 @@ _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) {
597	614	unw_cursor_t cursor = (unw_cursor_t )context;
598	615	unw_word_t result;
599	616	__unw_get_reg(cursor, UNW_REG_IP, &result);
	617
	618	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	619	// If we are in an arm64e frame, then the PC should have been signed with the
	620	// sp
	621	{
	622	unw_word_t sp;
	623	__unw_get_reg(cursor, UNW_REG_SP, &sp);
	624	result = (unw_word_t)ptrauth_auth_data((void *)result,
	625	ptrauth_key_return_address, sp);
	626	}
	627	#endif
	628
600	629	_LIBUNWIND_TRACE_API("_Unwind_GetIP(context=%p) => 0x%" PRIxPTR,
601	630	(void *)context, result);
602	631	return (uintptr_t)result;

lib/libunwind/src/UnwindRegistersRestore.S+46-5

...	...	@@ -6,6 +6,8 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8
	9	#if !defined(__wasm__)
	10
9	11	#include "assembly.h"
10	12
11	13	#define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
...	...	@@ -16,13 +18,17 @@
16	18
17	19	#if defined(_AIX)
18	20	.toc
	21	#elif defined(__aarch64__) && defined(__ELF__) && defined(_LIBUNWIND_EXECUTE_ONLY_CODE)
	22	.section .text,"axy",@progbits,unique,0
19	23	#else
20	24	.text
21	25	#endif
22	26
23		#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__)
	27	#if !defined(__USING_SJLJ_EXCEPTIONS__)
24	28
25	29	#if defined(__i386__)
	30	.att_syntax
	31
26	32	DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_jumpto)
27	33	#
28	34	# extern "C" void __libunwind_Registers_x86_jumpto(Registers_x86 *);
...	...	@@ -67,6 +73,7 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_jumpto)
67	73	# skip gs
68	74
69	75	#elif defined(__x86_64__) && !defined(__arm64ec__)
	76	.att_syntax
70	77
71	78	DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_64_jumpto)
72	79	#
...	...	@@ -629,18 +636,35 @@ Lnovec:
629	636
630	637	#elif defined(__aarch64__)
631	638
	639	#ifndef __has_feature
	640	#define __has_feature(__feature) 0
	641	#endif
	642
632	643	#if defined(__ARM_FEATURE_GCS_DEFAULT)
633	644	.arch_extension gcs
634	645	#endif
635	646
636	647	//
637		// extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *);
	648	// extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *, unsigned);
638	649	//
639	650	// On entry:
640	651	// thread_state pointer is in x0
	652	// walked_frames counter is in x1
641	653	//
642	654	.p2align 2
643	655	DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
	656
	657	#if defined(_LIBUNWIND_TRACE_RET_INJECT)
	658	cbz w1, 1f
	659	0:
	660	subs w1, w1, #1
	661	adr x16, #8
	662	ret x16
	663
	664	b.ne 0b
	665	1:
	666	#endif
	667
644	668	// skip restore of x0,x1 for now
645	669	ldp x2, x3, [x0, #0x010]
646	670	ldp x4, x5, [x0, #0x020]
...	...	@@ -657,7 +681,7 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
657	681	ldp x24,x25, [x0, #0x0C0]
658	682	ldp x26,x27, [x0, #0x0D0]
659	683	ldp x28,x29, [x0, #0x0E0]
660		ldr x30, [x0, #0x100] // restore pc into lr
	684
661	685	#if defined(__ARM_FP) && __ARM_FP != 0
662	686	ldp d0, d1, [x0, #0x110]
663	687	ldp d2, d3, [x0, #0x120]
...	...	@@ -681,7 +705,18 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
681	705	// context struct, because it is allocated on the stack, and an exception
682	706	// could clobber the de-allocated portion of the stack after sp has been
683	707	// restored.
684		ldr x16, [x0, #0x0F8]
	708
	709	ldr x16, [x0, #0x0F8] // load sp into scratch
	710	ldr lr, [x0, #0x100] // restore pc into lr
	711
	712	#if __has_feature(ptrauth_calls)
	713	// The LR is signed with its address inside the register state. Time
	714	// to resign to be a regular ROP protected signed pointer
	715	add x1, x0, #0x100
	716	autib lr, x1
	717	pacib lr, x16 // signed the scratch register for sp
	718	#endif
	719
685	720	ldp x0, x1, [x0, #0x000] // restore x0,x1
686	721	mov sp,x16 // restore sp
687	722	#if defined(__ARM_FEATURE_GCS_DEFAULT)
...	...	@@ -694,7 +729,12 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto)
694	729	gcspushm x30
695	730	Lnogcs:
696	731	#endif
	732
	733	#if __has_feature(ptrauth_calls)
	734	retab
	735	#else
697	736	ret x30 // jump to pc
	737	#endif
698	738
699	739	#elif defined(__arm__) && !defined(__APPLE__)
700	740
...	...	@@ -1253,7 +1293,8 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind19Registers_loongarch6jumptoEv)
1253	1293
1254	1294	#endif
1255	1295
1256		#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */
	1296	#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */
1257	1297
1258	1298	NO_EXEC_STACK_DIRECTIVE
1259	1299
	1300	#endif /* !defined(__wasm__) */

lib/libunwind/src/UnwindRegistersSave.S+86-2

...	...	@@ -6,6 +6,8 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8
	9	#if !defined(__wasm__)
	10
9	11	#include "assembly.h"
10	12
11	13	#define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
...	...	@@ -16,13 +18,16 @@
16	18
17	19	#if defined(_AIX)
18	20	.toc
	21	#elif defined(__aarch64__) && defined(__ELF__) && defined(_LIBUNWIND_EXECUTE_ONLY_CODE)
	22	.section .text,"axy",@progbits,unique,0
19	23	#else
20	24	.text
21	25	#endif
22	26
23		#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__)
	27	#if !defined(__USING_SJLJ_EXCEPTIONS__)
24	28
25	29	#if defined(__i386__)
	30	.att_syntax
26	31
27	32	#
28	33	# extern int __unw_getcontext(unw_context_t* thread_state)
...	...	@@ -107,6 +112,7 @@ DEFINE_LIBUNWIND_FUNCTION("#__unw_getcontext")
107	112	.text
108	113
109	114	#elif defined(__x86_64__)
	115	.att_syntax
110	116
111	117	#
112	118	# extern int __unw_getcontext(unw_context_t* thread_state)
...	...	@@ -759,6 +765,10 @@ LnoR2Fix:
759	765
760	766	#elif defined(__aarch64__)
761	767
	768	#ifndef __has_feature
	769	#define __has_feature(__feature) 0
	770	#endif
	771
762	772	//
763	773	// extern int __unw_getcontext(unw_context_t* thread_state)
764	774	//
...	...	@@ -767,6 +777,11 @@ LnoR2Fix:
767	777	//
768	778	.p2align 2
769	779	DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext)
	780
	781	#if __has_feature(ptrauth_calls)
	782	pacibsp
	783	#endif
	784
770	785	stp x0, x1, [x0, #0x000]
771	786	stp x2, x3, [x0, #0x010]
772	787	stp x4, x5, [x0, #0x020]
...	...	@@ -807,7 +822,74 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext)
807	822	str d31, [x0, #0x208]
808	823	#endif
809	824	mov x0, #0 // return UNW_ESUCCESS
	825
	826	#if __has_feature(ptrauth_calls)
	827	retab
	828	#else
	829	ret
	830	#endif
	831
	832	//
	833	// extern "C" int64_t __libunwind_Registers_arm64_za_disable()
	834	//
	835	// This function implements the requirements of the __arm_za_disable ABI
	836	// routine, except that it will not abort; it will return a non-zero value
	837	// to signify the routine failed.
	838	//
	839	// Note: This function uses SME instructions. It must only be called if SME
	840	// has been confirmed to be available.
	841	//
	842	// On return:
	843	//
	844	// A status is placed in x0. A zero value indicates success; any non-zero
	845	// value indicates failure.
	846	//
	847	.p2align 2
	848	DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_za_disable)
	849	.variant_pcs __libunwind_Registers_arm64_za_disable
	850	#if __has_feature(ptrauth_calls)
	851	pacibsp
	852	#endif
	853	// If TPIDR2_EL0 is null, the subroutine just disables ZA.
	854	.inst 0xd53bd0b0 // mrs x16, TPIDR2_EL0
	855	cbz x16, 1f
	856
	857	// If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
	858	// nonzero, return a non-zero value (libunwind will then abort).
	859	ldrh w0, [x16, #10]
	860	cbnz w0, 2f
	861	ldr w0, [x16, #12]
	862	cbnz w0, 2f
	863
	864	// If num_za_save_slices is zero, the subroutine just disables ZA.
	865	ldrh w0, [x16, #8]
	866	cbz x0, 1f
	867
	868	// If za_save_buffer is NULL, the subroutine just disables ZA.
	869	ldr x16, [x16]
	870	cbz x16, 1f
	871
	872	// Store ZA to za_save_buffer.
	873	mov x15, xzr
	874	0:
	875	.inst 0xe1206200 // str za[w15,0], [x16]
	876	.inst 0x04305830 // addsvl x16, x16, #1
	877	add x15, x15, #1
	878	cmp x0, x15
	879	b.ne 0b
	880	1:
	881	// * Set TPIDR2_EL0 to null.
	882	.inst 0xd51bd0bf // msr TPIDR2_EL0, xzr
	883	// * Set PSTATE.ZA to 0.
	884	.inst 0xd503447f // smstop za
	885	// * Return zero (success)
	886	mov x0, xzr
	887	2:
	888	#if __has_feature(ptrauth_calls)
	889	retab
	890	#else
810	891	ret
	892	#endif
811	893
812	894	#elif defined(__arm__) && !defined(__APPLE__)
813	895
...	...	@@ -1232,6 +1314,8 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext)
1232	1314	WEAK_ALIAS(__unw_getcontext, unw_getcontext)
1233	1315	#endif
1234	1316
1235		#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */
	1317	#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */
1236	1318
1237	1319	NO_EXEC_STACK_DIRECTIVE
	1320
	1321	#endif /* !defined(__wasm__) */

lib/libunwind/src/assembly.h+5-1

...	...	@@ -15,7 +15,7 @@
15	15	#ifndef UNWIND_ASSEMBLY_H
16	16	#define UNWIND_ASSEMBLY_H
17	17
18		#if defined(__linux__) && defined(__CET__)
	18	#if defined(__CET__)
19	19	#include <cet.h>
20	20	#define _LIBUNWIND_CET_ENDBR _CET_ENDBR
21	21	#else
...	...	@@ -132,6 +132,10 @@
132	132
133	133	#if defined(__APPLE__)
134	134
	135	#if defined(__aarch64__) \|\| defined(__arm64__) \|\| defined(__arm64e__)
	136	#define _LIBUNWIND_TRACE_RET_INJECT 1
	137	#endif
	138
135	139	#define SYMBOL_IS_FUNC(name)
136	140	#define HIDDEN_SYMBOL(name) .private_extern name
137	141	#if defined(_LIBUNWIND_HIDE_SYMBOLS)

lib/libunwind/src/config.h+9

...	...	@@ -28,6 +28,9 @@
28	28	#define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 1
29	29	#define _LIBUNWIND_SUPPORT_DWARF_UNWIND 1
30	30	#endif
	31	#if defined(__aarch64__) \|\| defined(__arm64__) \|\| defined(__arm64e__)
	32	#define _LIBUNWIND_TRACE_RET_INJECT 1
	33	#endif
31	34	#elif defined(_WIN32)
32	35	#ifdef __SEH__
33	36	#define _LIBUNWIND_SUPPORT_SEH_UNWIND 1
...	...	@@ -61,6 +64,12 @@
61	64	#endif
62	65	#endif
63	66
	67	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	68	#define _LIBUNWIND_TRACE_NO_INLINE __attribute__((noinline, disable_tail_calls))
	69	#else
	70	#define _LIBUNWIND_TRACE_NO_INLINE
	71	#endif
	72
64	73	#if defined(_LIBUNWIND_HIDE_SYMBOLS)
65	74	// The CMake file passes -fvisibility=hidden to control ELF/Mach-O visibility.
66	75	#define _LIBUNWIND_EXPORT

lib/libunwind/src/gcc_personality_v0.c+79-6

...	...	@@ -31,6 +31,58 @@ EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, void *, PCONTEXT,
31	31	_Unwind_Personality_Fn);
32	32	#endif
33	33
	34	#ifndef __has_feature
	35	#define __has_feature(__feature) 0
	36	#endif
	37
	38	#if __has_feature(ptrauth_calls)
	39	#include <ptrauth.h>
	40
	41	// `__ptrauth_restricted_intptr` is a feature of apple clang that predates
	42	// support for direct application of `__ptrauth` to integer types. This
	43	// guard is necessary to support compilation with those compiler.
	44	#if __has_feature(ptrauth_restricted_intptr_qualifier)
	45	#define __ptrauth_gcc_personality_intptr(key, addressDiscriminated, \
	46	discriminator) \
	47	__ptrauth_restricted_intptr(key, addressDiscriminated, discriminator)
	48	#else
	49	#define __ptrauth_gcc_personality_intptr(key, addressDiscriminated, \
	50	discriminator) \
	51	__ptrauth(key, addressDiscriminated, discriminator)
	52	#endif
	53	#else
	54	#define __ptrauth_gcc_personality_intptr(...)
	55	#endif
	56
	57	#define __ptrauth_gcc_personality_func_key ptrauth_key_function_pointer
	58
	59	// ptrauth_string_discriminator("__gcc_personality_v0'funcStart") == 0xDFEB
	60	#define __ptrauth_gcc_personality_func_start \
	61	__ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \
	62	0xDFEB)
	63
	64	// ptrauth_string_discriminator("__gcc_personality_v0'start") == 0x52DC
	65	#define __ptrauth_gcc_personality_start \
	66	__ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \
	67	0x52DC)
	68
	69	// ptrauth_string_discriminator("__gcc_personality_v0'length") == 0xFFF7
	70	#define __ptrauth_gcc_personality_length \
	71	__ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \
	72	0xFFF7)
	73
	74	// ptrauth_string_discriminator("__gcc_personality_v0'landingPadOffset") ==
	75	// 0x6498
	76	#define __ptrauth_gcc_personality_lpoffset \
	77	__ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \
	78	0x6498)
	79
	80	// ptrauth_string_discriminator("__gcc_personality_v0'landingPad") == 0xA134
	81	#define __ptrauth_gcc_personality_lpad_disc 0xA134
	82	#define __ptrauth_gcc_personality_lpad \
	83	__ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \
	84	__ptrauth_gcc_personality_lpad_disc)
	85
34	86	// Pointer encodings documented at:
35	87	// http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html
36	88
...	...	@@ -206,7 +258,8 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
206	258	return continueUnwind(exceptionObject, context);
207	259
208	260	uintptr_t pc = (uintptr_t)_Unwind_GetIP(context) - 1;
209		uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context);
	261	uintptr_t __ptrauth_gcc_personality_func_start funcStart =
	262	(uintptr_t)_Unwind_GetRegionStart(context);
210	263	uintptr_t pcOffset = pc - funcStart;
211	264
212	265	// Parse LSDA header.
...	...	@@ -225,11 +278,14 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
225	278	const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength;
226	279	const uint8_t *p = callSiteTableStart;
227	280	while (p < callSiteTableEnd) {
228		uintptr_t start = readEncodedPointer(&p, callSiteEncoding);
229		size_t length = readEncodedPointer(&p, callSiteEncoding);
230		size_t landingPad = readEncodedPointer(&p, callSiteEncoding);
	281	uintptr_t __ptrauth_gcc_personality_start start =
	282	readEncodedPointer(&p, callSiteEncoding);
	283	size_t __ptrauth_gcc_personality_length length =
	284	readEncodedPointer(&p, callSiteEncoding);
	285	size_t __ptrauth_gcc_personality_lpoffset landingPadOffset =
	286	readEncodedPointer(&p, callSiteEncoding);
231	287	readULEB128(&p); // action value not used for C code
232		if (landingPad == 0)
	288	if (landingPadOffset == 0)
233	289	continue; // no landing pad for this entry
234	290	if ((start <= pcOffset) && (pcOffset < (start + length))) {
235	291	// Found landing pad for the PC.
...	...	@@ -239,7 +295,24 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
239	295	_Unwind_SetGR(context, __builtin_eh_return_data_regno(0),
240	296	(uintptr_t)exceptionObject);
241	297	_Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0);
242		_Unwind_SetIP(context, (funcStart + landingPad));
	298	size_t __ptrauth_gcc_personality_lpad landingPad =
	299	funcStart + landingPadOffset;
	300	#if __has_feature(ptrauth_calls)
	301	uintptr_t stackPointer = _Unwind_GetGR(context, -2);
	302	const uintptr_t existingDiscriminator = ptrauth_blend_discriminator(
	303	&landingPad, __ptrauth_gcc_personality_lpad_disc);
	304	// newIP is authenticated as if it were qualified with a pseudo qualifier
	305	// along the lines of:
	306	// __ptrauth(ptrauth_key_return_address, <stackPointer>, 0)
	307	// where the stack pointer is used in place of the strict storage
	308	// address.
	309	uintptr_t newIP = (uintptr_t)ptrauth_auth_and_resign(
	310	(void *)&landingPad, __ptrauth_gcc_personality_func_key,
	311	existingDiscriminator, ptrauth_key_return_address, stackPointer);
	312	_Unwind_SetIP(context, newIP);
	313	#else
	314	_Unwind_SetIP(context, landingPad);
	315	#endif
243	316	return _URC_INSTALL_CONTEXT;
244	317	}
245	318	}

lib/libunwind/src/libunwind.cpp+104-4

...	...	@@ -118,14 +118,55 @@ _LIBUNWIND_HIDDEN int __unw_set_reg(unw_cursor_t *cursor, unw_regnum_t regNum,
118	118	typedef LocalAddressSpace::pint_t pint_t;
119	119	AbstractUnwindCursor co = (AbstractUnwindCursor )cursor;
120	120	if (co->validReg(regNum)) {
121		co->setReg(regNum, (pint_t)value);
122	121	// special case altering IP to re-find info (being called by personality
123	122	// function)
124	123	if (regNum == UNW_REG_IP) {
125	124	unw_proc_info_t info;
126	125	// First, get the FDE for the old location and then update it.
127	126	co->getInfo(&info);
128		co->setInfoBasedOnIPRegister(false);
	127
	128	pint_t sp = (pint_t)co->getReg(UNW_REG_SP);
	129
	130	#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING)
	131	{
	132	// It is only valid to set the IP within the current function. This is
	133	// important for ptrauth, otherwise the IP cannot be correctly signed.
	134	// The current signature of `value` is via the schema:
	135	// __ptrauth(ptrauth_key_return_address, <<sp>>, 0)
	136	// For this to be generally usable we manually re-sign it to the
	137	// directly supported schema:
	138	// __ptrauth(ptrauth_key_return_address, 1, 0)
	139	unw_word_t
	140	__unwind_ptrauth_restricted_intptr(ptrauth_key_return_address, 1,
	141	0) authenticated_value;
	142	unw_word_t opaque_value = (uint64_t)ptrauth_auth_and_resign(
	143	(void *)value, ptrauth_key_return_address, sp,
	144	ptrauth_key_return_address, &authenticated_value);
	145	memmove(reinterpret_cast<void *>(&authenticated_value),
	146	reinterpret_cast<void *>(&opaque_value),
	147	sizeof(authenticated_value));
	148	if (authenticated_value < info.start_ip \|\|
	149	authenticated_value > info.end_ip)
	150	_LIBUNWIND_ABORT("PC vs frame info mismatch");
	151
	152	// PC should have been signed with the sp, so we verify that
	153	// roundtripping does not fail. The `ptrauth_auth_and_resign` is
	154	// guaranteed to trap on authentication failure even without FPAC
	155	// feature.
	156	pint_t pc = (pint_t)co->getReg(UNW_REG_IP);
	157	if (ptrauth_auth_and_resign((void *)pc, ptrauth_key_return_address, sp,
	158	ptrauth_key_return_address,
	159	sp) != (void *)pc) {
	160	_LIBUNWIND_LOG(
	161	"Bad unwind with PAuth-enabled ABI (0x%zX, 0x%zX)->0x%zX\n", pc,
	162	sp,
	163	(pint_t)ptrauth_auth_data((void *)pc, ptrauth_key_return_address,
	164	sp));
	165	_LIBUNWIND_ABORT("Bad unwind with PAuth-enabled ABI");
	166	}
	167	}
	168	#endif
	169
129	170	// If the original call expects stack adjustment, perform this now.
130	171	// Normal frame unwinding would have included the offset already in the
131	172	// CFA computation.
...	...	@@ -133,7 +174,11 @@ _LIBUNWIND_HIDDEN int __unw_set_reg(unw_cursor_t *cursor, unw_regnum_t regNum,
133	174	// this should actually be - info.gp. LLVM doesn't currently support
134	175	// any such platforms and Clang doesn't export a macro for them.
135	176	if (info.gp)
136		co->setReg(UNW_REG_SP, co->getReg(UNW_REG_SP) + info.gp);
	177	co->setReg(UNW_REG_SP, sp + info.gp);
	178	co->setReg(UNW_REG_IP, value);
	179	co->setInfoBasedOnIPRegister(false);
	180	} else {
	181	co->setReg(regNum, (pint_t)value);
137	182	}
138	183	return UNW_ESUCCESS;
139	184	}
...	...	@@ -205,7 +250,27 @@ _LIBUNWIND_HIDDEN int __unw_get_proc_info(unw_cursor_t *cursor,
205	250	}
206	251	_LIBUNWIND_WEAK_ALIAS(__unw_get_proc_info, unw_get_proc_info)
207	252
208		/// Resume execution at cursor position (aka longjump).
	253	/// Rebalance the execution flow by injecting the right amount of `ret`
	254	/// instruction relatively to the amount of `walkedFrames` then resume execution
	255	/// at cursor position (aka longjump).
	256	_LIBUNWIND_HIDDEN int __unw_resume_with_frames_walked(unw_cursor_t *cursor,
	257	unsigned walkedFrames) {
	258	_LIBUNWIND_TRACE_API("__unw_resume(cursor=%p, walkedFrames=%u)",
	259	static_cast<void *>(cursor), walkedFrames);
	260	#if __has_feature(address_sanitizer) \|\| defined(__SANITIZE_ADDRESS__)
	261	// Inform the ASan runtime that now might be a good time to clean stuff up.
	262	__asan_handle_no_return();
	263	#endif
	264	#ifdef _LIBUNWIND_TRACE_RET_INJECT
	265	AbstractUnwindCursor co = (AbstractUnwindCursor )cursor;
	266	co->setWalkedFrames(walkedFrames);
	267	#endif
	268	return __unw_resume(cursor);
	269	}
	270	_LIBUNWIND_WEAK_ALIAS(__unw_resume_with_frames_walked,
	271	unw_resume_with_frames_walked)
	272
	273	/// Legacy function. Resume execution at cursor position (aka longjump).
209	274	_LIBUNWIND_HIDDEN int __unw_resume(unw_cursor_t *cursor) {
210	275	_LIBUNWIND_TRACE_API("__unw_resume(cursor=%p)", static_cast<void *>(cursor));
211	276	#if __has_feature(address_sanitizer) \|\| defined(__SANITIZE_ADDRESS__)
...	...	@@ -347,6 +412,41 @@ void __unw_remove_dynamic_eh_frame_section(unw_word_t eh_frame_start) {
347	412	}
348	413
349	414	#endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
	415
	416	/// Maps the UNW_* error code to a textual representation
	417	_LIBUNWIND_HIDDEN const char *__unw_strerror(int error_code) {
	418	switch (error_code) {
	419	case UNW_ESUCCESS:
	420	return "no error";
	421	case UNW_EUNSPEC:
	422	return "unspecified (general) error";
	423	case UNW_ENOMEM:
	424	return "out of memory";
	425	case UNW_EBADREG:
	426	return "bad register number";
	427	case UNW_EREADONLYREG:
	428	return "attempt to write read-only register";
	429	case UNW_ESTOPUNWIND:
	430	return "stop unwinding";
	431	case UNW_EINVALIDIP:
	432	return "invalid IP";
	433	case UNW_EBADFRAME:
	434	return "bad frame";
	435	case UNW_EINVAL:
	436	return "unsupported operation or bad value";
	437	case UNW_EBADVERSION:
	438	return "unwind info has unsupported version";
	439	case UNW_ENOINFO:
	440	return "no unwind info found";
	441	#if defined(_LIBUNWIND_TARGET_AARCH64) && !defined(_LIBUNWIND_IS_NATIVE_ONLY)
	442	case UNW_ECROSSRASIGNING:
	443	return "cross unwind with return address signing";
	444	#endif
	445	}
	446	return "invalid error code";
	447	}
	448	_LIBUNWIND_WEAK_ALIAS(__unw_strerror, unw_strerror)
	449
350	450	#endif // !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__)
351	451
352	452	#ifdef __APPLE__

lib/libunwind/src/libunwind_ext.h+7-1

...	...	@@ -26,11 +26,16 @@ extern "C" {
26	26	extern int __unw_getcontext(unw_context_t *);
27	27	extern int __unw_init_local(unw_cursor_t , unw_context_t );
28	28	extern int __unw_step(unw_cursor_t *);
	29	extern int __unw_step_stage2(unw_cursor_t *);
29	30	extern int __unw_get_reg(unw_cursor_t , unw_regnum_t, unw_word_t );
30	31	extern int __unw_get_fpreg(unw_cursor_t , unw_regnum_t, unw_fpreg_t );
31	32	extern int __unw_set_reg(unw_cursor_t *, unw_regnum_t, unw_word_t);
32	33	extern int __unw_set_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t);
33		extern int __unw_resume(unw_cursor_t *);
	34	_LIBUNWIND_TRACE_NO_INLINE
	35	extern int __unw_resume_with_frames_walked(unw_cursor_t *, unsigned);
	36	// `__unw_resume` is a legacy function. Use `__unw_resume_with_frames_walked` instead.
	37	_LIBUNWIND_TRACE_NO_INLINE
	38	extern int __unw_resume(unw_cursor_t *);
34	39
35	40	#ifdef __arm__
36	41	/* Save VFP registers in FSTMX format (instead of FSTMD). */
...	...	@@ -42,6 +47,7 @@ extern int __unw_get_proc_info(unw_cursor_t , unw_proc_info_t );
42	47	extern int __unw_is_fpreg(unw_cursor_t *, unw_regnum_t);
43	48	extern int __unw_is_signal_frame(unw_cursor_t *);
44	49	extern int __unw_get_proc_name(unw_cursor_t , char , size_t, unw_word_t *);
	50	extern const char *__unw_strerror(int);
45	51
46	52	#if defined(_AIX)
47	53	extern uintptr_t __unw_get_data_rel_base(unw_cursor_t *);

lib/libunwind/src/shadow_stack_unwind.h+2-2

...	...	@@ -12,8 +12,8 @@
12	12
13	13	#include "libunwind.h"
14	14
15		// Currently, CET is implemented on Linux x86 platforms.
16		#if defined(_LIBUNWIND_TARGET_LINUX) && defined(__CET__) && defined(__SHSTK__)
	15	// Currently, CET is implemented on some ELF x86 platforms.
	16	#if defined(__CET__) && defined(__SHSTK__)
17	17	#define _LIBUNWIND_USE_CET 1
18	18	#endif
19	19

lib/std/Io/test.zig-1

...	...	@@ -137,7 +137,6 @@ test "File.setLength" {
137	137	test "legacy setLength" {
138	138	// https://github.com/ziglang/zig/issues/20747 (open fd does not have write permission)
139	139	if (builtin.os.tag == .wasi and builtin.link_libc) return error.SkipZigTest;
140		if (builtin.cpu.arch.isMIPS64() and (builtin.abi == .gnuabin32 or builtin.abi == .muslabin32)) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23806
141	140
142	141	const io = testing.io;
143	142

lib/std/Target.zig+11-2

...	...	@@ -845,6 +845,12 @@ pub const Abi = enum {
845	845	=> .eabi,
846	846	else => .none,
847	847	},
	848	.fuchsia => switch (arch) {
	849	.arm,
	850	.thumb,
	851	=> .eabihf,
	852	else => .none,
	853	},
848	854	.haiku => switch (arch) {
849	855	.arm,
850	856	.powerpc,
...	...	@@ -928,7 +934,6 @@ pub const Abi = enum {
928	934	.wasi, .emscripten => .musl,
929	935
930	936	.contiki,
931		.fuchsia,
932	937	.hermit,
933	938	.illumos,
934	939	.managarm,
...	...	@@ -1014,6 +1019,7 @@ pub const Abi = enum {
1014	1019	.gnueabi,
1015	1020	.musleabi,
1016	1021	.gnusf,
	1022	.muslsf,
1017	1023	.ohoseabi,
1018	1024	=> .soft,
1019	1025	else => .hard,
...	...	@@ -1219,7 +1225,7 @@ pub const Cpu = struct {
1219	1225	pub const Set = struct {
1220	1226	ints: [usize_count]usize,
1221	1227
1222		pub const needed_bit_count = 317;
	1228	pub const needed_bit_count = 347;
1223	1229	pub const byte_count = (needed_bit_count + 7) / 8;
1224	1230	pub const usize_count = (byte_count + (@sizeOf(usize) - 1)) / @sizeOf(usize);
1225	1231	pub const Index = std.math.Log2Int(std.meta.Int(.unsigned, usize_count * @bitSizeOf(usize)));
...	...	@@ -2055,6 +2061,7 @@ pub const Cpu = struct {
2055	2061	.hppa => &hppa.cpu.pa_7300lc,
2056	2062	.kvx => &kvx.cpu.coolidge_v2,
2057	2063	.lanai => &lanai.cpu.v11, // clang does not have a generic lanai model.
	2064	.loongarch32 => &loongarch.cpu.la32v1_0,
2058	2065	.loongarch64 => &loongarch.cpu.la64v1_0,
2059	2066	.m68k => &m68k.cpu.M68000,
2060	2067	.mips => &mips.cpu.mips32r2,
...	...	@@ -2442,8 +2449,10 @@ pub const DynamicLinker = struct {
2442	2449	pub fn standard(cpu: Cpu, os: Os, abi: Abi) DynamicLinker {
2443	2450	return switch (os.tag) {
2444	2451	.fuchsia => switch (cpu.arch) {
	2452	.arm,
2445	2453	.aarch64,
2446	2454	.riscv64,
	2455	.thumb,
2447	2456	.x86_64,
2448	2457	=> init("ld.so.1"), // Fuchsia is unusual in that `DT_INTERP` is just a basename.
2449	2458	else => none,

lib/std/Target/aarch64.zig+620-77

...	...	@@ -9,6 +9,7 @@ pub const Feature = enum {
9	9	addr_lsl_slow_14,
10	10	aes,
11	11	aggressive_fma,
	12	aggressive_interleaving,
12	13	alternate_sextload_cvt_f32_pattern,
13	14	altnzcv,
14	15	alu_lsl_fast,
...	...	@@ -22,6 +23,7 @@ pub const Feature = enum {
22	23	bf16,
23	24	brbe,
24	25	bti,
	26	btie,
25	27	call_saved_x10,
26	28	call_saved_x11,
27	29	call_saved_x12,
...	...	@@ -36,6 +38,7 @@ pub const Feature = enum {
36	38	ccpp,
37	39	chk,
38	40	clrbhb,
	41	cmh,
39	42	cmp_bcc_fusion,
40	43	cmpbr,
41	44	complxnum,
...	...	@@ -48,7 +51,9 @@ pub const Feature = enum {
48	51	disable_fast_inc_vl,
49	52	disable_latency_sched_heuristic,
50	53	disable_ldp,
	54	disable_maximize_scalable_bandwidth,
51	55	disable_stp,
	56	disable_unpredicated_ld_st_lower,
52	57	dit,
53	58	dotprod,
54	59	ecv,
...	...	@@ -58,6 +63,9 @@ pub const Feature = enum {
58	63	ete,
59	64	execute_only,
60	65	exynos_cheap_as_move,
	66	f16f32dot,
	67	f16f32mm,
	68	f16mm,
61	69	f32mm,
62	70	f64mm,
63	71	f8f16mm,
...	...	@@ -86,7 +94,9 @@ pub const Feature = enum {
86	94	fuse_arith_logic,
87	95	fuse_crypto_eor,
88	96	fuse_csel,
	97	fuse_cset,
89	98	fuse_literals,
	99	gcie,
90	100	gcs,
91	101	harden_sls_blr,
92	102	harden_sls_nocomdat,
...	...	@@ -99,22 +109,27 @@ pub const Feature = enum {
99	109	ldp_aligned_only,
100	110	lor,
101	111	ls64,
	112	lscp,
102	113	lse,
103	114	lse128,
104	115	lse2,
105	116	lsfe,
106	117	lsui,
107	118	lut,
	119	max_interleave_factor_4,
108	120	mec,
109	121	mops,
	122	mops_go,
110	123	mpam,
	124	mpamv2,
111	125	mte,
	126	mtetc,
112	127	neon,
113	128	nmi,
114	129	no_bti_at_return_twice,
115	130	no_neg_immediates,
116	131	no_sve_fp_ld1r,
117		no_zcz_fp,
	132	no_zcz_fpr64,
118	133	nv,
119	134	occmo,
120	135	olympus,
...	...	@@ -125,6 +140,7 @@ pub const Feature = enum {
125	140	pauth_lr,
126	141	pcdphint,
127	142	perfmon,
	143	poe2,
128	144	pops,
129	145	predictable_select_expensive,
130	146	predres,
...	...	@@ -174,6 +190,7 @@ pub const Feature = enum {
174	190	sme2,
175	191	sme2p1,
176	192	sme2p2,
	193	sme2p3,
177	194	sme_b16b16,
178	195	sme_f16f16,
179	196	sme_f64f64,
...	...	@@ -206,19 +223,22 @@ pub const Feature = enum {
206	223	sve2_sm4,
207	224	sve2p1,
208	225	sve2p2,
	226	sve2p3,
209	227	sve_aes,
210	228	sve_aes2,
211	229	sve_b16b16,
	230	sve_b16mm,
212	231	sve_bfscale,
213	232	sve_bitperm,
214	233	sve_f16f32mm,
215	234	sve_sha3,
216	235	sve_sm4,
217	236	tagged_globals,
	237	tev,
218	238	the,
219	239	tlb_rmi,
	240	tlbid,
220	241	tlbiw,
221		tme,
222	242	tpidr_el1,
223	243	tpidr_el2,
224	244	tpidr_el3,
...	...	@@ -230,6 +250,7 @@ pub const Feature = enum {
230	250	use_fixed_over_scalable_if_equal_cost,
231	251	use_postra_scheduler,
232	252	use_reciprocal_square_root,
	253	use_wzr_to_vec_move,
233	254	v8_1a,
234	255	v8_2a,
235	256	v8_3a,
...	...	@@ -247,17 +268,20 @@ pub const Feature = enum {
247	268	v9_4a,
248	269	v9_5a,
249	270	v9_6a,
	271	v9_7a,
250	272	v9a,
251	273	vh,
252	274	wfxt,
253	275	xs,
	276	zcm_fpr128,
254	277	zcm_fpr32,
255	278	zcm_fpr64,
256	279	zcm_gpr32,
257	280	zcm_gpr64,
258		zcz,
259	281	zcz_fp_workaround,
260		zcz_gp,
	282	zcz_fpr128,
	283	zcz_gpr32,
	284	zcz_gpr64,
261	285	};
262	286
263	287	pub const featureSet = CpuFeature.FeatureSetFns(Feature).featureSet;
...	...	@@ -274,9 +298,12 @@ pub const all_features = blk: {
274	298	.llvm_name = "a320",
275	299	.description = "Cortex-A320 ARM processors",
276	300	.dependencies = featureSet(&[_]Feature{
	301	.aggressive_interleaving,
277	302	.fuse_adrp_add,
278	303	.fuse_aes,
	304	.use_fixed_over_scalable_if_equal_cost,
279	305	.use_postra_scheduler,
	306	.use_wzr_to_vec_move,
280	307	}),
281	308	};
282	309	result[@intFromEnum(Feature.addr_lsl_slow_14)] = .{
...	...	@@ -296,6 +323,11 @@ pub const all_features = blk: {
296	323	.description = "Enable Aggressive FMA for floating-point.",
297	324	.dependencies = featureSet(&[_]Feature{}),
298	325	};
	326	result[@intFromEnum(Feature.aggressive_interleaving)] = .{
	327	.llvm_name = "aggressive-interleaving",
	328	.description = "Make use of aggressive interleaving during vectorization",
	329	.dependencies = featureSet(&[_]Feature{}),
	330	};
299	331	result[@intFromEnum(Feature.alternate_sextload_cvt_f32_pattern)] = .{
300	332	.llvm_name = "alternate-sextload-cvt-f32-pattern",
301	333	.description = "Use alternative pattern for sextload convert to f32",
...	...	@@ -367,6 +399,11 @@ pub const all_features = blk: {
367	399	.description = "Enable Branch Target Identification",
368	400	.dependencies = featureSet(&[_]Feature{}),
369	401	};
	402	result[@intFromEnum(Feature.btie)] = .{
	403	.llvm_name = "btie",
	404	.description = "Enable Enhanced Branch Target Identification extension",
	405	.dependencies = featureSet(&[_]Feature{}),
	406	};
370	407	result[@intFromEnum(Feature.call_saved_x10)] = .{
371	408	.llvm_name = "call-saved-x10",
372	409	.description = "Make X10 callee saved.",
...	...	@@ -439,6 +476,11 @@ pub const all_features = blk: {
439	476	.description = "Enable Clear BHB instruction",
440	477	.dependencies = featureSet(&[_]Feature{}),
441	478	};
	479	result[@intFromEnum(Feature.cmh)] = .{
	480	.llvm_name = "cmh",
	481	.description = "Enable Armv9.7-A Contention Management Hints",
	482	.dependencies = featureSet(&[_]Feature{}),
	483	};
442	484	result[@intFromEnum(Feature.cmp_bcc_fusion)] = .{
443	485	.llvm_name = "cmp-bcc-fusion",
444	486	.description = "CPU fuses cmp+bcc operations",
...	...	@@ -506,11 +548,21 @@ pub const all_features = blk: {
506	548	.description = "Do not emit ldp",
507	549	.dependencies = featureSet(&[_]Feature{}),
508	550	};
	551	result[@intFromEnum(Feature.disable_maximize_scalable_bandwidth)] = .{
	552	.llvm_name = "disable-maximize-scalable-bandwidth",
	553	.description = "Determine the maximum scalable vector length for a loop by the largest scalar type rather than the smallest",
	554	.dependencies = featureSet(&[_]Feature{}),
	555	};
509	556	result[@intFromEnum(Feature.disable_stp)] = .{
510	557	.llvm_name = "disable-stp",
511	558	.description = "Do not emit stp",
512	559	.dependencies = featureSet(&[_]Feature{}),
513	560	};
	561	result[@intFromEnum(Feature.disable_unpredicated_ld_st_lower)] = .{
	562	.llvm_name = "disable-unpredicated-ld-st-lower",
	563	.description = "Disable lowering unpredicated loads/stores as LDR/STR",
	564	.dependencies = featureSet(&[_]Feature{}),
	565	};
514	566	result[@intFromEnum(Feature.dit)] = .{
515	567	.llvm_name = "dit",
516	568	.description = "Enable Armv8.4-A Data Independent Timing instructions",
...	...	@@ -560,6 +612,30 @@ pub const all_features = blk: {
560	612	.description = "Use Exynos specific handling of cheap instructions",
561	613	.dependencies = featureSet(&[_]Feature{}),
562	614	};
	615	result[@intFromEnum(Feature.f16f32dot)] = .{
	616	.llvm_name = "f16f32dot",
	617	.description = "Enable Armv9.7-A Advanced SIMD half-precision dot product accumulate to single-precision",
	618	.dependencies = featureSet(&[_]Feature{
	619	.fullfp16,
	620	.neon,
	621	}),
	622	};
	623	result[@intFromEnum(Feature.f16f32mm)] = .{
	624	.llvm_name = "f16f32mm",
	625	.description = "Enable Armv9.7-A Advanced SIMD half-precision matrix multiply-accumulate to single-precision",
	626	.dependencies = featureSet(&[_]Feature{
	627	.fullfp16,
	628	.neon,
	629	}),
	630	};
	631	result[@intFromEnum(Feature.f16mm)] = .{
	632	.llvm_name = "f16mm",
	633	.description = "Enable Armv9.7-A non-widening half-precision matrix multiply-accumulate",
	634	.dependencies = featureSet(&[_]Feature{
	635	.fullfp16,
	636	.neon,
	637	}),
	638	};
563	639	result[@intFromEnum(Feature.f32mm)] = .{
564	640	.llvm_name = "f32mm",
565	641	.description = "Enable Matrix Multiply FP32 Extension",
...	...	@@ -729,7 +805,12 @@ pub const all_features = blk: {
729	805	};
730	806	result[@intFromEnum(Feature.fuse_csel)] = .{
731	807	.llvm_name = "fuse-csel",
732		.description = "CPU fuses conditional select operations",
	808	.description = "CPU can fuse CMP and CSEL operations",
	809	.dependencies = featureSet(&[_]Feature{}),
	810	};
	811	result[@intFromEnum(Feature.fuse_cset)] = .{
	812	.llvm_name = "fuse-cset",
	813	.description = "CPU can fuse CMP and CSET operations",
733	814	.dependencies = featureSet(&[_]Feature{}),
734	815	};
735	816	result[@intFromEnum(Feature.fuse_literals)] = .{
...	...	@@ -737,6 +818,11 @@ pub const all_features = blk: {
737	818	.description = "CPU fuses literal generation operations",
738	819	.dependencies = featureSet(&[_]Feature{}),
739	820	};
	821	result[@intFromEnum(Feature.gcie)] = .{
	822	.llvm_name = "gcie",
	823	.description = "Enable GICv5 (Generic Interrupt Controller) CPU Interface Extension",
	824	.dependencies = featureSet(&[_]Feature{}),
	825	};
740	826	result[@intFromEnum(Feature.gcs)] = .{
741	827	.llvm_name = "gcs",
742	828	.description = "Enable Armv9.4-A Guarded Call Stack Extension",
...	...	@@ -805,6 +891,11 @@ pub const all_features = blk: {
805	891	.description = "Enable Armv8.7-A LD64B/ST64B Accelerator Extension",
806	892	.dependencies = featureSet(&[_]Feature{}),
807	893	};
	894	result[@intFromEnum(Feature.lscp)] = .{
	895	.llvm_name = "lscp",
	896	.description = "Enable Armv9.7-A Load-acquire and store-release pair extension",
	897	.dependencies = featureSet(&[_]Feature{}),
	898	};
808	899	result[@intFromEnum(Feature.lse)] = .{
809	900	.llvm_name = "lse",
810	901	.description = "Enable Armv8.1-A Large System Extension (LSE) atomic instructions",
...	...	@@ -841,6 +932,11 @@ pub const all_features = blk: {
841	932	.neon,
842	933	}),
843	934	};
	935	result[@intFromEnum(Feature.max_interleave_factor_4)] = .{
	936	.llvm_name = "max-interleave-factor-4",
	937	.description = "Set the MaxInterleaveFactor to 4 (from the default 2)",
	938	.dependencies = featureSet(&[_]Feature{}),
	939	};
844	940	result[@intFromEnum(Feature.mec)] = .{
845	941	.llvm_name = "mec",
846	942	.description = "Enable Memory Encryption Contexts Extension",
...	...	@@ -853,16 +949,36 @@ pub const all_features = blk: {
853	949	.description = "Enable Armv8.8-A memcpy and memset acceleration instructions",
854	950	.dependencies = featureSet(&[_]Feature{}),
855	951	};
	952	result[@intFromEnum(Feature.mops_go)] = .{
	953	.llvm_name = "mops-go",
	954	.description = "Enable memset acceleration granule only",
	955	.dependencies = featureSet(&[_]Feature{
	956	.mops,
	957	.mte,
	958	}),
	959	};
856	960	result[@intFromEnum(Feature.mpam)] = .{
857	961	.llvm_name = "mpam",
858	962	.description = "Enable Armv8.4-A Memory system Partitioning and Monitoring extension",
859	963	.dependencies = featureSet(&[_]Feature{}),
860	964	};
	965	result[@intFromEnum(Feature.mpamv2)] = .{
	966	.llvm_name = "mpamv2",
	967	.description = "Enable Armv9.7-A MPAMv2 Lookaside Buffer Invalidate instructions",
	968	.dependencies = featureSet(&[_]Feature{}),
	969	};
861	970	result[@intFromEnum(Feature.mte)] = .{
862	971	.llvm_name = "mte",
863	972	.description = "Enable Memory Tagging Extension",
864	973	.dependencies = featureSet(&[_]Feature{}),
865	974	};
	975	result[@intFromEnum(Feature.mtetc)] = .{
	976	.llvm_name = "mtetc",
	977	.description = "Enable Virtual Memory Tagging Extension",
	978	.dependencies = featureSet(&[_]Feature{
	979	.mte,
	980	}),
	981	};
866	982	result[@intFromEnum(Feature.neon)] = .{
867	983	.llvm_name = "neon",
868	984	.description = "Enable Advanced SIMD instructions",
...	...	@@ -890,9 +1006,9 @@ pub const all_features = blk: {
890	1006	.description = "Avoid using LD1RX instructions for FP",
891	1007	.dependencies = featureSet(&[_]Feature{}),
892	1008	};
893		result[@intFromEnum(Feature.no_zcz_fp)] = .{
894		.llvm_name = "no-zcz-fp",
895		.description = "Has no zero-cycle zeroing instructions for FP registers",
	1009	result[@intFromEnum(Feature.no_zcz_fpr64)] = .{
	1010	.llvm_name = "no-zcz-fpr64",
	1011	.description = "Has no zero-cycle zeroing instructions for FPR64 registers",
896	1012	.dependencies = featureSet(&[_]Feature{}),
897	1013	};
898	1014	result[@intFromEnum(Feature.nv)] = .{
...	...	@@ -914,6 +1030,7 @@ pub const all_features = blk: {
914	1030	.enable_select_opt,
915	1031	.fuse_adrp_add,
916	1032	.fuse_aes,
	1033	.max_interleave_factor_4,
917	1034	.predictable_select_expensive,
918	1035	.use_fixed_over_scalable_if_equal_cost,
919	1036	.use_postra_scheduler,
...	...	@@ -956,6 +1073,11 @@ pub const all_features = blk: {
956	1073	.description = "Enable Armv8.0-A PMUv3 Performance Monitors extension",
957	1074	.dependencies = featureSet(&[_]Feature{}),
958	1075	};
	1076	result[@intFromEnum(Feature.poe2)] = .{
	1077	.llvm_name = "poe2",
	1078	.description = "Enable Stage 1 Permission Overlays Extension 2 instructions",
	1079	.dependencies = featureSet(&[_]Feature{}),
	1080	};
959	1081	result[@intFromEnum(Feature.pops)] = .{
960	1082	.llvm_name = "pops",
961	1083	.description = "Enable Armv9.6-A Point Of Physical Storage (PoPS) DC instructions",
...	...	@@ -1224,6 +1346,13 @@ pub const all_features = blk: {
1224	1346	.sme2p1,
1225	1347	}),
1226	1348	};
	1349	result[@intFromEnum(Feature.sme2p3)] = .{
	1350	.llvm_name = "sme2p3",
	1351	.description = "Enable Armv9.7-A Scalable Matrix Extension 2.3 instructions",
	1352	.dependencies = featureSet(&[_]Feature{
	1353	.sme2p2,
	1354	}),
	1355	};
1227	1356	result[@intFromEnum(Feature.sme_b16b16)] = .{
1228	1357	.llvm_name = "sme-b16b16",
1229	1358	.description = "Enable SME2.1 ZA-targeting non-widening BFloat16 instructions",
...	...	@@ -1447,6 +1576,13 @@ pub const all_features = blk: {
1447	1576	.sve2p1,
1448	1577	}),
1449	1578	};
	1579	result[@intFromEnum(Feature.sve2p3)] = .{
	1580	.llvm_name = "sve2p3",
	1581	.description = "Enable Armv9.7-A Scalable Vector Extension 2.3 instructions",
	1582	.dependencies = featureSet(&[_]Feature{
	1583	.sve2p2,
	1584	}),
	1585	};
1450	1586	result[@intFromEnum(Feature.sve_aes)] = .{
1451	1587	.llvm_name = "sve-aes",
1452	1588	.description = "Enable SVE AES and quadword SVE polynomial multiply instructions",
...	...	@@ -1464,6 +1600,13 @@ pub const all_features = blk: {
1464	1600	.description = "Enable SVE2 non-widening and SME2 Z-targeting non-widening BFloat16 instructions",
1465	1601	.dependencies = featureSet(&[_]Feature{}),
1466	1602	};
	1603	result[@intFromEnum(Feature.sve_b16mm)] = .{
	1604	.llvm_name = "sve-b16mm",
	1605	.description = "Enable Armv9.7-A SVE non-widening BFloat16 matrix multiply-accumulate",
	1606	.dependencies = featureSet(&[_]Feature{
	1607	.sve,
	1608	}),
	1609	};
1467	1610	result[@intFromEnum(Feature.sve_bfscale)] = .{
1468	1611	.llvm_name = "sve-bfscale",
1469	1612	.description = "Enable Armv9.6-A SVE BFloat16 scaling instructions",
...	...	@@ -1500,6 +1643,11 @@ pub const all_features = blk: {
1500	1643	.description = "Use an instruction sequence for taking the address of a global that allows a memory tag in the upper address bits",
1501	1644	.dependencies = featureSet(&[_]Feature{}),
1502	1645	};
	1646	result[@intFromEnum(Feature.tev)] = .{
	1647	.llvm_name = "tev",
	1648	.description = "Enable TIndex Exception-like Vector instructions",
	1649	.dependencies = featureSet(&[_]Feature{}),
	1650	};
1503	1651	result[@intFromEnum(Feature.the)] = .{
1504	1652	.llvm_name = "the",
1505	1653	.description = "Enable Armv8.9-A Translation Hardening Extension",
...	...	@@ -1510,16 +1658,16 @@ pub const all_features = blk: {
1510	1658	.description = "Enable Armv8.4-A TLB Range and Maintenance instructions",
1511	1659	.dependencies = featureSet(&[_]Feature{}),
1512	1660	};
	1661	result[@intFromEnum(Feature.tlbid)] = .{
	1662	.llvm_name = "tlbid",
	1663	.description = "Enable Armv9.7-A TLBI Domains extension",
	1664	.dependencies = featureSet(&[_]Feature{}),
	1665	};
1513	1666	result[@intFromEnum(Feature.tlbiw)] = .{
1514	1667	.llvm_name = "tlbiw",
1515	1668	.description = "Enable Armv9.5-A TLBI VMALL for Dirty State",
1516	1669	.dependencies = featureSet(&[_]Feature{}),
1517	1670	};
1518		result[@intFromEnum(Feature.tme)] = .{
1519		.llvm_name = "tme",
1520		.description = "Enable Transactional Memory Extension",
1521		.dependencies = featureSet(&[_]Feature{}),
1522		};
1523	1671	result[@intFromEnum(Feature.tpidr_el1)] = .{
1524	1672	.llvm_name = "tpidr-el1",
1525	1673	.description = "Permit use of TPIDR_EL1 for the TLS base",
...	...	@@ -1575,6 +1723,11 @@ pub const all_features = blk: {
1575	1723	.description = "Use the reciprocal square root approximation",
1576	1724	.dependencies = featureSet(&[_]Feature{}),
1577	1725	};
	1726	result[@intFromEnum(Feature.use_wzr_to_vec_move)] = .{
	1727	.llvm_name = "use-wzr-to-vec-move",
	1728	.description = "Move from WZR to insert 0 into vector registers",
	1729	.dependencies = featureSet(&[_]Feature{}),
	1730	};
1578	1731	result[@intFromEnum(Feature.v8_1a)] = .{
1579	1732	.llvm_name = "v8.1a",
1580	1733	.description = "Support ARM v8.1a architecture",
...	...	@@ -1783,6 +1936,16 @@ pub const all_features = blk: {
1783	1936	.v9_5a,
1784	1937	}),
1785	1938	};
	1939	result[@intFromEnum(Feature.v9_7a)] = .{
	1940	.llvm_name = "v9.7a",
	1941	.description = "Support ARM v9.7a architecture",
	1942	.dependencies = featureSet(&[_]Feature{
	1943	.f16f32dot,
	1944	.fprcvt,
	1945	.sve2p3,
	1946	.v9_6a,
	1947	}),
	1948	};
1786	1949	result[@intFromEnum(Feature.v9a)] = .{
1787	1950	.llvm_name = "v9a",
1788	1951	.description = "Support ARM v9a architecture",
...	...	@@ -1808,6 +1971,11 @@ pub const all_features = blk: {
1808	1971	.description = "Enable Armv8.7-A limited-TLB-maintenance instruction",
1809	1972	.dependencies = featureSet(&[_]Feature{}),
1810	1973	};
	1974	result[@intFromEnum(Feature.zcm_fpr128)] = .{
	1975	.llvm_name = "zcm-fpr128",
	1976	.description = "Has zero-cycle register moves for FPR128 registers",
	1977	.dependencies = featureSet(&[_]Feature{}),
	1978	};
1811	1979	result[@intFromEnum(Feature.zcm_fpr32)] = .{
1812	1980	.llvm_name = "zcm-fpr32",
1813	1981	.description = "Has zero-cycle register moves for FPR32 registers",
...	...	@@ -1828,21 +1996,24 @@ pub const all_features = blk: {
1828	1996	.description = "Has zero-cycle register moves for GPR64 registers",
1829	1997	.dependencies = featureSet(&[_]Feature{}),
1830	1998	};
1831		result[@intFromEnum(Feature.zcz)] = .{
1832		.llvm_name = "zcz",
1833		.description = "Has zero-cycle zeroing instructions",
1834		.dependencies = featureSet(&[_]Feature{
1835		.zcz_gp,
1836		}),
1837		};
1838	1999	result[@intFromEnum(Feature.zcz_fp_workaround)] = .{
1839	2000	.llvm_name = "zcz-fp-workaround",
1840	2001	.description = "The zero-cycle floating-point zeroing instruction has a bug",
1841	2002	.dependencies = featureSet(&[_]Feature{}),
1842	2003	};
1843		result[@intFromEnum(Feature.zcz_gp)] = .{
1844		.llvm_name = "zcz-gp",
1845		.description = "Has zero-cycle zeroing instructions for generic registers",
	2004	result[@intFromEnum(Feature.zcz_fpr128)] = .{
	2005	.llvm_name = "zcz-fpr128",
	2006	.description = "Has zero-cycle zeroing instructions for FPR128 registers",
	2007	.dependencies = featureSet(&[_]Feature{}),
	2008	};
	2009	result[@intFromEnum(Feature.zcz_gpr32)] = .{
	2010	.llvm_name = "zcz-gpr32",
	2011	.description = "Has zero-cycle zeroing instructions for GPR32 registers",
	2012	.dependencies = featureSet(&[_]Feature{}),
	2013	};
	2014	result[@intFromEnum(Feature.zcz_gpr64)] = .{
	2015	.llvm_name = "zcz-gpr64",
	2016	.description = "Has zero-cycle zeroing instructions for GPR64 registers",
1846	2017	.dependencies = featureSet(&[_]Feature{}),
1847	2018	};
1848	2019	const ti = @typeInfo(Feature);
...	...	@@ -1862,6 +2033,8 @@ pub const cpu = struct {
1862	2033	.aggressive_fma,
1863	2034	.arith_bcc_fusion,
1864	2035	.complxnum,
	2036	.disable_unpredicated_ld_st_lower,
	2037	.max_interleave_factor_4,
1865	2038	.perfmon,
1866	2039	.predictable_select_expensive,
1867	2040	.sha2,
...	...	@@ -1886,6 +2059,7 @@ pub const cpu = struct {
1886	2059	.fuse_aes,
1887	2060	.fuse_literals,
1888	2061	.ldp_aligned_only,
	2062	.max_interleave_factor_4,
1889	2063	.perfmon,
1890	2064	.rand,
1891	2065	.sha3,
...	...	@@ -1911,6 +2085,7 @@ pub const cpu = struct {
1911	2085	.fuse_aes,
1912	2086	.fuse_literals,
1913	2087	.ldp_aligned_only,
	2088	.max_interleave_factor_4,
1914	2089	.mte,
1915	2090	.perfmon,
1916	2091	.rand,
...	...	@@ -1939,6 +2114,7 @@ pub const cpu = struct {
1939	2114	.fuse_aes,
1940	2115	.fuse_literals,
1941	2116	.ldp_aligned_only,
	2117	.max_interleave_factor_4,
1942	2118	.mte,
1943	2119	.perfmon,
1944	2120	.predictable_select_expensive,
...	...	@@ -1951,6 +2127,38 @@ pub const cpu = struct {
1951	2127	.v8_7a,
1952	2128	}),
1953	2129	};
	2130	pub const ampere1c: CpuModel = .{
	2131	.name = "ampere1c",
	2132	.llvm_name = "ampere1c",
	2133	.features = featureSet(&[_]Feature{
	2134	.aggressive_fma,
	2135	.alu_lsl_fast,
	2136	.arith_bcc_fusion,
	2137	.cmp_bcc_fusion,
	2138	.cssc,
	2139	.enable_select_opt,
	2140	.faminmax,
	2141	.fp16fml,
	2142	.fp8fma,
	2143	.fuse_address,
	2144	.fuse_adrp_add,
	2145	.fuse_aes,
	2146	.fuse_literals,
	2147	.lut,
	2148	.max_interleave_factor_4,
	2149	.mte,
	2150	.perfmon,
	2151	.predictable_select_expensive,
	2152	.rand,
	2153	.store_pair_suppress,
	2154	.sve_aes,
	2155	.sve_b16b16,
	2156	.sve_sha3,
	2157	.sve_sm4,
	2158	.use_postra_scheduler,
	2159	.v9_2a,
	2160	}),
	2161	};
1954	2162	pub const apple_a10: CpuModel = .{
1955	2163	.name = "apple_a10",
1956	2164	.llvm_name = "apple-a10",
...	...	@@ -1964,6 +2172,7 @@ pub const cpu = struct {
1964	2172	.fuse_aes,
1965	2173	.fuse_crypto_eor,
1966	2174	.lor,
	2175	.no_zcz_fpr64,
1967	2176	.pan,
1968	2177	.perfmon,
1969	2178	.rdm,
...	...	@@ -1971,9 +2180,11 @@ pub const cpu = struct {
1971	2180	.store_pair_suppress,
1972	2181	.v8a,
1973	2182	.vh,
1974		.zcm_fpr64,
	2183	.zcm_fpr128,
1975	2184	.zcm_gpr64,
1976		.zcz,
	2185	.zcz_fpr128,
	2186	.zcz_gpr32,
	2187	.zcz_gpr64,
1977	2188	}),
1978	2189	};
1979	2190	pub const apple_a11: CpuModel = .{
...	...	@@ -1988,13 +2199,16 @@ pub const cpu = struct {
1988	2199	.fullfp16,
1989	2200	.fuse_aes,
1990	2201	.fuse_crypto_eor,
	2202	.no_zcz_fpr64,
1991	2203	.perfmon,
1992	2204	.sha2,
1993	2205	.store_pair_suppress,
1994	2206	.v8_2a,
1995		.zcm_fpr64,
	2207	.zcm_fpr128,
1996	2208	.zcm_gpr64,
1997		.zcz,
	2209	.zcz_fpr128,
	2210	.zcz_gpr32,
	2211	.zcz_gpr64,
1998	2212	}),
1999	2213	};
2000	2214	pub const apple_a12: CpuModel = .{
...	...	@@ -2009,13 +2223,16 @@ pub const cpu = struct {
2009	2223	.fullfp16,
2010	2224	.fuse_aes,
2011	2225	.fuse_crypto_eor,
	2226	.no_zcz_fpr64,
2012	2227	.perfmon,
2013	2228	.sha2,
2014	2229	.store_pair_suppress,
2015	2230	.v8_3a,
2016		.zcm_fpr64,
	2231	.zcm_fpr128,
2017	2232	.zcm_gpr64,
2018		.zcz,
	2233	.zcz_fpr128,
	2234	.zcz_gpr32,
	2235	.zcz_gpr64,
2019	2236	}),
2020	2237	};
2021	2238	pub const apple_a13: CpuModel = .{
...	...	@@ -2030,13 +2247,16 @@ pub const cpu = struct {
2030	2247	.fp16fml,
2031	2248	.fuse_aes,
2032	2249	.fuse_crypto_eor,
	2250	.no_zcz_fpr64,
2033	2251	.perfmon,
2034	2252	.sha3,
2035	2253	.store_pair_suppress,
2036	2254	.v8_4a,
2037		.zcm_fpr64,
	2255	.zcm_fpr128,
2038	2256	.zcm_gpr64,
2039		.zcz,
	2257	.zcz_fpr128,
	2258	.zcz_gpr32,
	2259	.zcz_gpr64,
2040	2260	}),
2041	2261	};
2042	2262	pub const apple_a14: CpuModel = .{
...	...	@@ -2059,6 +2279,8 @@ pub const cpu = struct {
2059	2279	.fuse_crypto_eor,
2060	2280	.fuse_csel,
2061	2281	.fuse_literals,
	2282	.max_interleave_factor_4,
	2283	.no_zcz_fpr64,
2062	2284	.perfmon,
2063	2285	.predres,
2064	2286	.sb,
...	...	@@ -2067,9 +2289,11 @@ pub const cpu = struct {
2067	2289	.ssbs,
2068	2290	.store_pair_suppress,
2069	2291	.v8_4a,
2070		.zcm_fpr64,
	2292	.zcm_fpr128,
2071	2293	.zcm_gpr64,
2072		.zcz,
	2294	.zcz_fpr128,
	2295	.zcz_gpr32,
	2296	.zcz_gpr64,
2073	2297	}),
2074	2298	};
2075	2299	pub const apple_a15: CpuModel = .{
...	...	@@ -2090,13 +2314,17 @@ pub const cpu = struct {
2090	2314	.fuse_crypto_eor,
2091	2315	.fuse_csel,
2092	2316	.fuse_literals,
	2317	.max_interleave_factor_4,
	2318	.no_zcz_fpr64,
2093	2319	.perfmon,
2094	2320	.sha3,
2095	2321	.store_pair_suppress,
2096	2322	.v8_6a,
2097		.zcm_fpr64,
	2323	.zcm_fpr128,
2098	2324	.zcm_gpr64,
2099		.zcz,
	2325	.zcz_fpr128,
	2326	.zcz_gpr32,
	2327	.zcz_gpr64,
2100	2328	}),
2101	2329	};
2102	2330	pub const apple_a16: CpuModel = .{
...	...	@@ -2118,13 +2346,17 @@ pub const cpu = struct {
2118	2346	.fuse_csel,
2119	2347	.fuse_literals,
2120	2348	.hcx,
	2349	.max_interleave_factor_4,
	2350	.no_zcz_fpr64,
2121	2351	.perfmon,
2122	2352	.sha3,
2123	2353	.store_pair_suppress,
2124	2354	.v8_6a,
2125		.zcm_fpr64,
	2355	.zcm_fpr128,
2126	2356	.zcm_gpr64,
2127		.zcz,
	2357	.zcz_fpr128,
	2358	.zcz_gpr32,
	2359	.zcz_gpr64,
2128	2360	}),
2129	2361	};
2130	2362	pub const apple_a17: CpuModel = .{
...	...	@@ -2146,13 +2378,17 @@ pub const cpu = struct {
2146	2378	.fuse_csel,
2147	2379	.fuse_literals,
2148	2380	.hcx,
	2381	.max_interleave_factor_4,
	2382	.no_zcz_fpr64,
2149	2383	.perfmon,
2150	2384	.sha3,
2151	2385	.store_pair_suppress,
2152	2386	.v8_6a,
2153		.zcm_fpr64,
	2387	.zcm_fpr128,
2154	2388	.zcm_gpr64,
2155		.zcz,
	2389	.zcz_fpr128,
	2390	.zcz_gpr32,
	2391	.zcz_gpr64,
2156	2392	}),
2157	2393	};
2158	2394	pub const apple_a18: CpuModel = .{
...	...	@@ -2173,15 +2409,58 @@ pub const cpu = struct {
2173	2409	.fuse_crypto_eor,
2174	2410	.fuse_csel,
2175	2411	.fuse_literals,
	2412	.max_interleave_factor_4,
	2413	.no_zcz_fpr64,
2176	2414	.perfmon,
2177	2415	.sha3,
2178	2416	.sme2,
2179	2417	.sme_f64f64,
2180	2418	.sme_i16i64,
2181	2419	.v8_7a,
2182		.zcm_fpr64,
	2420	.zcm_fpr128,
2183	2421	.zcm_gpr64,
2184		.zcz,
	2422	.zcz_fpr128,
	2423	.zcz_gpr32,
	2424	.zcz_gpr64,
	2425	}),
	2426	};
	2427	pub const apple_a19: CpuModel = .{
	2428	.name = "apple_a19",
	2429	.llvm_name = "apple-a19",
	2430	.features = featureSet(&[_]Feature{
	2431	.aes,
	2432	.alternate_sextload_cvt_f32_pattern,
	2433	.arith_bcc_fusion,
	2434	.arith_cbz_fusion,
	2435	.cssc,
	2436	.disable_latency_sched_heuristic,
	2437	.fp16fml,
	2438	.fpac,
	2439	.fuse_address,
	2440	.fuse_adrp_add,
	2441	.fuse_aes,
	2442	.fuse_arith_logic,
	2443	.fuse_crypto_eor,
	2444	.fuse_csel,
	2445	.fuse_literals,
	2446	.hbc,
	2447	.max_interleave_factor_4,
	2448	.mte,
	2449	.no_zcz_fpr64,
	2450	.perfmon,
	2451	.sha3,
	2452	.sme2p1,
	2453	.sme_b16b16,
	2454	.sme_f16f16,
	2455	.sme_f64f64,
	2456	.sme_i16i64,
	2457	.specres2,
	2458	.v8_7a,
	2459	.zcm_fpr128,
	2460	.zcm_gpr64,
	2461	.zcz_fpr128,
	2462	.zcz_gpr32,
	2463	.zcz_gpr64,
2185	2464	}),
2186	2465	};
2187	2466	pub const apple_a7: CpuModel = .{
...	...	@@ -2195,14 +2474,17 @@ pub const cpu = struct {
2195	2474	.disable_latency_sched_heuristic,
2196	2475	.fuse_aes,
2197	2476	.fuse_crypto_eor,
	2477	.no_zcz_fpr64,
2198	2478	.perfmon,
2199	2479	.sha2,
2200	2480	.store_pair_suppress,
2201	2481	.v8a,
2202		.zcm_fpr64,
	2482	.zcm_fpr128,
2203	2483	.zcm_gpr64,
2204		.zcz,
2205	2484	.zcz_fp_workaround,
	2485	.zcz_fpr128,
	2486	.zcz_gpr32,
	2487	.zcz_gpr64,
2206	2488	}),
2207	2489	};
2208	2490	pub const apple_a8: CpuModel = .{
...	...	@@ -2216,14 +2498,17 @@ pub const cpu = struct {
2216	2498	.disable_latency_sched_heuristic,
2217	2499	.fuse_aes,
2218	2500	.fuse_crypto_eor,
	2501	.no_zcz_fpr64,
2219	2502	.perfmon,
2220	2503	.sha2,
2221	2504	.store_pair_suppress,
2222	2505	.v8a,
2223		.zcm_fpr64,
	2506	.zcm_fpr128,
2224	2507	.zcm_gpr64,
2225		.zcz,
2226	2508	.zcz_fp_workaround,
	2509	.zcz_fpr128,
	2510	.zcz_gpr32,
	2511	.zcz_gpr64,
2227	2512	}),
2228	2513	};
2229	2514	pub const apple_a9: CpuModel = .{
...	...	@@ -2237,14 +2522,17 @@ pub const cpu = struct {
2237	2522	.disable_latency_sched_heuristic,
2238	2523	.fuse_aes,
2239	2524	.fuse_crypto_eor,
	2525	.no_zcz_fpr64,
2240	2526	.perfmon,
2241	2527	.sha2,
2242	2528	.store_pair_suppress,
2243	2529	.v8a,
2244		.zcm_fpr64,
	2530	.zcm_fpr128,
2245	2531	.zcm_gpr64,
2246		.zcz,
2247	2532	.zcz_fp_workaround,
	2533	.zcz_fpr128,
	2534	.zcz_gpr32,
	2535	.zcz_gpr64,
2248	2536	}),
2249	2537	};
2250	2538	pub const apple_m1: CpuModel = .{
...	...	@@ -2267,6 +2555,8 @@ pub const cpu = struct {
2267	2555	.fuse_crypto_eor,
2268	2556	.fuse_csel,
2269	2557	.fuse_literals,
	2558	.max_interleave_factor_4,
	2559	.no_zcz_fpr64,
2270	2560	.perfmon,
2271	2561	.predres,
2272	2562	.sb,
...	...	@@ -2275,9 +2565,11 @@ pub const cpu = struct {
2275	2565	.ssbs,
2276	2566	.store_pair_suppress,
2277	2567	.v8_4a,
2278		.zcm_fpr64,
	2568	.zcm_fpr128,
2279	2569	.zcm_gpr64,
2280		.zcz,
	2570	.zcz_fpr128,
	2571	.zcz_gpr32,
	2572	.zcz_gpr64,
2281	2573	}),
2282	2574	};
2283	2575	pub const apple_m2: CpuModel = .{
...	...	@@ -2298,13 +2590,17 @@ pub const cpu = struct {
2298	2590	.fuse_crypto_eor,
2299	2591	.fuse_csel,
2300	2592	.fuse_literals,
	2593	.max_interleave_factor_4,
	2594	.no_zcz_fpr64,
2301	2595	.perfmon,
2302	2596	.sha3,
2303	2597	.store_pair_suppress,
2304	2598	.v8_6a,
2305		.zcm_fpr64,
	2599	.zcm_fpr128,
2306	2600	.zcm_gpr64,
2307		.zcz,
	2601	.zcz_fpr128,
	2602	.zcz_gpr32,
	2603	.zcz_gpr64,
2308	2604	}),
2309	2605	};
2310	2606	pub const apple_m3: CpuModel = .{
...	...	@@ -2326,13 +2622,17 @@ pub const cpu = struct {
2326	2622	.fuse_csel,
2327	2623	.fuse_literals,
2328	2624	.hcx,
	2625	.max_interleave_factor_4,
	2626	.no_zcz_fpr64,
2329	2627	.perfmon,
2330	2628	.sha3,
2331	2629	.store_pair_suppress,
2332	2630	.v8_6a,
2333		.zcm_fpr64,
	2631	.zcm_fpr128,
2334	2632	.zcm_gpr64,
2335		.zcz,
	2633	.zcz_fpr128,
	2634	.zcz_gpr32,
	2635	.zcz_gpr64,
2336	2636	}),
2337	2637	};
2338	2638	pub const apple_m4: CpuModel = .{
...	...	@@ -2353,15 +2653,58 @@ pub const cpu = struct {
2353	2653	.fuse_crypto_eor,
2354	2654	.fuse_csel,
2355	2655	.fuse_literals,
	2656	.max_interleave_factor_4,
	2657	.no_zcz_fpr64,
2356	2658	.perfmon,
2357	2659	.sha3,
2358	2660	.sme2,
2359	2661	.sme_f64f64,
2360	2662	.sme_i16i64,
2361	2663	.v8_7a,
2362		.zcm_fpr64,
	2664	.zcm_fpr128,
	2665	.zcm_gpr64,
	2666	.zcz_fpr128,
	2667	.zcz_gpr32,
	2668	.zcz_gpr64,
	2669	}),
	2670	};
	2671	pub const apple_m5: CpuModel = .{
	2672	.name = "apple_m5",
	2673	.llvm_name = "apple-m5",
	2674	.features = featureSet(&[_]Feature{
	2675	.aes,
	2676	.alternate_sextload_cvt_f32_pattern,
	2677	.arith_bcc_fusion,
	2678	.arith_cbz_fusion,
	2679	.cssc,
	2680	.disable_latency_sched_heuristic,
	2681	.fp16fml,
	2682	.fpac,
	2683	.fuse_address,
	2684	.fuse_adrp_add,
	2685	.fuse_aes,
	2686	.fuse_arith_logic,
	2687	.fuse_crypto_eor,
	2688	.fuse_csel,
	2689	.fuse_literals,
	2690	.hbc,
	2691	.max_interleave_factor_4,
	2692	.mte,
	2693	.no_zcz_fpr64,
	2694	.perfmon,
	2695	.sha3,
	2696	.sme2p1,
	2697	.sme_b16b16,
	2698	.sme_f16f16,
	2699	.sme_f64f64,
	2700	.sme_i16i64,
	2701	.specres2,
	2702	.v8_7a,
	2703	.zcm_fpr128,
2363	2704	.zcm_gpr64,
2364		.zcz,
	2705	.zcz_fpr128,
	2706	.zcz_gpr32,
	2707	.zcz_gpr64,
2365	2708	}),
2366	2709	};
2367	2710	pub const apple_s10: CpuModel = .{
...	...	@@ -2383,13 +2726,17 @@ pub const cpu = struct {
2383	2726	.fuse_csel,
2384	2727	.fuse_literals,
2385	2728	.hcx,
	2729	.max_interleave_factor_4,
	2730	.no_zcz_fpr64,
2386	2731	.perfmon,
2387	2732	.sha3,
2388	2733	.store_pair_suppress,
2389	2734	.v8_6a,
2390		.zcm_fpr64,
	2735	.zcm_fpr128,
2391	2736	.zcm_gpr64,
2392		.zcz,
	2737	.zcz_fpr128,
	2738	.zcz_gpr32,
	2739	.zcz_gpr64,
2393	2740	}),
2394	2741	};
2395	2742	pub const apple_s4: CpuModel = .{
...	...	@@ -2404,13 +2751,16 @@ pub const cpu = struct {
2404	2751	.fullfp16,
2405	2752	.fuse_aes,
2406	2753	.fuse_crypto_eor,
	2754	.no_zcz_fpr64,
2407	2755	.perfmon,
2408	2756	.sha2,
2409	2757	.store_pair_suppress,
2410	2758	.v8_3a,
2411		.zcm_fpr64,
	2759	.zcm_fpr128,
2412	2760	.zcm_gpr64,
2413		.zcz,
	2761	.zcz_fpr128,
	2762	.zcz_gpr32,
	2763	.zcz_gpr64,
2414	2764	}),
2415	2765	};
2416	2766	pub const apple_s5: CpuModel = .{
...	...	@@ -2425,13 +2775,16 @@ pub const cpu = struct {
2425	2775	.fullfp16,
2426	2776	.fuse_aes,
2427	2777	.fuse_crypto_eor,
	2778	.no_zcz_fpr64,
2428	2779	.perfmon,
2429	2780	.sha2,
2430	2781	.store_pair_suppress,
2431	2782	.v8_3a,
2432		.zcm_fpr64,
	2783	.zcm_fpr128,
2433	2784	.zcm_gpr64,
2434		.zcz,
	2785	.zcz_fpr128,
	2786	.zcz_gpr32,
	2787	.zcz_gpr64,
2435	2788	}),
2436	2789	};
2437	2790	pub const apple_s6: CpuModel = .{
...	...	@@ -2446,13 +2799,16 @@ pub const cpu = struct {
2446	2799	.fp16fml,
2447	2800	.fuse_aes,
2448	2801	.fuse_crypto_eor,
	2802	.no_zcz_fpr64,
2449	2803	.perfmon,
2450	2804	.sha3,
2451	2805	.store_pair_suppress,
2452	2806	.v8_4a,
2453		.zcm_fpr64,
	2807	.zcm_fpr128,
2454	2808	.zcm_gpr64,
2455		.zcz,
	2809	.zcz_fpr128,
	2810	.zcz_gpr32,
	2811	.zcz_gpr64,
2456	2812	}),
2457	2813	};
2458	2814	pub const apple_s7: CpuModel = .{
...	...	@@ -2467,13 +2823,16 @@ pub const cpu = struct {
2467	2823	.fp16fml,
2468	2824	.fuse_aes,
2469	2825	.fuse_crypto_eor,
	2826	.no_zcz_fpr64,
2470	2827	.perfmon,
2471	2828	.sha3,
2472	2829	.store_pair_suppress,
2473	2830	.v8_4a,
2474		.zcm_fpr64,
	2831	.zcm_fpr128,
2475	2832	.zcm_gpr64,
2476		.zcz,
	2833	.zcz_fpr128,
	2834	.zcz_gpr32,
	2835	.zcz_gpr64,
2477	2836	}),
2478	2837	};
2479	2838	pub const apple_s8: CpuModel = .{
...	...	@@ -2488,13 +2847,16 @@ pub const cpu = struct {
2488	2847	.fp16fml,
2489	2848	.fuse_aes,
2490	2849	.fuse_crypto_eor,
	2850	.no_zcz_fpr64,
2491	2851	.perfmon,
2492	2852	.sha3,
2493	2853	.store_pair_suppress,
2494	2854	.v8_4a,
2495		.zcm_fpr64,
	2855	.zcm_fpr128,
2496	2856	.zcm_gpr64,
2497		.zcz,
	2857	.zcz_fpr128,
	2858	.zcz_gpr32,
	2859	.zcz_gpr64,
2498	2860	}),
2499	2861	};
2500	2862	pub const apple_s9: CpuModel = .{
...	...	@@ -2516,13 +2878,126 @@ pub const cpu = struct {
2516	2878	.fuse_csel,
2517	2879	.fuse_literals,
2518	2880	.hcx,
	2881	.max_interleave_factor_4,
	2882	.no_zcz_fpr64,
2519	2883	.perfmon,
2520	2884	.sha3,
2521	2885	.store_pair_suppress,
2522	2886	.v8_6a,
2523		.zcm_fpr64,
	2887	.zcm_fpr128,
2524	2888	.zcm_gpr64,
2525		.zcz,
	2889	.zcz_fpr128,
	2890	.zcz_gpr32,
	2891	.zcz_gpr64,
	2892	}),
	2893	};
	2894	pub const c1_nano: CpuModel = .{
	2895	.name = "c1_nano",
	2896	.llvm_name = "c1-nano",
	2897	.features = featureSet(&[_]Feature{
	2898	.chk,
	2899	.clrbhb,
	2900	.ete,
	2901	.fp16fml,
	2902	.fpac,
	2903	.fuse_adrp_add,
	2904	.fuse_aes,
	2905	.mte,
	2906	.perfmon,
	2907	.rcpc3,
	2908	.sme2,
	2909	.specres2,
	2910	.sve_bitperm,
	2911	.use_fixed_over_scalable_if_equal_cost,
	2912	.use_postra_scheduler,
	2913	.use_wzr_to_vec_move,
	2914	.v9_3a,
	2915	}),
	2916	};
	2917	pub const c1_premium: CpuModel = .{
	2918	.name = "c1_premium",
	2919	.llvm_name = "c1-premium",
	2920	.features = featureSet(&[_]Feature{
	2921	.alu_lsl_fast,
	2922	.avoid_ldapur,
	2923	.chk,
	2924	.clrbhb,
	2925	.enable_select_opt,
	2926	.ete,
	2927	.fp16fml,
	2928	.fpac,
	2929	.fuse_adrp_add,
	2930	.fuse_aes,
	2931	.fuse_csel,
	2932	.fuse_cset,
	2933	.mte,
	2934	.perfmon,
	2935	.predictable_select_expensive,
	2936	.rcpc3,
	2937	.sme2,
	2938	.spe,
	2939	.specres2,
	2940	.sve_bitperm,
	2941	.use_fixed_over_scalable_if_equal_cost,
	2942	.use_postra_scheduler,
	2943	.v9_3a,
	2944	}),
	2945	};
	2946	pub const c1_pro: CpuModel = .{
	2947	.name = "c1_pro",
	2948	.llvm_name = "c1-pro",
	2949	.features = featureSet(&[_]Feature{
	2950	.alu_lsl_fast,
	2951	.chk,
	2952	.clrbhb,
	2953	.cmp_bcc_fusion,
	2954	.enable_select_opt,
	2955	.ete,
	2956	.fp16fml,
	2957	.fpac,
	2958	.fuse_adrp_add,
	2959	.fuse_aes,
	2960	.fuse_csel,
	2961	.fuse_cset,
	2962	.mte,
	2963	.perfmon,
	2964	.predictable_select_expensive,
	2965	.rcpc3,
	2966	.sme2,
	2967	.spe,
	2968	.specres2,
	2969	.sve_bitperm,
	2970	.use_postra_scheduler,
	2971	.v9_3a,
	2972	}),
	2973	};
	2974	pub const c1_ultra: CpuModel = .{
	2975	.name = "c1_ultra",
	2976	.llvm_name = "c1-ultra",
	2977	.features = featureSet(&[_]Feature{
	2978	.alu_lsl_fast,
	2979	.avoid_ldapur,
	2980	.chk,
	2981	.clrbhb,
	2982	.enable_select_opt,
	2983	.ete,
	2984	.fp16fml,
	2985	.fpac,
	2986	.fuse_adrp_add,
	2987	.fuse_aes,
	2988	.fuse_csel,
	2989	.fuse_cset,
	2990	.mte,
	2991	.perfmon,
	2992	.predictable_select_expensive,
	2993	.rcpc3,
	2994	.sme2,
	2995	.spe,
	2996	.specres2,
	2997	.sve_bitperm,
	2998	.use_fixed_over_scalable_if_equal_cost,
	2999	.use_postra_scheduler,
	3000	.v9_3a,
2526	3001	}),
2527	3002	};
2528	3003	pub const carmel: CpuModel = .{
...	...	@@ -2541,12 +3016,15 @@ pub const cpu = struct {
2541	3016	.features = featureSet(&[_]Feature{
2542	3017	.alu_lsl_fast,
2543	3018	.bf16,
	3019	.disable_maximize_scalable_bandwidth,
2544	3020	.enable_select_opt,
2545	3021	.ete,
2546	3022	.fp16fml,
2547	3023	.fpac,
2548	3024	.fuse_adrp_add,
2549	3025	.fuse_aes,
	3026	.fuse_csel,
	3027	.fuse_cset,
2550	3028	.i8mm,
2551	3029	.mte,
2552	3030	.perfmon,
...	...	@@ -2607,6 +3085,7 @@ pub const cpu = struct {
2607	3085	.sve_bitperm,
2608	3086	.use_fixed_over_scalable_if_equal_cost,
2609	3087	.use_postra_scheduler,
	3088	.use_wzr_to_vec_move,
2610	3089	.v9a,
2611	3090	}),
2612	3091	};
...	...	@@ -2624,6 +3103,7 @@ pub const cpu = struct {
2624	3103	.sve_bitperm,
2625	3104	.use_fixed_over_scalable_if_equal_cost,
2626	3105	.use_postra_scheduler,
	3106	.use_wzr_to_vec_move,
2627	3107	.v9_2a,
2628	3108	}),
2629	3109	};
...	...	@@ -2640,6 +3120,7 @@ pub const cpu = struct {
2640	3120	.perfmon,
2641	3121	.sve_bitperm,
2642	3122	.use_postra_scheduler,
	3123	.use_wzr_to_vec_move,
2643	3124	.v9_2a,
2644	3125	}),
2645	3126	};
...	...	@@ -2655,6 +3136,7 @@ pub const cpu = struct {
2655	3136	.perfmon,
2656	3137	.sha2,
2657	3138	.use_postra_scheduler,
	3139	.use_wzr_to_vec_move,
2658	3140	.v8a,
2659	3141	}),
2660	3142	};
...	...	@@ -2672,6 +3154,7 @@ pub const cpu = struct {
2672	3154	.rcpc,
2673	3155	.sha2,
2674	3156	.use_postra_scheduler,
	3157	.use_wzr_to_vec_move,
2675	3158	.v8_2a,
2676	3159	}),
2677	3160	};
...	...	@@ -2687,6 +3170,7 @@ pub const cpu = struct {
2687	3170	.fuse_adrp_add,
2688	3171	.fuse_aes,
2689	3172	.fuse_literals,
	3173	.max_interleave_factor_4,
2690	3174	.perfmon,
2691	3175	.predictable_select_expensive,
2692	3176	.sha2,
...	...	@@ -2747,6 +3231,8 @@ pub const cpu = struct {
2747	3231	.fpac,
2748	3232	.fuse_adrp_add,
2749	3233	.fuse_aes,
	3234	.fuse_csel,
	3235	.fuse_cset,
2750	3236	.i8mm,
2751	3237	.mte,
2752	3238	.perfmon,
...	...	@@ -2769,6 +3255,8 @@ pub const cpu = struct {
2769	3255	.fpac,
2770	3256	.fuse_adrp_add,
2771	3257	.fuse_aes,
	3258	.fuse_csel,
	3259	.fuse_cset,
2772	3260	.i8mm,
2773	3261	.mte,
2774	3262	.perfmon,
...	...	@@ -2808,6 +3296,8 @@ pub const cpu = struct {
2808	3296	.fpac,
2809	3297	.fuse_adrp_add,
2810	3298	.fuse_aes,
	3299	.fuse_csel,
	3300	.fuse_cset,
2811	3301	.mte,
2812	3302	.perfmon,
2813	3303	.predictable_select_expensive,
...	...	@@ -2829,6 +3319,8 @@ pub const cpu = struct {
2829	3319	.fpac,
2830	3320	.fuse_adrp_add,
2831	3321	.fuse_aes,
	3322	.fuse_csel,
	3323	.fuse_cset,
2832	3324	.mte,
2833	3325	.perfmon,
2834	3326	.predictable_select_expensive,
...	...	@@ -2850,6 +3342,8 @@ pub const cpu = struct {
2850	3342	.fpac,
2851	3343	.fuse_adrp_add,
2852	3344	.fuse_aes,
	3345	.fuse_csel,
	3346	.fuse_cset,
2853	3347	.mte,
2854	3348	.perfmon,
2855	3349	.predictable_select_expensive,
...	...	@@ -2967,6 +3461,8 @@ pub const cpu = struct {
2967	3461	.fullfp16,
2968	3462	.fuse_adrp_add,
2969	3463	.fuse_aes,
	3464	.fuse_csel,
	3465	.fuse_cset,
2970	3466	.perfmon,
2971	3467	.predictable_select_expensive,
2972	3468	.rcpc,
...	...	@@ -2990,6 +3486,8 @@ pub const cpu = struct {
2990	3486	.fullfp16,
2991	3487	.fuse_adrp_add,
2992	3488	.fuse_aes,
	3489	.fuse_csel,
	3490	.fuse_cset,
2993	3491	.perfmon,
2994	3492	.predictable_select_expensive,
2995	3493	.rcpc,
...	...	@@ -3014,6 +3512,8 @@ pub const cpu = struct {
3014	3512	.fullfp16,
3015	3513	.fuse_adrp_add,
3016	3514	.fuse_aes,
	3515	.fuse_csel,
	3516	.fuse_cset,
3017	3517	.pauth,
3018	3518	.perfmon,
3019	3519	.predictable_select_expensive,
...	...	@@ -3157,6 +3657,8 @@ pub const cpu = struct {
3157	3657	.fpac,
3158	3658	.fuse_adrp_add,
3159	3659	.fuse_aes,
	3660	.fuse_csel,
	3661	.fuse_cset,
3160	3662	.mte,
3161	3663	.perfmon,
3162	3664	.predictable_select_expensive,
...	...	@@ -3179,6 +3681,8 @@ pub const cpu = struct {
3179	3681	.fpac,
3180	3682	.fuse_adrp_add,
3181	3683	.fuse_aes,
	3684	.fuse_csel,
	3685	.fuse_cset,
3182	3686	.mte,
3183	3687	.perfmon,
3184	3688	.predictable_select_expensive,
...	...	@@ -3200,14 +3704,17 @@ pub const cpu = struct {
3200	3704	.disable_latency_sched_heuristic,
3201	3705	.fuse_aes,
3202	3706	.fuse_crypto_eor,
	3707	.no_zcz_fpr64,
3203	3708	.perfmon,
3204	3709	.sha2,
3205	3710	.store_pair_suppress,
3206	3711	.v8a,
3207		.zcm_fpr64,
	3712	.zcm_fpr128,
3208	3713	.zcm_gpr64,
3209		.zcz,
3210	3714	.zcz_fp_workaround,
	3715	.zcz_fpr128,
	3716	.zcz_gpr32,
	3717	.zcz_gpr64,
3211	3718	}),
3212	3719	};
3213	3720	pub const emag: CpuModel = .{
...	...	@@ -3267,6 +3774,7 @@ pub const cpu = struct {
3267	3774	.fuse_aes,
3268	3775	.fuse_csel,
3269	3776	.fuse_literals,
	3777	.max_interleave_factor_4,
3270	3778	.perfmon,
3271	3779	.predictable_select_expensive,
3272	3780	.sha2,
...	...	@@ -3293,12 +3801,14 @@ pub const cpu = struct {
3293	3801	.fuse_arith_logic,
3294	3802	.fuse_csel,
3295	3803	.fuse_literals,
	3804	.max_interleave_factor_4,
3296	3805	.perfmon,
3297	3806	.sha2,
3298	3807	.store_pair_suppress,
3299	3808	.use_postra_scheduler,
3300	3809	.v8_2a,
3301		.zcz,
	3810	.zcz_gpr32,
	3811	.zcz_gpr64,
3302	3812	}),
3303	3813	};
3304	3814	pub const exynos_m5: CpuModel = .{
...	...	@@ -3319,12 +3829,14 @@ pub const cpu = struct {
3319	3829	.fuse_arith_logic,
3320	3830	.fuse_csel,
3321	3831	.fuse_literals,
	3832	.max_interleave_factor_4,
3322	3833	.perfmon,
3323	3834	.sha2,
3324	3835	.store_pair_suppress,
3325	3836	.use_postra_scheduler,
3326	3837	.v8_2a,
3327		.zcz,
	3838	.zcz_gpr32,
	3839	.zcz_gpr64,
3328	3840	}),
3329	3841	};
3330	3842	pub const falkor: CpuModel = .{
...	...	@@ -3334,6 +3846,7 @@ pub const cpu = struct {
3334	3846	.aes,
3335	3847	.alu_lsl_fast,
3336	3848	.crc,
	3849	.max_interleave_factor_4,
3337	3850	.perfmon,
3338	3851	.predictable_select_expensive,
3339	3852	.rdm,
...	...	@@ -3342,7 +3855,8 @@ pub const cpu = struct {
3342	3855	.store_pair_suppress,
3343	3856	.use_postra_scheduler,
3344	3857	.v8a,
3345		.zcz,
	3858	.zcz_gpr32,
	3859	.zcz_gpr64,
3346	3860	}),
3347	3861	};
3348	3862	pub const fujitsu_monaka: CpuModel = .{
...	...	@@ -3382,6 +3896,8 @@ pub const cpu = struct {
3382	3896	.fpac,
3383	3897	.fuse_adrp_add,
3384	3898	.fuse_aes,
	3899	.fuse_csel,
	3900	.fuse_cset,
3385	3901	.mte,
3386	3902	.perfmon,
3387	3903	.predictable_select_expensive,
...	...	@@ -3422,11 +3938,13 @@ pub const cpu = struct {
3422	3938	.fpac,
3423	3939	.fuse_adrp_add,
3424	3940	.fuse_aes,
	3941	.fuse_csel,
	3942	.fuse_cset,
3425	3943	.i8mm,
	3944	.max_interleave_factor_4,
3426	3945	.mte,
3427	3946	.perfmon,
3428	3947	.predictable_select_expensive,
3429		.rand,
3430	3948	.spe,
3431	3949	.sve_aes,
3432	3950	.sve_bitperm,
...	...	@@ -3444,13 +3962,15 @@ pub const cpu = struct {
3444	3962	.aes,
3445	3963	.alu_lsl_fast,
3446	3964	.crc,
	3965	.max_interleave_factor_4,
3447	3966	.perfmon,
3448	3967	.predictable_select_expensive,
3449	3968	.sha2,
3450	3969	.store_pair_suppress,
3451	3970	.use_postra_scheduler,
3452	3971	.v8a,
3453		.zcz,
	3972	.zcz_gpr32,
	3973	.zcz_gpr64,
3454	3974	}),
3455	3975	};
3456	3976	pub const neoverse_512tvb: CpuModel = .{
...	...	@@ -3467,6 +3987,7 @@ pub const cpu = struct {
3467	3987	.fuse_adrp_add,
3468	3988	.fuse_aes,
3469	3989	.i8mm,
	3990	.max_interleave_factor_4,
3470	3991	.perfmon,
3471	3992	.predictable_select_expensive,
3472	3993	.rand,
...	...	@@ -3524,12 +4045,15 @@ pub const cpu = struct {
3524	4045	.features = featureSet(&[_]Feature{
3525	4046	.alu_lsl_fast,
3526	4047	.bf16,
	4048	.disable_maximize_scalable_bandwidth,
3527	4049	.enable_select_opt,
3528	4050	.ete,
3529	4051	.fp16fml,
3530	4052	.fpac,
3531	4053	.fuse_adrp_add,
3532	4054	.fuse_aes,
	4055	.fuse_csel,
	4056	.fuse_cset,
3533	4057	.i8mm,
3534	4058	.mte,
3535	4059	.perfmon,
...	...	@@ -3550,6 +4074,8 @@ pub const cpu = struct {
3550	4074	.fpac,
3551	4075	.fuse_adrp_add,
3552	4076	.fuse_aes,
	4077	.fuse_csel,
	4078	.fuse_cset,
3553	4079	.mte,
3554	4080	.perfmon,
3555	4081	.predictable_select_expensive,
...	...	@@ -3569,10 +4095,13 @@ pub const cpu = struct {
3569	4095	.alu_lsl_fast,
3570	4096	.bf16,
3571	4097	.ccdp,
	4098	.disable_maximize_scalable_bandwidth,
3572	4099	.enable_select_opt,
3573	4100	.fp16fml,
3574	4101	.fuse_adrp_add,
3575	4102	.fuse_aes,
	4103	.fuse_csel,
	4104	.fuse_cset,
3576	4105	.i8mm,
3577	4106	.no_sve_fp_ld1r,
3578	4107	.perfmon,
...	...	@@ -3602,7 +4131,10 @@ pub const cpu = struct {
3602	4131	.fpac,
3603	4132	.fuse_adrp_add,
3604	4133	.fuse_aes,
	4134	.fuse_csel,
	4135	.fuse_cset,
3605	4136	.i8mm,
	4137	.max_interleave_factor_4,
3606	4138	.mte,
3607	4139	.perfmon,
3608	4140	.predictable_select_expensive,
...	...	@@ -3627,7 +4159,10 @@ pub const cpu = struct {
3627	4159	.fpac,
3628	4160	.fuse_adrp_add,
3629	4161	.fuse_aes,
	4162	.fuse_csel,
	4163	.fuse_cset,
3630	4164	.ls64,
	4165	.max_interleave_factor_4,
3631	4166	.mte,
3632	4167	.perfmon,
3633	4168	.predictable_select_expensive,
...	...	@@ -3651,7 +4186,10 @@ pub const cpu = struct {
3651	4186	.fpac,
3652	4187	.fuse_adrp_add,
3653	4188	.fuse_aes,
	4189	.fuse_csel,
	4190	.fuse_cset,
3654	4191	.ls64,
	4192	.max_interleave_factor_4,
3655	4193	.mte,
3656	4194	.perfmon,
3657	4195	.predictable_select_expensive,
...	...	@@ -3700,6 +4238,7 @@ pub const cpu = struct {
3700	4238	.fuse_adrp_add,
3701	4239	.fuse_aes,
3702	4240	.fuse_crypto_eor,
	4241	.max_interleave_factor_4,
3703	4242	.perfmon,
3704	4243	.rand,
3705	4244	.sha3,
...	...	@@ -3715,6 +4254,7 @@ pub const cpu = struct {
3715	4254	.features = featureSet(&[_]Feature{
3716	4255	.aes,
3717	4256	.alu_lsl_fast,
	4257	.max_interleave_factor_4,
3718	4258	.perfmon,
3719	4259	.predictable_select_expensive,
3720	4260	.sha2,
...	...	@@ -3722,7 +4262,8 @@ pub const cpu = struct {
3722	4262	.store_pair_suppress,
3723	4263	.use_postra_scheduler,
3724	4264	.v8_4a,
3725		.zcz,
	4265	.zcz_gpr32,
	4266	.zcz_gpr64,
3726	4267	}),
3727	4268	};
3728	4269	pub const thunderx: CpuModel = .{
...	...	@@ -3746,6 +4287,7 @@ pub const cpu = struct {
3746	4287	.aes,
3747	4288	.aggressive_fma,
3748	4289	.arith_bcc_fusion,
	4290	.max_interleave_factor_4,
3749	4291	.predictable_select_expensive,
3750	4292	.sha2,
3751	4293	.store_pair_suppress,
...	...	@@ -3761,6 +4303,7 @@ pub const cpu = struct {
3761	4303	.aggressive_fma,
3762	4304	.arith_bcc_fusion,
3763	4305	.balance_fp_ops,
	4306	.max_interleave_factor_4,
3764	4307	.perfmon,
3765	4308	.predictable_select_expensive,
3766	4309	.sha2,

lib/std/Target/amdgcn.zig+421-21

...	...	@@ -5,12 +5,17 @@ const CpuFeature = std.Target.Cpu.Feature;
5	5	const CpuModel = std.Target.Cpu.Model;
6	6
7	7	pub const Feature = enum {
	8	@"1024_addressable_vgprs",
8	9	@"16_bit_insts",
	10	@"45_bit_num_records_buffer_resource",
9	11	@"64_bit_literals",
10	12	a16,
	13	add_min_max_insts,
11	14	add_no_carry_insts,
	15	add_sub_u64_insts,
12	16	addressablelocalmemorysize163840,
13	17	addressablelocalmemorysize32768,
	18	addressablelocalmemorysize327680,
14	19	addressablelocalmemorysize65536,
15	20	agent_scope_fine_grained_remote_memory_atomics,
16	21	allocate1_5xvgprs,
...	...	@@ -18,6 +23,7 @@ pub const Feature = enum {
18	23	architected_flat_scratch,
19	24	architected_sgprs,
20	25	ashr_pk_insts,
	26	assembler_permissive_wavesize,
21	27	atomic_buffer_global_pk_add_f16_insts,
22	28	atomic_buffer_global_pk_add_f16_no_rtn_insts,
23	29	atomic_buffer_pk_add_bf16_inst,
...	...	@@ -34,15 +40,22 @@ pub const Feature = enum {
34	40	auto_waitcnt_before_barrier,
35	41	back_off_barrier,
36	42	bf16_cvt_insts,
	43	bf16_pk_insts,
37	44	bf16_trans_insts,
38	45	bf8_cvt_scale_insts,
39	46	bitop3_insts,
40	47	block_vgpr_csr,
41	48	bvh_dual_bvh_8_insts,
42	49	ci_insts,
	50	clusters,
	51	cube_insts,
43	52	cumode,
44	53	cvt_fp8_vop1_bug,
	54	cvt_norm_insts,
45	55	cvt_pk_f16_f32_inst,
	56	cvt_pknorm_vop2_insts,
	57	cvt_pknorm_vop3_insts,
	58	d16_write_vgpr32,
46	59	default_component_broadcast,
47	60	default_component_zero,
48	61	dl_insts,
...	...	@@ -65,8 +78,7 @@ pub const Feature = enum {
65	78	dpp_src1_sgpr,
66	79	ds128,
67	80	ds_src2_insts,
68		dynamic_vgpr,
69		dynamic_vgpr_block_size_32,
	81	emulated_system_scope_atomics,
70	82	extended_image_insts,
71	83	f16bf16_to_fp6bf6_cvt_scale_insts,
72	84	f32_to_f16bf16_cvt_sr_insts,
...	...	@@ -77,10 +89,12 @@ pub const Feature = enum {
77	89	flat_buffer_global_fadd_f64_inst,
78	90	flat_for_global,
79	91	flat_global_insts,
	92	flat_gvs_mode,
80	93	flat_inst_offsets,
81	94	flat_scratch,
82	95	flat_scratch_insts,
83	96	flat_segment_offset_bug,
	97	fma_mix_bf16_insts,
84	98	fma_mix_insts,
85	99	fmacf64_inst,
86	100	fmaf,
...	...	@@ -113,6 +127,7 @@ pub const Feature = enum {
113	127	gfx940_insts,
114	128	gfx950_insts,
115	129	gfx9_insts,
	130	globally_addressable_scratch,
116	131	gws,
117	132	half_rate_64_ops,
118	133	ieee_minimum_maximum_insts,
...	...	@@ -128,20 +143,24 @@ pub const Feature = enum {
128	143	lds_misaligned_bug,
129	144	ldsbankcount16,
130	145	ldsbankcount32,
	146	lerp_inst,
131	147	load_store_opt,
132	148	lshl_add_u64_inst,
133	149	mad_intra_fwd_bug,
134	150	mad_mac_f32_insts,
135	151	mad_mix_insts,
	152	mad_u32_inst,
136	153	mai_insts,
137	154	max_hard_clause_length_32,
138	155	max_hard_clause_length_63,
139	156	max_private_element_size_16,
140	157	max_private_element_size_4,
141	158	max_private_element_size_8,
	159	mcast_load_insts,
142	160	memory_atomic_fadd_f32_denormal_support,
143	161	mfma_inline_literal_bug,
144	162	mimg_r128,
	163	min3_max3_pkf16,
145	164	minimum3_maximum3_f16,
146	165	minimum3_maximum3_f32,
147	166	minimum3_maximum3_pkf16,
...	...	@@ -160,6 +179,7 @@ pub const Feature = enum {
160	179	partial_nsa_encoding,
161	180	permlane16_swap,
162	181	permlane32_swap,
	182	pk_add_min_max_insts,
163	183	pk_fmac_f16_inst,
164	184	point_sample_accel,
165	185	precise_memory,
...	...	@@ -168,6 +188,7 @@ pub const Feature = enum {
168	188	promote_alloca,
169	189	prt_strict_null,
170	190	pseudo_scalar_trans,
	191	qsad_insts,
171	192	r128_a16,
172	193	real_true16,
173	194	relaxed_buffer_oob_mode,
...	...	@@ -176,6 +197,9 @@ pub const Feature = enum {
176	197	restricted_soffset,
177	198	s_memrealtime,
178	199	s_memtime_inst,
	200	s_wakeup_barrier_inst,
	201	sad_insts,
	202	safe_cu_prefetch,
179	203	safe_smem_prefetch,
180	204	salu_float,
181	205	scalar_atomics,
...	...	@@ -190,6 +214,7 @@ pub const Feature = enum {
190	214	sdwa_sdst,
191	215	sea_islands,
192	216	setprio_inc_wg_inst,
	217	setreg_vgpr_msb_fixup,
193	218	sgpr_init_bug,
194	219	shader_cycles_hi_lo_registers,
195	220	shader_cycles_register,
...	...	@@ -198,6 +223,8 @@ pub const Feature = enum {
198	223	southern_islands,
199	224	sramecc,
200	225	sramecc_support,
	226	tanh_insts,
	227	tensor_cvt_lut_insts,
201	228	tgsplit,
202	229	transpose_load_f4f6_insts,
203	230	trap_handler,
...	...	@@ -213,7 +240,9 @@ pub const Feature = enum {
213	240	valu_trans_use_hazard,
214	241	vcmpx_exec_war_hazard,
215	242	vcmpx_permlane_hazard,
	243	vgpr_align2,
216	244	vgpr_index_mode,
	245	vmem_pref_insts,
217	246	vmem_to_lds_load_insts,
218	247	vmem_to_scalar_write_hazard,
219	248	vmem_write_vgpr_in_order,
...	...	@@ -223,6 +252,7 @@ pub const Feature = enum {
223	252	vopd,
224	253	vscnt,
225	254	wait_xcnt,
	255	waits_before_system_scope_stores,
226	256	wavefrontsize16,
227	257	wavefrontsize32,
228	258	wavefrontsize64,
...	...	@@ -241,11 +271,21 @@ pub const all_features = blk: {
241	271	const len = @typeInfo(Feature).@"enum".fields.len;
242	272	std.debug.assert(len <= CpuFeature.Set.needed_bit_count);
243	273	var result: [len]CpuFeature = undefined;
	274	result[@intFromEnum(Feature.@"1024_addressable_vgprs")] = .{
	275	.llvm_name = "1024-addressable-vgprs",
	276	.description = "Has 1024 addressable VGPRs",
	277	.dependencies = featureSet(&[_]Feature{}),
	278	};
244	279	result[@intFromEnum(Feature.@"16_bit_insts")] = .{
245	280	.llvm_name = "16-bit-insts",
246	281	.description = "Has i16/f16 instructions",
247	282	.dependencies = featureSet(&[_]Feature{}),
248	283	};
	284	result[@intFromEnum(Feature.@"45_bit_num_records_buffer_resource")] = .{
	285	.llvm_name = "45-bit-num-records-buffer-resource",
	286	.description = "The buffer resource (V#) supports 45-bit num_records",
	287	.dependencies = featureSet(&[_]Feature{}),
	288	};
249	289	result[@intFromEnum(Feature.@"64_bit_literals")] = .{
250	290	.llvm_name = "64-bit-literals",
251	291	.description = "Can use 64-bit literals with single DWORD instructions",
...	...	@@ -256,11 +296,21 @@ pub const all_features = blk: {
256	296	.description = "Support A16 for 16-bit coordinates/gradients/lod/clamp/mip image operands",
257	297	.dependencies = featureSet(&[_]Feature{}),
258	298	};
	299	result[@intFromEnum(Feature.add_min_max_insts)] = .{
	300	.llvm_name = "add-min-max-insts",
	301	.description = "Has v_add_{min\|max}_{i\|u}32 instructions",
	302	.dependencies = featureSet(&[_]Feature{}),
	303	};
259	304	result[@intFromEnum(Feature.add_no_carry_insts)] = .{
260	305	.llvm_name = "add-no-carry-insts",
261	306	.description = "Have VALU add/sub instructions without carry out",
262	307	.dependencies = featureSet(&[_]Feature{}),
263	308	};
	309	result[@intFromEnum(Feature.add_sub_u64_insts)] = .{
	310	.llvm_name = "add-sub-u64-insts",
	311	.description = "Has v_add_u64 and v_sub_u64 instructions",
	312	.dependencies = featureSet(&[_]Feature{}),
	313	};
264	314	result[@intFromEnum(Feature.addressablelocalmemorysize163840)] = .{
265	315	.llvm_name = "addressablelocalmemorysize163840",
266	316	.description = "The size of local memory in bytes",
...	...	@@ -271,6 +321,11 @@ pub const all_features = blk: {
271	321	.description = "The size of local memory in bytes",
272	322	.dependencies = featureSet(&[_]Feature{}),
273	323	};
	324	result[@intFromEnum(Feature.addressablelocalmemorysize327680)] = .{
	325	.llvm_name = "addressablelocalmemorysize327680",
	326	.description = "The size of local memory in bytes",
	327	.dependencies = featureSet(&[_]Feature{}),
	328	};
274	329	result[@intFromEnum(Feature.addressablelocalmemorysize65536)] = .{
275	330	.llvm_name = "addressablelocalmemorysize65536",
276	331	.description = "The size of local memory in bytes",
...	...	@@ -306,6 +361,11 @@ pub const all_features = blk: {
306	361	.description = "Has Arithmetic Shift Pack instructions",
307	362	.dependencies = featureSet(&[_]Feature{}),
308	363	};
	364	result[@intFromEnum(Feature.assembler_permissive_wavesize)] = .{
	365	.llvm_name = "assembler-permissive-wavesize",
	366	.description = "allow parsing wave32 and wave64 variants of instructions",
	367	.dependencies = featureSet(&[_]Feature{}),
	368	};
309	369	result[@intFromEnum(Feature.atomic_buffer_global_pk_add_f16_insts)] = .{
310	370	.llvm_name = "atomic-buffer-global-pk-add-f16-insts",
311	371	.description = "Has buffer_atomic_pk_add_f16 and global_atomic_pk_add_f16 instructions that can return original value",
...	...	@@ -357,12 +417,16 @@ pub const all_features = blk: {
357	417	result[@intFromEnum(Feature.atomic_fmin_fmax_flat_f32)] = .{
358	418	.llvm_name = "atomic-fmin-fmax-flat-f32",
359	419	.description = "Has flat memory instructions for atomicrmw fmin/fmax for float",
360		.dependencies = featureSet(&[_]Feature{}),
	420	.dependencies = featureSet(&[_]Feature{
	421	.flat_address_space,
	422	}),
361	423	};
362	424	result[@intFromEnum(Feature.atomic_fmin_fmax_flat_f64)] = .{
363	425	.llvm_name = "atomic-fmin-fmax-flat-f64",
364	426	.description = "Has flat memory instructions for atomicrmw fmin/fmax for double",
365		.dependencies = featureSet(&[_]Feature{}),
	427	.dependencies = featureSet(&[_]Feature{
	428	.flat_address_space,
	429	}),
366	430	};
367	431	result[@intFromEnum(Feature.atomic_fmin_fmax_global_f32)] = .{
368	432	.llvm_name = "atomic-fmin-fmax-global-f32",
...	...	@@ -396,6 +460,11 @@ pub const all_features = blk: {
396	460	.description = "Has bf16 conversion instructions",
397	461	.dependencies = featureSet(&[_]Feature{}),
398	462	};
	463	result[@intFromEnum(Feature.bf16_pk_insts)] = .{
	464	.llvm_name = "bf16-pk-insts",
	465	.description = "Has bf16 packed instructions (fma, add, mul, max, min)",
	466	.dependencies = featureSet(&[_]Feature{}),
	467	};
399	468	result[@intFromEnum(Feature.bf16_trans_insts)] = .{
400	469	.llvm_name = "bf16-trans-insts",
401	470	.description = "Has bf16 transcendental instructions",
...	...	@@ -426,6 +495,16 @@ pub const all_features = blk: {
426	495	.description = "Additional instructions for CI+",
427	496	.dependencies = featureSet(&[_]Feature{}),
428	497	};
	498	result[@intFromEnum(Feature.clusters)] = .{
	499	.llvm_name = "clusters",
	500	.description = "Has clusters of workgroups support",
	501	.dependencies = featureSet(&[_]Feature{}),
	502	};
	503	result[@intFromEnum(Feature.cube_insts)] = .{
	504	.llvm_name = "cube-insts",
	505	.description = "Has v_cube* instructions",
	506	.dependencies = featureSet(&[_]Feature{}),
	507	};
429	508	result[@intFromEnum(Feature.cumode)] = .{
430	509	.llvm_name = "cumode",
431	510	.description = "Enable CU wavefront execution mode",
...	...	@@ -438,11 +517,31 @@ pub const all_features = blk: {
438	517	.fp8_conversion_insts,
439	518	}),
440	519	};
	520	result[@intFromEnum(Feature.cvt_norm_insts)] = .{
	521	.llvm_name = "cvt-norm-insts",
	522	.description = "Has v_cvt_norm* instructions",
	523	.dependencies = featureSet(&[_]Feature{}),
	524	};
441	525	result[@intFromEnum(Feature.cvt_pk_f16_f32_inst)] = .{
442	526	.llvm_name = "cvt-pk-f16-f32-inst",
443	527	.description = "Has cvt_pk_f16_f32 instruction",
444	528	.dependencies = featureSet(&[_]Feature{}),
445	529	};
	530	result[@intFromEnum(Feature.cvt_pknorm_vop2_insts)] = .{
	531	.llvm_name = "cvt-pknorm-vop2-insts",
	532	.description = "Has v_cvt_pk_norm_f32 instructions/Has v_cvt_pk_norm__f16 instructions",
	533	.dependencies = featureSet(&[_]Feature{}),
	534	};
	535	result[@intFromEnum(Feature.cvt_pknorm_vop3_insts)] = .{
	536	.llvm_name = "cvt-pknorm-vop3-insts",
	537	.description = "Has v_cvt_pk_norm_f32 instructions/Has v_cvt_pk_norm__f16 instructions",
	538	.dependencies = featureSet(&[_]Feature{}),
	539	};
	540	result[@intFromEnum(Feature.d16_write_vgpr32)] = .{
	541	.llvm_name = "d16-write-vgpr32",
	542	.description = "D16 instructions potentially have 32-bit data dependencies",
	543	.dependencies = featureSet(&[_]Feature{}),
	544	};
446	545	result[@intFromEnum(Feature.default_component_broadcast)] = .{
447	546	.llvm_name = "default-component-broadcast",
448	547	.description = "BUFFER/IMAGE store instructions set unspecified components to x component (GFX12)",
...	...	@@ -553,14 +652,9 @@ pub const all_features = blk: {
553	652	.description = "Has ds_*_src2 instructions",
554	653	.dependencies = featureSet(&[_]Feature{}),
555	654	};
556		result[@intFromEnum(Feature.dynamic_vgpr)] = .{
557		.llvm_name = "dynamic-vgpr",
558		.description = "Enable dynamic VGPR mode",
559		.dependencies = featureSet(&[_]Feature{}),
560		};
561		result[@intFromEnum(Feature.dynamic_vgpr_block_size_32)] = .{
562		.llvm_name = "dynamic-vgpr-block-size-32",
563		.description = "Use a block size of 32 for dynamic VGPR allocation (default is 16)",
	655	result[@intFromEnum(Feature.emulated_system_scope_atomics)] = .{
	656	.llvm_name = "emulated-system-scope-atomics",
	657	.description = "System scope atomics unsupported by the PCI-e are emulated in HW via CAS loop and functional.",
564	658	.dependencies = featureSet(&[_]Feature{}),
565	659	};
566	660	result[@intFromEnum(Feature.extended_image_insts)] = .{
...	...	@@ -596,7 +690,9 @@ pub const all_features = blk: {
596	690	result[@intFromEnum(Feature.flat_atomic_fadd_f32_inst)] = .{
597	691	.llvm_name = "flat-atomic-fadd-f32-inst",
598	692	.description = "Has flat_atomic_add_f32 instruction",
599		.dependencies = featureSet(&[_]Feature{}),
	693	.dependencies = featureSet(&[_]Feature{
	694	.flat_address_space,
	695	}),
600	696	};
601	697	result[@intFromEnum(Feature.flat_buffer_global_fadd_f64_inst)] = .{
602	698	.llvm_name = "flat-buffer-global-fadd-f64-inst",
...	...	@@ -611,7 +707,16 @@ pub const all_features = blk: {
611	707	result[@intFromEnum(Feature.flat_global_insts)] = .{
612	708	.llvm_name = "flat-global-insts",
613	709	.description = "Have global_* flat memory instructions",
614		.dependencies = featureSet(&[_]Feature{}),
	710	.dependencies = featureSet(&[_]Feature{
	711	.flat_address_space,
	712	}),
	713	};
	714	result[@intFromEnum(Feature.flat_gvs_mode)] = .{
	715	.llvm_name = "flat-gvs-mode",
	716	.description = "Have GVS addressing mode with flat_* instructions",
	717	.dependencies = featureSet(&[_]Feature{
	718	.flat_address_space,
	719	}),
615	720	};
616	721	result[@intFromEnum(Feature.flat_inst_offsets)] = .{
617	722	.llvm_name = "flat-inst-offsets",
...	...	@@ -626,13 +731,20 @@ pub const all_features = blk: {
626	731	result[@intFromEnum(Feature.flat_scratch_insts)] = .{
627	732	.llvm_name = "flat-scratch-insts",
628	733	.description = "Have scratch_* flat memory instructions",
629		.dependencies = featureSet(&[_]Feature{}),
	734	.dependencies = featureSet(&[_]Feature{
	735	.flat_address_space,
	736	}),
630	737	};
631	738	result[@intFromEnum(Feature.flat_segment_offset_bug)] = .{
632	739	.llvm_name = "flat-segment-offset-bug",
633	740	.description = "GFX10 bug where inst_offset is ignored when flat instructions access global memory",
634	741	.dependencies = featureSet(&[_]Feature{}),
635	742	};
	743	result[@intFromEnum(Feature.fma_mix_bf16_insts)] = .{
	744	.llvm_name = "fma-mix-bf16-insts",
	745	.description = "Has v_fma_mix_f32_bf16, v_fma_mixlo_bf16, v_fma_mixhi_bf16 instructions",
	746	.dependencies = featureSet(&[_]Feature{}),
	747	};
636	748	result[@intFromEnum(Feature.fma_mix_insts)] = .{
637	749	.llvm_name = "fma-mix-insts",
638	750	.description = "Has v_fma_mix_f32, v_fma_mixlo_f16, v_fma_mixhi_f16 instructions",
...	...	@@ -722,13 +834,16 @@ pub const all_features = blk: {
722	834	.atomic_fmin_fmax_global_f32,
723	835	.atomic_fmin_fmax_global_f64,
724	836	.ci_insts,
	837	.cube_insts,
	838	.cvt_norm_insts,
	839	.cvt_pknorm_vop2_insts,
	840	.cvt_pknorm_vop3_insts,
725	841	.default_component_zero,
726	842	.dpp,
727	843	.dpp8,
728	844	.extended_image_insts,
729	845	.fast_denormal_f32,
730	846	.fast_fmaf,
731		.flat_address_space,
732	847	.flat_global_insts,
733	848	.flat_inst_offsets,
734	849	.flat_scratch_insts,
...	...	@@ -743,14 +858,17 @@ pub const all_features = blk: {
743	858	.image_insts,
744	859	.int_clamp_insts,
745	860	.inv_2pi_inline_imm,
	861	.lerp_inst,
746	862	.max_hard_clause_length_63,
747	863	.mimg_r128,
748	864	.movrel,
749	865	.no_data_dep_hazard,
750	866	.no_sdst_cmpx,
751	867	.pk_fmac_f16_inst,
	868	.qsad_insts,
752	869	.s_memrealtime,
753	870	.s_memtime_inst,
	871	.sad_insts,
754	872	.sdwa,
755	873	.sdwa_omod,
756	874	.sdwa_scalar,
...	...	@@ -797,13 +915,16 @@ pub const all_features = blk: {
797	915	.atomic_fmin_fmax_flat_f32,
798	916	.atomic_fmin_fmax_global_f32,
799	917	.ci_insts,
	918	.cube_insts,
	919	.cvt_norm_insts,
	920	.cvt_pknorm_vop2_insts,
	921	.cvt_pknorm_vop3_insts,
800	922	.default_component_zero,
801	923	.dpp,
802	924	.dpp8,
803	925	.extended_image_insts,
804	926	.fast_denormal_f32,
805	927	.fast_fmaf,
806		.flat_address_space,
807	928	.flat_global_insts,
808	929	.flat_inst_offsets,
809	930	.flat_scratch_insts,
...	...	@@ -821,12 +942,15 @@ pub const all_features = blk: {
821	942	.gws,
822	943	.int_clamp_insts,
823	944	.inv_2pi_inline_imm,
	945	.lerp_inst,
824	946	.max_hard_clause_length_32,
825	947	.mimg_r128,
826	948	.movrel,
827	949	.no_data_dep_hazard,
828	950	.no_sdst_cmpx,
829	951	.pk_fmac_f16_inst,
	952	.qsad_insts,
	953	.sad_insts,
830	954	.true16,
831	955	.unaligned_buffer_access,
832	956	.unaligned_ds_access,
...	...	@@ -850,7 +974,6 @@ pub const all_features = blk: {
850	974	.@"16_bit_insts",
851	975	.a16,
852	976	.add_no_carry_insts,
853		.addressablelocalmemorysize65536,
854	977	.agent_scope_fine_grained_remote_memory_atomics,
855	978	.aperture_regs,
856	979	.atomic_fmin_fmax_flat_f32,
...	...	@@ -861,7 +984,6 @@ pub const all_features = blk: {
861	984	.dpp8,
862	985	.fast_denormal_f32,
863	986	.fast_fmaf,
864		.flat_address_space,
865	987	.flat_global_insts,
866	988	.flat_inst_offsets,
867	989	.flat_scratch_insts,
...	...	@@ -926,11 +1048,14 @@ pub const all_features = blk: {
926	1048	.add_no_carry_insts,
927	1049	.aperture_regs,
928	1050	.ci_insts,
	1051	.cube_insts,
	1052	.cvt_norm_insts,
	1053	.cvt_pknorm_vop2_insts,
	1054	.cvt_pknorm_vop3_insts,
929	1055	.default_component_zero,
930	1056	.dpp,
931	1057	.fast_denormal_f32,
932	1058	.fast_fmaf,
933		.flat_address_space,
934	1059	.flat_global_insts,
935	1060	.flat_inst_offsets,
936	1061	.flat_scratch_insts,
...	...	@@ -942,10 +1067,13 @@ pub const all_features = blk: {
942	1067	.gws,
943	1068	.int_clamp_insts,
944	1069	.inv_2pi_inline_imm,
	1070	.lerp_inst,
945	1071	.negative_scratch_offset_bug,
	1072	.qsad_insts,
946	1073	.r128_a16,
947	1074	.s_memrealtime,
948	1075	.s_memtime_inst,
	1076	.sad_insts,
949	1077	.scalar_atomics,
950	1078	.scalar_flat_scratch_insts,
951	1079	.scalar_stores,
...	...	@@ -997,6 +1125,11 @@ pub const all_features = blk: {
997	1125	.description = "Additional instructions for GFX9+",
998	1126	.dependencies = featureSet(&[_]Feature{}),
999	1127	};
	1128	result[@intFromEnum(Feature.globally_addressable_scratch)] = .{
	1129	.llvm_name = "globally-addressable-scratch",
	1130	.description = "FLAT instructions can access scratch memory for any thread in any wave",
	1131	.dependencies = featureSet(&[_]Feature{}),
	1132	};
1000	1133	result[@intFromEnum(Feature.gws)] = .{
1001	1134	.llvm_name = "gws",
1002	1135	.description = "Has Global Wave Sync",
...	...	@@ -1072,6 +1205,11 @@ pub const all_features = blk: {
1072	1205	.description = "The number of LDS banks per compute unit.",
1073	1206	.dependencies = featureSet(&[_]Feature{}),
1074	1207	};
	1208	result[@intFromEnum(Feature.lerp_inst)] = .{
	1209	.llvm_name = "lerp-inst",
	1210	.description = "Has v_lerp_u8 instruction",
	1211	.dependencies = featureSet(&[_]Feature{}),
	1212	};
1075	1213	result[@intFromEnum(Feature.load_store_opt)] = .{
1076	1214	.llvm_name = "load-store-opt",
1077	1215	.description = "Enable SI load/store optimizer pass",
...	...	@@ -1097,6 +1235,11 @@ pub const all_features = blk: {
1097	1235	.description = "Has v_mad_mix_f32, v_mad_mixlo_f16, v_mad_mixhi_f16 instructions",
1098	1236	.dependencies = featureSet(&[_]Feature{}),
1099	1237	};
	1238	result[@intFromEnum(Feature.mad_u32_inst)] = .{
	1239	.llvm_name = "mad-u32-inst",
	1240	.description = "Has v_mad_u32 instruction",
	1241	.dependencies = featureSet(&[_]Feature{}),
	1242	};
1100	1243	result[@intFromEnum(Feature.mai_insts)] = .{
1101	1244	.llvm_name = "mai-insts",
1102	1245	.description = "Has mAI instructions",
...	...	@@ -1127,6 +1270,11 @@ pub const all_features = blk: {
1127	1270	.description = "Maximum private access size may be 8",
1128	1271	.dependencies = featureSet(&[_]Feature{}),
1129	1272	};
	1273	result[@intFromEnum(Feature.mcast_load_insts)] = .{
	1274	.llvm_name = "mcast-load-insts",
	1275	.description = "Has multicast load instructions",
	1276	.dependencies = featureSet(&[_]Feature{}),
	1277	};
1130	1278	result[@intFromEnum(Feature.memory_atomic_fadd_f32_denormal_support)] = .{
1131	1279	.llvm_name = "memory-atomic-fadd-f32-denormal-support",
1132	1280	.description = "global/flat/buffer atomic fadd for float supports denormal handling",
...	...	@@ -1142,6 +1290,11 @@ pub const all_features = blk: {
1142	1290	.description = "Support 128-bit texture resources",
1143	1291	.dependencies = featureSet(&[_]Feature{}),
1144	1292	};
	1293	result[@intFromEnum(Feature.min3_max3_pkf16)] = .{
	1294	.llvm_name = "min3-max3-pkf16",
	1295	.description = "Has v_pk_min3_num_f16 and v_pk_max3_num_f16 instructions",
	1296	.dependencies = featureSet(&[_]Feature{}),
	1297	};
1145	1298	result[@intFromEnum(Feature.minimum3_maximum3_f16)] = .{
1146	1299	.llvm_name = "minimum3-maximum3-f16",
1147	1300	.description = "Has v_minimum3_f16 and v_maximum3_f16 instructions",
...	...	@@ -1232,6 +1385,11 @@ pub const all_features = blk: {
1232	1385	.description = "Has v_permlane32_swap_b32 instructions",
1233	1386	.dependencies = featureSet(&[_]Feature{}),
1234	1387	};
	1388	result[@intFromEnum(Feature.pk_add_min_max_insts)] = .{
	1389	.llvm_name = "pk-add-min-max-insts",
	1390	.description = "Has v_pk_add_{min\|max}_{i\|u}16 instructions",
	1391	.dependencies = featureSet(&[_]Feature{}),
	1392	};
1235	1393	result[@intFromEnum(Feature.pk_fmac_f16_inst)] = .{
1236	1394	.llvm_name = "pk-fmac-f16-inst",
1237	1395	.description = "Has v_pk_fmac_f16 instruction",
...	...	@@ -1272,6 +1430,11 @@ pub const all_features = blk: {
1272	1430	.description = "Has Pseudo Scalar Transcendental instructions",
1273	1431	.dependencies = featureSet(&[_]Feature{}),
1274	1432	};
	1433	result[@intFromEnum(Feature.qsad_insts)] = .{
	1434	.llvm_name = "qsad-insts",
	1435	.description = "Has v_qsad* instructions",
	1436	.dependencies = featureSet(&[_]Feature{}),
	1437	};
1275	1438	result[@intFromEnum(Feature.r128_a16)] = .{
1276	1439	.llvm_name = "r128-a16",
1277	1440	.description = "Support gfx9-style A16 for 16-bit coordinates/gradients/lod/clamp/mip image operands, where a16 is aliased with r128",
...	...	@@ -1312,6 +1475,21 @@ pub const all_features = blk: {
1312	1475	.description = "Has s_memtime instruction",
1313	1476	.dependencies = featureSet(&[_]Feature{}),
1314	1477	};
	1478	result[@intFromEnum(Feature.s_wakeup_barrier_inst)] = .{
	1479	.llvm_name = "s-wakeup-barrier-inst",
	1480	.description = "Has s_wakeup_barrier instruction.",
	1481	.dependencies = featureSet(&[_]Feature{}),
	1482	};
	1483	result[@intFromEnum(Feature.sad_insts)] = .{
	1484	.llvm_name = "sad-insts",
	1485	.description = "Has v_sad* instructions",
	1486	.dependencies = featureSet(&[_]Feature{}),
	1487	};
	1488	result[@intFromEnum(Feature.safe_cu_prefetch)] = .{
	1489	.llvm_name = "safe-cu-prefetch",
	1490	.description = "VMEM CU scope prefetches do not fail on illegal address",
	1491	.dependencies = featureSet(&[_]Feature{}),
	1492	};
1315	1493	result[@intFromEnum(Feature.safe_smem_prefetch)] = .{
1316	1494	.llvm_name = "safe-smem-prefetch",
1317	1495	.description = "SMEM prefetches do not fail on illegal address",
...	...	@@ -1382,19 +1560,23 @@ pub const all_features = blk: {
1382	1560	.atomic_fmin_fmax_global_f32,
1383	1561	.atomic_fmin_fmax_global_f64,
1384	1562	.ci_insts,
	1563	.cube_insts,
	1564	.cvt_pknorm_vop2_insts,
1385	1565	.default_component_zero,
1386	1566	.ds_src2_insts,
1387	1567	.extended_image_insts,
1388		.flat_address_space,
1389	1568	.fp64,
1390	1569	.gds,
1391	1570	.gfx7_gfx8_gfx9_insts,
1392	1571	.gws,
1393	1572	.image_insts,
	1573	.lerp_inst,
1394	1574	.mad_mac_f32_insts,
1395	1575	.mimg_r128,
1396	1576	.movrel,
	1577	.qsad_insts,
1397	1578	.s_memtime_inst,
	1579	.sad_insts,
1398	1580	.trig_reduced_range,
1399	1581	.unaligned_buffer_access,
1400	1582	.vmem_write_vgpr_in_order,
...	...	@@ -1406,6 +1588,11 @@ pub const all_features = blk: {
1406	1588	.description = "Has s_setprio_inc_wg instruction.",
1407	1589	.dependencies = featureSet(&[_]Feature{}),
1408	1590	};
	1591	result[@intFromEnum(Feature.setreg_vgpr_msb_fixup)] = .{
	1592	.llvm_name = "setreg-vgpr-msb-fixup",
	1593	.description = "S_SETREG to MODE clobbers VGPR MSB bits, requires fixup",
	1594	.dependencies = featureSet(&[_]Feature{}),
	1595	};
1409	1596	result[@intFromEnum(Feature.sgpr_init_bug)] = .{
1410	1597	.llvm_name = "sgpr-init-bug",
1411	1598	.description = "VI SGPR initialization bug requiring a fixed SGPR allocation size",
...	...	@@ -1438,6 +1625,8 @@ pub const all_features = blk: {
1438	1625	.addressablelocalmemorysize32768,
1439	1626	.atomic_fmin_fmax_global_f32,
1440	1627	.atomic_fmin_fmax_global_f64,
	1628	.cube_insts,
	1629	.cvt_pknorm_vop2_insts,
1441	1630	.default_component_zero,
1442	1631	.ds_src2_insts,
1443	1632	.extended_image_insts,
...	...	@@ -1446,10 +1635,12 @@ pub const all_features = blk: {
1446	1635	.gws,
1447	1636	.image_insts,
1448	1637	.ldsbankcount32,
	1638	.lerp_inst,
1449	1639	.mad_mac_f32_insts,
1450	1640	.mimg_r128,
1451	1641	.movrel,
1452	1642	.s_memtime_inst,
	1643	.sad_insts,
1453	1644	.trig_reduced_range,
1454	1645	.vmem_write_vgpr_in_order,
1455	1646	.wavefrontsize64,
...	...	@@ -1465,6 +1656,16 @@ pub const all_features = blk: {
1465	1656	.description = "Hardware supports SRAMECC",
1466	1657	.dependencies = featureSet(&[_]Feature{}),
1467	1658	};
	1659	result[@intFromEnum(Feature.tanh_insts)] = .{
	1660	.llvm_name = "tanh-insts",
	1661	.description = "Has v_tanh_f32/f16 instructions",
	1662	.dependencies = featureSet(&[_]Feature{}),
	1663	};
	1664	result[@intFromEnum(Feature.tensor_cvt_lut_insts)] = .{
	1665	.llvm_name = "tensor-cvt-lut-insts",
	1666	.description = "Has v_perm_pk16* instructions",
	1667	.dependencies = featureSet(&[_]Feature{}),
	1668	};
1468	1669	result[@intFromEnum(Feature.tgsplit)] = .{
1469	1670	.llvm_name = "tgsplit",
1470	1671	.description = "Enable threadgroup split execution",
...	...	@@ -1540,11 +1741,21 @@ pub const all_features = blk: {
1540	1741	.description = "TODO: describe me",
1541	1742	.dependencies = featureSet(&[_]Feature{}),
1542	1743	};
	1744	result[@intFromEnum(Feature.vgpr_align2)] = .{
	1745	.llvm_name = "vgpr-align2",
	1746	.description = "VGPR and AGPR tuple operands require even alignment",
	1747	.dependencies = featureSet(&[_]Feature{}),
	1748	};
1543	1749	result[@intFromEnum(Feature.vgpr_index_mode)] = .{
1544	1750	.llvm_name = "vgpr-index-mode",
1545	1751	.description = "Has VGPR mode register indexing",
1546	1752	.dependencies = featureSet(&[_]Feature{}),
1547	1753	};
	1754	result[@intFromEnum(Feature.vmem_pref_insts)] = .{
	1755	.llvm_name = "vmem-pref-insts",
	1756	.description = "Has flat_prefect_b8 and global_prefetch_b8 instructions",
	1757	.dependencies = featureSet(&[_]Feature{}),
	1758	};
1548	1759	result[@intFromEnum(Feature.vmem_to_lds_load_insts)] = .{
1549	1760	.llvm_name = "vmem-to-lds-load-insts",
1550	1761	.description = "The platform has memory to lds instructions (global_load w/lds bit set, buffer_load w/lds bit set or global_load_lds. This does not include scratch_load_lds.",
...	...	@@ -1567,6 +1778,8 @@ pub const all_features = blk: {
1567	1778	.@"16_bit_insts",
1568	1779	.addressablelocalmemorysize65536,
1569	1780	.ci_insts,
	1781	.cube_insts,
	1782	.cvt_pknorm_vop2_insts,
1570	1783	.default_component_zero,
1571	1784	.dpp,
1572	1785	.ds_src2_insts,
...	...	@@ -1582,11 +1795,14 @@ pub const all_features = blk: {
1582	1795	.image_insts,
1583	1796	.int_clamp_insts,
1584	1797	.inv_2pi_inline_imm,
	1798	.lerp_inst,
1585	1799	.mad_mac_f32_insts,
1586	1800	.mimg_r128,
1587	1801	.movrel,
	1802	.qsad_insts,
1588	1803	.s_memrealtime,
1589	1804	.s_memtime_inst,
	1805	.sad_insts,
1590	1806	.scalar_stores,
1591	1807	.sdwa,
1592	1808	.sdwa_mav,
...	...	@@ -1623,6 +1839,11 @@ pub const all_features = blk: {
1623	1839	.description = "Has s_wait_xcnt instruction",
1624	1840	.dependencies = featureSet(&[_]Feature{}),
1625	1841	};
	1842	result[@intFromEnum(Feature.waits_before_system_scope_stores)] = .{
	1843	.llvm_name = "waits-before-system-scope-stores",
	1844	.description = "Target requires waits for loads and atomics before system scope stores",
	1845	.dependencies = featureSet(&[_]Feature{}),
	1846	};
1626	1847	result[@intFromEnum(Feature.wavefrontsize16)] = .{
1627	1848	.llvm_name = "wavefrontsize16",
1628	1849	.description = "The number of threads per wavefront",
...	...	@@ -2044,6 +2265,8 @@ pub const cpu = struct {
2044	2265	.architected_flat_scratch,
2045	2266	.atomic_fadd_no_rtn_insts,
2046	2267	.atomic_fadd_rtn_insts,
	2268	.back_off_barrier,
	2269	.d16_write_vgpr32,
2047	2270	.dl_insts,
2048	2271	.dot10_insts,
2049	2272	.dot12_insts,
...	...	@@ -2077,6 +2300,8 @@ pub const cpu = struct {
2077	2300	.architected_flat_scratch,
2078	2301	.atomic_fadd_no_rtn_insts,
2079	2302	.atomic_fadd_rtn_insts,
	2303	.back_off_barrier,
	2304	.d16_write_vgpr32,
2080	2305	.dl_insts,
2081	2306	.dot10_insts,
2082	2307	.dot12_insts,
...	...	@@ -2108,6 +2333,8 @@ pub const cpu = struct {
2108	2333	.architected_flat_scratch,
2109	2334	.atomic_fadd_no_rtn_insts,
2110	2335	.atomic_fadd_rtn_insts,
	2336	.back_off_barrier,
	2337	.d16_write_vgpr32,
2111	2338	.dl_insts,
2112	2339	.dot10_insts,
2113	2340	.dot12_insts,
...	...	@@ -2140,6 +2367,8 @@ pub const cpu = struct {
2140	2367	.architected_flat_scratch,
2141	2368	.atomic_fadd_no_rtn_insts,
2142	2369	.atomic_fadd_rtn_insts,
	2370	.back_off_barrier,
	2371	.d16_write_vgpr32,
2143	2372	.dl_insts,
2144	2373	.dot10_insts,
2145	2374	.dot12_insts,
...	...	@@ -2171,6 +2400,8 @@ pub const cpu = struct {
2171	2400	.architected_flat_scratch,
2172	2401	.atomic_fadd_no_rtn_insts,
2173	2402	.atomic_fadd_rtn_insts,
	2403	.back_off_barrier,
	2404	.d16_write_vgpr32,
2174	2405	.dl_insts,
2175	2406	.dot10_insts,
2176	2407	.dot12_insts,
...	...	@@ -2188,6 +2419,7 @@ pub const cpu = struct {
2188	2419	.packed_tid,
2189	2420	.partial_nsa_encoding,
2190	2421	.point_sample_accel,
	2422	.real_true16,
2191	2423	.required_export_priority,
2192	2424	.salu_float,
2193	2425	.shader_cycles_register,
...	...	@@ -2202,6 +2434,8 @@ pub const cpu = struct {
2202	2434	.architected_flat_scratch,
2203	2435	.atomic_fadd_no_rtn_insts,
2204	2436	.atomic_fadd_rtn_insts,
	2437	.back_off_barrier,
	2438	.d16_write_vgpr32,
2205	2439	.dl_insts,
2206	2440	.dot10_insts,
2207	2441	.dot12_insts,
...	...	@@ -2219,6 +2453,7 @@ pub const cpu = struct {
2219	2453	.packed_tid,
2220	2454	.partial_nsa_encoding,
2221	2455	.point_sample_accel,
	2456	.real_true16,
2222	2457	.required_export_priority,
2223	2458	.salu_float,
2224	2459	.shader_cycles_register,
...	...	@@ -2232,6 +2467,8 @@ pub const cpu = struct {
2232	2467	.architected_flat_scratch,
2233	2468	.atomic_fadd_no_rtn_insts,
2234	2469	.atomic_fadd_rtn_insts,
	2470	.back_off_barrier,
	2471	.d16_write_vgpr32,
2235	2472	.dl_insts,
2236	2473	.dot10_insts,
2237	2474	.dot12_insts,
...	...	@@ -2249,6 +2486,7 @@ pub const cpu = struct {
2249	2486	.packed_tid,
2250	2487	.partial_nsa_encoding,
2251	2488	.point_sample_accel,
	2489	.real_true16,
2252	2490	.required_export_priority,
2253	2491	.salu_float,
2254	2492	.shader_cycles_register,
...	...	@@ -2262,6 +2500,8 @@ pub const cpu = struct {
2262	2500	.architected_flat_scratch,
2263	2501	.atomic_fadd_no_rtn_insts,
2264	2502	.atomic_fadd_rtn_insts,
	2503	.back_off_barrier,
	2504	.d16_write_vgpr32,
2265	2505	.dl_insts,
2266	2506	.dot10_insts,
2267	2507	.dot12_insts,
...	...	@@ -2278,6 +2518,7 @@ pub const cpu = struct {
2278	2518	.nsa_encoding,
2279	2519	.packed_tid,
2280	2520	.partial_nsa_encoding,
	2521	.real_true16,
2281	2522	.required_export_priority,
2282	2523	.salu_float,
2283	2524	.shader_cycles_register,
...	...	@@ -2291,6 +2532,8 @@ pub const cpu = struct {
2291	2532	.architected_flat_scratch,
2292	2533	.atomic_fadd_no_rtn_insts,
2293	2534	.atomic_fadd_rtn_insts,
	2535	.back_off_barrier,
	2536	.d16_write_vgpr32,
2294	2537	.dl_insts,
2295	2538	.dot10_insts,
2296	2539	.dot12_insts,
...	...	@@ -2309,6 +2552,7 @@ pub const cpu = struct {
2309	2552	.packed_tid,
2310	2553	.partial_nsa_encoding,
2311	2554	.priv_enabled_trap2_nop_bug,
	2555	.real_true16,
2312	2556	.required_export_priority,
2313	2557	.requires_cov6,
2314	2558	.shader_cycles_register,
...	...	@@ -2321,6 +2565,7 @@ pub const cpu = struct {
2321	2565	.name = "gfx1200",
2322	2566	.llvm_name = "gfx1200",
2323	2567	.features = featureSet(&[_]Feature{
	2568	.addressablelocalmemorysize65536,
2324	2569	.allocate1_5xvgprs,
2325	2570	.architected_flat_scratch,
2326	2571	.architected_sgprs,
...	...	@@ -2331,7 +2576,13 @@ pub const cpu = struct {
2331	2576	.atomic_fadd_rtn_insts,
2332	2577	.atomic_flat_pk_add_16_insts,
2333	2578	.atomic_global_pk_add_bf16_inst,
	2579	.back_off_barrier,
2334	2580	.bvh_dual_bvh_8_insts,
	2581	.cube_insts,
	2582	.cvt_norm_insts,
	2583	.cvt_pknorm_vop2_insts,
	2584	.cvt_pknorm_vop3_insts,
	2585	.d16_write_vgpr32,
2335	2586	.dl_insts,
2336	2587	.dot10_insts,
2337	2588	.dot11_insts,
...	...	@@ -2346,22 +2597,27 @@ pub const cpu = struct {
2346	2597	.gfx12,
2347	2598	.image_insts,
2348	2599	.ldsbankcount32,
	2600	.lerp_inst,
2349	2601	.memory_atomic_fadd_f32_denormal_support,
2350	2602	.nsa_encoding,
2351	2603	.packed_tid,
2352	2604	.partial_nsa_encoding,
2353	2605	.pseudo_scalar_trans,
	2606	.qsad_insts,
2354	2607	.restricted_soffset,
	2608	.sad_insts,
2355	2609	.salu_float,
2356	2610	.scalar_dwordx3_loads,
2357	2611	.shader_cycles_hi_lo_registers,
2358	2612	.vcmpx_permlane_hazard,
	2613	.waits_before_system_scope_stores,
2359	2614	}),
2360	2615	};
2361	2616	pub const gfx1201: CpuModel = .{
2362	2617	.name = "gfx1201",
2363	2618	.llvm_name = "gfx1201",
2364	2619	.features = featureSet(&[_]Feature{
	2620	.addressablelocalmemorysize65536,
2365	2621	.allocate1_5xvgprs,
2366	2622	.architected_flat_scratch,
2367	2623	.architected_sgprs,
...	...	@@ -2372,7 +2628,13 @@ pub const cpu = struct {
2372	2628	.atomic_fadd_rtn_insts,
2373	2629	.atomic_flat_pk_add_16_insts,
2374	2630	.atomic_global_pk_add_bf16_inst,
	2631	.back_off_barrier,
2375	2632	.bvh_dual_bvh_8_insts,
	2633	.cube_insts,
	2634	.cvt_norm_insts,
	2635	.cvt_pknorm_vop2_insts,
	2636	.cvt_pknorm_vop3_insts,
	2637	.d16_write_vgpr32,
2376	2638	.dl_insts,
2377	2639	.dot10_insts,
2378	2640	.dot11_insts,
...	...	@@ -2387,23 +2649,32 @@ pub const cpu = struct {
2387	2649	.gfx12,
2388	2650	.image_insts,
2389	2651	.ldsbankcount32,
	2652	.lerp_inst,
2390	2653	.memory_atomic_fadd_f32_denormal_support,
2391	2654	.nsa_encoding,
2392	2655	.packed_tid,
2393	2656	.partial_nsa_encoding,
2394	2657	.pseudo_scalar_trans,
	2658	.qsad_insts,
2395	2659	.restricted_soffset,
	2660	.sad_insts,
2396	2661	.salu_float,
2397	2662	.scalar_dwordx3_loads,
2398	2663	.shader_cycles_hi_lo_registers,
2399	2664	.vcmpx_permlane_hazard,
	2665	.waits_before_system_scope_stores,
2400	2666	}),
2401	2667	};
2402	2668	pub const gfx1250: CpuModel = .{
2403	2669	.name = "gfx1250",
2404	2670	.llvm_name = "gfx1250",
2405	2671	.features = featureSet(&[_]Feature{
	2672	.@"1024_addressable_vgprs",
	2673	.@"45_bit_num_records_buffer_resource",
2406	2674	.@"64_bit_literals",
	2675	.add_min_max_insts,
	2676	.add_sub_u64_insts,
	2677	.addressablelocalmemorysize327680,
2407	2678	.architected_flat_scratch,
2408	2679	.architected_sgprs,
2409	2680	.ashr_pk_insts,
...	...	@@ -2417,49 +2688,164 @@ pub const cpu = struct {
2417	2688	.atomic_fmin_fmax_global_f64,
2418	2689	.atomic_global_pk_add_bf16_inst,
2419	2690	.bf16_cvt_insts,
	2691	.bf16_pk_insts,
2420	2692	.bf16_trans_insts,
2421	2693	.bitop3_insts,
	2694	.clusters,
	2695	.cube_insts,
2422	2696	.cumode,
	2697	.cvt_norm_insts,
2423	2698	.cvt_pk_f16_f32_inst,
	2699	.cvt_pknorm_vop2_insts,
	2700	.cvt_pknorm_vop3_insts,
	2701	.d16_write_vgpr32,
2424	2702	.dl_insts,
2425	2703	.dot7_insts,
2426	2704	.dot8_insts,
2427	2705	.dpp_src1_sgpr,
	2706	.emulated_system_scope_atomics,
2428	2707	.flat_atomic_fadd_f32_inst,
2429	2708	.flat_buffer_global_fadd_f64_inst,
	2709	.flat_gvs_mode,
	2710	.fma_mix_bf16_insts,
2430	2711	.fmacf64_inst,
2431	2712	.fp8_conversion_insts,
2432	2713	.fp8e5m3_insts,
2433	2714	.gfx12,
2434	2715	.gfx1250_insts,
	2716	.globally_addressable_scratch,
2435	2717	.kernarg_preload,
2436	2718	.lds_barrier_arrive_atomic,
2437	2719	.ldsbankcount32,
	2720	.lerp_inst,
2438	2721	.lshl_add_u64_inst,
	2722	.mad_u32_inst,
2439	2723	.max_hard_clause_length_63,
	2724	.mcast_load_insts,
2440	2725	.memory_atomic_fadd_f32_denormal_support,
	2726	.min3_max3_pkf16,
2441	2727	.minimum3_maximum3_pkf16,
2442	2728	.packed_fp32_ops,
2443	2729	.packed_tid,
2444	2730	.permlane16_swap,
	2731	.pk_add_min_max_insts,
2445	2732	.prng_inst,
2446	2733	.pseudo_scalar_trans,
	2734	.qsad_insts,
2447	2735	.restricted_soffset,
	2736	.s_wakeup_barrier_inst,
	2737	.sad_insts,
2448	2738	.salu_float,
2449	2739	.scalar_dwordx3_loads,
2450	2740	.setprio_inc_wg_inst,
	2741	.setreg_vgpr_msb_fixup,
2451	2742	.shader_cycles_hi_lo_registers,
2452	2743	.sramecc_support,
	2744	.tanh_insts,
	2745	.tensor_cvt_lut_insts,
2453	2746	.transpose_load_f4f6_insts,
2454	2747	.vcmpx_permlane_hazard,
	2748	.vgpr_align2,
	2749	.vmem_pref_insts,
2455	2750	.wait_xcnt,
2456	2751	.wavefrontsize32,
	2752	.xnack,
	2753	.xnack_support,
	2754	}),
	2755	};
	2756	pub const gfx1251: CpuModel = .{
	2757	.name = "gfx1251",
	2758	.llvm_name = "gfx1251",
	2759	.features = featureSet(&[_]Feature{
	2760	.@"1024_addressable_vgprs",
	2761	.@"45_bit_num_records_buffer_resource",
	2762	.@"64_bit_literals",
	2763	.add_min_max_insts,
	2764	.add_sub_u64_insts,
	2765	.addressablelocalmemorysize327680,
	2766	.architected_flat_scratch,
	2767	.architected_sgprs,
	2768	.ashr_pk_insts,
	2769	.atomic_buffer_global_pk_add_f16_insts,
	2770	.atomic_buffer_pk_add_bf16_inst,
	2771	.atomic_ds_pk_add_16_insts,
	2772	.atomic_fadd_no_rtn_insts,
	2773	.atomic_fadd_rtn_insts,
	2774	.atomic_flat_pk_add_16_insts,
	2775	.atomic_fmin_fmax_flat_f64,
	2776	.atomic_fmin_fmax_global_f64,
	2777	.atomic_global_pk_add_bf16_inst,
	2778	.bf16_cvt_insts,
	2779	.bf16_pk_insts,
	2780	.bf16_trans_insts,
	2781	.bitop3_insts,
	2782	.clusters,
	2783	.cube_insts,
	2784	.cumode,
	2785	.cvt_norm_insts,
	2786	.cvt_pk_f16_f32_inst,
	2787	.cvt_pknorm_vop2_insts,
	2788	.cvt_pknorm_vop3_insts,
	2789	.d16_write_vgpr32,
	2790	.dl_insts,
	2791	.dot7_insts,
	2792	.dot8_insts,
	2793	.dpp_64bit,
	2794	.dpp_src1_sgpr,
	2795	.emulated_system_scope_atomics,
	2796	.flat_atomic_fadd_f32_inst,
	2797	.flat_buffer_global_fadd_f64_inst,
	2798	.flat_gvs_mode,
	2799	.fma_mix_bf16_insts,
	2800	.fmacf64_inst,
	2801	.fp8_conversion_insts,
	2802	.fp8e5m3_insts,
	2803	.gfx12,
	2804	.gfx1250_insts,
	2805	.globally_addressable_scratch,
	2806	.kernarg_preload,
	2807	.lds_barrier_arrive_atomic,
	2808	.ldsbankcount32,
	2809	.lerp_inst,
	2810	.lshl_add_u64_inst,
	2811	.mad_u32_inst,
	2812	.max_hard_clause_length_63,
	2813	.mcast_load_insts,
	2814	.memory_atomic_fadd_f32_denormal_support,
	2815	.min3_max3_pkf16,
	2816	.minimum3_maximum3_pkf16,
	2817	.packed_fp32_ops,
	2818	.packed_tid,
	2819	.permlane16_swap,
	2820	.pk_add_min_max_insts,
	2821	.prng_inst,
	2822	.pseudo_scalar_trans,
	2823	.qsad_insts,
	2824	.restricted_soffset,
	2825	.s_wakeup_barrier_inst,
	2826	.sad_insts,
	2827	.salu_float,
	2828	.scalar_dwordx3_loads,
	2829	.setprio_inc_wg_inst,
	2830	.shader_cycles_hi_lo_registers,
	2831	.sramecc_support,
	2832	.tanh_insts,
	2833	.tensor_cvt_lut_insts,
	2834	.transpose_load_f4f6_insts,
	2835	.vcmpx_permlane_hazard,
	2836	.vgpr_align2,
	2837	.vmem_pref_insts,
	2838	.wait_xcnt,
	2839	.wavefrontsize32,
	2840	.xnack,
	2841	.xnack_support,
2457	2842	}),
2458	2843	};
2459	2844	pub const gfx12_generic: CpuModel = .{
2460	2845	.name = "gfx12_generic",
2461	2846	.llvm_name = "gfx12-generic",
2462	2847	.features = featureSet(&[_]Feature{
	2848	.addressablelocalmemorysize65536,
2463	2849	.allocate1_5xvgprs,
2464	2850	.architected_flat_scratch,
2465	2851	.architected_sgprs,
...	...	@@ -2470,7 +2856,13 @@ pub const cpu = struct {
2470	2856	.atomic_fadd_rtn_insts,
2471	2857	.atomic_flat_pk_add_16_insts,
2472	2858	.atomic_global_pk_add_bf16_inst,
	2859	.back_off_barrier,
2473	2860	.bvh_dual_bvh_8_insts,
	2861	.cube_insts,
	2862	.cvt_norm_insts,
	2863	.cvt_pknorm_vop2_insts,
	2864	.cvt_pknorm_vop3_insts,
	2865	.d16_write_vgpr32,
2474	2866	.dl_insts,
2475	2867	.dot10_insts,
2476	2868	.dot11_insts,
...	...	@@ -2485,17 +2877,21 @@ pub const cpu = struct {
2485	2877	.gfx12,
2486	2878	.image_insts,
2487	2879	.ldsbankcount32,
	2880	.lerp_inst,
2488	2881	.memory_atomic_fadd_f32_denormal_support,
2489	2882	.nsa_encoding,
2490	2883	.packed_tid,
2491	2884	.partial_nsa_encoding,
2492	2885	.pseudo_scalar_trans,
	2886	.qsad_insts,
2493	2887	.requires_cov6,
2494	2888	.restricted_soffset,
	2889	.sad_insts,
2495	2890	.salu_float,
2496	2891	.scalar_dwordx3_loads,
2497	2892	.shader_cycles_hi_lo_registers,
2498	2893	.vcmpx_permlane_hazard,
	2894	.waits_before_system_scope_stores,
2499	2895	}),
2500	2896	};
2501	2897	pub const gfx600: CpuModel = .{
...	...	@@ -2779,6 +3175,7 @@ pub const cpu = struct {
2779	3175	.packed_tid,
2780	3176	.pk_fmac_f16_inst,
2781	3177	.sramecc_support,
	3178	.vgpr_align2,
2782	3179	}),
2783	3180	};
2784	3181	pub const gfx90c: CpuModel = .{
...	...	@@ -2842,6 +3239,7 @@ pub const cpu = struct {
2842	3239	.packed_tid,
2843	3240	.pk_fmac_f16_inst,
2844	3241	.sramecc_support,
	3242	.vgpr_align2,
2845	3243	.xf32_insts,
2846	3244	}),
2847	3245	};
...	...	@@ -2897,6 +3295,7 @@ pub const cpu = struct {
2897	3295	.pk_fmac_f16_inst,
2898	3296	.prng_inst,
2899	3297	.sramecc_support,
	3298	.vgpr_align2,
2900	3299	}),
2901	3300	};
2902	3301	pub const gfx9_4_generic: CpuModel = .{
...	...	@@ -2943,6 +3342,7 @@ pub const cpu = struct {
2943	3342	.pk_fmac_f16_inst,
2944	3343	.requires_cov6,
2945	3344	.sramecc_support,
	3345	.vgpr_align2,
2946	3346	}),
2947	3347	};
2948	3348	pub const gfx9_generic: CpuModel = .{

lib/std/Target/arm.zig+40-6

...	...	@@ -88,6 +88,7 @@ pub const Feature = enum {
88	88	has_v9_4a,
89	89	has_v9_5a,
90	90	has_v9_6a,
	91	has_v9_7a,
91	92	has_v9a,
92	93	hwdiv,
93	94	hwdiv_arm,
...	...	@@ -107,7 +108,6 @@ pub const Feature = enum {
107	108	mve2beat,
108	109	mve4beat,
109	110	mve_fp,
110		nacl_trap,
111	111	neon,
112	112	neon_fpmovs,
113	113	neonfp,
...	...	@@ -187,6 +187,7 @@ pub const Feature = enum {
187	187	v9_4a,
188	188	v9_5a,
189	189	v9_6a,
	190	v9_7a,
190	191	v9a,
191	192	vfp2,
192	193	vfp2sp,
...	...	@@ -748,6 +749,13 @@ pub const all_features = blk: {
748	749	.has_v9_5a,
749	750	}),
750	751	};
	752	result[@intFromEnum(Feature.has_v9_7a)] = .{
	753	.llvm_name = "v9.7a",
	754	.description = "Support ARM v9.7a instructions",
	755	.dependencies = featureSet(&[_]Feature{
	756	.has_v9_6a,
	757	}),
	758	};
751	759	result[@intFromEnum(Feature.has_v9a)] = .{
752	760	.llvm_name = "v9a",
753	761	.description = "Support ARM v9a instructions",
...	...	@@ -859,11 +867,6 @@ pub const all_features = blk: {
859	867	.mve,
860	868	}),
861	869	};
862		result[@intFromEnum(Feature.nacl_trap)] = .{
863		.llvm_name = "nacl-trap",
864		.description = "NaCl trap",
865		.dependencies = featureSet(&[_]Feature{}),
866		};
867	870	result[@intFromEnum(Feature.neon)] = .{
868	871	.llvm_name = "neon",
869	872	.description = "Enable NEON instructions",
...	...	@@ -1579,6 +1582,22 @@ pub const all_features = blk: {
1579	1582	.virtualization,
1580	1583	}),
1581	1584	};
	1585	result[@intFromEnum(Feature.v9_7a)] = .{
	1586	.llvm_name = "armv9.7-a",
	1587	.description = "ARMv97a architecture",
	1588	.dependencies = featureSet(&[_]Feature{
	1589	.aclass,
	1590	.crc,
	1591	.db,
	1592	.dsp,
	1593	.fp_armv8,
	1594	.has_v9_7a,
	1595	.mp,
	1596	.ras,
	1597	.trustzone,
	1598	.virtualization,
	1599	}),
	1600	};
1582	1601	result[@intFromEnum(Feature.v9a)] = .{
1583	1602	.llvm_name = "armv9-a",
1584	1603	.description = "ARMv9a architecture",
...	...	@@ -2658,6 +2677,21 @@ pub const cpu = struct {
2658	2677	.v8m_main,
2659	2678	}),
2660	2679	};
	2680	pub const star_mc3: CpuModel = .{
	2681	.name = "star_mc3",
	2682	.llvm_name = "star-mc3",
	2683	.features = featureSet(&[_]Feature{
	2684	.fp_armv8d16,
	2685	.loop_align,
	2686	.mve1beat,
	2687	.mve_fp,
	2688	.no_branch_predictor,
	2689	.pacbti,
	2690	.slowfpvmlx,
	2691	.use_misched,
	2692	.v8_1m_main,
	2693	}),
	2694	};
2661	2695	pub const strongarm: CpuModel = .{
2662	2696	.name = "strongarm",
2663	2697	.llvm_name = "strongarm",

lib/std/Target/bpf.zig+6

...	...	@@ -5,6 +5,7 @@ const CpuFeature = std.Target.Cpu.Feature;
5	5	const CpuModel = std.Target.Cpu.Model;
6	6
7	7	pub const Feature = enum {
	8	allows_misaligned_mem_access,
8	9	alu32,
9	10	dummy,
10	11	dwarfris,
...	...	@@ -19,6 +20,11 @@ pub const all_features = blk: {
19	20	const len = @typeInfo(Feature).@"enum".fields.len;
20	21	std.debug.assert(len <= CpuFeature.Set.needed_bit_count);
21	22	var result: [len]CpuFeature = undefined;
	23	result[@intFromEnum(Feature.allows_misaligned_mem_access)] = .{
	24	.llvm_name = "allows-misaligned-mem-access",
	25	.description = "Allows misaligned memory access",
	26	.dependencies = featureSet(&[_]Feature{}),
	27	};
22	28	result[@intFromEnum(Feature.alu32)] = .{
23	29	.llvm_name = "alu32",
24	30	.description = "Enable ALU32 instructions",

lib/std/Target/hexagon.zig+41-6

...	...	@@ -25,6 +25,7 @@ pub const Feature = enum {
25	25	hvxv73,
26	26	hvxv75,
27	27	hvxv79,
	28	hvxv81,
28	29	long_calls,
29	30	mem_noshuf,
30	31	memops,
...	...	@@ -36,7 +37,6 @@ pub const Feature = enum {
36	37	reserved_r19,
37	38	small_data,
38	39	tinycore,
39		unsafe_fp,
40	40	v5,
41	41	v55,
42	42	v60,
...	...	@@ -50,6 +50,7 @@ pub const Feature = enum {
50	50	v73,
51	51	v75,
52	52	v79,
	53	v81,
53	54	zreg,
54	55	};
55	56
...	...	@@ -189,6 +190,13 @@ pub const all_features = blk: {
189	190	.hvxv75,
190	191	}),
191	192	};
	193	result[@intFromEnum(Feature.hvxv81)] = .{
	194	.llvm_name = "hvxv81",
	195	.description = "Hexagon HVX instructions",
	196	.dependencies = featureSet(&[_]Feature{
	197	.hvxv79,
	198	}),
	199	};
192	200	result[@intFromEnum(Feature.long_calls)] = .{
193	201	.llvm_name = "long-calls",
194	202	.description = "Use constant-extended calls",
...	...	@@ -248,11 +256,6 @@ pub const all_features = blk: {
248	256	.description = "Hexagon Tiny Core",
249	257	.dependencies = featureSet(&[_]Feature{}),
250	258	};
251		result[@intFromEnum(Feature.unsafe_fp)] = .{
252		.llvm_name = "unsafe-fp",
253		.description = "Use unsafe FP math",
254		.dependencies = featureSet(&[_]Feature{}),
255		};
256	259	result[@intFromEnum(Feature.v5)] = .{
257	260	.llvm_name = "v5",
258	261	.description = "Enable Hexagon V5 architecture",
...	...	@@ -318,6 +321,11 @@ pub const all_features = blk: {
318	321	.description = "Enable Hexagon V79 architecture",
319	322	.dependencies = featureSet(&[_]Feature{}),
320	323	};
	324	result[@intFromEnum(Feature.v81)] = .{
	325	.llvm_name = "v81",
	326	.description = "Enable Hexagon V81 architecture",
	327	.dependencies = featureSet(&[_]Feature{}),
	328	};
321	329	result[@intFromEnum(Feature.zreg)] = .{
322	330	.llvm_name = "zreg",
323	331	.description = "Hexagon ZReg extension instructions",
...	...	@@ -662,4 +670,31 @@ pub const cpu = struct {
662	670	.v79,
663	671	}),
664	672	};
	673	pub const hexagonv81: CpuModel = .{
	674	.name = "hexagonv81",
	675	.llvm_name = "hexagonv81",
	676	.features = featureSet(&[_]Feature{
	677	.compound,
	678	.duplex,
	679	.mem_noshuf,
	680	.memops,
	681	.nvj,
	682	.nvs,
	683	.small_data,
	684	.v5,
	685	.v55,
	686	.v60,
	687	.v62,
	688	.v65,
	689	.v66,
	690	.v67,
	691	.v68,
	692	.v69,
	693	.v71,
	694	.v73,
	695	.v75,
	696	.v79,
	697	.v81,
	698	}),
	699	};
665	700	};

lib/std/Target/loongarch.zig+18

...	...	@@ -175,6 +175,24 @@ pub const cpu = struct {
175	175	.ual,
176	176	}),
177	177	};
	178	pub const la32rv1_0: CpuModel = .{
	179	.name = "la32rv1_0",
	180	.llvm_name = null,
	181	.features = featureSet(&[_]Feature{
	182	.@"32bit",
	183	.ual,
	184	}),
	185	};
	186	pub const la32v1_0: CpuModel = .{
	187	.name = "la32v1_0",
	188	.llvm_name = null,
	189	.features = featureSet(&[_]Feature{
	190	.@"32bit",
	191	.@"32s",
	192	.d,
	193	.ual,
	194	}),
	195	};
178	196	pub const la464: CpuModel = .{
179	197	.name = "la464",
180	198	.llvm_name = "la464",

lib/std/Target/mips.zig+6

...	...	@@ -56,6 +56,7 @@ pub const Feature = enum {
56	56	soft_float,
57	57	strict_align,
58	58	sym32,
	59	use_compact_branches,
59	60	use_indirect_jump_hazard,
60	61	use_tcc_in_div,
61	62	vfpu,
...	...	@@ -391,6 +392,11 @@ pub const all_features = blk: {
391	392	.description = "Symbols are 32 bit on Mips64",
392	393	.dependencies = featureSet(&[_]Feature{}),
393	394	};
	395	result[@intFromEnum(Feature.use_compact_branches)] = .{
	396	.llvm_name = "use-compact-branches",
	397	.description = "Use compact branch instructions for MIPS32R6/MIPS64R6",
	398	.dependencies = featureSet(&[_]Feature{}),
	399	};
394	400	result[@intFromEnum(Feature.use_indirect_jump_hazard)] = .{
395	401	.llvm_name = "use-indirect-jump-hazard",
396	402	.description = "Use indirect jump guards to prevent certain speculation based attacks",

lib/std/Target/nvptx.zig+58-35

...	...	@@ -35,6 +35,7 @@ pub const Feature = enum {
35	35	ptx86,
36	36	ptx87,
37	37	ptx88,
	38	ptx90,
38	39	sm_100,
39	40	sm_100a,
40	41	sm_100f,
...	...	@@ -44,6 +45,9 @@ pub const Feature = enum {
44	45	sm_103,
45	46	sm_103a,
46	47	sm_103f,
	48	sm_110,
	49	sm_110a,
	50	sm_110f,
47	51	sm_120,
48	52	sm_120a,
49	53	sm_120f,
...	...	@@ -68,6 +72,7 @@ pub const Feature = enum {
68	72	sm_80,
69	73	sm_86,
70	74	sm_87,
	75	sm_88,
71	76	sm_89,
72	77	sm_90,
73	78	sm_90a,
...	...	@@ -232,6 +237,11 @@ pub const all_features = blk: {
232	237	.description = "Use PTX version 88",
233	238	.dependencies = featureSet(&[_]Feature{}),
234	239	};
	240	result[@intFromEnum(Feature.ptx90)] = .{
	241	.llvm_name = "ptx90",
	242	.description = "Use PTX version 90",
	243	.dependencies = featureSet(&[_]Feature{}),
	244	};
235	245	result[@intFromEnum(Feature.sm_100)] = .{
236	246	.llvm_name = "sm_100",
237	247	.description = "Target SM 100",
...	...	@@ -277,6 +287,21 @@ pub const all_features = blk: {
277	287	.description = "Target SM 103f",
278	288	.dependencies = featureSet(&[_]Feature{}),
279	289	};
	290	result[@intFromEnum(Feature.sm_110)] = .{
	291	.llvm_name = "sm_110",
	292	.description = "Target SM 110",
	293	.dependencies = featureSet(&[_]Feature{}),
	294	};
	295	result[@intFromEnum(Feature.sm_110a)] = .{
	296	.llvm_name = "sm_110a",
	297	.description = "Target SM 110a",
	298	.dependencies = featureSet(&[_]Feature{}),
	299	};
	300	result[@intFromEnum(Feature.sm_110f)] = .{
	301	.llvm_name = "sm_110f",
	302	.description = "Target SM 110f",
	303	.dependencies = featureSet(&[_]Feature{}),
	304	};
280	305	result[@intFromEnum(Feature.sm_120)] = .{
281	306	.llvm_name = "sm_120",
282	307	.description = "Target SM 120",
...	...	@@ -397,6 +422,11 @@ pub const all_features = blk: {
397	422	.description = "Target SM 87",
398	423	.dependencies = featureSet(&[_]Feature{}),
399	424	};
	425	result[@intFromEnum(Feature.sm_88)] = .{
	426	.llvm_name = "sm_88",
	427	.description = "Target SM 88",
	428	.dependencies = featureSet(&[_]Feature{}),
	429	};
400	430	result[@intFromEnum(Feature.sm_89)] = .{
401	431	.llvm_name = "sm_89",
402	432	.description = "Target SM 89",
...	...	@@ -425,7 +455,6 @@ pub const cpu = struct {
425	455	.name = "sm_100",
426	456	.llvm_name = "sm_100",
427	457	.features = featureSet(&[_]Feature{
428		.ptx86,
429	458	.sm_100,
430	459	}),
431	460	};
...	...	@@ -433,7 +462,6 @@ pub const cpu = struct {
433	462	.name = "sm_100a",
434	463	.llvm_name = "sm_100a",
435	464	.features = featureSet(&[_]Feature{
436		.ptx86,
437	465	.sm_100a,
438	466	}),
439	467	};
...	...	@@ -441,7 +469,6 @@ pub const cpu = struct {
441	469	.name = "sm_100f",
442	470	.llvm_name = "sm_100f",
443	471	.features = featureSet(&[_]Feature{
444		.ptx88,
445	472	.sm_100f,
446	473	}),
447	474	};
...	...	@@ -449,7 +476,6 @@ pub const cpu = struct {
449	476	.name = "sm_101",
450	477	.llvm_name = "sm_101",
451	478	.features = featureSet(&[_]Feature{
452		.ptx86,
453	479	.sm_101,
454	480	}),
455	481	};
...	...	@@ -457,7 +483,6 @@ pub const cpu = struct {
457	483	.name = "sm_101a",
458	484	.llvm_name = "sm_101a",
459	485	.features = featureSet(&[_]Feature{
460		.ptx86,
461	486	.sm_101a,
462	487	}),
463	488	};
...	...	@@ -465,7 +490,6 @@ pub const cpu = struct {
465	490	.name = "sm_101f",
466	491	.llvm_name = "sm_101f",
467	492	.features = featureSet(&[_]Feature{
468		.ptx88,
469	493	.sm_101f,
470	494	}),
471	495	};
...	...	@@ -473,7 +497,6 @@ pub const cpu = struct {
473	497	.name = "sm_103",
474	498	.llvm_name = "sm_103",
475	499	.features = featureSet(&[_]Feature{
476		.ptx88,
477	500	.sm_103,
478	501	}),
479	502	};
...	...	@@ -481,7 +504,6 @@ pub const cpu = struct {
481	504	.name = "sm_103a",
482	505	.llvm_name = "sm_103a",
483	506	.features = featureSet(&[_]Feature{
484		.ptx88,
485	507	.sm_103a,
486	508	}),
487	509	};
...	...	@@ -489,15 +511,34 @@ pub const cpu = struct {
489	511	.name = "sm_103f",
490	512	.llvm_name = "sm_103f",
491	513	.features = featureSet(&[_]Feature{
492		.ptx88,
493	514	.sm_103f,
494	515	}),
495	516	};
	517	pub const sm_110: CpuModel = .{
	518	.name = "sm_110",
	519	.llvm_name = "sm_110",
	520	.features = featureSet(&[_]Feature{
	521	.sm_110,
	522	}),
	523	};
	524	pub const sm_110a: CpuModel = .{
	525	.name = "sm_110a",
	526	.llvm_name = "sm_110a",
	527	.features = featureSet(&[_]Feature{
	528	.sm_110a,
	529	}),
	530	};
	531	pub const sm_110f: CpuModel = .{
	532	.name = "sm_110f",
	533	.llvm_name = "sm_110f",
	534	.features = featureSet(&[_]Feature{
	535	.sm_110f,
	536	}),
	537	};
496	538	pub const sm_120: CpuModel = .{
497	539	.name = "sm_120",
498	540	.llvm_name = "sm_120",
499	541	.features = featureSet(&[_]Feature{
500		.ptx87,
501	542	.sm_120,
502	543	}),
503	544	};
...	...	@@ -505,7 +546,6 @@ pub const cpu = struct {
505	546	.name = "sm_120a",
506	547	.llvm_name = "sm_120a",
507	548	.features = featureSet(&[_]Feature{
508		.ptx87,
509	549	.sm_120a,
510	550	}),
511	551	};
...	...	@@ -513,7 +553,6 @@ pub const cpu = struct {
513	553	.name = "sm_120f",
514	554	.llvm_name = "sm_120f",
515	555	.features = featureSet(&[_]Feature{
516		.ptx88,
517	556	.sm_120f,
518	557	}),
519	558	};
...	...	@@ -521,7 +560,6 @@ pub const cpu = struct {
521	560	.name = "sm_121",
522	561	.llvm_name = "sm_121",
523	562	.features = featureSet(&[_]Feature{
524		.ptx88,
525	563	.sm_121,
526	564	}),
527	565	};
...	...	@@ -529,7 +567,6 @@ pub const cpu = struct {
529	567	.name = "sm_121a",
530	568	.llvm_name = "sm_121a",
531	569	.features = featureSet(&[_]Feature{
532		.ptx88,
533	570	.sm_121a,
534	571	}),
535	572	};
...	...	@@ -537,7 +574,6 @@ pub const cpu = struct {
537	574	.name = "sm_121f",
538	575	.llvm_name = "sm_121f",
539	576	.features = featureSet(&[_]Feature{
540		.ptx88,
541	577	.sm_121f,
542	578	}),
543	579	};
...	...	@@ -545,7 +581,6 @@ pub const cpu = struct {
545	581	.name = "sm_20",
546	582	.llvm_name = "sm_20",
547	583	.features = featureSet(&[_]Feature{
548		.ptx32,
549	584	.sm_20,
550	585	}),
551	586	};
...	...	@@ -553,7 +588,6 @@ pub const cpu = struct {
553	588	.name = "sm_21",
554	589	.llvm_name = "sm_21",
555	590	.features = featureSet(&[_]Feature{
556		.ptx32,
557	591	.sm_21,
558	592	}),
559	593	};
...	...	@@ -568,7 +602,6 @@ pub const cpu = struct {
568	602	.name = "sm_32",
569	603	.llvm_name = "sm_32",
570	604	.features = featureSet(&[_]Feature{
571		.ptx40,
572	605	.sm_32,
573	606	}),
574	607	};
...	...	@@ -576,7 +609,6 @@ pub const cpu = struct {
576	609	.name = "sm_35",
577	610	.llvm_name = "sm_35",
578	611	.features = featureSet(&[_]Feature{
579		.ptx32,
580	612	.sm_35,
581	613	}),
582	614	};
...	...	@@ -584,7 +616,6 @@ pub const cpu = struct {
584	616	.name = "sm_37",
585	617	.llvm_name = "sm_37",
586	618	.features = featureSet(&[_]Feature{
587		.ptx41,
588	619	.sm_37,
589	620	}),
590	621	};
...	...	@@ -592,7 +623,6 @@ pub const cpu = struct {
592	623	.name = "sm_50",
593	624	.llvm_name = "sm_50",
594	625	.features = featureSet(&[_]Feature{
595		.ptx40,
596	626	.sm_50,
597	627	}),
598	628	};
...	...	@@ -600,7 +630,6 @@ pub const cpu = struct {
600	630	.name = "sm_52",
601	631	.llvm_name = "sm_52",
602	632	.features = featureSet(&[_]Feature{
603		.ptx41,
604	633	.sm_52,
605	634	}),
606	635	};
...	...	@@ -608,7 +637,6 @@ pub const cpu = struct {
608	637	.name = "sm_53",
609	638	.llvm_name = "sm_53",
610	639	.features = featureSet(&[_]Feature{
611		.ptx42,
612	640	.sm_53,
613	641	}),
614	642	};
...	...	@@ -616,7 +644,6 @@ pub const cpu = struct {
616	644	.name = "sm_60",
617	645	.llvm_name = "sm_60",
618	646	.features = featureSet(&[_]Feature{
619		.ptx50,
620	647	.sm_60,
621	648	}),
622	649	};
...	...	@@ -624,7 +651,6 @@ pub const cpu = struct {
624	651	.name = "sm_61",
625	652	.llvm_name = "sm_61",
626	653	.features = featureSet(&[_]Feature{
627		.ptx50,
628	654	.sm_61,
629	655	}),
630	656	};
...	...	@@ -632,7 +658,6 @@ pub const cpu = struct {
632	658	.name = "sm_62",
633	659	.llvm_name = "sm_62",
634	660	.features = featureSet(&[_]Feature{
635		.ptx50,
636	661	.sm_62,
637	662	}),
638	663	};
...	...	@@ -640,7 +665,6 @@ pub const cpu = struct {
640	665	.name = "sm_70",
641	666	.llvm_name = "sm_70",
642	667	.features = featureSet(&[_]Feature{
643		.ptx60,
644	668	.sm_70,
645	669	}),
646	670	};
...	...	@@ -648,7 +672,6 @@ pub const cpu = struct {
648	672	.name = "sm_72",
649	673	.llvm_name = "sm_72",
650	674	.features = featureSet(&[_]Feature{
651		.ptx61,
652	675	.sm_72,
653	676	}),
654	677	};
...	...	@@ -656,7 +679,6 @@ pub const cpu = struct {
656	679	.name = "sm_75",
657	680	.llvm_name = "sm_75",
658	681	.features = featureSet(&[_]Feature{
659		.ptx63,
660	682	.sm_75,
661	683	}),
662	684	};
...	...	@@ -664,7 +686,6 @@ pub const cpu = struct {
664	686	.name = "sm_80",
665	687	.llvm_name = "sm_80",
666	688	.features = featureSet(&[_]Feature{
667		.ptx70,
668	689	.sm_80,
669	690	}),
670	691	};
...	...	@@ -672,7 +693,6 @@ pub const cpu = struct {
672	693	.name = "sm_86",
673	694	.llvm_name = "sm_86",
674	695	.features = featureSet(&[_]Feature{
675		.ptx71,
676	696	.sm_86,
677	697	}),
678	698	};
...	...	@@ -680,15 +700,20 @@ pub const cpu = struct {
680	700	.name = "sm_87",
681	701	.llvm_name = "sm_87",
682	702	.features = featureSet(&[_]Feature{
683		.ptx74,
684	703	.sm_87,
685	704	}),
686	705	};
	706	pub const sm_88: CpuModel = .{
	707	.name = "sm_88",
	708	.llvm_name = "sm_88",
	709	.features = featureSet(&[_]Feature{
	710	.sm_88,
	711	}),
	712	};
687	713	pub const sm_89: CpuModel = .{
688	714	.name = "sm_89",
689	715	.llvm_name = "sm_89",
690	716	.features = featureSet(&[_]Feature{
691		.ptx78,
692	717	.sm_89,
693	718	}),
694	719	};
...	...	@@ -696,7 +721,6 @@ pub const cpu = struct {
696	721	.name = "sm_90",
697	722	.llvm_name = "sm_90",
698	723	.features = featureSet(&[_]Feature{
699		.ptx78,
700	724	.sm_90,
701	725	}),
702	726	};
...	...	@@ -704,7 +728,6 @@ pub const cpu = struct {
704	728	.name = "sm_90a",
705	729	.llvm_name = "sm_90a",
706	730	.features = featureSet(&[_]Feature{
707		.ptx80,
708	731	.sm_90a,
709	732	}),
710	733	};

lib/std/Target/powerpc.zig+27-19

...	...	@@ -6,6 +6,7 @@ const CpuModel = std.Target.Cpu.Model;
6	6
7	7	pub const Feature = enum {
8	8	@"64bit",
	9	@"64bit_support",
9	10	@"64bitregs",
10	11	allow_unaligned_fp_access,
11	12	altivec,
...	...	@@ -97,7 +98,14 @@ pub const all_features = blk: {
97	98	var result: [len]CpuFeature = undefined;
98	99	result[@intFromEnum(Feature.@"64bit")] = .{
99	100	.llvm_name = "64bit",
100		.description = "Enable 64-bit instructions",
	101	.description = "Enable 64-bit mode",
	102	.dependencies = featureSet(&[_]Feature{
	103	.@"64bit_support",
	104	}),
	105	};
	106	result[@intFromEnum(Feature.@"64bit_support")] = .{
	107	.llvm_name = "64bit-support",
	108	.description = "Supports 64-bit instructions",
101	109	.dependencies = featureSet(&[_]Feature{}),
102	110	};
103	111	result[@intFromEnum(Feature.@"64bitregs")] = .{
...	...	@@ -705,7 +713,7 @@ pub const cpu = struct {
705	713	.name = "970",
706	714	.llvm_name = "970",
707	715	.features = featureSet(&[_]Feature{
708		.@"64bit",
	716	.@"64bit_support",
709	717	.altivec,
710	718	.fres,
711	719	.frsqrte,
...	...	@@ -718,7 +726,7 @@ pub const cpu = struct {
718	726	.name = "a2",
719	727	.llvm_name = "a2",
720	728	.features = featureSet(&[_]Feature{
721		.@"64bit",
	729	.@"64bit_support",
722	730	.booke,
723	731	.cmpb,
724	732	.fcpsgn,
...	...	@@ -761,7 +769,7 @@ pub const cpu = struct {
761	769	.name = "e5500",
762	770	.llvm_name = "e5500",
763	771	.features = featureSet(&[_]Feature{
764		.@"64bit",
	772	.@"64bit_support",
765	773	.booke,
766	774	.isel,
767	775	.mfocrf,
...	...	@@ -772,7 +780,7 @@ pub const cpu = struct {
772	780	.name = "future",
773	781	.llvm_name = "future",
774	782	.features = featureSet(&[_]Feature{
775		.@"64bit",
	783	.@"64bit_support",
776	784	.allow_unaligned_fp_access,
777	785	.bpermd,
778	786	.cmpb,
...	...	@@ -846,7 +854,7 @@ pub const cpu = struct {
846	854	.name = "g5",
847	855	.llvm_name = "g5",
848	856	.features = featureSet(&[_]Feature{
849		.@"64bit",
	857	.@"64bit_support",
850	858	.altivec,
851	859	.fres,
852	860	.frsqrte,
...	...	@@ -873,7 +881,7 @@ pub const cpu = struct {
873	881	.name = "ppc64",
874	882	.llvm_name = "ppc64",
875	883	.features = featureSet(&[_]Feature{
876		.@"64bit",
	884	.@"64bit_support",
877	885	.altivec,
878	886	.fres,
879	887	.frsqrte,
...	...	@@ -886,7 +894,7 @@ pub const cpu = struct {
886	894	.name = "ppc64le",
887	895	.llvm_name = "ppc64le",
888	896	.features = featureSet(&[_]Feature{
889		.@"64bit",
	897	.@"64bit_support",
890	898	.allow_unaligned_fp_access,
891	899	.bpermd,
892	900	.cmpb,
...	...	@@ -926,7 +934,7 @@ pub const cpu = struct {
926	934	.name = "pwr10",
927	935	.llvm_name = "pwr10",
928	936	.features = featureSet(&[_]Feature{
929		.@"64bit",
	937	.@"64bit_support",
930	938	.allow_unaligned_fp_access,
931	939	.bpermd,
932	940	.cmpb,
...	...	@@ -973,7 +981,7 @@ pub const cpu = struct {
973	981	.name = "pwr11",
974	982	.llvm_name = "pwr11",
975	983	.features = featureSet(&[_]Feature{
976		.@"64bit",
	984	.@"64bit_support",
977	985	.allow_unaligned_fp_access,
978	986	.bpermd,
979	987	.cmpb,
...	...	@@ -1020,7 +1028,7 @@ pub const cpu = struct {
1020	1028	.name = "pwr3",
1021	1029	.llvm_name = "pwr3",
1022	1030	.features = featureSet(&[_]Feature{
1023		.@"64bit",
	1031	.@"64bit_support",
1024	1032	.altivec,
1025	1033	.fres,
1026	1034	.frsqrte,
...	...	@@ -1032,7 +1040,7 @@ pub const cpu = struct {
1032	1040	.name = "pwr4",
1033	1041	.llvm_name = "pwr4",
1034	1042	.features = featureSet(&[_]Feature{
1035		.@"64bit",
	1043	.@"64bit_support",
1036	1044	.altivec,
1037	1045	.fres,
1038	1046	.frsqrte,
...	...	@@ -1045,7 +1053,7 @@ pub const cpu = struct {
1045	1053	.name = "pwr5",
1046	1054	.llvm_name = "pwr5",
1047	1055	.features = featureSet(&[_]Feature{
1048		.@"64bit",
	1056	.@"64bit_support",
1049	1057	.altivec,
1050	1058	.fre,
1051	1059	.fres,
...	...	@@ -1060,7 +1068,7 @@ pub const cpu = struct {
1060	1068	.name = "pwr5x",
1061	1069	.llvm_name = "pwr5x",
1062	1070	.features = featureSet(&[_]Feature{
1063		.@"64bit",
	1071	.@"64bit_support",
1064	1072	.altivec,
1065	1073	.fprnd,
1066	1074	.fre,
...	...	@@ -1076,7 +1084,7 @@ pub const cpu = struct {
1076	1084	.name = "pwr6",
1077	1085	.llvm_name = "pwr6",
1078	1086	.features = featureSet(&[_]Feature{
1079		.@"64bit",
	1087	.@"64bit_support",
1080	1088	.altivec,
1081	1089	.cmpb,
1082	1090	.fcpsgn,
...	...	@@ -1096,7 +1104,7 @@ pub const cpu = struct {
1096	1104	.name = "pwr6x",
1097	1105	.llvm_name = "pwr6x",
1098	1106	.features = featureSet(&[_]Feature{
1099		.@"64bit",
	1107	.@"64bit_support",
1100	1108	.altivec,
1101	1109	.cmpb,
1102	1110	.fcpsgn,
...	...	@@ -1116,7 +1124,7 @@ pub const cpu = struct {
1116	1124	.name = "pwr7",
1117	1125	.llvm_name = "pwr7",
1118	1126	.features = featureSet(&[_]Feature{
1119		.@"64bit",
	1127	.@"64bit_support",
1120	1128	.allow_unaligned_fp_access,
1121	1129	.bpermd,
1122	1130	.cmpb,
...	...	@@ -1145,7 +1153,7 @@ pub const cpu = struct {
1145	1153	.name = "pwr8",
1146	1154	.llvm_name = "pwr8",
1147	1155	.features = featureSet(&[_]Feature{
1148		.@"64bit",
	1156	.@"64bit_support",
1149	1157	.allow_unaligned_fp_access,
1150	1158	.bpermd,
1151	1159	.cmpb,
...	...	@@ -1185,7 +1193,7 @@ pub const cpu = struct {
1185	1193	.name = "pwr9",
1186	1194	.llvm_name = "pwr9",
1187	1195	.features = featureSet(&[_]Feature{
1188		.@"64bit",
	1196	.@"64bit_support",
1189	1197	.allow_unaligned_fp_access,
1190	1198	.bpermd,
1191	1199	.cmpb,

lib/std/Target/riscv.zig+415-177

...	...	@@ -8,49 +8,40 @@ pub const Feature = enum {
8	8	@"32bit",
9	9	@"64bit",
10	10	a,
	11	add_load_fusion,
	12	addi_load_fusion,
11	13	andes45,
12	14	auipc_addi_fusion,
	15	auipc_load_fusion,
13	16	b,
	17	bfext_fusion,
14	18	c,
15	19	conditional_cmv_fusion,
16	20	d,
17	21	disable_latency_sched_heuristic,
	22	disable_misched_load_clustering,
	23	disable_misched_store_clustering,
	24	disable_postmisched_load_clustering,
	25	disable_postmisched_store_clustering,
18	26	dlen_factor_2,
19	27	e,
	28	enable_vsetvli_sched_heuristic,
20	29	exact_asm,
21	30	experimental,
22	31	experimental_p,
23	32	experimental_rvm23u32,
24		experimental_smctr,
25		experimental_ssctr,
	33	experimental_smpmpmt,
26	34	experimental_svukte,
27		experimental_xqccmp,
28		experimental_xqcia,
29		experimental_xqciac,
30		experimental_xqcibi,
31		experimental_xqcibm,
32		experimental_xqcicli,
33		experimental_xqcicm,
34		experimental_xqcics,
35		experimental_xqcicsr,
36		experimental_xqciint,
37		experimental_xqciio,
38		experimental_xqcilb,
39		experimental_xqcili,
40		experimental_xqcilia,
41		experimental_xqcilo,
42		experimental_xqcilsm,
43		experimental_xqcisim,
44		experimental_xqcisls,
45		experimental_xqcisync,
46	35	experimental_xrivosvisni,
47	36	experimental_xrivosvizip,
48	37	experimental_xsfmclic,
49	38	experimental_xsfsclic,
50		experimental_zalasr,
	39	experimental_zibi,
51	40	experimental_zicfilp,
52	41	experimental_zicfiss,
53	42	experimental_zvbc32e,
	43	experimental_zvfbfa,
	44	experimental_zvfofp8min,
54	45	experimental_zvkgs,
55	46	experimental_zvqdotq,
56	47	f,
...	...	@@ -60,6 +51,7 @@ pub const Feature = enum {
60	51	ld_add_fusion,
61	52	log_vrgather,
62	53	lui_addi_fusion,
	54	lui_load_fusion,
63	55	m,
64	56	mips_p8700,
65	57	no_default_unroll,
...	...	@@ -73,6 +65,7 @@ pub const Feature = enum {
73	65	optimized_nf7_segment_load_store,
74	66	optimized_nf8_segment_load_store,
75	67	optimized_zero_stride_load,
	68	permissive_zalrsc,
76	69	predictable_select_expensive,
77	70	prefer_vsetvli_over_read_vlenb,
78	71	prefer_w_inst,
...	...	@@ -127,15 +120,21 @@ pub const Feature = enum {
127	120	shgatpa,
128	121	shifted_zextw_fusion,
129	122	shlcofideleg,
130		short_forward_branch_opt,
	123	short_forward_branch_ialu,
	124	short_forward_branch_iload,
	125	short_forward_branch_iminmax,
	126	short_forward_branch_imul,
131	127	shtvala,
132	128	shvsatpa,
133	129	shvstvala,
134	130	shvstvecd,
	131	shxadd_load_fusion,
	132	single_element_vec_fp64,
135	133	smaia,
136	134	smcdeleg,
137	135	smcntrpmf,
138	136	smcsrind,
	137	smctr,
139	138	smdbltrp,
140	139	smepmp,
141	140	smmpm,
...	...	@@ -148,6 +147,7 @@ pub const Feature = enum {
148	147	sscofpmf,
149	148	sscounterenw,
150	149	sscsrind,
	150	ssctr,
151	151	ssdbltrp,
152	152	ssnpm,
153	153	sspm,
...	...	@@ -179,6 +179,7 @@ pub const Feature = enum {
179	179	xandesvbfhcvt,
180	180	xandesvdot,
181	181	xandesvpackfph,
	182	xandesvsinth,
182	183	xandesvsintload,
183	184	xcvalu,
184	185	xcvbi,
...	...	@@ -189,7 +190,28 @@ pub const Feature = enum {
189	190	xcvsimd,
190	191	xmipscbop,
191	192	xmipscmov,
	193	xmipsexectl,
192	194	xmipslsp,
	195	xqccmp,
	196	xqci,
	197	xqcia,
	198	xqciac,
	199	xqcibi,
	200	xqcibm,
	201	xqcicli,
	202	xqcicm,
	203	xqcics,
	204	xqcicsr,
	205	xqciint,
	206	xqciio,
	207	xqcilb,
	208	xqcili,
	209	xqcilia,
	210	xqcilo,
	211	xqcilsm,
	212	xqcisim,
	213	xqcisls,
	214	xqcisync,
193	215	xsfcease,
194	216	xsfmm128t,
195	217	xsfmm16t,
...	...	@@ -202,12 +224,18 @@ pub const Feature = enum {
202	224	xsfmm64t,
203	225	xsfmmbase,
204	226	xsfvcp,
	227	xsfvfbfexp16e,
	228	xsfvfexp16e,
	229	xsfvfexp32e,
	230	xsfvfexpa,
	231	xsfvfexpa64e,
205	232	xsfvfnrclipxfqf,
206	233	xsfvfwmaccqqq,
207	234	xsfvqmaccdod,
208	235	xsfvqmaccqoq,
209	236	xsifivecdiscarddlone,
210	237	xsifivecflushdlone,
	238	xsmtvdot,
211	239	xtheadba,
212	240	xtheadbb,
213	241	xtheadbs,
...	...	@@ -226,6 +254,7 @@ pub const Feature = enum {
226	254	zaamo,
227	255	zabha,
228	256	zacas,
	257	zalasr,
229	258	zalrsc,
230	259	zama16b,
231	260	zawrs,
...	...	@@ -272,6 +301,7 @@ pub const Feature = enum {
272	301	zihintpause,
273	302	zihpm,
274	303	zilsd,
	304	zilsd_4byte_align,
275	305	zimop,
276	306	zk,
277	307	zkn,
...	...	@@ -352,6 +382,16 @@ pub const all_features = blk: {
352	382	.zalrsc,
353	383	}),
354	384	};
	385	result[@intFromEnum(Feature.add_load_fusion)] = .{
	386	.llvm_name = "add-load-fusion",
	387	.description = "Enable ADD(.UW) + load macrofusion",
	388	.dependencies = featureSet(&[_]Feature{}),
	389	};
	390	result[@intFromEnum(Feature.addi_load_fusion)] = .{
	391	.llvm_name = "addi-load-fusion",
	392	.description = "Enable ADDI + load macrofusion",
	393	.dependencies = featureSet(&[_]Feature{}),
	394	};
355	395	result[@intFromEnum(Feature.andes45)] = .{
356	396	.llvm_name = "andes45",
357	397	.description = "Andes 45-Series processors",
...	...	@@ -362,6 +402,11 @@ pub const all_features = blk: {
362	402	.description = "Enable AUIPC+ADDI macrofusion",
363	403	.dependencies = featureSet(&[_]Feature{}),
364	404	};
	405	result[@intFromEnum(Feature.auipc_load_fusion)] = .{
	406	.llvm_name = "auipc-load-fusion",
	407	.description = "Enable AUIPC + load macrofusion",
	408	.dependencies = featureSet(&[_]Feature{}),
	409	};
365	410	result[@intFromEnum(Feature.b)] = .{
366	411	.llvm_name = "b",
367	412	.description = "'B' (the collection of the Zba, Zbb, Zbs extensions)",
...	...	@@ -371,6 +416,11 @@ pub const all_features = blk: {
371	416	.zbs,
372	417	}),
373	418	};
	419	result[@intFromEnum(Feature.bfext_fusion)] = .{
	420	.llvm_name = "bfext-fusion",
	421	.description = "Enable SLLI+SRLI (bitfield extract) macrofusion",
	422	.dependencies = featureSet(&[_]Feature{}),
	423	};
374	424	result[@intFromEnum(Feature.c)] = .{
375	425	.llvm_name = "c",
376	426	.description = "'C' (Compressed Instructions)",
...	...	@@ -395,6 +445,26 @@ pub const all_features = blk: {
395	445	.description = "Disable latency scheduling heuristic",
396	446	.dependencies = featureSet(&[_]Feature{}),
397	447	};
	448	result[@intFromEnum(Feature.disable_misched_load_clustering)] = .{
	449	.llvm_name = "disable-misched-load-clustering",
	450	.description = "Disable load clustering in the machine scheduler",
	451	.dependencies = featureSet(&[_]Feature{}),
	452	};
	453	result[@intFromEnum(Feature.disable_misched_store_clustering)] = .{
	454	.llvm_name = "disable-misched-store-clustering",
	455	.description = "Disable store clustering in the machine scheduler",
	456	.dependencies = featureSet(&[_]Feature{}),
	457	};
	458	result[@intFromEnum(Feature.disable_postmisched_load_clustering)] = .{
	459	.llvm_name = "disable-postmisched-load-clustering",
	460	.description = "Disable PostRA load clustering in the machine scheduler",
	461	.dependencies = featureSet(&[_]Feature{}),
	462	};
	463	result[@intFromEnum(Feature.disable_postmisched_store_clustering)] = .{
	464	.llvm_name = "disable-postmisched-store-clustering",
	465	.description = "Disable PostRA store clustering in the machine scheduler",
	466	.dependencies = featureSet(&[_]Feature{}),
	467	};
398	468	result[@intFromEnum(Feature.dlen_factor_2)] = .{
399	469	.llvm_name = "dlen-factor-2",
400	470	.description = "Vector unit DLEN(data path width) is half of VLEN",
...	...	@@ -405,6 +475,11 @@ pub const all_features = blk: {
405	475	.description = "'E' (Embedded Instruction Set with 16 GPRs)",
406	476	.dependencies = featureSet(&[_]Feature{}),
407	477	};
	478	result[@intFromEnum(Feature.enable_vsetvli_sched_heuristic)] = .{
	479	.llvm_name = "enable-vsetvli-sched-heuristic",
	480	.description = "Enable vsetvli-based scheduling heuristic",
	481	.dependencies = featureSet(&[_]Feature{}),
	482	};
408	483	result[@intFromEnum(Feature.exact_asm)] = .{
409	484	.llvm_name = "exact-asm",
410	485	.description = "Enable Exact Assembly (Disables Compression and Relaxation)",
...	...	@@ -437,144 +512,16 @@ pub const all_features = blk: {
437	512	.zimop,
438	513	}),
439	514	};
440		result[@intFromEnum(Feature.experimental_smctr)] = .{
441		.llvm_name = "experimental-smctr",
442		.description = "'Smctr' (Control Transfer Records Machine Level)",
443		.dependencies = featureSet(&[_]Feature{
444		.sscsrind,
445		}),
446		};
447		result[@intFromEnum(Feature.experimental_ssctr)] = .{
448		.llvm_name = "experimental-ssctr",
449		.description = "'Ssctr' (Control Transfer Records Supervisor Level)",
450		.dependencies = featureSet(&[_]Feature{
451		.sscsrind,
452		}),
	515	result[@intFromEnum(Feature.experimental_smpmpmt)] = .{
	516	.llvm_name = "experimental-smpmpmt",
	517	.description = "'Smpmpmt' (PMP-based Memory Types Extension)",
	518	.dependencies = featureSet(&[_]Feature{}),
453	519	};
454	520	result[@intFromEnum(Feature.experimental_svukte)] = .{
455	521	.llvm_name = "experimental-svukte",
456	522	.description = "'Svukte' (Address-Independent Latency of User-Mode Faults to Supervisor Addresses)",
457	523	.dependencies = featureSet(&[_]Feature{}),
458	524	};
459		result[@intFromEnum(Feature.experimental_xqccmp)] = .{
460		.llvm_name = "experimental-xqccmp",
461		.description = "'Xqccmp' (Qualcomm 16-bit Push/Pop and Double Moves)",
462		.dependencies = featureSet(&[_]Feature{
463		.zca,
464		}),
465		};
466		result[@intFromEnum(Feature.experimental_xqcia)] = .{
467		.llvm_name = "experimental-xqcia",
468		.description = "'Xqcia' (Qualcomm uC Arithmetic Extension)",
469		.dependencies = featureSet(&[_]Feature{}),
470		};
471		result[@intFromEnum(Feature.experimental_xqciac)] = .{
472		.llvm_name = "experimental-xqciac",
473		.description = "'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)",
474		.dependencies = featureSet(&[_]Feature{
475		.zca,
476		}),
477		};
478		result[@intFromEnum(Feature.experimental_xqcibi)] = .{
479		.llvm_name = "experimental-xqcibi",
480		.description = "'Xqcibi' (Qualcomm uC Branch Immediate Extension)",
481		.dependencies = featureSet(&[_]Feature{
482		.zca,
483		}),
484		};
485		result[@intFromEnum(Feature.experimental_xqcibm)] = .{
486		.llvm_name = "experimental-xqcibm",
487		.description = "'Xqcibm' (Qualcomm uC Bit Manipulation Extension)",
488		.dependencies = featureSet(&[_]Feature{
489		.zca,
490		}),
491		};
492		result[@intFromEnum(Feature.experimental_xqcicli)] = .{
493		.llvm_name = "experimental-xqcicli",
494		.description = "'Xqcicli' (Qualcomm uC Conditional Load Immediate Extension)",
495		.dependencies = featureSet(&[_]Feature{}),
496		};
497		result[@intFromEnum(Feature.experimental_xqcicm)] = .{
498		.llvm_name = "experimental-xqcicm",
499		.description = "'Xqcicm' (Qualcomm uC Conditional Move Extension)",
500		.dependencies = featureSet(&[_]Feature{
501		.zca,
502		}),
503		};
504		result[@intFromEnum(Feature.experimental_xqcics)] = .{
505		.llvm_name = "experimental-xqcics",
506		.description = "'Xqcics' (Qualcomm uC Conditional Select Extension)",
507		.dependencies = featureSet(&[_]Feature{}),
508		};
509		result[@intFromEnum(Feature.experimental_xqcicsr)] = .{
510		.llvm_name = "experimental-xqcicsr",
511		.description = "'Xqcicsr' (Qualcomm uC CSR Extension)",
512		.dependencies = featureSet(&[_]Feature{}),
513		};
514		result[@intFromEnum(Feature.experimental_xqciint)] = .{
515		.llvm_name = "experimental-xqciint",
516		.description = "'Xqciint' (Qualcomm uC Interrupts Extension)",
517		.dependencies = featureSet(&[_]Feature{
518		.zca,
519		}),
520		};
521		result[@intFromEnum(Feature.experimental_xqciio)] = .{
522		.llvm_name = "experimental-xqciio",
523		.description = "'Xqciio' (Qualcomm uC External Input Output Extension)",
524		.dependencies = featureSet(&[_]Feature{}),
525		};
526		result[@intFromEnum(Feature.experimental_xqcilb)] = .{
527		.llvm_name = "experimental-xqcilb",
528		.description = "'Xqcilb' (Qualcomm uC Long Branch Extension)",
529		.dependencies = featureSet(&[_]Feature{
530		.zca,
531		}),
532		};
533		result[@intFromEnum(Feature.experimental_xqcili)] = .{
534		.llvm_name = "experimental-xqcili",
535		.description = "'Xqcili' (Qualcomm uC Load Large Immediate Extension)",
536		.dependencies = featureSet(&[_]Feature{
537		.zca,
538		}),
539		};
540		result[@intFromEnum(Feature.experimental_xqcilia)] = .{
541		.llvm_name = "experimental-xqcilia",
542		.description = "'Xqcilia' (Qualcomm uC Large Immediate Arithmetic Extension)",
543		.dependencies = featureSet(&[_]Feature{
544		.zca,
545		}),
546		};
547		result[@intFromEnum(Feature.experimental_xqcilo)] = .{
548		.llvm_name = "experimental-xqcilo",
549		.description = "'Xqcilo' (Qualcomm uC Large Offset Load Store Extension)",
550		.dependencies = featureSet(&[_]Feature{
551		.zca,
552		}),
553		};
554		result[@intFromEnum(Feature.experimental_xqcilsm)] = .{
555		.llvm_name = "experimental-xqcilsm",
556		.description = "'Xqcilsm' (Qualcomm uC Load Store Multiple Extension)",
557		.dependencies = featureSet(&[_]Feature{}),
558		};
559		result[@intFromEnum(Feature.experimental_xqcisim)] = .{
560		.llvm_name = "experimental-xqcisim",
561		.description = "'Xqcisim' (Qualcomm uC Simulation Hint Extension)",
562		.dependencies = featureSet(&[_]Feature{
563		.zca,
564		}),
565		};
566		result[@intFromEnum(Feature.experimental_xqcisls)] = .{
567		.llvm_name = "experimental-xqcisls",
568		.description = "'Xqcisls' (Qualcomm uC Scaled Load Store Extension)",
569		.dependencies = featureSet(&[_]Feature{}),
570		};
571		result[@intFromEnum(Feature.experimental_xqcisync)] = .{
572		.llvm_name = "experimental-xqcisync",
573		.description = "'Xqcisync' (Qualcomm uC Sync Delay Extension)",
574		.dependencies = featureSet(&[_]Feature{
575		.zca,
576		}),
577		};
578	525	result[@intFromEnum(Feature.experimental_xrivosvisni)] = .{
579	526	.llvm_name = "experimental-xrivosvisni",
580	527	.description = "'XRivosVisni' (Rivos Vector Integer Small New)",
...	...	@@ -595,9 +542,9 @@ pub const all_features = blk: {
595	542	.description = "'XSfsclic' (SiFive CLIC Supervisor-mode CSRs)",
596	543	.dependencies = featureSet(&[_]Feature{}),
597	544	};
598		result[@intFromEnum(Feature.experimental_zalasr)] = .{
599		.llvm_name = "experimental-zalasr",
600		.description = "'Zalasr' (Load-Acquire and Store-Release Instructions)",
	545	result[@intFromEnum(Feature.experimental_zibi)] = .{
	546	.llvm_name = "experimental-zibi",
	547	.description = "'Zibi' (Branch with Immediate)",
601	548	.dependencies = featureSet(&[_]Feature{}),
602	549	};
603	550	result[@intFromEnum(Feature.experimental_zicfilp)] = .{
...	...	@@ -622,6 +569,21 @@ pub const all_features = blk: {
622	569	.zve32x,
623	570	}),
624	571	};
	572	result[@intFromEnum(Feature.experimental_zvfbfa)] = .{
	573	.llvm_name = "experimental-zvfbfa",
	574	.description = "'Zvfbfa' (Additional BF16 vector compute support)",
	575	.dependencies = featureSet(&[_]Feature{
	576	.zfbfmin,
	577	.zve32f,
	578	}),
	579	};
	580	result[@intFromEnum(Feature.experimental_zvfofp8min)] = .{
	581	.llvm_name = "experimental-zvfofp8min",
	582	.description = "'Zvfofp8min' (Vector OFP8 Converts)",
	583	.dependencies = featureSet(&[_]Feature{
	584	.zve32f,
	585	}),
	586	};
625	587	result[@intFromEnum(Feature.experimental_zvkgs)] = .{
626	588	.llvm_name = "experimental-zvkgs",
627	589	.description = "'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)",
...	...	@@ -673,6 +635,11 @@ pub const all_features = blk: {
673	635	.description = "Enable LUI+ADDI macro fusion",
674	636	.dependencies = featureSet(&[_]Feature{}),
675	637	};
	638	result[@intFromEnum(Feature.lui_load_fusion)] = .{
	639	.llvm_name = "lui-load-fusion",
	640	.description = "Enable LUI + load macrofusion",
	641	.dependencies = featureSet(&[_]Feature{}),
	642	};
676	643	result[@intFromEnum(Feature.m)] = .{
677	644	.llvm_name = "m",
678	645	.description = "'M' (Integer Multiplication and Division)",
...	...	@@ -740,6 +707,11 @@ pub const all_features = blk: {
740	707	.description = "Optimized (perform fewer memory operations)zero-stride vector load",
741	708	.dependencies = featureSet(&[_]Feature{}),
742	709	};
	710	result[@intFromEnum(Feature.permissive_zalrsc)] = .{
	711	.llvm_name = "permissive-zalrsc",
	712	.description = "Implementation permits non-base instructions between LR/SC pairs",
	713	.dependencies = featureSet(&[_]Feature{}),
	714	};
743	715	result[@intFromEnum(Feature.predictable_select_expensive)] = .{
744	716	.llvm_name = "predictable-select-expensive",
745	717	.description = "Prefer likely predicted branches over selects",
...	...	@@ -1262,11 +1234,32 @@ pub const all_features = blk: {
1262	1234	.description = "'Shlcofideleg' (Delegating LCOFI Interrupts to VS-mode)",
1263	1235	.dependencies = featureSet(&[_]Feature{}),
1264	1236	};
1265		result[@intFromEnum(Feature.short_forward_branch_opt)] = .{
1266		.llvm_name = "short-forward-branch-opt",
1267		.description = "Enable short forward branch optimization",
	1237	result[@intFromEnum(Feature.short_forward_branch_ialu)] = .{
	1238	.llvm_name = "short-forward-branch-ialu",
	1239	.description = "Enable short forward branch optimization for RVI base instructions",
1268	1240	.dependencies = featureSet(&[_]Feature{}),
1269	1241	};
	1242	result[@intFromEnum(Feature.short_forward_branch_iload)] = .{
	1243	.llvm_name = "short-forward-branch-iload",
	1244	.description = "Enable short forward branch optimization for load instructions",
	1245	.dependencies = featureSet(&[_]Feature{
	1246	.short_forward_branch_ialu,
	1247	}),
	1248	};
	1249	result[@intFromEnum(Feature.short_forward_branch_iminmax)] = .{
	1250	.llvm_name = "short-forward-branch-iminmax",
	1251	.description = "Enable short forward branch optimization for MIN,MAX instructions in Zbb",
	1252	.dependencies = featureSet(&[_]Feature{
	1253	.short_forward_branch_ialu,
	1254	}),
	1255	};
	1256	result[@intFromEnum(Feature.short_forward_branch_imul)] = .{
	1257	.llvm_name = "short-forward-branch-imul",
	1258	.description = "Enable short forward branch optimization for MUL instruction",
	1259	.dependencies = featureSet(&[_]Feature{
	1260	.short_forward_branch_ialu,
	1261	}),
	1262	};
1270	1263	result[@intFromEnum(Feature.shtvala)] = .{
1271	1264	.llvm_name = "shtvala",
1272	1265	.description = "'Shtvala' (htval provides all needed values)",
...	...	@@ -1287,6 +1280,16 @@ pub const all_features = blk: {
1287	1280	.description = "'Shvstvecd' (vstvec supports Direct mode)",
1288	1281	.dependencies = featureSet(&[_]Feature{}),
1289	1282	};
	1283	result[@intFromEnum(Feature.shxadd_load_fusion)] = .{
	1284	.llvm_name = "shxadd-load-fusion",
	1285	.description = "Enable SH(1\|2\|3)ADD(.UW) + load macrofusion",
	1286	.dependencies = featureSet(&[_]Feature{}),
	1287	};
	1288	result[@intFromEnum(Feature.single_element_vec_fp64)] = .{
	1289	.llvm_name = "single-element-vec-fp64",
	1290	.description = "Certain vector FP64 operations produce a single result element per cycle",
	1291	.dependencies = featureSet(&[_]Feature{}),
	1292	};
1290	1293	result[@intFromEnum(Feature.smaia)] = .{
1291	1294	.llvm_name = "smaia",
1292	1295	.description = "'Smaia' (Advanced Interrupt Architecture Machine Level)",
...	...	@@ -1307,6 +1310,13 @@ pub const all_features = blk: {
1307	1310	.description = "'Smcsrind' (Indirect CSR Access Machine Level)",
1308	1311	.dependencies = featureSet(&[_]Feature{}),
1309	1312	};
	1313	result[@intFromEnum(Feature.smctr)] = .{
	1314	.llvm_name = "smctr",
	1315	.description = "'Smctr' (Control Transfer Records Machine Level)",
	1316	.dependencies = featureSet(&[_]Feature{
	1317	.sscsrind,
	1318	}),
	1319	};
1310	1320	result[@intFromEnum(Feature.smdbltrp)] = .{
1311	1321	.llvm_name = "smdbltrp",
1312	1322	.description = "'Smdbltrp' (Double Trap Machine Level)",
...	...	@@ -1369,6 +1379,13 @@ pub const all_features = blk: {
1369	1379	.description = "'Sscsrind' (Indirect CSR Access Supervisor Level)",
1370	1380	.dependencies = featureSet(&[_]Feature{}),
1371	1381	};
	1382	result[@intFromEnum(Feature.ssctr)] = .{
	1383	.llvm_name = "ssctr",
	1384	.description = "'Ssctr' (Control Transfer Records Supervisor Level)",
	1385	.dependencies = featureSet(&[_]Feature{
	1386	.sscsrind,
	1387	}),
	1388	};
1372	1389	result[@intFromEnum(Feature.ssdbltrp)] = .{
1373	1390	.llvm_name = "ssdbltrp",
1374	1391	.description = "'Ssdbltrp' (Double Trap Supervisor Level)",
...	...	@@ -1537,6 +1554,13 @@ pub const all_features = blk: {
1537	1554	.f,
1538	1555	}),
1539	1556	};
	1557	result[@intFromEnum(Feature.xandesvsinth)] = .{
	1558	.llvm_name = "xandesvsinth",
	1559	.description = "'XAndesVSIntH' (Andes Vector Small INT Handling Extension)",
	1560	.dependencies = featureSet(&[_]Feature{
	1561	.zve32x,
	1562	}),
	1563	};
1540	1564	result[@intFromEnum(Feature.xandesvsintload)] = .{
1541	1565	.llvm_name = "xandesvsintload",
1542	1566	.description = "'XAndesVSIntLoad' (Andes Vector INT4 Load Extension)",
...	...	@@ -1589,11 +1613,159 @@ pub const all_features = blk: {
1589	1613	.description = "'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov))",
1590	1614	.dependencies = featureSet(&[_]Feature{}),
1591	1615	};
	1616	result[@intFromEnum(Feature.xmipsexectl)] = .{
	1617	.llvm_name = "xmipsexectl",
	1618	.description = "'XMIPSEXECTL' (MIPS execution control)",
	1619	.dependencies = featureSet(&[_]Feature{}),
	1620	};
1592	1621	result[@intFromEnum(Feature.xmipslsp)] = .{
1593	1622	.llvm_name = "xmipslsp",
1594	1623	.description = "'XMIPSLSP' (MIPS optimization for hardware load-store bonding)",
1595	1624	.dependencies = featureSet(&[_]Feature{}),
1596	1625	};
	1626	result[@intFromEnum(Feature.xqccmp)] = .{
	1627	.llvm_name = "xqccmp",
	1628	.description = "'Xqccmp' (Qualcomm 16-bit Push/Pop and Double Moves)",
	1629	.dependencies = featureSet(&[_]Feature{
	1630	.zca,
	1631	}),
	1632	};
	1633	result[@intFromEnum(Feature.xqci)] = .{
	1634	.llvm_name = "xqci",
	1635	.description = "'Xqci' (Qualcomm uC Extension)",
	1636	.dependencies = featureSet(&[_]Feature{
	1637	.xqcia,
	1638	.xqciac,
	1639	.xqcibi,
	1640	.xqcibm,
	1641	.xqcicli,
	1642	.xqcicm,
	1643	.xqcics,
	1644	.xqcicsr,
	1645	.xqciint,
	1646	.xqciio,
	1647	.xqcilb,
	1648	.xqcili,
	1649	.xqcilia,
	1650	.xqcilo,
	1651	.xqcilsm,
	1652	.xqcisim,
	1653	.xqcisls,
	1654	.xqcisync,
	1655	}),
	1656	};
	1657	result[@intFromEnum(Feature.xqcia)] = .{
	1658	.llvm_name = "xqcia",
	1659	.description = "'Xqcia' (Qualcomm uC Arithmetic Extension)",
	1660	.dependencies = featureSet(&[_]Feature{}),
	1661	};
	1662	result[@intFromEnum(Feature.xqciac)] = .{
	1663	.llvm_name = "xqciac",
	1664	.description = "'Xqciac' (Qualcomm uC Load-Store Address Calculation Extension)",
	1665	.dependencies = featureSet(&[_]Feature{
	1666	.zca,
	1667	}),
	1668	};
	1669	result[@intFromEnum(Feature.xqcibi)] = .{
	1670	.llvm_name = "xqcibi",
	1671	.description = "'Xqcibi' (Qualcomm uC Branch Immediate Extension)",
	1672	.dependencies = featureSet(&[_]Feature{
	1673	.zca,
	1674	}),
	1675	};
	1676	result[@intFromEnum(Feature.xqcibm)] = .{
	1677	.llvm_name = "xqcibm",
	1678	.description = "'Xqcibm' (Qualcomm uC Bit Manipulation Extension)",
	1679	.dependencies = featureSet(&[_]Feature{
	1680	.zca,
	1681	}),
	1682	};
	1683	result[@intFromEnum(Feature.xqcicli)] = .{
	1684	.llvm_name = "xqcicli",
	1685	.description = "'Xqcicli' (Qualcomm uC Conditional Load Immediate Extension)",
	1686	.dependencies = featureSet(&[_]Feature{}),
	1687	};
	1688	result[@intFromEnum(Feature.xqcicm)] = .{
	1689	.llvm_name = "xqcicm",
	1690	.description = "'Xqcicm' (Qualcomm uC Conditional Move Extension)",
	1691	.dependencies = featureSet(&[_]Feature{
	1692	.zca,
	1693	}),
	1694	};
	1695	result[@intFromEnum(Feature.xqcics)] = .{
	1696	.llvm_name = "xqcics",
	1697	.description = "'Xqcics' (Qualcomm uC Conditional Select Extension)",
	1698	.dependencies = featureSet(&[_]Feature{}),
	1699	};
	1700	result[@intFromEnum(Feature.xqcicsr)] = .{
	1701	.llvm_name = "xqcicsr",
	1702	.description = "'Xqcicsr' (Qualcomm uC CSR Extension)",
	1703	.dependencies = featureSet(&[_]Feature{}),
	1704	};
	1705	result[@intFromEnum(Feature.xqciint)] = .{
	1706	.llvm_name = "xqciint",
	1707	.description = "'Xqciint' (Qualcomm uC Interrupts Extension)",
	1708	.dependencies = featureSet(&[_]Feature{
	1709	.zca,
	1710	}),
	1711	};
	1712	result[@intFromEnum(Feature.xqciio)] = .{
	1713	.llvm_name = "xqciio",
	1714	.description = "'Xqciio' (Qualcomm uC External Input Output Extension)",
	1715	.dependencies = featureSet(&[_]Feature{}),
	1716	};
	1717	result[@intFromEnum(Feature.xqcilb)] = .{
	1718	.llvm_name = "xqcilb",
	1719	.description = "'Xqcilb' (Qualcomm uC Long Branch Extension)",
	1720	.dependencies = featureSet(&[_]Feature{
	1721	.zca,
	1722	}),
	1723	};
	1724	result[@intFromEnum(Feature.xqcili)] = .{
	1725	.llvm_name = "xqcili",
	1726	.description = "'Xqcili' (Qualcomm uC Load Large Immediate Extension)",
	1727	.dependencies = featureSet(&[_]Feature{
	1728	.zca,
	1729	}),
	1730	};
	1731	result[@intFromEnum(Feature.xqcilia)] = .{
	1732	.llvm_name = "xqcilia",
	1733	.description = "'Xqcilia' (Qualcomm uC Large Immediate Arithmetic Extension)",
	1734	.dependencies = featureSet(&[_]Feature{
	1735	.zca,
	1736	}),
	1737	};
	1738	result[@intFromEnum(Feature.xqcilo)] = .{
	1739	.llvm_name = "xqcilo",
	1740	.description = "'Xqcilo' (Qualcomm uC Large Offset Load Store Extension)",
	1741	.dependencies = featureSet(&[_]Feature{
	1742	.zca,
	1743	}),
	1744	};
	1745	result[@intFromEnum(Feature.xqcilsm)] = .{
	1746	.llvm_name = "xqcilsm",
	1747	.description = "'Xqcilsm' (Qualcomm uC Load Store Multiple Extension)",
	1748	.dependencies = featureSet(&[_]Feature{}),
	1749	};
	1750	result[@intFromEnum(Feature.xqcisim)] = .{
	1751	.llvm_name = "xqcisim",
	1752	.description = "'Xqcisim' (Qualcomm uC Simulation Hint Extension)",
	1753	.dependencies = featureSet(&[_]Feature{
	1754	.zca,
	1755	}),
	1756	};
	1757	result[@intFromEnum(Feature.xqcisls)] = .{
	1758	.llvm_name = "xqcisls",
	1759	.description = "'Xqcisls' (Qualcomm uC Scaled Load Store Extension)",
	1760	.dependencies = featureSet(&[_]Feature{}),
	1761	};
	1762	result[@intFromEnum(Feature.xqcisync)] = .{
	1763	.llvm_name = "xqcisync",
	1764	.description = "'Xqcisync' (Qualcomm uC Sync Delay Extension)",
	1765	.dependencies = featureSet(&[_]Feature{
	1766	.zca,
	1767	}),
	1768	};
1597	1769	result[@intFromEnum(Feature.xsfcease)] = .{
1598	1770	.llvm_name = "xsfcease",
1599	1771	.description = "'XSfcease' (SiFive sf.cease Instruction)",
...	...	@@ -1684,6 +1856,40 @@ pub const all_features = blk: {
1684	1856	.zve32x,
1685	1857	}),
1686	1858	};
	1859	result[@intFromEnum(Feature.xsfvfbfexp16e)] = .{
	1860	.llvm_name = "xsfvfbfexp16e",
	1861	.description = "'XSfvfbfexp16e' (SiFive Vector Floating-Point Exponential Function Instruction, BFloat16)",
	1862	.dependencies = featureSet(&[_]Feature{}),
	1863	};
	1864	result[@intFromEnum(Feature.xsfvfexp16e)] = .{
	1865	.llvm_name = "xsfvfexp16e",
	1866	.description = "'XSfvfexp16e' (SiFive Vector Floating-Point Exponential Function Instruction, Half Precision)",
	1867	.dependencies = featureSet(&[_]Feature{
	1868	.zvfh,
	1869	}),
	1870	};
	1871	result[@intFromEnum(Feature.xsfvfexp32e)] = .{
	1872	.llvm_name = "xsfvfexp32e",
	1873	.description = "'XSfvfexp32e' (SiFive Vector Floating-Point Exponential Function Instruction, Single Precision)",
	1874	.dependencies = featureSet(&[_]Feature{
	1875	.zve32f,
	1876	}),
	1877	};
	1878	result[@intFromEnum(Feature.xsfvfexpa)] = .{
	1879	.llvm_name = "xsfvfexpa",
	1880	.description = "'XSfvfexpa' (SiFive Vector Floating-Point Exponential Approximation Instruction)",
	1881	.dependencies = featureSet(&[_]Feature{
	1882	.zve32f,
	1883	}),
	1884	};
	1885	result[@intFromEnum(Feature.xsfvfexpa64e)] = .{
	1886	.llvm_name = "xsfvfexpa64e",
	1887	.description = "'XSfvfexpa64e' (SiFive Vector Floating-Point Exponential Approximation Instruction with Double-Precision)",
	1888	.dependencies = featureSet(&[_]Feature{
	1889	.xsfvfexpa,
	1890	.zve64d,
	1891	}),
	1892	};
1687	1893	result[@intFromEnum(Feature.xsfvfnrclipxfqf)] = .{
1688	1894	.llvm_name = "xsfvfnrclipxfqf",
1689	1895	.description = "'XSfvfnrclipxfqf' (SiFive FP32-to-int8 Ranged Clip Instructions)",
...	...	@@ -1696,6 +1902,7 @@ pub const all_features = blk: {
1696	1902	.description = "'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4))",
1697	1903	.dependencies = featureSet(&[_]Feature{
1698	1904	.zvfbfmin,
	1905	.zvl128b,
1699	1906	}),
1700	1907	};
1701	1908	result[@intFromEnum(Feature.xsfvqmaccdod)] = .{
...	...	@@ -1703,6 +1910,7 @@ pub const all_features = blk: {
1703	1910	.description = "'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))",
1704	1911	.dependencies = featureSet(&[_]Feature{
1705	1912	.zve32x,
	1913	.zvl128b,
1706	1914	}),
1707	1915	};
1708	1916	result[@intFromEnum(Feature.xsfvqmaccqoq)] = .{
...	...	@@ -1710,6 +1918,7 @@ pub const all_features = blk: {
1710	1918	.description = "'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))",
1711	1919	.dependencies = featureSet(&[_]Feature{
1712	1920	.zve32x,
	1921	.zvl256b,
1713	1922	}),
1714	1923	};
1715	1924	result[@intFromEnum(Feature.xsifivecdiscarddlone)] = .{
...	...	@@ -1722,6 +1931,13 @@ pub const all_features = blk: {
1722	1931	.description = "'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction)",
1723	1932	.dependencies = featureSet(&[_]Feature{}),
1724	1933	};
	1934	result[@intFromEnum(Feature.xsmtvdot)] = .{
	1935	.llvm_name = "xsmtvdot",
	1936	.description = "'XSMTVDot' (SpacemiT Vector Dot Product Extension)",
	1937	.dependencies = featureSet(&[_]Feature{
	1938	.zve32f,
	1939	}),
	1940	};
1725	1941	result[@intFromEnum(Feature.xtheadba)] = .{
1726	1942	.llvm_name = "xtheadba",
1727	1943	.description = "'XTHeadBa' (T-Head address calculation instructions)",
...	...	@@ -1820,6 +2036,11 @@ pub const all_features = blk: {
1820	2036	.zaamo,
1821	2037	}),
1822	2038	};
	2039	result[@intFromEnum(Feature.zalasr)] = .{
	2040	.llvm_name = "zalasr",
	2041	.description = "'Zalasr' (Load-Acquire and Store-Release Instructions)",
	2042	.dependencies = featureSet(&[_]Feature{}),
	2043	};
1823	2044	result[@intFromEnum(Feature.zalrsc)] = .{
1824	2045	.llvm_name = "zalrsc",
1825	2046	.description = "'Zalrsc' (Load-Reserved/Store-Conditional)",
...	...	@@ -2092,6 +2313,11 @@ pub const all_features = blk: {
2092	2313	.description = "'Zilsd' (Load/Store Pair Instructions)",
2093	2314	.dependencies = featureSet(&[_]Feature{}),
2094	2315	};
	2316	result[@intFromEnum(Feature.zilsd_4byte_align)] = .{
	2317	.llvm_name = "zilsd-4byte-align",
	2318	.description = "Allow 4-byte alignment for Zilsd LD/SD instructions",
	2319	.dependencies = featureSet(&[_]Feature{}),
	2320	};
2095	2321	result[@intFromEnum(Feature.zimop)] = .{
2096	2322	.llvm_name = "zimop",
2097	2323	.description = "'Zimop' (May-Be-Operations)",
...	...	@@ -2461,7 +2687,7 @@ pub const cpu = struct {
2461	2687	.features = featureSet(&[_]Feature{
2462	2688	.andes45,
2463	2689	.no_default_unroll,
2464		.short_forward_branch_opt,
	2690	.short_forward_branch_ialu,
2465	2691	.use_postra_scheduler,
2466	2692	}),
2467	2693	};
...	...	@@ -2491,7 +2717,7 @@ pub const cpu = struct {
2491	2717	.i,
2492	2718	.m,
2493	2719	.no_default_unroll,
2494		.short_forward_branch_opt,
	2720	.short_forward_branch_ialu,
2495	2721	.use_postra_scheduler,
2496	2722	.xandesperf,
2497	2723	.zifencei,
...	...	@@ -2523,7 +2749,7 @@ pub const cpu = struct {
2523	2749	.i,
2524	2750	.m,
2525	2751	.no_default_unroll,
2526		.short_forward_branch_opt,
	2752	.short_forward_branch_ialu,
2527	2753	.use_postra_scheduler,
2528	2754	.xandesperf,
2529	2755	.zifencei,
...	...	@@ -2540,7 +2766,7 @@ pub const cpu = struct {
2540	2766	.i,
2541	2767	.m,
2542	2768	.no_default_unroll,
2543		.short_forward_branch_opt,
	2769	.short_forward_branch_ialu,
2544	2770	.use_postra_scheduler,
2545	2771	.v,
2546	2772	.xandesperf,
...	...	@@ -2559,7 +2785,7 @@ pub const cpu = struct {
2559	2785	.i,
2560	2786	.m,
2561	2787	.no_default_unroll,
2562		.short_forward_branch_opt,
	2788	.short_forward_branch_ialu,
2563	2789	.use_postra_scheduler,
2564	2790	.xandesperf,
2565	2791	.zifencei,
...	...	@@ -2577,7 +2803,7 @@ pub const cpu = struct {
2577	2803	.i,
2578	2804	.m,
2579	2805	.no_default_unroll,
2580		.short_forward_branch_opt,
	2806	.short_forward_branch_ialu,
2581	2807	.use_postra_scheduler,
2582	2808	.xandesperf,
2583	2809	.zifencei,
...	...	@@ -2648,6 +2874,7 @@ pub const cpu = struct {
2648	2874	.mips_p8700,
2649	2875	.xmipscbop,
2650	2876	.xmipscmov,
	2877	.xmipsexectl,
2651	2878	.xmipslsp,
2652	2879	.zba,
2653	2880	.zbb,
...	...	@@ -2703,7 +2930,7 @@ pub const cpu = struct {
2703	2930	.llvm_name = "sifive-7-series",
2704	2931	.features = featureSet(&[_]Feature{
2705	2932	.no_default_unroll,
2706		.short_forward_branch_opt,
	2933	.short_forward_branch_ialu,
2707	2934	.use_postra_scheduler,
2708	2935	}),
2709	2936	};
...	...	@@ -2782,7 +3009,7 @@ pub const cpu = struct {
2782	3009	.i,
2783	3010	.m,
2784	3011	.no_default_unroll,
2785		.short_forward_branch_opt,
	3012	.short_forward_branch_ialu,
2786	3013	.use_postra_scheduler,
2787	3014	.zifencei,
2788	3015	}),
...	...	@@ -2815,7 +3042,6 @@ pub const cpu = struct {
2815	3042	.ziccif,
2816	3043	.zicclsm,
2817	3044	.ziccrse,
2818		.zicntr,
2819	3045	.zifencei,
2820	3046	.zihintntl,
2821	3047	.zihintpause,
...	...	@@ -2855,7 +3081,6 @@ pub const cpu = struct {
2855	3081	.ziccif,
2856	3082	.zicclsm,
2857	3083	.ziccrse,
2858		.zicntr,
2859	3084	.zifencei,
2860	3085	.zihintntl,
2861	3086	.zihintpause,
...	...	@@ -2918,7 +3143,6 @@ pub const cpu = struct {
2918	3143	.ziccif,
2919	3144	.zicclsm,
2920	3145	.ziccrse,
2921		.zicntr,
2922	3146	.zifencei,
2923	3147	.zihintntl,
2924	3148	.zihintpause,
...	...	@@ -3035,7 +3259,7 @@ pub const cpu = struct {
3035	3259	.i,
3036	3260	.m,
3037	3261	.no_default_unroll,
3038		.short_forward_branch_opt,
	3262	.short_forward_branch_ialu,
3039	3263	.use_postra_scheduler,
3040	3264	.zifencei,
3041	3265	.zihintpause,
...	...	@@ -3065,7 +3289,7 @@ pub const cpu = struct {
3065	3289	.i,
3066	3290	.m,
3067	3291	.no_default_unroll,
3068		.short_forward_branch_opt,
	3292	.short_forward_branch_ialu,
3069	3293	.use_postra_scheduler,
3070	3294	.zifencei,
3071	3295	}),
...	...	@@ -3083,7 +3307,7 @@ pub const cpu = struct {
3083	3307	.no_default_unroll,
3084	3308	.optimized_nf2_segment_load_store,
3085	3309	.optimized_zero_stride_load,
3086		.short_forward_branch_opt,
	3310	.short_forward_branch_ialu,
3087	3311	.use_postra_scheduler,
3088	3312	.v,
3089	3313	.vl_dependent_latency,
...	...	@@ -3111,7 +3335,8 @@ pub const cpu = struct {
3111	3335	.no_default_unroll,
3112	3336	.optimized_nf2_segment_load_store,
3113	3337	.optimized_zero_stride_load,
3114		.short_forward_branch_opt,
	3338	.short_forward_branch_ialu,
	3339	.single_element_vec_fp64,
3115	3340	.use_postra_scheduler,
3116	3341	.v,
3117	3342	.vl_dependent_latency,
...	...	@@ -3173,6 +3398,7 @@ pub const cpu = struct {
3173	3398	.unaligned_scalar_mem,
3174	3399	.v,
3175	3400	.vxrm_pipeline_flush,
	3401	.xsmtvdot,
3176	3402	.za64rs,
3177	3403	.zbc,
3178	3404	.zbkc,
...	...	@@ -3341,6 +3567,13 @@ pub const cpu = struct {
3341	3567	.log_vrgather,
3342	3568	.m,
3343	3569	.no_default_unroll,
	3570	.optimized_nf2_segment_load_store,
	3571	.optimized_nf3_segment_load_store,
	3572	.optimized_nf4_segment_load_store,
	3573	.optimized_nf5_segment_load_store,
	3574	.optimized_nf6_segment_load_store,
	3575	.optimized_nf7_segment_load_store,
	3576	.optimized_nf8_segment_load_store,
3344	3577	.optimized_zero_stride_load,
3345	3578	.sha,
3346	3579	.smaia,
...	...	@@ -3400,12 +3633,17 @@ pub const cpu = struct {
3400	3633	.features = featureSet(&[_]Feature{
3401	3634	.@"64bit",
3402	3635	.a,
	3636	.add_load_fusion,
3403	3637	.auipc_addi_fusion,
	3638	.auipc_load_fusion,
3404	3639	.c,
3405	3640	.d,
	3641	.disable_misched_load_clustering,
	3642	.disable_postmisched_load_clustering,
	3643	.disable_postmisched_store_clustering,
3406	3644	.i,
3407		.ld_add_fusion,
3408	3645	.lui_addi_fusion,
	3646	.lui_load_fusion,
3409	3647	.m,
3410	3648	.shifted_zextw_fusion,
3411	3649	.ventana_veyron,

lib/std/Target/sparc.zig+17

...	...	@@ -5,6 +5,7 @@ const CpuFeature = std.Target.Cpu.Feature;
5	5	const CpuModel = std.Target.Cpu.Model;
6	6
7	7	pub const Feature = enum {
	8	@"64bit",
8	9	crypto,
9	10	deprecated_v8,
10	11	detectroundchange,
...	...	@@ -23,6 +24,7 @@ pub const Feature = enum {
23	24	leonpwrpsr,
24	25	no_fmuls,
25	26	no_fsmuld,
	27	no_predictor,
26	28	osa2011,
27	29	popc,
28	30	reserve_g1,
...	...	@@ -73,6 +75,13 @@ pub const all_features = blk: {
73	75	const len = @typeInfo(Feature).@"enum".fields.len;
74	76	std.debug.assert(len <= CpuFeature.Set.needed_bit_count);
75	77	var result: [len]CpuFeature = undefined;
	78	result[@intFromEnum(Feature.@"64bit")] = .{
	79	.llvm_name = "64bit",
	80	.description = "Enable 64-bit mode",
	81	.dependencies = featureSet(&[_]Feature{
	82	.v9,
	83	}),
	84	};
76	85	result[@intFromEnum(Feature.crypto)] = .{
77	86	.llvm_name = "crypto",
78	87	.description = "Enable cryptographic extensions",
...	...	@@ -165,6 +174,11 @@ pub const all_features = blk: {
165	174	.description = "Disable the fsmuld instruction.",
166	175	.dependencies = featureSet(&[_]Feature{}),
167	176	};
	177	result[@intFromEnum(Feature.no_predictor)] = .{
	178	.llvm_name = "no-predictor",
	179	.description = "Processor has no branch predictor, branches stall execution",
	180	.dependencies = featureSet(&[_]Feature{}),
	181	};
168	182	result[@intFromEnum(Feature.osa2011)] = .{
169	183	.llvm_name = "osa2011",
170	184	.description = "Enable Oracle SPARC Architecture 2011 extensions",
...	...	@@ -586,6 +600,7 @@ pub const cpu = struct {
586	600	.llvm_name = "niagara",
587	601	.features = featureSet(&[_]Feature{
588	602	.deprecated_v8,
	603	.no_predictor,
589	604	.ua2005,
590	605	}),
591	606	};
...	...	@@ -594,6 +609,7 @@ pub const cpu = struct {
594	609	.llvm_name = "niagara2",
595	610	.features = featureSet(&[_]Feature{
596	611	.deprecated_v8,
	612	.no_predictor,
597	613	.popc,
598	614	.ua2005,
599	615	}),
...	...	@@ -603,6 +619,7 @@ pub const cpu = struct {
603	619	.llvm_name = "niagara3",
604	620	.features = featureSet(&[_]Feature{
605	621	.deprecated_v8,
	622	.no_predictor,
606	623	.popc,
607	624	.ua2005,
608	625	.ua2007,

lib/std/Target/wasm.zig+7

...	...	@@ -12,6 +12,7 @@ pub const Feature = enum {
12	12	exception_handling,
13	13	extended_const,
14	14	fp16,
	15	gc,
15	16	multimemory,
16	17	multivalue,
17	18	mutable_globals,
...	...	@@ -71,6 +72,11 @@ pub const all_features = blk: {
71	72	.description = "Enable FP16 instructions",
72	73	.dependencies = featureSet(&[_]Feature{}),
73	74	};
	75	result[@intFromEnum(Feature.gc)] = .{
	76	.llvm_name = "gc",
	77	.description = "Enable wasm gc",
	78	.dependencies = featureSet(&[_]Feature{}),
	79	};
74	80	result[@intFromEnum(Feature.multimemory)] = .{
75	81	.llvm_name = "multimemory",
76	82	.description = "Enable multiple memories",
...	...	@@ -148,6 +154,7 @@ pub const cpu = struct {
148	154	.exception_handling,
149	155	.extended_const,
150	156	.fp16,
	157	.gc,
151	158	.multimemory,
152	159	.multivalue,
153	160	.mutable_globals,

lib/std/Target/x86.zig+200-31

...	...	@@ -22,7 +22,6 @@ pub const Feature = enum {
22	22	amx_movrs,
23	23	amx_tf32,
24	24	amx_tile,
25		amx_transpose,
26	25	avx,
27	26	avx10_1,
28	27	avx10_2,
...	...	@@ -67,7 +66,6 @@ pub const Feature = enum {
67	66	egpr,
68	67	enqcmd,
69	68	ermsb,
70		evex512,
71	69	f16c,
72	70	false_deps_getmant,
73	71	false_deps_lzcnt_tzcnt,
...	...	@@ -136,6 +134,7 @@ pub const Feature = enum {
136	134	ppx,
137	135	prefer_128_bit,
138	136	prefer_256_bit,
	137	prefer_legacy_setcc,
139	138	prefer_mask_registers,
140	139	prefer_movmsk_over_vtest,
141	140	prefer_no_gather,
...	...	@@ -168,6 +167,7 @@ pub const Feature = enum {
168	167	slow_lea,
169	168	slow_pmaddwd,
170	169	slow_pmulld,
	170	slow_pmullq,
171	171	slow_shld,
172	172	slow_two_mem_ops,
173	173	slow_unaligned_mem_16,
...	...	@@ -199,6 +199,7 @@ pub const Feature = enum {
199	199	waitpkg,
200	200	wbnoinvd,
201	201	widekl,
	202	x32,
202	203	x87,
203	204	xop,
204	205	xsave,
...	...	@@ -324,13 +325,6 @@ pub const all_features = blk: {
324	325	.description = "Support AMX-TILE instructions",
325	326	.dependencies = featureSet(&[_]Feature{}),
326	327	};
327		result[@intFromEnum(Feature.amx_transpose)] = .{
328		.llvm_name = "amx-transpose",
329		.description = "Support AMX amx-transpose instructions",
330		.dependencies = featureSet(&[_]Feature{
331		.amx_tile,
332		}),
333		};
334	328	result[@intFromEnum(Feature.avx)] = .{
335	329	.llvm_name = "avx",
336	330	.description = "Enable AVX instructions",
...	...	@@ -339,8 +333,8 @@ pub const all_features = blk: {
339	333	}),
340	334	};
341	335	result[@intFromEnum(Feature.avx10_1)] = .{
342		.llvm_name = "avx10.1-512",
343		.description = "Support AVX10.1 up to 512-bit instruction",
	336	.llvm_name = "avx10.1",
	337	.description = "Support AVX10.1 instruction",
344	338	.dependencies = featureSet(&[_]Feature{
345	339	.avx512bf16,
346	340	.avx512bitalg,
...	...	@@ -356,8 +350,8 @@ pub const all_features = blk: {
356	350	}),
357	351	};
358	352	result[@intFromEnum(Feature.avx10_2)] = .{
359		.llvm_name = "avx10.2-512",
360		.description = "Support AVX10.2 up to 512-bit instruction",
	353	.llvm_name = "avx10.2",
	354	.description = "Support AVX10.2 instruction",
361	355	.dependencies = featureSet(&[_]Feature{
362	356	.avx10_1,
363	357	}),
...	...	@@ -416,7 +410,6 @@ pub const all_features = blk: {
416	410	.description = "Enable AVX-512 instructions",
417	411	.dependencies = featureSet(&[_]Feature{
418	412	.avx2,
419		.evex512,
420	413	.f16c,
421	414	.fma,
422	415	}),
...	...	@@ -616,11 +609,6 @@ pub const all_features = blk: {
616	609	.description = "REP MOVS/STOS are fast",
617	610	.dependencies = featureSet(&[_]Feature{}),
618	611	};
619		result[@intFromEnum(Feature.evex512)] = .{
620		.llvm_name = "evex512",
621		.description = "Support ZMM and 64-bit mask instructions",
622		.dependencies = featureSet(&[_]Feature{}),
623		};
624	612	result[@intFromEnum(Feature.f16c)] = .{
625	613	.llvm_name = "f16c",
626	614	.description = "Support 16-bit floating point conversion instructions",
...	...	@@ -974,6 +962,11 @@ pub const all_features = blk: {
974	962	.description = "Prefer 256-bit AVX instructions",
975	963	.dependencies = featureSet(&[_]Feature{}),
976	964	};
	965	result[@intFromEnum(Feature.prefer_legacy_setcc)] = .{
	966	.llvm_name = "prefer-legacy-setcc",
	967	.description = "Prefer to emit legacy SetCC.",
	968	.dependencies = featureSet(&[_]Feature{}),
	969	};
977	970	result[@intFromEnum(Feature.prefer_mask_registers)] = .{
978	971	.llvm_name = "prefer-mask-registers",
979	972	.description = "Prefer AVX512 mask registers over PTEST/MOVMSK",
...	...	@@ -1145,6 +1138,11 @@ pub const all_features = blk: {
1145	1138	.description = "PMULLD instruction is slow (compared to PMULLW/PMULHW and PMULUDQ)",
1146	1139	.dependencies = featureSet(&[_]Feature{}),
1147	1140	};
	1141	result[@intFromEnum(Feature.slow_pmullq)] = .{
	1142	.llvm_name = "slow-pmullq",
	1143	.description = "PMULLQ instruction is slow",
	1144	.dependencies = featureSet(&[_]Feature{}),
	1145	};
1148	1146	result[@intFromEnum(Feature.slow_shld)] = .{
1149	1147	.llvm_name = "slow-shld",
1150	1148	.description = "SHLD instruction is slow",
...	...	@@ -1325,6 +1323,11 @@ pub const all_features = blk: {
1325	1323	.kl,
1326	1324	}),
1327	1325	};
	1326	result[@intFromEnum(Feature.x32)] = .{
	1327	.llvm_name = "x32",
	1328	.description = "64-bit with ILP32 programming model (e.g. x32 ABI)",
	1329	.dependencies = featureSet(&[_]Feature{}),
	1330	};
1328	1331	result[@intFromEnum(Feature.x87)] = .{
1329	1332	.llvm_name = "x87",
1330	1333	.description = "Enable X87 float instructions",
...	...	@@ -1393,7 +1396,6 @@ pub const cpu = struct {
1393	1396	.cx16,
1394	1397	.f16c,
1395	1398	.false_deps_perm,
1396		.false_deps_popcnt,
1397	1399	.fast_15bytenop,
1398	1400	.fast_gather,
1399	1401	.fast_scalar_fsqrt,
...	...	@@ -1432,6 +1434,7 @@ pub const cpu = struct {
1432	1434	.sha,
1433	1435	.shstk,
1434	1436	.slow_3ops_lea,
	1437	.slow_pmullq,
1435	1438	.smap,
1436	1439	.smep,
1437	1440	.tuning_fast_imm_vector_shift,
...	...	@@ -1490,7 +1493,6 @@ pub const cpu = struct {
1490	1493	.enqcmd,
1491	1494	.f16c,
1492	1495	.false_deps_perm,
1493		.false_deps_popcnt,
1494	1496	.fast_15bytenop,
1495	1497	.fast_gather,
1496	1498	.fast_scalar_fsqrt,
...	...	@@ -1529,6 +1531,7 @@ pub const cpu = struct {
1529	1531	.sha,
1530	1532	.shstk,
1531	1533	.slow_3ops_lea,
	1534	.slow_pmullq,
1532	1535	.smap,
1533	1536	.smep,
1534	1537	.tuning_fast_imm_vector_shift,
...	...	@@ -1566,7 +1569,6 @@ pub const cpu = struct {
1566	1569	.enqcmd,
1567	1570	.f16c,
1568	1571	.false_deps_perm,
1569		.false_deps_popcnt,
1570	1572	.fast_15bytenop,
1571	1573	.fast_gather,
1572	1574	.fast_scalar_fsqrt,
...	...	@@ -1606,6 +1608,7 @@ pub const cpu = struct {
1606	1608	.sha512,
1607	1609	.shstk,
1608	1610	.slow_3ops_lea,
	1611	.slow_pmullq,
1609	1612	.sm3,
1610	1613	.sm4,
1611	1614	.smap,
...	...	@@ -2204,6 +2207,7 @@ pub const cpu = struct {
2204	2207	.sahf,
2205	2208	.sha,
2206	2209	.slow_3ops_lea,
	2210	.slow_pmullq,
2207	2211	.smap,
2208	2212	.smep,
2209	2213	.tuning_fast_imm_vector_shift,
...	...	@@ -2297,7 +2301,6 @@ pub const cpu = struct {
2297	2301	.enqcmd,
2298	2302	.f16c,
2299	2303	.false_deps_perm,
2300		.false_deps_popcnt,
2301	2304	.fast_15bytenop,
2302	2305	.fast_gather,
2303	2306	.fast_scalar_fsqrt,
...	...	@@ -2338,6 +2341,7 @@ pub const cpu = struct {
2338	2341	.sha512,
2339	2342	.shstk,
2340	2343	.slow_3ops_lea,
	2344	.slow_pmullq,
2341	2345	.sm3,
2342	2346	.sm4,
2343	2347	.tuning_fast_imm_vector_shift,
...	...	@@ -2464,7 +2468,6 @@ pub const cpu = struct {
2464	2468	.amx_int8,
2465	2469	.amx_movrs,
2466	2470	.amx_tf32,
2467		.amx_transpose,
2468	2471	.avx10_2,
2469	2472	.avxifma,
2470	2473	.avxneconvert,
...	...	@@ -2475,7 +2478,6 @@ pub const cpu = struct {
2475	2478	.bmi2,
2476	2479	.branch_hint,
2477	2480	.ccmp,
2478		.cf,
2479	2481	.cldemote,
2480	2482	.clflushopt,
2481	2483	.clwb,
...	...	@@ -2533,12 +2535,12 @@ pub const cpu = struct {
2533	2535	.sha,
2534	2536	.sha512,
2535	2537	.shstk,
	2538	.slow_pmullq,
2536	2539	.sm3,
2537	2540	.sm4,
2538	2541	.tsxldtrk,
2539	2542	.tuning_fast_imm_vector_shift,
2540	2543	.uintr,
2541		.usermsr,
2542	2544	.vaes,
2543	2545	.vpclmulqdq,
2544	2546	.vzeroupper,
...	...	@@ -2622,6 +2624,7 @@ pub const cpu = struct {
2622	2624	.serialize,
2623	2625	.sha,
2624	2626	.shstk,
	2627	.slow_pmullq,
2625	2628	.smap,
2626	2629	.smep,
2627	2630	.tsxldtrk,
...	...	@@ -2935,6 +2938,7 @@ pub const cpu = struct {
2935	2938	.serialize,
2936	2939	.sha,
2937	2940	.shstk,
	2941	.slow_pmullq,
2938	2942	.tsxldtrk,
2939	2943	.tuning_fast_imm_vector_shift,
2940	2944	.uintr,
...	...	@@ -3024,6 +3028,7 @@ pub const cpu = struct {
3024	3028	.serialize,
3025	3029	.sha,
3026	3030	.shstk,
	3031	.slow_pmullq,
3027	3032	.tsxldtrk,
3028	3033	.tuning_fast_imm_vector_shift,
3029	3034	.uintr,
...	...	@@ -3181,6 +3186,7 @@ pub const cpu = struct {
3181	3186	.rdseed,
3182	3187	.sahf,
3183	3188	.sha,
	3189	.slow_pmullq,
3184	3190	.tuning_fast_imm_vector_shift,
3185	3191	.vaes,
3186	3192	.vpclmulqdq,
...	...	@@ -3245,6 +3251,7 @@ pub const cpu = struct {
3245	3251	.rdseed,
3246	3252	.sahf,
3247	3253	.sha,
	3254	.slow_pmullq,
3248	3255	.tuning_fast_imm_vector_shift,
3249	3256	.vaes,
3250	3257	.vpclmulqdq,
...	...	@@ -3475,7 +3482,6 @@ pub const cpu = struct {
3475	3482	.enqcmd,
3476	3483	.f16c,
3477	3484	.false_deps_perm,
3478		.false_deps_popcnt,
3479	3485	.fast_15bytenop,
3480	3486	.fast_gather,
3481	3487	.fast_scalar_fsqrt,
...	...	@@ -3515,6 +3521,7 @@ pub const cpu = struct {
3515	3521	.sha512,
3516	3522	.shstk,
3517	3523	.slow_3ops_lea,
	3524	.slow_pmullq,
3518	3525	.sm3,
3519	3526	.sm4,
3520	3527	.tuning_fast_imm_vector_shift,
...	...	@@ -3546,7 +3553,6 @@ pub const cpu = struct {
3546	3553	.cx16,
3547	3554	.f16c,
3548	3555	.false_deps_perm,
3549		.false_deps_popcnt,
3550	3556	.fast_15bytenop,
3551	3557	.fast_gather,
3552	3558	.fast_scalar_fsqrt,
...	...	@@ -3585,6 +3591,7 @@ pub const cpu = struct {
3585	3591	.sha,
3586	3592	.shstk,
3587	3593	.slow_3ops_lea,
	3594	.slow_pmullq,
3588	3595	.smap,
3589	3596	.smep,
3590	3597	.tuning_fast_imm_vector_shift,
...	...	@@ -3635,6 +3642,90 @@ pub const cpu = struct {
3635	3642	.x87,
3636	3643	}),
3637	3644	};
	3645	pub const novalake: CpuModel = .{
	3646	.name = "novalake",
	3647	.llvm_name = "novalake",
	3648	.features = featureSet(&[_]Feature{
	3649	.@"64bit",
	3650	.adx,
	3651	.allow_light_256_bit,
	3652	.avx10_2,
	3653	.avxifma,
	3654	.avxneconvert,
	3655	.avxvnni,
	3656	.avxvnniint16,
	3657	.avxvnniint8,
	3658	.bmi,
	3659	.bmi2,
	3660	.ccmp,
	3661	.clflushopt,
	3662	.clwb,
	3663	.cmov,
	3664	.cmpccxadd,
	3665	.cx16,
	3666	.egpr,
	3667	.enqcmd,
	3668	.false_deps_perm,
	3669	.fast_15bytenop,
	3670	.fast_gather,
	3671	.fast_scalar_fsqrt,
	3672	.fast_shld_rotate,
	3673	.fast_variable_crosslane_shuffle,
	3674	.fast_variable_perlane_shuffle,
	3675	.fast_vector_fsqrt,
	3676	.fsgsbase,
	3677	.fxsr,
	3678	.gfni,
	3679	.hreset,
	3680	.idivq_to_divl,
	3681	.invpcid,
	3682	.lzcnt,
	3683	.macrofusion,
	3684	.mmx,
	3685	.movbe,
	3686	.movdir64b,
	3687	.movdiri,
	3688	.movrs,
	3689	.ndd,
	3690	.nf,
	3691	.no_bypass_delay_blend,
	3692	.no_bypass_delay_mov,
	3693	.no_bypass_delay_shuffle,
	3694	.nopl,
	3695	.pconfig,
	3696	.pku,
	3697	.popcnt,
	3698	.ppx,
	3699	.prefer_movmsk_over_vtest,
	3700	.prefetchi,
	3701	.prfchw,
	3702	.ptwrite,
	3703	.push2pop2,
	3704	.rdpid,
	3705	.rdrnd,
	3706	.rdseed,
	3707	.sahf,
	3708	.serialize,
	3709	.sha,
	3710	.sha512,
	3711	.shstk,
	3712	.slow_3ops_lea,
	3713	.slow_pmullq,
	3714	.sm3,
	3715	.sm4,
	3716	.tuning_fast_imm_vector_shift,
	3717	.uintr,
	3718	.vaes,
	3719	.vpclmulqdq,
	3720	.vzeroupper,
	3721	.waitpkg,
	3722	.x87,
	3723	.xsavec,
	3724	.xsaveopt,
	3725	.xsaves,
	3726	.zu,
	3727	}),
	3728	};
3638	3729	pub const opteron: CpuModel = .{
3639	3730	.name = "opteron",
3640	3731	.llvm_name = "opteron",
...	...	@@ -3697,7 +3788,6 @@ pub const cpu = struct {
3697	3788	.enqcmd,
3698	3789	.f16c,
3699	3790	.false_deps_perm,
3700		.false_deps_popcnt,
3701	3791	.fast_15bytenop,
3702	3792	.fast_gather,
3703	3793	.fast_scalar_fsqrt,
...	...	@@ -3726,7 +3816,6 @@ pub const cpu = struct {
3726	3816	.pku,
3727	3817	.popcnt,
3728	3818	.prefer_movmsk_over_vtest,
3729		.prefetchi,
3730	3819	.prfchw,
3731	3820	.ptwrite,
3732	3821	.rdpid,
...	...	@@ -3738,6 +3827,7 @@ pub const cpu = struct {
3738	3827	.sha512,
3739	3828	.shstk,
3740	3829	.slow_3ops_lea,
	3830	.slow_pmullq,
3741	3831	.sm3,
3742	3832	.sm4,
3743	3833	.tuning_fast_imm_vector_shift,
...	...	@@ -3908,7 +3998,6 @@ pub const cpu = struct {
3908	3998	.cx16,
3909	3999	.f16c,
3910	4000	.false_deps_perm,
3911		.false_deps_popcnt,
3912	4001	.fast_15bytenop,
3913	4002	.fast_gather,
3914	4003	.fast_scalar_fsqrt,
...	...	@@ -3947,6 +4036,7 @@ pub const cpu = struct {
3947	4036	.sha,
3948	4037	.shstk,
3949	4038	.slow_3ops_lea,
	4039	.slow_pmullq,
3950	4040	.smap,
3951	4041	.smep,
3952	4042	.tuning_fast_imm_vector_shift,
...	...	@@ -4013,6 +4103,7 @@ pub const cpu = struct {
4013	4103	.rdseed,
4014	4104	.sahf,
4015	4105	.sha,
	4106	.slow_pmullq,
4016	4107	.smap,
4017	4108	.smep,
4018	4109	.tuning_fast_imm_vector_shift,
...	...	@@ -4124,6 +4215,7 @@ pub const cpu = struct {
4124	4215	.serialize,
4125	4216	.sha,
4126	4217	.shstk,
	4218	.slow_pmullq,
4127	4219	.smap,
4128	4220	.smep,
4129	4221	.tsxldtrk,
...	...	@@ -4494,6 +4586,7 @@ pub const cpu = struct {
4494	4586	.sahf,
4495	4587	.sha,
4496	4588	.shstk,
	4589	.slow_pmullq,
4497	4590	.smap,
4498	4591	.smep,
4499	4592	.tuning_fast_imm_vector_shift,
...	...	@@ -4567,6 +4660,82 @@ pub const cpu = struct {
4567	4660	.x87,
4568	4661	}),
4569	4662	};
	4663	pub const wildcatlake: CpuModel = .{
	4664	.name = "wildcatlake",
	4665	.llvm_name = "wildcatlake",
	4666	.features = featureSet(&[_]Feature{
	4667	.@"64bit",
	4668	.adx,
	4669	.allow_light_256_bit,
	4670	.avxifma,
	4671	.avxneconvert,
	4672	.avxvnni,
	4673	.avxvnniint16,
	4674	.avxvnniint8,
	4675	.bmi,
	4676	.bmi2,
	4677	.clflushopt,
	4678	.clwb,
	4679	.cmov,
	4680	.cmpccxadd,
	4681	.cx16,
	4682	.enqcmd,
	4683	.f16c,
	4684	.false_deps_perm,
	4685	.fast_15bytenop,
	4686	.fast_gather,
	4687	.fast_scalar_fsqrt,
	4688	.fast_shld_rotate,
	4689	.fast_variable_crosslane_shuffle,
	4690	.fast_variable_perlane_shuffle,
	4691	.fast_vector_fsqrt,
	4692	.fma,
	4693	.fsgsbase,
	4694	.fxsr,
	4695	.gfni,
	4696	.hreset,
	4697	.idivq_to_divl,
	4698	.invpcid,
	4699	.lzcnt,
	4700	.macrofusion,
	4701	.mmx,
	4702	.movbe,
	4703	.movdir64b,
	4704	.movdiri,
	4705	.no_bypass_delay_blend,
	4706	.no_bypass_delay_mov,
	4707	.no_bypass_delay_shuffle,
	4708	.nopl,
	4709	.pconfig,
	4710	.pku,
	4711	.popcnt,
	4712	.prefer_movmsk_over_vtest,
	4713	.prfchw,
	4714	.ptwrite,
	4715	.rdpid,
	4716	.rdrnd,
	4717	.rdseed,
	4718	.sahf,
	4719	.serialize,
	4720	.sha,
	4721	.sha512,
	4722	.shstk,
	4723	.slow_3ops_lea,
	4724	.slow_pmullq,
	4725	.sm3,
	4726	.sm4,
	4727	.tuning_fast_imm_vector_shift,
	4728	.uintr,
	4729	.vaes,
	4730	.vpclmulqdq,
	4731	.vzeroupper,
	4732	.waitpkg,
	4733	.x87,
	4734	.xsavec,
	4735	.xsaveopt,
	4736	.xsaves,
	4737	}),
	4738	};
4570	4739	pub const winchip2: CpuModel = .{
4571	4740	.name = "winchip2",
4572	4741	.llvm_name = "winchip2",

lib/std/Target/xtensa.zig+65

...	...	@@ -15,6 +15,7 @@ pub const Feature = enum {
15	15	div32,
16	16	exception,
17	17	extendedl32r,
	18	forced_atomics,
18	19	fp,
19	20	highpriinterrupts,
20	21	highpriinterrupts_level3,
...	...	@@ -34,6 +35,7 @@ pub const Feature = enum {
34	35	prid,
35	36	regprotect,
36	37	rvector,
	38	s32c1i,
37	39	sext,
38	40	threadptr,
39	41	timers1,
...	...	@@ -101,6 +103,11 @@ pub const all_features = blk: {
101	103	.description = "Enable Xtensa Extended L32R option",
102	104	.dependencies = featureSet(&[_]Feature{}),
103	105	};
	106	result[@intFromEnum(Feature.forced_atomics)] = .{
	107	.llvm_name = "forced-atomics",
	108	.description = "Assume that lock-free native-width atomics are available",
	109	.dependencies = featureSet(&[_]Feature{}),
	110	};
104	111	result[@intFromEnum(Feature.fp)] = .{
105	112	.llvm_name = "fp",
106	113	.description = "Enable Xtensa Single FP instructions",
...	...	@@ -206,6 +213,11 @@ pub const all_features = blk: {
206	213	.description = "Enable Xtensa Relocatable Vector option",
207	214	.dependencies = featureSet(&[_]Feature{}),
208	215	};
	216	result[@intFromEnum(Feature.s32c1i)] = .{
	217	.llvm_name = "s32c1i",
	218	.description = "Enable Xtensa S32C1I option",
	219	.dependencies = featureSet(&[_]Feature{}),
	220	};
209	221	result[@intFromEnum(Feature.sext)] = .{
210	222	.llvm_name = "sext",
211	223	.description = "Enable Xtensa Sign Extend option",
...	...	@@ -245,6 +257,59 @@ pub const all_features = blk: {
245	257	};
246	258
247	259	pub const cpu = struct {
	260	pub const esp32: CpuModel = .{
	261	.name = "esp32",
	262	.llvm_name = "esp32",
	263	.features = featureSet(&[_]Feature{
	264	.bool,
	265	.clamps,
	266	.coprocessor,
	267	.dcache,
	268	.debug,
	269	.density,
	270	.dfpaccel,
	271	.div32,
	272	.exception,
	273	.fp,
	274	.highpriinterrupts_level7,
	275	.interrupt,
	276	.loop,
	277	.mac16,
	278	.minmax,
	279	.miscsr,
	280	.mul16,
	281	.mul32,
	282	.mul32high,
	283	.nsa,
	284	.prid,
	285	.regprotect,
	286	.rvector,
	287	.s32c1i,
	288	.sext,
	289	.threadptr,
	290	.timers3,
	291	.windowed,
	292	}),
	293	};
	294	pub const esp8266: CpuModel = .{
	295	.name = "esp8266",
	296	.llvm_name = "esp8266",
	297	.features = featureSet(&[_]Feature{
	298	.debug,
	299	.density,
	300	.exception,
	301	.extendedl32r,
	302	.highpriinterrupts_level3,
	303	.interrupt,
	304	.mul16,
	305	.mul32,
	306	.nsa,
	307	.prid,
	308	.regprotect,
	309	.rvector,
	310	.timers1,
	311	}),
	312	};
248	313	pub const generic: CpuModel = .{
249	314	.name = "generic",
250	315	.llvm_name = "generic",

lib/std/hash/xxhash.zig-6

...	...	@@ -780,8 +780,6 @@ fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64)
780	780	}
781	781
782	782	test "xxhash3" {
783		if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23807
784
785	783	const H = XxHash3;
786	784	// Non-Seeded Tests
787	785	try testExpect(H, 0, "", 0x2d06800538d394c2);
...	...	@@ -813,8 +811,6 @@ test "xxhash3" {
813	811	}
814	812
815	813	test "xxhash3 smhasher" {
816		if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23807
817
818	814	const Test = struct {
819	815	fn do() !void {
820	816	try expectEqual(verify.smhasher(XxHash3.hash), 0x9a636405);
...	...	@@ -826,8 +822,6 @@ test "xxhash3 smhasher" {
826	822	}
827	823
828	824	test "xxhash3 iterative api" {
829		if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23807
830
831	825	const Test = struct {
832	826	fn do() !void {
833	827	try verify.iterativeApi(XxHash3);

lib/std/math/big/int_test.zig+1-3

...	...	@@ -484,7 +484,7 @@ fn toFloat(comptime Float: type) !void {
484	484	);
485	485	}
486	486	test toFloat {
487		if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/24191
	487	if (builtin.cpu.arch == .x86) return error.SkipZigTest;
488	488	try toFloat(f16);
489	489	try toFloat(f32);
490	490	try toFloat(f64);
...	...	@@ -2801,8 +2801,6 @@ test "bitNotWrap more than two limbs" {
2801	2801	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
2802	2802	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
2803	2803	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
2804		// LLVM: unexpected runtime library name: __umodei4
2805		if (builtin.zig_backend == .stage2_llvm and comptime builtin.target.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
2806	2804
2807	2805	var a = try Managed.initSet(testing.allocator, maxInt(Limb));
2808	2806	defer a.deinit();

lib/std/math/log10.zig-2

...	...	@@ -139,8 +139,6 @@ test log10_int {
139	139	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
140	140	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
141	141	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
142		if (builtin.zig_backend == .stage2_llvm and comptime builtin.target.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
143		if (builtin.zig_backend == .stage2_llvm and comptime builtin.target.cpu.arch == .hexagon) return error.SkipZigTest;
144	142
145	143	inline for (
146	144	.{ u8, u16, u32, u64, u128, u256, u512 },

lib/std/math/modf.zig-1

...	...	@@ -87,7 +87,6 @@ fn ModfTests(comptime T: type) type {
87	87	test "vector" {
88	88	if (builtin.os.tag.isDarwin() and builtin.cpu.arch == .aarch64) return error.SkipZigTest;
89	89	if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
90		if (comptime builtin.cpu.has(.loongarch, .lsx)) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/159529
91	90
92	91	const widths = [_]comptime_int{ 1, 2, 3, 4, 8, 16 };
93	92

lib/std/mem.zig+1-2

...	...	@@ -4809,8 +4809,7 @@ pub fn doNotOptimizeAway(val: anytype) void {
4809	4809	} else doNotOptimizeAway(&val);
4810	4810	},
4811	4811	.float => {
4812		// https://github.com/llvm/llvm-project/issues/159200
4813		if ((t.float.bits == 32 or t.float.bits == 64) and builtin.zig_backend != .stage2_c and !builtin.cpu.arch.isLoongArch()) {
	4812	if ((t.float.bits == 32 or t.float.bits == 64) and builtin.zig_backend != .stage2_c) {
4814	4813	asm volatile (""
4815	4814	:
4816	4815	: [_] "rm" (val),

lib/std/os/linux/test.zig-2

...	...	@@ -10,8 +10,6 @@ const expectEqual = std.testing.expectEqual;
10	10	const fs = std.fs;
11	11
12	12	test "fallocate" {
13		if (builtin.cpu.arch.isMIPS64() and (builtin.abi == .gnuabin32 or builtin.abi == .muslabin32)) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/30220
14
15	13	const io = std.testing.io;
16	14
17	15	var tmp = std.testing.tmpDir(.{});

lib/std/os/windows.zig+130-1

...	...	@@ -5665,7 +5665,136 @@ pub const PF = enum(DWORD) {
5665	5665	ARM_V83_JSCVT_INSTRUCTIONS_AVAILABLE = 44,
5666	5666
5667	5667	/// This Arm processor implements the Arm v8.3 LRCPC instructions (for example, LDAPR). Note that certain Arm v8.2 CPUs may optionally support the LRCPC instructions.
5668		ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE,
	5668	ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE = 45,
	5669
	5670	/// This Arm processor implements the SVE (Scalable Vector Extension) instructions (FEAT_SVE).
	5671	ARM_SVE_INSTRUCTIONS_AVAILABLE = 46,
	5672
	5673	/// This Arm processor implements the SVE2 instructions (FEAT_SVE2).
	5674	ARM_SVE2_INSTRUCTIONS_AVAILABLE = 47,
	5675
	5676	/// This Arm processor implements the SVE2.1 instructions (FEAT_SVE2p1).
	5677	ARM_SVE2_1_INSTRUCTIONS_AVAILABLE = 48,
	5678
	5679	/// This Arm processor implements the SVE AES instructions (FEAT_SVE_AES).
	5680	ARM_SVE_AES_INSTRUCTIONS_AVAILABLE = 49,
	5681
	5682	/// This Arm processor implements the SVE 128-bit polynomial multiply long instructions (FEAT_SVE_PMULL128).
	5683	ARM_SVE_PMULL128_INSTRUCTIONS_AVAILABLE = 50,
	5684
	5685	/// This Arm processor implements the SVE bit permute instructions (FEAT_SVE_BitPerm).
	5686	ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE = 51,
	5687
	5688	/// This Arm processor implements the SVE BF16 (BFloat16) instructions (FEAT_BF16).
	5689	ARM_SVE_BF16_INSTRUCTIONS_AVAILABLE = 52,
	5690
	5691	/// This Arm processor implements the SVE EBF16 (Extended BFloat16) instructions (FEAT_EBF16).
	5692	ARM_SVE_EBF16_INSTRUCTIONS_AVAILABLE = 53,
	5693
	5694	/// This Arm processor implements the SVE B16B16 instructions (FEAT_SVE_B16B16).
	5695	ARM_SVE_B16B16_INSTRUCTIONS_AVAILABLE = 54,
	5696
	5697	/// This Arm processor implements the SVE SHA-3 cryptographic instructions (FEAT_SVE_SHA3).
	5698	ARM_SVE_SHA3_INSTRUCTIONS_AVAILABLE = 55,
	5699
	5700	/// This Arm processor implements the SVE SM4 cryptographic instructions (FEAT_SVE_SM4).
	5701	ARM_SVE_SM4_INSTRUCTIONS_AVAILABLE = 56,
	5702
	5703	/// This Arm processor implements the SVE I8MM (Int8 matrix multiply) instructions (FEAT_I8MM).
	5704	ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE = 57,
	5705
	5706	/// This Arm processor implements the SVE F32MM (FP32 matrix multiply) instructions (FEAT_F32MM).
	5707	ARM_SVE_F32MM_INSTRUCTIONS_AVAILABLE = 58,
	5708
	5709	/// This Arm processor implements the SVE F64MM (FP64 matrix multiply) instructions (FEAT_F64MM).
	5710	ARM_SVE_F64MM_INSTRUCTIONS_AVAILABLE = 59,
	5711
	5712	/// This x64 processor implements the BMI2 instruction set.
	5713	BMI2_INSTRUCTIONS_AVAILABLE = 60,
	5714
	5715	/// This x64 processor implements the MOVDIR64B instruction.
	5716	MOVDIR64B_INSTRUCTION_AVAILABLE = 61,
	5717
	5718	/// This Arm processor implements the LSE2 atomic instructions (FEAT_LSE2).
	5719	ARM_LSE2_AVAILABLE = 62,
	5720
	5721	/// This Arm processor implements the SHA-3 cryptographic instructions (FEAT_SHA3).
	5722	ARM_SHA3_INSTRUCTIONS_AVAILABLE = 64,
	5723
	5724	/// This Arm processor implements the SHA-512 cryptographic instructions (FEAT_SHA512).
	5725	ARM_SHA512_INSTRUCTIONS_AVAILABLE = 65,
	5726
	5727	/// This Arm processor implements the I8MM (Int8 matrix multiply) NEON instructions (FEAT_I8MM).
	5728	ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE = 66,
	5729
	5730	/// This Arm processor implements the FP16 (half-precision floating point) NEON instructions (FEAT_FP16).
	5731	ARM_V82_FP16_INSTRUCTIONS_AVAILABLE = 67,
	5732
	5733	/// This Arm processor implements the BF16 (BFloat16) NEON instructions (FEAT_BF16).
	5734	ARM_V86_BF16_INSTRUCTIONS_AVAILABLE = 68,
	5735
	5736	/// This Arm processor implements the EBF16 (Extended BFloat16) NEON instructions (FEAT_EBF16).
	5737	ARM_V86_EBF16_INSTRUCTIONS_AVAILABLE = 69,
	5738
	5739	/// This Arm processor implements the SME (Scalable Matrix Extension) instructions (FEAT_SME).
	5740	ARM_SME_INSTRUCTIONS_AVAILABLE = 70,
	5741
	5742	/// This Arm processor implements the SME2 instructions (FEAT_SME2).
	5743	ARM_SME2_INSTRUCTIONS_AVAILABLE = 71,
	5744
	5745	/// This Arm processor implements the SME2.1 instructions (FEAT_SME2p1).
	5746	ARM_SME2_1_INSTRUCTIONS_AVAILABLE = 72,
	5747
	5748	/// This Arm processor implements the SME2.2 instructions (FEAT_SME2p2).
	5749	ARM_SME2_2_INSTRUCTIONS_AVAILABLE = 73,
	5750
	5751	/// This Arm processor implements the SVE AES instructions when in Streaming SVE mode (FEAT_SSVE_AES).
	5752	ARM_SME_AES_INSTRUCTIONS_AVAILABLE = 74,
	5753
	5754	/// This Arm processor implements the SVE bit permute instructions when in Streaming SVE mode (FEAT_SSVE_BitPerm).
	5755	ARM_SME_SBITPERM_INSTRUCTIONS_AVAILABLE = 75,
	5756
	5757	/// This Arm processor implements the SVE FMMLA (widening, 4-way, FP8 to FP16) instruction when in Streaming SVE mode (FEAT_SSVE_F8F16MM).
	5758	ARM_SME_SF8MM4_INSTRUCTIONS_AVAILABLE = 76,
	5759
	5760	/// This Arm processor implements the SVE FMMLA (widening, 8-way, FP8 to FP32) instruction when in Streaming SVE mode (FEAT_SSVE_F8F32MM).
	5761	ARM_SME_SF8MM8_INSTRUCTIONS_AVAILABLE = 77,
	5762
	5763	/// This Arm processor implements the SVE2 FP8DOT2 instructions when in Streaming SVE mode (FEAT_SSVE_FP8DOT2).
	5764	ARM_SME_SF8DP2_INSTRUCTIONS_AVAILABLE = 78,
	5765
	5766	/// This Arm processor implements the SVE2 FP8DOT4 instructions when in Streaming SVE mode (FEAT_SSVE_FP8DOT4).
	5767	ARM_SME_SF8DP4_INSTRUCTIONS_AVAILABLE = 79,
	5768
	5769	/// This Arm processor implements the SVE2 FP8FMA instructions when in Streaming SVE mode (FEAT_SSVE_FP8FMA).
	5770	ARM_SME_SF8FMA_INSTRUCTIONS_AVAILABLE = 80,
	5771
	5772	/// This Arm processor implements the SME F8F32 instructions (FEAT_SME_F8F32).
	5773	ARM_SME_F8F32_INSTRUCTIONS_AVAILABLE = 81,
	5774
	5775	/// This Arm processor implements the SME F8F16 instructions (FEAT_SME_F8F16).
	5776	ARM_SME_F8F16_INSTRUCTIONS_AVAILABLE = 82,
	5777
	5778	/// This Arm processor implements the SME F16F16 instructions (FEAT_SME_F16F16).
	5779	ARM_SME_F16F16_INSTRUCTIONS_AVAILABLE = 83,
	5780
	5781	/// This Arm processor implements the SME B16B16 instructions (FEAT_SME_B16B16).
	5782	ARM_SME_B16B16_INSTRUCTIONS_AVAILABLE = 84,
	5783
	5784	/// This Arm processor implements the SME F64F64 instructions (FEAT_SME_F64F64).
	5785	ARM_SME_F64F64_INSTRUCTIONS_AVAILABLE = 85,
	5786
	5787	/// This Arm processor implements the SME I16I64 instructions (FEAT_SME_I16I64).
	5788	ARM_SME_I16I64_INSTRUCTIONS_AVAILABLE = 86,
	5789
	5790	/// This Arm processor implements the SME LUTv2 instructions (FEAT_SME_LUTv2).
	5791	ARM_SME_LUTv2_INSTRUCTIONS_AVAILABLE = 87,
	5792
	5793	/// This Arm processor implements SME FA64 (Full AArch64 instruction set when in Streaming SVE mode) (FEAT_SME_FA64).
	5794	ARM_SME_FA64_INSTRUCTIONS_AVAILABLE = 88,
	5795
	5796	/// This x64 processor implements the UMONITOR instruction.
	5797	UMONITOR_INSTRUCTION_AVAILABLE = 89,
5669	5798	};
5670	5799
5671	5800	pub const MAX_WOW64_SHARED_ENTRIES = 16;

lib/std/simd.zig+4-21

...	...	@@ -157,13 +157,6 @@ pub fn join(a: anytype, b: anytype) @Vector(vectorLength(@TypeOf(a)) + vectorLen
157	157	/// Returns a vector whose elements alternates between those of each input vector.
158	158	/// For example, `interlace(.{[4]u32{11, 12, 13, 14}, [4]u32{21, 22, 23, 24}})` returns a vector containing `.{11, 21, 12, 22, 13, 23, 14, 24}`.
159	159	pub fn interlace(vecs: anytype) @Vector(vectorLength(@TypeOf(vecs[0])) * vecs.len, std.meta.Child(@TypeOf(vecs[0]))) {
160		// interlace doesn't work on MIPS, for some reason.
161		// Notes from earlier debug attempt:
162		// The indices are correct. The problem seems to be with the @shuffle builtin.
163		// On MIPS, the test that interlaces small_base gives { 0, 2, 0, 0, 64, 255, 248, 200, 0, 0 }.
164		// Calling this with two inputs seems to work fine, but I'll let the compile error trigger for all inputs, just to be safe.
165		if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why interlace() doesn't work on MIPS");
166
167	160	const VecType = @TypeOf(vecs[0]);
168	161	const vecs_arr = @as([vecs.len]VecType, vecs);
169	162	const Child = std.meta.Child(@TypeOf(vecs_arr[0]));
...	...	@@ -247,13 +240,11 @@ test "vector patterns" {
247	240	try std.testing.expectEqual([8]u32{ 10, 20, 30, 40, 55, 66, 77, 88 }, join(base, other_base));
248	241	try std.testing.expectEqual([2]u32{ 20, 30 }, extract(base, 1, 2));
249	242
250		if (!builtin.cpu.arch.isMIPS()) {
251		try std.testing.expectEqual([8]u32{ 10, 55, 20, 66, 30, 77, 40, 88 }, interlace(.{ base, other_base }));
	243	try std.testing.expectEqual([8]u32{ 10, 55, 20, 66, 30, 77, 40, 88 }, interlace(.{ base, other_base }));
252	244
253		const small_braid = interlace(small_bases);
254		try std.testing.expectEqual([10]u8{ 0, 2, 4, 6, 8, 1, 3, 5, 7, 9 }, small_braid);
255		try std.testing.expectEqual(small_bases, deinterlace(small_bases.len, small_braid));
256		}
	245	const small_braid = interlace(small_bases);
	246	try std.testing.expectEqual([10]u8{ 0, 2, 4, 6, 8, 1, 3, 5, 7, 9 }, small_braid);
	247	try std.testing.expectEqual(small_bases, deinterlace(small_bases.len, small_braid));
257	248	}
258	249
259	250	/// Joins two vectors, shifts them leftwards (towards lower indices) and extracts the leftmost elements into a vector the length of a and b.
...	...	@@ -384,9 +375,6 @@ pub fn prefixScanWithFunc(
384	375	/// For example, this should be 0 for addition or 1 for multiplication.
385	376	comptime identity: std.meta.Child(@TypeOf(vec)),
386	377	) if (ErrorType == void) @TypeOf(vec) else ErrorType!@TypeOf(vec) {
387		// I haven't debugged this, but it might be a cousin of sorts to what's going on with interlace.
388		if (builtin.cpu.arch.isMIPS()) @compileError("TODO: Find out why prefixScan doesn't work on MIPS");
389
390	378	const len = vectorLength(@TypeOf(vec));
391	379
392	380	if (hop == 0) @compileError("hop can not be 0; you'd be going nowhere forever!");
...	...	@@ -456,11 +444,6 @@ pub fn prefixScan(comptime op: std.builtin.ReduceOp, comptime hop: isize, vec: a
456	444	}
457	445
458	446	test "vector prefix scan" {
459		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21893
460		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .hexagon) return error.SkipZigTest;
461
462		if (builtin.cpu.arch.isMIPS()) return error.SkipZigTest;
463
464	447	const int_base = @Vector(4, i32){ 11, 23, 9, -21 };
465	448	const float_base = @Vector(4, f32){ 2, 0.5, -10, 6.54321 };
466	449	const bool_base = @Vector(4, bool){ true, false, true, false };

lib/std/start.zig+4-3

...	...	@@ -20,8 +20,9 @@ comptime {
20	20	_ = root;
21	21
22	22	if (builtin.output_mode == .Lib and builtin.link_mode == .dynamic) {
23		if (native_os == .windows and !@hasDecl(root, "_DllMainCRTStartup")) {
24		@export(&_DllMainCRTStartup, .{ .name = "_DllMainCRTStartup" });
	23	const dll_main_crt_startup = if (builtin.abi.isGnu()) "DllMainCRTStartup" else "_DllMainCRTStartup";
	24	if (native_os == .windows and !builtin.link_libc and !@hasDecl(root, dll_main_crt_startup)) {
	25	@export(&DllMainCRTStartup, .{ .name = dll_main_crt_startup });
25	26	}
26	27	} else if (builtin.output_mode == .Exe or @hasDecl(root, "main")) {
27	28	if (builtin.link_libc and @hasDecl(root, "main")) {
...	...	@@ -71,7 +72,7 @@ comptime {
71	72	}
72	73	}
73	74
74		fn _DllMainCRTStartup(
	75	fn DllMainCRTStartup(
75	76	hinstDLL: std.os.windows.HINSTANCE,
76	77	fdwReason: std.os.windows.DWORD,
77	78	lpReserved: std.os.windows.LPVOID,

lib/std/zig/llvm/Builder.zig+2-2

...	...	@@ -13947,8 +13947,8 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
13947	13947	const bit_count = extra.type.scalarBits(self);
13948	13948	const val: i64 = if (bit_count <= 64)
13949	13949	bigint.toInt(i64) catch unreachable
13950		else if (bigint.toInt(u64)) \|val\|
13951		@bitCast(val)
	13950	else if (bigint.toInt(u63)) \|val\|
	13951	@bitCast(@as(u64, val))
13952	13952	else \|_\| {
13953	13953	const limbs = try record.addManyAsSlice(
13954	13954	self.gpa,

lib/std/zig/system.zig-10

...	...	@@ -460,16 +460,6 @@ pub fn resolveTargetQuery(io: Io, query: Target.Query) DetectError!Target {
460	460	if (result.cpu.arch.isArm() and result.abi.float() == .soft) {
461	461	result.cpu.features.removeFeature(@intFromEnum(Target.arm.Feature.vfp2));
462	462	}
463
464		// https://github.com/llvm/llvm-project/issues/135283
465		if (result.cpu.arch.isMIPS() and result.abi.float() == .soft) {
466		result.cpu.features.addFeature(@intFromEnum(Target.mips.Feature.soft_float));
467		}
468
469		// https://github.com/llvm/llvm-project/issues/168992
470		if (result.cpu.arch == .s390x) {
471		result.cpu.features.removeFeature(@intFromEnum(Target.s390x.Feature.vector));
472		}
473	463	}
474	464
475	465	// It's possible that we detect the native ABI, but fail to detect the OS version or were told

lib/std/zig/system/arm.zig+56-2

...	...	@@ -21,7 +21,7 @@ pub const cpu_models = struct {
21	21	};
22	22
23	23	// implementer = 0x41
24		const ARM = [_]E{
	24	const Arm = [_]E{
25	25	E{ .part = 0x926, .m32 = &A32.arm926ej_s },
26	26	E{ .part = 0xb02, .m32 = &A32.mpcore },
27	27	E{ .part = 0xb36, .m32 = &A32.arm1136j_s },
...	...	@@ -88,8 +88,12 @@ pub const cpu_models = struct {
88	88	E{ .part = 0xd87, .m64 = &A64.cortex_a725 },
89	89	E{ .part = 0xd88, .m64 = &A64.cortex_a520ae },
90	90	E{ .part = 0xd89, .m64 = &A64.cortex_a720ae },
	91	E{ .part = 0xd8a, .m64 = &A64.c1_nano },
	92	E{ .part = 0xd8b, .m64 = &A64.c1_pro },
	93	E{ .part = 0xd8c, .m64 = &A64.c1_ultra },
91	94	E{ .part = 0xd8e, .m64 = &A64.neoverse_n3 },
92	95	E{ .part = 0xd8f, .m64 = &A64.cortex_a320 },
	96	E{ .part = 0xd90, .m64 = &A64.c1_premium },
93	97	};
94	98	// implementer = 0x42
95	99	const Broadcom = [_]E{
...	...	@@ -102,10 +106,17 @@ pub const cpu_models = struct {
102	106	E{ .part = 0x0a3, .m64 = &A64.thunderxt83 },
103	107	E{ .part = 0x0a1, .m64 = &A64.thunderxt88 },
104	108	E{ .part = 0x0af, .m64 = &A64.thunderx2t99 },
	109	E{ .part = 0x0b0, .m64 = &A64.cortex_a57 },
	110	E{ .part = 0x0b1, .m64 = &A64.cortex_a57 },
	111	E{ .part = 0x0b2, .m64 = &A64.cortex_a57 },
	112	E{ .part = 0x0b3, .m64 = &A64.cortex_a57 },
	113	E{ .part = 0x0b4, .m64 = &A64.cortex_a57 },
	114	E{ .part = 0x0b5, .m64 = &A64.cortex_a57 },
105	115	};
106	116	// implementer = 0x46
107	117	const Fujitsu = [_]E{
108	118	E{ .part = 0x001, .m64 = &A64.a64fx },
	119	E{ .part = 0x003, .m64 = &A64.fujitsu_monaka },
109	120	};
110	121	// implementer = 0x48
111	122	const HiSilicon = [_]E{
...	...	@@ -137,8 +148,14 @@ pub const cpu_models = struct {
137	148	E{ .part = 0xc00, .m64 = &A64.falkor },
138	149	E{ .part = 0xc01, .m64 = &A64.saphira },
139	150	};
	151	// implementer = 0x53
	152	const Samsung = [_]E{
	153	E{ .part = 0x000, .m64 = &A64.exynos_m1 },
	154	};
140	155	// implementer = 0x61
141	156	const Apple = [_]E{
	157	E{ .part = 0x020, .m64 = &A64.apple_m1 },
	158	E{ .part = 0x021, .m64 = &A64.apple_m1 },
142	159	E{ .part = 0x022, .m64 = &A64.apple_m1 },
143	160	E{ .part = 0x023, .m64 = &A64.apple_m1 },
144	161	E{ .part = 0x024, .m64 = &A64.apple_m1 },
...	...	@@ -151,11 +168,43 @@ pub const cpu_models = struct {
151	168	E{ .part = 0x035, .m64 = &A64.apple_m2 },
152	169	E{ .part = 0x038, .m64 = &A64.apple_m2 },
153	170	E{ .part = 0x039, .m64 = &A64.apple_m2 },
	171	E{ .part = 0x042, .m64 = &A64.apple_m3 },
	172	E{ .part = 0x043, .m64 = &A64.apple_m3 },
	173	E{ .part = 0x044, .m64 = &A64.apple_m3 },
	174	E{ .part = 0x045, .m64 = &A64.apple_m3 },
	175	E{ .part = 0x048, .m64 = &A64.apple_m3 },
	176	E{ .part = 0x049, .m64 = &A64.apple_m3 },
	177	E{ .part = 0x052, .m64 = &A64.apple_m4 },
	178	E{ .part = 0x053, .m64 = &A64.apple_m4 },
	179	E{ .part = 0x054, .m64 = &A64.apple_m4 },
	180	E{ .part = 0x055, .m64 = &A64.apple_m4 },
	181	E{ .part = 0x058, .m64 = &A64.apple_m4 },
	182	E{ .part = 0x059, .m64 = &A64.apple_m4 },
	183	};
	184	// implementer = 0x63
	185	const ArmChina = [_]E{
	186	E{ .part = 0x132, .m32 = &A32.star_mc1 },
	187	E{ .part = 0xd25, .m32 = &A32.star_mc3 },
	188	};
	189	// implementer = 0x68
	190	const Hxt = [_]E{
	191	E{ .part = 0x000, .m64 = &A64.cortex_a57 },
	192	};
	193	// implementer = 0x6d
	194	const Microsoft = [_]E{
	195	E{ .part = 0xd49, .m64 = &A64.neoverse_n2 },
	196	};
	197	// implementer = 0xC0
	198	const AmpereOne = [_]E{
	199	E{ .part = 0xac3, .m64 = &A64.ampere1 },
	200	E{ .part = 0xac4, .m64 = &A64.ampere1a },
	201	E{ .part = 0xac5, .m64 = &A64.ampere1b },
	202	E{ .part = 0xac7, .m64 = &A64.ampere1c },
154	203	};
155	204
156	205	pub fn isKnown(core: CoreInfo, is_64bit: bool) ?*const Target.Cpu.Model {
157	206	const models = switch (core.implementer) {
158		0x41 => &ARM,
	207	0x41 => &Arm,
159	208	0x42 => &Broadcom,
160	209	0x43 => &Cavium,
161	210	0x46 => &Fujitsu,
...	...	@@ -163,7 +212,12 @@ pub const cpu_models = struct {
163	212	0x4e => &Nvidia,
164	213	0x50 => &Ampere,
165	214	0x51 => &Qualcomm,
	215	0x53 => &Samsung,
166	216	0x61 => &Apple,
	217	0x63 => &ArmChina,
	218	0x68 => &Hxt,
	219	0x6d => &Microsoft,
	220	0xC0 => &AmpereOne,
167	221	else => return null,
168	222	};
169	223

lib/std/zig/system/windows.zig+34-1

...	...	@@ -215,10 +215,43 @@ fn genericCpuAndNativeFeatures(arch: Target.Cpu.Arch) Target.Cpu {
215	215	// Override any features that are either present or absent
216	216	setFeature(Feature, &cpu, .neon, IsProcessorFeaturePresent(PF.ARM_NEON_INSTRUCTIONS_AVAILABLE));
217	217	setFeature(Feature, &cpu, .crc, IsProcessorFeaturePresent(PF.ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE));
218		setFeature(Feature, &cpu, .crypto, IsProcessorFeaturePresent(PF.ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE));
	218	setFeature(Feature, &cpu, .aes, IsProcessorFeaturePresent(PF.ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE));
	219	setFeature(Feature, &cpu, .sha2, IsProcessorFeaturePresent(PF.ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE));
219	220	setFeature(Feature, &cpu, .lse, IsProcessorFeaturePresent(PF.ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE));
220	221	setFeature(Feature, &cpu, .dotprod, IsProcessorFeaturePresent(PF.ARM_V82_DP_INSTRUCTIONS_AVAILABLE));
221	222	setFeature(Feature, &cpu, .jsconv, IsProcessorFeaturePresent(PF.ARM_V83_JSCVT_INSTRUCTIONS_AVAILABLE));
	223	setFeature(Feature, &cpu, .rcpc, IsProcessorFeaturePresent(PF.ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE));
	224	setFeature(Feature, &cpu, .sve, IsProcessorFeaturePresent(PF.ARM_SVE_INSTRUCTIONS_AVAILABLE));
	225	setFeature(Feature, &cpu, .sve2, IsProcessorFeaturePresent(PF.ARM_SVE2_INSTRUCTIONS_AVAILABLE));
	226	setFeature(Feature, &cpu, .sve2p1, IsProcessorFeaturePresent(PF.ARM_SVE2_1_INSTRUCTIONS_AVAILABLE));
	227	setFeature(Feature, &cpu, .sve_aes, IsProcessorFeaturePresent(PF.ARM_SVE_AES_INSTRUCTIONS_AVAILABLE) or IsProcessorFeaturePresent(PF.ARM_SVE_PMULL128_INSTRUCTIONS_AVAILABLE));
	228	setFeature(Feature, &cpu, .sve_bitperm, IsProcessorFeaturePresent(PF.ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE));
	229	setFeature(Feature, &cpu, .bf16, IsProcessorFeaturePresent(PF.ARM_V86_BF16_INSTRUCTIONS_AVAILABLE));
	230	setFeature(Feature, &cpu, .sve_b16b16, IsProcessorFeaturePresent(PF.ARM_SVE_B16B16_INSTRUCTIONS_AVAILABLE));
	231	setFeature(Feature, &cpu, .sve_sha3, IsProcessorFeaturePresent(PF.ARM_SVE_SHA3_INSTRUCTIONS_AVAILABLE));
	232	setFeature(Feature, &cpu, .sve_sm4, IsProcessorFeaturePresent(PF.ARM_SVE_SM4_INSTRUCTIONS_AVAILABLE));
	233	setFeature(Feature, &cpu, .i8mm, IsProcessorFeaturePresent(PF.ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE));
	234	setFeature(Feature, &cpu, .f32mm, IsProcessorFeaturePresent(PF.ARM_SVE_F32MM_INSTRUCTIONS_AVAILABLE));
	235	setFeature(Feature, &cpu, .f64mm, IsProcessorFeaturePresent(PF.ARM_SVE_F64MM_INSTRUCTIONS_AVAILABLE));
	236	setFeature(Feature, &cpu, .sme, IsProcessorFeaturePresent(PF.ARM_SME_INSTRUCTIONS_AVAILABLE));
	237	setFeature(Feature, &cpu, .sme2, IsProcessorFeaturePresent(PF.ARM_SME2_INSTRUCTIONS_AVAILABLE));
	238	setFeature(Feature, &cpu, .lse2, IsProcessorFeaturePresent(PF.ARM_LSE2_AVAILABLE));
	239	setFeature(Feature, &cpu, .sha3, IsProcessorFeaturePresent(PF.ARM_SHA3_INSTRUCTIONS_AVAILABLE) and IsProcessorFeaturePresent(PF.ARM_SHA512_INSTRUCTIONS_AVAILABLE));
	240	setFeature(Feature, &cpu, .fullfp16, IsProcessorFeaturePresent(PF.ARM_V82_FP16_INSTRUCTIONS_AVAILABLE));
	241	setFeature(Feature, &cpu, .sme2p1, IsProcessorFeaturePresent(PF.ARM_SME2_1_INSTRUCTIONS_AVAILABLE));
	242	setFeature(Feature, &cpu, .sme2p2, IsProcessorFeaturePresent(PF.ARM_SME2_2_INSTRUCTIONS_AVAILABLE));
	243	setFeature(Feature, &cpu, .ssve_aes, IsProcessorFeaturePresent(PF.ARM_SME_AES_INSTRUCTIONS_AVAILABLE));
	244	setFeature(Feature, &cpu, .ssve_bitperm, IsProcessorFeaturePresent(PF.ARM_SME_SBITPERM_INSTRUCTIONS_AVAILABLE));
	245	setFeature(Feature, &cpu, .ssve_fp8dot2, IsProcessorFeaturePresent(PF.ARM_SME_SF8DP2_INSTRUCTIONS_AVAILABLE));
	246	setFeature(Feature, &cpu, .ssve_fp8dot4, IsProcessorFeaturePresent(PF.ARM_SME_SF8DP4_INSTRUCTIONS_AVAILABLE));
	247	setFeature(Feature, &cpu, .ssve_fp8fma, IsProcessorFeaturePresent(PF.ARM_SME_SF8FMA_INSTRUCTIONS_AVAILABLE));
	248	setFeature(Feature, &cpu, .sme_f8f32, IsProcessorFeaturePresent(PF.ARM_SME_F8F32_INSTRUCTIONS_AVAILABLE));
	249	setFeature(Feature, &cpu, .sme_f8f16, IsProcessorFeaturePresent(PF.ARM_SME_F8F16_INSTRUCTIONS_AVAILABLE));
	250	setFeature(Feature, &cpu, .sme_b16b16, IsProcessorFeaturePresent(PF.ARM_SME_B16B16_INSTRUCTIONS_AVAILABLE));
	251	setFeature(Feature, &cpu, .sme_f64f64, IsProcessorFeaturePresent(PF.ARM_SME_F64F64_INSTRUCTIONS_AVAILABLE));
	252	setFeature(Feature, &cpu, .sme_i16i64, IsProcessorFeaturePresent(PF.ARM_SME_I16I64_INSTRUCTIONS_AVAILABLE));
	253	setFeature(Feature, &cpu, .sme_lutv2, IsProcessorFeaturePresent(PF.ARM_SME_LUTv2_INSTRUCTIONS_AVAILABLE));
	254	setFeature(Feature, &cpu, .sme_fa64, IsProcessorFeaturePresent(PF.ARM_SME_FA64_INSTRUCTIONS_AVAILABLE));
222	255	},
223	256	else => {},
224	257	}

lib/std/zig/system/x86.zig+60-16

...	...	@@ -232,7 +232,7 @@ fn detectIntelProcessor(cpu: *Target.Cpu, family: u32, model: u32, brand_id: u32
232	232	cpu.model = &Target.x86.cpu.lunarlake;
233	233	return;
234	234	},
235		0xcc => {
	235	0xcc, 0xd5 => {
236	236	cpu.model = &Target.x86.cpu.pantherlake;
237	237	return;
238	238	},
...	...	@@ -307,6 +307,20 @@ fn detectIntelProcessor(cpu: *Target.Cpu, family: u32, model: u32, brand_id: u32
307	307	cpu.model = &Target.x86.cpu.pentium4;
308	308	return;
309	309	},
	310	18 => switch (model) {
	311	0x01, 0x03 => {
	312	cpu.model = &Target.x86.cpu.novalake;
	313	return;
	314	},
	315	else => return, // Unknown CPU Model
	316	},
	317	19 => switch (model) {
	318	0x01 => {
	319	cpu.model = &Target.x86.cpu.diamondrapids;
	320	return;
	321	},
	322	else => return, // Unknown CPU Model
	323	},
310	324	else => return, // Unknown CPU Model
311	325	}
312	326	}
...	...	@@ -412,6 +426,8 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
412	426	// AMX requires additional context to be saved by the OS.
413	427	const has_amx_save = xcr0.xtilecfg and xcr0.xtiledata;
414	428
	429	const has_apx_save = xcr0.apx;
	430
415	431	setFeature(cpu, .avx, has_avx_save);
416	432	setFeature(cpu, .fma, bit(leaf.ecx, 12) and has_avx_save);
417	433	// Only enable XSAVE if OS has enabled support for saving YMM state.
...	...	@@ -470,7 +486,20 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
470	486	}
471	487	}
472	488
473		if (max_level >= 0x7) {
	489	if (max_ext_level >= 0x80000021) {
	490	leaf = cpuid(0x80000021, 0);
	491
	492	// AMD uses a different bit for prefetchi.
	493	setFeature(cpu, .prefetchi, bit(leaf.eax, 20));
	494	} else {
	495	for ([_]Target.x86.Feature{
	496	.prefetchi,
	497	}) \|feat\| {
	498	setFeature(cpu, feat, false);
	499	}
	500	}
	501
	502	const has_avx10 = if (max_level >= 0x7) has_avx10: {
474	503	leaf = cpuid(0x7, 0);
475	504
476	505	setFeature(cpu, .fsgsbase, bit(leaf.ebx, 0));
...	...	@@ -484,7 +513,6 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
484	513	setFeature(cpu, .rtm, bit(leaf.ebx, 11));
485	514	// AVX512 is only supported if the OS supports the context save for it.
486	515	setFeature(cpu, .avx512f, bit(leaf.ebx, 16) and has_avx512_save);
487		setFeature(cpu, .evex512, bit(leaf.ebx, 16) and has_avx512_save);
488	516	setFeature(cpu, .avx512dq, bit(leaf.ebx, 17) and has_avx512_save);
489	517	setFeature(cpu, .rdseed, bit(leaf.ebx, 18));
490	518	setFeature(cpu, .adx, bit(leaf.ebx, 19));
...	...	@@ -556,15 +584,20 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
556	584	setFeature(cpu, .avxneconvert, bit(leaf.edx, 5) and has_avx_save);
557	585	setFeature(cpu, .amx_complex, bit(leaf.edx, 8) and has_amx_save);
558	586	setFeature(cpu, .avxvnniint16, bit(leaf.edx, 10) and has_avx_save);
559		setFeature(cpu, .prefetchi, bit(leaf.edx, 14));
	587	// This needs to account for prefetchi already being detected above on AMD.
	588	setFeature(cpu, .prefetchi, cpu.has(.x86, .prefetchi) or bit(leaf.edx, 14));
560	589	setFeature(cpu, .usermsr, bit(leaf.edx, 15));
561	590	// APX
562		setFeature(cpu, .egpr, bit(leaf.edx, 21));
563		setFeature(cpu, .push2pop2, bit(leaf.edx, 21));
564		setFeature(cpu, .ppx, bit(leaf.edx, 21));
565		setFeature(cpu, .ndd, bit(leaf.edx, 21));
566		setFeature(cpu, .ccmp, bit(leaf.edx, 21));
567		setFeature(cpu, .cf, bit(leaf.edx, 21));
	591	setFeature(cpu, .egpr, bit(leaf.edx, 21) and has_apx_save);
	592	setFeature(cpu, .push2pop2, bit(leaf.edx, 21) and has_apx_save);
	593	setFeature(cpu, .ppx, bit(leaf.edx, 21) and has_apx_save);
	594	setFeature(cpu, .ndd, bit(leaf.edx, 21) and has_apx_save);
	595	setFeature(cpu, .ccmp, bit(leaf.edx, 21) and has_apx_save);
	596	setFeature(cpu, .nf, bit(leaf.edx, 21) and has_apx_save);
	597	setFeature(cpu, .cf, bit(leaf.edx, 21) and has_apx_save);
	598	setFeature(cpu, .zu, bit(leaf.edx, 21) and has_apx_save);
	599
	600	break :has_avx10 bit(leaf.edx, 19);
568	601	} else {
569	602	for ([_]Target.x86.Feature{
570	603	.sha512,
...	...	@@ -582,19 +615,23 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
582	615	.avxneconvert,
583	616	.amx_complex,
584	617	.avxvnniint16,
585		.prefetchi,
	618	// prefetchi already handled earlier.
586	619	.usermsr,
587	620	.egpr,
588	621	.push2pop2,
589	622	.ppx,
590	623	.ndd,
591	624	.ccmp,
	625	.nf,
592	626	.cf,
	627	.zu,
593	628	}) \|feat\| {
594	629	setFeature(cpu, feat, false);
595	630	}
596	631	}
597		} else {
	632
	633	break :has_avx10 false;
	634	} else has_avx10: {
598	635	for ([_]Target.x86.Feature{
599	636	.fsgsbase,
600	637	.sgx,
...	...	@@ -605,7 +642,6 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
605	642	.invpcid,
606	643	.rtm,
607	644	.avx512f,
608		.evex512,
609	645	.avx512dq,
610	646	.rdseed,
611	647	.adx,
...	...	@@ -664,18 +700,22 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
664	700	.avxneconvert,
665	701	.amx_complex,
666	702	.avxvnniint16,
667		.prefetchi,
	703	// prefetchi already handled earlier.
668	704	.usermsr,
669	705	.egpr,
670	706	.push2pop2,
671	707	.ppx,
672	708	.ndd,
673	709	.ccmp,
	710	.nf,
674	711	.cf,
	712	.zu,
675	713	}) \|feat\| {
676	714	setFeature(cpu, feat, false);
677	715	}
678		}
	716
	717	break :has_avx10 false;
	718	};
679	719
680	720	if (max_level >= 0xD and has_avx_save) {
681	721	leaf = cpuid(0xD, 0x1);
...	...	@@ -721,10 +761,14 @@ fn detectNativeFeatures(cpu: *Target.Cpu, os_tag: Target.Os.Tag) void {
721	761	if (max_level >= 0x24) {
722	762	leaf = cpuid(0x24, 0);
723	763
724		setFeature(cpu, .avx10_1, bit(leaf.ebx, 18));
	764	const avx_ver = leaf.ebx & 0xff;
	765
	766	setFeature(cpu, .avx10_1, has_avx10 and avx_ver >= 1);
	767	setFeature(cpu, .avx10_2, has_avx10 and avx_ver >= 2);
725	768	} else {
726	769	for ([_]Target.x86.Feature{
727	770	.avx10_1,
	771	.avx10_2,
728	772	}) \|feat\| {
729	773	setFeature(cpu, feat, false);
730	774	}

src/Compilation.zig+18-2

...	...	@@ -3013,6 +3013,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
3013	3013	try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{c_object.src.src_path}));
3014	3014	}
3015	3015
	3016	for (comp.link_inputs) \|input\| if (input.path()) \|path\| {
	3017	try comp.appendFileSystemInput(try .fromUnresolved(arena, comp.dirs, &.{
	3018	path.root_dir.path orelse ".",
	3019	path.sub_path,
	3020	}));
	3021	};
	3022
3016	3023	// For compiling Win32 resources, we rely on the cache hash system to avoid duplicating work.
3017	3024	// Add a Job for each Win32 resource file.
3018	3025	try comp.win32_resource_work_queue.ensureUnusedCapacity(gpa, comp.win32_resource_table.count());
...	...	@@ -4763,12 +4770,12 @@ fn writeDepFile(
4763	4770
4764	4771	try w.print("{f}:", .{bin_file});
4765	4772
4766		{
	4773	if (fsi.len > 0) {
4767	4774	var it = std.mem.splitScalar(u8, fsi, 0);
4768	4775	while (it.next()) \|input\| try w.print(" \\\n {f}{s}", .{ prefixes[input[0] - 1], input[1..] });
4769	4776	}
4770	4777
4771		{
	4778	if (fsi.len > 0) {
4772	4779	var it = std.mem.splitScalar(u8, fsi, 0);
4773	4780	while (it.next()) \|input\| try w.print("\n\n{f}{s}:", .{ prefixes[input[0] - 1], input[1..] });
4774	4781	}
...	...	@@ -6355,6 +6362,13 @@ fn addCommonCCArgs(
6355	6362	try argv.append(
6356	6363	try std.fmt.allocPrint(arena, "-D_WIN32_WINNT=0x{x:0>4}", .{minver}),
6357	6364	);
	6365
	6366	// MinGW-w64's inline functions in headers (e.g. `fabs`), which are emitted with `linkonce_odr`
	6367	// linkage, sometimes cause duplicate symbol errors due to us providing the same symbols with
	6368	// `weak` linkage in compiler-rt or libzigc. So just disable them. Besides, they undermine the
	6369	// goal of moving more libc code to Zig, and they're also just kind of unnecessary since LLVM is
	6370	// perfectly capable of recognizing and optimizing libcalls.
	6371	try argv.append("-D__CRT__NO_INLINE");
6358	6372	} else if (target.isFreeBSDLibC()) {
6359	6373	// https://docs.freebsd.org/en/books/porters-handbook/versions
6360	6374	const min_ver = target.os.version_range.semver.min;
...	...	@@ -6794,6 +6808,8 @@ pub fn addCCArgs(
6794	6808	// We communicate these to Clang through the dedicated options.
6795	6809	if (std.mem.startsWith(u8, llvm_name, "soft-float") or
6796	6810	std.mem.startsWith(u8, llvm_name, "hard-float") or
	6811	(target.cpu.arch.isPowerPC() and std.mem.startsWith(u8, llvm_name, "64bit")) or
	6812	(target.cpu.arch.isX86() and std.mem.startsWith(u8, llvm_name, "x32")) or
6797	6813	(target.cpu.arch == .s390x and std.mem.eql(u8, llvm_name, "backchain")))
6798	6814	continue;
6799	6815

src/Package/Module.zig+3

...	...	@@ -347,6 +347,9 @@ pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module {
347	347	// See https://github.com/ziglang/zig/issues/23539
348	348	if (target_util.isDynamicAMDGCNFeature(target, feature)) continue;
349	349
	350	if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue;
	351	if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue;
	352
350	353	var is_enabled = target.cpu.features.isEnabled(feature.index);
351	354	if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) {
352	355	is_enabled = !omit_frame_pointer;

src/clang_options.zon+93-17

...	...	@@ -273,6 +273,9 @@
273	273	.{ .name = "utf-8", .psl = true },
274	274	.{ .name = "validate-charset", .psl = true },
275	275	.{ .name = "validate-charset-", .psl = true },
	276	.{ .name = "vlen", .psl = true },
	277	.{ .name = "vlen=256", .psl = true },
	278	.{ .name = "vlen=512", .psl = true },
276	279	.{ .name = "vmb", .psl = true },
277	280	.{ .name = "vmg", .psl = true },
278	281	.{ .name = "vmm", .psl = true },
...	...	@@ -692,9 +695,9 @@
692	695	.{ .name = "fnew-alignment", .syntax = .separate },
693	696	.{ .name = "faligned-new" },
694	697	.{ .name = "fno-aligned-new" },
695		.{ .name = "fsched-interblock" },
696	698	.{ .name = "fcuda-rdc" },
697	699	.{ .name = "fno-cuda-rdc" },
	700	.{ .name = "fsched-interblock" },
698	701	.{ .name = "ftree-vectorize" },
699	702	.{ .name = "fno-tree-vectorize" },
700	703	.{ .name = "ftree-slp-vectorize" },
...	...	@@ -723,6 +726,8 @@
723	726	.{ .name = "mtune=help", .ze = .mcpu },
724	727	.{ .name = "integrated-as" },
725	728	.{ .name = "no-integrated-as" },
	729	.{ .name = "shared-libasan" },
	730	.{ .name = "static-libasan" },
726	731	.{ .name = "fopenmp-is-device" },
727	732	.{ .name = "fcuda-approx-transcendentals" },
728	733	.{ .name = "fno-cuda-approx-transcendentals" },
...	...	@@ -760,14 +765,12 @@
760	765	.{ .name = "Oy-", .psl = true },
761	766	.{ .name = "Qgather-", .psl = true },
762	767	.{ .name = "Qscatter-", .psl = true },
763		.{ .name = "shared-libasan" },
764	768	.{ .name = "Xmicrosoft-visualc-tools-root", .syntax = .separate },
765	769	.{ .name = "Xmicrosoft-visualc-tools-version", .syntax = .separate },
766	770	.{ .name = "Xmicrosoft-windows-sdk-root", .syntax = .separate },
767	771	.{ .name = "Xmicrosoft-windows-sdk-version", .syntax = .separate },
768	772	.{ .name = "Xmicrosoft-windows-sys-root", .syntax = .separate },
769	773	.{ .name = "Qembed_debug", .psl = true },
770		.{ .name = "static-libasan" },
771	774	.{ .name = "fslp-vectorize-aggressive" },
772	775	.{ .name = "fident" },
773	776	.{ .name = "fno-ident" },
...	...	@@ -787,13 +790,13 @@
787	790	.{ .name = "fno-sanitize-blacklist" },
788	791	.{ .name = "fhonor-infinites" },
789	792	.{ .name = "fno-honor-infinites" },
790		.{ .name = "findirect-virtual-calls" },
791	793	.{
792	794	.name = "config",
793	795	.syntax = .separate,
794	796	.pd1 = false,
795	797	.pd2 = true,
796	798	},
	799	.{ .name = "findirect-virtual-calls" },
797	800	.{ .name = "ansi", .pd2 = true },
798	801	.{ .name = "arch", .syntax = .separate },
799	802	.{ .name = "arch_errors_fatal" },
...	...	@@ -900,6 +903,7 @@
900	903	.{ .name = "dead_strip" },
901	904	.{ .name = "debug-forward-template-params" },
902	905	.{ .name = "debug-info-macro" },
	906	.{ .name = "debug-info-macro-expansion-loc" },
903	907	.{ .name = "default-function-attr", .syntax = .separate },
904	908	.{
905	909	.name = "defsym",
...	...	@@ -934,14 +938,19 @@
934	938	.{ .name = "dwarf-debug-producer", .syntax = .separate },
935	939	.{ .name = "dwarf-explicit-import" },
936	940	.{ .name = "dwarf-ext-refs" },
	941	.{ .name = "all-resources-bound", .psl = true },
	942	.{ .name = "Zpc", .psl = true },
937	943	.{ .name = "Vd", .psl = true },
938	944	.{ .name = "Gis", .psl = true },
939	945	.{ .name = "hlsl-no-stdinc", .psl = true },
	946	.{ .name = "rootsig-define", .syntax = .separate },
940	947	.{
941	948	.name = "force-rootsig-ver",
942	949	.syntax = .separate,
943	950	.psl = true,
944	951	},
	952	.{ .name = "Zpr", .psl = true },
	953	.{ .name = "Qstrip-rootsignature", .psl = true },
945	954	.{ .name = "dylib_file", .syntax = .separate },
946	955	.{ .name = "dylinker" },
947	956	.{ .name = "dynamic", .ze = .dynamic },
...	...	@@ -1089,6 +1098,7 @@
1089	1098	.{ .name = "fcheck-array-temporaries" },
1090	1099	.{ .name = "fcheck-new" },
1091	1100	.{ .name = "fclangir" },
	1101	.{ .name = "fcoarray" },
1092	1102	.{ .name = "fcodegen-data-generate" },
1093	1103	.{ .name = "fcodegen-data-use" },
1094	1104	.{ .name = "fcolor-diagnostics", .ze = .color_diagnostics },
...	...	@@ -1148,8 +1158,11 @@
1148	1158	.{ .name = "fdeclspec" },
1149	1159	.{ .name = "fdefault-double-8" },
1150	1160	.{ .name = "fdefault-inline" },
	1161	.{ .name = "fdefault-integer-4" },
1151	1162	.{ .name = "fdefault-integer-8" },
	1163	.{ .name = "fdefault-real-4" },
1152	1164	.{ .name = "fdefault-real-8" },
	1165	.{ .name = "fdefer-ts" },
1153	1166	.{ .name = "fdefine-target-os-macros" },
1154	1167	.{ .name = "fdelayed-template-parsing" },
1155	1168	.{ .name = "fdelete-null-pointer-checks" },
...	...	@@ -1205,9 +1218,11 @@
1205	1218	.{ .name = "ferror-limit", .syntax = .separate },
1206	1219	.{ .name = "fescaping-block-tail-calls" },
1207	1220	.{ .name = "fexceptions" },
	1221	.{ .name = "fexperimental-call-graph-section" },
1208	1222	.{ .name = "fexperimental-isel" },
1209	1223	.{ .name = "fexperimental-late-parse-attributes" },
1210	1224	.{ .name = "fexperimental-library" },
	1225	.{ .name = "fexperimental-loop-fusion" },
1211	1226	.{ .name = "fexperimental-new-constant-interpreter" },
1212	1227	.{ .name = "fexperimental-omit-vtable-rtti" },
1213	1228	.{ .name = "fexperimental-relative-c++-abi-vtables" },
...	...	@@ -1221,6 +1236,7 @@
1221	1236	.{ .name = "ff2c" },
1222	1237	.{ .name = "ffake-address-space-map" },
1223	1238	.{ .name = "ffast-math" },
	1239	.{ .name = "ffast-real-mod" },
1224	1240	.{ .name = "ffat-lto-objects" },
1225	1241	.{ .name = "ffile-reproducible" },
1226	1242	.{ .name = "fimplicit-modules-use-lock" },
...	...	@@ -1345,6 +1361,8 @@
1345	1361	.{ .name = "fhip-fp32-correctly-rounded-divide-sqrt" },
1346	1362	.{ .name = "fhip-kernel-arg-name" },
1347	1363	.{ .name = "fhip-new-launch-api" },
	1364	.{ .name = "fspv-enable-maximal-reconvergence" },
	1365	.{ .name = "fspv-use-unknown-image-format" },
1348	1366	.{ .name = "fhlsl-strict-availability" },
1349	1367	.{ .name = "fhonor-infinities" },
1350	1368	.{ .name = "fhonor-nans" },
...	...	@@ -1393,6 +1411,7 @@
1393	1411	.{ .name = "flat_namespace" },
1394	1412	.{ .name = "flax-vector-conversions" },
1395	1413	.{ .name = "fexperimental-lifetime-safety" },
	1414	.{ .name = "fexperimental-lifetime-safety-inference" },
1396	1415	.{ .name = "flimit-debug-info" },
1397	1416	.{ .name = "flogical-abbreviations" },
1398	1417	.{ .name = "floop-interchange" },
...	...	@@ -1428,7 +1447,9 @@
1428	1447	.{ .name = "fmodules-force-validate-user-headers" },
1429	1448	.{ .name = "fmodules-hash-content" },
1430	1449	.{ .name = "fmodules-local-submodule-visibility" },
	1450	.{ .name = "fmodules-reduced-bmi" },
1431	1451	.{ .name = "fmodules-search-all" },
	1452	.{ .name = "fmodules-single-module-parse-mode" },
1432	1453	.{ .name = "fmodules-skip-diagnostic-options" },
1433	1454	.{ .name = "fmodules-skip-header-search-paths" },
1434	1455	.{ .name = "fmodules-strict-context-hash" },
...	...	@@ -1448,8 +1469,10 @@
1448	1469	.{ .name = "fms-volatile" },
1449	1470	.{ .name = "fmudflap" },
1450	1471	.{ .name = "fmudflapth" },
	1472	.{ .name = "fnamed-loops" },
1451	1473	.{ .name = "fnative-half-arguments-and-returns" },
1452	1474	.{ .name = "fnative-half-type" },
	1475	.{ .name = "fnative-int16-type" },
1453	1476	.{ .name = "fnested-functions" },
1454	1477	.{ .name = "fnew-infallible" },
1455	1478	.{ .name = "fnext-runtime" },
...	...	@@ -1539,6 +1562,7 @@
1539	1562	.{ .name = "fno-debug-types-section" },
1540	1563	.{ .name = "fno-declspec" },
1541	1564	.{ .name = "fno-default-inline" },
	1565	.{ .name = "fno-defer-ts" },
1542	1566	.{ .name = "fno-define-target-os-macros" },
1543	1567	.{ .name = "fno-delayed-template-parsing" },
1544	1568	.{ .name = "fno-delete-null-pointer-checks" },
...	...	@@ -1573,14 +1597,17 @@
1573	1597	.{ .name = "fno-emulated-tls" },
1574	1598	.{ .name = "fno-escaping-block-tail-calls" },
1575	1599	.{ .name = "fno-exceptions" },
	1600	.{ .name = "fno-experimental-call-graph-section" },
1576	1601	.{ .name = "fno-experimental-isel" },
1577	1602	.{ .name = "fno-experimental-late-parse-attributes" },
1578	1603	.{ .name = "fno-experimental-library" },
	1604	.{ .name = "fno-experimental-loop-fusion" },
1579	1605	.{ .name = "fno-experimental-omit-vtable-rtti" },
1580	1606	.{ .name = "fno-experimental-relative-c++-abi-vtables" },
1581	1607	.{ .name = "fno-external-blas" },
1582	1608	.{ .name = "fno-f2c" },
1583	1609	.{ .name = "fno-fast-math" },
	1610	.{ .name = "fno-fast-real-mod" },
1584	1611	.{ .name = "fno-fat-lto-objects" },
1585	1612	.{ .name = "fno-file-reproducible" },
1586	1613	.{ .name = "fno-implicit-modules-use-lock" },
...	...	@@ -1646,6 +1673,7 @@
1646	1673	.{ .name = "fno-knr-functions" },
1647	1674	.{ .name = "fno-lax-vector-conversions" },
1648	1675	.{ .name = "fno-experimental-lifetime-safety" },
	1676	.{ .name = "fno-experimental-lifetime-safety-inference" },
1649	1677	.{ .name = "fno-limit-debug-info" },
1650	1678	.{ .name = "fno-logical-abbreviations" },
1651	1679	.{ .name = "fno-loop-interchange" },
...	...	@@ -1671,6 +1699,7 @@
1671	1699	.{ .name = "fno-modules-force-validate-user-headers" },
1672	1700	.{ .name = "fno-modules-global-index" },
1673	1701	.{ .name = "fno-modules-prune-non-affecting-module-map-files" },
	1702	.{ .name = "fno-modules-reduced-bmi" },
1674	1703	.{ .name = "fno-modules-search-all" },
1675	1704	.{ .name = "fno-modules-share-filemanager" },
1676	1705	.{ .name = "fno-modules-skip-diagnostic-options" },
...	...	@@ -1685,12 +1714,14 @@
1685	1714	.{ .name = "fno-ms-extensions" },
1686	1715	.{ .name = "fno-ms-tls-guards" },
1687	1716	.{ .name = "fno-ms-volatile" },
	1717	.{ .name = "fno-named-loops" },
1688	1718	.{ .name = "fno-new-infallible" },
1689	1719	.{ .name = "fno-non-call-exceptions" },
1690	1720	.{ .name = "fno-objc-arc" },
1691	1721	.{ .name = "fno-objc-arc-exceptions" },
1692	1722	.{ .name = "fno-objc-avoid-heapify-local-blocks" },
1693	1723	.{ .name = "fno-objc-convert-messages-to-runtime-calls" },
	1724	.{ .name = "fno-objc-direct-precondition-thunk" },
1694	1725	.{ .name = "fno-objc-encode-cxx-class-template-spec" },
1695	1726	.{ .name = "fno-objc-exceptions" },
1696	1727	.{ .name = "fno-objc-infer-related-result-type" },
...	...	@@ -1718,6 +1749,7 @@
1718	1749	.{ .name = "fno-pack-derived" },
1719	1750	.{ .name = "fno-pack-struct" },
1720	1751	.{ .name = "fno-padding-on-unsigned-fixed-point" },
	1752	.{ .name = "fno-partition-static-data-sections" },
1721	1753	.{ .name = "fno-pascal-strings" },
1722	1754	.{ .name = "fno-pch-codegen" },
1723	1755	.{ .name = "fno-pch-debuginfo" },
...	...	@@ -1802,9 +1834,13 @@
1802	1834	.{ .name = "fno-sanitize-address-poison-custom-array-cookie" },
1803	1835	.{ .name = "fno-sanitize-address-use-after-scope" },
1804	1836	.{ .name = "fno-sanitize-address-use-odr-indicator" },
	1837	.{ .name = "fno-sanitize-alloc-token-extended" },
	1838	.{ .name = "fno-sanitize-alloc-token-fast-abi" },
1805	1839	.{ .name = "fno-sanitize-annotate-debug-info" },
1806	1840	.{ .name = "fno-sanitize-cfi-canonical-jump-tables" },
1807	1841	.{ .name = "fno-sanitize-cfi-cross-dso" },
	1842	.{ .name = "fno-sanitize-debug-trap-reasons" },
	1843	.{ .name = "fno-sanitize-handler-preserve-all-regs" },
1808	1844	.{ .name = "fno-sanitize-hwaddress-experimental-aliasing" },
1809	1845	.{ .name = "fno-sanitize-ignorelist" },
1810	1846	.{ .name = "fno-sanitize-link-c++-runtime" },
...	...	@@ -1821,6 +1857,7 @@
1821	1857	.{ .name = "fno-sanitize-thread-func-entry-exit" },
1822	1858	.{ .name = "fno-sanitize-thread-memory-access" },
1823	1859	.{ .name = "fno-sanitize-trap", .ze = .no_sanitize_trap },
	1860	.{ .name = "fno-sanitize-type-outline-instrumentation" },
1824	1861	.{ .name = "fno-sanitize-undefined-trap-on-error" },
1825	1862	.{ .name = "fno-save-main-program" },
1826	1863	.{ .name = "fno-save-optimization-record" },
...	...	@@ -1887,6 +1924,7 @@
1887	1924	.{ .name = "fno-unique-source-file-names" },
1888	1925	.{ .name = "fno-unroll-all-loops" },
1889	1926	.{ .name = "fno-unroll-loops" },
	1927	.{ .name = "fno-unsafe-cray-pointers" },
1890	1928	.{ .name = "fno-unsafe-loop-optimizations" },
1891	1929	.{ .name = "fno-unsafe-math-optimizations" },
1892	1930	.{ .name = "fno-unsigned" },
...	...	@@ -1938,6 +1976,7 @@
1938	1976	.{ .name = "fobjc-avoid-heapify-local-blocks" },
1939	1977	.{ .name = "fobjc-call-cxx-cdtors" },
1940	1978	.{ .name = "fobjc-convert-messages-to-runtime-calls" },
	1979	.{ .name = "fobjc-direct-precondition-thunk" },
1941	1980	.{ .name = "fobjc-disable-direct-methods-for-testing" },
1942	1981	.{ .name = "fobjc-encode-cxx-class-template-spec" },
1943	1982	.{ .name = "fobjc-exceptions" },
...	...	@@ -1987,6 +2026,7 @@
1987	2026	.{ .name = "fpack-struct" },
1988	2027	.{ .name = "fpadding-on-unsigned-fixed-point" },
1989	2028	.{ .name = "fparse-all-comments" },
	2029	.{ .name = "fpartition-static-data-sections" },
1990	2030	.{ .name = "fpascal-strings" },
1991	2031	.{ .name = "fpass-by-value-is-noalias" },
1992	2032	.{ .name = "fpcc-struct-return" },
...	...	@@ -2081,6 +2121,8 @@
2081	2121	.{ .name = "fsanitize-address-poison-custom-array-cookie" },
2082	2122	.{ .name = "fsanitize-address-use-after-scope" },
2083	2123	.{ .name = "fsanitize-address-use-odr-indicator" },
	2124	.{ .name = "fsanitize-alloc-token-extended" },
	2125	.{ .name = "fsanitize-alloc-token-fast-abi" },
2084	2126	.{ .name = "fsanitize-annotate-debug-info" },
2085	2127	.{ .name = "fsanitize-cfi-canonical-jump-tables" },
2086	2128	.{ .name = "fsanitize-cfi-cross-dso" },
...	...	@@ -2102,6 +2144,8 @@
2102	2144	.{ .name = "fsanitize-coverage-trace-pc" },
2103	2145	.{ .name = "fsanitize-coverage-trace-pc-guard", .ze = .san_cov_trace_pc_guard },
2104	2146	.{ .name = "fsanitize-coverage-trace-stores" },
	2147	.{ .name = "fsanitize-debug-trap-reasons" },
	2148	.{ .name = "fsanitize-handler-preserve-all-regs" },
2105	2149	.{ .name = "fsanitize-hwaddress-experimental-aliasing" },
2106	2150	.{ .name = "fsanitize-kcfi-arity" },
2107	2151	.{ .name = "fsanitize-link-c++-runtime" },
...	...	@@ -2118,6 +2162,7 @@
2118	2162	.{ .name = "fsanitize-thread-func-entry-exit" },
2119	2163	.{ .name = "fsanitize-thread-memory-access" },
2120	2164	.{ .name = "fsanitize-trap", .ze = .sanitize_trap },
	2165	.{ .name = "fsanitize-type-outline-instrumentation" },
2121	2166	.{ .name = "fsanitize-undefined-trap-on-error" },
2122	2167	.{ .name = "fsave-main-program" },
2123	2168	.{ .name = "fsave-optimization-record" },
...	...	@@ -2205,6 +2250,7 @@
2205	2250	.{ .name = "funknown-anytype" },
2206	2251	.{ .name = "funroll-all-loops" },
2207	2252	.{ .name = "funroll-loops" },
	2253	.{ .name = "funsafe-cray-pointers" },
2208	2254	.{ .name = "funsafe-loop-optimizations" },
2209	2255	.{ .name = "funsafe-math-optimizations" },
2210	2256	.{ .name = "funsigned" },
...	...	@@ -2259,6 +2305,7 @@
2259	2305	.{ .name = "g2" },
2260	2306	.{ .name = "g3" },
2261	2307	.{ .name = "g", .ze = .debug },
	2308	.{ .name = "gcall-site-info" },
2262	2309	.{ .name = "gcodeview" },
2263	2310	.{ .name = "gcodeview-command-line" },
2264	2311	.{ .name = "gcodeview-ghash" },
...	...	@@ -2271,6 +2318,7 @@
2271	2318	.{ .name = "gdwarf-3", .ze = .debug },
2272	2319	.{ .name = "gdwarf-4", .ze = .debug },
2273	2320	.{ .name = "gdwarf-5", .ze = .debug },
	2321	.{ .name = "gdwarf-6" },
2274	2322	.{ .name = "gdwarf-aranges" },
2275	2323	.{ .name = "gembed-source" },
2276	2324	.{ .name = "gen-cdb-fragment-path", .syntax = .separate },
...	...	@@ -2289,6 +2337,7 @@
2289	2337	.{ .name = "glldb" },
2290	2338	.{ .name = "gmlt" },
2291	2339	.{ .name = "gmodules" },
	2340	.{ .name = "gno-call-site-info" },
2292	2341	.{ .name = "gno-codeview-command-line" },
2293	2342	.{ .name = "gno-codeview-ghash" },
2294	2343	.{ .name = "gno-column-info" },
...	...	@@ -2303,6 +2352,7 @@
2303	2352	.{ .name = "gno-simple-template-names" },
2304	2353	.{ .name = "gno-split-dwarf" },
2305	2354	.{ .name = "gno-strict-dwarf" },
	2355	.{ .name = "gno-structor-decl-linkage-names" },
2306	2356	.{ .name = "gno-template-alias" },
2307	2357	.{ .name = "gomit-unreferenced-methods" },
2308	2358	.{
...	...	@@ -2319,9 +2369,15 @@
2319	2369	.{ .name = "gpulibc" },
2320	2370	.{ .name = "grecord-command-line" },
2321	2371	.{ .name = "gsce" },
	2372	.{
	2373	.name = "gsframe",
	2374	.pd1 = false,
	2375	.pd2 = true,
	2376	},
2322	2377	.{ .name = "gsimple-template-names" },
2323	2378	.{ .name = "gsplit-dwarf" },
2324	2379	.{ .name = "gstrict-dwarf" },
	2380	.{ .name = "gstructor-decl-linkage-names" },
2325	2381	.{ .name = "gtemplate-alias" },
2326	2382	.{ .name = "gtoggle" },
2327	2383	.{ .name = "gused" },
...	...	@@ -2347,6 +2403,7 @@
2347	2403	.pd1 = false,
2348	2404	.pd2 = true,
2349	2405	},
	2406	.{ .name = "hlsl-all-resources-bound", .psl = true },
2350	2407	.{ .name = "ibuiltininc" },
2351	2408	.{ .name = "ignore-pch" },
2352	2409	.{ .name = "image_base", .syntax = .separate },
...	...	@@ -2397,6 +2454,7 @@
2397	2454	.{ .name = "maix-struct-return" },
2398	2455	.{ .name = "malign-double" },
2399	2456	.{ .name = "maltivec", .ze = .m },
	2457	.{ .name = "mamdgpu-expand-waitcnt-profiling" },
2400	2458	.{ .name = "mamdgpu-ieee" },
2401	2459	.{ .name = "mamdgpu-precise-memory-op" },
2402	2460	.{ .name = "mamx-avx512", .ze = .m },
...	...	@@ -2408,7 +2466,6 @@
2408	2466	.{ .name = "mamx-movrs", .ze = .m },
2409	2467	.{ .name = "mamx-tf32", .ze = .m },
2410	2468	.{ .name = "mamx-tile", .ze = .m },
2411		.{ .name = "mamx-transpose", .ze = .m },
2412	2469	.{ .name = "mannotate-tablejump" },
2413	2470	.{ .name = "mapx-inline-asm-use-gpr32" },
2414	2471	.{ .name = "mapxf" },
...	...	@@ -2418,11 +2475,7 @@
2418	2475	.{ .name = "matomics", .ze = .m },
2419	2476	.{ .name = "mavx", .ze = .m },
2420	2477	.{ .name = "mavx10.1" },
2421		.{ .name = "mavx10.1-256" },
2422		.{ .name = "mavx10.1-512", .ze = .m },
2423	2478	.{ .name = "mavx10.2" },
2424		.{ .name = "mavx10.2-256" },
2425		.{ .name = "mavx10.2-512", .ze = .m },
2426	2479	.{ .name = "mavx2", .ze = .m },
2427	2480	.{ .name = "mavx512bf16", .ze = .m },
2428	2481	.{ .name = "mavx512bitalg", .ze = .m },
...	...	@@ -2485,7 +2538,6 @@
2485	2538	.{ .name = "menable-no-nans" },
2486	2539	.{ .name = "menqcmd", .ze = .m },
2487	2540	.{ .name = "metal", .psl = true },
2488		.{ .name = "mevex512", .ze = .m },
2489	2541	.{ .name = "mexception-handling", .ze = .m },
2490	2542	.{ .name = "mexecute-only", .ze = .m },
2491	2543	.{ .name = "mextended-const", .ze = .m },
...	...	@@ -2517,6 +2569,7 @@
2517	2569	.{ .name = "mfsgsbase", .ze = .m },
2518	2570	.{ .name = "mfsmuld" },
2519	2571	.{ .name = "mfxsr", .ze = .m },
	2572	.{ .name = "mgc" },
2520	2573	.{ .name = "mgeneral-regs-only" },
2521	2574	.{ .name = "mgfni", .ze = .m },
2522	2575	.{ .name = "mginv", .ze = .m },
...	...	@@ -2614,6 +2667,7 @@
2614	2667	.{ .name = "mno-adx", .ze = .m },
2615	2668	.{ .name = "mno-aes", .ze = .m },
2616	2669	.{ .name = "mno-altivec", .ze = .m },
	2670	.{ .name = "mno-amdgpu-expand-waitcnt-profiling" },
2617	2671	.{ .name = "mno-amdgpu-ieee" },
2618	2672	.{ .name = "mno-amdgpu-precise-memory-op" },
2619	2673	.{ .name = "mno-amx-avx512", .ze = .m },
...	...	@@ -2625,14 +2679,11 @@
2625	2679	.{ .name = "mno-amx-movrs", .ze = .m },
2626	2680	.{ .name = "mno-amx-tf32", .ze = .m },
2627	2681	.{ .name = "mno-amx-tile", .ze = .m },
2628		.{ .name = "mno-amx-transpose", .ze = .m },
2629	2682	.{ .name = "mno-annotate-tablejump" },
2630	2683	.{ .name = "mno-apxf" },
2631	2684	.{ .name = "mno-atomics", .ze = .m },
2632	2685	.{ .name = "mno-avx", .ze = .m },
2633	2686	.{ .name = "mno-avx10.1" },
2634		.{ .name = "mno-avx10.1-256" },
2635		.{ .name = "mno-avx10.1-512", .ze = .m },
2636	2687	.{ .name = "mno-avx10.2" },
2637	2688	.{ .name = "mno-avx2", .ze = .m },
2638	2689	.{ .name = "mno-avx512bf16", .ze = .m },
...	...	@@ -2682,7 +2733,6 @@
2682	2733	.{ .name = "mno-dspr2", .ze = .m },
2683	2734	.{ .name = "mno-embedded-data" },
2684	2735	.{ .name = "mno-enqcmd", .ze = .m },
2685		.{ .name = "mno-evex512", .ze = .m },
2686	2736	.{ .name = "mno-exception-handling", .ze = .m },
2687	2737	.{ .name = "mnoexecstack" },
2688	2738	.{ .name = "mno-execute-only", .ze = .m },
...	...	@@ -2707,6 +2757,7 @@
2707	2757	.{ .name = "mno-fsmuld", .ze = .m },
2708	2758	.{ .name = "mno-fxsr", .ze = .m },
2709	2759	.{ .name = "mno-gather" },
	2760	.{ .name = "mno-gc" },
2710	2761	.{ .name = "mno-gfni", .ze = .m },
2711	2762	.{ .name = "mno-ginv", .ze = .m },
2712	2763	.{ .name = "mno-global-merge" },
...	...	@@ -2798,6 +2849,7 @@
2798	2849	.{ .name = "mno-relax-all" },
2799	2850	.{ .name = "mno-relax-pic-calls" },
2800	2851	.{ .name = "mno-relaxed-simd", .ze = .m },
	2852	.{ .name = "mno-reserve-frame-pointer-reg" },
2801	2853	.{ .name = "mno-restrict-it" },
2802	2854	.{ .name = "mno-retpoline", .ze = .m },
2803	2855	.{ .name = "mno-retpoline-external-thunk", .ze = .m },
...	...	@@ -2886,7 +2938,6 @@
2886	2938	.{ .name = "module-file-deps" },
2887	2939	.{ .name = "module-file-info" },
2888	2940	.{ .name = "module-suffix", .syntax = .separate },
2889		.{ .name = "fmodules-reduced-bmi" },
2890	2941	.{ .name = "momit-leaf-frame-pointer" },
2891	2942	.{ .name = "moutline" },
2892	2943	.{ .name = "moutline-atomics", .ze = .m },
...	...	@@ -2931,6 +2982,7 @@
2931	2982	.{ .name = "mrelax-relocations=no" },
2932	2983	.{ .name = "mrelaxed-simd", .ze = .m },
2933	2984	.{ .name = "mrelocation-model", .syntax = .separate },
	2985	.{ .name = "mreserve-frame-pointer-reg" },
2934	2986	.{ .name = "mrestrict-it" },
2935	2987	.{ .name = "mretpoline", .ze = .m },
2936	2988	.{ .name = "mretpoline-external-thunk", .ze = .m },
...	...	@@ -3007,6 +3059,7 @@
3007	3059	.{ .name = "mv73", .ze = .m },
3008	3060	.{ .name = "mv75", .ze = .m },
3009	3061	.{ .name = "mv79", .ze = .m },
	3062	.{ .name = "mv81" },
3010	3063	.{ .name = "mv8plus", .ze = .m },
3011	3064	.{ .name = "mvaes", .ze = .m },
3012	3065	.{ .name = "mvector-strict-align" },
...	...	@@ -3101,6 +3154,7 @@
3101	3154	.{ .name = "no-pthread" },
3102	3155	.{ .name = "no-round-trip-args" },
3103	3156	.{ .name = "no-struct-path-tbaa" },
	3157	.{ .name = "no-use-spirv-backend" },
3104	3158	.{
3105	3159	.name = "no-wasm-opt",
3106	3160	.pd1 = false,
...	...	@@ -3311,6 +3365,7 @@
3311	3365	.{ .name = "static-openmp" },
3312	3366	.{ .name = "static-pie" },
3313	3367	.{ .name = "stats-file-append" },
	3368	.{ .name = "stats-file-timers" },
3314	3369	.{ .name = "stdlib" },
3315	3370	.{
3316	3371	.name = "sycl-link",
...	...	@@ -3368,6 +3423,7 @@
3368	3423	.{ .name = "Wrealloc-lhs-all", .pd2 = true },
3369	3424	.{ .name = "Wfrontend-loop-interchange", .pd2 = true },
3370	3425	.{ .name = "Wtarget-lifetime", .pd2 = true },
	3426	.{ .name = "use-spirv-backend" },
3371	3427	.{ .name = "v", .ze = .verbose },
3372	3428	.{ .name = "vectorize-loops" },
3373	3429	.{ .name = "vectorize-slp" },
...	...	@@ -3469,6 +3525,7 @@
3469	3525	.{ .name = "fobjc-nonfragile-abi-version=", .syntax = .joined },
3470	3526	.{ .name = "fprofile-instrument-use-path=", .syntax = .joined },
3471	3527	.{ .name = "fsanitize-coverage-allowlist=", .syntax = .joined },
	3528	.{ .name = "fsanitize-debug-trap-reasons=", .syntax = .joined },
3472	3529	.{ .name = "fxray-instrumentation-bundle=", .syntax = .joined },
3473	3530	.{ .name = "fsanitize-address-destructor=", .syntax = .joined },
3474	3531	.{
...	...	@@ -3567,6 +3624,8 @@
3567	3624	.{ .name = "analyzer-disable-checker=", .syntax = .joined },
3568	3625	.{ .name = "fbuild-session-timestamp=", .syntax = .joined },
3569	3626	.{ .name = "fdo-concurrent-to-openmp=", .syntax = .joined },
	3627	.{ .name = "fdx-rootsignature-define=", .syntax = .joined },
	3628	.{ .name = "fms-layout-compatibility=", .syntax = .joined },
3570	3629	.{ .name = "fprofile-function-groups=", .syntax = .joined },
3571	3630	.{ .name = "fprofile-instrument-path=", .syntax = .joined },
3572	3631	.{ .name = "header-include-filtering=", .syntax = .joined },
...	...	@@ -3589,11 +3648,13 @@
3589	3648	.{ .name = "dump-minimization-hints=", .syntax = .joined },
3590	3649	.{ .name = "fapinotes-swift-version=", .syntax = .joined },
3591	3650	.{ .name = "fcomment-block-commands=", .syntax = .comma_joined },
	3651	.{ .name = "fintrinsic-modules-path=", .syntax = .joined },
3592	3652	.{ .name = "flax-vector-conversions=", .syntax = .joined },
3593	3653	.{ .name = "fmodules-embed-all-files", .syntax = .joined },
3594	3654	.{ .name = "fmodules-prune-interval=", .syntax = .joined },
3595	3655	.{ .name = "foverride-record-layout=", .syntax = .joined },
3596	3656	.{ .name = "fprofile-instr-generate=", .syntax = .joined },
	3657	.{ .name = "fprofile-instrument-use=", .syntax = .joined },
3597	3658	.{ .name = "fprofile-remapping-file=", .syntax = .joined },
3598	3659	.{ .name = "fsanitize-coverage-type=", .syntax = .joined },
3599	3660	.{ .name = "fsanitize-hwaddress-abi=", .syntax = .joined },
...	...	@@ -3619,6 +3680,7 @@
3619	3680	.{ .name = "fdebug-compilation-dir=", .syntax = .joined },
3620	3681	.{ .name = "fdebug-default-version=", .syntax = .joined },
3621	3682	.{ .name = "ffp-exception-behavior=", .syntax = .joined },
	3683	.{ .name = "finitial-counter-value=", .syntax = .joined },
3622	3684	.{ .name = "fmacro-backtrace-limit=", .syntax = .joined },
3623	3685	.{ .name = "fmax-array-constructor=", .syntax = .joined },
3624	3686	.{ .name = "fmcdc-max-test-vectors=", .syntax = .joined },
...	...	@@ -3640,6 +3702,7 @@
3640	3702	.{ .name = "ffile-compilation-dir=", .syntax = .joined },
3641	3703	.{ .name = "fgpu-inline-threshold=", .syntax = .joined },
3642	3704	.{ .name = "finline-max-stacksize=", .syntax = .joined },
	3705	.{ .name = "fmatrix-memory-layout=", .syntax = .joined },
3643	3706	.{ .name = "fmax-subrecord-length=", .syntax = .joined },
3644	3707	.{ .name = "fmodules-ignore-macro=", .syntax = .joined },
3645	3708	.{
...	...	@@ -3732,6 +3795,7 @@
3732	3795	.{ .name = "fms-omit-default-lib", .syntax = .joined },
3733	3796	.{ .name = "fprofile-instrument=", .syntax = .joined },
3734	3797	.{ .name = "fprofile-sample-use=", .syntax = .joined },
	3798	.{ .name = "fsanitize-kcfi-hash=", .syntax = .joined },
3735	3799	.{ .name = "fstrict-flex-arrays=", .syntax = .joined },
3736	3800	.{
3737	3801	.name = "hipstdpar-prim-path=",
...	...	@@ -3817,6 +3881,7 @@
3817	3881	.pd1 = false,
3818	3882	.pd2 = true,
3819	3883	},
	3884	.{ .name = "falloc-token-mode=", .syntax = .joined },
3820	3885	.{ .name = "fbinutils-version=", .syntax = .joined },
3821	3886	.{ .name = "fclang-abi-compat=", .syntax = .joined },
3822	3887	.{ .name = "fcodegen-data-use=", .syntax = .joined },
...	...	@@ -3865,6 +3930,7 @@
3865	3930	.pd2 = true,
3866	3931	},
3867	3932	.{ .name = "falign-functions=", .syntax = .joined },
	3933	.{ .name = "falloc-token-max=", .syntax = .joined },
3868	3934	.{ .name = "fconstexpr-depth=", .syntax = .joined },
3869	3935	.{ .name = "fconstexpr-steps=", .syntax = .joined },
3870	3936	.{ .name = "ffile-prefix-map=", .syntax = .joined },
...	...	@@ -3951,7 +4017,7 @@
3951	4017	.{ .name = "mzos-hlq-csslib=", .syntax = .joined },
3952	4018	.{
3953	4019	.name = "no-offload-arch=",
3954		.syntax = .joined,
	4020	.syntax = .comma_joined,
3955	4021	.pd1 = false,
3956	4022	.pd2 = true,
3957	4023	},
...	...	@@ -4296,6 +4362,12 @@
4296	4362	.{ .name = "inline-asm=", .syntax = .joined },
4297	4363	.{ .name = "ivfsoverlay", .syntax = .joined_or_separate },
4298	4364	.{ .name = "iwithprefix", .syntax = .joined_or_separate },
	4365	.{
	4366	.name = "libclc-lib=",
	4367	.syntax = .joined,
	4368	.pd1 = false,
	4369	.pd2 = true,
	4370	},
4299	4371	.{ .name = "mfloat-abi=", .syntax = .joined },
4300	4372	.{ .name = "plugin-arg-", .syntax = .joined_and_separate },
4301	4373	.{
...	...	@@ -4457,7 +4529,6 @@
4457	4529	.pd1 = false,
4458	4530	.pd2 = true,
4459	4531	},
4460		.{ .name = "fcoarray=", .syntax = .joined },
4461	4532	.{ .name = "fconvert=", .syntax = .joined },
4462	4533	.{ .name = "fc++-abi=", .syntax = .joined },
4463	4534	.{ .name = "fextdirs=", .syntax = .joined },
...	...	@@ -4782,6 +4853,11 @@
4782	4853	.syntax = .joined,
4783	4854	.psl = true,
4784	4855	},
	4856	.{
	4857	.name = "Frs",
	4858	.syntax = .joined_or_separate,
	4859	.psl = true,
	4860	},
4785	4861	.{ .name = "gz=", .syntax = .joined },
4786	4862	.{ .name = "A-", .syntax = .joined },
4787	4863	.{ .name = "G=", .syntax = .joined },

src/codegen/c.zig+19-1

...	...	@@ -3497,9 +3497,27 @@ fn airOverflow(f: *Function, inst: Air.Inst.Index, operation: []const u8, info:
3497	3497	try w.writeAll(operation);
3498	3498	try w.writeAll("o_");
3499	3499	try f.dg.renderTypeForBuiltinFnName(w, scalar_ty);
3500		try w.writeAll("(&");
	3500	try w.writeByte('(');
	3501
	3502	// '&dest', possibly preceded by a cast
	3503	switch (zcu.intern_pool.indexToKey(scalar_ty.toIntern())) {
	3504	.int_type => {}, // we already have a '[u]intX_t *'
	3505	.simple_type => {
	3506	// '&dest' will be something like a 'uintptr_t *', which might be a different C type to
	3507	// the equivalent sized integer (e.g. 'uint64_t *'), so we need a cast. We don't need a
	3508	// cast on the operands because they are passed by value (except for big integers,
	3509	// where this issue doesn't exist because no "simple" int type needs bigint repr).
	3510	try w.print("({s}int{d}_t *)", .{
	3511	if (scalar_ty.isUnsignedInt(zcu)) "u" else "",
	3512	scalar_ty.abiSize(zcu) * 8,
	3513	});
	3514	},
	3515	else => unreachable,
	3516	}
	3517	try w.writeByte('&');
3501	3518	try f.writeCValueMember(w, local, .{ .field = 0 });
3502	3519	try v.elem(f, w);
	3520
3503	3521	try w.writeAll(", ");
3504	3522	if (ref_arg) try w.writeByte('&');
3505	3523	try f.writeCValue(w, lhs, .other);

src/codegen/llvm.zig+15-35

...	...	@@ -162,6 +162,7 @@ pub fn targetTriple(allocator: Allocator, target: *const std.Target) ![]const u8
162	162	.{ .v9_4a, "v9.4a" },
163	163	.{ .v9_5a, "v9.5a" },
164	164	.{ .v9_6a, "v9.6a" },
	165	.{ .v9_7a, "v9.7a" },
165	166	}),
166	167	.powerpc => subArchName(target, .powerpc, .{
167	168	.{ .spe, "spe" },
...	...	@@ -255,11 +256,15 @@ pub fn targetTriple(allocator: Allocator, target: *const std.Target) ![]const u8
255	256	.none,
256	257	.windows,
257	258	=> {},
258		.semver => \|ver\| try llvm_triple.print("{d}.{d}.{d}", .{
259		ver.min.major,
260		ver.min.minor,
261		ver.min.patch,
262		}),
	259	.semver => \|ver\| if (target.os.tag == .wasi and ver.min.major == 0) {
	260	try llvm_triple.print("p{d}", .{ver.min.minor});
	261	} else {
	262	try llvm_triple.print("{d}.{d}.{d}", .{
	263	ver.min.major,
	264	ver.min.minor,
	265	ver.min.patch,
	266	});
	267	},
263	268	inline .linux, .hurd => \|ver\| try llvm_triple.print("{d}.{d}.{d}", .{
264	269	ver.range.min.major,
265	270	ver.range.min.minor,
...	...	@@ -383,16 +388,9 @@ pub fn dataLayout(target: *const std.Target) []const u8 {
383	388	.powerpc => "E-m:e-p:32:32-Fn32-i64:64-n32",
384	389	.powerpcle => "e-m:e-p:32:32-Fn32-i64:64-n32",
385	390	.powerpc64 => switch (target.os.tag) {
386		.linux => if (target.abi.isMusl())
387		"E-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
388		else
389		"E-m:e-Fi64-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512",
	391	.linux => "E-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512",
390	392	.ps3 => "E-m:e-p:32:32-Fi64-i64:64-i128:128-n32:64",
391		else => if (target.os.tag == .openbsd or
392		(target.os.tag == .freebsd and target.os.version_range.semver.isAtLeast(.{ .major = 13, .minor = 0, .patch = 0 }) orelse false))
393		"E-m:e-Fn32-i64:64-i128:128-n32:64"
394		else
395		"E-m:e-Fi64-i64:64-i128:128-n32:64",
	393	else => "E-m:e-Fn32-i64:64-i128:128-n32:64",
396	394	},
397	395	.powerpc64le => if (target.os.tag == .linux)
398	396	"e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
...	...	@@ -400,7 +398,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 {
400	398	"e-m:e-Fn32-i64:64-i128:128-n32:64",
401	399	.nvptx => "e-p:32:32-p6:32:32-p7:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64",
402	400	.nvptx64 => "e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64",
403		.amdgcn => "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9",
	401	.amdgcn => "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9",
404	402	.riscv32 => if (target.cpu.has(.riscv, .e))
405	403	"e-m:e-p:32:32-i64:64-n32-S32"
406	404	else
...	...	@@ -1748,6 +1746,7 @@ pub const Object = struct {
1748	1746	if (name.eqlSlice("WinMainCRTStartup", ip)) flags.winmain_crt_startup = true;
1749	1747	if (name.eqlSlice("wWinMainCRTStartup", ip)) flags.wwinmain_crt_startup = true;
1750	1748	if (name.eqlSlice("DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;
	1749	if (name.eqlSlice("_DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;
1751	1750	}
1752	1751	}
1753	1752
...	...	@@ -4652,20 +4651,6 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ
4652	4651	/// or if it produces miscompilations.
4653	4652	pub fn backendSupportsF16(target: *const std.Target) bool {
4654	4653	return switch (target.cpu.arch) {
4655		// https://github.com/llvm/llvm-project/issues/97981
4656		.csky,
4657		// https://github.com/llvm/llvm-project/issues/97981
4658		.powerpc,
4659		.powerpcle,
4660		.powerpc64,
4661		.powerpc64le,
4662		// https://github.com/llvm/llvm-project/issues/97981
4663		.wasm32,
4664		.wasm64,
4665		// https://github.com/llvm/llvm-project/issues/97981
4666		.sparc,
4667		.sparc64,
4668		=> false,
4669	4654	.arm,
4670	4655	.armeb,
4671	4656	.thumb,
...	...	@@ -4692,11 +4677,6 @@ pub fn backendSupportsF128(target: *const std.Target) bool {
4692	4677	return switch (target.cpu.arch) {
4693	4678	// https://github.com/llvm/llvm-project/issues/121122
4694	4679	.amdgcn,
4695		// Test failures all over the place.
4696		.mips64,
4697		.mips64el,
4698		// https://github.com/llvm/llvm-project/issues/41838
4699		.sparc,
4700	4680	=> false,
4701	4681	.arm,
4702	4682	.armeb,
...	...	@@ -4839,7 +4819,7 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
4839	4819	bindings.LLVMInitializeXtensaTarget();
4840	4820	bindings.LLVMInitializeXtensaTargetInfo();
4841	4821	bindings.LLVMInitializeXtensaTargetMC();
4842		// There is no LLVMInitializeXtensaAsmPrinter function.
	4822	bindings.LLVMInitializeXtensaAsmPrinter();
4843	4823	bindings.LLVMInitializeXtensaAsmParser();
4844	4824	}
4845	4825	},

src/codegen/llvm/FuncGen.zig+4-4

...	...	@@ -6630,11 +6630,11 @@ const ParamTypeIterator = struct {
6630	6630	} else if (isByRef(ty, zcu)) {
6631	6631	return .byref;
6632	6632	} else if (target.cpu.arch.isX86() and
6633		!target.cpu.has(.x86, .evex512) and
	6633	!target.cpu.has(.x86, .avx512f) and
6634	6634	ty.totalVectorBits(zcu) >= 512)
6635	6635	{
6636	6636	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6637		// "512-bit vector arguments require 'evex512' for AVX512"
	6637	// "512-bit vector arguments require 'avx512f' for AVX512"
6638	6638	return .byref;
6639	6639	} else {
6640	6640	return .byval;
...	...	@@ -6903,11 +6903,11 @@ fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
6903	6903	if (isByRef(ty, zcu)) {
6904	6904	return true;
6905	6905	} else if (target.cpu.arch.isX86() and
6906		!target.cpu.has(.x86, .evex512) and
	6906	!target.cpu.has(.x86, .avx512f) and
6907	6907	ty.totalVectorBits(zcu) >= 512)
6908	6908	{
6909	6909	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6910		// "512-bit vector arguments require 'evex512' for AVX512"
	6910	// "512-bit vector arguments require 'avx512f' for AVX512"
6911	6911	return true;
6912	6912	} else {
6913	6913	return false;

src/codegen/llvm/bindings.zig+1

...	...	@@ -279,6 +279,7 @@ pub extern fn LLVMInitializeSystemZAsmPrinter() void;
279	279	pub extern fn LLVMInitializeWebAssemblyAsmPrinter() void;
280	280	pub extern fn LLVMInitializeX86AsmPrinter() void;
281	281	pub extern fn LLVMInitializeXCoreAsmPrinter() void;
	282	pub extern fn LLVMInitializeXtensaAsmPrinter() void;
282	283	pub extern fn LLVMInitializeM68kAsmPrinter() void;
283	284	pub extern fn LLVMInitializeVEAsmPrinter() void;
284	285	pub extern fn LLVMInitializeARCAsmPrinter() void;

src/libs/libcxx.zig+14-5

...	...	@@ -538,11 +538,20 @@ pub fn addCxxArgs(
538	538	// Compilation.addCCArgs. This option makes it use serial backend which
539	539	// is simple and works everywhere.
540	540	try cflags.append("-D_LIBCPP_PSTL_BACKEND_SERIAL");
541		try cflags.append(switch (optimize_mode) {
542		.Debug => "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG",
543		.ReleaseFast, .ReleaseSmall => "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE",
544		.ReleaseSafe => "-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST",
545		});
	541	switch (optimize_mode) {
	542	.Debug => {
	543	try cflags.append("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG");
	544	try cflags.append("-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE");
	545	},
	546	.ReleaseFast, .ReleaseSmall => {
	547	try cflags.append("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_NONE");
	548	try cflags.append("-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_IGNORE");
	549	},
	550	.ReleaseSafe => {
	551	try cflags.append("-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST");
	552	try cflags.append("-D_LIBCPP_ASSERTION_SEMANTIC_DEFAULT=_LIBCPP_ASSERTION_SEMANTIC_ENFORCE");
	553	},
	554	}
546	555	if (target.isGnuLibC()) {
547	556	// glibc 2.16 introduced aligned_alloc
548	557	if (target.os.versionRange().gnuLibCVersion().?.order(.{ .major = 2, .minor = 16, .patch = 0 }) == .lt) {

src/link/Lld.zig+3-12

...	...	@@ -637,17 +637,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void {
637	637	try argv.append("-ALTERNATENAME:__image_base__=__ImageBase");
638	638	}
639	639
640		if (is_dyn_lib) {
641		try argv.append(try comp.crtFileAsString(arena, "dllcrt2.obj"));
642		if (target.cpu.arch == .x86) {
643		try argv.append("-ALTERNATENAME:__DllMainCRTStartup@12=_DllMainCRTStartup@12");
644		} else {
645		try argv.append("-ALTERNATENAME:_DllMainCRTStartup=DllMainCRTStartup");
646		}
647		} else {
648		try argv.append(try comp.crtFileAsString(arena, "crt2.obj"));
649		}
650
	640	try argv.append(try comp.crtFileAsString(arena, if (is_dyn_lib) "dllcrt2.obj" else "crt2.obj"));
651	641	try argv.append(try comp.crtFileAsString(arena, "libmingw32.lib"));
652	642	} else {
653	643	try argv.append(switch (comp.config.link_mode) {
...	...	@@ -812,7 +802,8 @@ fn elfLink(lld: *Lld, arena: Allocator) !void {
812	802	target.cpu.arch == .m68k or
813	803	target.cpu.arch.isSPARC() or
814	804	target.cpu.arch == .ve or
815		target.cpu.arch == .xcore))
	805	target.cpu.arch == .xcore or
	806	target.cpu.arch == .xtensa))
816	807	{
817	808	// In this case we must do a simple file copy
818	809	// here. TODO: think carefully about how we can avoid this redundant operation when doing

src/link/Wasm.zig+3

...	...	@@ -2829,6 +2829,7 @@ pub const Feature = packed struct(u8) {
2829	2829	@"exception-handling",
2830	2830	@"extended-const",
2831	2831	fp16,
	2832	gc,
2832	2833	memory64,
2833	2834	multimemory,
2834	2835	multivalue,
...	...	@@ -2852,6 +2853,7 @@ pub const Feature = packed struct(u8) {
2852	2853	.exception_handling => .@"exception-handling",
2853	2854	.extended_const => .@"extended-const",
2854	2855	.fp16 => .fp16,
	2856	.gc => .gc,
2855	2857	.multimemory => .multimemory,
2856	2858	.multivalue => .multivalue,
2857	2859	.mutable_globals => .@"mutable-globals",
...	...	@@ -2875,6 +2877,7 @@ pub const Feature = packed struct(u8) {
2875	2877	.@"exception-handling" => .exception_handling,
2876	2878	.@"extended-const" => .extended_const,
2877	2879	.fp16 => .fp16,
	2880	.gc => .gc,
2878	2881	.memory64 => null, // Linker-only feature.
2879	2882	.multimemory => .multimemory,
2880	2883	.multivalue => .multivalue,

src/main.zig+5-2

...	...	@@ -977,6 +977,11 @@ fn buildOutputType(
977	977	var cc_argv: std.ArrayList([]const u8) = .empty;
978	978	var deps: std.ArrayList(CliModule.Dep) = .empty;
979	979
	980	// We need to raise the FD limit before CLI parsing, because we open link inputs during CLI
	981	// parsing (in `createModule`), so a large number of link inputs could push us past the limit on
	982	// targets with a low soft limit (e.g. macOS has a default limit of 256).
	983	process.raiseFileDescriptorLimit();
	984
980	985	// Contains every module specified via -M. The dependencies are added
981	986	// after argument parsing is completed. We use a StringArrayHashMap to make
982	987	// error output consistent. "root" is special.
...	...	@@ -3534,8 +3539,6 @@ fn buildOutputType(
3534	3539	break :b .whole;
3535	3540	};
3536	3541
3537		process.raiseFileDescriptorLimit();
3538
3539	3542	var file_system_inputs: std.ArrayList(u8) = .empty;
3540	3543	defer file_system_inputs.deinit(gpa);
3541	3544

src/target.zig+5-3

...	...	@@ -227,11 +227,11 @@ pub fn hasLlvmSupport(target: *const std.Target, ofmt: std.Target.ObjectFormat)
227	227	.wasm32,
228	228	.wasm64,
229	229	.ve,
	230	.xtensa,
230	231	=> true,
231	232
232	233	// LLVM backend exists but can produce neither assembly nor object files.
233	234	.csky,
234		.xtensa,
235	235	=> false,
236	236
237	237	// Third-party LLVM backend exists.
...	...	@@ -388,6 +388,8 @@ pub fn hasDebugInfo(target: *const std.Target) bool {
388	388	.ptx85,
389	389	.ptx86,
390	390	.ptx87,
	391	.ptx88,
	392	.ptx90,
391	393	}),
392	394	.bpfel, .bpfeb => false,
393	395	else => true,
...	...	@@ -684,8 +686,8 @@ pub fn llvmMachineAbi(target: *const std.Target) ?[:0]const u8 {
684	686	else => "lp64d",
685	687	},
686	688	.loongarch32 => switch (target.abi) {
687		.gnusf => "ilp32s",
688		.gnuf32 => "ilp32f",
	689	.gnusf, .muslsf => "ilp32s",
	690	.gnuf32, .muslf32 => "ilp32f",
689	691	else => "ilp32d",
690	692	},
691	693	.mips, .mipsel => "o32",

src/zig_clang_cc1_main.cpp+26-27

...	...	@@ -12,19 +12,20 @@
12	12	//
13	13	//===----------------------------------------------------------------------===//
14	14
	15	#include "clang/Basic/DiagnosticFrontend.h"
15	16	#include "clang/Basic/Stack.h"
16	17	#include "clang/Basic/TargetOptions.h"
17	18	#include "clang/CodeGen/ObjectFilePCHContainerWriter.h"
18	19	#include "clang/Config/config.h"
	20	#include "clang/Driver/Driver.h"
19	21	#include "clang/Driver/DriverDiagnostic.h"
20		#include "clang/Driver/Options.h"
21	22	#include "clang/Frontend/CompilerInstance.h"
22	23	#include "clang/Frontend/CompilerInvocation.h"
23		#include "clang/Frontend/FrontendDiagnostic.h"
24	24	#include "clang/Frontend/TextDiagnosticBuffer.h"
25	25	#include "clang/Frontend/TextDiagnosticPrinter.h"
26	26	#include "clang/Frontend/Utils.h"
27	27	#include "clang/FrontendTool/Utils.h"
	28	#include "clang/Options/Options.h"
28	29	#include "clang/Serialization/ObjectFilePCHContainerReader.h"
29	30	#include "llvm/ADT/Statistic.h"
30	31	#include "llvm/ADT/StringExtras.h"
...	...	@@ -38,6 +39,7 @@
38	39	#include "llvm/Support/BuryPointer.h"
39	40	#include "llvm/Support/Compiler.h"
40	41	#include "llvm/Support/ErrorHandling.h"
	42	#include "llvm/Support/IOSandbox.h"
41	43	#include "llvm/Support/ManagedStatic.h"
42	44	#include "llvm/Support/Path.h"
43	45	#include "llvm/Support/Process.h"
...	...	@@ -217,7 +219,7 @@ static int PrintEnabledExtensions(const TargetOptions& TargetOpts) {
217	219	int cc1_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
218	220	ensureSufficientStack();
219	221
220		IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
	222	IntrusiveRefCntPtr<DiagnosticIDs> DiagID = DiagnosticIDs::create();
221	223
222	224	// Register the support for object-file-wrapped Clang modules.
223	225	auto PCHOps = std::make_shared<PCHContainerOperations>();
...	...	@@ -269,12 +271,17 @@ int cc1_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
269	271	if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
270	272	Clang->getHeaderSearchOpts().ResourceDir.empty())
271	273	Clang->getHeaderSearchOpts().ResourceDir =
272		CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
	274	GetResourcesPath(Argv0, MainAddr);
	275
	276	/// Create the actual file system.
	277	auto VFS = [] {
	278	auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
	279	return llvm::vfs::getRealFileSystem();
	280	}();
	281	Clang->createVirtualFileSystem(std::move(VFS), DiagsBuffer);
273	282
274	283	// Create the actual diagnostics engine.
275		Clang->createDiagnostics(*llvm::vfs::getRealFileSystem());
276		if (!Clang->hasDiagnostics())
277		return 1;
	284	Clang->createDiagnostics();
278	285
279	286	// Set an error handler, so that any LLVM backend diagnostics go through our
280	287	// error handler.
...	...	@@ -299,29 +306,21 @@ int cc1_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
299	306
300	307	// If any timers were active but haven't been destroyed yet, print their
301	308	// results now. This happens in -disable-free mode.
302		std::unique_ptr<raw_ostream> IOFile = llvm::CreateInfoOutputFile();
303		if (Clang->getCodeGenOpts().TimePassesJson) {
304		*IOFile << "{\n";
305		llvm::TimerGroup::printAllJSONValues(*IOFile, "");
306		*IOFile << "\n}\n";
307		} else {
308		llvm::TimerGroup::printAll(*IOFile);
	309	{
	310	// This isn't a formal input or output of the compiler.
	311	auto BypassSandbox = llvm::sys::sandbox::scopedDisable();
	312	std::unique_ptr<raw_ostream> IOFile = llvm::CreateInfoOutputFile();
	313	if (Clang->getCodeGenOpts().TimePassesJson) {
	314	*IOFile << "{\n";
	315	llvm::TimerGroup::printAllJSONValues(*IOFile, "");
	316	*IOFile << "\n}\n";
	317	} else if (!Clang->getCodeGenOpts().TimePassesStatsFile) {
	318	llvm::TimerGroup::printAll(*IOFile);
	319	}
	320	llvm::TimerGroup::clearAll();
309	321	}
310		llvm::TimerGroup::clearAll();
311	322
312	323	if (llvm::timeTraceProfilerEnabled()) {
313		// It is possible that the compiler instance doesn't own a file manager here
314		// if we're compiling a module unit. Since the file manager are owned by AST
315		// when we're compiling a module unit. So the file manager may be invalid
316		// here.
317		//
318		// It should be fine to create file manager here since the file system
319		// options are stored in the compiler invocation and we can recreate the VFS
320		// from the compiler invocation.
321		if (!Clang->hasFileManager())
322		Clang->createFileManager(createVFSFromCompilerInvocation(
323		Clang->getInvocation(), Clang->getDiagnostics()));
324
325	324	if (auto profilerOutput = Clang->createOutputFile(
326	325	Clang->getFrontendOpts().TimeTracePath, /Binary=/false,
327	326	/RemoveFileOnSignal=/false,

src/zig_clang_cc1as_main.cpp+48-27

...	...	@@ -12,12 +12,12 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "clang/Basic/Diagnostic.h"
	15	#include "clang/Basic/DiagnosticFrontend.h"
15	16	#include "clang/Basic/DiagnosticOptions.h"
16	17	#include "clang/Driver/DriverDiagnostic.h"
17		#include "clang/Driver/Options.h"
18		#include "clang/Frontend/FrontendDiagnostic.h"
19	18	#include "clang/Frontend/TextDiagnosticPrinter.h"
20	19	#include "clang/Frontend/Utils.h"
	20	#include "clang/Options/Options.h"
21	21	#include "llvm/ADT/STLExtras.h"
22	22	#include "llvm/ADT/StringExtras.h"
23	23	#include "llvm/ADT/StringSwitch.h"
...	...	@@ -45,6 +45,7 @@
45	45	#include "llvm/Support/ErrorHandling.h"
46	46	#include "llvm/Support/FileSystem.h"
47	47	#include "llvm/Support/FormattedStream.h"
	48	#include "llvm/Support/IOSandbox.h"
48	49	#include "llvm/Support/MemoryBuffer.h"
49	50	#include "llvm/Support/Path.h"
50	51	#include "llvm/Support/Process.h"
...	...	@@ -59,8 +60,7 @@
59	60	#include <optional>
60	61	#include <system_error>
61	62	using namespace clang;
62		using namespace clang::driver;
63		using namespace clang::driver::options;
	63	using namespace clang::options;
64	64	using namespace llvm;
65	65	using namespace llvm::opt;
66	66
...	...	@@ -71,8 +71,8 @@ struct AssemblerInvocation {
71	71	/// @name Target Options
72	72	/// @{
73	73
74		/// The name of the target triple to assemble for.
75		std::string Triple;
	74	/// The target triple to assemble for.
	75	llvm::Triple Triple;
76	76
77	77	/// If given, the name of the target CPU to determine which instructions
78	78	/// are legal.
...	...	@@ -163,6 +163,10 @@ struct AssemblerInvocation {
163	163	LLVM_PREFERRED_TYPE(bool)
164	164	unsigned EmitCompactUnwindNonCanonical : 1;
165	165
	166	// Whether to emit sframe unwind sections.
	167	LLVM_PREFERRED_TYPE(bool)
	168	unsigned EmitSFrameUnwind : 1;
	169
166	170	LLVM_PREFERRED_TYPE(bool)
167	171	unsigned Crel : 1;
168	172	LLVM_PREFERRED_TYPE(bool)
...	...	@@ -192,9 +196,12 @@ struct AssemblerInvocation {
192	196	std::string AsSecureLogFile;
193	197	/// @}
194	198
	199	void setTriple(llvm::StringRef Str) {
	200	Triple = llvm::Triple(llvm::Triple::normalize(Str));
	201	}
	202
195	203	public:
196	204	AssemblerInvocation() {
197		Triple = "";
198	205	NoInitialTextSection = 0;
199	206	InputFile = "-";
200	207	OutputPath = "-";
...	...	@@ -261,7 +268,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
261	268	// Construct the invocation.
262	269
263	270	// Target Options
264		Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
	271	Opts.setTriple(Args.getLastArgValue(OPT_triple));
265	272	if (Arg *A = Args.getLastArg(options::OPT_darwin_target_variant_triple))
266	273	Opts.DarwinTargetVariantTriple = llvm::Triple(A->getValue());
267	274	if (Arg *A = Args.getLastArg(OPT_darwin_target_variant_sdk_version_EQ)) {
...	...	@@ -278,7 +285,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
278	285
279	286	// Use the default target triple if unspecified.
280	287	if (Opts.Triple.empty())
281		Opts.Triple = llvm::sys::getDefaultTargetTriple();
	288	Opts.setTriple(llvm::sys::getDefaultTargetTriple());
282	289
283	290	// Language Options
284	291	Opts.IncludePaths = Args.getAllArgValues(OPT_I);
...	...	@@ -385,6 +392,7 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
385	392
386	393	Opts.EmitCompactUnwindNonCanonical =
387	394	Args.hasArg(OPT_femit_compact_unwind_non_canonical);
	395	Opts.EmitSFrameUnwind = Args.hasArg(OPT_gsframe);
388	396	Opts.Crel = Args.hasArg(OPT_crel);
389	397	Opts.ImplicitMapsyms = Args.hasArg(OPT_mmapsyms_implicit);
390	398	Opts.X86RelaxRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
...	...	@@ -414,15 +422,19 @@ getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) {
414	422	}
415	423
416	424	static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
417		DiagnosticsEngine &Diags) {
	425	DiagnosticsEngine &Diags,
	426	IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
418	427	// Get the target specific parser.
419	428	std::string Error;
420	429	const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
421	430	if (!TheTarget)
422		return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
	431	return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple.str();
423	432
424		ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
425		MemoryBuffer::getFileOrSTDIN(Opts.InputFile, /IsText=/true);
	433	ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer = [&] {
	434	// FIXME(sandboxing): Make this a proper input file.
	435	auto BypassSandbox = sys::sandbox::scopedDisable();
	436	return MemoryBuffer::getFileOrSTDIN(Opts.InputFile, /IsText=/true);
	437	}();
426	438
427	439	if (std::error_code EC = Buffer.getError()) {
428	440	return Diags.Report(diag::err_fe_error_reading)
...	...	@@ -437,6 +449,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
437	449	// Record the location of the include directories so that the lexer can find
438	450	// it later.
439	451	SrcMgr.setIncludeDirs(Opts.IncludePaths);
	452	SrcMgr.setVirtualFileSystem(VFS);
440	453
441	454	std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
442	455	assert(MRI && "Unable to create target register info!");
...	...	@@ -445,11 +458,13 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
445	458	MCOptions.MCRelaxAll = Opts.RelaxAll;
446	459	MCOptions.EmitDwarfUnwind = Opts.EmitDwarfUnwind;
447	460	MCOptions.EmitCompactUnwindNonCanonical = Opts.EmitCompactUnwindNonCanonical;
	461	MCOptions.EmitSFrameUnwind = Opts.EmitSFrameUnwind;
448	462	MCOptions.MCSaveTempLabels = Opts.SaveTemporaryLabels;
449	463	MCOptions.Crel = Opts.Crel;
450	464	MCOptions.ImplicitMapSyms = Opts.ImplicitMapsyms;
451	465	MCOptions.X86RelaxRelocations = Opts.X86RelaxRelocations;
452	466	MCOptions.X86Sse2Avx = Opts.X86Sse2Avx;
	467	MCOptions.MCNoExecStack = Opts.NoExecStack;
453	468	MCOptions.CompressDebugSections = Opts.CompressDebugSections;
454	469	MCOptions.AsSecureLogFile = Opts.AsSecureLogFile;
455	470
...	...	@@ -477,7 +492,10 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
477	492
478	493	std::unique_ptr<MCSubtargetInfo> STI(
479	494	TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
480		assert(STI && "Unable to create subtarget info!");
	495	if (!STI) {
	496	return Diags.Report(diag::err_fe_unable_to_create_subtarget)
	497	<< Opts.CPU << FS.empty() << FS;
	498	}
481	499
482	500	MCContext Ctx(Triple(Opts.Triple), MAI.get(), MRI.get(), STI.get(), &SrcMgr,
483	501	&MCOptions);
...	...	@@ -509,9 +527,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
509	527	Ctx.setCompilationDir(Opts.DebugCompilationDir);
510	528	else {
511	529	// If no compilation dir is set, try to use the current directory.
512		SmallString<128> CWD;
513		if (!sys::fs::current_path(CWD))
514		Ctx.setCompilationDir(CWD);
	530	if (auto CWD = VFS->getCurrentWorkingDirectory())
	531	Ctx.setCompilationDir(*CWD);
515	532	}
516	533	if (!Opts.DebugPrefixMap.empty())
517	534	for (const auto &KV : Opts.DebugPrefixMap)
...	...	@@ -577,7 +594,6 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
577	594	Triple T(Opts.Triple);
578	595	Str.reset(TheTarget->createMCObjectStreamer(
579	596	T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI));
580		Str->initSections(Opts.NoExecStack, *STI);
581	597	if (T.isOSBinFormatMachO() && T.isOSDarwin()) {
582	598	Triple *TVT = Opts.DarwinTargetVariantTriple
583	599	? &*Opts.DarwinTargetVariantTriple
...	...	@@ -605,7 +621,7 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
605	621	std::unique_ptr<MCTargetAsmParser> TAP(
606	622	TheTarget->createMCAsmParser(STI, Parser, *MCII, MCOptions));
607	623	if (!TAP)
608		Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
	624	Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple.str();
609	625
610	626	// Set values for symbols, if any.
611	627	for (auto &S : Opts.SymbolDefs) {
...	...	@@ -627,8 +643,9 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
627	643	}
628	644
629	645	static bool ExecuteAssembler(AssemblerInvocation &Opts,
630		DiagnosticsEngine &Diags) {
631		bool Failed = ExecuteAssemblerImpl(Opts, Diags);
	646	DiagnosticsEngine &Diags,
	647	IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
	648	bool Failed = ExecuteAssemblerImpl(Opts, Diags, VFS);
632	649
633	650	// Delete output file if there were errors.
634	651	if (Failed) {
...	...	@@ -662,8 +679,12 @@ int cc1as_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
662	679	TextDiagnosticPrinter *DiagClient =
663	680	new TextDiagnosticPrinter(errs(), DiagOpts);
664	681	DiagClient->setPrefix("clang -cc1as");
665		IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
666		DiagnosticsEngine Diags(DiagID, DiagOpts, DiagClient);
	682	DiagnosticsEngine Diags(DiagnosticIDs::create(), DiagOpts, DiagClient);
	683
	684	auto VFS = [] {
	685	auto BypassSandbox = sys::sandbox::scopedDisable();
	686	return vfs::getRealFileSystem();
	687	}();
667	688
668	689	// Set an error handler, so that any LLVM backend diagnostics go through our
669	690	// error handler.
...	...	@@ -679,8 +700,7 @@ int cc1as_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
679	700	getDriverOptTable().printHelp(
680	701	llvm::outs(), "clang -cc1as [options] file...",
681	702	"Clang Integrated Assembler", /ShowHidden=/false,
682		/ShowAllAliases=/false,
683		llvm::opt::Visibility(driver::options::CC1AsOption));
	703	/ShowAllAliases=/false, llvm::opt::Visibility(options::CC1AsOption));
684	704
685	705	return 0;
686	706	}
...	...	@@ -703,11 +723,12 @@ int cc1as_main(ArrayRef<const char > Argv, const char Argv0, void *MainAddr) {
703	723	for (unsigned i = 0; i != NumArgs; ++i)
704	724	Args[i + 1] = Asm.LLVMArgs[i].c_str();
705	725	Args[NumArgs + 1] = nullptr;
706		llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
	726	llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get(), /Overview=/"",
	727	/Errs=/nullptr, /VFS=/VFS.get());
707	728	}
708	729
709	730	// Execute the invocation, unless there were parsing errors.
710		bool Failed = Diags.hasErrorOccurred() \|\| ExecuteAssembler(Asm, Diags);
	731	bool Failed = Diags.hasErrorOccurred() \|\| ExecuteAssembler(Asm, Diags, VFS);
711	732
712	733	// If any timers were active but haven't been destroyed yet, print their
713	734	// results now.

src/zig_clang_driver.cpp+22-14

...	...	@@ -18,13 +18,13 @@
18	18	#include "clang/Config/config.h"
19	19	#include "clang/Driver/Compilation.h"
20	20	#include "clang/Driver/DriverDiagnostic.h"
21		#include "clang/Driver/Options.h"
22	21	#include "clang/Driver/ToolChain.h"
23	22	#include "clang/Frontend/ChainedDiagnosticConsumer.h"
24	23	#include "clang/Frontend/CompilerInvocation.h"
25	24	#include "clang/Frontend/SerializedDiagnosticPrinter.h"
26	25	#include "clang/Frontend/TextDiagnosticPrinter.h"
27	26	#include "clang/Frontend/Utils.h"
	27	#include "clang/Options/Options.h"
28	28	#include "llvm/ADT/ArrayRef.h"
29	29	#include "llvm/ADT/SmallString.h"
30	30	#include "llvm/ADT/SmallVector.h"
...	...	@@ -38,6 +38,7 @@
38	38	#include "llvm/Support/CrashRecoveryContext.h"
39	39	#include "llvm/Support/ErrorHandling.h"
40	40	#include "llvm/Support/FileSystem.h"
	41	#include "llvm/Support/IOSandbox.h"
41	42	#include "llvm/Support/LLVMDriver.h"
42	43	#include "llvm/Support/Path.h"
43	44	#include "llvm/Support/PrettyStackTrace.h"
...	...	@@ -201,7 +202,8 @@ static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
201	202	}
202	203
203	204	static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV,
204		const llvm::ToolContext &ToolContext) {
	205	const llvm::ToolContext &ToolContext,
	206	IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
205	207	// If we call the cc1 tool from the clangDriver library (through
206	208	// Driver::CC1Main), we need to clean up the options usage count. The options
207	209	// are currently global, and they might have been used previously by the
...	...	@@ -209,7 +211,8 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV,
209	211	llvm::cl::ResetAllOptionOccurrences();
210	212
211	213	llvm::BumpPtrAllocator A;
212		llvm::cl::ExpansionContext ECtx(A, llvm::cl::TokenizeGNUCommandLine);
	214	llvm::cl::ExpansionContext ECtx(A, llvm::cl::TokenizeGNUCommandLine,
	215	VFS.get());
213	216	if (llvm::Error Err = ECtx.expandResponseFiles(ArgV)) {
214	217	llvm::errs() << toString(std::move(Err)) << '\n';
215	218	return 1;
...	...	@@ -249,14 +252,22 @@ static int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContex
249	252	bool ClangCLMode =
250	253	IsClangCL(getDriverMode(ProgName, llvm::ArrayRef(Args).slice(1)));
251	254
252		if (llvm::Error Err = expandResponseFiles(Args, ClangCLMode, A)) {
	255	auto VFS = llvm::vfs::getRealFileSystem();
	256
	257	if (llvm::Error Err = expandResponseFiles(Args, ClangCLMode, A, VFS.get())) {
253	258	llvm::errs() << toString(std::move(Err)) << '\n';
254	259	return 1;
255	260	}
256	261
257	262	// Handle -cc1 integrated tools.
258		if (Args.size() >= 2 && StringRef(Args[1]).starts_with("-cc1"))
259		return ExecuteCC1Tool(Args, ToolContext);
	263	if (Args.size() >= 2 && StringRef(Args[1]).starts_with("-cc1")) {
	264	// Note that this only enables the sandbox for direct -cc1 invocations and
	265	// out-of-process -cc1 invocations launched by the driver. For in-process
	266	// -cc1 invocations launched by the driver, the sandbox is enabled in
	267	// CC1Command::Execute() for better crash recovery.
	268	auto EnableSandbox = llvm::sys::sandbox::scopedEnable();
	269	return ExecuteCC1Tool(Args, ToolContext, VFS);
	270	}
260	271
261	272	// Handle options that need handling before the real command line parsing in
262	273	// Driver::BuildCompilation()
...	...	@@ -326,9 +337,7 @@ static int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContex
326	337	new TextDiagnosticPrinter(llvm::errs(), *DiagOpts);
327	338	FixupDiagPrefixExeName(DiagClient, ProgName);
328	339
329		IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
330
331		DiagnosticsEngine Diags(DiagID, *DiagOpts, DiagClient);
	340	DiagnosticsEngine Diags(DiagnosticIDs::create(), *DiagOpts, DiagClient);
332	341
333	342	if (!DiagOpts->DiagnosticSerializationFile.empty()) {
334	343	auto SerializedConsumer =
...	...	@@ -338,7 +347,6 @@ static int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContex
338	347	Diags.takeClient(), std::move(SerializedConsumer)));
339	348	}
340	349
341		auto VFS = llvm::vfs::getRealFileSystem();
342	350	ProcessWarningOptions(Diags, DiagOpts, VFS, /ReportDiags=/false);
343	351
344	352	Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags,
...	...	@@ -358,10 +366,10 @@ static int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContex
358	366	if (!SetBackdoorDriverOutputsFromEnvVars(TheDriver))
359	367	return 1;
360	368
361		auto ExecuteCC1WithContext =
362		[&ToolContext](SmallVectorImpl<const char *> &ArgV) {
363		return ExecuteCC1Tool(ArgV, ToolContext);
364		};
	369	auto ExecuteCC1WithContext = [&ToolContext,
	370	&VFS](SmallVectorImpl<const char *> &ArgV) {
	371	return ExecuteCC1Tool(ArgV, ToolContext, VFS);
	372	};
365	373	if (!UseNewCC1Process) {
366	374	TheDriver.CC1Main = ExecuteCC1WithContext;
367	375	// Ensure the CC1Command actually catches cc1 crashes

src/zig_llvm-ar.cpp+1

...	...	@@ -12,6 +12,7 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "llvm/ADT/StringExtras.h"
	15	#include "llvm/ADT/StringMap.h"
15	16	#include "llvm/ADT/StringSwitch.h"
16	17	#include "llvm/BinaryFormat/Magic.h"
17	18	#include "llvm/IR/LLVMContext.h"

src/zig_llvm.cpp+1-1

...	...	@@ -439,7 +439,7 @@ ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machi
439	439
440	440	void ZigLLVMSetOptBisectLimit(LLVMContextRef context_ref, int limit) {
441	441	static OptBisect opt_bisect;
442		opt_bisect.setLimit(limit);
	442	opt_bisect.setIntervals({0, limit});
443	443	unwrap(context_ref)->setOptPassGate(opt_bisect);
444	444	}
445	445

stage1/zig.h+354-1

...	...	@@ -79,6 +79,9 @@
79	79	#elif defined(__I86__)
80	80	#define zig_x86_16
81	81	#define zig_x86
	82	#elif defined (__ez80)
	83	#define zig_ez80
	84	#define zig_z80
82	85	#endif
83	86
84	87	#if defined(zig_msvc) \|\| __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
...	...	@@ -409,6 +412,8 @@
409	412	#define zig_trap() __asm__ volatile("int $0x3")
410	413	#elif defined(zig_x86)
411	414	#define zig_trap() __asm__ volatile("ud2")
	415	#elif defined(zig_z80)
	416	#define zig_trap() __asm__ volatile("rst 00h")
412	417	#else
413	418	#define zig_trap() zig_trap_unavailable
414	419	#endif
...	...	@@ -511,7 +516,7 @@ zig_extern void memcpy (void zig_restrict, void const *zig_restrict, size_t);
511	516	zig_extern void memset (void , int, size_t);
512	517	zig_extern void memmove (void , void const *, size_t);
513	518
514		/* ================ Bool and 8/16/32/64-bit Integer Support ================= */
	519	/* ================ Bool and 8/16/24/32/48/64-bit Integer Support ================= */
515	520
516	521	#include <limits.h>
517	522
...	...	@@ -590,6 +595,16 @@ typedef signed long long int16_t;
590	595	#define INT16_MAX ( INT16_C(0x7FFF))
591	596	#define UINT16_MAX ( INT16_C(0xFFFF))
592	597
	598	#if defined(zig_ez80)
	599	typedef unsigned int uint24_t;
	600	typedef signed int int24_t;
	601	#define INT24_C(c) c
	602	#define UINT24_C(c) c##U
	603	#endif
	604	#define INT24_MIN (~INT24_C(0x7FFF))
	605	#define INT24_MAX ( INT24_C(0x7FFF))
	606	#define UINT24_MAX ( INT24_C(0xFFFF))
	607
593	608	#if SCHAR_MIN == ~0x7FFFFFFF && SCHAR_MAX == 0x7FFFFFFF && UCHAR_MAX == 0xFFFFFFFF
594	609	typedef unsigned char uint32_t;
595	610	typedef signed char int32_t;
...	...	@@ -620,6 +635,17 @@ typedef signed long long int32_t;
620	635	#define INT32_MAX ( INT32_C(0x7FFFFFFF))
621	636	#define UINT32_MAX ( INT32_C(0xFFFFFFFF))
622	637
	638	#if defined(zig_ez80)
	639	typedef unsigned __int48 uint48_t;
	640	typedef signed __int48 int48_t;
	641	#define INT48_C(c) c
	642	/* no suffix */
	643	#define UINT48_C(c) ((uint48_t)(c))
	644	#endif
	645	#define INT48_MIN (~INT48_C(0x7FFFFFFFFFFF))
	646	#define INT48_MAX ( INT48_C(0x7FFFFFFFFFFF))
	647	#define UINT48_MAX ( INT48_C(0xFFFFFFFFFFFF))
	648
623	649	#if SCHAR_MIN == ~0x7FFFFFFFFFFFFFFF && SCHAR_MAX == 0x7FFFFFFFFFFFFFFF && UCHAR_MAX == 0xFFFFFFFFFFFFFFFF
624	650	typedef unsigned char uint64_t;
625	651	typedef signed char int64_t;
...	...	@@ -663,10 +689,18 @@ typedef ptrdiff_t intptr_t;
663	689	#define zig_maxInt_i16 INT16_MAX
664	690	#define zig_minInt_u16 UINT16_C(0)
665	691	#define zig_maxInt_u16 UINT16_MAX
	692	#define zig_minInt_i24 INT24_MIN
	693	#define zig_maxInt_i24 INT24_MAX
	694	#define zig_minInt_u24 UINT24_C(0)
	695	#define zig_maxInt_u24 UINT24_MAX
666	696	#define zig_minInt_i32 INT32_MIN
667	697	#define zig_maxInt_i32 INT32_MAX
668	698	#define zig_minInt_u32 UINT32_C(0)
669	699	#define zig_maxInt_u32 UINT32_MAX
	700	#define zig_minInt_i48 INT48_MIN
	701	#define zig_maxInt_i48 INT48_MAX
	702	#define zig_minInt_u48 UINT48_C(0)
	703	#define zig_maxInt_u48 UINT48_MAX
670	704	#define zig_minInt_i64 INT64_MIN
671	705	#define zig_maxInt_i64 INT64_MAX
672	706	#define zig_minInt_u64 UINT64_C(0)
...	...	@@ -786,6 +820,17 @@ zig_int_helpers(16, unsigned long long)
786	820	#else
787	821	zig_int_helpers(16, uint16_t)
788	822	#endif
	823	#if defined(zig_ez80)
	824	#if UINT24_MAX <= UINT_MAX
	825	zig_int_helpers(24, unsigned int)
	826	#elif UINT24_MAX <= ULONG_MAX
	827	zig_int_helpers(24, unsigned long)
	828	#elif UINT24_MAX <= ULLONG_MAX
	829	zig_int_helpers(24, unsigned long long)
	830	#else
	831	zig_int_helpers(24, uint24_t)
	832	#endif
	833	#endif
789	834	#if UINT32_MAX <= UINT_MAX
790	835	zig_int_helpers(32, unsigned int)
791	836	#elif UINT32_MAX <= ULONG_MAX
...	...	@@ -795,6 +840,17 @@ zig_int_helpers(32, unsigned long long)
795	840	#else
796	841	zig_int_helpers(32, uint32_t)
797	842	#endif
	843	#if defined(zig_ez80)
	844	#if UINT24_MAX <= UINT_MAX
	845	zig_int_helpers(48, unsigned int)
	846	#elif UINT24_MAX <= ULONG_MAX
	847	zig_int_helpers(48, unsigned long)
	848	#elif UINT24_MAX <= ULLONG_MAX
	849	zig_int_helpers(48, unsigned long long)
	850	#else
	851	zig_int_helpers(48, uint48_t)
	852	#endif
	853	#endif
798	854	#if UINT64_MAX <= UINT_MAX
799	855	zig_int_helpers(64, unsigned int)
800	856	#elif UINT64_MAX <= ULONG_MAX
...	...	@@ -909,6 +965,66 @@ static inline bool zig_addo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t
909	965	#endif
910	966	}
911	967
	968	#if defined(zig_ez80)
	969	static inline bool zig_addo_u24(uint24_t *res, uint24_t lhs, uint24_t rhs, uint8_t bits) {
	970	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
	971	uint24_t full_res;
	972	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
	973	*res = zig_wrap_u24(full_res, bits);
	974	return overflow \|\| full_res < zig_minInt_u(24, bits) \|\| full_res > zig_maxInt_u(24, bits);
	975	#else
	976	uint32_t full_res;
	977	bool overflow = zig_addo_u32(&full_res, lhs, rhs, bits);
	978	*res = (uint24_t)full_res;
	979	return overflow;
	980	#endif
	981	}
	982
	983	static inline bool zig_addo_i24(int24_t *res, int24_t lhs, int24_t rhs, uint8_t bits) {
	984	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
	985	int24_t full_res;
	986	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
	987	*res = zig_wrap_i24(full_res, bits);
	988	return overflow \|\| full_res < zig_minInt_i(24, bits) \|\| full_res > zig_maxInt_i(24, bits);
	989	#else
	990	int32_t full_res;
	991	bool overflow = zig_addo_i32(&full_res, lhs, rhs, bits);
	992	*res = (int24_t)full_res;
	993	return overflow;
	994	#endif
	995	}
	996	#endif
	997
	998	#if defined(zig_ez80)
	999	static inline bool zig_addo_u48(uint48_t *res, uint48_t lhs, uint48_t rhs, uint8_t bits) {
	1000	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
	1001	uint48_t full_res;
	1002	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
	1003	*res = zig_wrap_u48(full_res, bits);
	1004	return overflow \|\| full_res < zig_minInt_u(48, bits) \|\| full_res > zig_maxInt_u(48, bits);
	1005	#else
	1006	uint64_t full_res;
	1007	bool overflow = zig_addo_u64(&full_res, lhs, rhs, bits);
	1008	*res = (uint48_t)full_res;
	1009	return overflow;
	1010	#endif
	1011	}
	1012
	1013	static inline bool zig_addo_i48(int48_t *res, int48_t lhs, int48_t rhs, uint8_t bits) {
	1014	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
	1015	int48_t full_res;
	1016	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
	1017	*res = zig_wrap_i48(full_res, bits);
	1018	return overflow \|\| full_res < zig_minInt_i(48, bits) \|\| full_res > zig_maxInt_i(48, bits);
	1019	#else
	1020	int64_t full_res;
	1021	bool overflow = zig_addo_i64(&full_res, lhs, rhs, bits);
	1022	*res = (int48_t)full_res;
	1023	return overflow;
	1024	#endif
	1025	}
	1026	#endif
	1027
912	1028	static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) {
913	1029	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
914	1030	uint32_t full_res;
...	...	@@ -933,6 +1049,7 @@ static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t
933	1049	return overflow \|\| full_res < zig_minInt_i(32, bits) \|\| full_res > zig_maxInt_i(32, bits);
934	1050	}
935	1051
	1052
936	1053	static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) {
937	1054	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
938	1055	uint64_t full_res;
...	...	@@ -1013,6 +1130,66 @@ static inline bool zig_subo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t
1013	1130	#endif
1014	1131	}
1015	1132
	1133	#if defined(zig_ez80)
	1134	static inline bool zig_subo_u24(uint24_t *res, uint24_t lhs, uint24_t rhs, uint8_t bits) {
	1135	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
	1136	uint24_t full_res;
	1137	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
	1138	*res = zig_wrap_u24(full_res, bits);
	1139	return overflow \|\| full_res < zig_minInt_u(24, bits) \|\| full_res > zig_maxInt_u(24, bits);
	1140	#else
	1141	uint32_t full_res;
	1142	bool overflow = zig_subo_u32(&full_res, lhs, rhs, bits);
	1143	*res = (uint24_t)full_res;
	1144	return overflow;
	1145	#endif
	1146	}
	1147
	1148	static inline bool zig_subo_i24(int24_t *res, int24_t lhs, int24_t rhs, uint8_t bits) {
	1149	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
	1150	int24_t full_res;
	1151	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
	1152	*res = zig_wrap_i24(full_res, bits);
	1153	return overflow \|\| full_res < zig_minInt_i(24, bits) \|\| full_res > zig_maxInt_i(24, bits);
	1154	#else
	1155	int32_t full_res;
	1156	bool overflow = zig_subo_i32(&full_res, lhs, rhs, bits);
	1157	*res = (int24_t)full_res;
	1158	return overflow;
	1159	#endif
	1160	}
	1161	#endif
	1162
	1163	#if defined(zig_ez80)
	1164	static inline bool zig_subo_u48(uint48_t *res, uint48_t lhs, uint48_t rhs, uint8_t bits) {
	1165	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
	1166	uint48_t full_res;
	1167	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
	1168	*res = zig_wrap_u48(full_res, bits);
	1169	return overflow \|\| full_res < zig_minInt_u(48, bits) \|\| full_res > zig_maxInt_u(48, bits);
	1170	#else
	1171	uint64_t full_res;
	1172	bool overflow = zig_subo_u64(&full_res, lhs, rhs, bits);
	1173	*res = (uint48_t)full_res;
	1174	return overflow;
	1175	#endif
	1176	}
	1177
	1178	static inline bool zig_subo_i48(int48_t *res, int48_t lhs, int48_t rhs, uint8_t bits) {
	1179	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
	1180	int48_t full_res;
	1181	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
	1182	*res = zig_wrap_i48(full_res, bits);
	1183	return overflow \|\| full_res < zig_minInt_i(48, bits) \|\| full_res > zig_maxInt_i(48, bits);
	1184	#else
	1185	int64_t full_res;
	1186	bool overflow = zig_subo_i64(&full_res, lhs, rhs, bits);
	1187	*res = (int48_t)full_res;
	1188	return overflow;
	1189	#endif
	1190	}
	1191	#endif
	1192
1016	1193	static inline bool zig_mulo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) {
1017	1194	#if zig_has_builtin(mul_overflow) \|\| defined(zig_gcc)
1018	1195	uint32_t full_res;
...	...	@@ -1121,6 +1298,66 @@ static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t
1121	1298	#endif
1122	1299	}
1123	1300
	1301	#if defined(zig_ez80)
	1302	static inline bool zig_mulo_u24(uint24_t *res, uint24_t lhs, uint24_t rhs, uint8_t bits) {
	1303	#if zig_has_builtin(mul_overflow) \|\| defined(zig_gcc)
	1304	uint24_t full_res;
	1305	bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res);
	1306	*res = zig_wrap_u24(full_res, bits);
	1307	return overflow \|\| full_res < zig_minInt_u(24, bits) \|\| full_res > zig_maxInt_u(24, bits);
	1308	#else
	1309	uint32_t full_res;
	1310	bool overflow = zig_mulo_u32(&full_res, lhs, rhs, bits);
	1311	*res = (uint24_t)full_res;
	1312	return overflow;
	1313	#endif
	1314	}
	1315
	1316	static inline bool zig_mulo_i24(int24_t *res, int24_t lhs, int24_t rhs, uint8_t bits) {
	1317	#if zig_has_builtin(mul_overflow) \|\| defined(zig_gcc)
	1318	int24_t full_res;
	1319	bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res);
	1320	*res = zig_wrap_i24(full_res, bits);
	1321	return overflow \|\| full_res < zig_minInt_i(24, bits) \|\| full_res > zig_maxInt_i(24, bits);
	1322	#else
	1323	int32_t full_res;
	1324	bool overflow = zig_mulo_i32(&full_res, lhs, rhs, bits);
	1325	*res = (int24_t)full_res;
	1326	return overflow;
	1327	#endif
	1328	}
	1329	#endif
	1330
	1331	#if defined(zig_ez80)
	1332	static inline bool zig_mulo_u48(uint48_t *res, uint48_t lhs, uint48_t rhs, uint8_t bits) {
	1333	#if zig_has_builtin(mul_overflow) \|\| defined(zig_gcc)
	1334	uint48_t full_res;
	1335	bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res);
	1336	*res = zig_wrap_u48(full_res, bits);
	1337	return overflow \|\| full_res < zig_minInt_u(48, bits) \|\| full_res > zig_maxInt_u(48, bits);
	1338	#else
	1339	uint64_t full_res;
	1340	bool overflow = zig_mulo_u64(&full_res, lhs, rhs, bits);
	1341	*res = (uint48_t)full_res;
	1342	return overflow;
	1343	#endif
	1344	}
	1345
	1346	static inline bool zig_mulo_i48(int48_t *res, int48_t lhs, int48_t rhs, uint8_t bits) {
	1347	#if zig_has_builtin(mul_overflow) \|\| defined(zig_gcc)
	1348	int48_t full_res;
	1349	bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res);
	1350	*res = zig_wrap_i48(full_res, bits);
	1351	return overflow \|\| full_res < zig_minInt_i(48, bits) \|\| full_res > zig_maxInt_i(48, bits);
	1352	#else
	1353	int64_t full_res;
	1354	bool overflow = zig_mulo_i64(&full_res, lhs, rhs, bits);
	1355	*res = (int48_t)full_res;
	1356	return overflow;
	1357	#endif
	1358	}
	1359	#endif
	1360
1124	1361	#define zig_int_builtins(w) \
1125	1362	static inline bool zig_shlo_u##w(uint##w##_t *res, uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \
1126	1363	*res = zig_shlw_u##w(lhs, rhs, bits); \
...	...	@@ -1180,7 +1417,13 @@ static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t
1180	1417	}
1181	1418	zig_int_builtins(8)
1182	1419	zig_int_builtins(16)
	1420	#if defined(zig_ez80)
	1421	zig_int_builtins(24)
	1422	#endif
1183	1423	zig_int_builtins(32)
	1424	#if defined(zig_ez80)
	1425	zig_int_builtins(48)
	1426	#endif
1184	1427	zig_int_builtins(64)
1185	1428
1186	1429	#define zig_builtin8(name, val) __builtin_##name(val)
...	...	@@ -1189,6 +1432,11 @@ typedef unsigned int zig_Builtin8;
1189	1432	#define zig_builtin16(name, val) __builtin_##name(val)
1190	1433	typedef unsigned int zig_Builtin16;
1191	1434
	1435	#if defined(zig_ez80)
	1436	#define zig_builtin24(name, val) __builtin_##name(val)
	1437	typedef unsigned int zig_Builtin24;
	1438	#endif
	1439
1192	1440	#if INT_MIN <= INT32_MIN
1193	1441	#define zig_builtin32(name, val) __builtin_##name(val)
1194	1442	typedef unsigned int zig_Builtin32;
...	...	@@ -1197,6 +1445,11 @@ typedef unsigned int zig_Builtin32;
1197	1445	typedef unsigned long zig_Builtin32;
1198	1446	#endif
1199	1447
	1448	#if defined(zig_ez80)
	1449	#define zig_builtin48(name, val) __builtin_##name(val)
	1450	typedef unsigned long long zig_Builtin48;
	1451	#endif
	1452
1200	1453	#if INT_MIN <= INT64_MIN
1201	1454	#define zig_builtin64(name, val) __builtin_##name(val)
1202	1455	typedef unsigned int zig_Builtin64;
...	...	@@ -1231,6 +1484,23 @@ static inline int16_t zig_byte_swap_i16(int16_t val, uint8_t bits) {
1231	1484	return zig_wrap_i16((int16_t)zig_byte_swap_u16((uint16_t)val, bits), bits);
1232	1485	}
1233	1486
	1487	#if defined(zig_ez80)
	1488	static inline uint16_t zig_byte_swap_u24(uint24_t val, uint8_t bits) {
	1489	uint24_t full_res;
	1490	#if zig_has_builtin(bswap24) \|\| defined(zig_gcc)
	1491	full_res = __builtin_bswap24(val);
	1492	#else
	1493	full_res = (uint24_t)zig_byte_swap_u8((uint8_t)(val >> 0), 8) << 16 \|
	1494	(uint24_t)zig_byte_swap_u16((uint16_t)(val >> 8), 16) >> 0;
	1495	#endif
	1496	return zig_wrap_u24(full_res >> (24 - bits), bits);
	1497	}
	1498
	1499	static inline int16_t zig_byte_swap_i24(int24_t val, uint8_t bits) {
	1500	return zig_wrap_i24((int24_t)zig_byte_swap_u24((uint24_t)val, bits), bits);
	1501	}
	1502	#endif
	1503
1234	1504	static inline uint32_t zig_byte_swap_u32(uint32_t val, uint8_t bits) {
1235	1505	uint32_t full_res;
1236	1506	#if zig_has_builtin(bswap32) \|\| defined(zig_gcc)
...	...	@@ -1246,6 +1516,23 @@ static inline int32_t zig_byte_swap_i32(int32_t val, uint8_t bits) {
1246	1516	return zig_wrap_i32((int32_t)zig_byte_swap_u32((uint32_t)val, bits), bits);
1247	1517	}
1248	1518
	1519	#if defined(zig_ez80)
	1520	static inline uint32_t zig_byte_swap_u48(uint48_t val, uint8_t bits) {
	1521	uint48_t full_res;
	1522	#if zig_has_builtin(bswap48) \|\| defined(zig_gcc)
	1523	full_res = __builtin_bswap48(val);
	1524	#else
	1525	full_res = (uint48_t)zig_byte_swap_u24((uint24_t)(val >> 0), 24) << 24 \|
	1526	(uint48_t)zig_byte_swap_u24((uint24_t)(val >> 24), 24) >> 0;
	1527	#endif
	1528	return zig_wrap_u48(full_res >> (48 - bits), bits);
	1529	}
	1530
	1531	static inline int32_t zig_byte_swap_i48(int48_t val, uint8_t bits) {
	1532	return zig_wrap_i48((int48_t)zig_byte_swap_u48((uint48_t)val, bits), bits);
	1533	}
	1534	#endif
	1535
1249	1536	static inline uint64_t zig_byte_swap_u64(uint64_t val, uint8_t bits) {
1250	1537	uint64_t full_res;
1251	1538	#if zig_has_builtin(bswap64) \|\| defined(zig_gcc)
...	...	@@ -1294,6 +1581,23 @@ static inline int16_t zig_bit_reverse_i16(int16_t val, uint8_t bits) {
1294	1581	return zig_wrap_i16((int16_t)zig_bit_reverse_u16((uint16_t)val, bits), bits);
1295	1582	}
1296	1583
	1584	#if defined(zig_ez80)
	1585	static inline uint24_t zig_bit_reverse_u24(uint24_t val, uint8_t bits) {
	1586	uint24_t full_res;
	1587	#if zig_has_builtin(bitreverse24)
	1588	full_res = __builtin_bitreverse24(val);
	1589	#else
	1590	full_res = (uint24_t)zig_bit_reverse_u8((uint8_t)(val >> 0), 8) << 16 \|
	1591	(uint24_t)zig_bit_reverse_u16((uint16_t)(val >> 8), 16) >> 0;
	1592	#endif
	1593	return zig_wrap_u24(full_res >> (24 - bits), bits);
	1594	}
	1595
	1596	static inline int24_t zig_bit_reverse_i24(int24_t val, uint8_t bits) {
	1597	return zig_wrap_i24((int24_t)zig_bit_reverse_u24((uint24_t)val, bits), bits);
	1598	}
	1599	#endif
	1600
1297	1601	static inline uint32_t zig_bit_reverse_u32(uint32_t val, uint8_t bits) {
1298	1602	uint32_t full_res;
1299	1603	#if zig_has_builtin(bitreverse32)
...	...	@@ -1309,6 +1613,23 @@ static inline int32_t zig_bit_reverse_i32(int32_t val, uint8_t bits) {
1309	1613	return zig_wrap_i32((int32_t)zig_bit_reverse_u32((uint32_t)val, bits), bits);
1310	1614	}
1311	1615
	1616	#if defined(zig_ez80)
	1617	static inline uint32_t zig_bit_reverse_u48(uint48_t val, uint8_t bits) {
	1618	uint48_t full_res;
	1619	#if zig_has_builtin(bitreverse48)
	1620	full_res = __builtin_bitreverse48(val);
	1621	#else
	1622	full_res = (uint48_t)zig_bit_reverse_u24((uint24_t)(val >> 0), 24) << 24 \|
	1623	(uint48_t)zig_bit_reverse_u24((uint24_t)(val >> 24), 24) >> 0;
	1624	#endif
	1625	return zig_wrap_u32(full_res >> (48 - bits), bits);
	1626	}
	1627
	1628	static inline int32_t zig_bit_reverse_i48(int48_t val, uint8_t bits) {
	1629	return zig_wrap_i48((int48_t)zig_bit_reverse_u48((uint48_t)val, bits), bits);
	1630	}
	1631	#endif
	1632
1312	1633	static inline uint64_t zig_bit_reverse_u64(uint64_t val, uint8_t bits) {
1313	1634	uint64_t full_res;
1314	1635	#if zig_has_builtin(bitreverse64)
...	...	@@ -1350,7 +1671,13 @@ static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) {
1350	1671	#endif
1351	1672	zig_builtin_popcount(8)
1352	1673	zig_builtin_popcount(16)
	1674	#if defined(zig_ez80)
	1675	zig_builtin_popcount(24)
	1676	#endif
1353	1677	zig_builtin_popcount(32)
	1678	#if defined(zig_ez80)
	1679	zig_builtin_popcount(48)
	1680	#endif
1354	1681	zig_builtin_popcount(64)
1355	1682
1356	1683	#define zig_builtin_ctz_common(w) \
...	...	@@ -1375,7 +1702,13 @@ zig_builtin_popcount(64)
1375	1702	#endif
1376	1703	zig_builtin_ctz(8)
1377	1704	zig_builtin_ctz(16)
	1705	#if defined(zig_ez80)
	1706	zig_builtin_ctz(24)
	1707	#endif
1378	1708	zig_builtin_ctz(32)
	1709	#if defined(zig_ez80)
	1710	zig_builtin_ctz(48)
	1711	#endif
1379	1712	zig_builtin_ctz(64)
1380	1713
1381	1714	#define zig_builtin_clz_common(w) \
...	...	@@ -1400,7 +1733,13 @@ zig_builtin_ctz(64)
1400	1733	#endif
1401	1734	zig_builtin_clz(8)
1402	1735	zig_builtin_clz(16)
	1736	#if defined(zig_ez80)
	1737	zig_builtin_clz(24)
	1738	#endif
1403	1739	zig_builtin_clz(32)
	1740	#if defined(zig_ez80)
	1741	zig_builtin_clz(48)
	1742	#endif
1404	1743	zig_builtin_clz(64)
1405	1744
1406	1745	/* ======================== 128-bit Integer Support ========================= */
...	...	@@ -1981,6 +2320,20 @@ static inline zig_i128 zig_bit_reverse_i128(zig_i128 val, uint8_t bits) {
1981	2320	return zig_bitCast_i128(zig_bit_reverse_u128(zig_bitCast_u128(val), bits));
1982	2321	}
1983	2322
	2323	#if zig_has_int128
	2324	#define zig_switch_int128(operand) switch (operand)
	2325	#define zig_switch_prong_begin_int128()
	2326	#define zig_switch_case_int128(Type, operand, value) case value:
	2327	#define zig_switch_prong_end_int128()
	2328	#define zig_switch_default_int128() default:
	2329	#else // zig_has_int128
	2330	#define zig_switch_int128(operand)
	2331	#define zig_switch_prong_begin_int128() if (0
	2332	#define zig_switch_case_int128(Type, operand, value) \|\| (zig_cmp_##Type(operand, value) == 0)
	2333	#define zig_switch_prong_end_int128() )
	2334	#define zig_switch_default_int128()
	2335	#endif // zig_has_int128
	2336
1984	2337	/* ========================== Big Integer Support =========================== */
1985	2338
1986	2339	static inline uint16_t zig_int_bytes(uint16_t bits) {

stage1/zig1.wasm

Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ

test/behavior/align.zig-1

...	...	@@ -317,7 +317,6 @@ test "@alignCast functions" {
317	317
318	318	// function alignment is a compile error on wasm
319	319	if (native_arch.isWasm()) return error.SkipZigTest;
320		if (native_arch.isThumb()) return error.SkipZigTest;
321	320
322	321	try expect(fnExpectsOnly1(simple4) == 0x19);
323	322	}

test/behavior/asm.zig+1-5

...	...	@@ -67,7 +67,6 @@ test "alternative constraints" {
67	67	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
68	68	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
69	69	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
70		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isLoongArch()) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/159200
71	70
72	71	if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly
73	72
...	...	@@ -247,12 +246,9 @@ test "extern output types (x86_64)" {
247	246	}
248	247	}
249	248
250		test "riscv abi register aliases as clobbers" {
	249	test "abi register aliases as clobbers (RISC-V)" {
251	250	if (!builtin.target.cpu.arch.isRISCV()) return error.SkipZigTest;
252	251	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
253		if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest;
254
255		if (!comptime builtin.cpu.arch.isRISCV()) return error.SkipZigTest;
256	252
257	253	// Verify that ABI alias names are accepted as clobbers for RISC-V.
258	254	asm volatile ("" ::: .{ .ra = true, .sp = true, .gp = true, .tp = true });

test/behavior/basic.zig+1-5

...	...	@@ -1407,11 +1407,7 @@ test "allocation and looping over 3-byte integer" {
1407	1407	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1408	1408	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1409	1409	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1410
1411		if (builtin.zig_backend == .stage2_llvm and builtin.os.tag.isDarwin()) {
1412		return error.SkipZigTest; // TODO
1413		}
1414		if (builtin.cpu.arch == .s390x and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // TODO
	1410	if (builtin.zig_backend == .stage2_llvm and builtin.os.tag.isDarwin()) return error.SkipZigTest; // TODO
1415	1411
1416	1412	try expect(@sizeOf(u24) == 4);
1417	1413	try expect(@sizeOf([1]u24) == 4);

test/behavior/field_parent_ptr.zig-1

...	...	@@ -1757,7 +1757,6 @@ test "@fieldParentPtr packed union" {
1757	1757	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1758	1758	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1759	1759	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1760		if (builtin.target.cpu.arch.endian() == .big) return error.SkipZigTest; // TODO
1761	1760
1762	1761	const C = packed union {
1763	1762	a: packed struct(u32) {

test/behavior/floatop.zig+4-10

...	...	@@ -136,7 +136,6 @@ test "cmp f32" {
136	136	test "cmp f64" {
137	137	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
138	138	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
139		if (builtin.cpu.arch.isArm() and builtin.target.abi.float() == .soft) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21234
140	139
141	140	try testCmp(f64);
142	141	try comptime testCmp(f64);
...	...	@@ -155,7 +154,6 @@ test "cmp f128" {
155	154	test "cmp f80/c_longdouble" {
156	155	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
157	156	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
158		if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
159	157	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
160	158	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
161	159
...	...	@@ -223,8 +221,7 @@ test "vector cmp f16" {
223	221	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
224	222	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
225	223	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
226		if (builtin.cpu.arch.isArm()) return error.SkipZigTest;
227		if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
	224	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isArm()) return error.SkipZigTest;
228	225	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .hexagon) return error.SkipZigTest;
229	226
230	227	try testCmpVector(f16);
...	...	@@ -236,8 +233,8 @@ test "vector cmp f32" {
236	233	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
237	234	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
238	235	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
239		if (builtin.cpu.arch.isArm()) return error.SkipZigTest;
240		if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
	236	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isArm()) return error.SkipZigTest;
	237	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
241	238	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .hexagon) return error.SkipZigTest;
242	239
243	240	try testCmpVector(f32);
...	...	@@ -248,8 +245,6 @@ test "vector cmp f64" {
248	245	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
249	246	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
250	247	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
251		if (builtin.cpu.arch.isArm()) return error.SkipZigTest;
252		if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
253	248	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .hexagon) return error.SkipZigTest;
254	249
255	250	try testCmpVector(f64);
...	...	@@ -262,8 +257,7 @@ test "vector cmp f128" {
262	257	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
263	258	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
264	259	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
265		if (builtin.cpu.arch.isArm()) return error.SkipZigTest;
266		if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
	260	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .powerpc64le) return error.SkipZigTest;
267	261	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .hexagon) return error.SkipZigTest;
268	262
269	263	try testCmpVector(f128);

test/behavior/generics.zig-4

...	...	@@ -305,8 +305,6 @@ test "generic function instantiation non-duplicates" {
305	305	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
306	306	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
307	307
308		if (builtin.os.tag == .wasi) return error.SkipZigTest;
309
310	308	const S = struct {
311	309	fn copy(comptime T: type, dest: []T, source: []const T) void {
312	310	@export(&foo, .{ .name = "test_generic_instantiation_non_dupe" });
...	...	@@ -323,8 +321,6 @@ test "generic function instantiation non-duplicates" {
323	321	test "generic instantiation of tagged union with only one field" {
324	322	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
325	323
326		if (builtin.os.tag == .wasi) return error.SkipZigTest;
327
328	324	const S = struct {
329	325	const U = union(enum) {
330	326	s: []const u8,

test/behavior/math.zig-6

...	...	@@ -110,8 +110,6 @@ test "@clz vectors" {
110	110	}
111	111
112	112	fn testClzVectors() !void {
113		if (comptime builtin.cpu.has(.loongarch, .lsx)) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/159529
114
115	113	const Vu4 = @Vector(64, u4);
116	114	const Vu8 = @Vector(64, u8);
117	115	const Vu128 = @Vector(64, u128);
...	...	@@ -193,8 +191,6 @@ test "@ctz vectors" {
193	191	}
194	192
195	193	fn testCtzVectors() !void {
196		if (comptime builtin.cpu.has(.loongarch, .lsx)) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/159529
197
198	194	const Vu4 = @Vector(64, u4);
199	195	const Vu8 = @Vector(64, u8);
200	196	@setEvalBranchQuota(10_000);
...	...	@@ -2429,7 +2425,6 @@ test "runtime comparison to NaN is comptime-known" {
2429	2425	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2430	2426	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
2431	2427	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
2432		if (builtin.cpu.arch.isArm() and builtin.target.abi.float() == .soft) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21234
2433	2428
2434	2429	const S = struct {
2435	2430	fn doTheTest(comptime F: type, x: F) void {
...	...	@@ -2458,7 +2453,6 @@ test "runtime int comparison to inf is comptime-known" {
2458	2453	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
2459	2454	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
2460	2455	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
2461		if (builtin.cpu.arch.isArm() and builtin.target.abi.float() == .soft) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21234
2462	2456
2463	2457	const S = struct {
2464	2458	fn doTheTest(comptime F: type, x: u32) void {

test/behavior/maximum_minimum.zig-2

...	...	@@ -138,8 +138,6 @@ test "@min/max for floats" {
138	138	};
139	139
140	140	inline for (.{ f16, f32, f64, f80, f128, c_longdouble }) \|T\| {
141		if (T == c_longdouble and builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21090
142
143	141	try S.doTheTest(T);
144	142	try comptime S.doTheTest(T);
145	143	}

test/behavior/packed-struct.zig-1

...	...	@@ -1196,7 +1196,6 @@ test "packed struct with signed field" {
1196	1196
1197	1197	test "assign packed struct initialized with RLS to packed struct literal field" {
1198	1198	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1199		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) return error.SkipZigTest;
1200	1199	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1201	1200	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1202	1201

test/behavior/saturating_arithmetic.zig-10

...	...	@@ -147,11 +147,6 @@ test "saturating multiplication <= 32 bits" {
147	147	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
148	148	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
149	149
150		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
151		// https://github.com/ziglang/zig/issues/9660
152		return error.SkipZigTest;
153		}
154
155	150	try testSatMul(u8, 0, maxInt(u8), 0);
156	151	try testSatMul(u8, 1 << 7, 1 << 7, maxInt(u8));
157	152	try testSatMul(u8, maxInt(u8) - 1, 2, maxInt(u8));
...	...	@@ -246,11 +241,6 @@ test "saturating multiplication" {
246	241	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
247	242	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
248	243
249		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isWasm()) {
250		// https://github.com/ziglang/zig/issues/9660
251		return error.SkipZigTest;
252		}
253
254	244	const S = struct {
255	245	fn doTheTest() !void {
256	246	try testSatMul(i8, -3, 10, -30);

test/behavior/struct.zig-2

...	...	@@ -404,7 +404,6 @@ test "packed struct 24bits" {
404	404	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
405	405	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
406	406	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
407		if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
408	407	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
409	408	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; // TODO
410	409
...	...	@@ -921,7 +920,6 @@ test "tuple assigned to variable" {
921	920
922	921	test "comptime struct field" {
923	922	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
924		if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // TODO
925	923
926	924	const T = struct {
927	925	a: i32,

test/behavior/switch.zig+1

...	...	@@ -683,6 +683,7 @@ test "switch on pointer type" {
683	683	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
684	684	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
685	685	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
	686	if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://github.com/llvm/llvm-project/issues/176634
686	687
687	688	const S = struct {
688	689	const X = struct {

test/behavior/union.zig-3

...	...	@@ -1592,7 +1592,6 @@ test "memset packed union" {
1592	1592
1593	1593	try comptime S.doTheTest();
1594	1594
1595		if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
1596	1595	try S.doTheTest();
1597	1596	}
1598	1597
...	...	@@ -1750,8 +1749,6 @@ test "reinterpret packed union" {
1750	1749	try comptime S.doTheTest();
1751	1750
1752	1751	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1753		if (builtin.cpu.arch.isWasm()) return error.SkipZigTest; // TODO
1754		if (builtin.cpu.arch.endian() == .big) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21050
1755	1752	try S.doTheTest();
1756	1753	}
1757	1754

test/behavior/vector.zig-23

...	...	@@ -7,7 +7,6 @@ const expect = std.testing.expect;
7	7	const expectEqual = std.testing.expectEqual;
8	8
9	9	test "implicit cast vector to array - bool" {
10		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
11	10	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
12	11
13	12	const S = struct {
...	...	@@ -622,11 +621,6 @@ test "vector bitwise not operator" {
622	621	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
623	622	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
624	623
625		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) {
626		// https://github.com/ziglang/zig/issues/24061
627		return error.SkipZigTest;
628		}
629
630	624	const S = struct {
631	625	fn doTheTestNot(comptime T: type, x: @Vector(4, T)) !void {
632	626	const y = ~x;
...	...	@@ -660,11 +654,6 @@ test "vector boolean not operator" {
660	654	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
661	655	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
662	656
663		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) {
664		// https://github.com/ziglang/zig/issues/24061
665		return error.SkipZigTest;
666		}
667
668	657	const S = struct {
669	658	fn doTheTestNot(comptime T: type, x: @Vector(4, T)) !void {
670	659	const y = !x;
...	...	@@ -759,7 +748,6 @@ test "vector reduce operation" {
759	748	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
760	749	if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
761	750	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
762		if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isMIPS64()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/21091
763	751	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch.isSPARC()) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23719
764	752
765	753	const S = struct {
...	...	@@ -1096,9 +1084,6 @@ test "saturating multiplication" {
1096	1084	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1097	1085	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1098	1086
1099		// TODO: once #9660 has been solved, remove this line
1100		if (builtin.target.cpu.arch.isWasm()) return error.SkipZigTest;
1101
1102	1087	const S = struct {
1103	1088	fn doTheTest() !void {
1104	1089	// Broken out to avoid https://github.com/ziglang/zig/issues/11251
...	...	@@ -1165,7 +1150,6 @@ test "@addWithOverflow" {
1165	1150	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1166	1151	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1167	1152	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1168		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1169	1153
1170	1154	const S = struct {
1171	1155	fn doTheTest() !void {
...	...	@@ -1213,7 +1197,6 @@ test "@subWithOverflow" {
1213	1197	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1214	1198	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1215	1199	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1216		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1217	1200
1218	1201	const S = struct {
1219	1202	fn doTheTest() !void {
...	...	@@ -1244,7 +1227,6 @@ test "@mulWithOverflow" {
1244	1227	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1245	1228	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1246	1229	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1247		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1248	1230
1249	1231	const S = struct {
1250	1232	fn doTheTest() !void {
...	...	@@ -1265,7 +1247,6 @@ test "@shlWithOverflow" {
1265	1247	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1266	1248	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1267	1249	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1268		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1269	1250
1270	1251	const S = struct {
1271	1252	fn doTheTest() !void {
...	...	@@ -1347,7 +1328,6 @@ test "byte vector initialized in inline function" {
1347	1328	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1348	1329	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1349	1330	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1350		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1351	1331	if (builtin.cpu.arch == .hexagon and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1352	1332
1353	1333	if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .x86_64 and comptime builtin.cpu.has(.x86, .avx512f)) {
...	...	@@ -1463,7 +1443,6 @@ test "store packed vector element" {
1463	1443	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1464	1444	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1465	1445	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1466		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1467	1446
1468	1447	var v = @Vector(4, u1){ 1, 1, 1, 1 };
1469	1448	try expectEqual(@Vector(4, u1){ 1, 1, 1, 1 }, v);
...	...	@@ -1496,7 +1475,6 @@ test "store vector with memset" {
1496	1475	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; // TODO
1497	1476	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
1498	1477	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1499		if (builtin.cpu.arch == .aarch64_be and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
1500	1478
1501	1479	var a: [5]@Vector(2, i1) = undefined;
1502	1480	var b: [5]@Vector(2, u2) = undefined;
...	...	@@ -1596,7 +1574,6 @@ test "bitcast to vector with different child type" {
1596	1574	}
1597	1575	};
1598	1576
1599		// Originally reported at https://github.com/ziglang/zig/issues/8184
1600	1577	try S.doTheTest();
1601	1578	try comptime S.doTheTest();
1602	1579	}

test/behavior/wrapping_arithmetic.zig-3

...	...	@@ -82,9 +82,6 @@ test "wrapping multiplication" {
82	82	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
83	83	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
84	84
85		// TODO: once #9660 has been solved, remove this line
86		if (builtin.cpu.arch.isWasm()) return error.SkipZigTest;
87
88	85	const S = struct {
89	86	fn doTheTest() !void {
90	87	try testWrapMul(i8, -3, 10, -30);

test/c_abi/cfuncs.c+4

...	...	@@ -65,6 +65,10 @@ static void assert_or_panic(bool ok) {
65	65	# define ZIG_NO_COMPLEX
66	66	#endif
67	67
	68	#if defined(__loongarch__) && defined(__loongarch_soft_float)
	69	# define ZIG_NO_COMPLEX
	70	#endif
	71
68	72	#ifdef __powerpc__
69	73	# define ZIG_NO_COMPLEX
70	74	#endif

test/c_abi/main.zig+3-1

...	...	@@ -187,7 +187,8 @@ extern fn c_cmultd(a: ComplexDouble, b: ComplexDouble) ComplexDouble;
187	187	const complex_abi_compatible = builtin.cpu.arch != .x86 and !builtin.cpu.arch.isMIPS() and
188	188	!builtin.cpu.arch.isArm() and !builtin.cpu.arch.isPowerPC32() and !builtin.cpu.arch.isRISCV() and
189	189	builtin.cpu.arch != .hexagon and
190		builtin.cpu.arch != .s390x;
	190	builtin.cpu.arch != .s390x and
	191	!(builtin.cpu.arch.isLoongArch() and builtin.abi.float() == .soft);
191	192
192	193	test "C ABI complex float" {
193	194	if (!complex_abi_compatible) return error.SkipZigTest;
...	...	@@ -5676,6 +5677,7 @@ test "C ABI pointer sized float struct" {
5676	5677	if (builtin.cpu.arch.isMIPS64()) return error.SkipZigTest;
5677	5678	if (builtin.cpu.arch.isPowerPC32()) return error.SkipZigTest;
5678	5679	if (builtin.cpu.arch.isArm() and builtin.abi.float() == .soft) return error.SkipZigTest;
	5680	if (builtin.cpu.arch.isLoongArch() and builtin.abi.float() == .soft) return error.SkipZigTest;
5679	5681	if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
5680	5682
5681	5683	c_ptr_size_float_struct(.{ .x = 1, .y = 2 });

test/llvm_targets.zig+14-4

...	...	@@ -52,6 +52,7 @@ const targets = [_]std.Target.Query{
52	52	.{ .cpu_arch = .arm, .os_tag = .freebsd, .abi = .eabihf },
53	53	.{ .cpu_arch = .arm, .os_tag = .freestanding, .abi = .eabi },
54	54	.{ .cpu_arch = .arm, .os_tag = .freestanding, .abi = .eabihf },
	55	.{ .cpu_arch = .arm, .os_tag = .fuchsia, .abi = .eabihf },
55	56	.{ .cpu_arch = .arm, .os_tag = .haiku, .abi = .eabi },
56	57	.{ .cpu_arch = .arm, .os_tag = .haiku, .abi = .eabihf },
57	58	.{ .cpu_arch = .arm, .os_tag = .linux, .abi = .androideabi },
...	...	@@ -99,12 +100,19 @@ const targets = [_]std.Target.Query{
99	100	// .{ .cpu_arch = .csky, .os_tag = .linux, .abi = .gnueabihf },
100	101
101	102	.{ .cpu_arch = .hexagon, .os_tag = .freestanding, .abi = .none },
	103	.{ .cpu_arch = .hexagon, .os_tag = .linux, .abi = .musl },
102	104	.{ .cpu_arch = .hexagon, .os_tag = .linux, .abi = .none },
103	105
104	106	.{ .cpu_arch = .lanai, .os_tag = .freestanding, .abi = .none },
105	107
106		// .{ .cpu_arch = .loongarch32, .os_tag = .freestanding, .abi = .none },
107		// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .none },
	108	.{ .cpu_arch = .loongarch32, .os_tag = .freestanding, .abi = .none },
	109	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .gnu },
	110	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .gnuf32 },
	111	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .gnusf },
	112	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .musl },
	113	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .muslf32 },
	114	// .{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .muslsf },
	115	.{ .cpu_arch = .loongarch32, .os_tag = .linux, .abi = .none },
108	116	// .{ .cpu_arch = .loongarch32, .os_tag = .uefi, .abi = .none },
109	117
110	118	.{ .cpu_arch = .loongarch64, .os_tag = .freestanding, .abi = .none },
...	...	@@ -172,6 +180,7 @@ const targets = [_]std.Target.Query{
172	180
173	181	.{ .cpu_arch = .nvptx, .os_tag = .cuda, .abi = .none },
174	182	.{ .cpu_arch = .nvptx, .os_tag = .nvcl, .abi = .none },
	183
175	184	.{ .cpu_arch = .nvptx64, .os_tag = .cuda, .abi = .none },
176	185	.{ .cpu_arch = .nvptx64, .os_tag = .nvcl, .abi = .none },
177	186
...	...	@@ -255,6 +264,7 @@ const targets = [_]std.Target.Query{
255	264
256	265	.{ .cpu_arch = .thumb, .os_tag = .freestanding, .abi = .eabi },
257	266	.{ .cpu_arch = .thumb, .os_tag = .freestanding, .abi = .eabihf },
	267	.{ .cpu_arch = .thumb, .os_tag = .fuchsia, .abi = .eabihf },
258	268	.{ .cpu_arch = .thumb, .os_tag = .linux, .abi = .eabi },
259	269	.{ .cpu_arch = .thumb, .os_tag = .linux, .abi = .eabihf },
260	270	.{ .cpu_arch = .thumb, .os_tag = .linux, .abi = .musleabi },
...	...	@@ -342,8 +352,8 @@ const targets = [_]std.Target.Query{
342	352
343	353	.{ .cpu_arch = .xcore, .os_tag = .freestanding, .abi = .none },
344	354
345		// .{ .cpu_arch = .xtensa, .os_tag = .freestanding, .abi = .none },
346		// .{ .cpu_arch = .xtensa, .os_tag = .linux, .abi = .none },
	355	.{ .cpu_arch = .xtensa, .os_tag = .freestanding, .abi = .none },
	356	.{ .cpu_arch = .xtensa, .os_tag = .linux, .abi = .none },
347	357	};
348	358
349	359	pub fn addCases(

test/tests.zig+50-8

...	...	@@ -410,6 +410,14 @@ const module_test_targets = blk: {
410	410	.extra_target = true,
411	411	},
412	412
	413	.{
	414	.target = .{
	415	.cpu_arch = .loongarch32,
	416	.os_tag = .linux,
	417	.abi = .none,
	418	},
	419	},
	420
413	421	.{
414	422	.target = .{
415	423	.cpu_arch = .loongarch64,
...	...	@@ -435,6 +443,25 @@ const module_test_targets = blk: {
435	443	.link_libc = true,
436	444	.extra_target = true,
437	445	},
	446	.{
	447	.target = .{
	448	.cpu_arch = .loongarch64,
	449	.os_tag = .linux,
	450	.abi = .muslsf,
	451	},
	452	.link_libc = true,
	453	.extra_target = true,
	454	},
	455	.{
	456	.target = .{
	457	.cpu_arch = .loongarch64,
	458	.os_tag = .linux,
	459	.abi = .muslsf,
	460	},
	461	.linkage = .dynamic,
	462	.link_libc = true,
	463	.extra_target = true,
	464	},
438	465	.{
439	466	.target = .{
440	467	.cpu_arch = .loongarch64,
...	...	@@ -443,6 +470,15 @@ const module_test_targets = blk: {
443	470	},
444	471	.link_libc = true,
445	472	},
	473	.{
	474	.target = .{
	475	.cpu_arch = .loongarch64,
	476	.os_tag = .linux,
	477	.abi = .gnusf,
	478	},
	479	.link_libc = true,
	480	.extra_target = true,
	481	},
446	482
447	483	.{
448	484	.target = .{
...	...	@@ -1713,14 +1749,13 @@ const c_abi_targets = blk: {
1713	1749	},
1714	1750	},
1715	1751
1716		// https://gitlab.com/qemu-project/qemu/-/issues/3291
1717		// .{
1718		// .target = .{
1719		// .cpu_arch = .hexagon,
1720		// .os_tag = .linux,
1721		// .abi = .musl,
1722		// },
1723		// },
	1752	.{
	1753	.target = .{
	1754	.cpu_arch = .hexagon,
	1755	.os_tag = .linux,
	1756	.abi = .musl,
	1757	},
	1758	},
1724	1759
1725	1760	.{
1726	1761	.target = .{
...	...	@@ -1729,6 +1764,13 @@ const c_abi_targets = blk: {
1729	1764	.abi = .musl,
1730	1765	},
1731	1766	},
	1767	.{
	1768	.target = .{
	1769	.cpu_arch = .loongarch64,
	1770	.os_tag = .linux,
	1771	.abi = .muslsf,
	1772	},
	1773	},
1732	1774
1733	1775	.{
1734	1776	.target = .{

tools/update_clang_options.zig+1-1

...	...	@@ -675,7 +675,7 @@ pub fn main(init: std.process.Init) !void {
675	675	const child_args = [_][]const u8{
676	676	llvm_tblgen_exe,
677	677	"--dump-json",
678		try std.fmt.allocPrint(arena, "{s}/clang/include/clang/Driver/Options.td", .{llvm_src_root}),
	678	try std.fmt.allocPrint(arena, "{s}/clang/include/clang/Options/Options.td", .{llvm_src_root}),
679	679	try std.fmt.allocPrint(arena, "-I={s}/llvm/include", .{llvm_src_root}),
680	680	try std.fmt.allocPrint(arena, "-I={s}/clang/include/clang/Driver", .{llvm_src_root}),
681	681	};

tools/update_cpu_features.zig+55-13

...	...	@@ -203,6 +203,10 @@ const targets = [_]ArchTarget{
203	203	.llvm_name = "ampere1a",
204	204	.flatten = true,
205	205	},
	206	.{
	207	.llvm_name = "ampere1c",
	208	.flatten = true,
	209	},
206	210	.{
207	211	.llvm_name = "apple-a7",
208	212	.flatten = true,
...	...	@@ -247,6 +251,26 @@ const targets = [_]ArchTarget{
247	251	.llvm_name = "apple-m4",
248	252	.flatten = true,
249	253	},
	254	.{
	255	.llvm_name = "apple-m5",
	256	.flatten = true,
	257	},
	258	.{
	259	.llvm_name = "c1-nano",
	260	.flatten = true,
	261	},
	262	.{
	263	.llvm_name = "c1-premium",
	264	.flatten = true,
	265	},
	266	.{
	267	.llvm_name = "c1-pro",
	268	.flatten = true,
	269	},
	270	.{
	271	.llvm_name = "c1-ultra",
	272	.flatten = true,
	273	},
250	274	.{
251	275	.llvm_name = "carmel",
252	276	.flatten = true,
...	...	@@ -862,6 +886,10 @@ const targets = [_]ArchTarget{
862	886	.llvm_name = "armv9.6-a",
863	887	.zig_name = "v9_6a",
864	888	},
	889	.{
	890	.llvm_name = "armv9.7-a",
	891	.zig_name = "v9_7a",
	892	},
865	893	.{
866	894	.llvm_name = "armv9-a",
867	895	.zig_name = "v9a",
...	...	@@ -982,6 +1010,10 @@ const targets = [_]ArchTarget{
982	1010	.llvm_name = "v9.6a",
983	1011	.zig_name = "has_v9_6a",
984	1012	},
	1013	.{
	1014	.llvm_name = "v9.7a",
	1015	.zig_name = "has_v9_7a",
	1016	},
985	1017	},
986	1018	.extra_cpus = &.{
987	1019	.{
...	...	@@ -1249,6 +1281,24 @@ const targets = [_]ArchTarget{
1249	1281	.td_name = "LoongArch",
1250	1282	},
1251	1283	.extra_cpus = &.{
	1284	.{
	1285	.llvm_name = null,
	1286	.zig_name = "la32v1_0",
	1287	.features = &.{
	1288	"32bit",
	1289	"32s",
	1290	"d",
	1291	"ual",
	1292	},
	1293	},
	1294	.{
	1295	.llvm_name = null,
	1296	.zig_name = "la32rv1_0",
	1297	.features = &.{
	1298	"32bit",
	1299	"ual",
	1300	},
	1301	},
1252	1302	.{
1253	1303	.llvm_name = null,
1254	1304	.zig_name = "la64v1_0",
...	...	@@ -1276,6 +1326,7 @@ const targets = [_]ArchTarget{
1276	1326	},
1277	1327	.omit_cpus = &.{
1278	1328	"generic",
	1329	"loongarch32",
1279	1330	"loongarch64",
1280	1331	},
1281	1332	},
...	...	@@ -1557,26 +1608,17 @@ const targets = [_]ArchTarget{
1557	1608	.llvm_name = "64bit-mode",
1558	1609	.omit = true,
1559	1610	},
1560		// Remove these when LLVM removes AVX10.N-256 support.
1561		.{
1562		.llvm_name = "avx10.1-256",
1563		.flatten = true,
1564		},
1565		.{
1566		.llvm_name = "avx10.2-256",
1567		.flatten = true,
1568		},
1569	1611	.{
1570	1612	.llvm_name = "avx10.1-512",
1571		.zig_name = "avx10_1",
	1613	.omit = true,
1572	1614	},
1573	1615	.{
1574	1616	.llvm_name = "avx10.2-512",
1575		.zig_name = "avx10_2",
	1617	.omit = true,
1576	1618	},
1577	1619	.{
1578		.llvm_name = "avx512f",
1579		.extra_deps = &.{"evex512"},
	1620	.llvm_name = "evex512",
	1621	.omit = true,
1580	1622	},
1581	1623	.{
1582	1624	.llvm_name = "alderlake",