From cdd6e9ee3ad32b68c3231488bc2b6a50e5b3ace4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Thu, 31 Jul 2025 21:47:02 +0200 Subject: [PATCH 1/6] glibc: update abilists file to 2.42 --- lib/libc/glibc/abilists | Bin 247133 -> 248210 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/lib/libc/glibc/abilists b/lib/libc/glibc/abilists index 303f84023351580301d56671aaa16c8f850e1c46..d8a3ab654eba21c868f8256626a0a23f131d1be1 100644 GIT binary patch delta 2352 zcmcIlUu;ul6yJBg?oYRtcK-*1(K5soVWVRX40db`%n%J}5EGM`OV{2?Lw9d?@9nlC z33t=Q1}EvbB%dL$fk1pg2vqO);KL;DMT3StCaBm`dK$#6UB%VzNL`elMREJ*Q^ z$J(DuR(0D9i0`4I1*;wU?qVaCjJrVympvDi*Pr0jOB4LZ+csIFBPsR zx#`3&xsBRsQ1Xx1XRDwt9M#b9gP~+3cRry;1=2l$WX02D%bq&7wW?mPSafrTc~1<7 z;)8LrFos*Q4=8V}H==q)Gv*(a4d+g&ET6LO)0!=q>Ew8A_(PYLQ z4U3#I{nwW4+Iw@h2BU8Nj;$tx;W#HTBDs7Q1+wdYj(Y1D#<(71S=MlqD`b^Ul_7AK z&HV5d0#zBW>!5QDC{t6NLue^5+3Q0y%aZZXplGI@BEdG)M9bEr#wKr!VQypmo?R zZnfV}HdSI9efJOadslrSElZW@5g*(sbs9XG+0xhAL2R)2Te_6q*aZP-w#G8AG0eeK zf|nHk{94-C5C7O!8>%$_ZKybARWWUU8BT%A@G5AYez6z0)phw7%=;qClIu3KgC61G z*~&(SF$BO#e8MUKdNl>FRuyhrnu(TAg3VJo%9#HEoN9I_V?&`aOM8U>z`X$%Oi2pxStgf}3!Q4tF-bbb); zE33_=TA0FP5rN!&9&MxhV_0zPU#i(~h)ppXRFTzmDv@3V`#nrFDaD3ZPSh&#VD23} zaXA^c4oHc37Y(}s(e7Y_bkpgy7*oi1k%1QC5{$ zt%4NrR-h+}{e~lZhQpI_L}S4f4_{ z;!xr~do37=BVb_n?%VNEB8e{KyO)!82-~>5jU=Kun? z8Swc&(z={_r6AJ{yRfg7j2JP}L4vA(_dA%=B;dwd<24cR;|i-LOW7uLs~c{Wqn9 z-#AMws2YNjG#aY(u}Hp)Z+H(H9DEBtVD1_fH?4d+<8Y{JrSsefNDsNv#SKyWtv@3}j8%$`tL}&Em49<1Y39V*|$^eaWdz8iV zPeas{WuFMrX4C-7(7XvT&*qa~1kXoTr2r4bgM+bl|Fo HbdUZoD= Date: Thu, 31 Jul 2025 22:02:37 +0200 Subject: [PATCH 2/6] glibc: update headers to 2.42 --- .../include/aarch64-linux-gnu/bits/fcntl.h | 14 +-- .../aarch64-linux-gnu/bits/math-vector.h | 32 +++++++ .../bits/pthreadtypes-arch.h | 24 ++---- .../aarch64-linux-gnu/bits/semaphore.h | 8 +- .../include/aarch64-linux-gnu/bits/wordsize.h | 9 +- lib/libc/include/generic-glibc/arpa/inet.h | 5 ++ .../include/generic-glibc/bits/fcntl-linux.h | 2 + .../generic-glibc/bits/inet-fortified-decl.h | 42 +++++++++ .../generic-glibc/bits/inet-fortified.h | 61 +++++++++++++ .../include/generic-glibc/bits/ioctl-types.h | 11 --- lib/libc/include/generic-glibc/bits/ioctls.h | 85 +------------------ .../generic-glibc/bits/libm-simd-decl-stubs.h | 44 ++++++++++ .../generic-glibc/bits/mathcalls-macros.h | 2 +- .../include/generic-glibc/bits/mathcalls.h | 21 +++++ .../include/generic-glibc/bits/mman-linux.h | 2 + .../include/generic-glibc/bits/mman-shared.h | 7 +- lib/libc/include/generic-glibc/bits/sched.h | 2 +- .../generic-glibc/bits/string_fortified.h | 2 +- lib/libc/include/generic-glibc/bits/syscall.h | 24 +++++- .../include/generic-glibc/bits/termios-baud.h | 77 +++++++++++------ .../generic-glibc/bits/termios-c_cflag.h | 4 +- .../generic-glibc/bits/termios-cbaud.h | 47 ++++++++++ .../generic-glibc/bits/termios-struct.h | 11 ++- lib/libc/include/generic-glibc/bits/termios.h | 56 ++++++------ .../generic-glibc/bits/types/struct_FILE.h | 10 ++- lib/libc/include/generic-glibc/dlfcn.h | 10 ++- lib/libc/include/generic-glibc/elf.h | 33 +++---- lib/libc/include/generic-glibc/fcntl.h | 11 +-- lib/libc/include/generic-glibc/features.h | 6 +- lib/libc/include/generic-glibc/glob.h | 3 +- lib/libc/include/generic-glibc/inttypes.h | 5 ++ lib/libc/include/generic-glibc/malloc.h | 4 +- lib/libc/include/generic-glibc/netinet/tcp.h | 3 + lib/libc/include/generic-glibc/pthread.h | 5 ++ lib/libc/include/generic-glibc/resolv.h | 4 +- lib/libc/include/generic-glibc/stdio.h | 6 +- lib/libc/include/generic-glibc/stdio_ext.h | 18 ++-- lib/libc/include/generic-glibc/stdlib.h | 12 ++- lib/libc/include/generic-glibc/string.h | 4 +- lib/libc/include/generic-glibc/sys/hwprobe.h | 40 ++++++--- lib/libc/include/generic-glibc/sys/ifunc.h | 65 ++++++++++++-- lib/libc/include/generic-glibc/sys/mount.h | 2 +- lib/libc/include/generic-glibc/sys/ttychars.h | 4 - lib/libc/include/generic-glibc/termio.h | 6 -- lib/libc/include/generic-glibc/termios.h | 20 +++++ lib/libc/include/generic-glibc/tgmath.h | 18 ++++ lib/libc/include/generic-glibc/unistd.h | 2 +- .../include/mips-linux-gnu/bits/ioctl-types.h | 12 --- .../mips-linux-gnu/bits/termios-struct.h | 34 -------- .../powerpc-linux-gnu/bits/ioctl-types.h | 11 --- .../powerpc-linux-gnu/bits/termios-c_cflag.h | 4 +- .../bits/{termios-baud.h => termios-cbaud.h} | 42 ++++----- .../bits/{termios-baud.h => termios-cbaud.h} | 39 +++++---- .../sparc-linux-gnu/bits/termios-struct.h | 34 -------- lib/libc/include/x86-linux-gnu/bits/floatn.h | 10 ++- 55 files changed, 651 insertions(+), 418 deletions(-) create mode 100644 lib/libc/include/generic-glibc/bits/inet-fortified-decl.h create mode 100644 lib/libc/include/generic-glibc/bits/inet-fortified.h create mode 100644 lib/libc/include/generic-glibc/bits/termios-cbaud.h delete mode 100644 lib/libc/include/generic-glibc/termio.h delete mode 100644 lib/libc/include/mips-linux-gnu/bits/termios-struct.h rename lib/libc/include/powerpc-linux-gnu/bits/{termios-baud.h => termios-cbaud.h} (58%) rename lib/libc/include/sparc-linux-gnu/bits/{termios-baud.h => termios-cbaud.h} (58%) delete mode 100644 lib/libc/include/sparc-linux-gnu/bits/termios-struct.h diff --git a/lib/libc/include/aarch64-linux-gnu/bits/fcntl.h b/lib/libc/include/aarch64-linux-gnu/bits/fcntl.h index aaecf2bbc04527dc74d67713c87ce7371c5bba02..52dee64f6f1db38ed4cfb31c5463206d8cb459c7 100644 --- a/lib/libc/include/aarch64-linux-gnu/bits/fcntl.h +++ b/lib/libc/include/aarch64-linux-gnu/bits/fcntl.h @@ -25,17 +25,11 @@ #define __O_NOFOLLOW 0100000 #define __O_DIRECT 0200000 -#ifdef __ILP32__ -# define __O_LARGEFILE 0400000 -#else -# define __O_LARGEFILE 0 -#endif +#define __O_LARGEFILE 0 -#ifdef __LP64__ -# define F_GETLK64 5 -# define F_SETLK64 6 -# define F_SETLKW64 7 -#endif +#define F_GETLK64 5 +#define F_SETLK64 6 +#define F_SETLKW64 7 struct flock { diff --git a/lib/libc/include/aarch64-linux-gnu/bits/math-vector.h b/lib/libc/include/aarch64-linux-gnu/bits/math-vector.h index 95d52f96fc412d13acfc0845af569f792c8903e6..c4506228796577d29ffb3feef1ae910a36221004 100644 --- a/lib/libc/include/aarch64-linux-gnu/bits/math-vector.h +++ b/lib/libc/include/aarch64-linux-gnu/bits/math-vector.h @@ -37,6 +37,10 @@ # define __DECL_SIMD_acosh __DECL_SIMD_aarch64 # undef __DECL_SIMD_acoshf # define __DECL_SIMD_acoshf __DECL_SIMD_aarch64 +# undef __DECL_SIMD_acospi +# define __DECL_SIMD_acospi __DECL_SIMD_aarch64 +# undef __DECL_SIMD_acospif +# define __DECL_SIMD_acospif __DECL_SIMD_aarch64 # undef __DECL_SIMD_asin # define __DECL_SIMD_asin __DECL_SIMD_aarch64 # undef __DECL_SIMD_asinf @@ -45,6 +49,10 @@ # define __DECL_SIMD_asinh __DECL_SIMD_aarch64 # undef __DECL_SIMD_asinhf # define __DECL_SIMD_asinhf __DECL_SIMD_aarch64 +# undef __DECL_SIMD_asinpi +# define __DECL_SIMD_asinpi __DECL_SIMD_aarch64 +# undef __DECL_SIMD_asinpif +# define __DECL_SIMD_asinpif __DECL_SIMD_aarch64 # undef __DECL_SIMD_atan # define __DECL_SIMD_atan __DECL_SIMD_aarch64 # undef __DECL_SIMD_atanf @@ -53,10 +61,18 @@ # define __DECL_SIMD_atanh __DECL_SIMD_aarch64 # undef __DECL_SIMD_atanhf # define __DECL_SIMD_atanhf __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atanpi +# define __DECL_SIMD_atanpi __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atanpif +# define __DECL_SIMD_atanpif __DECL_SIMD_aarch64 # undef __DECL_SIMD_atan2 # define __DECL_SIMD_atan2 __DECL_SIMD_aarch64 # undef __DECL_SIMD_atan2f # define __DECL_SIMD_atan2f __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atan2pi +# define __DECL_SIMD_atan2pi __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atan2pif +# define __DECL_SIMD_atan2pif __DECL_SIMD_aarch64 # undef __DECL_SIMD_cbrt # define __DECL_SIMD_cbrt __DECL_SIMD_aarch64 # undef __DECL_SIMD_cbrtf @@ -176,12 +192,16 @@ typedef __SVBool_t __sv_bool_t; # define __vpcs __attribute__ ((__aarch64_vector_pcs__)) __vpcs __f32x4_t _ZGVnN4vv_atan2f (__f32x4_t, __f32x4_t); +__vpcs __f32x4_t _ZGVnN4vv_atan2pif (__f32x4_t, __f32x4_t); __vpcs __f32x4_t _ZGVnN4v_acosf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_acoshf (__f32x4_t); +__vpcs __f32x4_t _ZGVnN4v_acospif (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_asinf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_asinhf (__f32x4_t); +__vpcs __f32x4_t _ZGVnN4v_asinpif (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_atanf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_atanhf (__f32x4_t); +__vpcs __f32x4_t _ZGVnN4v_atanpif (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_cbrtf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_cosf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_coshf (__f32x4_t); @@ -207,12 +227,16 @@ __vpcs __f32x4_t _ZGVnN4v_tanhf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_tanpif (__f32x4_t); __vpcs __f64x2_t _ZGVnN2vv_atan2 (__f64x2_t, __f64x2_t); +__vpcs __f64x2_t _ZGVnN2vv_atan2pi (__f64x2_t, __f64x2_t); __vpcs __f64x2_t _ZGVnN2v_acos (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_acosh (__f64x2_t); +__vpcs __f64x2_t _ZGVnN2v_acospi (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_asin (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_asinh (__f64x2_t); +__vpcs __f64x2_t _ZGVnN2v_asinpi (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_atan (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_atanh (__f64x2_t); +__vpcs __f64x2_t _ZGVnN2v_atanpi (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cbrt (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cos (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cosh (__f64x2_t); @@ -243,12 +267,16 @@ __vpcs __f64x2_t _ZGVnN2v_tanpi (__f64x2_t); #ifdef __SVE_VEC_MATH_SUPPORTED __sv_f32_t _ZGVsMxvv_atan2f (__sv_f32_t, __sv_f32_t, __sv_bool_t); +__sv_f32_t _ZGVsMxvv_atan2pif (__sv_f32_t, __sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_acosf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_acoshf (__sv_f32_t, __sv_bool_t); +__sv_f32_t _ZGVsMxv_acospif (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_asinf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_asinhf (__sv_f32_t, __sv_bool_t); +__sv_f32_t _ZGVsMxv_asinpif (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_atanf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_atanhf (__sv_f32_t, __sv_bool_t); +__sv_f32_t _ZGVsMxv_atanpif (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_cbrtf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_cosf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_coshf (__sv_f32_t, __sv_bool_t); @@ -274,12 +302,16 @@ __sv_f32_t _ZGVsMxv_tanhf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_tanpif (__sv_f32_t, __sv_bool_t); __sv_f64_t _ZGVsMxvv_atan2 (__sv_f64_t, __sv_f64_t, __sv_bool_t); +__sv_f64_t _ZGVsMxvv_atan2pi (__sv_f64_t, __sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_acos (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_acosh (__sv_f64_t, __sv_bool_t); +__sv_f64_t _ZGVsMxv_acospi (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_asin (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_asinh (__sv_f64_t, __sv_bool_t); +__sv_f64_t _ZGVsMxv_asinpi (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_atan (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_atanh (__sv_f64_t, __sv_bool_t); +__sv_f64_t _ZGVsMxv_atanpi (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cbrt (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cos (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cosh (__sv_f64_t, __sv_bool_t); diff --git a/lib/libc/include/aarch64-linux-gnu/bits/pthreadtypes-arch.h b/lib/libc/include/aarch64-linux-gnu/bits/pthreadtypes-arch.h index bee5cc5acf9d45fe5cd16f30ba13016ba0d4e3c7..2dbe12da0ca4cdde3426daf253d84375a36b9bd1 100644 --- a/lib/libc/include/aarch64-linux-gnu/bits/pthreadtypes-arch.h +++ b/lib/libc/include/aarch64-linux-gnu/bits/pthreadtypes-arch.h @@ -21,23 +21,13 @@ #include -#ifdef __ILP32__ -# define __SIZEOF_PTHREAD_ATTR_T 32 -# define __SIZEOF_PTHREAD_MUTEX_T 32 -# define __SIZEOF_PTHREAD_MUTEXATTR_T 4 -# define __SIZEOF_PTHREAD_CONDATTR_T 4 -# define __SIZEOF_PTHREAD_RWLOCK_T 48 -# define __SIZEOF_PTHREAD_BARRIER_T 20 -# define __SIZEOF_PTHREAD_BARRIERATTR_T 4 -#else -# define __SIZEOF_PTHREAD_ATTR_T 64 -# define __SIZEOF_PTHREAD_MUTEX_T 48 -# define __SIZEOF_PTHREAD_MUTEXATTR_T 8 -# define __SIZEOF_PTHREAD_CONDATTR_T 8 -# define __SIZEOF_PTHREAD_RWLOCK_T 56 -# define __SIZEOF_PTHREAD_BARRIER_T 32 -# define __SIZEOF_PTHREAD_BARRIERATTR_T 8 -#endif +#define __SIZEOF_PTHREAD_ATTR_T 64 +#define __SIZEOF_PTHREAD_MUTEX_T 48 +#define __SIZEOF_PTHREAD_MUTEXATTR_T 8 +#define __SIZEOF_PTHREAD_CONDATTR_T 8 +#define __SIZEOF_PTHREAD_RWLOCK_T 56 +#define __SIZEOF_PTHREAD_BARRIER_T 32 +#define __SIZEOF_PTHREAD_BARRIERATTR_T 8 #define __SIZEOF_PTHREAD_COND_T 48 #define __SIZEOF_PTHREAD_RWLOCKATTR_T 8 diff --git a/lib/libc/include/aarch64-linux-gnu/bits/semaphore.h b/lib/libc/include/aarch64-linux-gnu/bits/semaphore.h index 256f68bbfc7506da0744597c30f474d40a3dae30..7d3d93431e67f2a62e40c4fdc1c4e169aa71e096 100644 --- a/lib/libc/include/aarch64-linux-gnu/bits/semaphore.h +++ b/lib/libc/include/aarch64-linux-gnu/bits/semaphore.h @@ -20,13 +20,7 @@ # error "Never use directly; include instead." #endif - -#ifdef __ILP32__ -# define __SIZEOF_SEM_T 16 -#else -# define __SIZEOF_SEM_T 32 -#endif - +#define __SIZEOF_SEM_T 32 /* Value returned if `sem_open' failed. */ #define SEM_FAILED ((sem_t *) 0) diff --git a/lib/libc/include/aarch64-linux-gnu/bits/wordsize.h b/lib/libc/include/aarch64-linux-gnu/bits/wordsize.h index 30b14a9b640f351a7c4a7d030f7a5168213a7a38..3f84507cc16e18556599b2f90bda882e6e283173 100644 --- a/lib/libc/include/aarch64-linux-gnu/bits/wordsize.h +++ b/lib/libc/include/aarch64-linux-gnu/bits/wordsize.h @@ -17,12 +17,5 @@ License along with the GNU C Library; if not, see . */ -#ifdef __LP64__ -# define __WORDSIZE 64 -#else -# define __WORDSIZE 32 -# define __WORDSIZE32_SIZE_ULONG 1 -# define __WORDSIZE32_PTRDIFF_LONG 1 -#endif - +#define __WORDSIZE 64 #define __WORDSIZE_TIME64_COMPAT32 0 \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/arpa/inet.h b/lib/libc/include/generic-glibc/arpa/inet.h index 8d6490ef478cf3d243fd37dfcc496d0681dae311..84602130a9351f781e05e8fa551464150e8d67c8 100644 --- a/lib/libc/include/generic-glibc/arpa/inet.h +++ b/lib/libc/include/generic-glibc/arpa/inet.h @@ -101,6 +101,11 @@ extern char *inet_nsap_ntoa (int __len, const unsigned char *__cp, char *__buf) __THROW; #endif +#if __USE_FORTIFY_LEVEL > 0 && defined __fortify_function +/* Include functions with security checks. */ +# include +#endif + __END_DECLS #endif /* arpa/inet.h */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/fcntl-linux.h b/lib/libc/include/generic-glibc/bits/fcntl-linux.h index 8ebc3319b61e54b55af352975418758ef52e366f..56169a7ba49047e7442aadc3593b6592c04c028b 100644 --- a/lib/libc/include/generic-glibc/bits/fcntl-linux.h +++ b/lib/libc/include/generic-glibc/bits/fcntl-linux.h @@ -379,6 +379,8 @@ struct file_handle identity and may not be usable to open_by_handle_at. */ +# define AT_HANDLE_MNT_ID_UNIQUE 1 /* Return the 64-bit unique mount + ID. */ #endif __BEGIN_DECLS diff --git a/lib/libc/include/generic-glibc/bits/inet-fortified-decl.h b/lib/libc/include/generic-glibc/bits/inet-fortified-decl.h new file mode 100644 index 0000000000000000000000000000000000000000..a77a0477cda6cc1ee8f12046c3ae501db9eccf2c --- /dev/null +++ b/lib/libc/include/generic-glibc/bits/inet-fortified-decl.h @@ -0,0 +1,42 @@ +/* Declarations of checking macros for inet functions. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#ifndef _BITS_INET_FORTIFIED_DEC_H +#define _BITS_INET_FORTIFIED_DEC_H 1 + +#ifndef _ARPA_INET_H +# error "Never include directly; use instead." +#endif + +extern const char *__inet_ntop_chk (int, const void *, char *, socklen_t, size_t); + +extern const char *__REDIRECT_FORTIFY_NTH (__inet_ntop_alias, + (int, const void *, char *, socklen_t), inet_ntop); +extern const char *__REDIRECT_NTH (__inet_ntop_chk_warn, + (int, const void *, char *, socklen_t, size_t), __inet_ntop_chk) + __warnattr ("inet_ntop called with bigger length than " + "size of destination buffer"); + +extern int __inet_pton_chk (int, const char *, void *, size_t); + +extern int __REDIRECT_FORTIFY_NTH (__inet_pton_alias, + (int, const char *, void *), inet_pton); +extern int __REDIRECT_NTH (__inet_pton_chk_warn, + (int, const char *, void *, size_t), __inet_pton_chk) + __warnattr ("inet_pton called with a destination buffer size too small"); +#endif /* bits/inet-fortified-decl.h. */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/inet-fortified.h b/lib/libc/include/generic-glibc/bits/inet-fortified.h new file mode 100644 index 0000000000000000000000000000000000000000..88aafbe8b20b48dde9190733f0a80561259f4f66 --- /dev/null +++ b/lib/libc/include/generic-glibc/bits/inet-fortified.h @@ -0,0 +1,61 @@ +/* Checking macros for inet functions. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#ifndef _BITS_INET_FORTIFIED_H +#define _BITS_INET_FORTIFIED_H 1 + +#ifndef _ARPA_INET_H +# error "Never include directly; use instead." +#endif + +#include + +__fortify_function __attribute_overloadable__ const char * +__NTH (inet_ntop (int __af, + __fortify_clang_overload_arg (const void *, __restrict, __src), + char *__restrict __dst, socklen_t __dst_size)) + __fortify_clang_warning_only_if_bos_lt (__dst_size, __dst, + "inet_ntop called with bigger length " + "than size of destination buffer") +{ + return __glibc_fortify (inet_ntop, __dst_size, sizeof (char), + __glibc_objsize (__dst), + __af, __src, __dst, __dst_size); +}; + +__fortify_function __attribute_overloadable__ int +__NTH (inet_pton (int __af, + const char *__restrict __src, + __fortify_clang_overload_arg (void *, __restrict, __dst))) + __fortify_clang_warning_only_if_bos0_lt + (4, __dst, "inet_pton called with destination buffer size less than 4") +{ + size_t sz = 0; + if (__af == AF_INET) + sz = sizeof (struct in_addr); + else if (__af == AF_INET6) + sz = sizeof (struct in6_addr); + else + return __inet_pton_alias (__af, __src, __dst); + + return __glibc_fortify (inet_pton, sz, sizeof (char), + __glibc_objsize (__dst), + __af, __src, __dst); +}; + +#endif /* bits/inet-fortified.h. */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/ioctl-types.h b/lib/libc/include/generic-glibc/bits/ioctl-types.h index 08562e7ba199d0bb18616dc8d16003904d41fb37..13c5bc0d455f20c1beedf540540d869e075f9c7f 100644 --- a/lib/libc/include/generic-glibc/bits/ioctl-types.h +++ b/lib/libc/include/generic-glibc/bits/ioctl-types.h @@ -32,17 +32,6 @@ struct winsize unsigned short int ws_ypixel; }; -#define NCC 8 -struct termio - { - unsigned short int c_iflag; /* input mode flags */ - unsigned short int c_oflag; /* output mode flags */ - unsigned short int c_cflag; /* control mode flags */ - unsigned short int c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - /* modem lines */ #define TIOCM_LE 0x001 #define TIOCM_DTR 0x002 diff --git a/lib/libc/include/generic-glibc/bits/ioctls.h b/lib/libc/include/generic-glibc/bits/ioctls.h index 915563810f358d892ae659da462c7c559918c267..84594dd239475382b220094e13d871dd45b58682 100644 --- a/lib/libc/include/generic-glibc/bits/ioctls.h +++ b/lib/libc/include/generic-glibc/bits/ioctls.h @@ -22,87 +22,4 @@ /* Use the definitions from the kernel header files. */ #include -/* Routing table calls. */ -#define SIOCADDRT 0x890B /* add routing table entry */ -#define SIOCDELRT 0x890C /* delete routing table entry */ -#define SIOCRTMSG 0x890D /* call to routing system */ - -/* Socket configuration controls. */ -#define SIOCGIFNAME 0x8910 /* get iface name */ -#define SIOCSIFLINK 0x8911 /* set iface channel */ -#define SIOCGIFCONF 0x8912 /* get iface list */ -#define SIOCGIFFLAGS 0x8913 /* get flags */ -#define SIOCSIFFLAGS 0x8914 /* set flags */ -#define SIOCGIFADDR 0x8915 /* get PA address */ -#define SIOCSIFADDR 0x8916 /* set PA address */ -#define SIOCGIFDSTADDR 0x8917 /* get remote PA address */ -#define SIOCSIFDSTADDR 0x8918 /* set remote PA address */ -#define SIOCGIFBRDADDR 0x8919 /* get broadcast PA address */ -#define SIOCSIFBRDADDR 0x891a /* set broadcast PA address */ -#define SIOCGIFNETMASK 0x891b /* get network PA mask */ -#define SIOCSIFNETMASK 0x891c /* set network PA mask */ -#define SIOCGIFMETRIC 0x891d /* get metric */ -#define SIOCSIFMETRIC 0x891e /* set metric */ -#define SIOCGIFMEM 0x891f /* get memory address (BSD) */ -#define SIOCSIFMEM 0x8920 /* set memory address (BSD) */ -#define SIOCGIFMTU 0x8921 /* get MTU size */ -#define SIOCSIFMTU 0x8922 /* set MTU size */ -#define SIOCSIFNAME 0x8923 /* set interface name */ -#define SIOCSIFHWADDR 0x8924 /* set hardware address */ -#define SIOCGIFENCAP 0x8925 /* get/set encapsulations */ -#define SIOCSIFENCAP 0x8926 -#define SIOCGIFHWADDR 0x8927 /* Get hardware address */ -#define SIOCGIFSLAVE 0x8929 /* Driver slaving support */ -#define SIOCSIFSLAVE 0x8930 -#define SIOCADDMULTI 0x8931 /* Multicast address lists */ -#define SIOCDELMULTI 0x8932 -#define SIOCGIFINDEX 0x8933 /* name -> if_index mapping */ -#define SIOGIFINDEX SIOCGIFINDEX /* misprint compatibility :-) */ -#define SIOCSIFPFLAGS 0x8934 /* set/get extended flags set */ -#define SIOCGIFPFLAGS 0x8935 -#define SIOCDIFADDR 0x8936 /* delete PA address */ -#define SIOCSIFHWBROADCAST 0x8937 /* set hardware broadcast addr */ -#define SIOCGIFCOUNT 0x8938 /* get number of devices */ - -#define SIOCGIFBR 0x8940 /* Bridging support */ -#define SIOCSIFBR 0x8941 /* Set bridging options */ - -#define SIOCGIFTXQLEN 0x8942 /* Get the tx queue length */ -#define SIOCSIFTXQLEN 0x8943 /* Set the tx queue length */ - - -/* ARP cache control calls. */ - /* 0x8950 - 0x8952 * obsolete calls, don't re-use */ -#define SIOCDARP 0x8953 /* delete ARP table entry */ -#define SIOCGARP 0x8954 /* get ARP table entry */ -#define SIOCSARP 0x8955 /* set ARP table entry */ - -/* RARP cache control calls. */ -#define SIOCDRARP 0x8960 /* delete RARP table entry */ -#define SIOCGRARP 0x8961 /* get RARP table entry */ -#define SIOCSRARP 0x8962 /* set RARP table entry */ - -/* Driver configuration calls */ - -#define SIOCGIFMAP 0x8970 /* Get device parameters */ -#define SIOCSIFMAP 0x8971 /* Set device parameters */ - -/* DLCI configuration calls */ - -#define SIOCADDDLCI 0x8980 /* Create new DLCI device */ -#define SIOCDELDLCI 0x8981 /* Delete DLCI device */ - -/* Device private ioctl calls. */ - -/* These 16 ioctls are available to devices via the do_ioctl() device - vector. Each device should include this file and redefine these - names as their own. Because these are device dependent it is a good - idea _NOT_ to issue them to random objects and hope. */ - -#define SIOCDEVPRIVATE 0x89F0 /* to 89FF */ - -/* - * These 16 ioctl calls are protocol private - */ - -#define SIOCPROTOPRIVATE 0x89E0 /* to 89EF */ \ No newline at end of file +#include \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/libm-simd-decl-stubs.h b/lib/libc/include/generic-glibc/bits/libm-simd-decl-stubs.h index 436770e11f0aab84911a46f494a1f5dbe8eb7119..15df4ea88fd10afc9e4a47cbce89951aa76477b1 100644 --- a/lib/libc/include/generic-glibc/bits/libm-simd-decl-stubs.h +++ b/lib/libc/include/generic-glibc/bits/libm-simd-decl-stubs.h @@ -373,4 +373,48 @@ #define __DECL_SIMD_tanpif32x #define __DECL_SIMD_tanpif64x #define __DECL_SIMD_tanpif128x + +#define __DECL_SIMD_acospi +#define __DECL_SIMD_acospif +#define __DECL_SIMD_acospil +#define __DECL_SIMD_acospif16 +#define __DECL_SIMD_acospif32 +#define __DECL_SIMD_acospif64 +#define __DECL_SIMD_acospif128 +#define __DECL_SIMD_acospif32x +#define __DECL_SIMD_acospif64x +#define __DECL_SIMD_acospif128x + +#define __DECL_SIMD_asinpi +#define __DECL_SIMD_asinpif +#define __DECL_SIMD_asinpil +#define __DECL_SIMD_asinpif16 +#define __DECL_SIMD_asinpif32 +#define __DECL_SIMD_asinpif64 +#define __DECL_SIMD_asinpif128 +#define __DECL_SIMD_asinpif32x +#define __DECL_SIMD_asinpif64x +#define __DECL_SIMD_asinpif128x + +#define __DECL_SIMD_atanpi +#define __DECL_SIMD_atanpif +#define __DECL_SIMD_atanpil +#define __DECL_SIMD_atanpif16 +#define __DECL_SIMD_atanpif32 +#define __DECL_SIMD_atanpif64 +#define __DECL_SIMD_atanpif128 +#define __DECL_SIMD_atanpif32x +#define __DECL_SIMD_atanpif64x +#define __DECL_SIMD_atanpif128x + +#define __DECL_SIMD_atan2pi +#define __DECL_SIMD_atan2pif +#define __DECL_SIMD_atan2pil +#define __DECL_SIMD_atan2pif16 +#define __DECL_SIMD_atan2pif32 +#define __DECL_SIMD_atan2pif64 +#define __DECL_SIMD_atan2pif128 +#define __DECL_SIMD_atan2pif32x +#define __DECL_SIMD_atan2pif64x +#define __DECL_SIMD_atan2pif128x #endif \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/mathcalls-macros.h b/lib/libc/include/generic-glibc/bits/mathcalls-macros.h index 6c29a7785c1a046491258aa5f67f7d348942eb3b..d5b71472bb53796288a42f51a99525beefe49a70 100644 --- a/lib/libc/include/generic-glibc/bits/mathcalls-macros.h +++ b/lib/libc/include/generic-glibc/bits/mathcalls-macros.h @@ -34,7 +34,7 @@ #define __MATHCALLX(function,suffix, args, attrib) \ __MATHDECLX (_Mdouble_,function,suffix, args, attrib) #define __MATHDECLX(type, function,suffix, args, attrib) \ - __MATHDECL_1(type, function,suffix, args) __attribute__ (attrib); + __MATHDECL_1(type, function,suffix, args) __attribute__ (attrib) #define __MATHDECL_1_IMPL(type, function, suffix, args) \ extern type __MATH_PRECNAME(function,suffix) args __THROW #define __MATHDECL_1(type, function, suffix, args) \ diff --git a/lib/libc/include/generic-glibc/bits/mathcalls.h b/lib/libc/include/generic-glibc/bits/mathcalls.h index 4f93b693d825492d18b14311537fa626e9699d72..265c1b867503e5f4a763f9c7ae99f3251590667b 100644 --- a/lib/libc/include/generic-glibc/bits/mathcalls.h +++ b/lib/libc/include/generic-glibc/bits/mathcalls.h @@ -68,12 +68,16 @@ __MATHCALL_VEC (tan,, (_Mdouble_ __x)); #if __GLIBC_USE (IEC_60559_FUNCS_EXT_C23) /* Arc cosine of X, divided by pi. */ __MATHCALL (acospi,, (_Mdouble_ __x)); +__MATHCALL_VEC (acospi,, (_Mdouble_ __x)); /* Arc sine of X, divided by pi. */ __MATHCALL (asinpi,, (_Mdouble_ __x)); +__MATHCALL_VEC (asinpi,, (_Mdouble_ __x)); /* Arc tangent of X, divided by pi. */ __MATHCALL (atanpi,, (_Mdouble_ __x)); +__MATHCALL_VEC (atanpi,, (_Mdouble_ __x)); /* Arc tangent of Y/X, divided by pi. */ __MATHCALL (atan2pi,, (_Mdouble_ __y, _Mdouble_ __x)); +__MATHCALL_VEC (atan2pi,, (_Mdouble_ __y, _Mdouble_ __x)); /* Cosine of pi * X. */ __MATHCALL_VEC (cospi,, (_Mdouble_ __x)); @@ -185,6 +189,23 @@ __MATHCALL_VEC (hypot,, (_Mdouble_ __x, _Mdouble_ __y)); __MATHCALL_VEC (cbrt,, (_Mdouble_ __x)); #endif +#if __GLIBC_USE (IEC_60559_FUNCS_EXT_C23) +/* Return 1+X to the Y power. */ +__MATHCALL (compoundn,, (_Mdouble_ __x, long long int __y)); + +/* Return X to the Y power. */ +__MATHCALL (pown,, (_Mdouble_ __x, long long int __y)); + +/* Return X to the Y power. */ +__MATHCALL (powr,, (_Mdouble_ __x, _Mdouble_ __y)); + +/* Return the Yth root of X. */ +__MATHCALL (rootn,, (_Mdouble_ __x, long long int __y)); + +/* Return the reciprocal of the square root of X. */ +__MATHCALL (rsqrt,, (_Mdouble_ __x)); +#endif + /* Nearest integer, absolute value, and remainder functions. */ diff --git a/lib/libc/include/generic-glibc/bits/mman-linux.h b/lib/libc/include/generic-glibc/bits/mman-linux.h index 58ea3d02c80affa7089e9c200a900460e8071b1e..2dbd4d3a06f73383b43724bd45a013a488363277 100644 --- a/lib/libc/include/generic-glibc/bits/mman-linux.h +++ b/lib/libc/include/generic-glibc/bits/mman-linux.h @@ -113,6 +113,8 @@ locked pages too. */ # define MADV_COLLAPSE 25 /* Synchronous hugepage collapse. */ # define MADV_HWPOISON 100 /* Poison a page for testing. */ +# define MADV_GUARD_INSTALL 102 /* Fatal signal on access to range */ +# define MADV_GUARD_REMOVE 103 /* Unguard range */ #endif /* The POSIX people had to invent similar names for the same things. */ diff --git a/lib/libc/include/generic-glibc/bits/mman-shared.h b/lib/libc/include/generic-glibc/bits/mman-shared.h index a43ef000be1a7741b506cc906c3816ac163dc582..39e82407cd5a9c722e74b81db37c7c5895187457 100644 --- a/lib/libc/include/generic-glibc/bits/mman-shared.h +++ b/lib/libc/include/generic-glibc/bits/mman-shared.h @@ -43,10 +43,9 @@ # endif /* Access restrictions for pkey_alloc. */ -# ifndef PKEY_DISABLE_ACCESS -# define PKEY_DISABLE_ACCESS 0x1 -# define PKEY_DISABLE_WRITE 0x2 -# endif +# define PKEY_UNRESTRICTED 0x0 +# define PKEY_DISABLE_ACCESS 0x1 +# define PKEY_DISABLE_WRITE 0x2 __BEGIN_DECLS diff --git a/lib/libc/include/generic-glibc/bits/sched.h b/lib/libc/include/generic-glibc/bits/sched.h index 3ac3e6c15330018dfa844fddb60c6129463a3616..c0e7cd3ead0630fcec65b47a0878b5e57a27c581 100644 --- a/lib/libc/include/generic-glibc/bits/sched.h +++ b/lib/libc/include/generic-glibc/bits/sched.h @@ -152,7 +152,7 @@ int sched_setattr (pid_t tid, struct sched_attr *attr, unsigned int flags) store it in *ATTR. */ int sched_getattr (pid_t tid, struct sched_attr *attr, unsigned int size, unsigned int flags) - __THROW __nonnull ((2)) __attr_access ((__write_only__, 2, 3)); + __THROW __nonnull ((2)); #endif diff --git a/lib/libc/include/generic-glibc/bits/string_fortified.h b/lib/libc/include/generic-glibc/bits/string_fortified.h index 076ca3942d41c98cf1227cf7457b2eb1978c054c..9c36d02fb1c87876117aef872b857fdef724b000 100644 --- a/lib/libc/include/generic-glibc/bits/string_fortified.h +++ b/lib/libc/include/generic-glibc/bits/string_fortified.h @@ -151,7 +151,7 @@ __NTH (strncat (__fortify_clang_overload_arg (char *, __restrict, __dest), } /* - * strlcpy and strlcat introduced in glibc 2.38 + * zig patch: strlcpy and strlcat introduced in glibc 2.38 * https://sourceware.org/git/?p=glibc.git;a=commit;h=2e0bbbfbf95fc9e22692e93658a6fbdd2d4554da */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 38) || __GLIBC__ > 2 diff --git a/lib/libc/include/generic-glibc/bits/syscall.h b/lib/libc/include/generic-glibc/bits/syscall.h index 4594b36b79d5389d8d4cd176112c9486a899db35..3c70ff32d493f92febe6d4aaf567c305e35844b9 100644 --- a/lib/libc/include/generic-glibc/bits/syscall.h +++ b/lib/libc/include/generic-glibc/bits/syscall.h @@ -1,11 +1,11 @@ /* Generated at libc build time from syscall list. */ -/* The system call list corresponds to kernel 6.12. */ +/* The system call list corresponds to kernel 6.15. */ #ifndef _SYSCALL_H # error "Never use directly; include instead." #endif -#define __GLIBC_LINUX_VERSION_CODE 396288 +#define __GLIBC_LINUX_VERSION_CODE 397056 #ifdef __NR_FAST_atomic_update # define SYS_FAST_atomic_update __NR_FAST_atomic_update @@ -703,6 +703,10 @@ # define SYS_getxattr __NR_getxattr #endif +#ifdef __NR_getxattrat +# define SYS_getxattrat __NR_getxattrat +#endif + #ifdef __NR_getxgid # define SYS_getxgid __NR_getxgid #endif @@ -875,6 +879,10 @@ # define SYS_listxattr __NR_listxattr #endif +#ifdef __NR_listxattrat +# define SYS_listxattrat __NR_listxattrat +#endif + #ifdef __NR_llistxattr # define SYS_llistxattr __NR_llistxattr #endif @@ -1167,6 +1175,10 @@ # define SYS_open_tree __NR_open_tree #endif +#ifdef __NR_open_tree_attr +# define SYS_open_tree_attr __NR_open_tree_attr +#endif + #ifdef __NR_openat # define SYS_openat __NR_openat #endif @@ -1839,6 +1851,10 @@ # define SYS_removexattr __NR_removexattr #endif +#ifdef __NR_removexattrat +# define SYS_removexattrat __NR_removexattrat +#endif + #ifdef __NR_rename # define SYS_rename __NR_rename #endif @@ -2199,6 +2215,10 @@ # define SYS_setxattr __NR_setxattr #endif +#ifdef __NR_setxattrat +# define SYS_setxattrat __NR_setxattrat +#endif + #ifdef __NR_sgetmask # define SYS_sgetmask __NR_sgetmask #endif diff --git a/lib/libc/include/generic-glibc/bits/termios-baud.h b/lib/libc/include/generic-glibc/bits/termios-baud.h index dcd8be77901a91a84fbd0e7d0dde059cc8adc3d6..cd11a7e55b5d5a6454b20d66b51ca731104e4102 100644 --- a/lib/libc/include/generic-glibc/bits/termios-baud.h +++ b/lib/libc/include/generic-glibc/bits/termios-baud.h @@ -1,4 +1,4 @@ -/* termios baud rate selection definitions. Linux/generic version. +/* termios baud rate selection definitions. Universal version for sane speed_t. Copyright (C) 2019-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -20,29 +20,56 @@ # error "Never include directly; use instead." #endif -#ifdef __USE_MISC -# define CBAUD 000000010017 /* Baud speed mask (not in POSIX). */ -# define CBAUDEX 000000010000 /* Extra baud speed mask, included in CBAUD. - (not in POSIX). */ -# define CIBAUD 002003600000 /* Input baud rate (not used). */ -# define CMSPAR 010000000000 /* Mark or space (stick) parity. */ -# define CRTSCTS 020000000000 /* Flow control. */ +/* POSIX required baud rates */ +#define B0 0U /* Hang up or ispeed == ospeed */ +#define B50 50U +#define B75 75U +#define B110 110U +#define B134 134U /* Really 134.5 baud by POSIX spec */ +#define B150 150U +#define B200 200U +#define B300 300U +#define B600 600U +#define B1200 1200U +#define B1800 1800U +#define B2400 2400U +#define B4800 4800U +#define B9600 9600U +#define B19200 19200U +#define B38400 38400U +#ifdef __USE_MISC +# define EXTA B19200 +# define EXTB B38400 #endif -/* Extra output baud rates (not in POSIX). */ -#define B57600 0010001 -#define B115200 0010002 -#define B230400 0010003 -#define B460800 0010004 -#define B500000 0010005 -#define B576000 0010006 -#define B921600 0010007 -#define B1000000 0010010 -#define B1152000 0010011 -#define B1500000 0010012 -#define B2000000 0010013 -#define B2500000 0010014 -#define B3000000 0010015 -#define B3500000 0010016 -#define B4000000 0010017 -#define __MAX_BAUD B4000000 \ No newline at end of file +/* Other baud rates, "nonstandard" but known to be used */ +#define B7200 7200U +#define B14400 14400U +#define B28800 28800U +#define B33600 33600U +#define B57600 57600U +#define B76800 76800U +#define B115200 115200U +#define B153600 153600U +#define B230400 230400U +#define B307200 307200U +#define B460800 460800U +#define B500000 500000U +#define B576000 576000U +#define B614400 614400U +#define B921600 921600U +#define B1000000 1000000U +#define B1152000 1152000U +#define B1500000 1500000U +#define B2000000 2000000U +#define B2500000 2500000U +#define B3000000 3000000U +#define B3500000 3500000U +#define B4000000 4000000U +#define B5000000 5000000U +#define B10000000 10000000U + +#ifdef __USE_GNU +#define SPEED_MAX 4294967295U /* maximum valid speed_t value */ +#endif +#define __MAX_BAUD 4294967295U /* legacy alias for SPEED_MAX */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/termios-c_cflag.h b/lib/libc/include/generic-glibc/bits/termios-c_cflag.h index 0b0e40cd8a962b334b3a17cb74f2dafb54f14567..7a4c0a53fe348037f5412d2785151aa1c04fcc1e 100644 --- a/lib/libc/include/generic-glibc/bits/termios-c_cflag.h +++ b/lib/libc/include/generic-glibc/bits/termios-c_cflag.h @@ -34,5 +34,7 @@ #define CLOCAL 0004000 #ifdef __USE_MISC -# define ADDRB 04000000000 +# define ADDRB 04000000000 +# define CMSPAR 010000000000 /* Mark or space (stick) parity. */ +# define CRTSCTS 020000000000 /* Flow control. */ #endif \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/termios-cbaud.h b/lib/libc/include/generic-glibc/bits/termios-cbaud.h new file mode 100644 index 0000000000000000000000000000000000000000..38bf2ec1aae085543a8e9290793ebf14f6503de9 --- /dev/null +++ b/lib/libc/include/generic-glibc/bits/termios-cbaud.h @@ -0,0 +1,47 @@ +/* termios baud rate selection definitions. Linux/generic version. + Copyright (C) 2019-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library. If not, see + . */ + +#ifndef _TERMIOS_H +# error "Never include directly; use instead." +#endif + +#ifdef __USE_MISC +# define CBAUD 000000010017 /* Baud speed mask (not in POSIX). */ +# define CBAUDEX 000000010000 /* Extra baud speed mask, included in CBAUD. + (not in POSIX). */ +# define CIBAUD 002003600000 /* Input baud rate. */ +# define IBSHIFT 16 +#endif + +/* Extra output baud rates (not in POSIX). */ +#define __BOTHER 0010000 +#define __B57600 0010001 +#define __B115200 0010002 +#define __B230400 0010003 +#define __B460800 0010004 +#define __B500000 0010005 +#define __B576000 0010006 +#define __B921600 0010007 +#define __B1000000 0010010 +#define __B1152000 0010011 +#define __B1500000 0010012 +#define __B2000000 0010013 +#define __B2500000 0010014 +#define __B3000000 0010015 +#define __B3500000 0010016 +#define __B4000000 0010017 \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/termios-struct.h b/lib/libc/include/generic-glibc/bits/termios-struct.h index 51dda8eda38e4f17342da6ee41d1ee7277e13ec5..40d20e442d494026e28ffc6813a69f033beaddcf 100644 --- a/lib/libc/include/generic-glibc/bits/termios-struct.h +++ b/lib/libc/include/generic-glibc/bits/termios-struct.h @@ -29,8 +29,15 @@ struct termios tcflag_t c_lflag; /* local mode flags */ cc_t c_line; /* line discipline */ cc_t c_cc[NCCS]; /* control characters */ - speed_t c_ispeed; /* input speed */ - speed_t c_ospeed; /* output speed */ + /* Input and output baud rates. */ + __extension__ union { + speed_t __ispeed; + speed_t c_ispeed; + }; #define _HAVE_STRUCT_TERMIOS_C_ISPEED 1 + __extension__ union { + speed_t __ospeed; + speed_t c_ospeed; + }; #define _HAVE_STRUCT_TERMIOS_C_OSPEED 1 }; \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/termios.h b/lib/libc/include/generic-glibc/bits/termios.h index 5433d2d4554fa7874f4e877dfd5478a1a12d55f4..4af6204a30b0b073221afe8f85ee9209881a0e15 100644 --- a/lib/libc/include/generic-glibc/bits/termios.h +++ b/lib/libc/include/generic-glibc/bits/termios.h @@ -24,35 +24,41 @@ typedef unsigned char cc_t; typedef unsigned int speed_t; typedef unsigned int tcflag_t; -#include +#ifdef _TERMIOS_H +# include +#endif + #include #include #include /* c_cflag bit meaning */ -#define B0 0000000 /* hang up */ -#define B50 0000001 -#define B75 0000002 -#define B110 0000003 -#define B134 0000004 -#define B150 0000005 -#define B200 0000006 -#define B300 0000007 -#define B600 0000010 -#define B1200 0000011 -#define B1800 0000012 -#define B2400 0000013 -#define B4800 0000014 -#define B9600 0000015 -#define B19200 0000016 -#define B38400 0000017 -#ifdef __USE_MISC -# define EXTA B19200 -# define EXTB B38400 -#endif -#include - #include + +#ifdef __USE_MISC +#define __B0 0000000 /* hang up */ +#define __B50 0000001 +#define __B75 0000002 +#define __B110 0000003 +#define __B134 0000004 +#define __B150 0000005 +#define __B200 0000006 +#define __B300 0000007 +#define __B600 0000010 +#define __B1200 0000011 +#define __B1800 0000012 +#define __B2400 0000013 +#define __B4800 0000014 +#define __B9600 0000015 +#define __B19200 0000016 +#define __B38400 0000017 +#include + +# define __EXTA __B19200 +# define __EXTB __B38400 +# define BOTHER __BOTHER +#endif + #include #ifdef __USE_MISC @@ -73,4 +79,6 @@ typedef unsigned int tcflag_t; #include -#include \ No newline at end of file +#include + +#include \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/bits/types/struct_FILE.h b/lib/libc/include/generic-glibc/bits/types/struct_FILE.h index 9dfac48cb75baaa6add238093417d5920058c124..b02bb0c15400c5e8d177cbe0ab006b3c4427e4b6 100644 --- a/lib/libc/include/generic-glibc/bits/types/struct_FILE.h +++ b/lib/libc/include/generic-glibc/bits/types/struct_FILE.h @@ -32,6 +32,7 @@ #endif #include +#include struct _IO_FILE; struct _IO_marker; @@ -97,8 +98,15 @@ struct _IO_FILE_complete void *_freeres_buf; struct _IO_FILE **_prevchain; int _mode; +#if __WORDSIZE == 64 + int _unused3; +#endif + __uint64_t _total_written; +#if __WORDSIZE == 32 + int _unused3; +#endif /* Make sure we don't get into trouble again. */ - char _unused2[15 * sizeof (int) - 5 * sizeof (void *)]; + char _unused2[12 * sizeof (int) - 5 * sizeof (void *)]; }; /* These macros are used by bits/stdio.h and internal headers. */ diff --git a/lib/libc/include/generic-glibc/dlfcn.h b/lib/libc/include/generic-glibc/dlfcn.h index d1e540fc8aab5cbeb0efc48fc44bc2d847e9bdc8..fe5059b8d53fa8b6efecdf8a3f6d34c9b99c3134 100644 --- a/lib/libc/include/generic-glibc/dlfcn.h +++ b/lib/libc/include/generic-glibc/dlfcn.h @@ -217,16 +217,22 @@ struct dl_find_object int dlfo_eh_count; /* Number of exception handling entries. */ unsigned int __dlfo_eh_count_pad; # endif - __extension__ unsigned long long int __dflo_reserved[7]; + void *dlfo_sframe; /* SFrame stack trace data of the object. */ +#if __WORDSIZE == 32 + unsigned int __dlfo_sframe_pad; +#endif + __extension__ unsigned long long int __dlfo_reserved[6]; }; /* If ADDRESS is found in an object, fill in *RESULT and return 0. Otherwise, return -1. */ int _dl_find_object (void *__address, struct dl_find_object *__result) __THROW; +/* SFrame stack trace data is valid. */ +#define DLFO_FLAG_SFRAME (1ULL << 0) + #endif /* __USE_GNU */ - __END_DECLS #endif /* dlfcn.h */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/elf.h b/lib/libc/include/generic-glibc/elf.h index 690b047978d0456b868bc893b41559e125f4d426..11dc6453b50e5ac3fd639598dfb28ecb9253ab8f 100644 --- a/lib/libc/include/generic-glibc/elf.h +++ b/lib/libc/include/generic-glibc/elf.h @@ -837,12 +837,15 @@ typedef struct #define NT_ARM_ZT 0x40d /* ARM SME ZT registers. */ #define NT_ARM_FPMR 0x40e /* ARM floating point mode register. */ #define NT_ARM_POE 0x40f /* ARM POE registers. */ +#define NT_ARM_GCS 0x410 /* ARM GCS state. */ #define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note. */ #define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers. */ #define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode. */ #define NT_MIPS_MSA 0x802 /* MIPS SIMD registers. */ #define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */ #define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */ +#define NT_RISCV_TAGGED_ADDR_CTRL 0x902 /* RISC-V tagged + address control */ #define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers. */ #define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers. */ @@ -2906,19 +2909,6 @@ enum #define R_AARCH64_NONE 0 /* No relocation. */ -/* ILP32 AArch64 relocs. */ -#define R_AARCH64_P32_ABS32 1 /* Direct 32 bit. */ -#define R_AARCH64_P32_COPY 180 /* Copy symbol at runtime. */ -#define R_AARCH64_P32_GLOB_DAT 181 /* Create GOT entry. */ -#define R_AARCH64_P32_JUMP_SLOT 182 /* Create PLT entry. */ -#define R_AARCH64_P32_RELATIVE 183 /* Adjust by program base. */ -#define R_AARCH64_P32_TLS_DTPMOD 184 /* Module number, 32 bit. */ -#define R_AARCH64_P32_TLS_DTPREL 185 /* Module-relative offset, 32 bit. */ -#define R_AARCH64_P32_TLS_TPREL 186 /* TP-relative offset, 32 bit. */ -#define R_AARCH64_P32_TLSDESC 187 /* TLS Descriptor. */ -#define R_AARCH64_P32_IRELATIVE 188 /* STT_GNU_IFUNC relocation. */ - -/* LP64 AArch64 relocs. */ #define R_AARCH64_ABS64 257 /* Direct 64 bit. */ #define R_AARCH64_ABS32 258 /* Direct 32 bit. */ #define R_AARCH64_ABS16 259 /* Direct 16-bit. */ @@ -4091,6 +4081,7 @@ enum #define R_RISCV_TLS_DTPREL64 9 #define R_RISCV_TLS_TPREL32 10 #define R_RISCV_TLS_TPREL64 11 +#define R_RISCV_TLSDESC 12 #define R_RISCV_BRANCH 16 #define R_RISCV_JAL 17 #define R_RISCV_CALL 18 @@ -4116,16 +4107,10 @@ enum #define R_RISCV_SUB16 38 #define R_RISCV_SUB32 39 #define R_RISCV_SUB64 40 -#define R_RISCV_GNU_VTINHERIT 41 -#define R_RISCV_GNU_VTENTRY 42 +#define R_RISCV_GOT32_PCREL 41 #define R_RISCV_ALIGN 43 #define R_RISCV_RVC_BRANCH 44 #define R_RISCV_RVC_JUMP 45 -#define R_RISCV_RVC_LUI 46 -#define R_RISCV_GPREL_I 47 -#define R_RISCV_GPREL_S 48 -#define R_RISCV_TPREL_I 49 -#define R_RISCV_TPREL_S 50 #define R_RISCV_RELAX 51 #define R_RISCV_SUB6 52 #define R_RISCV_SET6 53 @@ -4137,8 +4122,12 @@ enum #define R_RISCV_PLT32 59 #define R_RISCV_SET_ULEB128 60 #define R_RISCV_SUB_ULEB128 61 +#define R_RISCV_TLSDESC_HI20 62 +#define R_RISCV_TLSDESC_LOAD_LO12 63 +#define R_RISCV_TLSDESC_ADD_LO12 64 +#define R_RISCV_TLSDESC_CALL 65 -#define R_RISCV_NUM 62 +#define R_RISCV_NUM 66 /* RISC-V specific values for the st_other field. */ #define STO_RISCV_VARIANT_CC 0x80 /* Function uses variant calling @@ -4147,7 +4136,7 @@ enum /* RISC-V specific values for the sh_type field. */ #define SHT_RISCV_ATTRIBUTES (SHT_LOPROC + 3) -/* RISC-V specific values for the p_type field. */ +/* RISC-V specific values for the p_type field (deprecated). */ #define PT_RISCV_ATTRIBUTES (PT_LOPROC + 3) /* RISC-V specific values for the d_tag field. */ diff --git a/lib/libc/include/generic-glibc/fcntl.h b/lib/libc/include/generic-glibc/fcntl.h index 117f1c3647cecbd112f230ffcb902dc3c3456f0e..51add4e1464e936bddd102d0cd39af9b9e023812 100644 --- a/lib/libc/include/generic-glibc/fcntl.h +++ b/lib/libc/include/generic-glibc/fcntl.h @@ -168,7 +168,7 @@ typedef __pid_t pid_t; #endif -/* fcntl was a simple symbol until glibc 2.27 inclusive. +/* zig patch: fcntl was a simple symbol until glibc 2.27 inclusive. * glibc 2.28 onwards converted it to a macro when compiled with * USE_LARGEFILE64. */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 28) || __GLIBC__ > 2 @@ -289,16 +289,17 @@ extern int creat64 (const char *__file, mode_t __mode) __nonnull ((1)); # define F_TEST 3 /* Test a region for other processes locks. */ # ifndef __USE_FILE_OFFSET64 -extern int lockf (int __fd, int __cmd, off_t __len); +extern int lockf (int __fd, int __cmd, off_t __len) __wur; # else # ifdef __REDIRECT -extern int __REDIRECT (lockf, (int __fd, int __cmd, __off64_t __len), lockf64); +extern int __REDIRECT (lockf, (int __fd, int __cmd, __off64_t __len), + lockf64) __wur; # else # define lockf lockf64 # endif # endif # ifdef __USE_LARGEFILE64 -extern int lockf64 (int __fd, int __cmd, off64_t __len); +extern int lockf64 (int __fd, int __cmd, off64_t __len) __wur; # endif #endif @@ -351,4 +352,4 @@ extern int posix_fallocate64 (int __fd, off64_t __offset, off64_t __len); __END_DECLS -#endif /* fcntl.h */ \ No newline at end of file +#endif /* fcntl.h */ diff --git a/lib/libc/include/generic-glibc/features.h b/lib/libc/include/generic-glibc/features.h index 7a830cdbac96ac28814cdc03c47d5310247487b2..8a918c60176d685f3d151830a0c173691f88538d 100644 --- a/lib/libc/include/generic-glibc/features.h +++ b/lib/libc/include/generic-glibc/features.h @@ -491,7 +491,7 @@ or without -D_GNU_SOURCE, but -std=c89 -D_GNU_SOURCE will have the old extension. */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ < 7) -/* support for ISOC99 was added in glibc-2.7 */ +/* zig patch: support for ISOC99 was added in glibc-2.7 */ # define __GLIBC_USE_DEPRECATED_SCANF 1 #elif (defined __USE_GNU \ && (defined __cplusplus \ @@ -503,7 +503,7 @@ #endif -/* support for ISO C2X strtol was added in 2.38 +/* zig patch: support for ISO C2X strtol was added in 2.38 * glibc commit 64924422a99690d147a166b4de3103f3bf3eaf6c */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 38) || __GLIBC__ > 2 @@ -564,4 +564,4 @@ #include -#endif /* features.h */ \ No newline at end of file +#endif /* features.h */ diff --git a/lib/libc/include/generic-glibc/glob.h b/lib/libc/include/generic-glibc/glob.h index 3577c9c7c4fef696a357c35f81f036483bc0cad2..14340f5fae38311cc1fdb7d72943fe30f037a626 100644 --- a/lib/libc/include/generic-glibc/glob.h +++ b/lib/libc/include/generic-glibc/glob.h @@ -195,7 +195,8 @@ extern void globfree64 (glob64_t *__pglob) __THROW; This function is not part of the interface specified by POSIX.2 but several programs want to use it. */ -extern int glob_pattern_p (const char *__pattern, int __quote) __THROW; +extern int glob_pattern_p (const char *__pattern, int __quote) __THROW + __nonnull ((1)); #endif __END_DECLS diff --git a/lib/libc/include/generic-glibc/inttypes.h b/lib/libc/include/generic-glibc/inttypes.h index 3127d5637e1525353510f4745f7b4eb4ab3afe09..be384f223e52627c68d4feb99c08ab0b3e282ed5 100644 --- a/lib/libc/include/generic-glibc/inttypes.h +++ b/lib/libc/include/generic-glibc/inttypes.h @@ -350,6 +350,11 @@ typedef struct /* Compute absolute value of N. */ extern intmax_t imaxabs (intmax_t __n) __THROW __attribute__ ((__const__)); + +#if __GLIBC_USE (ISOC2Y) +extern uintmax_t uimaxabs (intmax_t __n) __THROW __attribute__ ((__const__)); +#endif + /* Return the `imaxdiv_t' representation of the value of NUMER over DENOM. */ extern imaxdiv_t imaxdiv (intmax_t __numer, intmax_t __denom) __THROW __attribute__ ((__const__)); diff --git a/lib/libc/include/generic-glibc/malloc.h b/lib/libc/include/generic-glibc/malloc.h index a24d43c295d28ccafd580158e0ce70515be1b319..187457d42131e12c92649a85f52ef0d62c5ca5f9 100644 --- a/lib/libc/include/generic-glibc/malloc.h +++ b/lib/libc/include/generic-glibc/malloc.h @@ -52,7 +52,7 @@ extern void *realloc (void *__ptr, size_t __size) __THROW __attribute_warn_unused_result__ __attribute_alloc_size__ ((2)); /* - * reallocarray introduced in glibc 2.26 + * zig patch: reallocarray introduced in glibc 2.26 * https://sourceware.org/git/?p=glibc.git;a=commit;h=2e0bbbfbf95fc9e22692e93658a6fbdd2d4554da */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 26) || __GLIBC__ > 2 @@ -164,4 +164,4 @@ extern void malloc_stats (void) __THROW; extern int malloc_info (int __options, FILE *__fp) __THROW; __END_DECLS -#endif /* malloc.h */ \ No newline at end of file +#endif /* malloc.h */ diff --git a/lib/libc/include/generic-glibc/netinet/tcp.h b/lib/libc/include/generic-glibc/netinet/tcp.h index 65ee87a04126063108243ce773562ba50ff1ffb4..357da4371277a57ef430bd1adf5e55affc1d9d45 100644 --- a/lib/libc/include/generic-glibc/netinet/tcp.h +++ b/lib/libc/include/generic-glibc/netinet/tcp.h @@ -212,6 +212,9 @@ enum # define TCPI_OPT_ECN 8 /* ECN was negotiated at TCP session init */ # define TCPI_OPT_ECN_SEEN 16 /* we received at least one packet with ECT */ # define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */ +# define TCPI_OPT_USEC_TS 64 /* usec timestamps */ +# define TCPI_OPT_TFO_CHILD 128 /* child from a Fast Open option on SYN */ + /* Values for tcpi_state. */ enum tcp_ca_state diff --git a/lib/libc/include/generic-glibc/pthread.h b/lib/libc/include/generic-glibc/pthread.h index 1c7bf92e8cca40069049f8a3753a0e0de868af23..cfce06cef914646764069c7dfbf367fbe76f280e 100644 --- a/lib/libc/include/generic-glibc/pthread.h +++ b/lib/libc/include/generic-glibc/pthread.h @@ -1317,6 +1317,11 @@ extern int pthread_getcpuclockid (pthread_t __thread_id, __THROW __nonnull ((2)); #endif +#ifdef __USE_GNU +/* Return the Linux TID for THREAD_ID. Returns -1 on failure. */ +extern pid_t pthread_gettid_np (pthread_t __thread_id); +#endif + /* Install handlers to be called when a new process is created with FORK. The PREPARE handler is called in the parent process just before performing diff --git a/lib/libc/include/generic-glibc/resolv.h b/lib/libc/include/generic-glibc/resolv.h index 9bdbc09e1d6cfa47525616228d9dc5c5236ade0f..206ff771fb87c2372501d9d7a6345033c628c181 100644 --- a/lib/libc/include/generic-glibc/resolv.h +++ b/lib/libc/include/generic-glibc/resolv.h @@ -171,7 +171,7 @@ __END_DECLS #define res_init __res_init #define res_isourserver __res_isourserver -/* In glibc 2.33 and earlier res_search, res_nsearch, res_query, res_nquery, +/* zig patch: In glibc 2.33 and earlier res_search, res_nsearch, res_query, res_nquery, * res_querydomain, res_nquerydomain, dn_skipname, dn_comp, dn_expand were * #define'd to __res_search, __res_nsearch, etc. glibc 2.34 onwards removes * the macros and exposes the symbols directly. New glibc exposes compat @@ -336,4 +336,4 @@ void res_nclose (res_state) __THROW; __END_DECLS -#endif /* !_RESOLV_H_ */ \ No newline at end of file +#endif /* !_RESOLV_H_ */ diff --git a/lib/libc/include/generic-glibc/stdio.h b/lib/libc/include/generic-glibc/stdio.h index ad7dd9eb8e49baf44c2efff43fea02b23e86dda3..e80e85755094b569778e697af1a6c1f6bdfcb342 100644 --- a/lib/libc/include/generic-glibc/stdio.h +++ b/lib/libc/include/generic-glibc/stdio.h @@ -168,8 +168,11 @@ extern int renameat (int __oldfd, const char *__old, int __newfd, #ifdef __USE_GNU /* Flags for renameat2. */ # define RENAME_NOREPLACE (1 << 0) +# define AT_RENAME_NOREPLACE RENAME_NOREPLACE # define RENAME_EXCHANGE (1 << 1) +# define AT_RENAME_EXCHANGE RENAME_EXCHANGE # define RENAME_WHITEOUT (1 << 2) +# define AT_RENAME_WHITEOUT RENAME_WHITEOUT /* Rename file OLD relative to OLDFD to NEW relative to NEWFD, with additional flags. */ @@ -604,9 +607,6 @@ extern int fgetc_unlocked (FILE *__stream) __nonnull ((1)); /* Write a character to STREAM. These functions are possible cancellation points and therefore not - marked with __THROW. - - These functions is a possible cancellation point and therefore not marked with __THROW. */ extern int fputc (int __c, FILE *__stream) __nonnull ((2)); extern int putc (int __c, FILE *__stream) __nonnull ((2)); diff --git a/lib/libc/include/generic-glibc/stdio_ext.h b/lib/libc/include/generic-glibc/stdio_ext.h index 37c30ce0fa592cf4698667d491defbd5ed22bef9..35ba67d43e0065b63d01429884bec7883ff0c94f 100644 --- a/lib/libc/include/generic-glibc/stdio_ext.h +++ b/lib/libc/include/generic-glibc/stdio_ext.h @@ -43,43 +43,43 @@ __BEGIN_DECLS /* Return the size of the buffer of FP in bytes currently in use by the given stream. */ -extern size_t __fbufsize (FILE *__fp) __THROW; +extern size_t __fbufsize (FILE *__fp) __THROW __nonnull ((1)); /* Return non-zero value iff the stream FP is opened readonly, or if the last operation on the stream was a read operation. */ -extern int __freading (FILE *__fp) __THROW; +extern int __freading (FILE *__fp) __THROW __nonnull ((1)); /* Return non-zero value iff the stream FP is opened write-only or append-only, or if the last operation on the stream was a write operation. */ -extern int __fwriting (FILE *__fp) __THROW; +extern int __fwriting (FILE *__fp) __THROW __nonnull ((1)); /* Return non-zero value iff stream FP is not opened write-only or append-only. */ -extern int __freadable (FILE *__fp) __THROW; +extern int __freadable (FILE *__fp) __THROW __nonnull ((1)); /* Return non-zero value iff stream FP is not opened read-only. */ -extern int __fwritable (FILE *__fp) __THROW; +extern int __fwritable (FILE *__fp) __THROW __nonnull ((1)); /* Return non-zero value iff the stream FP is line-buffered. */ -extern int __flbf (FILE *__fp) __THROW; +extern int __flbf (FILE *__fp) __THROW __nonnull ((1)); /* Discard all pending buffered I/O on the stream FP. */ -extern void __fpurge (FILE *__fp) __THROW; +extern void __fpurge (FILE *__fp) __THROW __nonnull ((1)); /* Return amount of output in bytes pending on a stream FP. */ -extern size_t __fpending (FILE *__fp) __THROW; +extern size_t __fpending (FILE *__fp) __THROW __nonnull ((1)); /* Flush all line-buffered files. */ extern void _flushlbf (void); /* Set locking status of stream FP to TYPE. */ -extern int __fsetlocking (FILE *__fp, int __type) __THROW; +extern int __fsetlocking (FILE *__fp, int __type) __THROW __nonnull ((1)); __END_DECLS diff --git a/lib/libc/include/generic-glibc/stdlib.h b/lib/libc/include/generic-glibc/stdlib.h index 7ae0c0f7e9964063ea368b4a66f7094a2be6309a..b6b4ff601ff8b673f9a3de1cacf25723884c2c9d 100644 --- a/lib/libc/include/generic-glibc/stdlib.h +++ b/lib/libc/include/generic-glibc/stdlib.h @@ -654,7 +654,7 @@ extern int lcong48_r (unsigned short int __param[7], __THROW __nonnull ((1, 2)); /* - * arc4random* symbols introduced in glibc 2.36: + * zig patch: arc4random* symbols introduced in glibc 2.36: * https://sourceware.org/git/?p=glibc.git;a=blob;f=NEWS;h=8420a65cd06874ee09518366b8fba746a557212a;hb=6f4e0fcfa2d2b0915816a3a3a1d48b4763a7dee2 */ # if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 36) || __GLIBC__ > 2 @@ -693,7 +693,7 @@ extern void *realloc (void *__ptr, size_t __size) extern void free (void *__ptr) __THROW; /* - * reallocarray introduced in glibc 2.26 + * zig patch: reallocarray introduced in glibc 2.26 * https://sourceware.org/git/?p=glibc.git;a=commit;h=2e0bbbfbf95fc9e22692e93658a6fbdd2d4554da */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 26) || __GLIBC__ > 2 @@ -997,6 +997,12 @@ __extension__ extern long long int llabs (long long int __x) __THROW __attribute__ ((__const__)) __wur; #endif +#if __GLIBC_USE (ISOC2Y) +extern unsigned int uabs (int __x) __THROW __attribute__ ((__const__)) __wur; +extern unsigned long int ulabs (long int __x) __THROW __attribute__ ((__const__)) __wur; +__extension__ extern unsigned long long int ullabs (long long int __x) + __THROW __attribute__ ((__const__)) __wur; +#endif /* Return the `div_t', `ldiv_t' or `lldiv_t' representation of the value of NUMER over DENOM. */ @@ -1178,4 +1184,4 @@ extern int ttyslot (void) __THROW; __END_DECLS -#endif /* stdlib.h */ \ No newline at end of file +#endif /* stdlib.h */ diff --git a/lib/libc/include/generic-glibc/string.h b/lib/libc/include/generic-glibc/string.h index 68f695cc425cb2b07ed3e932c40c8498ce18f269..d6794d21d1763e1a946e3897e5b30ce8a3bca169 100644 --- a/lib/libc/include/generic-glibc/string.h +++ b/lib/libc/include/generic-glibc/string.h @@ -502,7 +502,7 @@ extern char *stpncpy (char *__restrict __dest, #endif /* - * strlcpy and strlcat introduced in glibc 2.38 + * zig patch: strlcpy and strlcat introduced in glibc 2.38 * https://sourceware.org/git/?p=glibc.git;a=commit;h=2e0bbbfbf95fc9e22692e93658a6fbdd2d4554da */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 38) || __GLIBC__ > 2 @@ -557,4 +557,4 @@ extern char *basename (const char *__filename) __THROW __nonnull ((1)); __END_DECLS -#endif /* string.h */ \ No newline at end of file +#endif /* string.h */ diff --git a/lib/libc/include/generic-glibc/sys/hwprobe.h b/lib/libc/include/generic-glibc/sys/hwprobe.h index 63986df08f2ffd69683fca0c6a6f11d39257b747..056a98dc5182784e8c74c40331cb5a8b4e7bb00d 100644 --- a/lib/libc/include/generic-glibc/sys/hwprobe.h +++ b/lib/libc/include/generic-glibc/sys/hwprobe.h @@ -21,6 +21,7 @@ #define _SYS_HWPROBE_H 1 #include +#include #include #include #ifdef __has_include @@ -63,22 +64,39 @@ struct riscv_hwprobe { __BEGIN_DECLS -extern int __riscv_hwprobe (struct riscv_hwprobe *__pairs, size_t __pair_count, - size_t __cpu_count, unsigned long int *__cpus, +#if defined __cplusplus || !__GNUC_PREREQ (2, 7) +# define __RISCV_HWPROBE_CPUS_TYPE cpu_set_t * +#else +/* The fourth argument to __riscv_hwprobe should be a null pointer or a + pointer to a cpu_set_t (either the fixed-size type or allocated with + CPU_ALLOC). However, early versions of this header file used the + argument type unsigned long int *. The transparent union allows + the argument to be either cpu_set_t * or unsigned long int * for + compatibility. The older header file requiring unsigned long int * + can be identified by the lack of the __RISCV_HWPROBE_CPUS_TYPE macro. + In C++ and with compilers that do not support transparent unions, the + argument type must be cpu_set_t *. */ +typedef union { + cpu_set_t *__cs; + unsigned long int *__ul; +} __RISCV_HWPROBE_CPUS_TYPE __attribute__ ((__transparent_union__)); +# define __RISCV_HWPROBE_CPUS_TYPE __RISCV_HWPROBE_CPUS_TYPE +#endif + +extern int __riscv_hwprobe (struct riscv_hwprobe *__pairs, + size_t __pair_count, size_t __cpusetsize, + __RISCV_HWPROBE_CPUS_TYPE __cpus, unsigned int __flags) - __nonnull ((1)) __wur - __fortified_attr_access (__read_write__, 1, 2) - __fortified_attr_access (__read_only__, 4, 3); + __THROW __nonnull ((1)) __attr_access ((__read_write__, 1, 2)); -/* A pointer to the __riscv_hwprobe vDSO function is passed as the second +/* A pointer to the __riscv_hwprobe function is passed as the second argument to ifunc selector routines. Include a function pointer type for convenience in calling the function in those settings. */ -typedef int (*__riscv_hwprobe_t) (struct riscv_hwprobe *__pairs, size_t __pair_count, - size_t __cpu_count, unsigned long int *__cpus, +typedef int (*__riscv_hwprobe_t) (struct riscv_hwprobe *__pairs, + size_t __pair_count, size_t __cpusetsize, + __RISCV_HWPROBE_CPUS_TYPE __cpus, unsigned int __flags) - __nonnull ((1)) __wur - __fortified_attr_access (__read_write__, 1, 2) - __fortified_attr_access (__read_only__, 4, 3); + __nonnull ((1)) __attr_access ((__read_write__, 1, 2)); /* Helper function usable from ifunc selectors that probes a single key. */ static __inline int diff --git a/lib/libc/include/generic-glibc/sys/ifunc.h b/lib/libc/include/generic-glibc/sys/ifunc.h index d86de9707de293ccb23b0fbca64043b6e6b5d08d..c4ca0a983ccc513588dc84ce3821e5b0086a7c32 100644 --- a/lib/libc/include/generic-glibc/sys/ifunc.h +++ b/lib/libc/include/generic-glibc/sys/ifunc.h @@ -19,24 +19,77 @@ #ifndef _SYS_IFUNC_H #define _SYS_IFUNC_H +#include + /* A second argument is passed to the ifunc resolver. */ #define _IFUNC_ARG_HWCAP (1ULL << 62) -/* The prototype of a gnu indirect function resolver on AArch64 is +/* Maximum number of HWCAP elements that are currently supported. */ +#define _IFUNC_HWCAP_MAX 4 + +/* The prototype of a GNU indirect function resolver on AArch64 is + + ElfW(Addr) ifunc_resolver (uint64_t, const uint64_t *); + + The following prototype is also compatible: ElfW(Addr) ifunc_resolver (uint64_t, const __ifunc_arg_t *); - the first argument should have the _IFUNC_ARG_HWCAP bit set and - the remaining bits should match the AT_HWCAP settings. */ + The first argument might have the _IFUNC_ARG_HWCAP bit set and + the remaining bits should match the AT_HWCAP settings. + + If the _IFUNC_ARG_HWCAP bit is set in the first argument, then + the second argument is passed to the resolver function. In + this case, the second argument is a const pointer to a buffer + that allows to access all available HWCAP elements. + + This buffer has its size in bytes at offset 0. The HWCAP elements + are available at offsets 8, 16, 24, 32... respectively for AT_HWCAP, + AT_HWCAP2, AT_HWCAP3, AT_HWCAP4... (these offsets are multiples of + sizeof (unsigned long)). + + Indirect function resolvers must check availability of HWCAP + elements at runtime before accessing them using the size of the + buffer. */ -/* Second argument to an ifunc resolver. */ struct __ifunc_arg_t { - unsigned long _size; /* Size of the struct, so it can grow. */ + unsigned long _size; /* Size of the struct, so it can grow. */ unsigned long _hwcap; - unsigned long _hwcap2; + unsigned long _hwcap2; /* End of 1st published struct. */ + unsigned long _hwcap3; + unsigned long _hwcap4; /* End of 2nd published struct. */ }; typedef struct __ifunc_arg_t __ifunc_arg_t; +/* Constants for IDs of HWCAP elements to be used with the + __ifunc_hwcap function below. */ +enum +{ + _IFUNC_ARG_AT_HWCAP = 1, + _IFUNC_ARG_AT_HWCAP2 = 2, + _IFUNC_ARG_AT_HWCAP3 = 3, + _IFUNC_ARG_AT_HWCAP4 = 4, +}; + +/* A helper function to obtain HWCAP element by its ID from the + parameters ARG0 and ARG1 passed to the ifunc resolver. Note that + ID 1 corresponds to AT_HWCAP, ID 2 corresponds to AT_HWCAP2, etc. + If there is no element available for the requested ID then 0 is + returned. If ID doesn't much any supported AT_HWCAP{,2,...} value, + then 0 is also returned. */ +static __inline unsigned long __attribute__ ((unused, always_inline)) +__ifunc_hwcap (unsigned long __id, + unsigned long __arg0, const unsigned long *__arg1) +{ + if (__glibc_likely (__arg0 & _IFUNC_ARG_HWCAP)) + { + const unsigned long size = __arg1[0]; + const unsigned long offset = __id * sizeof (unsigned long); + return offset < size && __id > 0 ? __arg1[__id] : 0; + } + return __id == 1 ? __arg0 : 0; +} + #endif \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/sys/mount.h b/lib/libc/include/generic-glibc/sys/mount.h index b3f0cd64ecab74495f8fa1cbf4bd566ca17ece1f..fbb157810e243939be3b90310458ee8eec739f6e 100644 --- a/lib/libc/include/generic-glibc/sys/mount.h +++ b/lib/libc/include/generic-glibc/sys/mount.h @@ -121,7 +121,7 @@ enum MS_ACTIVE = 1 << 30, #define MS_ACTIVE MS_ACTIVE #undef MS_NOUSER - MS_NOUSER = 1 << 31 + MS_NOUSER = 1U << 31 #define MS_NOUSER MS_NOUSER }; diff --git a/lib/libc/include/generic-glibc/sys/ttychars.h b/lib/libc/include/generic-glibc/sys/ttychars.h index 492732666731848889e2773637676d7176d414a3..c187c170c65a2b6f0b8ab3dad0eadec7d921312b 100644 --- a/lib/libc/include/generic-glibc/sys/ttychars.h +++ b/lib/libc/include/generic-glibc/sys/ttychars.h @@ -54,8 +54,4 @@ struct ttychars { char tc_lnextc; /* literal next character */ }; -#ifdef __USE_OLD_TTY -#include /* to pick up character defaults */ -#endif - #endif /* sys/ttychars.h */ \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/termio.h b/lib/libc/include/generic-glibc/termio.h deleted file mode 100644 index a3f32a79f55c72f8d94f3c23b999cf6f252e5704..0000000000000000000000000000000000000000 --- a/lib/libc/include/generic-glibc/termio.h +++ /dev/null @@ -1,6 +0,0 @@ -/* Compatible for old `struct termio' ioctl interface. - This is obsolete; use the POSIX.1 `struct termios' interface - defined in instead. */ - -#include -#include \ No newline at end of file diff --git a/lib/libc/include/generic-glibc/termios.h b/lib/libc/include/generic-glibc/termios.h index 94624586e7b414f52f17cfd492eecb071cb3679c..ef7b166b4c422e51270f1409f7c2abf8d546a797 100644 --- a/lib/libc/include/generic-glibc/termios.h +++ b/lib/libc/include/generic-glibc/termios.h @@ -61,6 +61,26 @@ extern int cfsetispeed (struct termios *__termios_p, speed_t __speed) __THROW; extern int cfsetspeed (struct termios *__termios_p, speed_t __speed) __THROW; #endif +#ifdef __USE_GNU +/* Interfaces that are explicitly numeric representations of baud rates */ +typedef speed_t baud_t; +#define BAUD_MAX SPEED_MAX + +/* Return the output baud rate stored in *TERMIOS_P. */ +extern baud_t cfgetobaud (const struct termios *__termios_p) __THROW; + +/* Return the input baud rate stored in *TERMIOS_P. */ +extern baud_t cfgetibaud (const struct termios *__termios_p) __THROW; + +/* Set the output baud rate stored in *TERMIOS_P to BAUD. */ +extern int cfsetobaud (struct termios *__termios_p, baud_t __baud) __THROW; + +/* Set the input baud rate stored in *TERMIOS_P to BAUD. */ +extern int cfsetibaud (struct termios *__termios_p, baud_t __baud) __THROW; + +/* Set both the input and output baud rates in *TERMIOS_OP to BAUD. */ +extern int cfsetbaud (struct termios *__termios_p, baud_t __baud) __THROW; +#endif /* Put the state of FD into *TERMIOS_P. */ extern int tcgetattr (int __fd, struct termios *__termios_p) __THROW; diff --git a/lib/libc/include/generic-glibc/tgmath.h b/lib/libc/include/generic-glibc/tgmath.h index c90b783891bf3803e28ab07742b1377f26b2ff7d..4eae43da5e045050d1b0652a308e74ae6c38fee0 100644 --- a/lib/libc/include/generic-glibc/tgmath.h +++ b/lib/libc/include/generic-glibc/tgmath.h @@ -923,6 +923,24 @@ /* Return the cube root of X. */ #define cbrt(Val) __TGMATH_UNARY_REAL_ONLY (Val, cbrt) +#if __GLIBC_USE (IEC_60559_FUNCS_EXT_C23) +/* Return 1+X to the Y power. */ +# define compoundn(Val1, Val2) \ + __TGMATH_BINARY_FIRST_REAL_ONLY (Val1, Val2, compoundn) + +/* Return X to the Y power. */ +# define pown(Val1, Val2) __TGMATH_BINARY_FIRST_REAL_ONLY (Val1, Val2, pown) + +/* Return X to the Y power. */ +# define powr(Val1, Val2) __TGMATH_BINARY_REAL_ONLY (Val1, Val2, powr) + +/* Return the Yth root of X. */ +# define rootn(Val1, Val2) __TGMATH_BINARY_FIRST_REAL_ONLY (Val1, Val2, rootn) + +/* Return 1/sqrt(X). */ +# define rsqrt(Val) __TGMATH_UNARY_REAL_ONLY (Val, rsqrt) +#endif + /* Nearest integer, absolute value, and remainder functions. */ diff --git a/lib/libc/include/generic-glibc/unistd.h b/lib/libc/include/generic-glibc/unistd.h index 7bc48ec48bdecb03ef83a2cf19933f1b60977113..6aaa1bfa90b37f2ad0071fe5eb3a5fb76623e1cd 100644 --- a/lib/libc/include/generic-glibc/unistd.h +++ b/lib/libc/include/generic-glibc/unistd.h @@ -1231,4 +1231,4 @@ extern int close_range (unsigned int __fd, unsigned int __max_fd, __END_DECLS -#endif /* unistd.h */ +#endif /* unistd.h */ \ No newline at end of file diff --git a/lib/libc/include/mips-linux-gnu/bits/ioctl-types.h b/lib/libc/include/mips-linux-gnu/bits/ioctl-types.h index 0b157b683cf8d9776513b0d912904889770ec06a..511fe1c96b30153f8acf0c2c8ca845dbb8e1bb3a 100644 --- a/lib/libc/include/mips-linux-gnu/bits/ioctl-types.h +++ b/lib/libc/include/mips-linux-gnu/bits/ioctl-types.h @@ -31,18 +31,6 @@ struct winsize unsigned short int ws_ypixel; }; -#define NCC 8 -struct termio - { - unsigned short int c_iflag; /* input mode flags */ - unsigned short int c_oflag; /* output mode flags */ - unsigned short int c_cflag; /* control mode flags */ - unsigned short int c_lflag; /* local mode flags */ - char c_line; /* line discipline */ - /* Yes, this is really NCCS. */ - unsigned char c_cc[32 /* NCCS */]; /* control characters */ - }; - /* modem lines */ #define TIOCM_LE 0x001 /* line enable */ #define TIOCM_DTR 0x002 /* data terminal ready */ diff --git a/lib/libc/include/mips-linux-gnu/bits/termios-struct.h b/lib/libc/include/mips-linux-gnu/bits/termios-struct.h deleted file mode 100644 index f6dd2de6185b1376ba29c62ab4d481187982462c..0000000000000000000000000000000000000000 --- a/lib/libc/include/mips-linux-gnu/bits/termios-struct.h +++ /dev/null @@ -1,34 +0,0 @@ -/* struct termios definition. Linux/mips version. - Copyright (C) 2019-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library. If not, see - . */ - -#ifndef _TERMIOS_H -# error "Never include directly; use instead." -#endif - -#define NCCS 32 -struct termios - { - tcflag_t c_iflag; /* input mode flags */ - tcflag_t c_oflag; /* output mode flags */ - tcflag_t c_cflag; /* control mode flags */ - tcflag_t c_lflag; /* local mode flags */ - cc_t c_line; /* line discipline */ - cc_t c_cc[NCCS]; /* control characters */ -#define _HAVE_STRUCT_TERMIOS_C_ISPEED 0 -#define _HAVE_STRUCT_TERMIOS_C_OSPEED 0 - }; \ No newline at end of file diff --git a/lib/libc/include/powerpc-linux-gnu/bits/ioctl-types.h b/lib/libc/include/powerpc-linux-gnu/bits/ioctl-types.h index 1744e55f3e05aa4b465b8c6c2d7d6dbc25cb7e78..ecb6295b249bb7a28b282c979ed11a8f38c4ad81 100644 --- a/lib/libc/include/powerpc-linux-gnu/bits/ioctl-types.h +++ b/lib/libc/include/powerpc-linux-gnu/bits/ioctl-types.h @@ -32,17 +32,6 @@ struct winsize unsigned short int ws_ypixel; }; -#define NCC 10 -struct termio - { - unsigned short int c_iflag; /* input mode flags */ - unsigned short int c_oflag; /* output mode flags */ - unsigned short int c_cflag; /* control mode flags */ - unsigned short int c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - /* modem lines */ #define TIOCM_LE 0x001 #define TIOCM_DTR 0x002 diff --git a/lib/libc/include/powerpc-linux-gnu/bits/termios-c_cflag.h b/lib/libc/include/powerpc-linux-gnu/bits/termios-c_cflag.h index 1532bbe4a7f1aac192ce7c46021c4b85dab5bb2c..86d5639c460e9a1b398633b1345c3f1fae8e5baf 100644 --- a/lib/libc/include/powerpc-linux-gnu/bits/termios-c_cflag.h +++ b/lib/libc/include/powerpc-linux-gnu/bits/termios-c_cflag.h @@ -35,5 +35,7 @@ #define CLOCAL 00100000 #ifdef __USE_MISC -# define ADDRB 04000000000 +# define ADDRB 04000000000 +# define CMSPAR 010000000000 /* Mark or space (stick) parity. */ +# define CRTSCTS 020000000000 /* Flow control. */ #endif \ No newline at end of file diff --git a/lib/libc/include/powerpc-linux-gnu/bits/termios-baud.h b/lib/libc/include/powerpc-linux-gnu/bits/termios-cbaud.h similarity index 58% rename from lib/libc/include/powerpc-linux-gnu/bits/termios-baud.h rename to lib/libc/include/powerpc-linux-gnu/bits/termios-cbaud.h index b61756d302d47ebf6654e8f1362b6804e3c24fcb..6e0257e59d42ad45f0ab7bd0e226195a9cb87c02 100644 --- a/lib/libc/include/powerpc-linux-gnu/bits/termios-baud.h +++ b/lib/libc/include/powerpc-linux-gnu/bits/termios-cbaud.h @@ -17,29 +17,29 @@ . */ #ifndef _TERMIOS_H -# error "Never include directly; use instead." +# error "Never include directly; use instead." #endif #ifdef __USE_MISC -# define CBAUD 0000377 -# define CBAUDEX 0000020 -# define CMSPAR 010000000000 /* mark or space (stick) parity */ -# define CRTSCTS 020000000000 /* flow control */ +# define CBAUD 000000377 +# define CBAUDEX 000000020 +# define CIBAUD 077600000 +# define IBSHIFT 16 #endif -#define B57600 00020 -#define B115200 00021 -#define B230400 00022 -#define B460800 00023 -#define B500000 00024 -#define B576000 00025 -#define B921600 00026 -#define B1000000 00027 -#define B1152000 00030 -#define B1500000 00031 -#define B2000000 00032 -#define B2500000 00033 -#define B3000000 00034 -#define B3500000 00035 -#define B4000000 00036 -#define __MAX_BAUD B4000000 \ No newline at end of file +#define __B57600 00020 +#define __B115200 00021 +#define __B230400 00022 +#define __B460800 00023 +#define __B500000 00024 +#define __B576000 00025 +#define __B921600 00026 +#define __B1000000 00027 +#define __B1152000 00030 +#define __B1500000 00031 +#define __B2000000 00032 +#define __B2500000 00033 +#define __B3000000 00034 +#define __B3500000 00035 +#define __B4000000 00036 +#define __BOTHER 00037 \ No newline at end of file diff --git a/lib/libc/include/sparc-linux-gnu/bits/termios-baud.h b/lib/libc/include/sparc-linux-gnu/bits/termios-cbaud.h similarity index 58% rename from lib/libc/include/sparc-linux-gnu/bits/termios-baud.h rename to lib/libc/include/sparc-linux-gnu/bits/termios-cbaud.h index 465e9bbb0065167445acb8e2dc21c6f0f926c754..e848a18538c9fb8c5880fc6f9cd76cf3e160b07d 100644 --- a/lib/libc/include/sparc-linux-gnu/bits/termios-baud.h +++ b/lib/libc/include/sparc-linux-gnu/bits/termios-cbaud.h @@ -17,30 +17,29 @@ . */ #ifndef _TERMIOS_H -# error "Never include directly; use instead." +# error "Never include directly; use instead." #endif #ifdef __USE_MISC # define CBAUD 0x0000100f # define CBAUDEX 0x00001000 -# define CIBAUD 0x100f0000 /* input baud rate (not used) */ -# define CMSPAR 0x40000000 /* mark or space (stick) parity */ -# define CRTSCTS 0x80000000 /* flow control */ +# define CIBAUD 0x100f0000 /* input baud rate */ +# define IBSHIFT 16 #endif -#define B57600 0x00001001 -#define B115200 0x00001002 -#define B230400 0x00001003 -#define B460800 0x00001004 -#define B76800 0x00001005 -#define B153600 0x00001006 -#define B307200 0x00001007 -#define B614400 0x00001008 -#define B921600 0x00001009 -#define B500000 0x0000100a -#define B576000 0x0000100b -#define B1000000 0x0000100c -#define B1152000 0x0000100d -#define B1500000 0x0000100e -#define B2000000 0x0000100f -#define __MAX_BAUD B2000000 \ No newline at end of file +#define __B57600 0x00001001 +#define __B115200 0x00001002 +#define __B230400 0x00001003 +#define __B460800 0x00001004 +#define __B76800 0x00001005 +#define __B153600 0x00001006 +#define __B307200 0x00001007 +#define __B614400 0x00001008 +#define __B921600 0x00001009 +#define __B500000 0x0000100a +#define __B576000 0x0000100b +#define __B1000000 0x0000100c +#define __B1152000 0x0000100d +#define __B1500000 0x0000100e +#define __B2000000 0x0000100f +#define __BOTHER 0x00001000 \ No newline at end of file diff --git a/lib/libc/include/sparc-linux-gnu/bits/termios-struct.h b/lib/libc/include/sparc-linux-gnu/bits/termios-struct.h deleted file mode 100644 index 815227e7d4156a648150a98d2d0cc739f8a7a98d..0000000000000000000000000000000000000000 --- a/lib/libc/include/sparc-linux-gnu/bits/termios-struct.h +++ /dev/null @@ -1,34 +0,0 @@ -/* struct termios definition. Linux/sparc version. - Copyright (C) 2019-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library. If not, see - . */ - -#ifndef _TERMIOS_H -# error "Never include directly; use instead." -#endif - -#define NCCS 17 -struct termios - { - tcflag_t c_iflag; /* input mode flags */ - tcflag_t c_oflag; /* output mode flags */ - tcflag_t c_cflag; /* control mode flags */ - tcflag_t c_lflag; /* local mode flags */ - cc_t c_line; /* line discipline */ - cc_t c_cc[NCCS]; /* control characters */ -#define _HAVE_STRUCT_TERMIOS_C_ISPEED 0 -#define _HAVE_STRUCT_TERMIOS_C_OSPEED 0 - }; \ No newline at end of file diff --git a/lib/libc/include/x86-linux-gnu/bits/floatn.h b/lib/libc/include/x86-linux-gnu/bits/floatn.h index b2716390f8c5457259a5a8bfe21f81ebc9d7ffb7..b0bab474a1aaffa8ded90c4e5426662d23a080d8 100644 --- a/lib/libc/include/x86-linux-gnu/bits/floatn.h +++ b/lib/libc/include/x86-linux-gnu/bits/floatn.h @@ -25,11 +25,15 @@ floating-point type with the IEEE 754 binary128 format, and this glibc includes corresponding *f128 interfaces for it. The required libgcc support was added some time after the basic compiler - support, for x86_64 and x86. */ + support, for x86_64 and x86. Intel SYCL compiler doesn't support + _Float128: https://github.com/intel/llvm/issues/16903 + */ #if (defined __x86_64__ \ ? __GNUC_PREREQ (4, 3) \ : (defined __GNU__ ? __GNUC_PREREQ (4, 5) : __GNUC_PREREQ (4, 4))) \ - || __glibc_clang_prereq (3, 4) + || (__glibc_clang_prereq (3, 9) \ + && (!defined __INTEL_LLVM_COMPILER \ + || !defined SYCL_LANGUAGE_VERSION)) # define __HAVE_FLOAT128 1 #else # define __HAVE_FLOAT128 0 @@ -89,7 +93,7 @@ typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__))); /* The type _Float128 exists only since GCC 7.0. */ # if !__GNUC_PREREQ (7, 0) \ || (defined __cplusplus && !__GNUC_PREREQ (13, 0)) \ - || __glibc_clang_prereq (3, 4) + || __glibc_clang_prereq (3, 9) typedef __float128 _Float128; # endif -- 2.54.0 From eb17d4562a6515baa7128eb4cd3ffdab3e6c4835 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 31 Jul 2025 22:36:52 -0700 Subject: [PATCH 3/6] std.Io.Writer.Hashed: fix bad assert --- lib/std/Io/Writer.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index ef487958e5c747cc597bab9b6baab9322521ddfe..a84077f8f31370ebb51eb6135fd7bb64cd03460f 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2350,7 +2350,7 @@ pub fn Hashed(comptime Hasher: type) type { this.hasher.update(slice); } const pattern = data[data.len - 1]; - assert(remaining == splat * pattern.len); + assert(remaining <= splat * pattern.len); switch (pattern.len) { 0 => { assert(remaining == 0); -- 2.54.0 From 675b1a15a17d796fa9e48902047c3c7b350a1d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Thu, 31 Jul 2025 22:08:36 +0200 Subject: [PATCH 4/6] glibc: update crt0 code to 2.42 --- lib/libc/glibc/elf/elf.h | 33 +++---- lib/libc/glibc/include/elf.h | 13 +++ lib/libc/glibc/include/libc-symbols.h | 8 +- lib/libc/glibc/include/stdlib.h | 15 ++++ lib/libc/glibc/io/fcntl.h | 9 +- lib/libc/glibc/posix/bits/types.h | 2 +- lib/libc/glibc/stdlib/stdlib.h | 6 ++ .../aarch64/nptl/bits/pthreadtypes-arch.h | 24 ++--- lib/libc/glibc/sysdeps/aarch64/start-2.33.S | 25 ++++-- lib/libc/glibc/sysdeps/aarch64/start.S | 14 +-- lib/libc/glibc/sysdeps/aarch64/sysdep.h | 87 ++----------------- lib/libc/glibc/sysdeps/generic/sysdep.h | 3 + lib/libc/glibc/sysdeps/htl/libc-lockP.h | 49 +++-------- lib/libc/glibc/sysdeps/mach/sysdep.h | 5 ++ lib/libc/glibc/sysdeps/nptl/pthread.h | 5 ++ lib/libc/glibc/sysdeps/sparc/sparc32/start.S | 11 ++- lib/libc/glibc/sysdeps/sparc/sparc64/start.S | 4 + .../sysdeps/unix/sysv/linux/aarch64/sysdep.h | 9 +- .../sysdeps/unix/sysv/linux/kernel-features.h | 4 + .../sysdeps/unix/sysv/linux/riscv/sysdep.h | 3 +- lib/libc/glibc/sysdeps/x86/sysdep.h | 29 +++++++ 21 files changed, 165 insertions(+), 193 deletions(-) diff --git a/lib/libc/glibc/elf/elf.h b/lib/libc/glibc/elf/elf.h index 96df2eec01ebbe02a13c0b0be6eababc5def0c75..2f29a47c0bb8c43d2ed0f8faab60dc8840b3f495 100644 --- a/lib/libc/glibc/elf/elf.h +++ b/lib/libc/glibc/elf/elf.h @@ -837,12 +837,15 @@ typedef struct #define NT_ARM_ZT 0x40d /* ARM SME ZT registers. */ #define NT_ARM_FPMR 0x40e /* ARM floating point mode register. */ #define NT_ARM_POE 0x40f /* ARM POE registers. */ +#define NT_ARM_GCS 0x410 /* ARM GCS state. */ #define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note. */ #define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers. */ #define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode. */ #define NT_MIPS_MSA 0x802 /* MIPS SIMD registers. */ #define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */ #define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */ +#define NT_RISCV_TAGGED_ADDR_CTRL 0x902 /* RISC-V tagged + address control */ #define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers. */ #define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers. */ @@ -2906,19 +2909,6 @@ enum #define R_AARCH64_NONE 0 /* No relocation. */ -/* ILP32 AArch64 relocs. */ -#define R_AARCH64_P32_ABS32 1 /* Direct 32 bit. */ -#define R_AARCH64_P32_COPY 180 /* Copy symbol at runtime. */ -#define R_AARCH64_P32_GLOB_DAT 181 /* Create GOT entry. */ -#define R_AARCH64_P32_JUMP_SLOT 182 /* Create PLT entry. */ -#define R_AARCH64_P32_RELATIVE 183 /* Adjust by program base. */ -#define R_AARCH64_P32_TLS_DTPMOD 184 /* Module number, 32 bit. */ -#define R_AARCH64_P32_TLS_DTPREL 185 /* Module-relative offset, 32 bit. */ -#define R_AARCH64_P32_TLS_TPREL 186 /* TP-relative offset, 32 bit. */ -#define R_AARCH64_P32_TLSDESC 187 /* TLS Descriptor. */ -#define R_AARCH64_P32_IRELATIVE 188 /* STT_GNU_IFUNC relocation. */ - -/* LP64 AArch64 relocs. */ #define R_AARCH64_ABS64 257 /* Direct 64 bit. */ #define R_AARCH64_ABS32 258 /* Direct 32 bit. */ #define R_AARCH64_ABS16 259 /* Direct 16-bit. */ @@ -4091,6 +4081,7 @@ enum #define R_RISCV_TLS_DTPREL64 9 #define R_RISCV_TLS_TPREL32 10 #define R_RISCV_TLS_TPREL64 11 +#define R_RISCV_TLSDESC 12 #define R_RISCV_BRANCH 16 #define R_RISCV_JAL 17 #define R_RISCV_CALL 18 @@ -4116,16 +4107,10 @@ enum #define R_RISCV_SUB16 38 #define R_RISCV_SUB32 39 #define R_RISCV_SUB64 40 -#define R_RISCV_GNU_VTINHERIT 41 -#define R_RISCV_GNU_VTENTRY 42 +#define R_RISCV_GOT32_PCREL 41 #define R_RISCV_ALIGN 43 #define R_RISCV_RVC_BRANCH 44 #define R_RISCV_RVC_JUMP 45 -#define R_RISCV_RVC_LUI 46 -#define R_RISCV_GPREL_I 47 -#define R_RISCV_GPREL_S 48 -#define R_RISCV_TPREL_I 49 -#define R_RISCV_TPREL_S 50 #define R_RISCV_RELAX 51 #define R_RISCV_SUB6 52 #define R_RISCV_SET6 53 @@ -4137,8 +4122,12 @@ enum #define R_RISCV_PLT32 59 #define R_RISCV_SET_ULEB128 60 #define R_RISCV_SUB_ULEB128 61 +#define R_RISCV_TLSDESC_HI20 62 +#define R_RISCV_TLSDESC_LOAD_LO12 63 +#define R_RISCV_TLSDESC_ADD_LO12 64 +#define R_RISCV_TLSDESC_CALL 65 -#define R_RISCV_NUM 62 +#define R_RISCV_NUM 66 /* RISC-V specific values for the st_other field. */ #define STO_RISCV_VARIANT_CC 0x80 /* Function uses variant calling @@ -4147,7 +4136,7 @@ enum /* RISC-V specific values for the sh_type field. */ #define SHT_RISCV_ATTRIBUTES (SHT_LOPROC + 3) -/* RISC-V specific values for the p_type field. */ +/* RISC-V specific values for the p_type field (deprecated). */ #define PT_RISCV_ATTRIBUTES (PT_LOPROC + 3) /* RISC-V specific values for the d_tag field. */ diff --git a/lib/libc/glibc/include/elf.h b/lib/libc/glibc/include/elf.h index 14ed67ff67d36349b12750c431478ce86c504ab0..1424982bb59073b878f347e5e178ff4c22ec6782 100644 --- a/lib/libc/glibc/include/elf.h +++ b/lib/libc/glibc/include/elf.h @@ -15,6 +15,19 @@ # define ELF_NOTE_NEXT_OFFSET(namesz, descsz, align) \ ALIGN_UP (ELF_NOTE_DESC_OFFSET ((namesz), (align)) + (descsz), (align)) +# ifdef HIDDEN_VAR_NEEDS_DYNAMIC_RELOC +# define DL_ADDRESS_WITHOUT_RELOC(expr) (expr) +# else +/* Evaluate EXPR without run-time relocation for it. EXPR should be an + array, an address of an object, or a string literal. */ +# define DL_ADDRESS_WITHOUT_RELOC(expr) \ + ({ \ + __auto_type _result = (expr); \ + asm ("" : "+r" (_result)); \ + _result; \ + }) +# endif + /* Some information which is not meant for the public and therefore not in . */ # include diff --git a/lib/libc/glibc/include/libc-symbols.h b/lib/libc/glibc/include/libc-symbols.h index b11460131dd5d3a7b416eaefc8f61f3fe8813560..7f2c8938b6cd9d5aaba9dd4ba1daa3d4cda1a1cf 100644 --- a/lib/libc/glibc/include/libc-symbols.h +++ b/lib/libc/glibc/include/libc-symbols.h @@ -155,7 +155,7 @@ extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))) \ __attribute_copy__ (name); -/* Zig patch. weak_hidden_alias was removed from glibc v2.36 (v2.37?), Zig +/* zig patch: weak_hidden_alias was removed from glibc v2.36 (v2.37?), Zig needs it for the v2.32 and earlier {f,l,}stat wrappers, so only include in this header for 2.32 and earlier. */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 32) || __GLIBC__ < 2 @@ -220,7 +220,7 @@ #define __make_section_unallocated(section_string) \ asm (".section " section_string "\n\t.previous"); -/* Tacking on "\n\t#" to the section name makes gcc put it's bogus +/* Tacking on "\n\t#" to the section name makes gcc put its bogus section attributes on what looks like a comment to the assembler. */ #ifdef HAVE_SECTION_QUOTES # define __sec_comment "\"\n\t#\"" @@ -280,7 +280,7 @@ for linking") /* - + */ #ifdef HAVE_GNU_RETAIN @@ -807,7 +807,7 @@ for linking") #define libm_ifunc_init() #define libm_ifunc(name, expr) \ __ifunc (name, name, expr, void, libm_ifunc_init) - + /* These macros facilitate sharing source files with gnulib. They are here instead of sys/cdefs.h because they should not be diff --git a/lib/libc/glibc/include/stdlib.h b/lib/libc/glibc/include/stdlib.h index 57f4ab8545023300fde008a80a1bcf13e0cec8bb..b7147ba59071a4abf4c80dadab288925ec4696b6 100644 --- a/lib/libc/glibc/include/stdlib.h +++ b/lib/libc/glibc/include/stdlib.h @@ -368,6 +368,21 @@ struct abort_msg_s extern struct abort_msg_s *__abort_msg; libc_hidden_proto (__abort_msg) +enum readonly_error_type +{ + readonly_noerror, + readonly_area_writable, + readonly_procfs_inaccessible, + readonly_procfs_open_fail, +}; + +extern enum readonly_error_type __readonly_area (const void *ptr, + size_t size) + attribute_hidden; +extern enum readonly_error_type __readonly_area_fallback (const void *ptr, + size_t size) + attribute_hidden; + # if IS_IN (rtld) extern __typeof (unsetenv) unsetenv attribute_hidden; extern __typeof (__strtoul_internal) __strtoul_internal attribute_hidden; diff --git a/lib/libc/glibc/io/fcntl.h b/lib/libc/glibc/io/fcntl.h index 2e07f8fc4b2cccc74de835b9e13b34fd4108420f..d99dc68a88742bf898f6ac501d20a8afb989e58e 100644 --- a/lib/libc/glibc/io/fcntl.h +++ b/lib/libc/glibc/io/fcntl.h @@ -168,7 +168,7 @@ typedef __pid_t pid_t; #endif -/* fcntl was a simple symbol until glibc 2.27 inclusive. glibc 2.28 onwards +/* zig patch: fcntl was a simple symbol until glibc 2.27 inclusive. glibc 2.28 onwards * re-defines it to fcntl64 (via #define) if _FILE_OFFSET_BITS == 64. */ #if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 28) || __GLIBC__ > 2 /* Do the file control operation described by CMD on FD. @@ -288,16 +288,17 @@ extern int creat64 (const char *__file, mode_t __mode) __nonnull ((1)); # define F_TEST 3 /* Test a region for other processes locks. */ # ifndef __USE_FILE_OFFSET64 -extern int lockf (int __fd, int __cmd, off_t __len); +extern int lockf (int __fd, int __cmd, off_t __len) __wur; # else # ifdef __REDIRECT -extern int __REDIRECT (lockf, (int __fd, int __cmd, __off64_t __len), lockf64); +extern int __REDIRECT (lockf, (int __fd, int __cmd, __off64_t __len), + lockf64) __wur; # else # define lockf lockf64 # endif # endif # ifdef __USE_LARGEFILE64 -extern int lockf64 (int __fd, int __cmd, off64_t __len); +extern int lockf64 (int __fd, int __cmd, off64_t __len) __wur; # endif #endif diff --git a/lib/libc/glibc/posix/bits/types.h b/lib/libc/glibc/posix/bits/types.h index 97d2b1be3566ef3519319aebb3b1176f320ac9e9..a6638467c8bcda24edde5bdea54d933dbd65feeb 100644 --- a/lib/libc/glibc/posix/bits/types.h +++ b/lib/libc/glibc/posix/bits/types.h @@ -217,7 +217,7 @@ typedef int __sig_atomic_t; /* Seconds since the Epoch, visible to user code when time_t is too narrow only for consistency with the old way of widening too-narrow types. User code should never use __time64_t. */ -/* Zig patch: Don't check __LIBC here because it breaks fstatat.c on x86. */ +/* zig patch: Don't check __LIBC here because it breaks fstatat.c on x86. */ #if __TIMESIZE == 64 # define __time64_t __time_t #elif __TIMESIZE != 64 diff --git a/lib/libc/glibc/stdlib/stdlib.h b/lib/libc/glibc/stdlib/stdlib.h index 975f5aeb0b26717560ed7541bc547f0f3f1a48e5..cd4503c761887324ddda8f0fd630e99c2bff196d 100644 --- a/lib/libc/glibc/stdlib/stdlib.h +++ b/lib/libc/glibc/stdlib/stdlib.h @@ -985,6 +985,12 @@ __extension__ extern long long int llabs (long long int __x) __THROW __attribute__ ((__const__)) __wur; #endif +#if __GLIBC_USE (ISOC2Y) +extern unsigned int uabs (int __x) __THROW __attribute__ ((__const__)) __wur; +extern unsigned long int ulabs (long int __x) __THROW __attribute__ ((__const__)) __wur; +__extension__ extern unsigned long long int ullabs (long long int __x) + __THROW __attribute__ ((__const__)) __wur; +#endif /* Return the `div_t', `ldiv_t' or `lldiv_t' representation of the value of NUMER over DENOM. */ diff --git a/lib/libc/glibc/sysdeps/aarch64/nptl/bits/pthreadtypes-arch.h b/lib/libc/glibc/sysdeps/aarch64/nptl/bits/pthreadtypes-arch.h index 7f11e82ff5720db9089d6fee1f95490243aec846..c2825603353ee2013726ac0ffd8678523453cafe 100644 --- a/lib/libc/glibc/sysdeps/aarch64/nptl/bits/pthreadtypes-arch.h +++ b/lib/libc/glibc/sysdeps/aarch64/nptl/bits/pthreadtypes-arch.h @@ -21,23 +21,13 @@ #include -#ifdef __ILP32__ -# define __SIZEOF_PTHREAD_ATTR_T 32 -# define __SIZEOF_PTHREAD_MUTEX_T 32 -# define __SIZEOF_PTHREAD_MUTEXATTR_T 4 -# define __SIZEOF_PTHREAD_CONDATTR_T 4 -# define __SIZEOF_PTHREAD_RWLOCK_T 48 -# define __SIZEOF_PTHREAD_BARRIER_T 20 -# define __SIZEOF_PTHREAD_BARRIERATTR_T 4 -#else -# define __SIZEOF_PTHREAD_ATTR_T 64 -# define __SIZEOF_PTHREAD_MUTEX_T 48 -# define __SIZEOF_PTHREAD_MUTEXATTR_T 8 -# define __SIZEOF_PTHREAD_CONDATTR_T 8 -# define __SIZEOF_PTHREAD_RWLOCK_T 56 -# define __SIZEOF_PTHREAD_BARRIER_T 32 -# define __SIZEOF_PTHREAD_BARRIERATTR_T 8 -#endif +#define __SIZEOF_PTHREAD_ATTR_T 64 +#define __SIZEOF_PTHREAD_MUTEX_T 48 +#define __SIZEOF_PTHREAD_MUTEXATTR_T 8 +#define __SIZEOF_PTHREAD_CONDATTR_T 8 +#define __SIZEOF_PTHREAD_RWLOCK_T 56 +#define __SIZEOF_PTHREAD_BARRIER_T 32 +#define __SIZEOF_PTHREAD_BARRIERATTR_T 8 #define __SIZEOF_PTHREAD_COND_T 48 #define __SIZEOF_PTHREAD_RWLOCKATTR_T 8 diff --git a/lib/libc/glibc/sysdeps/aarch64/start-2.33.S b/lib/libc/glibc/sysdeps/aarch64/start-2.33.S index d96cf57e2da901bc21e69bf5a87ed034820453f0..6d8d49ce239036a3c661bb5a46960d74d26001c3 100644 --- a/lib/libc/glibc/sysdeps/aarch64/start-2.33.S +++ b/lib/libc/glibc/sysdeps/aarch64/start-2.33.S @@ -54,8 +54,8 @@ _start: mov x5, x0 /* Load argc and a pointer to argv */ - ldr PTR_REG (1), [sp, #0] - add x2, sp, #PTR_SIZE + ldr x1, [sp, #0] + add x2, sp, 8 /* Setup stack limit in argument register */ mov x6, sp @@ -63,13 +63,13 @@ _start: #ifdef PIC # ifdef SHARED adrp x0, :got:main - ldr PTR_REG (0), [x0, #:got_lo12:main] + ldr x0, [x0, #:got_lo12:main] adrp x3, :got:__libc_csu_init - ldr PTR_REG (3), [x3, #:got_lo12:__libc_csu_init] + ldr x3, [x3, #:got_lo12:__libc_csu_init] adrp x4, :got:__libc_csu_fini - ldr PTR_REG (4), [x4, #:got_lo12:__libc_csu_fini] + ldr x4, [x4, #:got_lo12:__libc_csu_fini] # else adrp x0, __wrap_main add x0, x0, :lo12:__wrap_main @@ -80,9 +80,18 @@ _start: # endif #else /* Set up the other arguments in registers */ - MOVL (0, main) - MOVL (3, __libc_csu_init) - MOVL (4, __libc_csu_fini) + movz x0, :abs_g3:main + movk x0, :abs_g2_nc:main + movk x0, :abs_g1_nc:main + movk x0, :abs_g0_nc:main + movz x3, :abs_g3:__libc_csu_init + movk x3, :abs_g2_nc:__libc_csu_init + movk x3, :abs_g1_nc:__libc_csu_init + movk x3, :abs_g0_nc:__libc_csu_init + movz x4, :abs_g3:__libc_csu_fini + movk x4, :abs_g2_nc:__libc_csu_fini + movk x4, :abs_g1_nc:__libc_csu_fini + movk x4, :abs_g0_nc:__libc_csu_fini #endif /* __libc_start_main (main, argc, argv, init, fini, rtld_fini, diff --git a/lib/libc/glibc/sysdeps/aarch64/start.S b/lib/libc/glibc/sysdeps/aarch64/start.S index ef6b5ad686c87bd51f65bef6db43ff935d666a7e..694c338c8be3f6ab642e88f84eeec804d6946580 100644 --- a/lib/libc/glibc/sysdeps/aarch64/start.S +++ b/lib/libc/glibc/sysdeps/aarch64/start.S @@ -70,8 +70,8 @@ ENTRY(_start) mov x5, x0 /* Load argc and a pointer to argv */ - ldr PTR_REG (1), [sp, #0] - add x2, sp, #PTR_SIZE + ldr x1, [sp, #0] + add x2, sp, 8 /* Setup stack limit in argument register */ mov x6, sp @@ -79,14 +79,16 @@ ENTRY(_start) #ifdef PIC # ifdef SHARED adrp x0, :got:main - ldr PTR_REG (0), [x0, #:got_lo12:main] + ldr x0, [x0, #:got_lo12:main] # else adrp x0, __wrap_main add x0, x0, :lo12:__wrap_main # endif #else - /* Set up the other arguments in registers */ - MOVL (0, main) + movz x0, :abs_g3:main + movk x0, :abs_g2_nc:main + movk x0, :abs_g1_nc:main + movk x0, :abs_g0_nc:main #endif mov x3, #0 /* Used to be init. */ mov x4, #0 /* Used to be fini. */ @@ -106,7 +108,7 @@ ENTRY(_start) because crt1.o and rcrt1.o share code and the later must avoid the use of GOT relocations before __libc_start_main is called. */ __wrap_main: - BTI_C + bti c b main #endif END(_start) diff --git a/lib/libc/glibc/sysdeps/aarch64/sysdep.h b/lib/libc/glibc/sysdeps/aarch64/sysdep.h index 036eb125274e6ec4052f0bf0eb7a5daf889444b5..f5e28cb2427e0f7ce5e1de749b9f93ca86507131 100644 --- a/lib/libc/glibc/sysdeps/aarch64/sysdep.h +++ b/lib/libc/glibc/sysdeps/aarch64/sysdep.h @@ -21,59 +21,15 @@ #include -#ifdef __LP64__ -# define AARCH64_R(NAME) R_AARCH64_ ## NAME -# define PTR_REG(n) x##n -# define PTR_LOG_SIZE 3 -# define PTR_ARG(n) -# define SIZE_ARG(n) -#else -# define AARCH64_R(NAME) R_AARCH64_P32_ ## NAME -# define PTR_REG(n) w##n -# define PTR_LOG_SIZE 2 -# define PTR_ARG(n) mov w##n, w##n -# define SIZE_ARG(n) mov w##n, w##n -#endif - -#define PTR_SIZE (1< +/* This macro is defined in Mach system headers, but string functions use it + with different definitions depending on whether being compiled for + wide-characters or not. */ +#undef P2ALIGN + /* The Mach definitions assume underscores should be prepended to symbol names. Redefine them to do so only when appropriate. */ #undef EXT diff --git a/lib/libc/glibc/sysdeps/nptl/pthread.h b/lib/libc/glibc/sysdeps/nptl/pthread.h index 9ad36cabe92e96e172b60dbab67531b4ae9758e4..92957a620d9584685dbbd2fb3bf5dc8f050d1041 100644 --- a/lib/libc/glibc/sysdeps/nptl/pthread.h +++ b/lib/libc/glibc/sysdeps/nptl/pthread.h @@ -1317,6 +1317,11 @@ extern int pthread_getcpuclockid (pthread_t __thread_id, __THROW __nonnull ((2)); #endif +#ifdef __USE_GNU +/* Return the Linux TID for THREAD_ID. Returns -1 on failure. */ +extern pid_t pthread_gettid_np (pthread_t __thread_id); +#endif + /* Install handlers to be called when a new process is created with FORK. The PREPARE handler is called in the parent process just before performing diff --git a/lib/libc/glibc/sysdeps/sparc/sparc32/start.S b/lib/libc/glibc/sysdeps/sparc/sparc32/start.S index 694b020ce0defff4b73eb0415d8fdbe1b1c3e262..8393760da684d4f333090d2dfb2aecbe8a8e87e9 100644 --- a/lib/libc/glibc/sysdeps/sparc/sparc32/start.S +++ b/lib/libc/glibc/sysdeps/sparc/sparc32/start.S @@ -35,6 +35,7 @@ #include +#define FRAME_SIZE 104 .section ".text" .align 4 @@ -48,12 +49,12 @@ _start: /* Terminate the stack frame, and reserve space for functions to drop their arguments. */ mov %g0, %fp - sub %sp, 6*4, %sp + sub %sp, FRAME_SIZE, %sp /* Extract the arguments and environment as encoded on the stack. The argument info starts after one register window (16 words) past the SP. */ - ld [%sp+22*4], %o1 - add %sp, 23*4, %o2 + ld [%sp+168], %o1 + add %sp, 172, %o2 /* Load the addresses of the user entry points. */ #ifndef PIC @@ -73,6 +74,10 @@ _start: be NULL. */ mov %g1, %o5 + /* Provide the highest stack address to update the __libc_stack_end (used + to enable executable stacks if required). */ + st %sp, [%sp+23*4] + /* Let libc do the rest of the initialization, and call main. */ call __libc_start_main nop diff --git a/lib/libc/glibc/sysdeps/sparc/sparc64/start.S b/lib/libc/glibc/sysdeps/sparc/sparc64/start.S index c9c25c2e4794a0405996dd93d88e06cf5604206f..08e1e77210540cdac66493c22fe9b3c6859c9cd1 100644 --- a/lib/libc/glibc/sysdeps/sparc/sparc64/start.S +++ b/lib/libc/glibc/sysdeps/sparc/sparc64/start.S @@ -74,6 +74,10 @@ _start: be NULL. */ mov %g1, %o5 + /* Provide the highest stack address to update the __libc_stack_end (used + to enable executable stacks if required). */ + stx %sp, [%sp+STACK_BIAS+22*8] + /* Let libc do the rest of the initialization, and call main. */ call __libc_start_main nop diff --git a/lib/libc/glibc/sysdeps/unix/sysv/linux/aarch64/sysdep.h b/lib/libc/glibc/sysdeps/unix/sysv/linux/aarch64/sysdep.h index b81380593125920b4257367da204e6a043ab7a06..f0e8d64eefac3581108d93a96fc60941aec00dc1 100644 --- a/lib/libc/glibc/sysdeps/unix/sysv/linux/aarch64/sysdep.h +++ b/lib/libc/glibc/sysdeps/unix/sysv/linux/aarch64/sysdep.h @@ -152,13 +152,8 @@ #else /* not __ASSEMBLER__ */ -# ifdef __LP64__ -# define VDSO_NAME "LINUX_2.6.39" -# define VDSO_HASH 123718537 -# else -# define VDSO_NAME "LINUX_4.9" -# define VDSO_HASH 61765625 -# endif +# define VDSO_NAME "LINUX_2.6.39" +# define VDSO_HASH 123718537 /* List of system calls which are supported as vsyscalls. */ # define HAVE_CLOCK_GETRES64_VSYSCALL "__kernel_clock_getres" diff --git a/lib/libc/glibc/sysdeps/unix/sysv/linux/kernel-features.h b/lib/libc/glibc/sysdeps/unix/sysv/linux/kernel-features.h index 86b2d3ce512588c8823c11076c91116092836402..a49a9159cfa7d4496395581e64a2a7ad2d8b99a8 100644 --- a/lib/libc/glibc/sysdeps/unix/sysv/linux/kernel-features.h +++ b/lib/libc/glibc/sysdeps/unix/sysv/linux/kernel-features.h @@ -54,6 +54,10 @@ configurations). */ #define __ASSUME_SET_ROBUST_LIST 1 +/* The termios2 interface was introduced across all architectures except + Alpha in kernel 2.6.22. */ +#define __ASSUME_TERMIOS2 1 + /* Support for various CLOEXEC and NONBLOCK flags was added in 2.6.27. */ #define __ASSUME_IN_NONBLOCK 1 diff --git a/lib/libc/glibc/sysdeps/unix/sysv/linux/riscv/sysdep.h b/lib/libc/glibc/sysdeps/unix/sysv/linux/riscv/sysdep.h index ee015dfeb65b5131d1786a897aecddc5382953b1..05e0e0523d3f744cd7ec1341afc5126746bc80e8 100644 --- a/lib/libc/glibc/sysdeps/unix/sysv/linux/riscv/sysdep.h +++ b/lib/libc/glibc/sysdeps/unix/sysv/linux/riscv/sysdep.h @@ -145,11 +145,12 @@ # define HAVE_CLOCK_GETRES64_VSYSCALL "__vdso_clock_getres" # define HAVE_CLOCK_GETTIME64_VSYSCALL "__vdso_clock_gettime" # define HAVE_GETTIMEOFDAY_VSYSCALL "__vdso_gettimeofday" +# define HAVE_GETRANDOM_VSYSCALL "__vdso_getrandom" # else # define VDSO_NAME "LINUX_5.4" # define VDSO_HASH 61765876 -/* RV32 does not support the gettime VDSO syscalls. */ +/* RV32 does not support the gettime and getrandom VDSO syscalls. */ # endif # define HAVE_CLONE3_WRAPPER 1 diff --git a/lib/libc/glibc/sysdeps/x86/sysdep.h b/lib/libc/glibc/sysdeps/x86/sysdep.h index 541393f1dc3ec6ecabe091e5b9853427198780fc..b8e963b654c4cb6964e2cf8a10f1fe143ea2c2f2 100644 --- a/lib/libc/glibc/sysdeps/x86/sysdep.h +++ b/lib/libc/glibc/sysdeps/x86/sysdep.h @@ -102,6 +102,9 @@ | (1 << X86_XSTATE_ZMM_ID) \ | (1 << X86_XSTATE_APX_F_ID)) +/* The maximum supported xstate ID. */ +# define X86_XSTATE_MAX_ID X86_XSTATE_APX_F_ID + /* AMX state mask. */ # define AMX_STATE_SAVE_MASK \ ((1 << X86_XSTATE_TILECFG_ID) | (1 << X86_XSTATE_TILEDATA_ID)) @@ -123,6 +126,9 @@ | (1 << X86_XSTATE_K_ID) \ | (1 << X86_XSTATE_ZMM_H_ID)) +/* The maximum supported xstate ID. */ +# define X86_XSTATE_MAX_ID X86_XSTATE_ZMM_H_ID + /* States to be included in xsave_state_size. */ # define FULL_STATE_SAVE_MASK STATE_SAVE_MASK #endif @@ -177,6 +183,29 @@ #define atom_text_section .section ".text.atom", "ax" +#ifndef DL_STACK_ALIGNMENT +/* Due to GCC bug: + + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 + + __tls_get_addr may be called with 8-byte/4-byte stack alignment. + Although this bug has been fixed in GCC 4.9.4, 5.3 and 6, we can't + assume that stack will be always aligned at 16 bytes. */ +# ifdef __x86_64__ +# define DL_STACK_ALIGNMENT 8 +# define MINIMUM_ALIGNMENT 16 +# else +# define DL_STACK_ALIGNMENT 4 +# endif +#endif + +/* True if _dl_runtime_resolve/_dl_tlsdesc_dynamic should align stack for + STATE_SAVE or align stack to MINIMUM_ALIGNMENT bytes before calling + _dl_fixup/__tls_get_addr. */ +#define DL_RUNTIME_RESOLVE_REALIGN_STACK \ + (STATE_SAVE_ALIGNMENT > DL_STACK_ALIGNMENT \ + || MINIMUM_ALIGNMENT > DL_STACK_ALIGNMENT) + #endif /* __ASSEMBLER__ */ #endif /* _X86_SYSDEP_H */ -- 2.54.0 From a6f7927764fde807cb077fb4385be0729035e1db Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 1 Aug 2025 09:04:27 -0700 Subject: [PATCH 5/6] std.compress.flate.Decompress: use 64 buffered bits will have to find out why usize doesn't work for 32 bit targets some other time --- lib/std/compress/flate/Decompress.zig | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index a51fd0561062d23f0634a5fe8a6a906fb7600a33..90f89ea6c9d42c94b24c197a0173cbb51eb25f42 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -10,8 +10,8 @@ const Decompress = @This(); const Token = @import("Token.zig"); input: *Reader, -next_bits: usize, -remaining_bits: std.math.Log2Int(usize), +next_bits: Bits, +remaining_bits: std.math.Log2Int(Bits), reader: Reader, @@ -25,6 +25,9 @@ state: State, err: ?Error, +/// TODO: change this to usize +const Bits = u64; + const BlockType = enum(u2) { stored = 0, fixed = 1, @@ -498,14 +501,14 @@ fn takeBits(d: *Decompress, comptime U: type) !U { return u; } const in = d.input; - const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + const next_int = in.takeInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return takeBitsEnding(d, U), }; const needed_bits = @bitSizeOf(U) - remaining_bits; - const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; - d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits)); return u; } @@ -514,14 +517,14 @@ fn takeBitsEnding(d: *Decompress, comptime U: type) !U { const next_bits = d.next_bits; const in = d.input; const n = in.bufferedLen(); - assert(n < @sizeOf(usize)); + assert(n < @sizeOf(Bits)); const needed_bits = @bitSizeOf(U) - remaining_bits; if (n * 8 < needed_bits) return error.EndOfStream; - const next_int = in.takeVarInt(usize, .little, n) catch |err| switch (err) { + const next_int = in.takeVarInt(Bits, .little, n) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => unreachable, }; - const u: U = @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); d.next_bits = next_int >> needed_bits; d.remaining_bits = @intCast(n * 8 - @as(usize, needed_bits)); return u; @@ -532,37 +535,37 @@ fn peekBits(d: *Decompress, comptime U: type) !U { const next_bits = d.next_bits; if (remaining_bits >= @bitSizeOf(U)) return @truncate(next_bits); const in = d.input; - const next_int = in.peekInt(usize, .little) catch |err| switch (err) { + const next_int = in.peekInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return peekBitsEnding(d, U), }; const needed_bits = @bitSizeOf(U) - remaining_bits; - return @intCast(((next_int & ((@as(usize, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); + return @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) | next_bits); } fn peekBitsEnding(d: *Decompress, comptime U: type) !U { const remaining_bits = d.remaining_bits; const next_bits = d.next_bits; const in = d.input; - var u: usize = 0; + var u: Bits = 0; var remaining_needed_bits = @bitSizeOf(U) - remaining_bits; var i: usize = 0; while (remaining_needed_bits >= 8) { const byte = try specialPeek(in, next_bits, i); - u |= @as(usize, byte) << @intCast(i * 8); + u |= @as(Bits, byte) << @intCast(i * 8); remaining_needed_bits -= 8; i += 1; } if (remaining_needed_bits != 0) { const byte = try specialPeek(in, next_bits, i); - u |= @as(usize, byte) << @intCast((i * 8) + remaining_needed_bits); + u |= @as(Bits, byte) << @intCast((i * 8) + remaining_needed_bits); } return @truncate((u << remaining_bits) | next_bits); } /// If there is any unconsumed data, handles EndOfStream by pretending there /// are zeroes afterwards. -fn specialPeek(in: *Reader, next_bits: usize, i: usize) Reader.Error!u8 { +fn specialPeek(in: *Reader, next_bits: Bits, i: usize) Reader.Error!u8 { const peeked = in.peek(i + 1) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => if (next_bits == 0 and i == 0) return error.EndOfStream else return 0, @@ -578,13 +581,13 @@ fn tossBits(d: *Decompress, n: u4) !void { d.remaining_bits = remaining_bits - n; } else { const in = d.input; - const next_int = in.takeInt(usize, .little) catch |err| switch (err) { + const next_int = in.takeInt(Bits, .little) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => return tossBitsEnding(d, n), }; const needed_bits = n - remaining_bits; d.next_bits = next_int >> needed_bits; - d.remaining_bits = @intCast(@bitSizeOf(usize) - @as(usize, needed_bits)); + d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits)); } } @@ -593,9 +596,9 @@ fn tossBitsEnding(d: *Decompress, n: u4) !void { const in = d.input; const buffered_n = in.bufferedLen(); if (buffered_n == 0) return error.EndOfStream; - assert(buffered_n < @sizeOf(usize)); + assert(buffered_n < @sizeOf(Bits)); const needed_bits = n - remaining_bits; - const next_int = in.takeVarInt(usize, .little, buffered_n) catch |err| switch (err) { + const next_int = in.takeVarInt(Bits, .little, buffered_n) catch |err| switch (err) { error.ReadFailed => return error.ReadFailed, error.EndOfStream => unreachable, }; -- 2.54.0 From dcc3e6e1dd224f1719b0ad9ef6d8d9dc0ed497ec Mon Sep 17 00:00:00 2001 From: mlugg Date: Thu, 10 Jul 2025 09:18:10 +0100 Subject: [PATCH 6/6] build system: replace fuzzing UI with build UI, add time report This commit replaces the "fuzzer" UI, previously accessed with the `--fuzz` and `--port` flags, with a more interesting web UI which allows more interactions with the Zig build system. Most notably, it allows accessing the data emitted by a new "time report" system, which allows users to see which parts of Zig programs take the longest to compile. The option to expose the web UI is `--webui`. By default, it will listen on `[::1]` on a random port, but any IPv6 or IPv4 address can be specified with e.g. `--webui=[::1]:8000` or `--webui=127.0.0.1:8000`. The options `--fuzz` and `--time-report` both imply `--webui` if not given. Currently, `--webui` is incompatible with `--watch`; specifying both will cause `zig build` to exit with a fatal error. When the web UI is enabled, the build runner spawns the web server as soon as the configure phase completes. The frontend code consists of one HTML file, one JavaScript file, two CSS files, and a few Zig source files which are built into a WASM blob on-demand -- this is all very similar to the old fuzzer UI. Also inherited from the fuzzer UI is that the build system communicates with web clients over a WebSocket connection. When the build finishes, if `--webui` was passed (i.e. if the web server is running), the build runner does not terminate; it continues running to serve web requests, allowing interactive control of the build system. In the web interface is an overall "status" indicating whether a build is currently running, and also a list of all steps in this build. There are visual indicators (colors and spinners) for in-progress, succeeded, and failed steps. There is a "Rebuild" button which will cause the build system to reset the state of every step (note that this does not affect caching) and evaluate the step graph again. If `--time-report` is passed to `zig build`, a new section of the interface becomes visible, which associates every build step with a "time report". For most steps, this is just a simple "time taken" value. However, for `Compile` steps, the compiler communicates with the build system to provide it with much more interesting information: time taken for various pipeline phases, with a per-declaration and per-file breakdown, sorted by slowest declarations/files first. This feature is still in its early stages: the data can be a little tricky to understand, and there is no way to, for instance, sort by different properties, or filter to certain files. However, it has already given us some interesting statistics, and can be useful for spotting, for instance, particularly complex and slow compile-time logic. Additionally, if a compilation uses LLVM, its time report includes the "LLVM pass timing" information, which was previously accessible with the (now removed) `-ftime-report` compiler flag. To make time reports more useful, ZIR and compilation caches are ignored by the Zig compiler when they are enabled -- in other words, `Compile` steps *always* run, even if their result should be cached. This means that the flag can be used to analyze a project's compile time without having to repeatedly clear cache directory, for instance. However, when using `-fincremental`, updates other than the first will only show you the statistics for what changed on that particular update. Notably, this gives us a fairly nice way to see exactly which declarations were re-analyzed by an incremental update. If `--fuzz` is passed to `zig build`, another section of the web interface becomes visible, this time exposing the fuzzer. This is quite similar to the fuzzer UI this commit replaces, with only a few cosmetic tweaks. The interface is closer than before to supporting multiple fuzz steps at a time (in line with the overall strategy for this build UI, the goal will be for all of the fuzz steps to be accessible in the same interface), but still doesn't actually support it. The fuzzer UI looks quite different under the hood: as a result, various bugs are fixed, although other bugs remain. For instance, viewing the source code of any file other than the root of the main module is completely broken (as on master) due to some bogus file-to-module assignment logic in the fuzzer UI. Implementation notes: * The `lib/build-web/` directory holds the client side of the web UI. * The general server logic is in `std.Build.WebServer`. * Fuzzing-specific logic is in `std.Build.Fuzz`. * `std.Build.abi` is the new home of `std.Build.Fuzz.abi`, since it now relates to the build system web UI in general. * The build runner now has an **actual** general-purpose allocator, because thanks to `--watch` and `--webui`, the process can be arbitrarily long-lived. The gpa is `std.heap.DebugAllocator`, but the arena remains backed by `std.heap.page_allocator` for efficiency. I fixed several crashes caused by conflation of `gpa` and `arena` in the build runner and `std.Build`, but there may still be some I have missed. * The I/O logic in `std.Build.WebServer` is pretty gnarly; there are a *lot* of threads involved. I anticipate this situation improving significantly once the `std.Io` interface (with concurrency support) is introduced. --- .../web/main.zig => build-web/fuzz.zig} | 514 +++++------ lib/build-web/index.html | 202 +++++ lib/build-web/main.js | 346 ++++++++ lib/build-web/main.zig | 213 +++++ lib/build-web/style.css | 240 +++++ lib/build-web/time_report.css | 43 + lib/build-web/time_report.zig | 234 +++++ lib/compiler/build_runner.zig | 210 +++-- lib/fuzzer.zig | 2 +- lib/fuzzer/web/index.html | 161 ---- lib/fuzzer/web/main.js | 252 ------ lib/std/Build.zig | 3 + lib/std/Build/Fuzz.zig | 451 ++++++++-- lib/std/Build/Fuzz/WebServer.zig | 709 --------------- lib/std/Build/Fuzz/abi.zig | 112 --- lib/std/Build/Step.zig | 72 +- lib/std/Build/Step/Compile.zig | 9 +- lib/std/Build/Step/ObjCopy.zig | 2 +- lib/std/Build/Step/Options.zig | 1 + lib/std/Build/Step/Run.zig | 26 +- lib/std/Build/Step/TranslateC.zig | 2 +- lib/std/Build/WebServer.zig | 823 ++++++++++++++++++ lib/std/Build/abi.zig | 313 +++++++ lib/std/http/WebSocket.zig | 14 +- lib/std/net.zig | 41 + lib/std/zig/Server.zig | 15 + src/Compilation.zig | 196 ++++- src/Sema.zig | 40 +- src/Type.zig | 6 + src/Zcu.zig | 54 +- src/Zcu/PerThread.zig | 78 +- src/codegen/llvm.zig | 15 +- src/codegen/llvm/bindings.zig | 2 +- src/link.zig | 33 + src/main.zig | 90 +- src/zig_llvm.cpp | 15 +- src/zig_llvm.h | 5 +- tools/dump-cov.zig | 2 +- 38 files changed, 3755 insertions(+), 1791 deletions(-) rename lib/{fuzzer/web/main.zig => build-web/fuzz.zig} (52%) create mode 100644 lib/build-web/index.html create mode 100644 lib/build-web/main.js create mode 100644 lib/build-web/main.zig create mode 100644 lib/build-web/style.css create mode 100644 lib/build-web/time_report.css create mode 100644 lib/build-web/time_report.zig delete mode 100644 lib/fuzzer/web/index.html delete mode 100644 lib/fuzzer/web/main.js delete mode 100644 lib/std/Build/Fuzz/WebServer.zig delete mode 100644 lib/std/Build/Fuzz/abi.zig create mode 100644 lib/std/Build/WebServer.zig create mode 100644 lib/std/Build/abi.zig diff --git a/lib/fuzzer/web/main.zig b/lib/build-web/fuzz.zig similarity index 52% rename from lib/fuzzer/web/main.zig rename to lib/build-web/fuzz.zig index 5b16b0532ccfe5a8db6182614061f250b0fdc458..a05f9d2e649ac7edfe800b41d6aae9a3799c87ec 100644 --- a/lib/fuzzer/web/main.zig +++ b/lib/build-web/fuzz.zig @@ -1,229 +1,26 @@ -const std = @import("std"); -const assert = std.debug.assert; -const abi = std.Build.Fuzz.abi; -const gpa = std.heap.wasm_allocator; -const log = std.log; -const Coverage = std.debug.Coverage; -const Allocator = std.mem.Allocator; - -const Walk = @import("Walk"); -const Decl = Walk.Decl; -const html_render = @import("html_render"); - -/// Nanoseconds. -var server_base_timestamp: i64 = 0; -/// Milliseconds. -var client_base_timestamp: i64 = 0; -/// Relative to `server_base_timestamp`. +// Server timestamp. var start_fuzzing_timestamp: i64 = undefined; const js = struct { - extern "js" fn log(ptr: [*]const u8, len: usize) void; - extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn; - extern "js" fn timestamp() i64; - extern "js" fn emitSourceIndexChange() void; - extern "js" fn emitCoverageUpdate() void; - extern "js" fn emitEntryPointsUpdate() void; -}; + extern "fuzz" fn requestSources() void; + extern "fuzz" fn ready() void; -pub const std_options: std.Options = .{ - .logFn = logFn, + extern "fuzz" fn updateStats(html_ptr: [*]const u8, html_len: usize) void; + extern "fuzz" fn updateEntryPoints(html_ptr: [*]const u8, html_len: usize) void; + extern "fuzz" fn updateSource(html_ptr: [*]const u8, html_len: usize) void; + extern "fuzz" fn updateCoverage(covered_ptr: [*]const SourceLocationIndex, covered_len: u32) void; }; -pub fn panic(msg: []const u8, st: ?*std.builtin.StackTrace, addr: ?usize) noreturn { - _ = st; - _ = addr; - log.err("panic: {s}", .{msg}); - @trap(); -} - -fn logFn( - comptime message_level: log.Level, - comptime scope: @TypeOf(.enum_literal), - comptime format: []const u8, - args: anytype, -) void { - const level_txt = comptime message_level.asText(); - const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; - var buf: [500]u8 = undefined; - const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: { - buf[buf.len - 3 ..][0..3].* = "...".*; - break :l &buf; - }; - js.log(line.ptr, line.len); -} - -export fn alloc(n: usize) [*]u8 { - const slice = gpa.alloc(u8, n) catch @panic("OOM"); - return slice.ptr; -} - -var message_buffer: std.ArrayListAlignedUnmanaged(u8, .of(u64)) = .empty; - -/// Resizes the message buffer to be the correct length; returns the pointer to -/// the query string. -export fn message_begin(len: usize) [*]u8 { - message_buffer.resize(gpa, len) catch @panic("OOM"); - return message_buffer.items.ptr; -} - -export fn message_end() void { - const msg_bytes = message_buffer.items; - - const tag: abi.ToClientTag = @enumFromInt(msg_bytes[0]); - switch (tag) { - .current_time => return currentTimeMessage(msg_bytes), - .source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"), - .coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"), - .entry_points => return entryPointsMessage(msg_bytes) catch @panic("OOM"), - _ => unreachable, - } -} - -export fn unpack(tar_ptr: [*]u8, tar_len: usize) void { - const tar_bytes = tar_ptr[0..tar_len]; - log.debug("received {d} bytes of tar file", .{tar_bytes.len}); - - unpackInner(tar_bytes) catch |err| { - fatal("unable to unpack tar: {s}", .{@errorName(err)}); - }; -} - -/// Set by `set_input_string`. -var input_string: std.ArrayListUnmanaged(u8) = .empty; -var string_result: std.ArrayListUnmanaged(u8) = .empty; - -export fn set_input_string(len: usize) [*]u8 { - input_string.resize(gpa, len) catch @panic("OOM"); - return input_string.items.ptr; -} - -/// Looks up the root struct decl corresponding to a file by path. -/// Uses `input_string`. -export fn find_file_root() Decl.Index { - const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none); - return file.findRootDecl(); -} - -export fn decl_source_html(decl_index: Decl.Index) String { - const decl = decl_index.get(); - - string_result.clearRetainingCapacity(); - html_render.fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| { - fatal("unable to render source: {s}", .{@errorName(err)}); - }; - return String.init(string_result.items); -} - -export fn totalSourceLocations() usize { - return coverage_source_locations.items.len; -} - -export fn coveredSourceLocations() usize { - const covered_bits = recent_coverage_update.items[@sizeOf(abi.CoverageUpdateHeader)..]; - var count: usize = 0; - for (covered_bits) |byte| count += @popCount(byte); - return count; -} - -fn getCoverageUpdateHeader() *abi.CoverageUpdateHeader { - return @alignCast(@ptrCast(recent_coverage_update.items[0..@sizeOf(abi.CoverageUpdateHeader)])); -} - -export fn totalRuns() u64 { - const header = getCoverageUpdateHeader(); - return header.n_runs; -} - -export fn uniqueRuns() u64 { - const header = getCoverageUpdateHeader(); - return header.unique_runs; -} - -export fn totalRunsPerSecond() f64 { - @setFloatMode(.optimized); - const header = getCoverageUpdateHeader(); - const ns_elapsed: f64 = @floatFromInt(nsSince(start_fuzzing_timestamp)); - const n_runs: f64 = @floatFromInt(header.n_runs); - return n_runs / (ns_elapsed / std.time.ns_per_s); -} - -const String = Slice(u8); - -fn Slice(T: type) type { - return packed struct(u64) { - ptr: u32, - len: u32, - - fn init(s: []const T) @This() { - return .{ - .ptr = @intFromPtr(s.ptr), - .len = s.len, - }; - } - }; -} - -fn unpackInner(tar_bytes: []u8) !void { - var fbs = std.io.fixedBufferStream(tar_bytes); - var file_name_buffer: [1024]u8 = undefined; - var link_name_buffer: [1024]u8 = undefined; - var it = std.tar.iterator(fbs.reader(), .{ - .file_name_buffer = &file_name_buffer, - .link_name_buffer = &link_name_buffer, - }); - while (try it.next()) |tar_file| { - switch (tar_file.kind) { - .file => { - if (tar_file.size == 0 and tar_file.name.len == 0) break; - if (std.mem.endsWith(u8, tar_file.name, ".zig")) { - log.debug("found file: '{s}'", .{tar_file.name}); - const file_name = try gpa.dupe(u8, tar_file.name); - if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| { - const pkg_name = file_name[0..pkg_name_end]; - const gop = try Walk.modules.getOrPut(gpa, pkg_name); - const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len); - if (!gop.found_existing or - std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or - std.mem.eql(u8, file_name[pkg_name_end + 1 .. file_name.len - ".zig".len], pkg_name)) - { - gop.value_ptr.* = file; - } - const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)]; - assert(file == try Walk.add_file(file_name, file_bytes)); - } - } else { - log.warn("skipping: '{s}' - the tar creation should have done that", .{tar_file.name}); - } - }, - else => continue, - } - } -} - -fn fatal(comptime format: []const u8, args: anytype) noreturn { - var buf: [500]u8 = undefined; - const line = std.fmt.bufPrint(&buf, format, args) catch l: { - buf[buf.len - 3 ..][0..3].* = "...".*; - break :l &buf; - }; - js.panic(line.ptr, line.len); -} - -fn currentTimeMessage(msg_bytes: []u8) void { - client_base_timestamp = js.timestamp(); - server_base_timestamp = @bitCast(msg_bytes[1..][0..8].*); -} +pub fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void { + Walk.files.clearRetainingCapacity(); + Walk.decls.clearRetainingCapacity(); + Walk.modules.clearRetainingCapacity(); + recent_coverage_update.clearRetainingCapacity(); + selected_source_location = null; -/// Nanoseconds passed since a server timestamp. -fn nsSince(server_timestamp: i64) i64 { - const ms_passed = js.timestamp() - client_base_timestamp; - const ns_passed = server_base_timestamp - server_timestamp; - return ns_passed + ms_passed * std.time.ns_per_ms; -} + js.requestSources(); -fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void { - const Header = abi.SourceIndexHeader; + const Header = abi.fuzz.SourceIndexHeader; const header: Header = @bitCast(msg_bytes[0..@sizeOf(Header)].*); const directories_start = @sizeOf(Header); @@ -239,27 +36,55 @@ fn sourceIndexMessage(msg_bytes: []u8) error{OutOfMemory}!void { const source_locations: []const Coverage.SourceLocation = @alignCast(std.mem.bytesAsSlice(Coverage.SourceLocation, msg_bytes[source_locations_start..source_locations_end])); start_fuzzing_timestamp = header.start_timestamp; - try updateCoverage(directories, files, source_locations, string_bytes); - js.emitSourceIndexChange(); + try updateCoverageSources(directories, files, source_locations, string_bytes); + js.ready(); } -fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void { +var coverage = Coverage.init; +/// Index of type `SourceLocationIndex`. +var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .empty; +/// Contains the most recent coverage update message, unmodified. +var recent_coverage_update: std.ArrayListAlignedUnmanaged(u8, .of(u64)) = .empty; + +fn updateCoverageSources( + directories: []const Coverage.String, + files: []const Coverage.File, + source_locations: []const Coverage.SourceLocation, + string_bytes: []const u8, +) !void { + coverage.directories.clearRetainingCapacity(); + coverage.files.clearRetainingCapacity(); + coverage.string_bytes.clearRetainingCapacity(); + coverage_source_locations.clearRetainingCapacity(); + + try coverage_source_locations.appendSlice(gpa, source_locations); + try coverage.string_bytes.appendSlice(gpa, string_bytes); + + try coverage.files.entries.resize(gpa, files.len); + @memcpy(coverage.files.entries.items(.key), files); + try coverage.files.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items }); + + try coverage.directories.entries.resize(gpa, directories.len); + @memcpy(coverage.directories.entries.items(.key), directories); + try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items }); +} + +pub fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void { recent_coverage_update.clearRetainingCapacity(); recent_coverage_update.appendSlice(gpa, msg_bytes) catch @panic("OOM"); - js.emitCoverageUpdate(); + try updateStats(); + try updateCoverage(); } -var entry_points: std.ArrayListUnmanaged(u32) = .empty; +var entry_points: std.ArrayListUnmanaged(SourceLocationIndex) = .empty; -fn entryPointsMessage(msg_bytes: []u8) error{OutOfMemory}!void { - const header: abi.EntryPointHeader = @bitCast(msg_bytes[0..@sizeOf(abi.EntryPointHeader)].*); - entry_points.resize(gpa, header.flags.locs_len) catch @panic("OOM"); - @memcpy(entry_points.items, std.mem.bytesAsSlice(u32, msg_bytes[@sizeOf(abi.EntryPointHeader)..])); - js.emitEntryPointsUpdate(); -} - -export fn entryPoints() Slice(u32) { - return Slice(u32).init(entry_points.items); +pub fn entryPointsMessage(msg_bytes: []u8) error{OutOfMemory}!void { + const header: abi.fuzz.EntryPointHeader = @bitCast(msg_bytes[0..@sizeOf(abi.fuzz.EntryPointHeader)].*); + const slis: []align(1) const SourceLocationIndex = @ptrCast(msg_bytes[@sizeOf(abi.fuzz.EntryPointHeader)..]); + assert(slis.len == header.locsLen()); + try entry_points.resize(gpa, slis.len); + @memcpy(entry_points.items, slis); + try updateEntryPoints(); } /// Index into `coverage_source_locations`. @@ -277,11 +102,18 @@ const SourceLocationIndex = enum(u32) { fn sourceLocationLinkHtml( sli: SourceLocationIndex, out: *std.ArrayListUnmanaged(u8), + focused: bool, ) Allocator.Error!void { const sl = sli.ptr(); - try out.writer(gpa).print("", .{@intFromEnum(sli)}); + try out.writer(gpa).print("", .{ + @as([]const u8, if (focused) " class=\"status-running\"" else ""), + }); try sli.appendPath(out); - try out.writer(gpa).print(":{d}:{d}", .{ sl.line, sl.column }); + try out.writer(gpa).print(":{d}:{d} ", .{ + sl.line, + sl.column, + @intFromEnum(sli), + }); } fn appendPath(sli: SourceLocationIndex, out: *std.ArrayListUnmanaged(u8)) Allocator.Error!void { @@ -372,84 +204,174 @@ fn computeSourceAnnotations( } } -var coverage = Coverage.init; -/// Index of type `SourceLocationIndex`. -var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .empty; -/// Contains the most recent coverage update message, unmodified. -var recent_coverage_update: std.ArrayListAlignedUnmanaged(u8, .of(u64)) = .empty; +export fn fuzzUnpackSources(tar_ptr: [*]u8, tar_len: usize) void { + const tar_bytes = tar_ptr[0..tar_len]; + log.debug("received {d} bytes of sources.tar", .{tar_bytes.len}); -fn updateCoverage( - directories: []const Coverage.String, - files: []const Coverage.File, - source_locations: []const Coverage.SourceLocation, - string_bytes: []const u8, -) !void { - coverage.directories.clearRetainingCapacity(); - coverage.files.clearRetainingCapacity(); - coverage.string_bytes.clearRetainingCapacity(); - coverage_source_locations.clearRetainingCapacity(); - - try coverage_source_locations.appendSlice(gpa, source_locations); - try coverage.string_bytes.appendSlice(gpa, string_bytes); - - try coverage.files.entries.resize(gpa, files.len); - @memcpy(coverage.files.entries.items(.key), files); - try coverage.files.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items }); - - try coverage.directories.entries.resize(gpa, directories.len); - @memcpy(coverage.directories.entries.items(.key), directories); - try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items }); -} - -export fn sourceLocationLinkHtml(index: SourceLocationIndex) String { - string_result.clearRetainingCapacity(); - index.sourceLocationLinkHtml(&string_result) catch @panic("OOM"); - return String.init(string_result.items); -} - -/// Returns empty string if coverage metadata is not available for this source location. -export fn sourceLocationPath(sli: SourceLocationIndex) String { - string_result.clearRetainingCapacity(); - if (sli.haveCoverage()) sli.appendPath(&string_result) catch @panic("OOM"); - return String.init(string_result.items); -} - -export fn sourceLocationFileHtml(sli: SourceLocationIndex) String { - string_result.clearRetainingCapacity(); - sli.fileHtml(&string_result) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - error.SourceUnavailable => {}, + unpackSourcesInner(tar_bytes) catch |err| { + fatal("unable to unpack sources.tar: {s}", .{@errorName(err)}); }; - return String.init(string_result.items); } -export fn sourceLocationFileCoveredList(sli_file: SourceLocationIndex) Slice(SourceLocationIndex) { - const global = struct { - var result: std.ArrayListUnmanaged(SourceLocationIndex) = .empty; - fn add(i: u32, want_file: Coverage.File.Index) void { - const src_loc_index: SourceLocationIndex = @enumFromInt(i); - if (src_loc_index.ptr().file == want_file) result.appendAssumeCapacity(src_loc_index); +fn unpackSourcesInner(tar_bytes: []u8) !void { + var tar_reader: std.Io.Reader = .fixed(tar_bytes); + var file_name_buffer: [1024]u8 = undefined; + var link_name_buffer: [1024]u8 = undefined; + var it: std.tar.Iterator = .init(&tar_reader, .{ + .file_name_buffer = &file_name_buffer, + .link_name_buffer = &link_name_buffer, + }); + while (try it.next()) |tar_file| { + switch (tar_file.kind) { + .file => { + if (tar_file.size == 0 and tar_file.name.len == 0) break; + if (std.mem.endsWith(u8, tar_file.name, ".zig")) { + log.debug("found file: '{s}'", .{tar_file.name}); + const file_name = try gpa.dupe(u8, tar_file.name); + if (std.mem.indexOfScalar(u8, file_name, '/')) |pkg_name_end| { + const pkg_name = file_name[0..pkg_name_end]; + const gop = try Walk.modules.getOrPut(gpa, pkg_name); + const file: Walk.File.Index = @enumFromInt(Walk.files.entries.len); + if (!gop.found_existing or + std.mem.eql(u8, file_name[pkg_name_end..], "/root.zig") or + std.mem.eql(u8, file_name[pkg_name_end + 1 .. file_name.len - ".zig".len], pkg_name)) + { + gop.value_ptr.* = file; + } + const file_bytes = tar_reader.take(@intCast(tar_file.size)) catch unreachable; + it.unread_file_bytes = 0; // we have read the whole thing + assert(file == try Walk.add_file(file_name, file_bytes)); + } + } else { + log.warn("skipping: '{s}' - the tar creation should have done that", .{tar_file.name}); + } + }, + else => continue, } + } +} + +fn updateStats() error{OutOfMemory}!void { + @setFloatMode(.optimized); + + if (recent_coverage_update.items.len == 0) return; + + const hdr: *abi.fuzz.CoverageUpdateHeader = @alignCast(@ptrCast( + recent_coverage_update.items[0..@sizeOf(abi.fuzz.CoverageUpdateHeader)], + )); + + const covered_src_locs: usize = n: { + var n: usize = 0; + const covered_bits = recent_coverage_update.items[@sizeOf(abi.fuzz.CoverageUpdateHeader)..]; + for (covered_bits) |byte| n += @popCount(byte); + break :n n; }; - const want_file = sli_file.ptr().file; - global.result.clearRetainingCapacity(); + const total_src_locs = coverage_source_locations.items.len; + + const avg_speed: f64 = speed: { + const ns_elapsed: f64 = @floatFromInt(nsSince(start_fuzzing_timestamp)); + const n_runs: f64 = @floatFromInt(hdr.n_runs); + break :speed n_runs / (ns_elapsed / std.time.ns_per_s); + }; + + const html = try std.fmt.allocPrint(gpa, + \\{d} + \\{d} ({d:.1}%) + \\{d} / {d} ({d:.1}%) + \\{d:.0} + , .{ + hdr.n_runs, + hdr.unique_runs, + @as(f64, @floatFromInt(hdr.unique_runs)) / @as(f64, @floatFromInt(hdr.n_runs)), + covered_src_locs, + total_src_locs, + @as(f64, @floatFromInt(covered_src_locs)) / @as(f64, @floatFromInt(total_src_locs)), + avg_speed, + }); + defer gpa.free(html); + + js.updateStats(html.ptr, html.len); +} + +fn updateEntryPoints() error{OutOfMemory}!void { + var html: std.ArrayListUnmanaged(u8) = .empty; + defer html.deinit(gpa); + for (entry_points.items) |sli| { + try html.appendSlice(gpa, "
  • "); + try sli.sourceLocationLinkHtml(&html, selected_source_location == sli); + try html.appendSlice(gpa, "
    • \n"); + } + js.updateEntryPoints(html.items.ptr, html.items.len); +} + +fn updateCoverage() error{OutOfMemory}!void { + if (recent_coverage_update.items.len == 0) return; + const want_file = (selected_source_location orelse return).ptr().file; + + var covered: std.ArrayListUnmanaged(SourceLocationIndex) = .empty; + defer covered.deinit(gpa); // This code assumes 64-bit elements, which is incorrect if the executable // being fuzzed is not a 64-bit CPU. It also assumes little-endian which // can also be incorrect. - comptime assert(abi.CoverageUpdateHeader.trailing[0] == .pc_bits_usize); + comptime assert(abi.fuzz.CoverageUpdateHeader.trailing[0] == .pc_bits_usize); const n_bitset_elems = (coverage_source_locations.items.len + @bitSizeOf(u64) - 1) / @bitSizeOf(u64); const covered_bits = std.mem.bytesAsSlice( u64, - recent_coverage_update.items[@sizeOf(abi.CoverageUpdateHeader)..][0 .. n_bitset_elems * @sizeOf(u64)], + recent_coverage_update.items[@sizeOf(abi.fuzz.CoverageUpdateHeader)..][0 .. n_bitset_elems * @sizeOf(u64)], ); - var sli: u32 = 0; + var sli: SourceLocationIndex = @enumFromInt(0); for (covered_bits) |elem| { - global.result.ensureUnusedCapacity(gpa, 64) catch @panic("OOM"); + try covered.ensureUnusedCapacity(gpa, 64); for (0..@bitSizeOf(u64)) |i| { - if ((elem & (@as(u64, 1) << @intCast(i))) != 0) global.add(sli, want_file); - sli += 1; + if ((elem & (@as(u64, 1) << @intCast(i))) != 0) { + if (sli.ptr().file == want_file) { + covered.appendAssumeCapacity(sli); + } + } + sli = @enumFromInt(@intFromEnum(sli) + 1); } } - return Slice(SourceLocationIndex).init(global.result.items); + + js.updateCoverage(covered.items.ptr, covered.items.len); } + +fn updateSource() error{OutOfMemory}!void { + if (recent_coverage_update.items.len == 0) return; + const file_sli = selected_source_location.?; + var html: std.ArrayListUnmanaged(u8) = .empty; + defer html.deinit(gpa); + file_sli.fileHtml(&html) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.SourceUnavailable => {}, + }; + js.updateSource(html.items.ptr, html.items.len); +} + +var selected_source_location: ?SourceLocationIndex = null; + +/// This function is not used directly by `main.js`, but a reference to it is +/// emitted by `SourceLocationIndex.sourceLocationLinkHtml`. +export fn fuzzSelectSli(sli: SourceLocationIndex) void { + if (!sli.haveCoverage()) return; + selected_source_location = sli; + updateEntryPoints() catch @panic("out of memory"); // highlights the selected one green + updateSource() catch @panic("out of memory"); + updateCoverage() catch @panic("out of memory"); +} + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Coverage = std.debug.Coverage; +const abi = std.Build.abi; +const assert = std.debug.assert; +const gpa = std.heap.wasm_allocator; + +const Walk = @import("Walk"); +const html_render = @import("html_render"); + +const nsSince = @import("main.zig").nsSince; +const Slice = @import("main.zig").Slice; +const fatal = @import("main.zig").fatal; +const log = std.log; +const String = Slice(u8); diff --git a/lib/build-web/index.html b/lib/build-web/index.html new file mode 100644 index 0000000000000000000000000000000000000000..117ae14c209c045c23c5d623c92806e940e929f0 --- /dev/null +++ b/lib/build-web/index.html @@ -0,0 +1,202 @@ + + + +Zig Build System + + + + + + + + + + + + +

    Loading JavaScript...

    + +
    +

    Zig Build System

    + +

    | steps

    + + +
      + +
      + + + + + +

      Help

      +

      This is the Zig Build System web interface. It allows live interaction with the build system.

      +

      The following zig build flags can expose extra features of this interface:

      +
        +
      • --time-report: collect and show statistics about the time taken to evaluate a build graph
        • +
      • --fuzz: enable the fuzzer for any Zig test binaries in the build graph (experimental)
        • +
      +
      + + + + diff --git a/lib/build-web/main.js b/lib/build-web/main.js new file mode 100644 index 0000000000000000000000000000000000000000..481c33f024ba2b70ef1160a9ccae5647850caa4c --- /dev/null +++ b/lib/build-web/main.js @@ -0,0 +1,346 @@ +const domConnectionStatus = document.getElementById("connectionStatus"); +const domFirefoxWebSocketBullshitExplainer = document.getElementById("firefoxWebSocketBullshitExplainer"); + +const domMain = document.getElementsByTagName("main")[0]; +const domSummary = { + stepCount: document.getElementById("summaryStepCount"), + status: document.getElementById("summaryStatus"), +}; +const domButtonRebuild = document.getElementById("buttonRebuild"); +const domStepList = document.getElementById("stepList"); +let domSteps = []; + +let wasm_promise = fetch("main.wasm"); +let wasm_exports = null; + +const text_decoder = new TextDecoder(); +const text_encoder = new TextEncoder(); + +domButtonRebuild.addEventListener("click", () => wasm_exports.rebuild()); + +setConnectionStatus("Loading WebAssembly...", false); +WebAssembly.instantiateStreaming(wasm_promise, { + core: { + log: function(ptr, len) { + const msg = decodeString(ptr, len); + console.log(msg); + }, + panic: function (ptr, len) { + const msg = decodeString(ptr, len); + throw new Error("panic: " + msg); + }, + timestamp: function () { + return BigInt(new Date()); + }, + hello: hello, + updateBuildStatus: updateBuildStatus, + updateStepStatus: updateStepStatus, + sendWsMessage: (ptr, len) => ws.send(new Uint8Array(wasm_exports.memory.buffer, ptr, len)), + }, + fuzz: { + requestSources: fuzzRequestSources, + ready: fuzzReady, + updateStats: fuzzUpdateStats, + updateEntryPoints: fuzzUpdateEntryPoints, + updateSource: fuzzUpdateSource, + updateCoverage: fuzzUpdateCoverage, + }, + time_report: { + updateCompile: timeReportUpdateCompile, + updateGeneric: timeReportUpdateGeneric, + }, +}).then(function(obj) { + setConnectionStatus("Connecting to WebSocket...", true); + connectWebSocket(); + + wasm_exports = obj.instance.exports; + window.wasm = obj; // for debugging +}); + +function connectWebSocket() { + const host = document.location.host; + const pathname = document.location.pathname; + const isHttps = document.location.protocol === 'https:'; + const match = host.match(/^(.+):(\d+)$/); + const defaultPort = isHttps ? 443 : 80; + const port = match ? parseInt(match[2], 10) : defaultPort; + const hostName = match ? match[1] : host; + const wsProto = isHttps ? "wss:" : "ws:"; + const wsUrl = wsProto + '//' + hostName + ':' + port + pathname; + ws = new WebSocket(wsUrl); + ws.binaryType = "arraybuffer"; + ws.addEventListener('message', onWebSocketMessage, false); + ws.addEventListener('error', onWebSocketClose, false); + ws.addEventListener('close', onWebSocketClose, false); + ws.addEventListener('open', onWebSocketOpen, false); +} +function onWebSocketOpen() { + setConnectionStatus("Waiting for data...", false); +} +function onWebSocketMessage(ev) { + const jsArray = new Uint8Array(ev.data); + const ptr = wasm_exports.message_begin(jsArray.length); + const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, jsArray.length); + wasmArray.set(jsArray); + wasm_exports.message_end(); +} +function onWebSocketClose() { + setConnectionStatus("WebSocket connection closed. Re-connecting...", true); + ws.removeEventListener('message', onWebSocketMessage, false); + ws.removeEventListener('error', onWebSocketClose, false); + ws.removeEventListener('close', onWebSocketClose, false); + ws.removeEventListener('open', onWebSocketOpen, false); + ws = null; + setTimeout(connectWebSocket, 1000); +} + +function setConnectionStatus(msg, is_websocket_connect) { + domConnectionStatus.textContent = msg; + if (msg.length > 0) { + domConnectionStatus.classList.remove("hidden"); + domMain.classList.add("hidden"); + } else { + domConnectionStatus.classList.add("hidden"); + domMain.classList.remove("hidden"); + } + if (is_websocket_connect) { + domFirefoxWebSocketBullshitExplainer.classList.remove("hidden"); + } else { + domFirefoxWebSocketBullshitExplainer.classList.add("hidden"); + } +} + +function hello( + steps_len, + build_status, + time_report, +) { + domSummary.stepCount.textContent = steps_len; + updateBuildStatus(build_status); + setConnectionStatus("", false); + + { + let entries = []; + for (let i = 0; i < steps_len; i += 1) { + const step_name = unwrapString(wasm_exports.stepName(i)); + const code = document.createElement("code"); + code.textContent = step_name; + const li = document.createElement("li"); + li.appendChild(code); + entries.push(li); + } + domStepList.replaceChildren(...entries); + for (let i = 0; i < steps_len; i += 1) { + updateStepStatus(i); + } + } + + if (time_report) timeReportReset(steps_len); + fuzzReset(); +} + +function updateBuildStatus(s) { + let text; + let active = false; + let reset_time_reports = false; + if (s == 0) { + text = "Idle"; + } else if (s == 1) { + text = "Watching for changes..."; + } else if (s == 2) { + text = "Running..."; + active = true; + reset_time_reports = true; + } else if (s == 3) { + text = "Starting fuzzer..."; + active = true; + } else { + console.log(`bad build status: ${s}`); + } + domSummary.status.textContent = text; + if (active) { + domSummary.status.classList.add("status-running"); + domSummary.status.classList.remove("status-idle"); + domButtonRebuild.disabled = true; + } else { + domSummary.status.classList.remove("status-running"); + domSummary.status.classList.add("status-idle"); + domButtonRebuild.disabled = false; + } + if (reset_time_reports) { + // Grey out and collapse all the time reports + for (const time_report_host of domTimeReportList.children) { + const details = time_report_host.shadowRoot.querySelector(":host > details"); + details.classList.add("pending"); + details.open = false; + } + } +} +function updateStepStatus(step_idx) { + const li = domStepList.children[step_idx]; + const step_status = wasm_exports.stepStatus(step_idx); + li.classList.remove("step-wip", "step-success", "step-failure"); + if (step_status == 0) { + // pending + } else if (step_status == 1) { + li.classList.add("step-wip"); + } else if (step_status == 2) { + li.classList.add("step-success"); + } else if (step_status == 3) { + li.classList.add("step-failure"); + } else { + console.log(`bad step status: ${step_status}`); + } +} + +function decodeString(ptr, len) { + if (len === 0) return ""; + return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len)); +} +function getU32Array(ptr, len) { + if (len === 0) return new Uint32Array(); + return new Uint32Array(wasm_exports.memory.buffer, ptr, len); +} +function unwrapString(bigint) { + const ptr = Number(bigint & 0xffffffffn); + const len = Number(bigint >> 32n); + return decodeString(ptr, len); +} + +const time_report_entry_template = document.getElementById("timeReportEntryTemplate").content; +const domTimeReport = document.getElementById("timeReport"); +const domTimeReportList = document.getElementById("timeReportList"); +function timeReportReset(steps_len) { + let entries = []; + for (let i = 0; i < steps_len; i += 1) { + const step_name = unwrapString(wasm_exports.stepName(i)); + const host = document.createElement("div"); + const shadow = host.attachShadow({ mode: "open" }); + shadow.appendChild(time_report_entry_template.cloneNode(true)); + shadow.querySelector(":host > details").classList.add("pending"); + const slotted_name = document.createElement("code"); + slotted_name.setAttribute("slot", "step-name"); + slotted_name.textContent = step_name; + host.appendChild(slotted_name); + entries.push(host); + } + domTimeReportList.replaceChildren(...entries); + domTimeReport.classList.remove("hidden"); +} +function timeReportUpdateCompile( + step_idx, + inner_html_ptr, + inner_html_len, + file_table_html_ptr, + file_table_html_len, + decl_table_html_ptr, + decl_table_html_len, + use_llvm, +) { + const inner_html = decodeString(inner_html_ptr, inner_html_len); + const file_table_html = decodeString(file_table_html_ptr, file_table_html_len); + const decl_table_html = decodeString(decl_table_html_ptr, decl_table_html_len); + + const host = domTimeReportList.children.item(step_idx); + const shadow = host.shadowRoot; + + shadow.querySelector(":host > details").classList.remove("pending", "no-llvm"); + + shadow.getElementById("genericReport").classList.add("hidden"); + shadow.getElementById("compileReport").classList.remove("hidden"); + + if (!use_llvm) shadow.querySelector(":host > details").classList.add("no-llvm"); + host.innerHTML = inner_html; + shadow.getElementById("fileTableBody").innerHTML = file_table_html; + shadow.getElementById("declTableBody").innerHTML = decl_table_html; +} +function timeReportUpdateGeneric( + step_idx, + inner_html_ptr, + inner_html_len, +) { + const inner_html = decodeString(inner_html_ptr, inner_html_len); + const host = domTimeReportList.children.item(step_idx); + const shadow = host.shadowRoot; + shadow.querySelector(":host > details").classList.remove("pending", "no-llvm"); + shadow.getElementById("genericReport").classList.remove("hidden"); + shadow.getElementById("compileReport").classList.add("hidden"); + host.innerHTML = inner_html; +} + +const fuzz_entry_template = document.getElementById("fuzzEntryTemplate").content; +const domFuzz = document.getElementById("fuzz"); +const domFuzzStatus = document.getElementById("fuzzStatus"); +const domFuzzEntries = document.getElementById("fuzzEntries"); +let domFuzzInstance = null; +function fuzzRequestSources() { + domFuzzStatus.classList.remove("hidden"); + domFuzzStatus.textContent = "Loading sources tarball..."; + fetch("sources.tar").then(function(response) { + if (!response.ok) throw new Error("unable to download sources"); + domFuzzStatus.textContent = "Parsing fuzz test sources..."; + return response.arrayBuffer(); + }).then(function(buffer) { + if (buffer.length === 0) throw new Error("sources.tar was empty"); + const js_array = new Uint8Array(buffer); + const ptr = wasm_exports.alloc(js_array.length); + const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length); + wasm_array.set(js_array); + wasm_exports.fuzzUnpackSources(ptr, js_array.length); + domFuzzStatus.textContent = ""; + domFuzzStatus.classList.add("hidden"); + }); +} +function fuzzReady() { + domFuzz.classList.remove("hidden"); + + // TODO: multiple fuzzer instances + if (domFuzzInstance !== null) return; + + const host = document.createElement("div"); + const shadow = host.attachShadow({ mode: "open" }); + shadow.appendChild(fuzz_entry_template.cloneNode(true)); + + domFuzzInstance = host; + domFuzzEntries.appendChild(host); +} +function fuzzReset() { + domFuzz.classList.add("hidden"); + domFuzzEntries.replaceChildren(); + domFuzzInstance = null; +} +function fuzzUpdateStats(stats_html_ptr, stats_html_len) { + if (domFuzzInstance === null) throw new Error("fuzzUpdateStats called when fuzzer inactive"); + const stats_html = decodeString(stats_html_ptr, stats_html_len); + const host = domFuzzInstance; + host.innerHTML = stats_html; +} +function fuzzUpdateEntryPoints(entry_points_html_ptr, entry_points_html_len) { + if (domFuzzInstance === null) throw new Error("fuzzUpdateEntryPoints called when fuzzer inactive"); + const entry_points_html = decodeString(entry_points_html_ptr, entry_points_html_len); + const domEntryPointList = domFuzzInstance.shadowRoot.getElementById("entryPointList"); + domEntryPointList.innerHTML = entry_points_html; +} +function fuzzUpdateSource(source_html_ptr, source_html_len) { + if (domFuzzInstance === null) throw new Error("fuzzUpdateSource called when fuzzer inactive"); + const source_html = decodeString(source_html_ptr, source_html_len); + const domSourceText = domFuzzInstance.shadowRoot.getElementById("sourceText"); + domSourceText.innerHTML = source_html; + domFuzzInstance.shadowRoot.getElementById("source").classList.remove("hidden"); +} +function fuzzUpdateCoverage(covered_ptr, covered_len) { + if (domFuzzInstance === null) throw new Error("fuzzUpdateCoverage called when fuzzer inactive"); + const shadow = domFuzzInstance.shadowRoot; + const domSourceText = shadow.getElementById("sourceText"); + const covered = getU32Array(covered_ptr, covered_len); + for (let i = 0; i < domSourceText.children.length; i += 1) { + const childDom = domSourceText.children[i]; + if (childDom.id != null && childDom.id[0] == "l") { + childDom.classList.add("l"); + childDom.classList.remove("c"); + } + } + for (const sli of covered) { + shadow.getElementById(`l${sli}`).classList.add("c"); + } +} diff --git a/lib/build-web/main.zig b/lib/build-web/main.zig new file mode 100644 index 0000000000000000000000000000000000000000..e971f6de48e788de622410adf69aa6352061c307 --- /dev/null +++ b/lib/build-web/main.zig @@ -0,0 +1,213 @@ +const std = @import("std"); +const assert = std.debug.assert; +const abi = std.Build.abi; +const gpa = std.heap.wasm_allocator; +const log = std.log; +const Allocator = std.mem.Allocator; + +const fuzz = @import("fuzz.zig"); +const time_report = @import("time_report.zig"); + +/// Nanoseconds. +var server_base_timestamp: i64 = 0; +/// Milliseconds. +var client_base_timestamp: i64 = 0; + +pub var step_list: []Step = &.{}; +/// Not accessed after initialization, but must be freed alongside `step_list`. +pub var step_list_data: []u8 = &.{}; + +const Step = struct { + name: []const u8, + status: abi.StepUpdate.Status, +}; + +const js = struct { + extern "core" fn log(ptr: [*]const u8, len: usize) void; + extern "core" fn panic(ptr: [*]const u8, len: usize) noreturn; + extern "core" fn timestamp() i64; + extern "core" fn hello( + steps_len: u32, + status: abi.BuildStatus, + time_report: bool, + ) void; + extern "core" fn updateBuildStatus(status: abi.BuildStatus) void; + extern "core" fn updateStepStatus(step_idx: u32) void; + extern "core" fn sendWsMessage(ptr: [*]const u8, len: usize) void; +}; + +pub const std_options: std.Options = .{ + .logFn = logFn, +}; + +pub fn panic(msg: []const u8, st: ?*std.builtin.StackTrace, addr: ?usize) noreturn { + _ = st; + _ = addr; + log.err("panic: {s}", .{msg}); + @trap(); +} + +fn logFn( + comptime message_level: log.Level, + comptime scope: @TypeOf(.enum_literal), + comptime format: []const u8, + args: anytype, +) void { + const level_txt = comptime message_level.asText(); + const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + var buf: [500]u8 = undefined; + const line = std.fmt.bufPrint(&buf, level_txt ++ prefix2 ++ format, args) catch l: { + buf[buf.len - 3 ..][0..3].* = "...".*; + break :l &buf; + }; + js.log(line.ptr, line.len); +} + +export fn alloc(n: usize) [*]u8 { + const slice = gpa.alloc(u8, n) catch @panic("OOM"); + return slice.ptr; +} + +var message_buffer: std.ArrayListAlignedUnmanaged(u8, .of(u64)) = .empty; + +/// Resizes the message buffer to be the correct length; returns the pointer to +/// the query string. +export fn message_begin(len: usize) [*]u8 { + message_buffer.resize(gpa, len) catch @panic("OOM"); + return message_buffer.items.ptr; +} + +export fn message_end() void { + const msg_bytes = message_buffer.items; + + const tag: abi.ToClientTag = @enumFromInt(msg_bytes[0]); + switch (tag) { + _ => @panic("malformed message"), + + .hello => return helloMessage(msg_bytes) catch @panic("OOM"), + .status_update => return statusUpdateMessage(msg_bytes) catch @panic("OOM"), + .step_update => return stepUpdateMessage(msg_bytes) catch @panic("OOM"), + + .fuzz_source_index => return fuzz.sourceIndexMessage(msg_bytes) catch @panic("OOM"), + .fuzz_coverage_update => return fuzz.coverageUpdateMessage(msg_bytes) catch @panic("OOM"), + .fuzz_entry_points => return fuzz.entryPointsMessage(msg_bytes) catch @panic("OOM"), + + .time_report_generic_result => return time_report.genericResultMessage(msg_bytes) catch @panic("OOM"), + .time_report_compile_result => return time_report.compileResultMessage(msg_bytes) catch @panic("OOM"), + } +} + +const String = Slice(u8); + +pub fn Slice(T: type) type { + return packed struct(u64) { + ptr: u32, + len: u32, + + pub fn init(s: []const T) @This() { + return .{ + .ptr = @intFromPtr(s.ptr), + .len = s.len, + }; + } + }; +} + +pub fn fatal(comptime format: []const u8, args: anytype) noreturn { + var buf: [500]u8 = undefined; + const line = std.fmt.bufPrint(&buf, format, args) catch l: { + buf[buf.len - 3 ..][0..3].* = "...".*; + break :l &buf; + }; + js.panic(line.ptr, line.len); +} + +fn helloMessage(msg_bytes: []align(4) u8) Allocator.Error!void { + if (msg_bytes.len < @sizeOf(abi.Hello)) @panic("malformed Hello message"); + const hdr: *const abi.Hello = @ptrCast(msg_bytes[0..@sizeOf(abi.Hello)]); + const trailing = msg_bytes[@sizeOf(abi.Hello)..]; + + client_base_timestamp = js.timestamp(); + server_base_timestamp = hdr.timestamp; + + const steps = try gpa.alloc(Step, hdr.steps_len); + errdefer gpa.free(steps); + + const step_name_lens: []align(1) const u32 = @ptrCast(trailing[0 .. steps.len * 4]); + + const step_name_data_len: usize = len: { + var sum: usize = 0; + for (step_name_lens) |n| sum += n; + break :len sum; + }; + const step_name_data: []const u8 = trailing[steps.len * 4 ..][0..step_name_data_len]; + const step_status_bits: []const u8 = trailing[steps.len * 4 + step_name_data_len ..]; + + const duped_step_name_data = try gpa.dupe(u8, step_name_data); + errdefer gpa.free(duped_step_name_data); + + var name_off: usize = 0; + for (steps, step_name_lens, 0..) |*step_out, name_len, step_idx| { + step_out.* = .{ + .name = duped_step_name_data[name_off..][0..name_len], + .status = @enumFromInt(@as(u2, @truncate(step_status_bits[step_idx / 4] >> @intCast((step_idx % 4) * 2)))), + }; + name_off += name_len; + } + + gpa.free(step_list); + gpa.free(step_list_data); + step_list = steps; + step_list_data = duped_step_name_data; + + js.hello(step_list.len, hdr.status, hdr.flags.time_report); +} +fn statusUpdateMessage(msg_bytes: []u8) Allocator.Error!void { + if (msg_bytes.len < @sizeOf(abi.StatusUpdate)) @panic("malformed StatusUpdate message"); + const msg: *const abi.StatusUpdate = @ptrCast(msg_bytes[0..@sizeOf(abi.StatusUpdate)]); + js.updateBuildStatus(msg.new); +} +fn stepUpdateMessage(msg_bytes: []u8) Allocator.Error!void { + if (msg_bytes.len < @sizeOf(abi.StepUpdate)) @panic("malformed StepUpdate message"); + const msg: *const abi.StepUpdate = @ptrCast(msg_bytes[0..@sizeOf(abi.StepUpdate)]); + if (msg.step_idx >= step_list.len) @panic("malformed StepUpdate message"); + step_list[msg.step_idx].status = msg.bits.status; + js.updateStepStatus(msg.step_idx); +} + +export fn stepName(idx: usize) String { + return .init(step_list[idx].name); +} +export fn stepStatus(idx: usize) u8 { + return @intFromEnum(step_list[idx].status); +} + +export fn rebuild() void { + const msg: abi.Rebuild = .{}; + const raw: []const u8 = @ptrCast(&msg); + js.sendWsMessage(raw.ptr, raw.len); +} + +/// Nanoseconds passed since a server timestamp. +pub fn nsSince(server_timestamp: i64) i64 { + const ms_passed = js.timestamp() - client_base_timestamp; + const ns_passed = server_base_timestamp - server_timestamp; + return ns_passed + ms_passed * std.time.ns_per_ms; +} + +pub fn fmtEscapeHtml(unescaped: []const u8) HtmlEscaper { + return .{ .unescaped = unescaped }; +} +const HtmlEscaper = struct { + unescaped: []const u8, + pub fn format(he: HtmlEscaper, w: *std.Io.Writer) !void { + for (he.unescaped) |c| switch (c) { + '&' => try w.writeAll("&"), + '<' => try w.writeAll("<"), + '>' => try w.writeAll(">"), + '"' => try w.writeAll("""), + '\'' => try w.writeAll("'"), + else => try w.writeByte(c), + }; + } +}; diff --git a/lib/build-web/style.css b/lib/build-web/style.css new file mode 100644 index 0000000000000000000000000000000000000000..eb89ced5270713d581131110dd9adb07cff8ed06 --- /dev/null +++ b/lib/build-web/style.css @@ -0,0 +1,240 @@ +body { + font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif; + color: #000000; + padding: 1em 10%; +} +ul.no-marker { + list-style-type: none; + padding-left: 0; +} +hr { + margin: 2em 0; +} +.hidden { + display: none; +} +.empty-cell { + background: #ccc; +} +table.time-stats > tbody > tr > th { + text-align: left; +} +table.time-stats > tbody > tr > td { + text-align: right; +} +details > summary { + cursor: pointer; + font-size: 1.5em; +} +.tooltip { + text-decoration: underline; + cursor: help; +} +.tooltip-content { + border-radius: 6px; + display: none; + position: absolute; + background: #fff; + border: 1px solid black; + max-width: 500px; + padding: 1em; + text-align: left; + font-weight: normal; + pointer-events: none; +} +.tooltip:hover > .tooltip-content { + display: block; +} +table { + margin: 1.0em auto 1.5em 0; + border-collapse: collapse; +} +th, td { + padding: 0.5em 1em 0.5em 1em; + border: 1px solid; + border-color: black; +} +a, button { + color: #2A6286; +} +button { + background: #eee; + cursor: pointer; + border: none; + border-radius: 3px; + padding: 0.2em 0.5em; +} +button.big-btn { + font-size: 1.3em; +} +button.linkish { + background: none; + text-decoration: underline; + padding: 0; +} +button:disabled { + color: #888; + cursor: not-allowed; +} +pre { + font-family: "Source Code Pro", monospace; + font-size: 1em; + background-color: #F5F5F5; + padding: 1em; + margin: 0; + overflow-x: auto; +} +:not(pre) > code { + white-space: break-spaces; +} +code { + font-family: "Source Code Pro", monospace; + font-size: 0.9em; +} +code a { + color: #000000; +} +kbd { + color: #000; + background-color: #fafbfc; + border-color: #d1d5da; + border-bottom-color: #c6cbd1; + box-shadow-color: #c6cbd1; + display: inline-block; + padding: 0.3em 0.2em; + font: 1.2em monospace; + line-height: 0.8em; + vertical-align: middle; + border: solid 1px; + border-radius: 3px; + box-shadow: inset 0 -1px 0; + cursor: default; +} +.status-running { color: #181; } +.status-idle { color: #444; } +.step-success { color: #181; } +.step-failure { color: #d11; } +.step-wip::before { + content: ''; + position: absolute; + margin-left: -1.5em; + width: 1em; + text-align: center; + animation-name: spinner; + animation-duration: 0.5s; + animation-iteration-count: infinite; + animation-timing-function: step-start; +} +@keyframes spinner { + 0% { content: '|'; } + 25% { content: '/'; } + 50% { content: '-'; } + 75% { content: '\\'; } + 100% { content: '|'; } +} + +.l { + display: inline-block; + background: red; + width: 1em; + height: 1em; + border-radius: 1em; +} +.c { + background-color: green; +} + +.tok-kw { + color: #333; + font-weight: bold; +} +.tok-str { + color: #d14; +} +.tok-builtin { + color: #0086b3; +} +.tok-comment { + color: #777; + font-style: italic; +} +.tok-fn { + color: #900; + font-weight: bold; +} +.tok-null { + color: #008080; +} +.tok-number { + color: #008080; +} +.tok-type { + color: #458; + font-weight: bold; +} + +@media (prefers-color-scheme: dark) { + body { + background-color: #111; + color: #ddd; + } + pre { + background-color: #222; + } + a, button { + color: #88f; + } + button { + background: #333; + } + button:disabled { + color: #555; + } + code a { + color: #eee; + } + th, td { + border-color: white; + } + .empty-cell { + background: #000; + } + .tooltip-content { + background: #060606; + border-color: white; + } + .status-running { color: #90ee90; } + .status-idle { color: #bbb; } + .step-success { color: #90ee90; } + .step-failure { color: #f66; } + .l { + background-color: red; + } + .c { + background-color: green; + } + .tok-kw { + color: #eee; + } + .tok-str { + color: #2e5; + } + .tok-builtin { + color: #ff894c; + } + .tok-comment { + color: #aa7; + } + .tok-fn { + color: #B1A0F8; + } + .tok-null { + color: #ff8080; + } + .tok-number { + color: #ff8080; + } + .tok-type { + color: #68f; + } +} diff --git a/lib/build-web/time_report.css b/lib/build-web/time_report.css new file mode 100644 index 0000000000000000000000000000000000000000..27a7c4b0223824d2fef7812270e892da521b1ee6 --- /dev/null +++ b/lib/build-web/time_report.css @@ -0,0 +1,43 @@ +:host > details { + padding: 0.5em 1em; + background: #f2f2f2; + margin-bottom: 1.0em; + overflow-x: scroll; +} +:host > details.pending { + pointer-events: none; + background: #fafafa; + color: #666; +} +:host > details > div { + margin: 1em 2em; + overflow: scroll; /* we'll try to avoid overflow, but if it does happen, this makes sense */ +} +.stats { + font-size: 1.2em; +} +details.section { + margin: 1.0em 0 0 0; +} +details.section > summary { + font-weight: bold; +} +details.section > :not(summary) { + margin-left: 2em; +} +:host > details.no-llvm .llvm-only { + display: none; +} +@media (prefers-color-scheme: dark) { + :host > details { + background: #222; + } + :host > details.pending { + background: #181818; + color: #888; + } +} +th { + max-width: 20em; /* don't let the 'file' column get crazy long */ + overflow-wrap: anywhere; /* avoid overflow where possible */ +} diff --git a/lib/build-web/time_report.zig b/lib/build-web/time_report.zig new file mode 100644 index 0000000000000000000000000000000000000000..de5f9abd0c4a514ff0bd4cc6b1cee169680420ff --- /dev/null +++ b/lib/build-web/time_report.zig @@ -0,0 +1,234 @@ +const std = @import("std"); +const gpa = std.heap.wasm_allocator; +const abi = std.Build.abi.time_report; +const fmtEscapeHtml = @import("root").fmtEscapeHtml; +const step_list = &@import("root").step_list; + +const js = struct { + extern "time_report" fn updateGeneric( + /// The index of the step. + step_idx: u32, + // The HTML which will be used to populate the template slots. + inner_html_ptr: [*]const u8, + inner_html_len: usize, + ) void; + extern "time_report" fn updateCompile( + /// The index of the step. + step_idx: u32, + // The HTML which will be used to populate the template slots. + inner_html_ptr: [*]const u8, + inner_html_len: usize, + // The HTML which will populate the of the file table. + file_table_html_ptr: [*]const u8, + file_table_html_len: usize, + // The HTML which will populate the of the decl table. + decl_table_html_ptr: [*]const u8, + decl_table_html_len: usize, + /// Whether the LLVM backend was used. If not, LLVM-specific statistics are hidden. + use_llvm: bool, + ) void; +}; + +pub fn genericResultMessage(msg_bytes: []u8) error{OutOfMemory}!void { + if (msg_bytes.len != @sizeOf(abi.GenericResult)) @panic("malformed GenericResult message"); + const msg: *const abi.GenericResult = @ptrCast(msg_bytes); + if (msg.step_idx >= step_list.*.len) @panic("malformed GenericResult message"); + const inner_html = try std.fmt.allocPrint(gpa, + \\{[step_name]f} + \\{[stat_total_time]D} + , .{ + .step_name = fmtEscapeHtml(step_list.*[msg.step_idx].name), + .stat_total_time = msg.ns_total, + }); + defer gpa.free(inner_html); + js.updateGeneric(msg.step_idx, inner_html.ptr, inner_html.len); +} + +pub fn compileResultMessage(msg_bytes: []u8) error{OutOfMemory}!void { + const max_table_rows = 500; + + if (msg_bytes.len < @sizeOf(abi.CompileResult)) @panic("malformed CompileResult message"); + const hdr: *const abi.CompileResult = @ptrCast(msg_bytes[0..@sizeOf(abi.CompileResult)]); + if (hdr.step_idx >= step_list.*.len) @panic("malformed CompileResult message"); + var trailing = msg_bytes[@sizeOf(abi.CompileResult)..]; + + const llvm_pass_timings = trailing[0..hdr.llvm_pass_timings_len]; + trailing = trailing[hdr.llvm_pass_timings_len..]; + + const FileTimeReport = struct { + name: []const u8, + ns_sema: u64, + ns_codegen: u64, + ns_link: u64, + }; + const DeclTimeReport = struct { + file_name: []const u8, + name: []const u8, + sema_count: u32, + ns_sema: u64, + ns_codegen: u64, + ns_link: u64, + }; + + const slowest_files = try gpa.alloc(FileTimeReport, hdr.files_len); + defer gpa.free(slowest_files); + + const slowest_decls = try gpa.alloc(DeclTimeReport, hdr.decls_len); + defer gpa.free(slowest_decls); + + for (slowest_files) |*file_out| { + const i = std.mem.indexOfScalar(u8, trailing, 0) orelse @panic("malformed CompileResult message"); + file_out.* = .{ + .name = trailing[0..i], + .ns_sema = 0, + .ns_codegen = 0, + .ns_link = 0, + }; + trailing = trailing[i + 1 ..]; + } + + for (slowest_decls) |*decl_out| { + const i = std.mem.indexOfScalar(u8, trailing, 0) orelse @panic("malformed CompileResult message"); + const file_idx = std.mem.readInt(u32, trailing[i..][1..5], .little); + const sema_count = std.mem.readInt(u32, trailing[i..][5..9], .little); + const sema_ns = std.mem.readInt(u64, trailing[i..][9..17], .little); + const codegen_ns = std.mem.readInt(u64, trailing[i..][17..25], .little); + const link_ns = std.mem.readInt(u64, trailing[i..][25..33], .little); + const file = &slowest_files[file_idx]; + decl_out.* = .{ + .file_name = file.name, + .name = trailing[0..i], + .sema_count = sema_count, + .ns_sema = sema_ns, + .ns_codegen = codegen_ns, + .ns_link = link_ns, + }; + trailing = trailing[i + 33 ..]; + file.ns_sema += sema_ns; + file.ns_codegen += codegen_ns; + file.ns_link += link_ns; + } + + const S = struct { + fn fileLessThan(_: void, lhs: FileTimeReport, rhs: FileTimeReport) bool { + const lhs_ns = lhs.ns_sema + lhs.ns_codegen + lhs.ns_link; + const rhs_ns = rhs.ns_sema + rhs.ns_codegen + rhs.ns_link; + return lhs_ns > rhs_ns; // flipped to sort in reverse order + } + fn declLessThan(_: void, lhs: DeclTimeReport, rhs: DeclTimeReport) bool { + //if (true) return lhs.sema_count > rhs.sema_count; + const lhs_ns = lhs.ns_sema + lhs.ns_codegen + lhs.ns_link; + const rhs_ns = rhs.ns_sema + rhs.ns_codegen + rhs.ns_link; + return lhs_ns > rhs_ns; // flipped to sort in reverse order + } + }; + std.mem.sort(FileTimeReport, slowest_files, {}, S.fileLessThan); + std.mem.sort(DeclTimeReport, slowest_decls, {}, S.declLessThan); + + const stats = hdr.stats; + const inner_html = try std.fmt.allocPrint(gpa, + \\{[step_name]f} + \\{[stat_reachable_files]d} + \\{[stat_imported_files]d} + \\{[stat_generic_instances]d} + \\{[stat_inline_calls]d} + \\{[stat_compilation_time]D} + \\{[cpu_time_parse]D} + \\{[cpu_time_astgen]D} + \\{[cpu_time_sema]D} + \\{[cpu_time_codegen]D} + \\{[cpu_time_link]D} + \\{[real_time_files]D} + \\{[real_time_decls]D} + \\{[real_time_llvm_emit]D} + \\{[real_time_link_flush]D} + \\
      {[llvm_pass_timings]f}
      + \\ + , .{ + .step_name = fmtEscapeHtml(step_list.*[hdr.step_idx].name), + .stat_reachable_files = stats.n_reachable_files, + .stat_imported_files = stats.n_imported_files, + .stat_generic_instances = stats.n_generic_instances, + .stat_inline_calls = stats.n_inline_calls, + .stat_compilation_time = hdr.ns_total, + + .cpu_time_parse = stats.cpu_ns_parse, + .cpu_time_astgen = stats.cpu_ns_astgen, + .cpu_time_sema = stats.cpu_ns_sema, + .cpu_time_codegen = stats.cpu_ns_codegen, + .cpu_time_link = stats.cpu_ns_link, + .real_time_files = stats.real_ns_files, + .real_time_decls = stats.real_ns_decls, + .real_time_llvm_emit = stats.real_ns_llvm_emit, + .real_time_link_flush = stats.real_ns_link_flush, + + .llvm_pass_timings = fmtEscapeHtml(llvm_pass_timings), + }); + defer gpa.free(inner_html); + + var file_table_html: std.ArrayListUnmanaged(u8) = .empty; + defer file_table_html.deinit(gpa); + for (slowest_files[0..@min(max_table_rows, slowest_files.len)]) |file| { + try file_table_html.writer(gpa).print( + \\ + \\ {f} + \\ {D} + \\ {D} + \\ {D} + \\ + \\ + , .{ + fmtEscapeHtml(file.name), + file.ns_sema, + file.ns_codegen, + file.ns_link, + }); + } + if (slowest_files.len > max_table_rows) { + try file_table_html.writer(gpa).print( + \\{d} more rows omitted + \\ + , .{slowest_files.len - max_table_rows}); + } + + var decl_table_html: std.ArrayListUnmanaged(u8) = .empty; + defer decl_table_html.deinit(gpa); + + for (slowest_decls[0..@min(max_table_rows, slowest_decls.len)]) |decl| { + try decl_table_html.writer(gpa).print( + \\ + \\ {f} + \\ {f} + \\ {d} + \\ {D} + \\ {D} + \\ {D} + \\ + \\ + , .{ + fmtEscapeHtml(decl.file_name), + fmtEscapeHtml(decl.name), + decl.sema_count, + decl.ns_sema, + decl.ns_codegen, + decl.ns_link, + }); + } + if (slowest_decls.len > max_table_rows) { + try decl_table_html.writer(gpa).print( + \\{d} more rows omitted + \\ + , .{slowest_decls.len - max_table_rows}); + } + + js.updateCompile( + hdr.step_idx, + inner_html.ptr, + inner_html.len, + file_table_html.items.ptr, + file_table_html.items.len, + decl_table_html.items.ptr, + decl_table_html.items.len, + hdr.flags.use_llvm, + ); +} diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 6b7266ee710b974833db34f1a33410b8be384ddd..2e84309ccd494fc756f239cdaaf933bdd967d923 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -9,7 +9,7 @@ const ArrayList = std.ArrayList; const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; -const Fuzz = std.Build.Fuzz; +const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.io.Writer; @@ -25,15 +25,16 @@ pub const std_options: std.Options = .{ }; pub fn main() !void { - // Here we use an ArenaAllocator backed by a page allocator because a build is a short-lived, - // one shot program. We don't need to waste time freeing memory and finding places to squish - // bytes into. So we free everything all at once at the very end. - var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not + // always the case. So, we do need a true gpa for some things. + var debug_gpa_state: std.heap.DebugAllocator(.{}) = .init; + defer _ = debug_gpa_state.deinit(); + const gpa = debug_gpa_state.allocator(); + + // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. + var single_threaded_arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator); defer single_threaded_arena.deinit(); - - var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ - .child_allocator = single_threaded_arena.allocator(), - }; + var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ .child_allocator = single_threaded_arena.allocator() }; const arena = thread_safe_arena.allocator(); const args = try process.argsAlloc(arena); @@ -81,6 +82,7 @@ pub fn main() !void { .query = .{}, .result = try std.zig.system.resolveTargetQuery(.{}), }, + .time_report = false, }; graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); @@ -113,7 +115,7 @@ pub fn main() !void { var watch = false; var fuzz = false; var debounce_interval_ms: u16 = 50; - var listen_port: u16 = 0; + var webui_listen: ?std.net.Address = null; while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-Z")) { @@ -220,13 +222,13 @@ pub fn main() !void { next_arg, @errorName(err), }); }; - } else if (mem.eql(u8, arg, "--port")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); - listen_port = std.fmt.parseUnsigned(u16, next_arg, 10) catch |err| { - fatal("unable to parse port '{s}' as unsigned 16-bit integer: {s}\n", .{ - next_arg, @errorName(err), - }); + } else if (mem.eql(u8, arg, "--webui")) { + webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; + } else if (mem.startsWith(u8, arg, "--webui=")) { + const addr_str = arg["--webui=".len..]; + if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); + webui_listen = std.net.Address.parseIpAndPort(addr_str) catch |err| { + fatal("invalid web UI address '{s}': {s}", .{ addr_str, @errorName(err) }); }; } else if (mem.eql(u8, arg, "--debug-log")) { const next_arg = nextArgOrFatal(args, &arg_idx); @@ -267,8 +269,16 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "--time-report")) { + graph.time_report = true; + if (webui_listen == null) { + webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; + } } else if (mem.eql(u8, arg, "--fuzz")) { fuzz = true; + if (webui_listen == null) { + webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; + } } else if (mem.eql(u8, arg, "-fincremental")) { graph.incremental = true; } else if (mem.eql(u8, arg, "-fno-incremental")) { @@ -331,6 +341,10 @@ pub fn main() !void { } } + if (webui_listen != null and watch) fatal( + \\the build system does not yet support combining '--webui' and '--watch'; consider omitting '--watch' in favour of the web UI "Rebuild" button + , .{}); + const stderr: std.fs.File = .stderr(); const ttyconf = get_tty_conf(color, stderr); switch (ttyconf) { @@ -394,14 +408,16 @@ pub fn main() !void { } var run: Run = .{ + .gpa = gpa, + .max_rss = max_rss, .max_rss_is_default = false, .max_rss_mutex = .{}, .skip_oom_steps = skip_oom_steps, .watch = watch, - .fuzz = fuzz, - .memory_blocked_steps = std.ArrayList(*Step).init(arena), - .step_stack = .{}, + .web_server = undefined, // set after `prepare` + .memory_blocked_steps = .empty, + .step_stack = .empty, .prominent_compile_errors = prominent_compile_errors, .claimed_rss = 0, @@ -410,74 +426,81 @@ pub fn main() !void { .stderr = stderr, .thread_pool = undefined, }; + defer { + run.memory_blocked_steps.deinit(gpa); + run.step_stack.deinit(gpa); + } if (run.max_rss == 0) { run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64); run.max_rss_is_default = true; } - const gpa = arena; - prepare(gpa, arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { + prepare(arena, builder, targets.items, &run, graph.random_seed) catch |err| switch (err) { error.UncleanExit => process.exit(1), else => return err, }; - var w: Watch = if (watch and Watch.have_impl) try Watch.init() else undefined; + var w: Watch = w: { + if (!watch) break :w undefined; + if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); + break :w try .init(); + }; try run.thread_pool.init(thread_pool_options); defer run.thread_pool.deinit(); + run.web_server = if (webui_listen) |listen_address| .init(.{ + .gpa = gpa, + .thread_pool = &run.thread_pool, + .graph = &graph, + .all_steps = run.step_stack.keys(), + .ttyconf = run.ttyconf, + .root_prog_node = main_progress_node, + .watch = watch, + .listen_address = listen_address, + }) else null; + + if (run.web_server) |*ws| { + ws.start() catch |err| fatal("failed to start web server: {s}", .{@errorName(err)}); + } + rebuild: while (true) { + if (run.web_server) |*ws| ws.startBuild(); + runStepNames( - gpa, builder, targets.items, main_progress_node, &run, ) catch |err| switch (err) { error.UncleanExit => { - assert(!run.watch); + assert(!run.watch and run.web_server == null); process.exit(1); }, else => return err, }; - if (fuzz) { - if (builtin.single_threaded) { - fatal("--fuzz not yet implemented for single-threaded builds", .{}); - } - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - const listen_address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable; - try Fuzz.start( - gpa, - arena, - global_cache_directory, - zig_lib_directory, - zig_exe, - &run.thread_pool, - run.step_stack.keys(), - run.ttyconf, - listen_address, - main_progress_node, - ); + + if (run.web_server) |*web_server| { + web_server.finishBuild(.{ .fuzz = fuzz }); } - if (!watch) return cleanExit(); + if (!watch and run.web_server == null) { + return cleanExit(); + } - if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); + if (run.web_server) |*ws| { + assert(!watch); // fatal error after CLI parsing + while (true) switch (ws.wait()) { + .rebuild => { + for (run.step_stack.keys()) |step| { + step.state = .precheck_done; + step.reset(gpa); + } + continue :rebuild; + }, + }; + } try w.update(gpa, run.step_stack.keys()); @@ -491,15 +514,16 @@ pub fn main() !void { w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); - var debounce_timeout: Watch.Timeout = .none; - while (true) switch (try w.wait(gpa, debounce_timeout)) { + var in_debounce = false; + while (true) switch (try w.wait(gpa, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { .timeout => { + assert(in_debounce); debouncing_node.end(); markFailedStepsDirty(gpa, run.step_stack.keys()); continue :rebuild; }, - .dirty => if (debounce_timeout == .none) { - debounce_timeout = .{ .ms = debounce_interval_ms }; + .dirty => if (!in_debounce) { + in_debounce = true; debouncing_node.end(); debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); }, @@ -530,13 +554,16 @@ fn countSubProcesses(all_steps: []const *Step) usize { } const Run = struct { + gpa: Allocator, max_rss: u64, max_rss_is_default: bool, max_rss_mutex: std.Thread.Mutex, skip_oom_steps: bool, watch: bool, - fuzz: bool, - memory_blocked_steps: std.ArrayList(*Step), + web_server: ?WebServer, + /// Allocated into `gpa`. + memory_blocked_steps: std.ArrayListUnmanaged(*Step), + /// Allocated into `gpa`. step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), prominent_compile_errors: bool, thread_pool: std.Thread.Pool, @@ -547,19 +574,19 @@ const Run = struct { stderr: File, fn cleanExit(run: Run) void { - if (run.watch or run.fuzz) return; + if (run.watch or run.web_server != null) return; return runner.cleanExit(); } }; fn prepare( - gpa: Allocator, arena: Allocator, b: *std.Build, step_names: []const []const u8, run: *Run, seed: u32, ) !void { + const gpa = run.gpa; const step_stack = &run.step_stack; if (step_names.len == 0) { @@ -583,7 +610,7 @@ fn prepare( rand.shuffle(*Step, starting_steps); for (starting_steps) |s| { - constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) { + constructGraphAndCheckForDependencyLoop(gpa, b, s, &run.step_stack, rand) catch |err| switch (err) { error.DependencyLoopDetected => return uncleanExit(), else => |e| return e, }; @@ -614,12 +641,12 @@ fn prepare( } fn runStepNames( - gpa: Allocator, b: *std.Build, step_names: []const []const u8, parent_prog_node: std.Progress.Node, run: *Run, ) !void { + const gpa = run.gpa; const step_stack = &run.step_stack; const thread_pool = &run.thread_pool; @@ -675,6 +702,7 @@ fn runStepNames( // B will be marked as dependency_failure, while A may never be queued, and thus // remain in the initial state of precheck_done. s.state = .dependency_failure; + if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); pending_count += 1; }, .dependency_failure => pending_count += 1, @@ -768,7 +796,7 @@ fn runStepNames( } } - if (!run.watch) { + if (!run.watch and run.web_server == null) { // Signal to parent process that we have printed compile errors. The // parent process may choose to omit the "following command failed" // line in this case. @@ -777,7 +805,7 @@ fn runStepNames( } } - if (!run.watch) return uncleanExit(); + if (!run.watch and run.web_server == null) return uncleanExit(); } const PrintNode = struct { @@ -1022,6 +1050,7 @@ fn printTreeStep( /// when it finishes executing in `workerMakeOneStep`, it spawns next steps /// to run in random order fn constructGraphAndCheckForDependencyLoop( + gpa: Allocator, b: *std.Build, s: *Step, step_stack: *std.AutoArrayHashMapUnmanaged(*Step, void), @@ -1035,17 +1064,19 @@ fn constructGraphAndCheckForDependencyLoop( .precheck_unstarted => { s.state = .precheck_started; - try step_stack.ensureUnusedCapacity(b.allocator, s.dependencies.items.len); + try step_stack.ensureUnusedCapacity(gpa, s.dependencies.items.len); // We dupe to avoid shuffling the steps in the summary, it depends // on s.dependencies' order. - const deps = b.allocator.dupe(*Step, s.dependencies.items) catch @panic("OOM"); + const deps = gpa.dupe(*Step, s.dependencies.items) catch @panic("OOM"); + defer gpa.free(deps); + rand.shuffle(*Step, deps); for (deps) |dep| { - try step_stack.put(b.allocator, dep, {}); + try step_stack.put(gpa, dep, {}); try dep.dependants.append(b.allocator, s); - constructGraphAndCheckForDependencyLoop(b, dep, step_stack, rand) catch |err| { + constructGraphAndCheckForDependencyLoop(gpa, b, dep, step_stack, rand) catch |err| { if (err == error.DependencyLoopDetected) { std.debug.print(" {s}\n", .{s.name}); } @@ -1084,6 +1115,7 @@ fn workerMakeOneStep( .success, .skipped => continue, .failure, .dependency_failure, .skipped_oom => { @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst); + if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); return; }, .precheck_done, .running => { @@ -1109,7 +1141,7 @@ fn workerMakeOneStep( if (new_claimed_rss > run.max_rss) { // Running this step right now could possibly exceed the allotted RSS. // Add this step to the queue of memory-blocked steps. - run.memory_blocked_steps.append(s) catch @panic("OOM"); + run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM"); return; } @@ -1126,10 +1158,14 @@ fn workerMakeOneStep( const sub_prog_node = prog_node.start(s.name, 0); defer sub_prog_node.end(); + if (run.web_server) |*ws| ws.updateStepStatus(s, .wip); + const make_result = s.make(.{ .progress_node = sub_prog_node, .thread_pool = thread_pool, .watch = run.watch, + .web_server = if (run.web_server) |*ws| ws else null, + .gpa = run.gpa, }); // No matter the result, we want to display error/warning messages. @@ -1141,21 +1177,24 @@ fn workerMakeOneStep( if (show_error_msgs or show_compile_errors or show_stderr) { const bw = std.debug.lockStderrWriter(&stdio_buffer_allocation); defer std.debug.unlockStderrWriter(); - - const gpa = b.allocator; - printErrorMessages(gpa, s, .{ .ttyconf = run.ttyconf }, bw, run.prominent_compile_errors) catch {}; + printErrorMessages(run.gpa, s, .{ .ttyconf = run.ttyconf }, bw, run.prominent_compile_errors) catch {}; } handle_result: { if (make_result) |_| { @atomicStore(Step.State, &s.state, .success, .seq_cst); + if (run.web_server) |*ws| ws.updateStepStatus(s, .success); } else |err| switch (err) { error.MakeFailed => { @atomicStore(Step.State, &s.state, .failure, .seq_cst); + if (run.web_server) |*ws| ws.updateStepStatus(s, .failure); std.Progress.setStatus(.failure_working); break :handle_result; }, - error.MakeSkipped => @atomicStore(Step.State, &s.state, .skipped, .seq_cst), + error.MakeSkipped => { + @atomicStore(Step.State, &s.state, .skipped, .seq_cst); + if (run.web_server) |*ws| ws.updateStepStatus(s, .success); + }, } // Successful completion of a step, so we queue up its dependants as well. @@ -1255,10 +1294,10 @@ pub fn printErrorMessages( } fn printSteps(builder: *std.Build, w: *Writer) !void { - const allocator = builder.allocator; + const arena = builder.graph.arena; for (builder.top_level_steps.values()) |top_level_step| { const name = if (&top_level_step.step == builder.default_step) - try fmt.allocPrint(allocator, "{s} (default)", .{top_level_step.step.name}) + try fmt.allocPrint(arena, "{s} (default)", .{top_level_step.step.name}) else top_level_step.step.name; try w.print(" {s:<28} {s}\n", .{ name, top_level_step.description }); @@ -1319,8 +1358,11 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ needed (Default) Lazy dependencies are fetched as needed \\ all Lazy dependencies are always fetched \\ --watch Continuously rebuild when source files are modified - \\ --fuzz Continuously search for unit test failures \\ --debounce Delay before rebuilding after changed file detected + \\ --webui[=ip] Enable the web interface on the given IP address + \\ --fuzz Continuously search for unit test failures (implies '--webui') + \\ --time-report Force full rebuild and provide detailed information on + \\ compilation time of Zig source code (implies '--webui') \\ -fincremental Enable incremental compilation \\ -fno-incremental Disable incremental compilation \\ @@ -1328,7 +1370,7 @@ fn printUsage(b: *std.Build, w: *Writer) !void { \\ ); - const arena = b.allocator; + const arena = b.graph.arena; if (b.available_options_list.items.len == 0) { try w.print(" (none)\n", .{}); } else { diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index ce23f63421acc25e96736d0baa44316475201604..7f1592f0de7cd729a3aa934265a1d7d6f545802b 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -3,7 +3,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const assert = std.debug.assert; const fatal = std.process.fatal; -const SeenPcsHeader = std.Build.Fuzz.abi.SeenPcsHeader; +const SeenPcsHeader = std.Build.abi.fuzz.SeenPcsHeader; pub const std_options = std.Options{ .logFn = logOverride, diff --git a/lib/fuzzer/web/index.html b/lib/fuzzer/web/index.html deleted file mode 100644 index 325342e8ebf569ebe3fe90e523dfbfe2ed3d7a2c..0000000000000000000000000000000000000000 --- a/lib/fuzzer/web/index.html +++ /dev/null @@ -1,161 +0,0 @@ - - - - - Zig Build System Interface - - - -

      Loading JavaScript...

      - - - - - - diff --git a/lib/fuzzer/web/main.js b/lib/fuzzer/web/main.js deleted file mode 100644 index 94f09391bb6706a5d0f6262d241610897770f1ae..0000000000000000000000000000000000000000 --- a/lib/fuzzer/web/main.js +++ /dev/null @@ -1,252 +0,0 @@ -(function() { - const domStatus = document.getElementById("status"); - const domSectSource = document.getElementById("sectSource"); - const domSectStats = document.getElementById("sectStats"); - const domSourceText = document.getElementById("sourceText"); - const domStatTotalRuns = document.getElementById("statTotalRuns"); - const domStatUniqueRuns = document.getElementById("statUniqueRuns"); - const domStatSpeed = document.getElementById("statSpeed"); - const domStatCoverage = document.getElementById("statCoverage"); - const domEntryPointsList = document.getElementById("entryPointsList"); - - let wasm_promise = fetch("main.wasm"); - let sources_promise = fetch("sources.tar").then(function(response) { - if (!response.ok) throw new Error("unable to download sources"); - return response.arrayBuffer(); - }); - var wasm_exports = null; - var curNavSearch = null; - var curNavLocation = null; - - const text_decoder = new TextDecoder(); - const text_encoder = new TextEncoder(); - - domStatus.textContent = "Loading WebAssembly..."; - WebAssembly.instantiateStreaming(wasm_promise, { - js: { - log: function(ptr, len) { - const msg = decodeString(ptr, len); - console.log(msg); - }, - panic: function (ptr, len) { - const msg = decodeString(ptr, len); - throw new Error("panic: " + msg); - }, - timestamp: function () { - return BigInt(new Date()); - }, - emitSourceIndexChange: onSourceIndexChange, - emitCoverageUpdate: onCoverageUpdate, - emitEntryPointsUpdate: renderStats, - }, - }).then(function(obj) { - wasm_exports = obj.instance.exports; - window.wasm = obj; // for debugging - domStatus.textContent = "Loading sources tarball..."; - - sources_promise.then(function(buffer) { - domStatus.textContent = "Parsing sources..."; - const js_array = new Uint8Array(buffer); - const ptr = wasm_exports.alloc(js_array.length); - const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length); - wasm_array.set(js_array); - wasm_exports.unpack(ptr, js_array.length); - - window.addEventListener('popstate', onPopState, false); - onHashChange(null); - - domStatus.textContent = "Waiting for server to send source location metadata..."; - connectWebSocket(); - }); - }); - - function onPopState(ev) { - onHashChange(ev.state); - } - - function onHashChange(state) { - history.replaceState({}, ""); - navigate(location.hash); - if (state == null) window.scrollTo({top: 0}); - } - - function navigate(location_hash) { - domSectSource.classList.add("hidden"); - - curNavLocation = null; - curNavSearch = null; - - if (location_hash.length > 1 && location_hash[0] === '#') { - const query = location_hash.substring(1); - const qpos = query.indexOf("?"); - let nonSearchPart; - if (qpos === -1) { - nonSearchPart = query; - } else { - nonSearchPart = query.substring(0, qpos); - curNavSearch = decodeURIComponent(query.substring(qpos + 1)); - } - - if (nonSearchPart[0] == "l") { - curNavLocation = +nonSearchPart.substring(1); - renderSource(curNavLocation); - } - } - - render(); - } - - function connectWebSocket() { - const host = document.location.host; - const pathname = document.location.pathname; - const isHttps = document.location.protocol === 'https:'; - const match = host.match(/^(.+):(\d+)$/); - const defaultPort = isHttps ? 443 : 80; - const port = match ? parseInt(match[2], 10) : defaultPort; - const hostName = match ? match[1] : host; - const wsProto = isHttps ? "wss:" : "ws:"; - const wsUrl = wsProto + '//' + hostName + ':' + port + pathname; - ws = new WebSocket(wsUrl); - ws.binaryType = "arraybuffer"; - ws.addEventListener('message', onWebSocketMessage, false); - ws.addEventListener('error', timeoutThenCreateNew, false); - ws.addEventListener('close', timeoutThenCreateNew, false); - ws.addEventListener('open', onWebSocketOpen, false); - } - - function onWebSocketOpen() { - //console.log("web socket opened"); - } - - function onWebSocketMessage(ev) { - wasmOnMessage(ev.data); - } - - function timeoutThenCreateNew() { - ws.removeEventListener('message', onWebSocketMessage, false); - ws.removeEventListener('error', timeoutThenCreateNew, false); - ws.removeEventListener('close', timeoutThenCreateNew, false); - ws.removeEventListener('open', onWebSocketOpen, false); - ws = null; - setTimeout(connectWebSocket, 1000); - } - - function wasmOnMessage(data) { - const jsArray = new Uint8Array(data); - const ptr = wasm_exports.message_begin(jsArray.length); - const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, jsArray.length); - wasmArray.set(jsArray); - wasm_exports.message_end(); - } - - function onSourceIndexChange() { - render(); - if (curNavLocation != null) renderSource(curNavLocation); - } - - function onCoverageUpdate() { - renderStats(); - renderCoverage(); - } - - function render() { - domStatus.classList.add("hidden"); - } - - function renderStats() { - const totalRuns = wasm_exports.totalRuns(); - const uniqueRuns = wasm_exports.uniqueRuns(); - const totalSourceLocations = wasm_exports.totalSourceLocations(); - const coveredSourceLocations = wasm_exports.coveredSourceLocations(); - domStatTotalRuns.innerText = totalRuns; - domStatUniqueRuns.innerText = uniqueRuns + " (" + percent(uniqueRuns, totalRuns) + "%)"; - domStatCoverage.innerText = coveredSourceLocations + " / " + totalSourceLocations + " (" + percent(coveredSourceLocations, totalSourceLocations) + "%)"; - domStatSpeed.innerText = wasm_exports.totalRunsPerSecond().toFixed(0); - - const entryPoints = unwrapInt32Array(wasm_exports.entryPoints()); - resizeDomList(domEntryPointsList, entryPoints.length, "
    • "); - for (let i = 0; i < entryPoints.length; i += 1) { - const liDom = domEntryPointsList.children[i]; - liDom.innerHTML = unwrapString(wasm_exports.sourceLocationLinkHtml(entryPoints[i])); - } - - - domSectStats.classList.remove("hidden"); - } - - function renderCoverage() { - if (curNavLocation == null) return; - const sourceLocationIndex = curNavLocation; - - for (let i = 0; i < domSourceText.children.length; i += 1) { - const childDom = domSourceText.children[i]; - if (childDom.id != null && childDom.id[0] == "l") { - childDom.classList.add("l"); - childDom.classList.remove("c"); - } - } - const coveredList = unwrapInt32Array(wasm_exports.sourceLocationFileCoveredList(sourceLocationIndex)); - for (let i = 0; i < coveredList.length; i += 1) { - document.getElementById("l" + coveredList[i]).classList.add("c"); - } - } - - function resizeDomList(listDom, desiredLen, templateHtml) { - for (let i = listDom.childElementCount; i < desiredLen; i += 1) { - listDom.insertAdjacentHTML('beforeend', templateHtml); - } - while (desiredLen < listDom.childElementCount) { - listDom.removeChild(listDom.lastChild); - } - } - - function percent(a, b) { - return ((Number(a) / Number(b)) * 100).toFixed(1); - } - - function renderSource(sourceLocationIndex) { - const pathName = unwrapString(wasm_exports.sourceLocationPath(sourceLocationIndex)); - if (pathName.length === 0) return; - - const h2 = domSectSource.children[0]; - h2.innerText = pathName; - domSourceText.innerHTML = unwrapString(wasm_exports.sourceLocationFileHtml(sourceLocationIndex)); - - domSectSource.classList.remove("hidden"); - - // Empirically, Firefox needs this requestAnimationFrame in order for the scrollIntoView to work. - requestAnimationFrame(function() { - const slDom = document.getElementById("l" + sourceLocationIndex); - if (slDom != null) slDom.scrollIntoView({ - behavior: "smooth", - block: "center", - }); - }); - } - - function decodeString(ptr, len) { - if (len === 0) return ""; - return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len)); - } - - function unwrapInt32Array(bigint) { - const ptr = Number(bigint & 0xffffffffn); - const len = Number(bigint >> 32n); - if (len === 0) return new Uint32Array(); - return new Uint32Array(wasm_exports.memory.buffer, ptr, len); - } - - function setInputString(s) { - const jsArray = text_encoder.encode(s); - const len = jsArray.length; - const ptr = wasm_exports.set_input_string(len); - const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len); - wasmArray.set(jsArray); - } - - function unwrapString(bigint) { - const ptr = Number(bigint & 0xffffffffn); - const len = Number(bigint >> 32n); - return decodeString(ptr, len); - } -})(); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index d6b0e68f5d4abcf42d548a6bd40678c57b2b3942..db539e7b09ab8ca3dc03e75903ec385ac952b6f8 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -22,6 +22,8 @@ pub const Step = @import("Build/Step.zig"); pub const Module = @import("Build/Module.zig"); pub const Watch = @import("Build/Watch.zig"); pub const Fuzz = @import("Build/Fuzz.zig"); +pub const WebServer = @import("Build/WebServer.zig"); +pub const abi = @import("Build/abi.zig"); /// Shared state among all Build instances. graph: *Graph, @@ -125,6 +127,7 @@ pub const Graph = struct { random_seed: u32 = 0, dependency_cache: InitializedDepMap = .empty, allow_so_scripts: ?bool = null, + time_report: bool, }; const AvailableDeps = []const struct { []const u8, []const u8 }; diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 28f8781dd196eb3aab810ebf245b233dc942f55b..a25b501755974dfc1f0c2c8c9f9eaa115fbaa40c 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -1,108 +1,134 @@ -const builtin = @import("builtin"); const std = @import("../std.zig"); const Build = std.Build; +const Cache = Build.Cache; const Step = std.Build.Step; const assert = std.debug.assert; const fatal = std.process.fatal; const Allocator = std.mem.Allocator; const log = std.log; +const Coverage = std.debug.Coverage; +const abi = Build.abi.fuzz; const Fuzz = @This(); const build_runner = @import("root"); -pub const WebServer = @import("Fuzz/WebServer.zig"); -pub const abi = @import("Fuzz/abi.zig"); - -pub fn start( - gpa: Allocator, - arena: Allocator, - global_cache_directory: Build.Cache.Directory, - zig_lib_directory: Build.Cache.Directory, - zig_exe_path: []const u8, - thread_pool: *std.Thread.Pool, - all_steps: []const *Step, - ttyconf: std.io.tty.Config, - listen_address: std.net.Address, - prog_node: std.Progress.Node, -) Allocator.Error!void { - const fuzz_run_steps = block: { - const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); +ws: *Build.WebServer, + +/// Allocated into `ws.gpa`. +run_steps: []const *Step.Run, + +wait_group: std.Thread.WaitGroup, +prog_node: std.Progress.Node, + +/// Protects `coverage_files`. +coverage_mutex: std.Thread.Mutex, +coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), + +queue_mutex: std.Thread.Mutex, +queue_cond: std.Thread.Condition, +msg_queue: std.ArrayListUnmanaged(Msg), + +const Msg = union(enum) { + coverage: struct { + id: u64, + run: *Step.Run, + }, + entry_point: struct { + coverage_id: u64, + addr: u64, + }, +}; + +const CoverageMap = struct { + mapped_memory: []align(std.heap.page_size_min) const u8, + coverage: Coverage, + source_locations: []Coverage.SourceLocation, + /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. + entry_points: std.ArrayListUnmanaged(u32), + start_timestamp: i64, + + fn deinit(cm: *CoverageMap, gpa: Allocator) void { + std.posix.munmap(cm.mapped_memory); + cm.coverage.deinit(gpa); + cm.* = undefined; + } +}; + +pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz { + const gpa = ws.gpa; + + const run_steps: []const *Step.Run = steps: { + var steps: std.ArrayListUnmanaged(*Step.Run) = .empty; + defer steps.deinit(gpa); + const rebuild_node = ws.root_prog_node.start("Rebuilding Unit Tests", 0); defer rebuild_node.end(); - var wait_group: std.Thread.WaitGroup = .{}; - defer wait_group.wait(); - var fuzz_run_steps: std.ArrayListUnmanaged(*Step.Run) = .empty; - defer fuzz_run_steps.deinit(gpa); - for (all_steps) |step| { + var rebuild_wg: std.Thread.WaitGroup = .{}; + defer rebuild_wg.wait(); + + for (ws.all_steps) |step| { const run = step.cast(Step.Run) orelse continue; - if (run.fuzz_tests.items.len > 0 and run.producer != null) { - thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node }); - try fuzz_run_steps.append(gpa, run); - } + if (run.producer == null) continue; + if (run.fuzz_tests.items.len == 0) continue; + try steps.append(gpa, run); + ws.thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ws.ttyconf, rebuild_node }); } - if (fuzz_run_steps.items.len == 0) fatal("no fuzz tests found", .{}); - rebuild_node.setEstimatedTotalItems(fuzz_run_steps.items.len); - break :block try arena.dupe(*Step.Run, fuzz_run_steps.items); + + if (steps.items.len == 0) fatal("no fuzz tests found", .{}); + rebuild_node.setEstimatedTotalItems(steps.items.len); + break :steps try gpa.dupe(*Step.Run, steps.items); }; + errdefer gpa.free(run_steps); - // Detect failure. - for (fuzz_run_steps) |run| { + for (run_steps) |run| { assert(run.fuzz_tests.items.len > 0); if (run.rebuilt_executable == null) fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); } - var web_server: WebServer = .{ - .gpa = gpa, - .global_cache_directory = global_cache_directory, - .zig_lib_directory = zig_lib_directory, - .zig_exe_path = zig_exe_path, - .listen_address = listen_address, - .fuzz_run_steps = fuzz_run_steps, - - .msg_queue = .{}, - .mutex = .{}, - .condition = .{}, - - .coverage_files = .{}, + return .{ + .ws = ws, + .run_steps = run_steps, + .wait_group = .{}, + .prog_node = .none, + .coverage_files = .empty, .coverage_mutex = .{}, - .coverage_condition = .{}, - - .base_timestamp = std.time.nanoTimestamp(), + .queue_mutex = .{}, + .queue_cond = .{}, + .msg_queue = .empty, }; +} - // For accepting HTTP connections. - const web_server_thread = std.Thread.spawn(.{}, WebServer.run, .{&web_server}) catch |err| { - fatal("unable to spawn web server thread: {s}", .{@errorName(err)}); - }; - defer web_server_thread.join(); +pub fn start(fuzz: *Fuzz) void { + const ws = fuzz.ws; + fuzz.prog_node = ws.root_prog_node.start("Fuzzing", fuzz.run_steps.len); // For polling messages and sending updates to subscribers. - const coverage_thread = std.Thread.spawn(.{}, WebServer.coverageRun, .{&web_server}) catch |err| { + fuzz.wait_group.start(); + _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| { + fuzz.wait_group.finish(); fatal("unable to spawn coverage thread: {s}", .{@errorName(err)}); }; - defer coverage_thread.join(); - { - const fuzz_node = prog_node.start("Fuzzing", fuzz_run_steps.len); - defer fuzz_node.end(); - var wait_group: std.Thread.WaitGroup = .{}; - defer wait_group.wait(); - - for (fuzz_run_steps) |run| { - for (run.fuzz_tests.items) |unit_test_index| { - assert(run.rebuilt_executable != null); - thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ - run, &web_server, unit_test_index, ttyconf, fuzz_node, - }); - } + for (fuzz.run_steps) |run| { + for (run.fuzz_tests.items) |unit_test_index| { + assert(run.rebuilt_executable != null); + ws.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{ + fuzz, run, unit_test_index, + }); } } +} +pub fn deinit(fuzz: *Fuzz) void { + if (true) @panic("TODO: terminate the fuzzer processes"); + fuzz.wait_group.wait(); + fuzz.prog_node.end(); - log.err("all fuzz workers crashed", .{}); + const gpa = fuzz.ws.gpa; + gpa.free(fuzz.run_steps); } -fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { - rebuildTestsWorkerRunFallible(run, ttyconf, parent_prog_node) catch |err| { +fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { + rebuildTestsWorkerRunFallible(run, gpa, ttyconf, parent_prog_node) catch |err| { const compile = run.producer.?; log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{ compile.step.name, @errorName(err), @@ -110,14 +136,12 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog }; } -fn rebuildTestsWorkerRunFallible(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) !void { - const gpa = run.step.owner.allocator; - +fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) !void { const compile = run.producer.?; const prog_node = parent_prog_node.start(compile.step.name, 0); defer prog_node.end(); - const result = compile.rebuildInFuzzMode(prog_node); + const result = compile.rebuildInFuzzMode(gpa, prog_node); const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; const show_error_msgs = compile.step.result_error_msgs.items.len > 0; @@ -138,24 +162,22 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, ttyconf: std.io.tty.Config, par } fn fuzzWorkerRun( + fuzz: *Fuzz, run: *Step.Run, - web_server: *WebServer, unit_test_index: u32, - ttyconf: std.io.tty.Config, - parent_prog_node: std.Progress.Node, ) void { const gpa = run.step.owner.allocator; const test_name = run.cached_test_metadata.?.testName(unit_test_index); - const prog_node = parent_prog_node.start(test_name, 0); + const prog_node = fuzz.prog_node.start(test_name, 0); defer prog_node.end(); - run.rerunInFuzzMode(web_server, unit_test_index, prog_node) catch |err| switch (err) { + run.rerunInFuzzMode(fuzz, unit_test_index, prog_node) catch |err| switch (err) { error.MakeFailed => { var buf: [256]u8 = undefined; const w = std.debug.lockStderrWriter(&buf); defer std.debug.unlockStderrWriter(); - build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = ttyconf }, w, false) catch {}; + build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ws.ttyconf }, w, false) catch {}; return; }, else => { @@ -166,3 +188,270 @@ fn fuzzWorkerRun( }, }; } + +pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { + const gpa = fuzz.ws.gpa; + + var arena_state: std.heap.ArenaAllocator = .init(gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); + var dedup_table: DedupTable = .empty; + defer dedup_table.deinit(gpa); + + for (fuzz.run_steps) |run_step| { + const compile_inputs = run_step.producer.?.step.inputs.table; + for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { + try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len); + for (file_list.items) |sub_path| { + if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; + const joined_path = try dir_path.join(arena, sub_path); + dedup_table.putAssumeCapacity(joined_path, {}); + } + } + } + + const deduped_paths = dedup_table.keys(); + const SortContext = struct { + pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { + _ = this; + return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { + .lt => true, + .gt => false, + .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), + }; + } + }; + std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); + return fuzz.ws.serveTarFile(req, deduped_paths); +} + +pub const Previous = struct { + unique_runs: usize, + entry_points: usize, + pub const init: Previous = .{ .unique_runs = 0, .entry_points = 0 }; +}; +pub fn sendUpdate( + fuzz: *Fuzz, + socket: *std.http.WebSocket, + prev: *Previous, +) !void { + fuzz.coverage_mutex.lock(); + defer fuzz.coverage_mutex.unlock(); + + const coverage_maps = fuzz.coverage_files.values(); + if (coverage_maps.len == 0) return; + // TODO: handle multiple fuzz steps in the WebSocket packets + const coverage_map = &coverage_maps[0]; + const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the + // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the + // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass + // this data straight to the socket with sendfile... + const seen_pcs = cov_header.seenBits(); + const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); + const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); + if (prev.unique_runs != unique_runs) { + // There has been an update. + if (prev.unique_runs == 0) { + // We need to send initial context. + const header: abi.SourceIndexHeader = .{ + .directories_len = @intCast(coverage_map.coverage.directories.entries.len), + .files_len = @intCast(coverage_map.coverage.files.entries.len), + .source_locations_len = @intCast(coverage_map.source_locations.len), + .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), + .start_timestamp = coverage_map.start_timestamp, + }; + const iovecs: [5]std.posix.iovec_const = .{ + makeIov(@ptrCast(&header)), + makeIov(@ptrCast(coverage_map.coverage.directories.keys())), + makeIov(@ptrCast(coverage_map.coverage.files.keys())), + makeIov(@ptrCast(coverage_map.source_locations)), + makeIov(coverage_map.coverage.string_bytes.items), + }; + try socket.writeMessagev(&iovecs, .binary); + } + + const header: abi.CoverageUpdateHeader = .{ + .n_runs = n_runs, + .unique_runs = unique_runs, + }; + const iovecs: [2]std.posix.iovec_const = .{ + makeIov(@ptrCast(&header)), + makeIov(@ptrCast(seen_pcs)), + }; + try socket.writeMessagev(&iovecs, .binary); + + prev.unique_runs = unique_runs; + } + + if (prev.entry_points != coverage_map.entry_points.items.len) { + const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); + const iovecs: [2]std.posix.iovec_const = .{ + makeIov(@ptrCast(&header)), + makeIov(@ptrCast(coverage_map.entry_points.items)), + }; + try socket.writeMessagev(&iovecs, .binary); + + prev.entry_points = coverage_map.entry_points.items.len; + } +} + +fn coverageRun(fuzz: *Fuzz) void { + defer fuzz.wait_group.finish(); + + fuzz.queue_mutex.lock(); + defer fuzz.queue_mutex.unlock(); + + while (true) { + fuzz.queue_cond.wait(&fuzz.queue_mutex); + for (fuzz.msg_queue.items) |msg| switch (msg) { + .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { + error.AlreadyReported => continue, + else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}), + }, + .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { + error.AlreadyReported => continue, + else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}), + }, + }; + fuzz.msg_queue.clearRetainingCapacity(); + } +} +fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void { + const ws = fuzz.ws; + const gpa = ws.gpa; + + fuzz.coverage_mutex.lock(); + defer fuzz.coverage_mutex.unlock(); + + const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); + if (gop.found_existing) { + // We are fuzzing the same executable with multiple threads. + // Perhaps the same unit test; perhaps a different one. In any + // case, since the coverage file is the same, we only have to + // notice changes to that one file in order to learn coverage for + // this particular executable. + return; + } + errdefer _ = fuzz.coverage_files.pop(); + + gop.value_ptr.* = .{ + .coverage = std.debug.Coverage.init, + .mapped_memory = undefined, // populated below + .source_locations = undefined, // populated below + .entry_points = .{}, + .start_timestamp = ws.now(), + }; + errdefer gop.value_ptr.coverage.deinit(gpa); + + const rebuilt_exe_path = run_step.rebuilt_executable.?; + var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| { + log.err("step '{s}': failed to load debug information for '{f}': {s}", .{ + run_step.step.name, rebuilt_exe_path, @errorName(err), + }); + return error.AlreadyReported; + }; + defer debug_info.deinit(gpa); + + const coverage_file_path: Build.Cache.Path = .{ + .root_dir = run_step.step.owner.cache_root, + .sub_path = "v/" ++ std.fmt.hex(coverage_id), + }; + var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| { + log.err("step '{s}': failed to load coverage file '{f}': {s}", .{ + run_step.step.name, coverage_file_path, @errorName(err), + }); + return error.AlreadyReported; + }; + defer coverage_file.close(); + + const file_size = coverage_file.getEndPos() catch |err| { + log.err("unable to check len of coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) }); + return error.AlreadyReported; + }; + + const mapped_memory = std.posix.mmap( + null, + file_size, + std.posix.PROT.READ, + .{ .TYPE = .SHARED }, + coverage_file.handle, + 0, + ) catch |err| { + log.err("failed to map coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) }); + return error.AlreadyReported; + }; + gop.value_ptr.mapped_memory = mapped_memory; + + const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); + errdefer gpa.free(source_locations); + + // Unfortunately the PCs array that LLVM gives us from the 8-bit PC + // counters feature is not sorted. + var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{}; + defer sorted_pcs.deinit(gpa); + try sorted_pcs.resize(gpa, pcs.len); + @memcpy(sorted_pcs.items(.pc), pcs); + for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); + sorted_pcs.sortUnstable(struct { + addrs: []const u64, + + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.addrs[a_index] < ctx.addrs[b_index]; + } + }{ .addrs = sorted_pcs.items(.pc) }); + + debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { + log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)}); + return error.AlreadyReported; + }; + + for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; + gop.value_ptr.source_locations = source_locations; + + ws.notifyUpdate(); +} +fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void { + fuzz.coverage_mutex.lock(); + defer fuzz.coverage_mutex.unlock(); + + const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; + const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + + // Since this pcs list is unsorted, we must linear scan for the best index. + const index = i: { + var best: usize = 0; + for (pcs[1..], 1..) |elem_addr, i| { + if (elem_addr == addr) break :i i; + if (elem_addr > addr) continue; + if (elem_addr > pcs[best]) best = i; + } + break :i best; + }; + if (index >= pcs.len) { + log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ + addr, pcs[0], pcs[pcs.len - 1], + }); + return error.AlreadyReported; + } + if (false) { + const sl = coverage_map.source_locations[index]; + const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ + addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], + }); + } + try coverage_map.entry_points.append(fuzz.ws.gpa, @intCast(index)); +} + +fn makeIov(s: []const u8) std.posix.iovec_const { + return .{ + .base = s.ptr, + .len = s.len, + }; +} diff --git a/lib/std/Build/Fuzz/WebServer.zig b/lib/std/Build/Fuzz/WebServer.zig deleted file mode 100644 index 18582a60ef6a9464a847c529e1155af1d2fe0183..0000000000000000000000000000000000000000 --- a/lib/std/Build/Fuzz/WebServer.zig +++ /dev/null @@ -1,709 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("../../std.zig"); -const Allocator = std.mem.Allocator; -const Build = std.Build; -const Step = std.Build.Step; -const Coverage = std.debug.Coverage; -const abi = std.Build.Fuzz.abi; -const log = std.log; -const assert = std.debug.assert; -const Cache = std.Build.Cache; -const Path = Cache.Path; - -const WebServer = @This(); - -gpa: Allocator, -global_cache_directory: Build.Cache.Directory, -zig_lib_directory: Build.Cache.Directory, -zig_exe_path: []const u8, -listen_address: std.net.Address, -fuzz_run_steps: []const *Step.Run, - -/// Messages from fuzz workers. Protected by mutex. -msg_queue: std.ArrayListUnmanaged(Msg), -/// Protects `msg_queue` only. -mutex: std.Thread.Mutex, -/// Signaled when there is a message in `msg_queue`. -condition: std.Thread.Condition, - -coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), -/// Protects `coverage_files` only. -coverage_mutex: std.Thread.Mutex, -/// Signaled when `coverage_files` changes. -coverage_condition: std.Thread.Condition, - -/// Time at initialization of WebServer. -base_timestamp: i128, - -const fuzzer_bin_name = "fuzzer"; -const fuzzer_arch_os_abi = "wasm32-freestanding"; -const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; - -const CoverageMap = struct { - mapped_memory: []align(std.heap.page_size_min) const u8, - coverage: Coverage, - source_locations: []Coverage.SourceLocation, - /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. - entry_points: std.ArrayListUnmanaged(u32), - start_timestamp: i64, - - fn deinit(cm: *CoverageMap, gpa: Allocator) void { - std.posix.munmap(cm.mapped_memory); - cm.coverage.deinit(gpa); - cm.* = undefined; - } -}; - -const Msg = union(enum) { - coverage: struct { - id: u64, - run: *Step.Run, - }, - entry_point: struct { - coverage_id: u64, - addr: u64, - }, -}; - -pub fn run(ws: *WebServer) void { - var http_server = ws.listen_address.listen(.{ - .reuse_address = true, - }) catch |err| { - log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.in.getPort(), @errorName(err) }); - return; - }; - const port = http_server.listen_address.in.getPort(); - log.info("web interface listening at http://127.0.0.1:{d}/", .{port}); - if (ws.listen_address.in.getPort() == 0) - log.info("hint: pass --port {d} to use this same port next time", .{port}); - - while (true) { - const connection = http_server.accept() catch |err| { - log.err("failed to accept connection: {s}", .{@errorName(err)}); - return; - }; - _ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch |err| { - log.err("unable to spawn connection thread: {s}", .{@errorName(err)}); - connection.stream.close(); - continue; - }; - } -} - -fn now(s: *const WebServer) i64 { - return @intCast(std.time.nanoTimestamp() - s.base_timestamp); -} - -fn accept(ws: *WebServer, connection: std.net.Server.Connection) void { - defer connection.stream.close(); - - var read_buffer: [0x4000]u8 = undefined; - var server = std.http.Server.init(connection, &read_buffer); - var web_socket: std.http.WebSocket = undefined; - var send_buffer: [0x4000]u8 = undefined; - var ws_recv_buffer: [0x4000]u8 align(4) = undefined; - while (server.state == .ready) { - var request = server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => { - log.err("closing http connection: {s}", .{@errorName(err)}); - return; - }, - }; - if (web_socket.init(&request, &send_buffer, &ws_recv_buffer) catch |err| { - log.err("initializing web socket: {s}", .{@errorName(err)}); - return; - }) { - serveWebSocket(ws, &web_socket) catch |err| { - log.err("unable to serve web socket connection: {s}", .{@errorName(err)}); - return; - }; - } else { - serveRequest(ws, &request) catch |err| switch (err) { - error.AlreadyReported => return, - else => |e| { - log.err("unable to serve {s}: {s}", .{ request.head.target, @errorName(e) }); - return; - }, - }; - } - } -} - -fn serveRequest(ws: *WebServer, request: *std.http.Server.Request) !void { - if (std.mem.eql(u8, request.head.target, "/") or - std.mem.eql(u8, request.head.target, "/debug") or - std.mem.eql(u8, request.head.target, "/debug/")) - { - try serveFile(ws, request, "fuzzer/web/index.html", "text/html"); - } else if (std.mem.eql(u8, request.head.target, "/main.js") or - std.mem.eql(u8, request.head.target, "/debug/main.js")) - { - try serveFile(ws, request, "fuzzer/web/main.js", "application/javascript"); - } else if (std.mem.eql(u8, request.head.target, "/main.wasm")) { - try serveWasm(ws, request, .ReleaseFast); - } else if (std.mem.eql(u8, request.head.target, "/debug/main.wasm")) { - try serveWasm(ws, request, .Debug); - } else if (std.mem.eql(u8, request.head.target, "/sources.tar") or - std.mem.eql(u8, request.head.target, "/debug/sources.tar")) - { - try serveSourcesTar(ws, request); - } else { - try request.respond("not found", .{ - .status = .not_found, - .extra_headers = &.{ - .{ .name = "content-type", .value = "text/plain" }, - }, - }); - } -} - -fn serveFile( - ws: *WebServer, - request: *std.http.Server.Request, - name: []const u8, - content_type: []const u8, -) !void { - const gpa = ws.gpa; - // The desired API is actually sendfile, which will require enhancing std.http.Server. - // We load the file with every request so that the user can make changes to the file - // and refresh the HTML page without restarting this server. - const file_contents = ws.zig_lib_directory.handle.readFileAlloc(gpa, name, 10 * 1024 * 1024) catch |err| { - log.err("failed to read '{f}{s}': {s}", .{ ws.zig_lib_directory, name, @errorName(err) }); - return error.AlreadyReported; - }; - defer gpa.free(file_contents); - try request.respond(file_contents, .{ - .extra_headers = &.{ - .{ .name = "content-type", .value = content_type }, - cache_control_header, - }, - }); -} - -fn serveWasm( - ws: *WebServer, - request: *std.http.Server.Request, - optimize_mode: std.builtin.OptimizeMode, -) !void { - const gpa = ws.gpa; - - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // Do the compilation every request, so that the user can edit the files - // and see the changes without restarting the server. - const wasm_base_path = try buildWasmBinary(ws, arena, optimize_mode); - const bin_name = try std.zig.binNameAlloc(arena, .{ - .root_name = fuzzer_bin_name, - .target = &(std.zig.system.resolveTargetQuery(std.Build.parseTargetQuery(.{ - .arch_os_abi = fuzzer_arch_os_abi, - .cpu_features = fuzzer_cpu_features, - }) catch unreachable) catch unreachable), - .output_mode = .Exe, - }); - // std.http.Server does not have a sendfile API yet. - const bin_path = try wasm_base_path.join(arena, bin_name); - const file_contents = try bin_path.root_dir.handle.readFileAlloc(gpa, bin_path.sub_path, 10 * 1024 * 1024); - defer gpa.free(file_contents); - try request.respond(file_contents, .{ - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/wasm" }, - cache_control_header, - }, - }); -} - -fn buildWasmBinary( - ws: *WebServer, - arena: Allocator, - optimize_mode: std.builtin.OptimizeMode, -) !Path { - const gpa = ws.gpa; - - const main_src_path: Build.Cache.Path = .{ - .root_dir = ws.zig_lib_directory, - .sub_path = "fuzzer/web/main.zig", - }; - const walk_src_path: Build.Cache.Path = .{ - .root_dir = ws.zig_lib_directory, - .sub_path = "docs/wasm/Walk.zig", - }; - const html_render_src_path: Build.Cache.Path = .{ - .root_dir = ws.zig_lib_directory, - .sub_path = "docs/wasm/html_render.zig", - }; - - var argv: std.ArrayListUnmanaged([]const u8) = .empty; - - try argv.appendSlice(arena, &.{ - ws.zig_exe_path, "build-exe", // - "-fno-entry", // - "-O", @tagName(optimize_mode), // - "-target", fuzzer_arch_os_abi, // - "-mcpu", fuzzer_cpu_features, // - "--cache-dir", ws.global_cache_directory.path orelse ".", // - "--global-cache-dir", ws.global_cache_directory.path orelse ".", // - "--name", fuzzer_bin_name, // - "-rdynamic", // - "-fsingle-threaded", // - "--dep", "Walk", // - "--dep", "html_render", // - try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // - try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // - "--dep", "Walk", // - try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // - "--listen=-", - }); - - var child = std.process.Child.init(argv.items, gpa); - child.stdin_behavior = .Pipe; - child.stdout_behavior = .Pipe; - child.stderr_behavior = .Pipe; - try child.spawn(); - - var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ - .stdout = child.stdout.?, - .stderr = child.stderr.?, - }); - defer poller.deinit(); - - try sendMessage(child.stdin.?, .update); - try sendMessage(child.stdin.?, .exit); - - var result: ?Path = null; - var result_error_bundle = std.zig.ErrorBundle.empty; - - const stdout = poller.reader(.stdout); - - poll: while (true) { - const Header = std.zig.Server.Message.Header; - while (stdout.buffered().len < @sizeOf(Header)) if (!try poller.poll()) break :poll; - const header = stdout.takeStruct(Header, .little) catch unreachable; - while (stdout.buffered().len < header.bytes_len) if (!try poller.poll()) break :poll; - const body = stdout.take(header.bytes_len) catch unreachable; - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return error.ZigProtocolVersionMismatch; - } - }, - .error_bundle => { - const EbHdr = std.zig.Server.Message.ErrorBundle; - const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); - const extra_bytes = - body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; - const string_bytes = - body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; - // TODO: use @ptrCast when the compiler supports it - const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); - const extra_array = try arena.alloc(u32, unaligned_extra.len); - @memcpy(extra_array, unaligned_extra); - result_error_bundle = .{ - .string_bytes = try arena.dupe(u8, string_bytes), - .extra = extra_array, - }; - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const ebp_hdr = @as(*align(1) const EmitDigest, @ptrCast(body)); - if (!ebp_hdr.flags.cache_hit) { - log.info("source changes detected; rebuilt wasm component", .{}); - } - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = ws.global_cache_directory, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - else => {}, // ignore other messages - } - } - - const stderr_contents = try poller.toOwnedSlice(.stderr); - if (stderr_contents.len > 0) { - std.debug.print("{s}", .{stderr_contents}); - } - - // Send EOF to stdin. - child.stdin.?.close(); - child.stdin = null; - - switch (try child.wait()) { - .Exited => |code| { - if (code != 0) { - log.err( - "the following command exited with error code {d}:\n{s}", - .{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) }, - ); - return error.WasmCompilationFailed; - } - }, - .Signal, .Stopped, .Unknown => { - log.err( - "the following command terminated unexpectedly:\n{s}", - .{try Build.Step.allocPrintCmd(arena, null, argv.items)}, - ); - return error.WasmCompilationFailed; - }, - } - - if (result_error_bundle.errorMessageCount() > 0) { - const color = std.zig.Color.auto; - result_error_bundle.renderToStdErr(color.renderOptions()); - log.err("the following command failed with {d} compilation errors:\n{s}", .{ - result_error_bundle.errorMessageCount(), - try Build.Step.allocPrintCmd(arena, null, argv.items), - }); - return error.WasmCompilationFailed; - } - - return result orelse { - log.err("child process failed to report result\n{s}", .{ - try Build.Step.allocPrintCmd(arena, null, argv.items), - }); - return error.WasmCompilationFailed; - }; -} - -fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { - const header: std.zig.Client.Message.Header = .{ - .tag = tag, - .bytes_len = 0, - }; - try file.writeAll(std.mem.asBytes(&header)); -} - -fn serveWebSocket(ws: *WebServer, web_socket: *std.http.WebSocket) !void { - ws.coverage_mutex.lock(); - defer ws.coverage_mutex.unlock(); - - // On first connection, the client needs to know what time the server - // thinks it is to rebase timestamps. - { - const timestamp_message: abi.CurrentTime = .{ .base = ws.now() }; - try web_socket.writeMessage(std.mem.asBytes(×tamp_message), .binary); - } - - // On first connection, the client needs all the coverage information - // so that subsequent updates can contain only the updated bits. - var prev_unique_runs: usize = 0; - var prev_entry_points: usize = 0; - try sendCoverageContext(ws, web_socket, &prev_unique_runs, &prev_entry_points); - while (true) { - ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {}; - try sendCoverageContext(ws, web_socket, &prev_unique_runs, &prev_entry_points); - } -} - -fn sendCoverageContext( - ws: *WebServer, - web_socket: *std.http.WebSocket, - prev_unique_runs: *usize, - prev_entry_points: *usize, -) !void { - const coverage_maps = ws.coverage_files.values(); - if (coverage_maps.len == 0) return; - // TODO: make each events URL correspond to one coverage map - const coverage_map = &coverage_maps[0]; - const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const seen_pcs = cov_header.seenBits(); - const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); - const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); - if (prev_unique_runs.* != unique_runs) { - // There has been an update. - if (prev_unique_runs.* == 0) { - // We need to send initial context. - const header: abi.SourceIndexHeader = .{ - .flags = .{}, - .directories_len = @intCast(coverage_map.coverage.directories.entries.len), - .files_len = @intCast(coverage_map.coverage.files.entries.len), - .source_locations_len = @intCast(coverage_map.source_locations.len), - .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), - .start_timestamp = coverage_map.start_timestamp, - }; - const iovecs: [5]std.posix.iovec_const = .{ - makeIov(std.mem.asBytes(&header)), - makeIov(std.mem.sliceAsBytes(coverage_map.coverage.directories.keys())), - makeIov(std.mem.sliceAsBytes(coverage_map.coverage.files.keys())), - makeIov(std.mem.sliceAsBytes(coverage_map.source_locations)), - makeIov(coverage_map.coverage.string_bytes.items), - }; - try web_socket.writeMessagev(&iovecs, .binary); - } - - const header: abi.CoverageUpdateHeader = .{ - .n_runs = n_runs, - .unique_runs = unique_runs, - }; - const iovecs: [2]std.posix.iovec_const = .{ - makeIov(std.mem.asBytes(&header)), - makeIov(std.mem.sliceAsBytes(seen_pcs)), - }; - try web_socket.writeMessagev(&iovecs, .binary); - - prev_unique_runs.* = unique_runs; - } - - if (prev_entry_points.* != coverage_map.entry_points.items.len) { - const header: abi.EntryPointHeader = .{ - .flags = .{ - .locs_len = @intCast(coverage_map.entry_points.items.len), - }, - }; - const iovecs: [2]std.posix.iovec_const = .{ - makeIov(std.mem.asBytes(&header)), - makeIov(std.mem.sliceAsBytes(coverage_map.entry_points.items)), - }; - try web_socket.writeMessagev(&iovecs, .binary); - - prev_entry_points.* = coverage_map.entry_points.items.len; - } -} - -fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void { - const gpa = ws.gpa; - - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - var send_buffer: [0x4000]u8 = undefined; - var response = request.respondStreaming(.{ - .send_buffer = &send_buffer, - .respond_options = .{ - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/x-tar" }, - cache_control_header, - }, - }, - }); - - const DedupeTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); - var dedupe_table: DedupeTable = .{}; - defer dedupe_table.deinit(gpa); - - for (ws.fuzz_run_steps) |run_step| { - const compile_step_inputs = run_step.producer.?.step.inputs.table; - for (compile_step_inputs.keys(), compile_step_inputs.values()) |dir_path, *file_list| { - try dedupe_table.ensureUnusedCapacity(gpa, file_list.items.len); - for (file_list.items) |sub_path| { - // Special file "." means the entire directory. - if (std.mem.eql(u8, sub_path, ".")) continue; - const joined_path = try dir_path.join(arena, sub_path); - _ = dedupe_table.getOrPutAssumeCapacity(joined_path); - } - } - } - - const deduped_paths = dedupe_table.keys(); - const SortContext = struct { - pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { - _ = this; - return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { - .lt => true, - .gt => false, - .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), - }; - } - }; - std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); - - var cwd_cache: ?[]const u8 = null; - - var adapter = response.writer().adaptToNewApi(); - var archiver: std.tar.Writer = .{ .underlying_writer = &adapter.new_interface }; - var read_buffer: [1024]u8 = undefined; - - for (deduped_paths) |joined_path| { - var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch |err| { - log.err("failed to open {f}: {s}", .{ joined_path, @errorName(err) }); - continue; - }; - defer file.close(); - const stat = try file.stat(); - var file_reader: std.fs.File.Reader = .initSize(file, &read_buffer, stat.size); - archiver.prefix = joined_path.root_dir.path orelse try memoizedCwd(arena, &cwd_cache); - try archiver.writeFile(joined_path.sub_path, &file_reader, stat.mtime); - } - - // intentionally not calling `archiver.finishPedantically` - try adapter.new_interface.flush(); - try response.end(); -} - -fn memoizedCwd(arena: Allocator, opt_ptr: *?[]const u8) ![]const u8 { - if (opt_ptr.*) |cached| return cached; - const result = try std.process.getCwdAlloc(arena); - opt_ptr.* = result; - return result; -} - -const cache_control_header: std.http.Header = .{ - .name = "cache-control", - .value = "max-age=0, must-revalidate", -}; - -pub fn coverageRun(ws: *WebServer) void { - ws.mutex.lock(); - defer ws.mutex.unlock(); - - while (true) { - ws.condition.wait(&ws.mutex); - for (ws.msg_queue.items) |msg| switch (msg) { - .coverage => |coverage| prepareTables(ws, coverage.run, coverage.id) catch |err| switch (err) { - error.AlreadyReported => continue, - else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}), - }, - .entry_point => |entry_point| addEntryPoint(ws, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { - error.AlreadyReported => continue, - else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}), - }, - }; - ws.msg_queue.clearRetainingCapacity(); - } -} - -fn prepareTables( - ws: *WebServer, - run_step: *Step.Run, - coverage_id: u64, -) error{ OutOfMemory, AlreadyReported }!void { - const gpa = ws.gpa; - - ws.coverage_mutex.lock(); - defer ws.coverage_mutex.unlock(); - - const gop = try ws.coverage_files.getOrPut(gpa, coverage_id); - if (gop.found_existing) { - // We are fuzzing the same executable with multiple threads. - // Perhaps the same unit test; perhaps a different one. In any - // case, since the coverage file is the same, we only have to - // notice changes to that one file in order to learn coverage for - // this particular executable. - return; - } - errdefer _ = ws.coverage_files.pop(); - - gop.value_ptr.* = .{ - .coverage = std.debug.Coverage.init, - .mapped_memory = undefined, // populated below - .source_locations = undefined, // populated below - .entry_points = .{}, - .start_timestamp = ws.now(), - }; - errdefer gop.value_ptr.coverage.deinit(gpa); - - const rebuilt_exe_path = run_step.rebuilt_executable.?; - var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| { - log.err("step '{s}': failed to load debug information for '{f}': {s}", .{ - run_step.step.name, rebuilt_exe_path, @errorName(err), - }); - return error.AlreadyReported; - }; - defer debug_info.deinit(gpa); - - const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step.step.owner.cache_root, - .sub_path = "v/" ++ std.fmt.hex(coverage_id), - }; - var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| { - log.err("step '{s}': failed to load coverage file '{f}': {s}", .{ - run_step.step.name, coverage_file_path, @errorName(err), - }); - return error.AlreadyReported; - }; - defer coverage_file.close(); - - const file_size = coverage_file.getEndPos() catch |err| { - log.err("unable to check len of coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) }); - return error.AlreadyReported; - }; - - const mapped_memory = std.posix.mmap( - null, - file_size, - std.posix.PROT.READ, - .{ .TYPE = .SHARED }, - coverage_file.handle, - 0, - ) catch |err| { - log.err("failed to map coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) }); - return error.AlreadyReported; - }; - gop.value_ptr.mapped_memory = mapped_memory; - - const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); - errdefer gpa.free(source_locations); - - // Unfortunately the PCs array that LLVM gives us from the 8-bit PC - // counters feature is not sorted. - var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{}; - defer sorted_pcs.deinit(gpa); - try sorted_pcs.resize(gpa, pcs.len); - @memcpy(sorted_pcs.items(.pc), pcs); - for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); - sorted_pcs.sortUnstable(struct { - addrs: []const u64, - - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.addrs[a_index] < ctx.addrs[b_index]; - } - }{ .addrs = sorted_pcs.items(.pc) }); - - debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { - log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)}); - return error.AlreadyReported; - }; - - for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; - gop.value_ptr.source_locations = source_locations; - - ws.coverage_condition.broadcast(); -} - -fn addEntryPoint(ws: *WebServer, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void { - ws.coverage_mutex.lock(); - defer ws.coverage_mutex.unlock(); - - const coverage_map = ws.coverage_files.getPtr(coverage_id).?; - const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - // Since this pcs list is unsorted, we must linear scan for the best index. - const index = i: { - var best: usize = 0; - for (pcs[1..], 1..) |elem_addr, i| { - if (elem_addr == addr) break :i i; - if (elem_addr > addr) continue; - if (elem_addr > pcs[best]) best = i; - } - break :i best; - }; - if (index >= pcs.len) { - log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ - addr, pcs[0], pcs[pcs.len - 1], - }); - return error.AlreadyReported; - } - if (false) { - const sl = coverage_map.source_locations[index]; - const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ - addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], - }); - } - const gpa = ws.gpa; - try coverage_map.entry_points.append(gpa, @intCast(index)); -} - -fn makeIov(s: []const u8) std.posix.iovec_const { - return .{ - .base = s.ptr, - .len = s.len, - }; -} diff --git a/lib/std/Build/Fuzz/abi.zig b/lib/std/Build/Fuzz/abi.zig deleted file mode 100644 index a6abc13feebbacf36e71cd941f6d8069ee96c81b..0000000000000000000000000000000000000000 --- a/lib/std/Build/Fuzz/abi.zig +++ /dev/null @@ -1,112 +0,0 @@ -//! This file is shared among Zig code running in wildly different contexts: -//! libfuzzer, compiled alongside unit tests, the build runner, running on the -//! host computer, and the fuzzing web interface webassembly code running in -//! the browser. All of these components interface to some degree via an ABI. - -/// libfuzzer uses this and its usize is the one that counts. To match the ABI, -/// make the ints be the size of the target used with libfuzzer. -/// -/// Trailing: -/// * 1 bit per pc_addr, usize elements -/// * pc_addr: usize for each pcs_len -pub const SeenPcsHeader = extern struct { - n_runs: usize, - unique_runs: usize, - pcs_len: usize, - - /// Used for comptime assertions. Provides a mechanism for strategically - /// causing compile errors. - pub const trailing = .{ - .pc_bits_usize, - .pc_addr, - }; - - pub fn headerEnd(header: *const SeenPcsHeader) []const usize { - const ptr: [*]align(@alignOf(usize)) const u8 = @ptrCast(header); - const header_end_ptr: [*]const usize = @ptrCast(ptr + @sizeOf(SeenPcsHeader)); - const pcs_len = header.pcs_len; - return header_end_ptr[0 .. pcs_len + seenElemsLen(pcs_len)]; - } - - pub fn seenBits(header: *const SeenPcsHeader) []const usize { - return header.headerEnd()[0..seenElemsLen(header.pcs_len)]; - } - - pub fn seenElemsLen(pcs_len: usize) usize { - return (pcs_len + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); - } - - pub fn pcAddrs(header: *const SeenPcsHeader) []const usize { - const pcs_len = header.pcs_len; - return header.headerEnd()[seenElemsLen(pcs_len)..][0..pcs_len]; - } -}; - -pub const ToClientTag = enum(u8) { - current_time, - source_index, - coverage_update, - entry_points, - _, -}; - -pub const CurrentTime = extern struct { - tag: ToClientTag = .current_time, - /// Number of nanoseconds that all other timestamps are in reference to. - base: i64 align(1), -}; - -/// Sent to the fuzzer web client on first connection to the websocket URL. -/// -/// Trailing: -/// * std.debug.Coverage.String for each directories_len -/// * std.debug.Coverage.File for each files_len -/// * std.debug.Coverage.SourceLocation for each source_locations_len -/// * u8 for each string_bytes_len -pub const SourceIndexHeader = extern struct { - flags: Flags, - directories_len: u32, - files_len: u32, - source_locations_len: u32, - string_bytes_len: u32, - /// When, according to the server, fuzzing started. - start_timestamp: i64 align(4), - - pub const Flags = packed struct(u32) { - tag: ToClientTag = .source_index, - _: u24 = 0, - }; -}; - -/// Sent to the fuzzer web client whenever the set of covered source locations -/// changes. -/// -/// Trailing: -/// * one bit per source_locations_len, contained in u64 elements -pub const CoverageUpdateHeader = extern struct { - flags: Flags = .{}, - n_runs: u64, - unique_runs: u64, - - pub const Flags = packed struct(u64) { - tag: ToClientTag = .coverage_update, - _: u56 = 0, - }; - - pub const trailing = .{ - .pc_bits_usize, - }; -}; - -/// Sent to the fuzzer web client when the set of entry points is updated. -/// -/// Trailing: -/// * one u32 index of source_locations per locs_len -pub const EntryPointHeader = extern struct { - flags: Flags, - - pub const Flags = packed struct(u32) { - tag: ToClientTag = .entry_points, - locs_len: u24, - }; -}; diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 8583427aad0c145691b0412779de24e0103e0b00..ee883ee1522567f9f7aa96f18129082234fc0774 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -72,6 +72,14 @@ pub const MakeOptions = struct { progress_node: std.Progress.Node, thread_pool: *std.Thread.Pool, watch: bool, + web_server: switch (builtin.target.cpu.arch) { + else => ?*Build.WebServer, + // WASM code references `Build.abi` which happens to incidentally reference this type, but + // it currently breaks because `std.net.Address` doesn't work there. Work around for now. + .wasm32 => void, + }, + /// Not to be confused with `Build.allocator`, which is an alias of `Build.graph.arena`. + gpa: Allocator, }; pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; @@ -229,7 +237,17 @@ pub fn init(options: StepOptions) Step { pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { const arena = s.owner.allocator; - s.makeFn(s, options) catch |err| switch (err) { + var timer: ?std.time.Timer = t: { + if (!s.owner.graph.time_report) break :t null; + if (s.id == .compile) break :t null; + break :t std.time.Timer.start() catch @panic("--time-report not supported on this host"); + }; + const make_result = s.makeFn(s, options); + if (timer) |*t| { + options.web_server.?.updateTimeReportGeneric(s, t.read()); + } + + make_result catch |err| switch (err) { error.MakeFailed => return error.MakeFailed, error.MakeSkipped => return error.MakeSkipped, else => { @@ -372,18 +390,20 @@ pub fn evalZigProcess( argv: []const []const u8, prog_node: std.Progress.Node, watch: bool, + web_server: ?*Build.WebServer, + gpa: Allocator, ) !?Path { if (s.getZigProcess()) |zp| update: { assert(watch); if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd); - const result = zigProcessUpdate(s, zp, watch) catch |err| switch (err) { + const result = zigProcessUpdate(s, zp, watch, web_server, gpa) catch |err| switch (err) { error.BrokenPipe => { // Process restart required. const term = zp.child.wait() catch |e| { return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); }; _ = term; - s.clearZigProcess(); + s.clearZigProcess(gpa); break :update; }, else => |e| return e, @@ -398,7 +418,7 @@ pub fn evalZigProcess( return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); }; s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; - s.clearZigProcess(); + s.clearZigProcess(gpa); try handleChildProcessTerm(s, term, null, argv); return error.MakeFailed; } @@ -408,7 +428,6 @@ pub fn evalZigProcess( assert(argv.len != 0); const b = s.owner; const arena = b.allocator; - const gpa = arena; try handleChildProcUnsupported(s, null, argv); try handleVerbose(s.owner, null, argv); @@ -435,9 +454,12 @@ pub fn evalZigProcess( .progress_ipc_fd = if (std.Progress.have_ipc) child.progress_node.getIpcFd() else {}, }; if (watch) s.setZigProcess(zp); - defer if (!watch) zp.poller.deinit(); + defer if (!watch) { + zp.poller.deinit(); + gpa.destroy(zp); + }; - const result = try zigProcessUpdate(s, zp, watch); + const result = try zigProcessUpdate(s, zp, watch, web_server, gpa); if (!watch) { // Send EOF to stdin. @@ -499,7 +521,7 @@ pub fn installDir(s: *Step, dest_path: []const u8) !std.fs.Dir.MakePathStatus { }; } -fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path { +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool, web_server: ?*Build.WebServer, gpa: Allocator) !?Path { const b = s.owner; const arena = b.allocator; @@ -537,12 +559,14 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path { body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; // TODO: use @ptrCast when the compiler supports it const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); - const extra_array = try arena.alloc(u32, unaligned_extra.len); - @memcpy(extra_array, unaligned_extra); - s.result_error_bundle = .{ - .string_bytes = try arena.dupe(u8, string_bytes), - .extra = extra_array, - }; + { + s.result_error_bundle = .{ .string_bytes = &.{}, .extra = &.{} }; + errdefer s.result_error_bundle.deinit(gpa); + s.result_error_bundle.string_bytes = try gpa.dupe(u8, string_bytes); + const extra = try gpa.alloc(u32, unaligned_extra.len); + @memcpy(extra, unaligned_extra); + s.result_error_bundle.extra = extra; + } // This message indicates the end of the update. if (watch) break :poll; }, @@ -602,6 +626,20 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path { } } }, + .time_report => if (web_server) |ws| { + const TimeReport = std.zig.Server.Message.TimeReport; + const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); + ws.updateTimeReportCompile(.{ + .compile = s.cast(Step.Compile).?, + .use_llvm = tr.flags.use_llvm, + .stats = tr.stats, + .ns_total = timer.read(), + .llvm_pass_timings_len = tr.llvm_pass_timings_len, + .files_len = tr.files_len, + .decls_len = tr.decls_len, + .trailing = body[@sizeOf(TimeReport)..], + }); + }, else => {}, // ignore other messages } } @@ -630,8 +668,7 @@ fn setZigProcess(s: *Step, zp: *ZigProcess) void { } } -fn clearZigProcess(s: *Step) void { - const gpa = s.owner.allocator; +fn clearZigProcess(s: *Step, gpa: Allocator) void { switch (s.id) { .compile => { const compile = s.cast(Compile).?; @@ -947,7 +984,8 @@ fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const try gop.value_ptr.append(gpa, basename); } -fn reset(step: *Step, gpa: Allocator) void { +/// Implementation detail of file watching and forced rebuilds. Prepares the step for being re-evaluated. +pub fn reset(step: *Step, gpa: Allocator) void { assert(step.state == .precheck_done); step.result_error_msgs.clearRetainingCapacity(); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 141d18a7bff1139718ce05024dd1a1ce5bc93bc5..5e147967920e393ea493bec7ab0bae71cd2a07c3 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1491,6 +1491,7 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { if (b.verbose_link or compile.verbose_link) try zig_args.append("--verbose-link"); if (b.verbose_cc or compile.verbose_cc) try zig_args.append("--verbose-cc"); if (b.verbose_llvm_cpu_features) try zig_args.append("--verbose-llvm-cpu-features"); + if (b.graph.time_report) try zig_args.append("--time-report"); if (compile.generated_asm != null) try zig_args.append("-femit-asm"); if (compile.generated_bin == null) try zig_args.append("-fno-emit-bin"); @@ -1851,6 +1852,8 @@ fn make(step: *Step, options: Step.MakeOptions) !void { zig_args, options.progress_node, (b.graph.incremental == true) and options.watch, + options.web_server, + options.gpa, ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); @@ -1905,9 +1908,7 @@ fn outputPath(c: *Compile, out_dir: std.Build.Cache.Path, ea: std.zig.EmitArtifa return out_dir.joinString(arena, name) catch @panic("OOM"); } -pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path { - const gpa = c.step.owner.allocator; - +pub fn rebuildInFuzzMode(c: *Compile, gpa: Allocator, progress_node: std.Progress.Node) !Path { c.step.result_error_msgs.clearRetainingCapacity(); c.step.result_stderr = ""; @@ -1915,7 +1916,7 @@ pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path { c.step.result_error_bundle = std.zig.ErrorBundle.empty; const zig_args = try getZigArgs(c, true); - const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false, null, gpa); return maybe_output_bin_path.?; } diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index 74f871d2fc958e5db8af18b1c86d27bb424aa424..3c9eb2eaa7f1674c2a19070c25d4d0e7fedb6af2 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -236,7 +236,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { try argv.appendSlice(&.{ full_src_path, full_dest_path }); try argv.append("--listen=-"); - _ = try step.evalZigProcess(argv.items, prog_node, false); + _ = try step.evalZigProcess(argv.items, prog_node, false, options.web_server, options.gpa); objcopy.output_file.path = full_dest_path; if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug; diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 6f8c40b1d57d9378fea2fe8b26e52185067a2c06..fd6194f7ff7bd698fbf82ace71cfabd3af2a7299 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -549,6 +549,7 @@ test Options { .result = try std.zig.system.resolveTargetQuery(.{}), }, .zig_lib_directory = std.Build.Cache.Directory.cwd(), + .time_report = false, }; var builder = try std.Build.create( diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 819fc6745d4e10f94f7aca0b5e1cd1350d5ff1b0..d317422bd9abb91af84258239270d2547373baf4 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -944,7 +944,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { pub fn rerunInFuzzMode( run: *Run, - web_server: *std.Build.Fuzz.WebServer, + fuzz: *std.Build.Fuzz, unit_test_index: u32, prog_node: std.Progress.Node, ) !void { @@ -984,7 +984,7 @@ pub fn rerunInFuzzMode( const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, .{ .unit_test_index = unit_test_index, - .web_server = web_server, + .fuzz = fuzz, }); } @@ -1054,7 +1054,7 @@ fn termMatches(expected: ?std.process.Child.Term, actual: std.process.Child.Term } const FuzzContext = struct { - web_server: *std.Build.Fuzz.WebServer, + fuzz: *std.Build.Fuzz, unit_test_index: u32, }; @@ -1638,31 +1638,31 @@ fn evalZigTest( }; }, .coverage_id => { - const web_server = fuzz_context.?.web_server; + const fuzz = fuzz_context.?.fuzz; const msg_ptr: *align(1) const u64 = @ptrCast(body); coverage_id = msg_ptr.*; { - web_server.mutex.lock(); - defer web_server.mutex.unlock(); - try web_server.msg_queue.append(web_server.gpa, .{ .coverage = .{ + fuzz.queue_mutex.lock(); + defer fuzz.queue_mutex.unlock(); + try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .coverage = .{ .id = coverage_id.?, .run = run, } }); - web_server.condition.signal(); + fuzz.queue_cond.signal(); } }, .fuzz_start_addr => { - const web_server = fuzz_context.?.web_server; + const fuzz = fuzz_context.?.fuzz; const msg_ptr: *align(1) const u64 = @ptrCast(body); const addr = msg_ptr.*; { - web_server.mutex.lock(); - defer web_server.mutex.unlock(); - try web_server.msg_queue.append(web_server.gpa, .{ .entry_point = .{ + fuzz.queue_mutex.lock(); + defer fuzz.queue_mutex.unlock(); + try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .entry_point = .{ .addr = addr, .coverage_id = coverage_id.?, } }); - web_server.condition.signal(); + fuzz.queue_cond.signal(); } }, else => {}, // ignore other messages diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 7187aaf8c68a9806b84b13463748eb20a9cdb8cb..53c4007e26e724f61f599b66a382132dd686f67a 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -187,7 +187,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { const c_source_path = translate_c.source.getPath2(b, step); try argv_list.append(c_source_path); - const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false); + const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false, options.web_server, options.gpa); const basename = std.fs.path.stem(std.fs.path.basename(c_source_path)); translate_c.out_basename = b.fmt("{s}.zig", .{basename}); diff --git a/lib/std/Build/WebServer.zig b/lib/std/Build/WebServer.zig new file mode 100644 index 0000000000000000000000000000000000000000..9264d7473c61fe7db8b0b197ca7ded0980ef83a1 --- /dev/null +++ b/lib/std/Build/WebServer.zig @@ -0,0 +1,823 @@ +gpa: Allocator, +thread_pool: *std.Thread.Pool, +graph: *const Build.Graph, +all_steps: []const *Build.Step, +listen_address: std.net.Address, +ttyconf: std.io.tty.Config, +root_prog_node: std.Progress.Node, +watch: bool, + +tcp_server: ?std.net.Server, +serve_thread: ?std.Thread, + +base_timestamp: i128, +/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. +step_names_trailing: []u8, + +/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. +/// Accessed atomically. +step_status_bits: []u8, + +fuzz: ?Fuzz, +time_report_mutex: std.Thread.Mutex, +time_report_msgs: [][]u8, +time_report_update_times: []i64, + +build_status: std.atomic.Value(abi.BuildStatus), +/// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` +/// to increment this value. Each client thread waits for this increment with `std.Thread.Futex`, so +/// `notifyUpdate` will wake those threads. Updates are sent on a short interval regardless, so it +/// is recommended to only use `notifyUpdate` for changes which the user should see immediately. For +/// instance, we do not call `notifyUpdate` when the number of "unique runs" in the fuzzer changes, +/// because this value changes quickly so this would result in constantly spamming all clients with +/// an unreasonable number of packets. +update_id: std.atomic.Value(u32), + +runner_request_mutex: std.Thread.Mutex, +runner_request_ready_cond: std.Thread.Condition, +runner_request_empty_cond: std.Thread.Condition, +runner_request: ?RunnerRequest, + +/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates +/// on a fixed interval of this many milliseconds. +const default_update_interval_ms = 500; + +/// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. +pub fn notifyUpdate(ws: *WebServer) void { + _ = ws.update_id.rmw(.Add, 1, .release); + std.Thread.Futex.wake(&ws.update_id, 16); +} + +pub const Options = struct { + gpa: Allocator, + thread_pool: *std.Thread.Pool, + graph: *const std.Build.Graph, + all_steps: []const *Build.Step, + ttyconf: std.io.tty.Config, + root_prog_node: std.Progress.Node, + watch: bool, + listen_address: std.net.Address, +}; +pub fn init(opts: Options) WebServer { + if (builtin.single_threaded) { + // The upcoming `std.Io` interface should allow us to use `Io.async` and `Io.concurrent` + // instead of threads, so that the web server can function in single-threaded builds. + std.process.fatal("--webui not yet implemented for single-threaded builds", .{}); + } + + if (builtin.os.tag == .windows) { + // At the time of writing, there are two bugs in the standard library which break this feature on Windows: + // * Reading from a socket on one thread while writing to it on another seems to deadlock. + // * Vectored writes to sockets currently trigger an infinite loop when a buffer has length 0. + // + // Both of these bugs are expected to be solved by changes which are currently in the unmerged + // 'wrangle-writer-buffering' branch. Until that makes it in, this must remain disabled. + std.process.fatal("--webui is currently disabled on Windows due to bugs", .{}); + } + + const all_steps = opts.all_steps; + + const step_names_trailing = opts.gpa.alloc(u8, len: { + var name_bytes: usize = 0; + for (all_steps) |step| name_bytes += step.name.len; + break :len name_bytes + all_steps.len * 4; + }) catch @panic("out of memory"); + { + const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); + var idx: usize = all_steps.len * 4; + for (all_steps, step_name_lens) |step, *name_len| { + name_len.* = @intCast(step.name.len); + @memcpy(step_names_trailing[idx..][0..step.name.len], step.name); + idx += step.name.len; + } + assert(idx == step_names_trailing.len); + } + + const step_status_bits = opts.gpa.alloc( + u8, + std.math.divCeil(usize, all_steps.len, 4) catch unreachable, + ) catch @panic("out of memory"); + @memset(step_status_bits, 0); + + const time_reports_len: usize = if (opts.graph.time_report) all_steps.len else 0; + const time_report_msgs = opts.gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); + const time_report_update_times = opts.gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); + @memset(time_report_msgs, &.{}); + @memset(time_report_update_times, std.math.minInt(i64)); + + return .{ + .gpa = opts.gpa, + .thread_pool = opts.thread_pool, + .graph = opts.graph, + .all_steps = all_steps, + .listen_address = opts.listen_address, + .ttyconf = opts.ttyconf, + .root_prog_node = opts.root_prog_node, + .watch = opts.watch, + + .tcp_server = null, + .serve_thread = null, + + .base_timestamp = std.time.nanoTimestamp(), + .step_names_trailing = step_names_trailing, + + .step_status_bits = step_status_bits, + + .fuzz = null, + .time_report_mutex = .{}, + .time_report_msgs = time_report_msgs, + .time_report_update_times = time_report_update_times, + + .build_status = .init(.idle), + .update_id = .init(0), + + .runner_request_mutex = .{}, + .runner_request_ready_cond = .{}, + .runner_request_empty_cond = .{}, + .runner_request = null, + }; +} +pub fn deinit(ws: *WebServer) void { + const gpa = ws.gpa; + + gpa.free(ws.step_names_trailing); + gpa.free(ws.step_status_bits); + + if (ws.fuzz) |*f| f.deinit(); + for (ws.time_report_msgs) |msg| gpa.free(msg); + gpa.free(ws.time_report_msgs); + gpa.free(ws.time_report_update_times); + + if (ws.serve_thread) |t| { + if (ws.tcp_server) |*s| s.stream.close(); + t.join(); + } + if (ws.tcp_server) |*s| s.deinit(); + + gpa.free(ws.step_names_trailing); +} +pub fn start(ws: *WebServer) error{AlreadyReported}!void { + assert(ws.tcp_server == null); + assert(ws.serve_thread == null); + + ws.tcp_server = ws.listen_address.listen(.{ .reuse_address = true }) catch |err| { + log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.getPort(), @errorName(err) }); + return error.AlreadyReported; + }; + ws.serve_thread = std.Thread.spawn(.{}, serve, .{ws}) catch |err| { + log.err("unable to spawn web server thread: {s}", .{@errorName(err)}); + ws.tcp_server.?.deinit(); + ws.tcp_server = null; + return error.AlreadyReported; + }; + + log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.listen_address}); + if (ws.listen_address.getPort() == 0) { + log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.listen_address}); + } +} +fn serve(ws: *WebServer) void { + while (true) { + const connection = ws.tcp_server.?.accept() catch |err| { + log.err("failed to accept connection: {s}", .{@errorName(err)}); + return; + }; + _ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch |err| { + log.err("unable to spawn connection thread: {s}", .{@errorName(err)}); + connection.stream.close(); + continue; + }; + } +} + +pub fn startBuild(ws: *WebServer) void { + if (ws.fuzz) |*fuzz| { + fuzz.deinit(); + ws.fuzz = null; + } + for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); + ws.build_status.store(.running, .monotonic); + ws.notifyUpdate(); +} + +pub fn updateStepStatus(ws: *WebServer, step: *Build.Step, new_status: abi.StepUpdate.Status) void { + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == step) break @intCast(i); + } else unreachable; + const ptr = &ws.step_status_bits[step_idx / 4]; + const bit_offset: u3 = @intCast((step_idx % 4) * 2); + const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); + const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; + _ = @atomicRmw(u8, ptr, .Xor, mask, .monotonic); + ws.notifyUpdate(); +} + +pub fn finishBuild(ws: *WebServer, opts: struct { + fuzz: bool, +}) void { + if (opts.fuzz) { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => std.process.fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + assert(ws.fuzz == null); + + ws.build_status.store(.fuzz_init, .monotonic); + ws.notifyUpdate(); + + ws.fuzz = Fuzz.init(ws) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)}); + ws.fuzz.?.start(); + } + + ws.build_status.store(if (ws.watch) .watching else .idle, .monotonic); + ws.notifyUpdate(); +} + +pub fn now(s: *const WebServer) i64 { + return @intCast(std.time.nanoTimestamp() - s.base_timestamp); +} + +fn accept(ws: *WebServer, connection: std.net.Server.Connection) void { + defer connection.stream.close(); + + var read_buf: [0x4000]u8 = undefined; + var server: std.http.Server = .init(connection, &read_buf); + + while (true) { + var request = server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => { + log.err("failed to receive http request: {s}", .{@errorName(err)}); + return; + }, + }; + var ws_send_buf: [0x4000]u8 = undefined; + var ws_recv_buf: [0x4000]u8 align(4) = undefined; + if (std.http.WebSocket.init(&request, &ws_send_buf, &ws_recv_buf) catch |err| { + log.err("failed to initialize websocket connection: {s}", .{@errorName(err)}); + return; + }) |ws_init| { + var web_socket = ws_init; + ws.serveWebSocket(&web_socket) catch |err| { + log.err("failed to serve websocket: {s}", .{@errorName(err)}); + return; + }; + comptime unreachable; + } else { + ws.serveRequest(&request) catch |err| switch (err) { + error.AlreadyReported => return, + else => { + log.err("failed to serve '{s}': {s}", .{ request.head.target, @errorName(err) }); + return; + }, + }; + } + } +} + +fn makeIov(s: []const u8) std.posix.iovec_const { + return .{ + .base = s.ptr, + .len = s.len, + }; +} +fn serveWebSocket(ws: *WebServer, sock: *std.http.WebSocket) !noreturn { + var prev_build_status = ws.build_status.load(.monotonic); + + const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len); + defer ws.gpa.free(prev_step_status_bits); + for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { + copy.* = @atomicLoad(u8, shared, .monotonic); + } + + _ = try std.Thread.spawn(.{}, recvWebSocketMessages, .{ ws, sock }); + + { + const hello_header: abi.Hello = .{ + .status = prev_build_status, + .flags = .{ + .time_report = ws.graph.time_report, + }, + .timestamp = ws.now(), + .steps_len = @intCast(ws.all_steps.len), + }; + try sock.writeMessagev(&.{ + makeIov(@ptrCast(&hello_header)), + makeIov(ws.step_names_trailing), + makeIov(prev_step_status_bits), + }, .binary); + } + + var prev_fuzz: Fuzz.Previous = .init; + var prev_time: i64 = std.math.minInt(i64); + while (true) { + const start_time = ws.now(); + const start_update_id = ws.update_id.load(.acquire); + + if (ws.fuzz) |*fuzz| { + try fuzz.sendUpdate(sock, &prev_fuzz); + } + + { + ws.time_report_mutex.lock(); + defer ws.time_report_mutex.unlock(); + for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { + if (update_time <= prev_time) continue; + // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so + // that we don't hold up the build system on the client accepting this packet. + const owned_msg = try ws.gpa.dupe(u8, msg); + defer ws.gpa.free(owned_msg); + // Temporarily unlock, then re-lock after the message is sent. + ws.time_report_mutex.unlock(); + defer ws.time_report_mutex.lock(); + try sock.writeMessage(msg, .binary); + } + } + + { + const build_status = ws.build_status.load(.monotonic); + if (build_status != prev_build_status) { + prev_build_status = build_status; + const msg: abi.StatusUpdate = .{ .new = build_status }; + try sock.writeMessage(@ptrCast(&msg), .binary); + } + } + + for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { + const cur_byte = @atomicLoad(u8, shared, .monotonic); + if (prev_byte.* == cur_byte) continue; + const cur: [4]abi.StepUpdate.Status = .{ + @enumFromInt(@as(u2, @truncate(cur_byte >> 0))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 2))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 4))), + @enumFromInt(@as(u2, @truncate(cur_byte >> 6))), + }; + const prev: [4]abi.StepUpdate.Status = .{ + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 0))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 2))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 4))), + @enumFromInt(@as(u2, @truncate(prev_byte.* >> 6))), + }; + for (cur, prev, byte_idx * 4..) |cur_status, prev_status, step_idx| { + const msg: abi.StepUpdate = .{ .step_idx = @intCast(step_idx), .bits = .{ .status = cur_status } }; + if (cur_status != prev_status) try sock.writeMessage(@ptrCast(&msg), .binary); + } + prev_byte.* = cur_byte; + } + + prev_time = start_time; + std.Thread.Futex.timedWait(&ws.update_id, start_update_id, std.time.ns_per_ms * default_update_interval_ms) catch {}; + } +} +fn recvWebSocketMessages(ws: *WebServer, sock: *std.http.WebSocket) void { + while (true) { + const msg = sock.readSmallMessage() catch return; + if (msg.opcode != .binary) continue; + if (msg.data.len == 0) continue; + const tag: abi.ToServerTag = @enumFromInt(msg.data[0]); + switch (tag) { + _ => continue, + .rebuild => while (true) { + ws.runner_request_mutex.lock(); + defer ws.runner_request_mutex.unlock(); + if (ws.runner_request == null) { + ws.runner_request = .rebuild; + ws.runner_request_ready_cond.signal(); + break; + } + ws.runner_request_empty_cond.wait(&ws.runner_request_mutex); + }, + } + } +} + +fn serveRequest(ws: *WebServer, req: *std.http.Server.Request) !void { + // Strip an optional leading '/debug' component from the request. + const target: []const u8, const debug: bool = target: { + if (mem.eql(u8, req.head.target, "/debug")) break :target .{ "/", true }; + if (mem.eql(u8, req.head.target, "/debug/")) break :target .{ "/", true }; + if (mem.startsWith(u8, req.head.target, "/debug/")) break :target .{ req.head.target["/debug".len..], true }; + break :target .{ req.head.target, false }; + }; + + if (mem.eql(u8, target, "/")) return serveLibFile(ws, req, "build-web/index.html", "text/html"); + if (mem.eql(u8, target, "/main.js")) return serveLibFile(ws, req, "build-web/main.js", "application/javascript"); + if (mem.eql(u8, target, "/style.css")) return serveLibFile(ws, req, "build-web/style.css", "text/css"); + if (mem.eql(u8, target, "/time_report.css")) return serveLibFile(ws, req, "build-web/time_report.css", "text/css"); + if (mem.eql(u8, target, "/main.wasm")) return serveClientWasm(ws, req, if (debug) .Debug else .ReleaseFast); + + if (ws.fuzz) |*fuzz| { + if (mem.eql(u8, target, "/sources.tar")) return fuzz.serveSourcesTar(req); + } + + try req.respond("not found", .{ + .status = .not_found, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "text/plain" }, + }, + }); +} + +fn serveLibFile( + ws: *WebServer, + request: *std.http.Server.Request, + sub_path: []const u8, + content_type: []const u8, +) !void { + return serveFile(ws, request, .{ + .root_dir = ws.graph.zig_lib_directory, + .sub_path = sub_path, + }, content_type); +} +fn serveClientWasm( + ws: *WebServer, + req: *std.http.Server.Request, + optimize_mode: std.builtin.OptimizeMode, +) !void { + var arena_state: std.heap.ArenaAllocator = .init(ws.gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // We always rebuild the wasm on-the-fly, so that if it is edited the user can just refresh the page. + const bin_path = try buildClientWasm(ws, arena, optimize_mode); + return serveFile(ws, req, bin_path, "application/wasm"); +} + +pub fn serveFile( + ws: *WebServer, + request: *std.http.Server.Request, + path: Cache.Path, + content_type: []const u8, +) !void { + const gpa = ws.gpa; + // The desired API is actually sendfile, which will require enhancing std.http.Server. + // We load the file with every request so that the user can make changes to the file + // and refresh the HTML page without restarting this server. + const file_contents = path.root_dir.handle.readFileAlloc(gpa, path.sub_path, 10 * 1024 * 1024) catch |err| { + log.err("failed to read '{f}': {s}", .{ path, @errorName(err) }); + return error.AlreadyReported; + }; + defer gpa.free(file_contents); + try request.respond(file_contents, .{ + .extra_headers = &.{ + .{ .name = "Content-Type", .value = content_type }, + cache_control_header, + }, + }); +} +pub fn serveTarFile( + ws: *WebServer, + request: *std.http.Server.Request, + paths: []const Cache.Path, +) !void { + const gpa = ws.gpa; + + var send_buf: [0x4000]u8 = undefined; + var response = request.respondStreaming(.{ + .send_buffer = &send_buf, + .respond_options = .{ + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-tar" }, + cache_control_header, + }, + }, + }); + + var cached_cwd_path: ?[]const u8 = null; + defer if (cached_cwd_path) |p| gpa.free(p); + + var response_buf: [1024]u8 = undefined; + var adapter = response.writer().adaptToNewApi(); + adapter.new_interface.buffer = &response_buf; + var archiver: std.tar.Writer = .{ .underlying_writer = &adapter.new_interface }; + + for (paths) |path| { + var file = path.root_dir.handle.openFile(path.sub_path, .{}) catch |err| { + log.err("failed to open '{f}': {s}", .{ path, @errorName(err) }); + continue; + }; + defer file.close(); + const stat = try file.stat(); + var read_buffer: [1024]u8 = undefined; + var file_reader: std.fs.File.Reader = .initSize(file, &read_buffer, stat.size); + + // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can + // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: + // it turns out the WASM treats the first path component as the module name, typically + // resulting in modules named "" and "src". The compiler needs to tell the build system + // about the module graph so that the build system can correctly encode this information in + // the tar file. + archiver.prefix = path.root_dir.path orelse cwd: { + if (cached_cwd_path == null) cached_cwd_path = try std.process.getCwdAlloc(gpa); + break :cwd cached_cwd_path.?; + }; + try archiver.writeFile(path.sub_path, &file_reader, stat.mtime); + } + + // intentionally not calling `archiver.finishPedantically` + try adapter.new_interface.flush(); + try response.end(); +} + +fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.OptimizeMode) !Cache.Path { + const root_name = "build-web"; + const arch_os_abi = "wasm32-freestanding"; + const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; + + const gpa = ws.gpa; + const graph = ws.graph; + + const main_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "build-web/main.zig", + }; + const walk_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "docs/wasm/Walk.zig", + }; + const html_render_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "docs/wasm/html_render.zig", + }; + + var argv: std.ArrayListUnmanaged([]const u8) = .empty; + + try argv.appendSlice(arena, &.{ + graph.zig_exe, "build-exe", // + "-fno-entry", // + "-O", @tagName(optimize), // + "-target", arch_os_abi, // + "-mcpu", cpu_features, // + "--cache-dir", graph.global_cache_root.path orelse ".", // + "--global-cache-dir", graph.global_cache_root.path orelse ".", // + "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // + "--name", root_name, // + "-rdynamic", // + "-fsingle-threaded", // + "--dep", "Walk", // + "--dep", "html_render", // + try std.fmt.allocPrint(arena, "-Mroot={f}", .{main_src_path}), // + try std.fmt.allocPrint(arena, "-MWalk={f}", .{walk_src_path}), // + "--dep", "Walk", // + try std.fmt.allocPrint(arena, "-Mhtml_render={f}", .{html_render_src_path}), // + "--listen=-", + }); + + var child: std.process.Child = .init(argv.items, gpa); + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + try child.spawn(); + + var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try child.stdin.?.writeAll(@ptrCast(@as([]const std.zig.Client.Message.Header, &.{ + .{ .tag = .update, .bytes_len = 0 }, + .{ .tag = .exit, .bytes_len = 0 }, + }))); + + const Header = std.zig.Server.Message.Header; + var result: ?Cache.Path = null; + var result_error_bundle = std.zig.ErrorBundle.empty; + + const stdout = poller.reader(.stdout); + + poll: while (true) { + while (stdout.buffered().len < @sizeOf(Header)) if (!(try poller.poll())) break :poll; + const header = stdout.takeStruct(Header, .little) catch unreachable; + while (stdout.buffered().len < header.bytes_len) if (!try poller.poll()) break :poll; + const body = stdout.take(header.bytes_len) catch unreachable; + + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return error.ZigProtocolVersionMismatch; + } + }, + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + const unaligned_extra: []align(1) const u32 = @ptrCast(extra_bytes); + const extra_array = try arena.alloc(u32, unaligned_extra.len); + @memcpy(extra_array, unaligned_extra); + result_error_bundle = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .extra = extra_array, + }; + }, + .emit_digest => { + const EmitDigest = std.zig.Server.Message.EmitDigest; + const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); + if (!ebp_hdr.flags.cache_hit) { + log.info("source changes detected; rebuilt wasm component", .{}); + } + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + result = .{ + .root_dir = graph.global_cache_root, + .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), + }; + }, + else => {}, // ignore other messages + } + } + + const stderr_contents = try poller.toOwnedSlice(.stderr); + if (stderr_contents.len > 0) { + std.debug.print("{s}", .{stderr_contents}); + } + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + switch (try child.wait()) { + .Exited => |code| { + if (code != 0) { + log.err( + "the following command exited with error code {d}:\n{s}", + .{ code, try Build.Step.allocPrintCmd(arena, null, argv.items) }, + ); + return error.WasmCompilationFailed; + } + }, + .Signal, .Stopped, .Unknown => { + log.err( + "the following command terminated unexpectedly:\n{s}", + .{try Build.Step.allocPrintCmd(arena, null, argv.items)}, + ); + return error.WasmCompilationFailed; + }, + } + + if (result_error_bundle.errorMessageCount() > 0) { + const color = std.zig.Color.auto; + result_error_bundle.renderToStdErr(color.renderOptions()); + log.err("the following command failed with {d} compilation errors:\n{s}", .{ + result_error_bundle.errorMessageCount(), + try Build.Step.allocPrintCmd(arena, null, argv.items), + }); + return error.WasmCompilationFailed; + } + + const base_path = result orelse { + log.err("child process failed to report result\n{s}", .{ + try Build.Step.allocPrintCmd(arena, null, argv.items), + }); + return error.WasmCompilationFailed; + }; + const bin_name = try std.zig.binNameAlloc(arena, .{ + .root_name = root_name, + .target = &(std.zig.system.resolveTargetQuery(std.Build.parseTargetQuery(.{ + .arch_os_abi = arch_os_abi, + .cpu_features = cpu_features, + }) catch unreachable) catch unreachable), + .output_mode = .Exe, + }); + return base_path.join(arena, bin_name); +} + +pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { + compile: *Build.Step.Compile, + + use_llvm: bool, + stats: abi.time_report.CompileResult.Stats, + ns_total: u64, + + llvm_pass_timings_len: u32, + files_len: u32, + decls_len: u32, + + /// The trailing data of `abi.time_report.CompileResult`, except the step name. + trailing: []const u8, +}) void { + const gpa = ws.gpa; + + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == &opts.compile.step) break @intCast(i); + } else unreachable; + + const old_buf = old: { + ws.time_report_mutex.lock(); + defer ws.time_report_mutex.unlock(); + const old = ws.time_report_msgs[step_idx]; + ws.time_report_msgs[step_idx] = &.{}; + break :old old; + }; + const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); + + const out_header: *align(1) abi.time_report.CompileResult = @ptrCast(buf[0..@sizeOf(abi.time_report.CompileResult)]); + out_header.* = .{ + .step_idx = step_idx, + .flags = .{ + .use_llvm = opts.use_llvm, + }, + .stats = opts.stats, + .ns_total = opts.ns_total, + .llvm_pass_timings_len = opts.llvm_pass_timings_len, + .files_len = opts.files_len, + .decls_len = opts.decls_len, + }; + @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); + + { + ws.time_report_mutex.lock(); + defer ws.time_report_mutex.unlock(); + assert(ws.time_report_msgs[step_idx].len == 0); + ws.time_report_msgs[step_idx] = buf; + ws.time_report_update_times[step_idx] = ws.now(); + } + ws.notifyUpdate(); +} + +pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, ns_total: u64) void { + const gpa = ws.gpa; + + const step_idx: u32 = for (ws.all_steps, 0..) |s, i| { + if (s == step) break @intCast(i); + } else unreachable; + + const old_buf = old: { + ws.time_report_mutex.lock(); + defer ws.time_report_mutex.unlock(); + const old = ws.time_report_msgs[step_idx]; + ws.time_report_msgs[step_idx] = &.{}; + break :old old; + }; + const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); + const out: *align(1) abi.time_report.GenericResult = @ptrCast(buf); + out.* = .{ + .step_idx = step_idx, + .ns_total = ns_total, + }; + { + ws.time_report_mutex.lock(); + defer ws.time_report_mutex.unlock(); + assert(ws.time_report_msgs[step_idx].len == 0); + ws.time_report_msgs[step_idx] = buf; + ws.time_report_update_times[step_idx] = ws.now(); + } + ws.notifyUpdate(); +} + +const RunnerRequest = union(enum) { + rebuild, +}; +pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { + ws.runner_request_mutex.lock(); + defer ws.runner_request_mutex.unlock(); + if (ws.runner_request) |req| { + ws.runner_request = null; + ws.runner_request_empty_cond.signal(); + return req; + } + return null; +} +pub fn wait(ws: *WebServer) RunnerRequest { + ws.runner_request_mutex.lock(); + defer ws.runner_request_mutex.unlock(); + while (true) { + if (ws.runner_request) |req| { + ws.runner_request = null; + ws.runner_request_empty_cond.signal(); + return req; + } + ws.runner_request_ready_cond.wait(&ws.runner_request_mutex); + } +} + +const cache_control_header: std.http.Header = .{ + .name = "Cache-Control", + .value = "max-age=0, must-revalidate", +}; + +const builtin = @import("builtin"); +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const log = std.log.scoped(.web_server); +const Allocator = std.mem.Allocator; +const Build = std.Build; +const Cache = Build.Cache; +const Fuzz = Build.Fuzz; +const abi = Build.abi; + +const WebServer = @This(); diff --git a/lib/std/Build/abi.zig b/lib/std/Build/abi.zig new file mode 100644 index 0000000000000000000000000000000000000000..d5b02d951abfd83f6323fad316a95a4d4ec7ac3f --- /dev/null +++ b/lib/std/Build/abi.zig @@ -0,0 +1,313 @@ +//! This file is shared among Zig code running in wildly different contexts: +//! * The build runner, running on the host computer +//! * The build system web interface Wasm code, running in the browser +//! * `libfuzzer`, compiled alongside unit tests +//! +//! All of these components interface to some degree via an ABI: +//! * The build runner communicates with the web interface over a WebSocket connection +//! * The build runner communicates with `libfuzzer` over a shared memory-mapped file + +// Check that no WebSocket message type has implicit padding bits. This ensures we never send any +// undefined bits over the wire, and also helps validate that the layout doesn't differ between, for +// instance, the web server in `std.Build` and the Wasm client. +comptime { + const check = struct { + fn check(comptime T: type) void { + const std = @import("std"); + std.debug.assert(@typeInfo(T) == .@"struct"); + std.debug.assert(@typeInfo(T).@"struct".layout == .@"extern"); + std.debug.assert(std.meta.hasUniqueRepresentation(T)); + } + }.check; + + // server->client + check(Hello); + check(StatusUpdate); + check(StepUpdate); + check(fuzz.SourceIndexHeader); + check(fuzz.CoverageUpdateHeader); + check(fuzz.EntryPointHeader); + check(time_report.GenericResult); + check(time_report.CompileResult); + + // client->server + check(Rebuild); +} + +/// All WebSocket messages sent by the server to the client begin with a `ToClientTag` byte. This +/// enum is non-exhaustive only to avoid Illegal Behavior when malformed messages are sent over the +/// socket; unnamed tags are an error condition and should terminate the connection. +/// +/// Every tag has a curresponding `extern struct` representing the full message (or a header of the +/// message if it is variable-length). For instance, `.hello` corresponds to `Hello`. +/// +/// When introducing a tag, make sure to add a corresponding `extern struct` whose first field is +/// this enum, and `check` its layout in the `comptime` block above. +pub const ToClientTag = enum(u8) { + hello, + status_update, + step_update, + + // `--fuzz` + fuzz_source_index, + fuzz_coverage_update, + fuzz_entry_points, + + // `--time-report` + time_report_generic_result, + time_report_compile_result, + + _, +}; + +/// Like `ToClientTag`, but for messages sent by the client to the server. +pub const ToServerTag = enum(u8) { + rebuild, + + _, +}; + +/// The current overall status of the build runner. +/// Keep in sync with indices in web UI `main.js:updateBuildStatus`. +pub const BuildStatus = enum(u8) { + idle, + watching, + running, + fuzz_init, +}; + +/// WebSocket server->client. +/// +/// Sent by the server as the first message after a WebSocket connection opens to provide basic +/// information about the server, the build graph, etc. +/// +/// Trailing: +/// * `step_name_len: u32` for each `steps_len` +/// * `step_name: [step_name_len]u8` for each `step_name_len` +/// * `step_status: u8` for every 4 `steps_len`; every 2 bits is a `StepUpdate.Status`, LSBs first +pub const Hello = extern struct { + tag: ToClientTag = .hello, + + status: BuildStatus, + flags: Flags, + + /// Any message containing a timestamp represents it as a number of nanoseconds relative to when + /// the build began. This field is the current timestamp, represented in that form. + timestamp: i64 align(4), + + /// The number of steps in the build graph which are reachable from the top-level step[s] being + /// run; in other words, the number of steps which will be executed by this build. The name of + /// each step trails this message. + steps_len: u32 align(1), + + pub const Flags = packed struct(u16) { + /// Whether time reporting is enabled. + time_report: bool, + _: u15 = 0, + }; +}; +/// WebSocket server->client. +/// +/// Indicates that the build status has changed. +pub const StatusUpdate = extern struct { + tag: ToClientTag = .status_update, + new: BuildStatus, +}; +/// WebSocket server->client. +/// +/// Indicates a change in a step's status. +pub const StepUpdate = extern struct { + tag: ToClientTag = .step_update, + step_idx: u32 align(1), + bits: packed struct(u8) { + status: Status, + _: u6 = 0, + }, + /// Keep in sync with indices in web UI `main.js:updateStepStatus`. + pub const Status = enum(u2) { + pending, + wip, + success, + failure, + }; +}; + +pub const Rebuild = extern struct { + tag: ToServerTag = .rebuild, +}; + +/// ABI bits specifically relating to the fuzzer interface. +pub const fuzz = struct { + /// libfuzzer uses this and its usize is the one that counts. To match the ABI, + /// make the ints be the size of the target used with libfuzzer. + /// + /// Trailing: + /// * 1 bit per pc_addr, usize elements + /// * pc_addr: usize for each pcs_len + pub const SeenPcsHeader = extern struct { + n_runs: usize, + unique_runs: usize, + pcs_len: usize, + + /// Used for comptime assertions. Provides a mechanism for strategically + /// causing compile errors. + pub const trailing = .{ + .pc_bits_usize, + .pc_addr, + }; + + pub fn headerEnd(header: *const SeenPcsHeader) []const usize { + const ptr: [*]align(@alignOf(usize)) const u8 = @ptrCast(header); + const header_end_ptr: [*]const usize = @ptrCast(ptr + @sizeOf(SeenPcsHeader)); + const pcs_len = header.pcs_len; + return header_end_ptr[0 .. pcs_len + seenElemsLen(pcs_len)]; + } + + pub fn seenBits(header: *const SeenPcsHeader) []const usize { + return header.headerEnd()[0..seenElemsLen(header.pcs_len)]; + } + + pub fn seenElemsLen(pcs_len: usize) usize { + return (pcs_len + @bitSizeOf(usize) - 1) / @bitSizeOf(usize); + } + + pub fn pcAddrs(header: *const SeenPcsHeader) []const usize { + const pcs_len = header.pcs_len; + return header.headerEnd()[seenElemsLen(pcs_len)..][0..pcs_len]; + } + }; + + /// WebSocket server->client. + /// + /// Sent once, when fuzzing starts, to indicate the available coverage data. + /// + /// Trailing: + /// * std.debug.Coverage.String for each directories_len + /// * std.debug.Coverage.File for each files_len + /// * std.debug.Coverage.SourceLocation for each source_locations_len + /// * u8 for each string_bytes_len + pub const SourceIndexHeader = extern struct { + tag: ToClientTag = .fuzz_source_index, + _: [3]u8 = @splat(0), + directories_len: u32, + files_len: u32, + source_locations_len: u32, + string_bytes_len: u32, + /// When, according to the server, fuzzing started. + start_timestamp: i64 align(4), + }; + + /// WebSocket server->client. + /// + /// Sent whenever the set of covered source locations is updated. + /// + /// Trailing: + /// * one bit per source_locations_len, contained in u64 elements + pub const CoverageUpdateHeader = extern struct { + tag: ToClientTag = .fuzz_coverage_update, + _: [7]u8 = @splat(0), + n_runs: u64, + unique_runs: u64, + + pub const trailing = .{ + .pc_bits_usize, + }; + }; + + /// WebSocket server->client. + /// + /// Sent whenever the set of entry points is updated. + /// + /// Trailing: + /// * one u32 index of source_locations per locsLen() + pub const EntryPointHeader = extern struct { + tag: ToClientTag = .fuzz_entry_points, + locs_len_raw: [3]u8, + + pub fn locsLen(hdr: EntryPointHeader) u24 { + return @bitCast(hdr.locs_len_raw); + } + pub fn init(locs_len: u24) EntryPointHeader { + return .{ .locs_len_raw = @bitCast(locs_len) }; + } + }; +}; + +/// ABI bits specifically relating to the time report interface. +pub const time_report = struct { + /// WebSocket server->client. + /// + /// Sent after a `Step` finishes, providing the time taken to execute the step. + pub const GenericResult = extern struct { + tag: ToClientTag = .time_report_generic_result, + step_idx: u32 align(1), + ns_total: u64 align(1), + }; + + /// WebSocket server->client. + /// + /// Sent after a `Step.Compile` finishes, providing the step's time report. + /// + /// Trailing: + /// * `llvm_pass_timings: [llvm_pass_timings_len]u8` (ASCII-encoded) + /// * for each `files_len`: + /// * `name` (null-terminated UTF-8 string) + /// * for each `decls_len`: + /// * `name` (null-terminated UTF-8 string) + /// * `file: u32` (index of file this decl is in) + /// * `sema_ns: u64` (nanoseconds spent semantically analyzing this decl) + /// * `codegen_ns: u64` (nanoseconds spent semantically analyzing this decl) + /// * `link_ns: u64` (nanoseconds spent semantically analyzing this decl) + pub const CompileResult = extern struct { + tag: ToClientTag = .time_report_compile_result, + + step_idx: u32 align(1), + + flags: Flags, + stats: Stats align(1), + ns_total: u64 align(1), + + llvm_pass_timings_len: u32 align(1), + files_len: u32 align(1), + decls_len: u32 align(1), + + pub const Flags = packed struct(u8) { + use_llvm: bool, + _: u7 = 0, + }; + + pub const Stats = extern struct { + n_reachable_files: u32, + n_imported_files: u32, + n_generic_instances: u32, + n_inline_calls: u32, + + cpu_ns_parse: u64, + cpu_ns_astgen: u64, + cpu_ns_sema: u64, + cpu_ns_codegen: u64, + cpu_ns_link: u64, + + real_ns_files: u64, + real_ns_decls: u64, + real_ns_llvm_emit: u64, + real_ns_link_flush: u64, + + pub const init: Stats = .{ + .n_reachable_files = 0, + .n_imported_files = 0, + .n_generic_instances = 0, + .n_inline_calls = 0, + .cpu_ns_parse = 0, + .cpu_ns_astgen = 0, + .cpu_ns_sema = 0, + .cpu_ns_codegen = 0, + .cpu_ns_link = 0, + .real_ns_files = 0, + .real_ns_decls = 0, + .real_ns_llvm_emit = 0, + .real_ns_link_flush = 0, + }; + }; + }; +}; diff --git a/lib/std/http/WebSocket.zig b/lib/std/http/WebSocket.zig index 8ab434ceae05107b29b733c51f5da7549967e0b3..b9a66cdbd6607c61d55f15f0586a3772fd911046 100644 --- a/lib/std/http/WebSocket.zig +++ b/lib/std/http/WebSocket.zig @@ -18,14 +18,13 @@ pub const InitError = error{WebSocketUpgradeMissingKey} || std.http.Server.Request.ReaderError; pub fn init( - ws: *WebSocket, request: *std.http.Server.Request, send_buffer: []u8, recv_buffer: []align(4) u8, -) InitError!bool { +) InitError!?WebSocket { switch (request.head.version) { - .@"HTTP/1.0" => return false, - .@"HTTP/1.1" => if (request.head.method != .GET) return false, + .@"HTTP/1.0" => return null, + .@"HTTP/1.1" => if (request.head.method != .GET) return null, } var sec_websocket_key: ?[]const u8 = null; @@ -36,12 +35,12 @@ pub fn init( sec_websocket_key = header.value; } else if (std.ascii.eqlIgnoreCase(header.name, "upgrade")) { if (!std.ascii.eqlIgnoreCase(header.value, "websocket")) - return false; + return null; upgrade_websocket = true; } } if (!upgrade_websocket) - return false; + return null; const key = sec_websocket_key orelse return error.WebSocketUpgradeMissingKey; @@ -55,7 +54,7 @@ pub fn init( request.head.content_length = std.math.maxInt(u64); - ws.* = .{ + return .{ .key = key, .recv_fifo = std.fifo.LinearFifo(u8, .Slice).init(recv_buffer), .reader = try request.reader(), @@ -74,7 +73,6 @@ pub fn init( .request = request, .outstanding_len = 0, }; - return true; } pub const Header0 = packed struct(u8) { diff --git a/lib/std/net.zig b/lib/std/net.zig index d7387662c004311223de6466eab77dd9eb462404..ac851059d28aebc5d4f8422db64be04d922a165d 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -42,6 +42,47 @@ pub const Address = extern union { in6: Ip6Address, un: if (has_unix_sockets) posix.sockaddr.un else void, + /// Parse an IP address which may include a port. For IPv4, this is just written `address:port`. + /// For IPv6, RFC 3986 defines this as an "IP literal", and the port is differentiated from the + /// address by surrounding the address part in brackets '[addr]:port'. Even if the port is not + /// given, the brackets are mandatory. + pub fn parseIpAndPort(str: []const u8) error{ InvalidAddress, InvalidPort }!Address { + if (str.len == 0) return error.InvalidAddress; + if (str[0] == '[') { + const addr_end = std.mem.indexOfScalar(u8, str, ']') orelse + return error.InvalidAddress; + const addr_str = str[1..addr_end]; + const port: u16 = p: { + if (addr_end == str.len - 1) break :p 0; + if (str[addr_end + 1] != ':') return error.InvalidAddress; + break :p parsePort(str[addr_end + 2 ..]) orelse return error.InvalidPort; + }; + return parseIp6(addr_str, port) catch error.InvalidAddress; + } else { + if (std.mem.indexOfScalar(u8, str, ':')) |idx| { + // hold off on `error.InvalidPort` since `error.InvalidAddress` might make more sense + const port: ?u16 = parsePort(str[idx + 1 ..]); + const addr = parseIp4(str[0..idx], port orelse 0) catch return error.InvalidAddress; + if (port == null) return error.InvalidPort; + return addr; + } else { + return parseIp4(str, 0) catch error.InvalidAddress; + } + } + } + fn parsePort(str: []const u8) ?u16 { + var p: u16 = 0; + for (str) |c| switch (c) { + '0'...'9' => { + const shifted = std.math.mul(u16, p, 10) catch return null; + p = std.math.add(u16, shifted, c - '0') catch return null; + }, + else => return null, + }; + if (p == 0) return null; + return p; + } + /// Parse the given IP address string into an Address value. /// It is recommended to use `resolveIp` instead, to handle /// IPv6 link-local unix addresses. diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 12bd259b16d8b3ab01ac365f052c8e0bd5f9b629..ea603547411143d55bb81e71d1bf62ded016216e 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -50,6 +50,8 @@ pub const Message = struct { /// address of the fuzz unit test. This is used to provide a starting /// point to view coverage. fuzz_start_addr, + /// Body is a TimeReport. + time_report, _, }; @@ -95,6 +97,19 @@ pub const Message = struct { }; }; + /// Trailing is the same as in `std.Build.abi.time_report.CompileResult`, excluding `step_name`. + pub const TimeReport = extern struct { + stats: std.Build.abi.time_report.CompileResult.Stats align(4), + llvm_pass_timings_len: u32, + files_len: u32, + decls_len: u32, + flags: Flags, + pub const Flags = packed struct(u32) { + use_llvm: bool, + _: u31 = 0, + }; + }; + /// Trailing: /// * the hex digest of the cache directory within the /o/ subdirectory. pub const EmitDigest = extern struct { diff --git a/src/Compilation.zig b/src/Compilation.zig index 3796ed6acc53ccc82cbbbfac112eb8e7e32b13a0..419c56019fd009459cb01862f3fc103f2e7014a4 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -173,7 +173,6 @@ verbose_cimport: bool, verbose_llvm_cpu_features: bool, verbose_link: bool, disable_c_depfile: bool, -time_report: bool, stack_report: bool, debug_compiler_runtime_libs: bool, debug_compile_errors: bool, @@ -263,6 +262,8 @@ link_prog_node: std.Progress.Node = std.Progress.Node.none, llvm_opt_bisect_limit: c_int, +time_report: ?TimeReport, + file_system_inputs: ?*std.ArrayListUnmanaged(u8), /// This is the digest of the cache for the current compilation. @@ -322,6 +323,72 @@ const QueuedJobs = struct { zigc_lib: bool = false, }; +pub const Timer = union(enum) { + unused, + active: struct { + start: std.time.Instant, + saved_ns: u64, + }, + paused: u64, + stopped, + + pub fn pause(t: *Timer) void { + switch (t.*) { + .unused => return, + .active => |a| { + const current = std.time.Instant.now() catch unreachable; + const new_ns = switch (current.order(a.start)) { + .lt, .eq => 0, + .gt => current.since(a.start), + }; + t.* = .{ .paused = a.saved_ns + new_ns }; + }, + .paused => unreachable, + .stopped => unreachable, + } + } + pub fn @"resume"(t: *Timer) void { + switch (t.*) { + .unused => return, + .active => unreachable, + .paused => |saved_ns| t.* = .{ .active = .{ + .start = std.time.Instant.now() catch unreachable, + .saved_ns = saved_ns, + } }, + .stopped => unreachable, + } + } + pub fn finish(t: *Timer) ?u64 { + defer t.* = .stopped; + switch (t.*) { + .unused => return null, + .active => |a| { + const current = std.time.Instant.now() catch unreachable; + const new_ns = switch (current.order(a.start)) { + .lt, .eq => 0, + .gt => current.since(a.start), + }; + return a.saved_ns + new_ns; + }, + .paused => |ns| return ns, + .stopped => unreachable, + } + } +}; + +/// Starts a timer for measuring a `--time-report` value. If `comp.time_report` is `null`, the +/// returned timer does nothing. When the thing being timed is done, call `Timer.finish`. If that +/// function returns non-`null`, then the value is a number of nanoseconds, and `comp.time_report` +/// is set. +pub fn startTimer(comp: *Compilation) Timer { + if (comp.time_report == null) return .unused; + const now = std.time.Instant.now() catch @panic("std.time.Timer unsupported; cannot emit time report"); + return .{ .active = .{ + .start = now, + .saved_ns = 0, + } }; +} + /// A filesystem path, represented relative to one of a few specific directories where possible. /// Every path (considering symlinks as distinct paths) has a canonical representation in this form. /// This abstraction allows us to: @@ -787,6 +854,58 @@ pub inline fn debugIncremental(comp: *const Compilation) bool { return comp.debug_incremental; } +pub const TimeReport = struct { + stats: std.Build.abi.time_report.CompileResult.Stats, + + /// Allocated into `gpa`. The pass time statistics emitted by LLVM's "time-passes" option. + /// LLVM provides this data in ASCII form as a table, which can be directly shown to users. + /// + /// Ideally, we would be able to use `printAllJSONValues` to get *structured* data which we can + /// then display more nicely. Unfortunately, that function seems to trip an assertion on one of + /// the pass timer names at the time of writing. + llvm_pass_timings: []u8, + + /// Key is a ZIR `declaration` instruction; value is the number of nanoseconds spent analyzing + /// it. This is the total across all instances of the generic parent namespace, and (if this is + /// a function) all generic instances of this function. It also includes time spent analyzing + /// function bodies if this is a function (generic or otherwise). + /// An entry not existing means the declaration has not been analyzed (so far). + decl_sema_info: std.AutoArrayHashMapUnmanaged(InternPool.TrackedInst.Index, struct { + ns: u64, + count: u32, + }), + + /// Key is a ZIR `declaration` instruction which is a function or test; value is the number of + /// nanoseconds spent running codegen on it. As above, this is the total across all generic + /// instances, both of this function itself and of its parent namespace. + /// An entry not existing means the declaration has not been codegenned (so far). + /// Every key in `decl_codegen_ns` is also in `decl_sema_ns`. + decl_codegen_ns: std.AutoArrayHashMapUnmanaged(InternPool.TrackedInst.Index, u64), + + /// Key is a ZIR `declaration` instruction which is anything other than a `comptime` decl; value + /// is the number of nanoseconds spent linking it into the binary. As above, this is the total + /// across all generic instances. + /// An entry not existing means the declaration has not been linked (so far). + /// Every key in `decl_link_ns` is also in `decl_sema_ns`. + decl_link_ns: std.AutoArrayHashMapUnmanaged(InternPool.TrackedInst.Index, u64), + + pub fn deinit(tr: *TimeReport, gpa: Allocator) void { + tr.stats = undefined; + gpa.free(tr.llvm_pass_timings); + tr.decl_sema_info.deinit(gpa); + tr.decl_codegen_ns.deinit(gpa); + tr.decl_link_ns.deinit(gpa); + } + + pub const init: TimeReport = .{ + .stats = .init, + .llvm_pass_timings = &.{}, + .decl_sema_info = .empty, + .decl_codegen_ns = .empty, + .decl_link_ns = .empty, + }; +}; + pub const default_stack_protector_buffer_size = target_util.default_stack_protector_buffer_size; pub const SemaError = Zcu.SemaError; @@ -2027,7 +2146,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .verbose_link = options.verbose_link, .disable_c_depfile = options.disable_c_depfile, .reference_trace = options.reference_trace, - .time_report = options.time_report, + .time_report = if (options.time_report) .init else null, .stack_report = options.stack_report, .test_filters = options.test_filters, .test_name_prefix = options.test_name_prefix, @@ -2561,6 +2680,8 @@ pub fn destroy(comp: *Compilation) void { } comp.failed_win32_resources.deinit(gpa); + if (comp.time_report) |*tr| tr.deinit(gpa); + comp.link_diags.deinit(); comp.clearMiscFailures(); @@ -2657,6 +2778,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { comp.clearMiscFailures(); comp.last_update_was_cache_hit = false; + if (comp.time_report) |*tr| { + tr.deinit(gpa); // this is information about an old update + tr.* = .init; + } var tmp_dir_rand_int: u64 = undefined; var man: Cache.Manifest = undefined; @@ -2688,6 +2813,14 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { whole.cache_manifest = &man; try addNonIncrementalStuffToCacheManifest(comp, arena, &man); + // Under `--time-report`, ignore cache hits; do the work anyway for those juicy numbers. + const ignore_hit = comp.time_report != null; + + if (ignore_hit) { + // We're going to do the work regardless of whether this is a hit or a miss. + man.want_shared_lock = false; + } + const is_hit = man.hit() catch |err| switch (err) { error.CacheCheckFailed => switch (man.diagnostic) { .none => unreachable, @@ -2713,7 +2846,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { .{}, ), }; - if (is_hit) { + if (is_hit and !ignore_hit) { // In this case the cache hit contains the full set of file system inputs. Nice! if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf); if (comp.parent_whole_cache) |pwc| { @@ -2734,6 +2867,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { } log.debug("CacheMode.whole cache miss for {s}", .{comp.root_name}); + if (ignore_hit) { + // Okay, now set this back so that `writeManifest` will downgrade our lock later. + man.want_shared_lock = true; + } + // Compile the artifacts to a temporary directory. whole.tmp_artifact_directory = d: { tmp_dir_rand_int = std.crypto.random.int(u64); @@ -2786,6 +2924,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { const pt: Zcu.PerThread = .activate(zcu, .main); defer pt.deactivate(); + assert(zcu.cur_analysis_timer == null); + zcu.skip_analysis_this_update = false; // TODO: doing this in `resolveReferences` later could avoid adding inputs for dead embedfiles. Investigate! @@ -2829,6 +2969,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { const pt: Zcu.PerThread = .activate(zcu, .main); defer pt.deactivate(); + assert(zcu.cur_analysis_timer == null); + if (!zcu.skip_analysis_this_update) { if (comp.config.is_test) { // The `test_functions` decl has been intentionally postponed until now, @@ -3040,11 +3182,22 @@ fn flush( ) !void { if (comp.zcu) |zcu| { if (zcu.llvm_object) |llvm_object| { + const pt: Zcu.PerThread = .activate(zcu, tid); + defer pt.deactivate(); + // Emit the ZCU object from LLVM now; it's required to flush the output file. // If there's an output file, it wants to decide where the LLVM object goes! const sub_prog_node = comp.link_prog_node.start("LLVM Emit Object", 0); defer sub_prog_node.end(); - try llvm_object.emit(.{ .zcu = zcu, .tid = tid }, .{ + + var timer = comp.startTimer(); + defer if (timer.finish()) |ns| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.real_ns_llvm_emit = ns; + }; + + try llvm_object.emit(pt, .{ .pre_ir_path = comp.verbose_llvm_ir, .pre_bc_path = comp.verbose_llvm_bc, @@ -3071,7 +3224,7 @@ fn flush( .is_debug = comp.root_mod.optimize_mode == .Debug, .is_small = comp.root_mod.optimize_mode == .ReleaseSmall, - .time_report = comp.time_report, + .time_report = if (comp.time_report) |*p| p else null, .sanitize_thread = comp.config.any_sanitize_thread, .fuzz = comp.config.any_fuzz, .lto = comp.config.lto, @@ -3079,6 +3232,12 @@ fn flush( } } if (comp.bin_file) |lf| { + var timer = comp.startTimer(); + defer if (timer.finish()) |ns| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.real_ns_link_flush = ns; + }; // This is needed before reading the error flags. lf.flush(arena, tid, comp.link_prog_node) catch |err| switch (err) { error.LinkFailure => {}, // Already reported. @@ -4223,6 +4382,17 @@ fn performAllTheWork( zcu.generation += 1; }; + // This is awkward: we don't want to start the timer until later, but we won't want to stop it + // until the wait groups finish. That means we need do do this. + var decl_work_timer: ?Timer = null; + defer commit_timer: { + const t = &(decl_work_timer orelse break :commit_timer); + const ns = t.finish() orelse break :commit_timer; + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.real_ns_decls = ns; + } + // Here we queue up all the AstGen tasks first, followed by C object compilation. // We wait until the AstGen tasks are all completed before proceeding to the // (at least for now) single-threaded main work queue. However, C object compilation @@ -4431,6 +4601,13 @@ fn performAllTheWork( const zir_prog_node = main_progress_node.start("AST Lowering", 0); defer zir_prog_node.end(); + var timer = comp.startTimer(); + defer if (timer.finish()) |ns| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.real_ns_files = ns; + }; + var astgen_wait_group: WaitGroup = .{}; defer astgen_wait_group.wait(); @@ -4556,6 +4733,10 @@ fn performAllTheWork( return; } + if (comp.time_report) |*tr| { + tr.stats.n_reachable_files = @intCast(zcu.alive_files.count()); + } + if (comp.incremental) { const update_zir_refs_node = main_progress_node.start("Update ZIR References", 0); defer update_zir_refs_node.end(); @@ -4599,6 +4780,11 @@ fn performAllTheWork( } } + if (comp.zcu != null) { + // Start the timer for the "decls" part of the pipeline (Sema, CodeGen, link). + decl_work_timer = comp.startTimer(); + } + work: while (true) { for (&comp.work_queues) |*work_queue| if (work_queue.readItem()) |job| { try processOneJob(@intFromEnum(Zcu.PerThread.Id.main), comp, job); diff --git a/src/Sema.zig b/src/Sema.zig index d7add0724d7e6462040703f8450a20241ab376f8..1d2b658fd3418a0b7a4b5ca70dc2091ed8dc3426 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -3246,21 +3246,25 @@ fn zirEnumDecl( wip_ty.prepare(ip, new_namespace_index); done = true; - try Sema.resolveDeclaredEnum( - pt, - wip_ty, - inst, - tracked_inst, - new_namespace_index, - type_name.name, - small, - body, - tag_type_ref, - any_values, - fields_len, - sema.code, - body_end, - ); + { + const tracked_unit = zcu.trackUnitSema(type_name.name.toSlice(ip), null); + defer tracked_unit.end(zcu); + try Sema.resolveDeclaredEnum( + pt, + wip_ty, + inst, + tracked_inst, + new_namespace_index, + type_name.name, + small, + body, + tag_type_ref, + any_values, + fields_len, + sema.code, + body_end, + ); + } codegen_type: { if (zcu.comp.config.use_llvm) break :codegen_type; @@ -7577,6 +7581,12 @@ fn analyzeCall( // This is an inline call. The function must be comptime-known. We will analyze its body directly using this `Sema`. + if (zcu.comp.time_report) |*tr| { + if (!block.isComptime()) { + tr.stats.n_inline_calls += 1; + } + } + if (func_ty_info.is_noinline and !block.isComptime()) { return sema.fail(block, call_src, "inline call of noinline function", .{}); } diff --git a/src/Type.zig b/src/Type.zig index 9316bec11e12092aab5e2cd2b0760fba830deb2e..0677b8d227293761a05508dadda45d34b527164e 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -3797,6 +3797,9 @@ fn resolveStructInner( return error.AnalysisFail; } + const tracked_unit = zcu.trackUnitSema(struct_obj.name.toSlice(&zcu.intern_pool), null); + defer tracked_unit.end(zcu); + if (zcu.comp.debugIncremental()) { const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner); info.last_update_gen = zcu.generation; @@ -3856,6 +3859,9 @@ fn resolveUnionInner( return error.AnalysisFail; } + const tracked_unit = zcu.trackUnitSema(union_obj.name.toSlice(&zcu.intern_pool), null); + defer tracked_unit.end(zcu); + if (zcu.comp.debugIncremental()) { const info = try zcu.incremental_debug_state.getUnitInfo(gpa, owner); info.last_update_gen = zcu.generation; diff --git a/src/Zcu.zig b/src/Zcu.zig index c13f7aaac9be01da55c2bbff74bdc4c9d54a9416..90b3edd00138a2c356689434a122ea434d1b22aa 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -312,6 +312,10 @@ builtin_decl_values: BuiltinDecl.Memoized = .initFill(.none), incremental_debug_state: if (build_options.enable_debug_extensions) IncrementalDebugState else void = if (build_options.enable_debug_extensions) .init else {}, +/// Times semantic analysis of the current `AnalUnit`. When we pause to analyze a different unit, +/// this timer must be temporarily paused and resumed later. +cur_analysis_timer: ?Compilation.Timer = null, + generation: u32 = 0, pub const IncrementalDebugState = struct { @@ -4683,26 +4687,56 @@ fn explainWhyFileIsInModule( } } -const SemaProgNode = struct { +const TrackedUnitSema = struct { /// `null` means we created the node, so should end it. old_name: ?[std.Progress.Node.max_name_len]u8, - pub fn end(spn: SemaProgNode, zcu: *Zcu) void { - if (spn.old_name) |old_name| { + old_analysis_timer: ?Compilation.Timer, + analysis_timer_decl: ?InternPool.TrackedInst.Index, + pub fn end(tus: TrackedUnitSema, zcu: *Zcu) void { + const comp = zcu.comp; + if (tus.old_name) |old_name| { zcu.sema_prog_node.completeOne(); // we're just renaming, but it's effectively completion zcu.cur_sema_prog_node.setName(&old_name); } else { zcu.cur_sema_prog_node.end(); zcu.cur_sema_prog_node = .none; } + report_time: { + const sema_ns = zcu.cur_analysis_timer.?.finish() orelse break :report_time; + const zir_decl = tus.analysis_timer_decl orelse break :report_time; + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.cpu_ns_sema += sema_ns; + const gop = comp.time_report.?.decl_sema_info.getOrPut(comp.gpa, zir_decl) catch |err| switch (err) { + error.OutOfMemory => { + comp.setAllocFailure(); + break :report_time; + }, + }; + if (!gop.found_existing) gop.value_ptr.* = .{ .ns = 0, .count = 0 }; + gop.value_ptr.ns += sema_ns; + gop.value_ptr.count += 1; + } + zcu.cur_analysis_timer = tus.old_analysis_timer; + if (zcu.cur_analysis_timer) |*t| t.@"resume"(); } }; -pub fn startSemaProgNode(zcu: *Zcu, name: []const u8) SemaProgNode { - if (zcu.cur_sema_prog_node.index != .none) { +pub fn trackUnitSema(zcu: *Zcu, name: []const u8, zir_inst: ?InternPool.TrackedInst.Index) TrackedUnitSema { + if (zcu.cur_analysis_timer) |*t| t.pause(); + const old_analysis_timer = zcu.cur_analysis_timer; + zcu.cur_analysis_timer = zcu.comp.startTimer(); + const old_name: ?[std.Progress.Node.max_name_len]u8 = old_name: { + if (zcu.cur_sema_prog_node.index == .none) { + zcu.cur_sema_prog_node = zcu.sema_prog_node.start(name, 0); + break :old_name null; + } const old_name = zcu.cur_sema_prog_node.getName(); zcu.cur_sema_prog_node.setName(name); - return .{ .old_name = old_name }; - } else { - zcu.cur_sema_prog_node = zcu.sema_prog_node.start(name, 0); - return .{ .old_name = null }; - } + break :old_name old_name; + }; + return .{ + .old_name = old_name, + .old_analysis_timer = old_analysis_timer, + .analysis_timer_decl = zir_inst, + }; } diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 119b742a89ea295cf72c763d05fc37877357cf2b..de4be438f507b42dd897fa628f4975a1cdbebb8e 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -215,12 +215,15 @@ pub fn updateFile( }; defer cache_file.close(); + // Under `--time-report`, ignore cache hits; do the work anyway for those juicy numbers. + const ignore_hit = comp.time_report != null; + const need_update = while (true) { const result = switch (file.getMode()) { inline else => |mode| try loadZirZoirCache(zcu, cache_file, stat, file, mode), }; switch (result) { - .success => { + .success => if (!ignore_hit) { log.debug("AstGen cached success: {f}", .{file.path.fmt(comp)}); break false; }, @@ -260,9 +263,16 @@ pub fn updateFile( file.source = source; + var timer = comp.startTimer(); // Any potential AST errors are converted to ZIR errors when we run AstGen/ZonGen. file.tree = try Ast.parse(gpa, source, file.getMode()); + if (timer.finish()) |ns_parse| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.cpu_ns_parse += ns_parse; + } + timer = comp.startTimer(); switch (file.getMode()) { .zig => { file.zir = try AstGen.generate(gpa, file.tree.?); @@ -282,6 +292,11 @@ pub fn updateFile( }; }, } + if (timer.finish()) |ns_astgen| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.cpu_ns_astgen += ns_astgen; + } log.debug("AstGen fresh success: {f}", .{file.path.fmt(comp)}); } @@ -801,8 +816,11 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU info.deps.clearRetainingCapacity(); } - const unit_prog_node = zcu.startSemaProgNode("comptime"); - defer unit_prog_node.end(zcu); + const unit_tracking = zcu.trackUnitSema( + "comptime", + zcu.intern_pool.getComptimeUnit(cu_id).zir_index, + ); + defer unit_tracking.end(zcu); return pt.analyzeComptimeUnit(cu_id) catch |err| switch (err) { error.AnalysisFail => { @@ -981,8 +999,8 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu info.deps.clearRetainingCapacity(); } - const unit_prog_node = zcu.startSemaProgNode(nav.fqn.toSlice(ip)); - defer unit_prog_node.end(zcu); + const unit_tracking = zcu.trackUnitSema(nav.fqn.toSlice(ip), nav.srcInst(ip)); + defer unit_tracking.end(zcu); const invalidate_value: bool, const new_failed: bool = if (pt.analyzeNavVal(nav_id)) |result| res: { break :res .{ @@ -1381,8 +1399,8 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc info.deps.clearRetainingCapacity(); } - const unit_prog_node = zcu.startSemaProgNode(nav.fqn.toSlice(ip)); - defer unit_prog_node.end(zcu); + const unit_tracking = zcu.trackUnitSema(nav.fqn.toSlice(ip), nav.srcInst(ip)); + defer unit_tracking.end(zcu); const invalidate_type: bool, const new_failed: bool = if (pt.analyzeNavType(nav_id)) |result| res: { break :res .{ @@ -1601,8 +1619,12 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, maybe_coerced_func_index: Inter info.deps.clearRetainingCapacity(); } - const func_prog_node = zcu.startSemaProgNode(ip.getNav(func.owner_nav).fqn.toSlice(ip)); - defer func_prog_node.end(zcu); + const owner_nav = ip.getNav(func.owner_nav); + const unit_tracking = zcu.trackUnitSema( + owner_nav.fqn.toSlice(ip), + owner_nav.srcInst(ip), + ); + defer unit_tracking.end(zcu); const ies_outdated, const new_failed = if (pt.analyzeFuncBody(func_index)) |result| .{ prev_failed or result.ies_outdated, false } @@ -1847,6 +1869,10 @@ fn semaFile(pt: Zcu.PerThread, file_index: Zcu.File.Index) Zcu.SemaError!void { }); const struct_ty = try pt.createFileRootStruct(file_index, new_namespace_index, false); errdefer zcu.intern_pool.remove(pt.tid, struct_ty); + + if (zcu.comp.time_report) |*tr| { + tr.stats.n_imported_files += 1; + } } /// Called by AstGen worker threads when an import is seen. If `new_file` is returned, the caller is @@ -2520,6 +2546,12 @@ pub fn scanNamespace( const gpa = zcu.gpa; const namespace = zcu.namespacePtr(namespace_index); + const tracked_unit = zcu.trackUnitSema( + Type.fromInterned(namespace.owner_type).containerTypeName(ip).toSlice(ip), + null, + ); + defer tracked_unit.end(zcu); + // For incremental updates, `scanDecl` wants to look up existing decls by their ZIR index rather // than their name. We'll build an efficient mapping now, then discard the current `decls`. // We map to the `AnalUnit`, since not every declaration has a `Nav`. @@ -2755,6 +2787,12 @@ fn analyzeFnBodyInner(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaE func.setResolvedErrorSet(ip, .none); } + if (zcu.comp.time_report) |*tr| { + if (func.generic_owner != .none) { + tr.stats.n_generic_instances += 1; + } + } + // This is the `Nau` corresponding to the `declaration` instruction which the function or its generic owner originates from. const decl_nav = ip.getNav(if (func.generic_owner == .none) func.owner_nav @@ -4307,6 +4345,9 @@ pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dep /// codegen thread, depending on whether the backend supports `Zcu.Feature.separate_thread`. pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, out: *@import("../link.zig").ZcuTask.LinkFunc.SharedMir) void { const zcu = pt.zcu; + + var timer = zcu.comp.startTimer(); + const success: bool = if (runCodegenInner(pt, func_index, air)) |mir| success: { out.value = mir; break :success true; @@ -4327,6 +4368,25 @@ pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, ou } break :success false; }; + + if (timer.finish()) |ns_codegen| report_time: { + const ip = &zcu.intern_pool; + const nav = ip.indexToKey(func_index).func.owner_nav; + const zir_decl = ip.getNav(nav).srcInst(ip); + zcu.comp.mutex.lock(); + defer zcu.comp.mutex.unlock(); + const tr = &zcu.comp.time_report.?; + tr.stats.cpu_ns_codegen += ns_codegen; + const gop = tr.decl_codegen_ns.getOrPut(zcu.gpa, zir_decl) catch |err| switch (err) { + error.OutOfMemory => { + zcu.comp.setAllocFailure(); + break :report_time; + }, + }; + if (!gop.found_existing) gop.value_ptr.* = 0; + gop.value_ptr.* += ns_codegen; + } + // release `out.value` with this store; synchronizes with acquire loads in `link` out.status.store(if (success) .ready else .failed, .release); zcu.comp.link_task_queue.mirReady(zcu.comp, func_index, out); diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 5e522c3d73d0ca9e0469995306ec7c4b2043c98a..6bec900aecaf3c8b828764362316f11c5d4e68eb 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -764,7 +764,7 @@ pub const Object = struct { is_debug: bool, is_small: bool, - time_report: bool, + time_report: ?*Compilation.TimeReport, sanitize_thread: bool, fuzz: bool, lto: std.zig.LtoMode, @@ -1063,7 +1063,7 @@ pub const Object = struct { var lowered_options: llvm.TargetMachine.EmitOptions = .{ .is_debug = options.is_debug, .is_small = options.is_small, - .time_report = options.time_report, + .time_report_out = null, // set below to make sure it's only set for a single `emitToFile` .tsan = options.sanitize_thread, .lto = switch (options.lto) { .none => .None, @@ -1118,6 +1118,11 @@ pub const Object = struct { lowered_options.llvm_ir_filename = null; } + var time_report_c_str: [*:0]u8 = undefined; + if (options.time_report != null) { + lowered_options.time_report_out = &time_report_c_str; + } + lowered_options.asm_filename = options.asm_path; if (target_machine.emitToFile(module, &error_message, &lowered_options)) { defer llvm.disposeMessage(error_message); @@ -1125,6 +1130,12 @@ pub const Object = struct { emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message, }); } + if (options.time_report) |tr| { + defer std.c.free(time_report_c_str); + const time_report_data = std.mem.span(time_report_c_str); + assert(tr.llvm_pass_timings.len == 0); + tr.llvm_pass_timings = try comp.gpa.dupe(u8, time_report_data); + } } pub fn updateFunc( diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig index 7fa2a7da4356ec55c1e61cf11aedb12ef3cb44ee..89bdb4b3da54ac34b15acadf9596d3e4a7e9cde2 100644 --- a/src/codegen/llvm/bindings.zig +++ b/src/codegen/llvm/bindings.zig @@ -88,7 +88,7 @@ pub const TargetMachine = opaque { pub const EmitOptions = extern struct { is_debug: bool, is_small: bool, - time_report: bool, + time_report_out: ?*[*:0]u8, tsan: bool, sancov: bool, lto: LtoPhase, diff --git a/src/link.zig b/src/link.zig index 51bb9330f1094a47d1935f01bf0b86c20aeb9600..1b63e7793b5ae4a7c93b2bee7572f7309442166c 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1302,6 +1302,14 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void { comp.link_prog_node.completeOne(); return; }; + + var timer = comp.startTimer(); + defer if (timer.finish()) |ns| { + comp.mutex.lock(); + defer comp.mutex.unlock(); + comp.time_report.?.stats.cpu_ns_link += ns; + }; + switch (task) { .load_explicitly_provided => { const prog_node = comp.link_prog_node.start("Parse Inputs", comp.link_inputs.len); @@ -1428,6 +1436,9 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void { const ip = &zcu.intern_pool; const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid)); defer pt.deactivate(); + + var timer = comp.startTimer(); + switch (task) { .link_nav => |nav_index| { const fqn_slice = ip.getNav(nav_index).fqn.toSlice(ip); @@ -1502,6 +1513,28 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void { } }, } + + if (timer.finish()) |ns_link| report_time: { + const zir_decl: ?InternPool.TrackedInst.Index = switch (task) { + .link_type, .update_line_number => null, + .link_nav => |nav| ip.getNav(nav).srcInst(ip), + .link_func => |f| ip.getNav(ip.indexToKey(f.func).func.owner_nav).srcInst(ip), + }; + comp.mutex.lock(); + defer comp.mutex.unlock(); + const tr = &zcu.comp.time_report.?; + tr.stats.cpu_ns_link += ns_link; + if (zir_decl) |inst| { + const gop = tr.decl_link_ns.getOrPut(zcu.gpa, inst) catch |err| switch (err) { + error.OutOfMemory => { + zcu.comp.setAllocFailure(); + break :report_time; + }, + }; + if (!gop.found_existing) gop.value_ptr.* = 0; + gop.value_ptr.* += ns_link; + } + } } /// After the main pipeline is done, but before flush, the compilation may need to link one final /// `Nav` into the binary: the `builtin.test_functions` value. Since the link thread isn't running diff --git a/src/main.zig b/src/main.zig index 9349899a561a1f9b024e7b9afcf61393acc86367..2a143c30aefe0dc8b05e5644870699942bbbe46d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -484,6 +484,7 @@ const usage_build_generic = \\ -fno-structured-cfg (SPIR-V) force SPIR-V kernels to not use structured control flow \\ -mexec-model=[value] (WASI) Execution model \\ -municode (Windows) Use wmain/wWinMain as entry point + \\ --time-report Send timing diagnostics to '--listen' clients \\ \\Per-Module Compile Options: \\ -target [name] -- see the targets command @@ -678,7 +679,6 @@ const usage_build_generic = \\ \\Debug Options (Zig Compiler Development): \\ -fopt-bisect-limit=[limit] Only run [limit] first LLVM optimization passes - \\ -ftime-report Print timing diagnostics \\ -fstack-report Print stack size diagnostics \\ --verbose-link Display linker invocations \\ --verbose-cc Display C compiler invocations @@ -1403,7 +1403,7 @@ fn buildOutputType( try test_exec_args.append(arena, null); } else if (mem.eql(u8, arg, "--test-no-exec")) { test_no_exec = true; - } else if (mem.eql(u8, arg, "-ftime-report")) { + } else if (mem.eql(u8, arg, "--time-report")) { time_report = true; } else if (mem.eql(u8, arg, "-fstack-report")) { stack_report = true; @@ -2899,6 +2899,10 @@ fn buildOutputType( fatal("test-obj requires --test-no-exec", .{}); } + if (time_report and listen == .none) { + fatal("--time-report requires --listen", .{}); + } + if (arg_mode == .translate_c and create_module.c_source_files.items.len != 1) { fatal("translate-c expects exactly 1 source file (found {d})", .{create_module.c_source_files.items.len}); } @@ -4208,6 +4212,84 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void { } } + if (comp.time_report) |*tr| { + var decls_len: u32 = 0; + + var file_name_bytes: std.ArrayListUnmanaged(u8) = .empty; + defer file_name_bytes.deinit(gpa); + var files: std.AutoArrayHashMapUnmanaged(Zcu.File.Index, void) = .empty; + defer files.deinit(gpa); + var decl_data: std.ArrayListUnmanaged(u8) = .empty; + defer decl_data.deinit(gpa); + + // Each decl needs at least 34 bytes: + // * 2 for 1-byte name plus null terminator + // * 4 for `file` + // * 4 for `sema_count` + // * 8 for `sema_ns` + // * 8 for `codegen_ns` + // * 8 for `link_ns` + // Most, if not all, decls in `tr.decl_sema_ns` are valid, so we have a good size estimate. + try decl_data.ensureUnusedCapacity(gpa, tr.decl_sema_info.count() * 34); + + for (tr.decl_sema_info.keys(), tr.decl_sema_info.values()) |tracked_inst, sema_info| { + const resolved = tracked_inst.resolveFull(&comp.zcu.?.intern_pool) orelse continue; + const file = comp.zcu.?.fileByIndex(resolved.file); + const zir = file.zir orelse continue; + const decl_name = zir.nullTerminatedString(zir.getDeclaration(resolved.inst).name); + + const gop = try files.getOrPut(gpa, resolved.file); + if (!gop.found_existing) try file_name_bytes.writer(gpa).print("{f}\x00", .{file.path.fmt(comp)}); + + const codegen_ns = tr.decl_codegen_ns.get(tracked_inst) orelse 0; + const link_ns = tr.decl_link_ns.get(tracked_inst) orelse 0; + + decls_len += 1; + + try decl_data.ensureUnusedCapacity(gpa, 33 + decl_name.len); + decl_data.appendSliceAssumeCapacity(decl_name); + decl_data.appendAssumeCapacity(0); + + const out_file = decl_data.addManyAsArrayAssumeCapacity(4); + const out_sema_count = decl_data.addManyAsArrayAssumeCapacity(4); + const out_sema_ns = decl_data.addManyAsArrayAssumeCapacity(8); + const out_codegen_ns = decl_data.addManyAsArrayAssumeCapacity(8); + const out_link_ns = decl_data.addManyAsArrayAssumeCapacity(8); + std.mem.writeInt(u32, out_file, @intCast(gop.index), .little); + std.mem.writeInt(u32, out_sema_count, sema_info.count, .little); + std.mem.writeInt(u64, out_sema_ns, sema_info.ns, .little); + std.mem.writeInt(u64, out_codegen_ns, codegen_ns, .little); + std.mem.writeInt(u64, out_link_ns, link_ns, .little); + } + + const header: std.zig.Server.Message.TimeReport = .{ + .stats = tr.stats, + .llvm_pass_timings_len = @intCast(tr.llvm_pass_timings.len), + .files_len = @intCast(files.count()), + .decls_len = decls_len, + .flags = .{ + .use_llvm = comp.zcu != null and comp.zcu.?.llvm_object != null, + }, + }; + + var slices: [4][]const u8 = .{ + @ptrCast(&header), + tr.llvm_pass_timings, + file_name_bytes.items, + decl_data.items, + }; + try s.serveMessageHeader(.{ + .tag = .time_report, + .bytes_len = len: { + var len: u32 = 0; + for (slices) |slice| len += @intCast(slice.len); + break :len len; + }, + }); + try s.out.writeVecAll(&slices); + try s.out.flush(); + } + if (error_bundle.errorMessageCount() > 0) { try s.serveErrorBundle(error_bundle); return; @@ -5277,7 +5359,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var windows_libs: std.StringArrayHashMapUnmanaged(void) = .empty; if (resolved_target.result.os.tag == .windows) { - try windows_libs.put(arena, "advapi32", {}); + try windows_libs.ensureUnusedCapacity(arena, 2); + windows_libs.putAssumeCapacity("advapi32", {}); + windows_libs.putAssumeCapacity("ws2_32", {}); // for `--listen` (web interface) } const comp = Compilation.create(gpa, arena, .{ diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp index c9a30bb167e5319f944f998fd8a8991c1b98ad46..65dae8edcdbb12db52436044bb1d4c39e13993bc 100644 --- a/src/zig_llvm.cpp +++ b/src/zig_llvm.cpp @@ -220,7 +220,7 @@ static SanitizerCoverageOptions getSanCovOptions(ZigLLVMCoverageOptions z) { ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref, char **error_message, const ZigLLVMEmitOptions *options) { - TimePassesIsEnabled = options->time_report; + TimePassesIsEnabled = options->time_report_out != nullptr; raw_fd_ostream *dest_asm_ptr = nullptr; raw_fd_ostream *dest_bin_ptr = nullptr; @@ -418,10 +418,17 @@ ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machi WriteBitcodeToFile(llvm_module, *dest_bitcode); } - if (options->time_report) { - TimerGroup::printAll(errs()); + // This must only happen once we know we've succeeded and will be returning `false`, because + // this code `malloc`s memory which will become owned by the caller (in Zig code). + if (options->time_report_out != nullptr) { + std::string out_str; + auto os = raw_string_ostream(out_str); + TimerGroup::printAll(os); + TimerGroup::clearAll(); + auto c_str = (char *)malloc(out_str.length() + 1); + strcpy(c_str, out_str.c_str()); + *options->time_report_out = c_str; } - return false; } diff --git a/src/zig_llvm.h b/src/zig_llvm.h index db331512bd908d7e7e73b50d097314a617726153..f1940bb3408f69fa0d237802bef96ce9f94d77ff 100644 --- a/src/zig_llvm.h +++ b/src/zig_llvm.h @@ -66,7 +66,10 @@ enum ZigLLVMThinOrFullLTOPhase { struct ZigLLVMEmitOptions { bool is_debug; bool is_small; - bool time_report; + // If not null, and `ZigLLVMTargetMachineEmitToFile` returns `false` indicating success, this + // `char *` will be populated with a `malloc`-allocated string containing the serialized (as + // JSON) time report data. The caller is responsible for freeing that memory. + char **time_report_out; bool tsan; bool sancov; ZigLLVMThinOrFullLTOPhase lto; diff --git a/tools/dump-cov.zig b/tools/dump-cov.zig index 16df7e34793e83067e07814f82bbd581acae237c..1e24447011832e6887f7617c3f6d613b5101645b 100644 --- a/tools/dump-cov.zig +++ b/tools/dump-cov.zig @@ -5,7 +5,7 @@ const std = @import("std"); const fatal = std.process.fatal; const Path = std.Build.Cache.Path; const assert = std.debug.assert; -const SeenPcsHeader = std.Build.Fuzz.abi.SeenPcsHeader; +const SeenPcsHeader = std.Build.abi.fuzz.SeenPcsHeader; pub fn main() !void { var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .init; -- 2.54.0