authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-03-11 10:26:08-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-03-11 10:26:08-04:00
log3a6f19de48366a616eaffd9dd6c4d4712e0b6c27
tree60e0f7451f1dc6ac03153e0b3471f171825f35e3
parentfec4555476e38d2eeb1dfb02572404b243acd0b2
signaturelock-open Commit is signed but in an unrecognized format.

stage1 caching system: detect problematic mtimes

closes #2045

8 files changed, 189 insertions(+), 70 deletions(-)

src/cache_hash.cpp+71-26
......@@ -158,8 +158,10 @@ static void base64_encode(Slice<uint8_t> dest, Slice<uint8_t> source) {
158158
159159// Ported from std/base64.zig
160160static Error base64_decode(Slice<uint8_t> dest, Slice<uint8_t> source) {
161 assert(source.len % 4 == 0);
162 assert(dest.len == (source.len / 4) * 3);
161 if (source.len % 4 != 0)
162 return ErrorInvalidFormat;
163 if (dest.len != (source.len / 4) * 3)
164 return ErrorInvalidFormat;
163165
164166 // In Zig this is comptime computed. In C++ it's not worth it to do that.
165167 uint8_t char_to_index[256];
......@@ -218,15 +220,41 @@ static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) {
218220 }
219221}
220222
223// If the wall clock time, rounded to the same precision as the
224// mtime, is equal to the mtime, then we cannot rely on this mtime
225// yet. We will instead save an mtime value that indicates the hash
226// must be unconditionally computed.
227static bool is_problematic_timestamp(const OsTimeStamp *fs_clock) {
228 OsTimeStamp wall_clock = os_timestamp_calendar();
229 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
230 if (fs_clock->nsec == 0) {
231 wall_clock.nsec = 0;
232 if (fs_clock->sec == 0) {
233 wall_clock.sec = 0;
234 } else {
235 wall_clock.sec &= (-1ull) << ctzll(fs_clock->sec);
236 }
237 } else {
238 wall_clock.nsec &= (-1ull) << ctzll(fs_clock->nsec);
239 }
240 return wall_clock.nsec == fs_clock->nsec && wall_clock.sec == fs_clock->sec;
241}
242
221243static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) {
222244 Error err;
223245
224246 assert(chf->path != nullptr);
225247
226248 OsFile this_file;
227 if ((err = os_file_open_r(chf->path, &this_file, &chf->mtime)))
249 if ((err = os_file_open_r(chf->path, &this_file, &chf->attr)))
228250 return err;
229251
252 if (is_problematic_timestamp(&chf->attr.mtime)) {
253 chf->attr.mtime.sec = 0;
254 chf->attr.mtime.nsec = 0;
255 chf->attr.inode = 0;
256 }
257
230258 if ((err = hash_file(chf->bin_digest, this_file, contents))) {
231259 os_file_close(this_file);
232260 return err;
......@@ -278,6 +306,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
278306
279307 size_t input_file_count = ch->files.length;
280308 bool any_file_changed = false;
309 Error return_code = ErrorNone;
281310 size_t file_i = 0;
282311 SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n"));
283312 for (;; file_i += 1) {
......@@ -299,7 +328,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
299328 blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48);
300329 }
301330 // caller can notice that out_digest is unmodified.
302 return ErrorNone;
331 return return_code;
303332 } else if (!opt_line.is_some) {
304333 break;
305334 } else {
......@@ -312,57 +341,73 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
312341
313342 SplitIterator it = memSplit(opt_line.value, str(" "));
314343
344 Optional<Slice<uint8_t>> opt_inode = SplitIterator_next(&it);
345 if (!opt_inode.is_some) {
346 return_code = ErrorInvalidFormat;
347 break;
348 }
349 chf->attr.inode = strtoull((const char *)opt_inode.value.ptr, nullptr, 10);
350
315351 Optional<Slice<uint8_t>> opt_mtime_sec = SplitIterator_next(&it);
316352 if (!opt_mtime_sec.is_some) {
317 os_file_close(ch->manifest_file);
318 return ErrorInvalidFormat;
353 return_code = ErrorInvalidFormat;
354 break;
319355 }
320 chf->mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10);
356 chf->attr.mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10);
321357
322358 Optional<Slice<uint8_t>> opt_mtime_nsec = SplitIterator_next(&it);
323359 if (!opt_mtime_nsec.is_some) {
324 os_file_close(ch->manifest_file);
325 return ErrorInvalidFormat;
360 return_code = ErrorInvalidFormat;
361 break;
326362 }
327 chf->mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10);
363 chf->attr.mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10);
328364
329365 Optional<Slice<uint8_t>> opt_digest = SplitIterator_next(&it);
330366 if (!opt_digest.is_some) {
331 os_file_close(ch->manifest_file);
332 return ErrorInvalidFormat;
367 return_code = ErrorInvalidFormat;
368 break;
333369 }
334370 if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) {
335 os_file_close(ch->manifest_file);
336 return ErrorInvalidFormat;
371 return_code = ErrorInvalidFormat;
372 break;
337373 }
338374
339375 Slice<uint8_t> file_path = SplitIterator_rest(&it);
340376 if (file_path.len == 0) {
341 os_file_close(ch->manifest_file);
342 return ErrorInvalidFormat;
377 return_code = ErrorInvalidFormat;
378 break;
343379 }
344380 Buf *this_path = buf_create_from_slice(file_path);
345381 if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) {
346 os_file_close(ch->manifest_file);
347 return ErrorInvalidFormat;
382 return_code = ErrorInvalidFormat;
383 break;
348384 }
349385 chf->path = this_path;
350386
351387 // if the mtime matches we can trust the digest
352388 OsFile this_file;
353 OsTimeStamp actual_mtime;
354 if ((err = os_file_open_r(chf->path, &this_file, &actual_mtime))) {
389 OsFileAttr actual_attr;
390 if ((err = os_file_open_r(chf->path, &this_file, &actual_attr))) {
355391 fprintf(stderr, "Unable to open %s\n: %s", buf_ptr(chf->path), err_str(err));
356392 os_file_close(ch->manifest_file);
357393 return ErrorCacheUnavailable;
358394 }
359 if (chf->mtime.sec == actual_mtime.sec && chf->mtime.nsec == actual_mtime.nsec) {
395 if (chf->attr.mtime.sec == actual_attr.mtime.sec &&
396 chf->attr.mtime.nsec == actual_attr.mtime.nsec &&
397 chf->attr.inode == actual_attr.inode)
398 {
360399 os_file_close(this_file);
361400 } else {
362401 // we have to recompute the digest.
363402 // later we'll rewrite the manifest with the new mtime/digest values
364403 ch->manifest_dirty = true;
365 chf->mtime = actual_mtime;
404 chf->attr = actual_attr;
405
406 if (is_problematic_timestamp(&actual_attr.mtime)) {
407 chf->attr.mtime.sec = 0;
408 chf->attr.mtime.nsec = 0;
409 chf->attr.inode = 0;
410 }
366411
367412 uint8_t actual_digest[48];
368413 if ((err = hash_file(actual_digest, this_file, nullptr))) {
......@@ -381,7 +426,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
381426 blake2b_update(&ch->blake, chf->bin_digest, 48);
382427 }
383428 }
384 if (file_i < input_file_count || file_i == 0) {
429 if (file_i < input_file_count || file_i == 0 || return_code != ErrorNone) {
385430 // manifest file is empty or missing entries, so this is a cache miss
386431 ch->manifest_dirty = true;
387432 for (; file_i < input_file_count; file_i += 1) {
......@@ -392,7 +437,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
392437 return ErrorCacheUnavailable;
393438 }
394439 }
395 return ErrorNone;
440 return return_code;
396441 }
397442 // Cache Hit
398443 return cache_final(ch, out_digest);
......@@ -499,8 +544,8 @@ static Error write_manifest_file(CacheHash *ch) {
499544 for (size_t i = 0; i < ch->files.length; i += 1) {
500545 CacheHashFile *chf = &ch->files.at(i);
501546 base64_encode({encoded_digest, 64}, {chf->bin_digest, 48});
502 buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n",
503 chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path));
547 buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n",
548 chf->attr.inode, chf->attr.mtime.sec, chf->attr.mtime.nsec, encoded_digest, buf_ptr(chf->path));
504549 }
505550 if ((err = os_file_overwrite(ch->manifest_file, &contents)))
506551 return err;
src/cache_hash.hpp+3-1
......@@ -15,7 +15,7 @@ struct LinkLib;
1515
1616struct CacheHashFile {
1717 Buf *path;
18 OsTimeStamp mtime;
18 OsFileAttr attr;
1919 uint8_t bin_digest[48];
2020 Buf *contents;
2121};
......@@ -57,6 +57,8 @@ void cache_file_opt(CacheHash *ch, Buf *path);
5757// added any files before calling cache_hit. CacheHash::b64_digest becomes
5858// available for use after this call, even in the case of a miss, and it
5959// is a hash of the input parameters only.
60// If this function returns ErrorInvalidFormat, that error may be treated
61// as a cache miss.
6062Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);
6163
6264// If you did not get a cache hit, call this function for every file
src/codegen.cpp+20-10
......@@ -7724,8 +7724,11 @@ static Error define_builtin_compile_vars(CodeGen *g) {
77247724
77257725 Buf digest = BUF_INIT;
77267726 buf_resize(&digest, 0);
7727 if ((err = cache_hit(&cache_hash, &digest)))
7728 return err;
7727 if ((err = cache_hit(&cache_hash, &digest))) {
7728 // Treat an invalid format error as a cache miss.
7729 if (err != ErrorInvalidFormat)
7730 return err;
7731 }
77297732
77307733 // We should always get a cache hit because there are no
77317734 // files in the input hash.
......@@ -8342,12 +8345,14 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {
83428345 Buf digest = BUF_INIT;
83438346 buf_resize(&digest, 0);
83448347 if ((err = cache_hit(cache_hash, &digest))) {
8345 if (err == ErrorCacheUnavailable) {
8346 // already printed error
8347 } else {
8348 fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err));
8348 if (err != ErrorInvalidFormat) {
8349 if (err == ErrorCacheUnavailable) {
8350 // already printed error
8351 } else {
8352 fprintf(stderr, "unable to check cache when compiling C object: %s\n", err_str(err));
8353 }
8354 exit(1);
83498355 }
8350 exit(1);
83518356 }
83528357 bool is_cache_miss = (buf_len(&digest) == 0);
83538358 if (is_cache_miss) {
......@@ -8993,7 +8998,10 @@ void codegen_print_timing_report(CodeGen *g, FILE *f) {
89938998}
89948999
89959000void codegen_add_time_event(CodeGen *g, const char *name) {
8996 g->timing_events.append({os_get_time(), name});
9001 OsTimeStamp timestamp = os_timestamp_monotonic();
9002 double seconds = (double)timestamp.sec;
9003 seconds += ((double)timestamp.nsec) / 1000000000.0;
9004 g->timing_events.append({seconds, name});
89979005}
89989006
89999007static void add_cache_pkg(CodeGen *g, CacheHash *ch, ZigPackage *pkg) {
......@@ -9090,8 +9098,10 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) {
90909098 cache_list_of_file(ch, g->link_objects.items, g->link_objects.length);
90919099
90929100 buf_resize(digest, 0);
9093 if ((err = cache_hit(ch, digest)))
9094 return err;
9101 if ((err = cache_hit(ch, digest))) {
9102 if (err != ErrorInvalidFormat)
9103 return err;
9104 }
90959105
90969106 if (ch->manifest_file_path != nullptr) {
90979107 g->caches_to_release.append(ch);
src/compiler.cpp+4-2
......@@ -75,8 +75,10 @@ Error get_compiler_id(Buf **result) {
7575 cache_file(ch, &self_exe_path);
7676
7777 buf_resize(&saved_compiler_id, 0);
78 if ((err = cache_hit(ch, &saved_compiler_id)))
79 return err;
78 if ((err = cache_hit(ch, &saved_compiler_id))) {
79 if (err != ErrorInvalidFormat)
80 return err;
81 }
8082 if (buf_len(&saved_compiler_id) != 0) {
8183 cache_release(ch);
8284 *result = &saved_compiler_id;
src/ir.cpp+4-2
......@@ -18734,8 +18734,10 @@ static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstruct
1873418734 Buf tmp_c_file_digest = BUF_INIT;
1873518735 buf_resize(&tmp_c_file_digest, 0);
1873618736 if ((err = cache_hit(cache_hash, &tmp_c_file_digest))) {
18737 ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err)));
18738 return ira->codegen->invalid_instruction;
18737 if (err != ErrorInvalidFormat) {
18738 ir_add_error_node(ira, node, buf_sprintf("C import failed: unable to check cache: %s", err_str(err)));
18739 return ira->codegen->invalid_instruction;
18740 }
1873918741 }
1874018742 ira->codegen->caches_to_release.append(cache_hash);
1874118743
src/os.cpp+62-27
......@@ -71,8 +71,10 @@ typedef SSIZE_T ssize_t;
7171
7272#if defined(ZIG_OS_WINDOWS)
7373static double win32_time_resolution;
74static LARGE_INTEGER windows_perf_freq;
7475#elif defined(__MACH__)
75static clock_serv_t cclock;
76static clock_serv_t macos_calendar_clock;
77static clock_serv_t macos_monotonic_clock;
7678#endif
7779
7880#include <stdlib.h>
......@@ -1233,28 +1235,60 @@ Error os_rename(Buf *src_path, Buf *dest_path) {
12331235 return ErrorNone;
12341236}
12351237
1236double os_get_time(void) {
12371238#if defined(ZIG_OS_WINDOWS)
1238 unsigned __int64 time;
1239 QueryPerformanceCounter((LARGE_INTEGER*) &time);
1240 return time * win32_time_resolution;
1239static void windows_filetime_to_os_timestamp(FILETIME *ft, OsTimeStamp *mtime) {
1240 mtime->sec = (((ULONGLONG) ft->dwHighDateTime) << 32) + ft->dwLowDateTime;
1241 mtime->nsec = 0;
1242}
1243#endif
1244
1245OsTimeStamp os_timestamp_calendar(void) {
1246 OsTimeStamp result;
1247#if defined(ZIG_OS_WINDOWS)
1248 FILETIME ft;
1249 GetSystemTimeAsFileTime(&ft);
1250 windows_filetime_to_os_timestamp(&ft, &result);
12411251#elif defined(__MACH__)
12421252 mach_timespec_t mts;
12431253
1244 kern_return_t err = clock_get_time(cclock, &mts);
1254 kern_return_t err = clock_get_time(macos_calendar_clock, &mts);
12451255 assert(!err);
12461256
1247 double seconds = (double)mts.tv_sec;
1248 seconds += ((double)mts.tv_nsec) / 1000000000.0;
1257 result.sec = mts.tv_sec;
1258 result.nsec = mts.tv_nsec;
1259#else
1260 struct timespec tms;
1261 clock_gettime(CLOCK_REALTIME, &tms);
1262
1263 result.sec = tms.tv_sec;
1264 result.nsec = tms.tv_nsec;
1265#endif
1266 return result;
1267}
1268
1269OsTimeStamp os_timestamp_monotonic(void) {
1270 OsTimeStamp result;
1271#if defined(ZIG_OS_WINDOWS)
1272 LARGE_INTEGER counts;
1273 QueryPerformanceCounter(&counts);
1274 result.sec = counts / windows_perf_freq;
1275 result.nsec = (counts % windows_perf_freq) * 1000000000u / windows_perf_freq;
1276#elif defined(__MACH__)
1277 mach_timespec_t mts;
12491278
1250 return seconds;
1279 kern_return_t err = clock_get_time(macos_monotonic_clock, &mts);
1280 assert(!err);
1281
1282 result.sec = mts.tv_sec;
1283 result.nsec = mts.tv_nsec;
12511284#else
12521285 struct timespec tms;
12531286 clock_gettime(CLOCK_MONOTONIC, &tms);
1254 double seconds = (double)tms.tv_sec;
1255 seconds += ((double)tms.tv_nsec) / 1000000000.0;
1256 return seconds;
1287
1288 result.sec = tms.tv_sec;
1289 result.nsec = tms.tv_nsec;
12571290#endif
1291 return result;
12581292}
12591293
12601294Error os_make_path(Buf *path) {
......@@ -1352,14 +1386,14 @@ int os_init(void) {
13521386#if defined(ZIG_OS_WINDOWS)
13531387 _setmode(fileno(stdout), _O_BINARY);
13541388 _setmode(fileno(stderr), _O_BINARY);
1355 unsigned __int64 frequency;
1356 if (QueryPerformanceFrequency((LARGE_INTEGER*) &frequency)) {
1357 win32_time_resolution = 1.0 / (double) frequency;
1389 if (QueryPerformanceFrequency(&windows_perf_freq)) {
1390 win32_time_resolution = 1.0 / (double) windows_perf_freq;
13581391 } else {
13591392 return ErrorSystemResources;
13601393 }
13611394#elif defined(__MACH__)
1362 host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock);
1395 host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &macos_monotonic_clock);
1396 host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &macos_calendar_clock);
13631397#endif
13641398 return 0;
13651399}
......@@ -1780,7 +1814,7 @@ Error os_self_exe_shared_libs(ZigList<Buf *> &paths) {
17801814#endif
17811815}
17821816
1783Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {
1817Error os_file_open_r(Buf *full_path, OsFile *out_file, OsFileAttr *attr) {
17841818#if defined(ZIG_OS_WINDOWS)
17851819 // TODO use CreateFileW
17861820 HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
......@@ -1808,14 +1842,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {
18081842 }
18091843 *out_file = result;
18101844
1811 if (mtime != nullptr) {
1812 FILETIME last_write_time;
1813 if (!GetFileTime(result, nullptr, nullptr, &last_write_time)) {
1845 if (attr != nullptr) {
1846 BY_HANDLE_FILE_INFORMATION file_info;
1847 if (!GetFileInformationByHandle(result, &file_info)) {
18141848 CloseHandle(result);
18151849 return ErrorUnexpected;
18161850 }
1817 mtime->sec = (((ULONGLONG) last_write_time.dwHighDateTime) << 32) + last_write_time.dwLowDateTime;
1818 mtime->nsec = 0;
1851 windows_filetime_to_os_timestamp(&file_info.ftLastWriteTime, &attr->mtime);
1852 attr->inode = (((uint64_t)file_info.nFileIndexHigh) << 32) | file_info.nFileIndexLow;
18191853 }
18201854
18211855 return ErrorNone;
......@@ -1851,13 +1885,14 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime) {
18511885 }
18521886 *out_file = fd;
18531887
1854 if (mtime != nullptr) {
1888 if (attr != nullptr) {
1889 attr->inode = statbuf.st_ino;
18551890#if defined(ZIG_OS_DARWIN)
1856 mtime->sec = statbuf.st_mtimespec.tv_sec;
1857 mtime->nsec = statbuf.st_mtimespec.tv_nsec;
1891 attr->mtime.sec = statbuf.st_mtimespec.tv_sec;
1892 attr->mtime.nsec = statbuf.st_mtimespec.tv_nsec;
18581893#else
1859 mtime->sec = statbuf.st_mtim.tv_sec;
1860 mtime->nsec = statbuf.st_mtim.tv_nsec;
1894 attr->mtime.sec = statbuf.st_mtim.tv_sec;
1895 attr->mtime.nsec = statbuf.st_mtim.tv_nsec;
18611896#endif
18621897 }
18631898 return ErrorNone;
src/os.hpp+8-2
......@@ -85,6 +85,11 @@ struct OsTimeStamp {
8585 uint64_t nsec;
8686};
8787
88struct OsFileAttr {
89 OsTimeStamp mtime;
90 uint64_t inode;
91};
92
8893int os_init(void);
8994
9095void os_spawn_process(const char *exe, ZigList<const char *> &args, Termination *term);
......@@ -103,7 +108,7 @@ bool os_path_is_absolute(Buf *path);
103108Error ATTRIBUTE_MUST_USE os_make_path(Buf *path);
104109Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path);
105110
106Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file, OsTimeStamp *mtime);
111Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file, OsFileAttr *attr);
107112Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file);
108113Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len);
109114Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents);
......@@ -126,7 +131,8 @@ Error os_delete_file(Buf *path);
126131Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result);
127132
128133Error os_rename(Buf *src_path, Buf *dest_path);
129double os_get_time(void);
134OsTimeStamp os_timestamp_monotonic(void);
135OsTimeStamp os_timestamp_calendar(void);
130136
131137bool os_is_sep(uint8_t c);
132138
src/util.hpp+17
......@@ -63,10 +63,27 @@ static inline int clzll(unsigned long long mask) {
6363 return 63 - lz;
6464#endif
6565}
66static inline int ctzll(unsigned long long mask) {
67 unsigned long result;
68#if defined(_WIN64)
69 if (_BitScanForward64(&result, mask))
70 return result;
71 zig_unreachable();
72#else
73 if (_BitScanForward(&result, mask & 0xffffffff))
74 return result;
75 }
76 if (_BitScanForward(&result, mask >> 32))
77 return 32 + result;
78 zig_unreachable();
79#endif
80}
6681#else
6782#define clzll(x) __builtin_clzll(x)
83#define ctzll(x) __builtin_ctzll(x)
6884#endif
6985
86
7087template<typename T>
7188ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
7289#ifndef NDEBUG