authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-04-22 03:22:15-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-04-23 03:20:38-07:00
log84f4c5d9ccbebb6675c7366c4e1fdb661003356e
tree52c35bf6456f5a26e3e4a2083d2b62e772770642
parenta2b834e8c7152f70d71c71107db40b9182909647

std.unicode: Fix ArrayList functions when using populated ArrayLists

ensureTotalCapacityPrecise only satisfies the assumptions made in the ArrayListImpl functions (that there's already enough capacity for the entire converted string if it's all ASCII) when the ArrayList has no items, otherwise it would hit illegal behavior.

1 files changed, 64 insertions(+), 6 deletions(-)

lib/std/unicode.zig+64-6
......@@ -934,7 +934,7 @@ fn utf16LeToUtf8ArrayListImpl(
934934 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
935935 .can_encode_surrogate_half => mem.Allocator.Error,
936936})!void {
937 assert(result.capacity >= utf16le.len);
937 assert(result.unusedCapacitySlice().len >= utf16le.len);
938938
939939 var remaining = utf16le;
940940 vectorized: {
......@@ -979,7 +979,7 @@ fn utf16LeToUtf8ArrayListImpl(
979979pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;
980980
981981pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
982 try result.ensureTotalCapacityPrecise(utf16le.len);
982 try result.ensureUnusedCapacity(utf16le.len);
983983 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
984984}
985985
......@@ -1138,7 +1138,7 @@ test utf16LeToUtf8 {
11381138}
11391139
11401140fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
1141 assert(result.capacity >= utf8.len);
1141 assert(result.unusedCapacitySlice().len >= utf8.len);
11421142
11431143 var remaining = utf8;
11441144 vectorized: {
......@@ -1176,7 +1176,7 @@ fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, com
11761176}
11771177
11781178pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
1179 try result.ensureTotalCapacityPrecise(utf8.len);
1179 try result.ensureUnusedCapacity(utf8.len);
11801180 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
11811181}
11821182
......@@ -1351,6 +1351,64 @@ test utf8ToUtf16LeAllocZ {
13511351 }
13521352}
13531353
1354test "ArrayList functions on a re-used list" {
1355 // utf8ToUtf16LeArrayList
1356 {
1357 var list = std.ArrayList(u16).init(testing.allocator);
1358 defer list.deinit();
1359
1360 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1361 try list.ensureTotalCapacityPrecise(init_slice.len);
1362 list.appendSliceAssumeCapacity(init_slice);
1363
1364 try utf8ToUtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1365
1366 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1367 }
1368
1369 // utf16LeToUtf8ArrayList
1370 {
1371 var list = std.ArrayList(u8).init(testing.allocator);
1372 defer list.deinit();
1373
1374 const init_slice = "abcdefg";
1375 try list.ensureTotalCapacityPrecise(init_slice.len);
1376 list.appendSliceAssumeCapacity(init_slice);
1377
1378 try utf16LeToUtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1379
1380 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1381 }
1382
1383 // wtf8ToWtf16LeArrayList
1384 {
1385 var list = std.ArrayList(u16).init(testing.allocator);
1386 defer list.deinit();
1387
1388 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1389 try list.ensureTotalCapacityPrecise(init_slice.len);
1390 list.appendSliceAssumeCapacity(init_slice);
1391
1392 try wtf8ToWtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1393
1394 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1395 }
1396
1397 // wtf16LeToWtf8ArrayList
1398 {
1399 var list = std.ArrayList(u8).init(testing.allocator);
1400 defer list.deinit();
1401
1402 const init_slice = "abcdefg";
1403 try list.ensureTotalCapacityPrecise(init_slice.len);
1404 list.appendSliceAssumeCapacity(init_slice);
1405
1406 try wtf16LeToWtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1407
1408 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1409 }
1410}
1411
13541412/// Converts a UTF-8 string literal into a UTF-16LE string literal.
13551413pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
13561414 return comptime blk: {
......@@ -1685,7 +1743,7 @@ pub const Wtf8Iterator = struct {
16851743};
16861744
16871745pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
1688 try result.ensureTotalCapacityPrecise(utf16le.len);
1746 try result.ensureUnusedCapacity(utf16le.len);
16891747 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
16901748}
16911749
......@@ -1714,7 +1772,7 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
17141772}
17151773
17161774pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
1717 try result.ensureTotalCapacityPrecise(wtf8.len);
1775 try result.ensureUnusedCapacity(wtf8.len);
17181776 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
17191777}
17201778