authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-04-22 03:22:15-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-04-23 03:20:38-07:00
log84f4c5d9ccbebb6675c7366c4e1fdb661003356e
tree52c35bf6456f5a26e3e4a2083d2b62e772770642
parenta2b834e8c7152f70d71c71107db40b9182909647

std.unicode: Fix ArrayList functions when using populated ArrayLists

ensureTotalCapacityPrecise only satisfies the assumptions made in the ArrayListImpl functions (that there's already enough capacity for the entire converted string if it's all ASCII) when the ArrayList has no items, otherwise it would hit illegal behavior.

1 files changed, 64 insertions(+), 6 deletions(-)

lib/std/unicode.zig+64-6
...@@ -934,7 +934,7 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -934,7 +934,7 @@ fn utf16LeToUtf8ArrayListImpl(
934 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,934 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
935 .can_encode_surrogate_half => mem.Allocator.Error,935 .can_encode_surrogate_half => mem.Allocator.Error,
936})!void {936})!void {
937 assert(result.capacity >= utf16le.len);937 assert(result.unusedCapacitySlice().len >= utf16le.len);
938938
939 var remaining = utf16le;939 var remaining = utf16le;
940 vectorized: {940 vectorized: {
...@@ -979,7 +979,7 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -979,7 +979,7 @@ fn utf16LeToUtf8ArrayListImpl(
979pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;979pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;
980980
981pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {981pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
982 try result.ensureTotalCapacityPrecise(utf16le.len);982 try result.ensureUnusedCapacity(utf16le.len);
983 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);983 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
984}984}
985985
...@@ -1138,7 +1138,7 @@ test utf16LeToUtf8 {...@@ -1138,7 +1138,7 @@ test utf16LeToUtf8 {
1138}1138}
11391139
1140fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {1140fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
1141 assert(result.capacity >= utf8.len);1141 assert(result.unusedCapacitySlice().len >= utf8.len);
11421142
1143 var remaining = utf8;1143 var remaining = utf8;
1144 vectorized: {1144 vectorized: {
...@@ -1176,7 +1176,7 @@ fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, com...@@ -1176,7 +1176,7 @@ fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, com
1176}1176}
11771177
1178pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {1178pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
1179 try result.ensureTotalCapacityPrecise(utf8.len);1179 try result.ensureUnusedCapacity(utf8.len);
1180 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);1180 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1181}1181}
11821182
...@@ -1351,6 +1351,64 @@ test utf8ToUtf16LeAllocZ {...@@ -1351,6 +1351,64 @@ test utf8ToUtf16LeAllocZ {
1351 }1351 }
1352}1352}
13531353
1354test "ArrayList functions on a re-used list" {
1355 // utf8ToUtf16LeArrayList
1356 {
1357 var list = std.ArrayList(u16).init(testing.allocator);
1358 defer list.deinit();
1359
1360 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1361 try list.ensureTotalCapacityPrecise(init_slice.len);
1362 list.appendSliceAssumeCapacity(init_slice);
1363
1364 try utf8ToUtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1365
1366 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1367 }
1368
1369 // utf16LeToUtf8ArrayList
1370 {
1371 var list = std.ArrayList(u8).init(testing.allocator);
1372 defer list.deinit();
1373
1374 const init_slice = "abcdefg";
1375 try list.ensureTotalCapacityPrecise(init_slice.len);
1376 list.appendSliceAssumeCapacity(init_slice);
1377
1378 try utf16LeToUtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1379
1380 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1381 }
1382
1383 // wtf8ToWtf16LeArrayList
1384 {
1385 var list = std.ArrayList(u16).init(testing.allocator);
1386 defer list.deinit();
1387
1388 const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
1389 try list.ensureTotalCapacityPrecise(init_slice.len);
1390 list.appendSliceAssumeCapacity(init_slice);
1391
1392 try wtf8ToWtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
1393
1394 try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
1395 }
1396
1397 // wtf16LeToWtf8ArrayList
1398 {
1399 var list = std.ArrayList(u8).init(testing.allocator);
1400 defer list.deinit();
1401
1402 const init_slice = "abcdefg";
1403 try list.ensureTotalCapacityPrecise(init_slice.len);
1404 list.appendSliceAssumeCapacity(init_slice);
1405
1406 try wtf16LeToWtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
1407
1408 try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
1409 }
1410}
1411
1354/// Converts a UTF-8 string literal into a UTF-16LE string literal.1412/// Converts a UTF-8 string literal into a UTF-16LE string literal.
1355pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {1413pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
1356 return comptime blk: {1414 return comptime blk: {
...@@ -1685,7 +1743,7 @@ pub const Wtf8Iterator = struct {...@@ -1685,7 +1743,7 @@ pub const Wtf8Iterator = struct {
1685};1743};
16861744
1687pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {1745pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
1688 try result.ensureTotalCapacityPrecise(utf16le.len);1746 try result.ensureUnusedCapacity(utf16le.len);
1689 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);1747 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1690}1748}
16911749
...@@ -1714,7 +1772,7 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {...@@ -1714,7 +1772,7 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
1714}1772}
17151773
1716pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {1774pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
1717 try result.ensureTotalCapacityPrecise(wtf8.len);1775 try result.ensureUnusedCapacity(wtf8.len);
1718 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);1776 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1719}1777}
17201778