authorgravatar for julien.philippon@epitech.euErsikan <julien.philippon@epitech.eu> 2021-03-14 18:07:09+01:00
committergravatar for julien.philippon@epitech.euErsikan <julien.philippon@epitech.eu> 2021-03-17 10:27:26+01:00
log8942243f7a825e42c16c8d210f5f9dc3baa76b2f
treeae67b24aeacd992e1a382e349d9b51d474bf3307
parenta3540000909bdc6a59ba07c85d21afeb3a7e54e2

zig fmt: factorize source file reading and decoding

Now reading a source file and decoding it from UTF-16LE to UTF-8 is done in a single function. Error messages are improved, and an error is emitted when the source file has a BOM not supported (UTF-16BE, UTF-32). Please note that the BOM of UTF-32 is composed of the same bytes as the BOM of UTF-16 followed by a null character. Therefore a source file in UTF-16LE starting with a null byte will be interpreted as an UTF-32, and rejeted because of an invalid format. In pratice this is not a problem, as the code would have been rejected later anyway because of the null character.

1 files changed, 55 insertions(+), 47 deletions(-)

src/main.zig+55-47
...@@ -2637,6 +2637,50 @@ fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 {...@@ -2637,6 +2637,50 @@ fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 {
2637 return cmd.toOwnedSlice();2637 return cmd.toOwnedSlice();
2638}2638}
26392639
2640fn readSourceFileToEndAlloc(allocator: *mem.Allocator, input: *const fs.File, size_hint: ?usize) ![]const u8 {
2641 const source_code = input.readToEndAllocOptions(
2642 allocator,
2643 max_src_size,
2644 size_hint,
2645 @alignOf(u16),
2646 null,
2647 ) catch |err| switch (err) {
2648 error.ConnectionResetByPeer => unreachable,
2649 error.ConnectionTimedOut => unreachable,
2650 error.NotOpenForReading => unreachable,
2651 else => |e| return e,
2652 };
2653 errdefer allocator.free(source_code);
2654
2655 // Detect unsupported file types with their Byte Order Mark
2656 const unsupported_boms = [_][]const u8{
2657 "\xff\xfe\x00\x00", // UTF-32 little endian
2658 "\xfe\xff\x00\x00", // UTF-32 big endian
2659 "\xfe\xff", // UTF-16 big endian
2660 };
2661 for (unsupported_boms) |bom| {
2662 if (mem.startsWith(u8, source_code, bom)) {
2663 return error.UnsupportedEncoding;
2664 }
2665 }
2666
2667 // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
2668 if (mem.startsWith(u8, source_code, "\xff\xfe")) {
2669 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2670 const source_code_utf8 = std.unicode.utf16leToUtf8Alloc(allocator, source_code_utf16_le) catch |err| switch (err) {
2671 error.DanglingSurrogateHalf => error.UnsupportedEncoding,
2672 error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2673 error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2674 else => |e| return e,
2675 };
2676
2677 allocator.free(source_code);
2678 return source_code_utf8;
2679 }
2680
2681 return source_code;
2682}
2683
2640pub const usage_fmt =2684pub const usage_fmt =
2641 \\Usage: zig fmt [file]...2685 \\Usage: zig fmt [file]...
2642 \\2686 \\
...@@ -2709,20 +2753,8 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {...@@ -2709,20 +2753,8 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
2709 }2753 }
27102754
2711 const stdin = io.getStdIn();2755 const stdin = io.getStdIn();
27122756 const source_code = readSourceFileToEndAlloc(gpa, &stdin, null) catch |err| {
2713 const source_code = blk: {2757 fatal("unable to read stdin: {s}", .{err});
2714 const source_code = try stdin.readToEndAllocOptions(gpa, max_src_size, null, @alignOf(u16), null);
2715 errdefer gpa.free(source_code);
2716
2717 // If the file starts with a UTF-16 BOM, translate it to UTF-8
2718 if (mem.startsWith(u8, source_code, "\xff\xfe")) {
2719 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2720 const source_code_utf8 = try std.unicode.utf16leToUtf8Alloc(gpa, source_code_utf16_le);
2721 gpa.free(source_code);
2722 break :blk source_code_utf8;
2723 } else {
2724 break :blk source_code;
2725 }
2726 };2758 };
2727 defer gpa.free(source_code);2759 defer gpa.free(source_code);
27282760
...@@ -2798,7 +2830,7 @@ const FmtError = error{...@@ -2798,7 +2830,7 @@ const FmtError = error{
2798 EndOfStream,2830 EndOfStream,
2799 Unseekable,2831 Unseekable,
2800 NotOpenForWriting,2832 NotOpenForWriting,
2801 UnknownTextFormat,2833 UnsupportedEncoding,
2802} || fs.File.OpenError;2834} || fs.File.OpenError;
28032835
2804fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {2836fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {
...@@ -2864,40 +2896,16 @@ fn fmtPathFile(...@@ -2864,40 +2896,16 @@ fn fmtPathFile(
2864 if (stat.kind == .Directory)2896 if (stat.kind == .Directory)
2865 return error.IsDir;2897 return error.IsDir;
28662898
2867 const source_code = blk: {2899 const source_code = try readSourceFileToEndAlloc(
2868 const source_code = source_file.readToEndAllocOptions(2900 fmt.gpa,
2869 fmt.gpa,2901 &source_file,
2870 max_src_size,2902 std.math.cast(usize, stat.size) catch return error.FileTooBig,
2871 std.math.cast(usize, stat.size) catch return error.FileTooBig,2903 );
2872 @alignOf(u16),
2873 null,
2874 ) catch |err| switch (err) {
2875 error.ConnectionResetByPeer => unreachable,
2876 error.ConnectionTimedOut => unreachable,
2877 error.NotOpenForReading => unreachable,
2878 else => |e| return e,
2879 };
2880 source_file.close();
2881 file_closed = true;
2882 errdefer fmt.gpa.free(source_code);
2883
2884 // If the file starts with a UTF-16 BOM, translate it to UTF-8
2885 if (mem.eql(u8, source_code[0..2], "\xff\xfe")) {
2886 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2887 const source_code_utf8 = std.unicode.utf16leToUtf8Alloc(fmt.gpa, source_code_utf16_le) catch |err| return switch (err) {
2888 error.DanglingSurrogateHalf => FmtError.UnknownTextFormat,
2889 error.ExpectedSecondSurrogateHalf => FmtError.UnknownTextFormat,
2890 error.UnexpectedSecondSurrogateHalf => FmtError.UnknownTextFormat,
2891 else => |e| e,
2892 };
2893 fmt.gpa.free(source_code);
2894 break :blk source_code_utf8;
2895 } else {
2896 break :blk source_code;
2897 }
2898 };
2899 defer fmt.gpa.free(source_code);2904 defer fmt.gpa.free(source_code);
29002905
2906 source_file.close();
2907 file_closed = true;
2908
2901 // Add to set after no longer possible to get error.IsDir.2909 // Add to set after no longer possible to get error.IsDir.
2902 if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;2910 if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;
29032911