authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-16 18:26:02-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-06-16 18:26:02-04:00
log593db7e8d0a5ad3cc1693944503f718ee96bda38
tree69c8ed14ace2b8f30031eb9040c9ac9e1d9f6079
parent04c3fae720b57e72dedbaf28982e7e9bfed47db6
parent2c8a3aaf855475cabef4e8275581a12b06417a02
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5608 from alexnask/windows_utf16_dir

Use PathSpace and wide FS calls on windows in stage1

3 files changed, 222 insertions(+), 30 deletions(-)

src/os.cpp+213-29
......@@ -6,8 +6,13 @@
66 */
77
88#include "os.hpp"
9#include "buffer.hpp"
10#include "heap.hpp"
911#include "util.hpp"
1012#include "error.hpp"
13#include "util_base.hpp"
14#include <stdint.h>
15#include <stdio.h>
1116
1217#if defined(_WIN32)
1318
......@@ -73,6 +78,8 @@ typedef SSIZE_T ssize_t;
7378#endif
7479
7580#if defined(ZIG_OS_WINDOWS)
81static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le);
82static size_t utf8_to_utf16le(WCHAR *utf16_le, Slice<uint8_t> utf8);
7683static uint64_t windows_perf_freq;
7784#elif defined(__MACH__)
7885static clock_serv_t macos_calendar_clock;
......@@ -148,15 +155,21 @@ static void os_spawn_process_windows(ZigList<const char *> &args, Termination *t
148155 os_windows_create_command_line(&command_line, args);
149156
150157 PROCESS_INFORMATION piProcInfo = {0};
151 STARTUPINFO siStartInfo = {0};
152 siStartInfo.cb = sizeof(STARTUPINFO);
158 STARTUPINFOW siStartInfo = {0};
159 siStartInfo.cb = sizeof(STARTUPINFOW);
153160
154 const char *exe = args.at(0);
155 BOOL success = CreateProcessA(exe, buf_ptr(&command_line), nullptr, nullptr, TRUE, 0, nullptr, nullptr,
161 Slice<uint8_t> exe_slice = str(args.at(0));
162 auto exe_utf16_slice = Slice<WCHAR>::alloc(exe_slice.len + 1);
163 exe_utf16_slice.ptr[utf8_to_utf16le(exe_utf16_slice.ptr, exe_slice)] = 0;
164
165 auto command_line_utf16 = Slice<WCHAR>::alloc(buf_len(&command_line) + 1);
166 command_line_utf16.ptr[utf8_to_utf16le(command_line_utf16.ptr, buf_to_slice(&command_line))] = 0;
167
168 BOOL success = CreateProcessW(exe_utf16_slice.ptr, command_line_utf16.ptr, nullptr, nullptr, TRUE, CREATE_UNICODE_ENVIRONMENT, nullptr, nullptr,
156169 &siStartInfo, &piProcInfo);
157170
158171 if (!success) {
159 zig_panic("CreateProcess failed. exe: %s command_line: %s", exe, buf_ptr(&command_line));
172 zig_panic("CreateProcess failed. exe: %s command_line: %s", args.at(0), buf_ptr(&command_line));
160173 }
161174
162175 WaitForSingleObject(piProcInfo.hProcess, INFINITE);
......@@ -269,11 +282,13 @@ void os_path_join(Buf *dirname, Buf *basename, Buf *out_full_path) {
269282
270283Error os_path_real(Buf *rel_path, Buf *out_abs_path) {
271284#if defined(ZIG_OS_WINDOWS)
272 buf_resize(out_abs_path, 4096);
273 if (_fullpath(buf_ptr(out_abs_path), buf_ptr(rel_path), buf_len(out_abs_path)) == nullptr) {
274 zig_panic("_fullpath failed");
285 PathSpace rel_path_space = slice_to_prefixed_file_w(buf_to_slice(rel_path));
286 PathSpace out_abs_path_space;
287
288 if (_wfullpath(&out_abs_path_space.data.items[0], &rel_path_space.data.items[0], PATH_MAX_WIDE) == nullptr) {
289 zig_panic("_wfullpath failed");
275290 }
276 buf_resize(out_abs_path, strlen(buf_ptr(out_abs_path)));
291 utf16le_ptr_to_utf8(out_abs_path, &out_abs_path_space.data.items[0]);
277292 return ErrorNone;
278293#elif defined(ZIG_OS_POSIX)
279294 buf_resize(out_abs_path, PATH_MAX + 1);
......@@ -773,7 +788,8 @@ Error os_fetch_file(FILE *f, Buf *out_buf) {
773788
774789Error os_file_exists(Buf *full_path, bool *result) {
775790#if defined(ZIG_OS_WINDOWS)
776 *result = GetFileAttributes(buf_ptr(full_path)) != INVALID_FILE_ATTRIBUTES;
791 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(full_path));
792 *result = GetFileAttributesW(&path_space.data.items[0]) != INVALID_FILE_ATTRIBUTES;
777793 return ErrorNone;
778794#else
779795 *result = access(buf_ptr(full_path), F_OK) != -1;
......@@ -1021,7 +1037,12 @@ Error os_exec_process(ZigList<const char *> &args,
10211037}
10221038
10231039Error os_write_file(Buf *full_path, Buf *contents) {
1040#if defined(ZIG_OS_WINDOWS)
1041 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(full_path));
1042 FILE *f = _wfopen(&path_space.data.items[0], L"wb");
1043#else
10241044 FILE *f = fopen(buf_ptr(full_path), "wb");
1045#endif
10251046 if (!f) {
10261047 zig_panic("os_write_file failed for %s", buf_ptr(full_path));
10271048 }
......@@ -1056,7 +1077,12 @@ static Error copy_open_files(FILE *src_f, FILE *dest_f) {
10561077Error os_dump_file(Buf *src_path, FILE *dest_file) {
10571078 Error err;
10581079
1080#if defined(ZIG_OS_WINDOWS)
1081 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(src_path));
1082 FILE *src_f = _wfopen(&path_space.data.items[0], L"rb");
1083#else
10591084 FILE *src_f = fopen(buf_ptr(src_path), "rb");
1085#endif
10601086 if (!src_f) {
10611087 int err = errno;
10621088 if (err == ENOENT) {
......@@ -1173,7 +1199,12 @@ Error os_update_file(Buf *src_path, Buf *dst_path) {
11731199}
11741200
11751201Error os_copy_file(Buf *src_path, Buf *dest_path) {
1202#if defined(ZIG_OS_WINDOWS)
1203 PathSpace src_path_space = slice_to_prefixed_file_w(buf_to_slice(src_path));
1204 FILE *src_f = _wfopen(&src_path_space.data.items[0], L"rb");
1205#else
11761206 FILE *src_f = fopen(buf_ptr(src_path), "rb");
1207#endif
11771208 if (!src_f) {
11781209 int err = errno;
11791210 if (err == ENOENT) {
......@@ -1184,7 +1215,12 @@ Error os_copy_file(Buf *src_path, Buf *dest_path) {
11841215 return ErrorFileSystem;
11851216 }
11861217 }
1218#if defined(ZIG_OS_WINDOWS)
1219 PathSpace dest_path_space = slice_to_prefixed_file_w(buf_to_slice(dest_path));
1220 FILE *dest_f = _wfopen(&dest_path_space.data.items[0], L"wb");
1221#else
11871222 FILE *dest_f = fopen(buf_ptr(dest_path), "wb");
1223#endif
11881224 if (!dest_f) {
11891225 int err = errno;
11901226 if (err == ENOENT) {
......@@ -1205,7 +1241,12 @@ Error os_copy_file(Buf *src_path, Buf *dest_path) {
12051241}
12061242
12071243Error os_fetch_file_path(Buf *full_path, Buf *out_contents) {
1244#if defined(ZIG_OS_WINDOWS)
1245 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(full_path));
1246 FILE *f = _wfopen(&path_space.data.items[0], L"rb");
1247#else
12081248 FILE *f = fopen(buf_ptr(full_path), "rb");
1249#endif
12091250 if (!f) {
12101251 switch (errno) {
12111252 case EACCES:
......@@ -1230,11 +1271,11 @@ Error os_fetch_file_path(Buf *full_path, Buf *out_contents) {
12301271
12311272Error os_get_cwd(Buf *out_cwd) {
12321273#if defined(ZIG_OS_WINDOWS)
1233 char buf[4096];
1234 if (GetCurrentDirectory(4096, buf) == 0) {
1274 PathSpace path_space;
1275 if (GetCurrentDirectoryW(PATH_MAX_WIDE, &path_space.data.items[0]) == 0) {
12351276 zig_panic("GetCurrentDirectory failed");
12361277 }
1237 buf_init_from_str(out_cwd, buf);
1278 utf16le_ptr_to_utf8(out_cwd, &path_space.data.items[0]);
12381279 return ErrorNone;
12391280#elif defined(ZIG_OS_POSIX)
12401281 char buf[PATH_MAX];
......@@ -1330,7 +1371,9 @@ Error os_rename(Buf *src_path, Buf *dest_path) {
13301371 return ErrorNone;
13311372 }
13321373#if defined(ZIG_OS_WINDOWS)
1333 if (!MoveFileExA(buf_ptr(src_path), buf_ptr(dest_path), MOVEFILE_REPLACE_EXISTING)) {
1374 PathSpace src_path_space = slice_to_prefixed_file_w(buf_to_slice(src_path));
1375 PathSpace dest_path_space = slice_to_prefixed_file_w(buf_to_slice(dest_path));
1376 if (!MoveFileExW(&src_path_space.data.items[0], &dest_path_space.data.items[0], MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) {
13341377 return ErrorFileSystem;
13351378 }
13361379#else
......@@ -1426,7 +1469,15 @@ Error os_make_path(Buf *path) {
14261469
14271470Error os_make_dir(Buf *path) {
14281471#if defined(ZIG_OS_WINDOWS)
1429 if (!CreateDirectory(buf_ptr(path), NULL)) {
1472 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(path));
1473 if (memEql(buf_to_slice(path), str("C:\\dev\\tést"))) {
1474 for (size_t i = 0; i < path_space.len; i++) {
1475 fprintf(stderr, "%d ", path_space.data.items[i]);
1476 }
1477 fprintf(stderr, "\n");
1478 }
1479
1480 if (!CreateDirectoryW(&path_space.data.items[0], NULL)) {
14301481 if (GetLastError() == ERROR_ALREADY_EXISTS)
14311482 return ErrorPathAlreadyExists;
14321483 if (GetLastError() == ERROR_PATH_NOT_FOUND)
......@@ -1531,18 +1582,13 @@ int os_init(void) {
15311582
15321583Error os_self_exe_path(Buf *out_path) {
15331584#if defined(ZIG_OS_WINDOWS)
1534 buf_resize(out_path, 256);
1535 for (;;) {
1536 DWORD copied_amt = GetModuleFileName(nullptr, buf_ptr(out_path), buf_len(out_path));
1537 if (copied_amt <= 0) {
1538 return ErrorFileNotFound;
1539 }
1540 if (copied_amt < buf_len(out_path)) {
1541 buf_resize(out_path, copied_amt);
1542 return ErrorNone;
1543 }
1544 buf_resize(out_path, buf_len(out_path) * 2);
1585 PathSpace path_space;
1586 DWORD copied_amt = GetModuleFileNameW(nullptr, &path_space.data.items[0], PATH_MAX_WIDE);
1587 if (copied_amt <= 0) {
1588 return ErrorFileNotFound;
15451589 }
1590 utf16le_ptr_to_utf8(out_path, &path_space.data.items[0]);
1591 return ErrorNone;
15461592
15471593#elif defined(ZIG_OS_DARWIN)
15481594 // How long is the executable's path?
......@@ -1719,6 +1765,15 @@ static uint8_t utf8CodepointSequenceLength(uint32_t c) {
17191765 zig_unreachable();
17201766}
17211767
1768// Ported from std.unicode.utf8ByteSequenceLength
1769static uint8_t utf8ByteSequenceLength(uint8_t first_byte) {
1770 if (first_byte < 0b10000000) return 1;
1771 if ((first_byte & 0b11100000) == 0b11000000) return 2;
1772 if ((first_byte & 0b11110000) == 0b11100000) return 3;
1773 if ((first_byte & 0b11111000) == 0b11110000) return 4;
1774 zig_unreachable();
1775}
1776
17221777// Ported from std/unicode.zig
17231778static size_t utf8Encode(uint32_t c, Slice<uint8_t> out) {
17241779 size_t length = utf8CodepointSequenceLength(c);
......@@ -1753,6 +1808,80 @@ static size_t utf8Encode(uint32_t c, Slice<uint8_t> out) {
17531808 return length;
17541809}
17551810
1811// Ported from std.unicode.utf8Decode2
1812static uint32_t utf8Decode2(Slice<uint8_t> bytes) {
1813 assert(bytes.len == 2);
1814 assert((bytes.at(0) & 0b11100000) == 0b11000000);
1815
1816 uint32_t value = bytes.at(0) & 0b00011111;
1817 assert((bytes.at(1) & 0b11000000) == 0b10000000);
1818 value <<= 6;
1819 value |= bytes.at(1) & 0b00111111;
1820
1821 assert(value >= 0x80);
1822 return value;
1823}
1824
1825// Ported from std.unicode.utf8Decode3
1826static uint32_t utf8Decode3(Slice<uint8_t> bytes) {
1827 assert(bytes.len == 3);
1828 assert((bytes.at(0) & 0b11110000) == 0b11100000);
1829
1830 uint32_t value = bytes.at(0) & 0b00001111;
1831 assert((bytes.at(1) & 0b11000000) == 0b10000000);
1832 value <<= 6;
1833 value |= bytes.at(1) & 0b00111111;
1834
1835 assert((bytes.at(2) & 0b11000000) == 0b10000000);
1836 value <<= 6;
1837 value |= bytes.at(2) & 0b00111111;
1838
1839 assert(value >= 0x80);
1840 assert(value < 0xd800 || value > 0xdfff);
1841 return value;
1842}
1843
1844// Ported from std.unicode.utf8Decode4
1845static uint32_t utf8Decode4(Slice<uint8_t> bytes) {
1846 assert(bytes.len == 4);
1847 assert((bytes.at(0) & 0b11111000) == 0b11110000);
1848
1849 uint32_t value = bytes.at(0) & 0b00000111;
1850 assert((bytes.at(1) & 0b11000000) == 0b10000000);
1851 value <<= 6;
1852 value |= bytes.at(1) & 0b00111111;
1853
1854 assert((bytes.at(2) & 0b11000000) == 0b10000000);
1855 value <<= 6;
1856 value |= bytes.at(2) & 0b00111111;
1857
1858 assert((bytes.at(3) & 0b11000000) == 0b10000000);
1859 value <<= 6;
1860 value |= bytes.at(3) & 0b00111111;
1861
1862 assert(value >= 0x10000 && value <= 0x10FFFF);
1863 return value;
1864}
1865
1866// Ported from std.unicode.utf8Decode
1867static uint32_t utf8Decode(Slice<uint8_t> bytes) {
1868 switch (bytes.len) {
1869 case 1:
1870 return bytes.at(0);
1871 break;
1872 case 2:
1873 return utf8Decode2(bytes);
1874 break;
1875 case 3:
1876 return utf8Decode3(bytes);
1877 break;
1878 case 4:
1879 return utf8Decode4(bytes);
1880 break;
1881 default:
1882 zig_unreachable();
1883 }
1884}
17561885// Ported from std.unicode.utf16leToUtf8Alloc
17571886static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le) {
17581887 // optimistically guess that it will all be ascii.
......@@ -1770,6 +1899,60 @@ static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le) {
17701899 out_index += utf8_len;
17711900 }
17721901}
1902
1903// Ported from std.unicode.utf8ToUtf16Le
1904static size_t utf8_to_utf16le(WCHAR *utf16_le, Slice<uint8_t> utf8) {
1905 size_t dest_i = 0;
1906 size_t src_i = 0;
1907 while (src_i < utf8.len) {
1908 uint8_t n = utf8ByteSequenceLength(utf8.at(src_i));
1909 size_t next_src_i = src_i + n;
1910 uint32_t codepoint = utf8Decode(utf8.slice(src_i, next_src_i));
1911 if (codepoint < 0x10000) {
1912 utf16_le[dest_i] = codepoint;
1913 dest_i += 1;
1914 } else {
1915 WCHAR high = ((codepoint - 0x10000) >> 10) + 0xD800;
1916 WCHAR low = (codepoint & 0x3FF) + 0xDC00;
1917 utf16_le[dest_i] = high;
1918 utf16_le[dest_i + 1] = low;
1919 dest_i += 2;
1920 }
1921 src_i = next_src_i;
1922 }
1923 return dest_i;
1924}
1925
1926// Ported from std.os.windows.sliceToPrefixedFileW
1927PathSpace slice_to_prefixed_file_w(Slice<uint8_t> path) {
1928 PathSpace path_space;
1929 for (size_t idx = 0; idx < path.len; idx++) {
1930 assert(path.ptr[idx] != '*' && path.ptr[idx] != '?' && path.ptr[idx] != '"' &&
1931 path.ptr[idx] != '<' && path.ptr[idx] != '>' && path.ptr[idx] != '|');
1932 }
1933
1934 size_t start_index;
1935 if (memStartsWith(path, str("\\?")) || !isAbsoluteWindows(path)) {
1936 start_index = 0;
1937 } else {
1938 static WCHAR prefix[4] = { u'\\', u'?', u'?', u'\\' };
1939 memCopy(path_space.data.slice(), Slice<WCHAR> { prefix, 4 });
1940 start_index = 4;
1941 }
1942
1943 path_space.len = start_index + utf8_to_utf16le(path_space.data.slice().sliceFrom(start_index).ptr, path);
1944 assert(path_space.len <= path_space.data.len);
1945
1946 Slice<WCHAR> path_slice = path_space.data.slice().slice(0, path_space.len);
1947 for (size_t elem_idx = 0; elem_idx < path_slice.len; elem_idx += 1) {
1948 if (path_slice.at(elem_idx) == '/') {
1949 path_slice.at(elem_idx) = '\\';
1950 }
1951 }
1952
1953 path_space.data.items[path_space.len] = 0;
1954 return path_space;
1955}
17731956#endif
17741957
17751958// Ported from std.os.getAppDataDir
......@@ -1862,8 +2045,8 @@ Error os_self_exe_shared_libs(ZigList<Buf *> &paths) {
18622045
18632046Error os_file_open_rw(Buf *full_path, OsFile *out_file, OsFileAttr *attr, bool need_write, uint32_t mode) {
18642047#if defined(ZIG_OS_WINDOWS)
1865 // TODO use CreateFileW
1866 HANDLE result = CreateFileA(buf_ptr(full_path),
2048 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(full_path));
2049 HANDLE result = CreateFileW(&path_space.data.items[0],
18672050 need_write ? (GENERIC_READ|GENERIC_WRITE) : GENERIC_READ,
18682051 need_write ? 0 : FILE_SHARE_READ,
18692052 nullptr,
......@@ -1967,8 +2150,9 @@ Error os_file_open_w(Buf *full_path, OsFile *out_file, OsFileAttr *attr, uint32_
19672150
19682151Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
19692152#if defined(ZIG_OS_WINDOWS)
2153 PathSpace path_space = slice_to_prefixed_file_w(buf_to_slice(full_path));
19702154 for (;;) {
1971 HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ | GENERIC_WRITE,
2155 HANDLE result = CreateFileW(&path_space.data.items[0], GENERIC_READ | GENERIC_WRITE,
19722156 0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
19732157
19742158 if (result == INVALID_HANDLE_VALUE) {
src/os.hpp+8
......@@ -155,4 +155,12 @@ Error ATTRIBUTE_MUST_USE os_get_app_data_dir(Buf *out_path, const char *appname)
155155
156156Error ATTRIBUTE_MUST_USE os_self_exe_shared_libs(ZigList<Buf *> &paths);
157157
158const size_t PATH_MAX_WIDE = 32767;
159
160struct PathSpace {
161 Array<wchar_t, PATH_MAX_WIDE> data;
162 size_t len;
163};
164
165PathSpace slice_to_prefixed_file_w(Slice<uint8_t> path);
158166#endif
src/util.hpp+1-1
......@@ -159,7 +159,7 @@ struct Slice {
159159
160160 inline T &at(size_t i) {
161161 assert(i < len);
162 return &ptr[i];
162 return ptr[i];
163163 }
164164
165165 inline Slice<T> slice(size_t start, size_t end) {