authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-24 16:22:45-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-24 16:22:45-07:00
log054fafd7d9a5226f21e7be1737c6d352fe39f795
treec0e10eb14eca2b7e8cccabe4e2cdfd36fe1d6119
parent1123c909872569e9a956f9110685c417036118bd

stage2: implement @cImport

Also rename Cache.CacheHash to Cache.Manifest

8 files changed, 311 insertions(+), 86 deletions(-)

BRANCH_TODO+1-2
......@@ -1,6 +1,4 @@
1 * repair @cImport
21 * tests passing with -Dskip-non-native
3 * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj]
42 * make sure zig cc works
53 - using it as a preprocessor (-E)
64 - try building some software
......@@ -24,6 +22,7 @@
2422 * audit the base cache hash
2523 * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process.
2624 * restore error messages for stage2_add_link_lib
25 * windows CUSTOMBUILD : error : unable to build compiler_rt: FileNotFound [D:\a\1\s\build\zig_install_lib_files.vcxproj]
2726
2827 * implement proper parsing of clang stderr/stdout and exposing compile errors with the Compilation API
2928 * implement proper parsing of LLD stderr/stdout and exposing compile errors with the Compilation API
src/Cache.zig+24-24
......@@ -12,9 +12,9 @@ const mem = std.mem;
1212const fmt = std.fmt;
1313const Allocator = std.mem.Allocator;
1414
15/// Be sure to call `CacheHash.deinit` after successful initialization.
16pub fn obtain(cache: *const Cache) CacheHash {
17 return CacheHash{
15/// Be sure to call `Manifest.deinit` after successful initialization.
16pub fn obtain(cache: *const Cache) Manifest {
17 return Manifest{
1818 .cache = cache,
1919 .hash = cache.hash,
2020 .manifest_file = null,
......@@ -30,7 +30,7 @@ pub const hex_digest_len = bin_digest_len * 2;
3030const manifest_file_size_max = 50 * 1024 * 1024;
3131
3232/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
33/// provides enough collision resistance for the CacheHash use cases, while being one of our
33/// provides enough collision resistance for the Manifest use cases, while being one of our
3434/// fastest options right now.
3535pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
3636
......@@ -147,10 +147,10 @@ pub const Lock = struct {
147147 }
148148};
149149
150/// CacheHash manages project-local `zig-cache` directories.
150/// Manifest manages project-local `zig-cache` directories.
151151/// This is not a general-purpose cache.
152152/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
153pub const CacheHash = struct {
153pub const Manifest = struct {
154154 cache: *const Cache,
155155 /// Current state for incremental hashing.
156156 hash: HashHelper,
......@@ -173,7 +173,7 @@ pub const CacheHash = struct {
173173 /// ```
174174 /// var file_contents = cache_hash.files.items[file_index].contents.?;
175175 /// ```
176 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
176 pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
177177 assert(self.manifest_file == null);
178178
179179 try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1);
......@@ -193,13 +193,13 @@ pub const CacheHash = struct {
193193 return idx;
194194 }
195195
196 pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void {
196 pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
197197 self.hash.add(optional_file_path != null);
198198 const file_path = optional_file_path orelse return;
199199 _ = try self.addFile(file_path, null);
200200 }
201201
202 pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void {
202 pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
203203 self.hash.add(list_of_files.len);
204204 for (list_of_files) |file_path| {
205205 _ = try self.addFile(file_path, null);
......@@ -210,13 +210,13 @@ pub const CacheHash = struct {
210210 /// A hex encoding of its hash is available by calling `final`.
211211 ///
212212 /// This function will also acquire an exclusive lock to the manifest file. This means
213 /// that a process holding a CacheHash will block any other process attempting to
213 /// that a process holding a Manifest will block any other process attempting to
214214 /// acquire the lock.
215215 ///
216216 /// The lock on the manifest file is released when `deinit` is called. As another
217217 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
218218 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
219 pub fn hit(self: *CacheHash) !bool {
219 pub fn hit(self: *Manifest) !bool {
220220 assert(self.manifest_file == null);
221221
222222 const ext = ".txt";
......@@ -361,7 +361,7 @@ pub const CacheHash = struct {
361361 return true;
362362 }
363363
364 pub fn unhit(self: *CacheHash, bin_digest: [bin_digest_len]u8, input_file_count: usize) void {
364 pub fn unhit(self: *Manifest, bin_digest: [bin_digest_len]u8, input_file_count: usize) void {
365365 // Reset the hash.
366366 self.hash.hasher = hasher_init;
367367 self.hash.hasher.update(&bin_digest);
......@@ -377,7 +377,7 @@ pub const CacheHash = struct {
377377 }
378378 }
379379
380 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
380 fn populateFileHash(self: *Manifest, ch_file: *File) !void {
381381 const file = try fs.cwd().openFile(ch_file.path.?, .{});
382382 defer file.close();
383383
......@@ -421,7 +421,7 @@ pub const CacheHash = struct {
421421 /// calculated. This is useful for processes that don't know the all the files that
422422 /// are depended on ahead of time. For example, a source file that can import other files
423423 /// will need to be recompiled if the imported file is changed.
424 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]const u8 {
424 pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
425425 assert(self.manifest_file != null);
426426
427427 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
......@@ -446,7 +446,7 @@ pub const CacheHash = struct {
446446 /// calculated. This is useful for processes that don't know the all the files that
447447 /// are depended on ahead of time. For example, a source file that can import other files
448448 /// will need to be recompiled if the imported file is changed.
449 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
449 pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
450450 assert(self.manifest_file != null);
451451
452452 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
......@@ -465,7 +465,7 @@ pub const CacheHash = struct {
465465 try self.populateFileHash(new_ch_file);
466466 }
467467
468 pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void {
468 pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
469469 assert(self.manifest_file != null);
470470
471471 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
......@@ -501,7 +501,7 @@ pub const CacheHash = struct {
501501 }
502502
503503 /// Returns a hex encoded hash of the inputs.
504 pub fn final(self: *CacheHash) [hex_digest_len]u8 {
504 pub fn final(self: *Manifest) [hex_digest_len]u8 {
505505 assert(self.manifest_file != null);
506506
507507 // We don't close the manifest file yet, because we want to
......@@ -519,7 +519,7 @@ pub const CacheHash = struct {
519519 return out_digest;
520520 }
521521
522 pub fn writeManifest(self: *CacheHash) !void {
522 pub fn writeManifest(self: *Manifest) !void {
523523 assert(self.manifest_file != null);
524524 if (!self.manifest_dirty) return;
525525
......@@ -544,18 +544,18 @@ pub const CacheHash = struct {
544544 }
545545
546546 /// Obtain only the data needed to maintain a lock on the manifest file.
547 /// The `CacheHash` remains safe to deinit.
547 /// The `Manifest` remains safe to deinit.
548548 /// Don't forget to call `writeManifest` before this!
549 pub fn toOwnedLock(self: *CacheHash) Lock {
549 pub fn toOwnedLock(self: *Manifest) Lock {
550550 const manifest_file = self.manifest_file.?;
551551 self.manifest_file = null;
552552 return Lock{ .manifest_file = manifest_file };
553553 }
554554
555 /// Releases the manifest file and frees any memory the CacheHash was using.
556 /// `CacheHash.hit` must be called first.
555 /// Releases the manifest file and frees any memory the Manifest was using.
556 /// `Manifest.hit` must be called first.
557557 /// Don't forget to call `writeManifest` before this!
558 pub fn deinit(self: *CacheHash) void {
558 pub fn deinit(self: *Manifest) void {
559559 if (self.manifest_file) |file| {
560560 file.close();
561561 }
......@@ -808,7 +808,7 @@ test "no file inputs" {
808808 testing.expectEqual(digest1, digest2);
809809}
810810
811test "CacheHashes with files added after initial hash work" {
811test "Manifest with files added after initial hash work" {
812812 if (std.Target.current.os.tag == .wasi) {
813813 // https://github.com/ziglang/zig/issues/5437
814814 return error.SkipZigTest;
src/Compilation.zig+209-53
......@@ -22,6 +22,7 @@ const fatal = @import("main.zig").fatal;
2222const Module = @import("Module.zig");
2323const Cache = @import("Cache.zig");
2424const stage1 = @import("stage1.zig");
25const translate_c = @import("translate_c.zig");
2526
2627/// General-purpose allocator. Used for both temporary and long-term storage.
2728gpa: *Allocator,
......@@ -30,7 +31,7 @@ arena_state: std.heap.ArenaAllocator.State,
3031bin_file: *link.File,
3132c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
3233stage1_lock: ?Cache.Lock = null,
33stage1_cache_hash: *Cache.CacheHash = undefined,
34stage1_cache_manifest: *Cache.Manifest = undefined,
3435
3536link_error_flags: link.File.ErrorFlags = .{},
3637
......@@ -1198,29 +1199,182 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
11981199 };
11991200}
12001201
1201fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
1202fn obtainCObjectCacheManifest(comp: *Compilation) Cache.Manifest {
1203 var man = comp.cache_parent.obtain();
1204
1205 // Only things that need to be added on top of the base hash, and only things
1206 // that apply both to @cImport and compiling C objects. No linking stuff here!
1207 // Also nothing that applies only to compiling .zig code.
1208
1209 man.hash.add(comp.sanitize_c);
1210 man.hash.addListOfBytes(comp.clang_argv);
1211 man.hash.add(comp.bin_file.options.link_libcpp);
1212 man.hash.addListOfBytes(comp.libc_include_dir_list);
1213
1214 return man;
1215}
1216
1217test "cImport" {
1218 _ = cImport;
1219}
1220
1221const CImportResult = struct {
1222 out_zig_path: []u8,
1223 errors: []translate_c.ClangErrMsg,
1224};
1225
1226/// Caller owns returned memory.
1227/// This API is currently coupled pretty tightly to stage1's needs; it will need to be reworked
1228/// a bit when we want to start using it from self-hosted.
1229pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
1230 if (!build_options.have_llvm)
1231 return error.ZigCompilerNotBuiltWithLLVMExtensions;
1232
12021233 const tracy = trace(@src());
12031234 defer tracy.end();
12041235
1236 const cimport_zig_basename = "cimport.zig";
1237
1238 var man = comp.obtainCObjectCacheManifest();
1239 defer man.deinit();
1240
1241 man.hash.addBytes(c_src);
1242
1243 // If the previous invocation resulted in clang errors, we will see a hit
1244 // here with 0 files in the manifest, in which case it is actually a miss.
1245 const actual_hit = (try man.hit()) and man.files.items.len != 0;
1246 const digest = if (!actual_hit) digest: {
1247 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
1248 defer arena_allocator.deinit();
1249 const arena = &arena_allocator.allocator;
1250
1251 // We need a place to leave the .h file so we can can log it in case of verbose_cimport.
1252 // This block is so that the defers for closing the tmp directory handle can run before
1253 // we try to delete the directory after the block.
1254 const result: struct { tmp_dir_sub_path: []const u8, digest: [Cache.hex_digest_len]u8 } = blk: {
1255 const tmp_digest = man.hash.peek();
1256 const tmp_dir_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &tmp_digest });
1257 var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
1258 defer zig_cache_tmp_dir.close();
1259 const cimport_c_basename = "cimport.c";
1260 const out_h_path = try comp.local_cache_directory.join(arena, &[_][]const u8{
1261 tmp_dir_sub_path, cimport_c_basename,
1262 });
1263 const out_dep_path = try std.fmt.allocPrint(arena, "{}.d", .{out_h_path});
1264
1265 try zig_cache_tmp_dir.writeFile(cimport_c_basename, c_src);
1266 if (comp.verbose_cimport) {
1267 log.info("C import source: {}", .{out_h_path});
1268 }
1269
1270 var argv = std.ArrayList([]const u8).init(comp.gpa);
1271 defer argv.deinit();
1272
1273 try comp.addTranslateCCArgs(arena, &argv, .c, out_dep_path);
1274
1275 try argv.append(out_h_path);
1276
1277 if (comp.verbose_cc) {
1278 dump_argv(argv.items);
1279 }
1280
1281 // Convert to null terminated args.
1282 const new_argv_with_sentinel = try arena.alloc(?[*:0]const u8, argv.items.len + 1);
1283 new_argv_with_sentinel[argv.items.len] = null;
1284 const new_argv = new_argv_with_sentinel[0..argv.items.len :null];
1285 for (argv.items) |arg, i| {
1286 new_argv[i] = try arena.dupeZ(u8, arg);
1287 }
1288
1289 const c_headers_dir_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{"include"});
1290 const c_headers_dir_path_z = try arena.dupeZ(u8, c_headers_dir_path);
1291 var clang_errors: []translate_c.ClangErrMsg = &[0]translate_c.ClangErrMsg{};
1292 const tree = translate_c.translate(
1293 comp.gpa,
1294 new_argv.ptr,
1295 new_argv.ptr + new_argv.len,
1296 &clang_errors,
1297 c_headers_dir_path_z,
1298 ) catch |err| switch (err) {
1299 error.OutOfMemory => return error.OutOfMemory,
1300 error.ASTUnitFailure => {
1301 log.warn("clang API returned errors but due to a clang bug, it is not exposing the errors for zig to see. For more details: https://github.com/ziglang/zig/issues/4455", .{});
1302 return error.ASTUnitFailure;
1303 },
1304 error.SemanticAnalyzeFail => {
1305 return CImportResult{
1306 .out_zig_path = "",
1307 .errors = clang_errors,
1308 };
1309 },
1310 };
1311 defer tree.deinit();
1312
1313 if (comp.verbose_cimport) {
1314 log.info("C import .d file: {}", .{out_dep_path});
1315 }
1316
1317 const dep_basename = std.fs.path.basename(out_dep_path);
1318 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
1319
1320 const digest = man.final();
1321 const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
1322 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
1323 defer o_dir.close();
1324
1325 var out_zig_file = try o_dir.createFile(cimport_zig_basename, .{});
1326 defer out_zig_file.close();
1327
1328 var bos = std.io.bufferedOutStream(out_zig_file.writer());
1329 _ = try std.zig.render(comp.gpa, bos.writer(), tree);
1330 try bos.flush();
1331
1332 man.writeManifest() catch |err| {
1333 log.warn("failed to write cache manifest for C import: {}", .{@errorName(err)});
1334 };
1335
1336 break :blk .{ .tmp_dir_sub_path = tmp_dir_sub_path, .digest = digest };
1337 };
1338 if (!comp.verbose_cimport) {
1339 // Remove the tmp dir and files to save space because we don't need them again.
1340 comp.local_cache_directory.handle.deleteTree(result.tmp_dir_sub_path) catch |err| {
1341 log.warn("failed to delete tmp files for C import: {}", .{@errorName(err)});
1342 };
1343 }
1344 break :digest result.digest;
1345 } else man.final();
1346
1347 const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{
1348 "o", &digest, cimport_zig_basename,
1349 });
1350 if (comp.verbose_cimport) {
1351 log.info("C import output: {}\n", .{out_zig_path});
1352 }
1353 return CImportResult{
1354 .out_zig_path = out_zig_path,
1355 .errors = &[0]translate_c.ClangErrMsg{},
1356 };
1357}
1358
1359fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
12051360 if (!build_options.have_llvm) {
12061361 return comp.failCObj(c_object, "clang not available: compiler built without LLVM extensions", .{});
12071362 }
12081363 const self_exe_path = comp.self_exe_path orelse
12091364 return comp.failCObj(c_object, "clang compilation disabled", .{});
12101365
1366 const tracy = trace(@src());
1367 defer tracy.end();
1368
12111369 if (c_object.clearStatus(comp.gpa)) {
12121370 // There was previous failure.
12131371 comp.failed_c_objects.removeAssertDiscard(c_object);
12141372 }
12151373
1216 var ch = comp.cache_parent.obtain();
1217 defer ch.deinit();
1374 var man = comp.obtainCObjectCacheManifest();
1375 defer man.deinit();
12181376
1219 ch.hash.add(comp.sanitize_c);
1220 ch.hash.addListOfBytes(comp.clang_argv);
1221 ch.hash.add(comp.bin_file.options.link_libcpp);
1222 ch.hash.addListOfBytes(comp.libc_include_dir_list);
1223 _ = try ch.addFile(c_object.src.src_path, null);
1377 _ = try man.addFile(c_object.src.src_path, null);
12241378 {
12251379 // Hash the extra flags, with special care to call addFile for file parameters.
12261380 // TODO this logic can likely be improved by utilizing clang_options_data.zig.
......@@ -1228,11 +1382,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
12281382 var arg_i: usize = 0;
12291383 while (arg_i < c_object.src.extra_flags.len) : (arg_i += 1) {
12301384 const arg = c_object.src.extra_flags[arg_i];
1231 ch.hash.addBytes(arg);
1385 man.hash.addBytes(arg);
12321386 for (file_args) |file_arg| {
12331387 if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_object.src.extra_flags.len) {
12341388 arg_i += 1;
1235 _ = try ch.addFile(c_object.src.extra_flags[arg_i], null);
1389 _ = try man.addFile(c_object.src.extra_flags[arg_i], null);
12361390 }
12371391 }
12381392 }
......@@ -1254,7 +1408,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
12541408 mem.split(c_source_basename, ".").next().?;
12551409 const o_basename = try std.fmt.allocPrint(arena, "{}{}", .{ o_basename_noext, comp.getTarget().oFileExt() });
12561410
1257 const digest = if ((try ch.hit()) and !comp.disable_c_depfile) ch.final() else blk: {
1411 const digest = if ((try man.hit()) and !comp.disable_c_depfile) man.final() else blk: {
12581412 var argv = std.ArrayList([]const u8).init(comp.gpa);
12591413 defer argv.deinit();
12601414
......@@ -1270,7 +1424,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
12701424 null
12711425 else
12721426 try std.fmt.allocPrint(arena, "{}.d", .{out_obj_path});
1273 try comp.addCCArgs(arena, &argv, ext, false, out_dep_path);
1427 try comp.addCCArgs(arena, &argv, ext, out_dep_path);
12741428
12751429 try argv.append("-o");
12761430 try argv.append(out_obj_path);
......@@ -1325,12 +1479,12 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
13251479 if (code != 0) {
13261480 // TODO parse clang stderr and turn it into an error message
13271481 // and then call failCObjWithOwnedErrorMsg
1328 std.log.err("clang failed with stderr: {}", .{stderr});
1482 log.err("clang failed with stderr: {}", .{stderr});
13291483 return comp.failCObj(c_object, "clang exited with code {}", .{code});
13301484 }
13311485 },
13321486 else => {
1333 std.log.err("clang terminated with stderr: {}", .{stderr});
1487 log.err("clang terminated with stderr: {}", .{stderr});
13341488 return comp.failCObj(c_object, "clang terminated unexpectedly", .{});
13351489 },
13361490 }
......@@ -1339,15 +1493,15 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
13391493 if (out_dep_path) |dep_file_path| {
13401494 const dep_basename = std.fs.path.basename(dep_file_path);
13411495 // Add the files depended on to the cache system.
1342 try ch.addDepFilePost(zig_cache_tmp_dir, dep_basename);
1496 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
13431497 // Just to save disk space, we delete the file because it is never needed again.
13441498 zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| {
1345 std.log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) });
1499 log.warn("failed to delete '{}': {}", .{ dep_file_path, @errorName(err) });
13461500 };
13471501 }
13481502
13491503 // Rename into place.
1350 const digest = ch.final();
1504 const digest = man.final();
13511505 const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
13521506 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
13531507 defer o_dir.close();
......@@ -1355,8 +1509,8 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
13551509 const tmp_basename = std.fs.path.basename(out_obj_path);
13561510 try std.os.renameat(zig_cache_tmp_dir.fd, tmp_basename, o_dir.fd, o_basename);
13571511
1358 ch.writeManifest() catch |err| {
1359 std.log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) });
1512 man.writeManifest() catch |err| {
1513 log.warn("failed to write cache manifest when compiling '{}': {}", .{ c_object.src.src_path, @errorName(err) });
13601514 };
13611515 break :blk digest;
13621516 };
......@@ -1369,7 +1523,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void {
13691523 c_object.status = .{
13701524 .success = .{
13711525 .object_path = try std.fs.path.join(comp.gpa, components),
1372 .lock = ch.toOwnedLock(),
1526 .lock = man.toOwnedLock(),
13731527 },
13741528 };
13751529}
......@@ -1384,21 +1538,28 @@ fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{
13841538 }
13851539}
13861540
1541pub fn addTranslateCCArgs(
1542 comp: *Compilation,
1543 arena: *Allocator,
1544 argv: *std.ArrayList([]const u8),
1545 ext: FileExt,
1546 out_dep_path: ?[]const u8,
1547) !void {
1548 try comp.addCCArgs(arena, argv, ext, out_dep_path);
1549 // This gives us access to preprocessing entities, presumably at the cost of performance.
1550 try argv.appendSlice(&[_][]const u8{ "-Xclang", "-detailed-preprocessing-record" });
1551}
1552
13871553/// Add common C compiler args between translate-c and C object compilation.
13881554pub fn addCCArgs(
13891555 comp: *Compilation,
13901556 arena: *Allocator,
13911557 argv: *std.ArrayList([]const u8),
13921558 ext: FileExt,
1393 translate_c: bool,
13941559 out_dep_path: ?[]const u8,
13951560) !void {
13961561 const target = comp.getTarget();
13971562
1398 if (translate_c) {
1399 try argv.appendSlice(&[_][]const u8{ "-x", "c" });
1400 }
1401
14021563 if (ext == .cpp) {
14031564 try argv.append("-nostdinc++");
14041565 }
......@@ -1488,11 +1649,6 @@ pub fn addCCArgs(
14881649 if (mcmodel != .default) {
14891650 try argv.append(try std.fmt.allocPrint(arena, "-mcmodel={}", .{@tagName(mcmodel)}));
14901651 }
1491 if (translate_c) {
1492 // This gives us access to preprocessing entities, presumably at the cost of performance.
1493 try argv.append("-Xclang");
1494 try argv.append("-detailed-preprocessing-record");
1495 }
14961652
14971653 // windows.h has files such as pshpack1.h which do #pragma packing, triggering a clang warning.
14981654 // So for this target, we disable this warning.
......@@ -2118,7 +2274,7 @@ pub fn updateSubCompilation(sub_compilation: *Compilation) !void {
21182274
21192275 if (errors.list.len != 0) {
21202276 for (errors.list) |full_err_msg| {
2121 std.log.err("{}:{}:{}: {}\n", .{
2277 log.err("{}:{}:{}: {}\n", .{
21222278 full_err_msg.src_path,
21232279 full_err_msg.line + 1,
21242280 full_err_msg.column + 1,
......@@ -2231,23 +2387,23 @@ fn updateStage1Module(comp: *Compilation) !void {
22312387 // the artifact directory the same, however, so we take the same strategy as linking
22322388 // does where we have a file which specifies the hash of the output directory so that we can
22332389 // skip the expensive compilation step if the hash matches.
2234 var ch = comp.cache_parent.obtain();
2235 defer ch.deinit();
2390 var man = comp.cache_parent.obtain();
2391 defer man.deinit();
22362392
2237 _ = try ch.addFile(main_zig_file, null);
2238 ch.hash.add(comp.bin_file.options.valgrind);
2239 ch.hash.add(comp.bin_file.options.single_threaded);
2240 ch.hash.add(target.os.getVersionRange());
2241 ch.hash.add(comp.bin_file.options.dll_export_fns);
2242 ch.hash.add(comp.bin_file.options.function_sections);
2243 ch.hash.add(comp.is_test);
2393 _ = try man.addFile(main_zig_file, null);
2394 man.hash.add(comp.bin_file.options.valgrind);
2395 man.hash.add(comp.bin_file.options.single_threaded);
2396 man.hash.add(target.os.getVersionRange());
2397 man.hash.add(comp.bin_file.options.dll_export_fns);
2398 man.hash.add(comp.bin_file.options.function_sections);
2399 man.hash.add(comp.is_test);
22442400
22452401 // Capture the state in case we come back from this branch where the hash doesn't match.
2246 const prev_hash_state = ch.hash.peekBin();
2247 const input_file_count = ch.files.items.len;
2402 const prev_hash_state = man.hash.peekBin();
2403 const input_file_count = man.files.items.len;
22482404
2249 if (try ch.hit()) {
2250 const digest = ch.final();
2405 if (try man.hit()) {
2406 const digest = man.final();
22512407
22522408 var prev_digest_buf: [digest.len]u8 = undefined;
22532409 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
......@@ -2257,11 +2413,11 @@ fn updateStage1Module(comp: *Compilation) !void {
22572413 };
22582414 if (mem.eql(u8, prev_digest, &digest)) {
22592415 log.debug("stage1 {} digest={} match - skipping invocation", .{ mod.root_pkg.root_src_path, digest });
2260 comp.stage1_lock = ch.toOwnedLock();
2416 comp.stage1_lock = man.toOwnedLock();
22612417 return;
22622418 }
22632419 log.debug("stage1 {} prev_digest={} new_digest={}", .{ mod.root_pkg.root_src_path, prev_digest, digest });
2264 ch.unhit(prev_hash_state, input_file_count);
2420 man.unhit(prev_hash_state, input_file_count);
22652421 }
22662422
22672423 // We are about to change the output file to be different, so we invalidate the build hash now.
......@@ -2285,7 +2441,7 @@ fn updateStage1Module(comp: *Compilation) !void {
22852441 defer main_progress_node.end();
22862442 if (comp.color == .Off) progress.terminal = null;
22872443
2288 comp.stage1_cache_hash = &ch;
2444 comp.stage1_cache_manifest = &man;
22892445
22902446 const main_pkg_path = mod.root_pkg.root_src_directory.path orelse "";
22912447
......@@ -2350,22 +2506,22 @@ fn updateStage1Module(comp: *Compilation) !void {
23502506 stage1_module.build_object();
23512507 stage1_module.destroy();
23522508
2353 const digest = ch.final();
2509 const digest = man.final();
23542510
23552511 log.debug("stage1 {} final digest={}", .{ mod.root_pkg.root_src_path, digest });
23562512
23572513 // Update the dangling symlink with the digest. If it fails we can continue; it only
23582514 // means that the next invocation will have an unnecessary cache miss.
23592515 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
2360 std.log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)});
2516 log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)});
23612517 };
23622518 // Again failure here only means an unnecessary cache miss.
2363 ch.writeManifest() catch |err| {
2364 std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
2519 man.writeManifest() catch |err| {
2520 log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
23652521 };
23662522 // We hang on to this lock so that the output file path can be used without
23672523 // other processes clobbering it.
2368 comp.stage1_lock = ch.toOwnedLock();
2524 comp.stage1_lock = man.toOwnedLock();
23692525}
23702526
23712527fn createStage1Pkg(
src/main.zig+1-1
......@@ -1617,7 +1617,7 @@ fn cmdTranslateC(comp: *Compilation, arena: *Allocator) !void {
16171617
16181618 const c_source_file = comp.c_source_files[0];
16191619 const file_ext = Compilation.classifyFileExt(c_source_file.src_path);
1620 try comp.addCCArgs(arena, &argv, file_ext, true, null);
1620 try comp.addTranslateCCArgs(arena, &argv, file_ext, null);
16211621 try argv.append(c_source_file.src_path);
16221622
16231623 if (comp.verbose_cc) {
src/stage1.zig+34-3
......@@ -11,10 +11,12 @@ const fatal = stage2.fatal;
1111const CrossTarget = std.zig.CrossTarget;
1212const Target = std.Target;
1313const Compilation = @import("Compilation.zig");
14const translate_c = @import("translate_c.zig");
1415
1516comptime {
1617 assert(std.builtin.link_libc);
1718 assert(build_options.is_stage1);
19 assert(build_options.have_llvm);
1820 _ = @import("compiler_rt");
1921}
2022
......@@ -322,8 +324,37 @@ const Stage2SemVer = extern struct {
322324};
323325
324326// ABI warning
325export fn stage2_cimport(stage1: *Module) [*:0]const u8 {
326 @panic("TODO implement stage2_cimport");
327export fn stage2_cimport(
328 stage1: *Module,
329 c_src_ptr: [*]const u8,
330 c_src_len: usize,
331 out_zig_path_ptr: *[*]const u8,
332 out_zig_path_len: *usize,
333 out_errors_ptr: *[*]translate_c.ClangErrMsg,
334 out_errors_len: *usize,
335) Error {
336 const comp = @intToPtr(*Compilation, stage1.userdata);
337 const c_src = c_src_ptr[0..c_src_len];
338 const result = comp.cImport(c_src) catch |err| switch (err) {
339 error.SystemResources => return .SystemResources,
340 error.OperationAborted => return .OperationAborted,
341 error.BrokenPipe => return .BrokenPipe,
342 error.DiskQuota => return .DiskQuota,
343 error.FileTooBig => return .FileTooBig,
344 error.NoSpaceLeft => return .NoSpaceLeft,
345 error.AccessDenied => return .AccessDenied,
346 error.OutOfMemory => return .OutOfMemory,
347 error.Unexpected => return .Unexpected,
348 error.InputOutput => return .FileSystem,
349 error.ASTUnitFailure => return .ASTUnitFailure,
350 else => return .Unexpected,
351 };
352 out_zig_path_ptr.* = result.out_zig_path.ptr;
353 out_zig_path_len.* = result.out_zig_path.len;
354 out_errors_ptr.* = result.errors.ptr;
355 out_errors_len.* = result.errors.len;
356 if (result.errors.len != 0) return .CCompileErrors;
357 return Error.None;
327358}
328359
329360export fn stage2_add_link_lib(
......@@ -345,7 +376,7 @@ export fn stage2_fetch_file(
345376 const comp = @intToPtr(*Compilation, stage1.userdata);
346377 const file_path = path_ptr[0..path_len];
347378 const max_file_size = std.math.maxInt(u32);
348 const contents = comp.stage1_cache_hash.addFilePostFetch(file_path, max_file_size) catch return null;
379 const contents = comp.stage1_cache_manifest.addFilePostFetch(file_path, max_file_size) catch return null;
349380 result_len.* = contents.len;
350381 return contents.ptr;
351382}
src/stage1/ir.cpp+35-1
......@@ -26382,7 +26382,41 @@ static IrInstGen *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstSrcCImpo
2638226382 cimport_pkg->package_table.put(buf_create_from_str("std"), ira->codegen->std_package);
2638326383 buf_init_from_buf(&cimport_pkg->pkg_path, namespace_name);
2638426384
26385 Buf *out_zig_path = buf_create_from_str(stage2_cimport(&ira->codegen->stage1));
26385 const char *out_zig_path_ptr;
26386 size_t out_zig_path_len;
26387 Stage2ErrorMsg *errors_ptr;
26388 size_t errors_len;
26389 if ((err = stage2_cimport(&ira->codegen->stage1,
26390 buf_ptr(&cimport_scope->buf), buf_len(&cimport_scope->buf),
26391 &out_zig_path_ptr, &out_zig_path_len,
26392 &errors_ptr, &errors_len)))
26393 {
26394 if (err != ErrorCCompileErrors) {
26395 ir_add_error_node(ira, node, buf_sprintf("C import failed: %s", err_str(err)));
26396 return ira->codegen->invalid_inst_gen;
26397 }
26398
26399 ErrorMsg *parent_err_msg = ir_add_error_node(ira, node, buf_sprintf("C import failed"));
26400 if (!ira->codegen->stage1.link_libc) {
26401 add_error_note(ira->codegen, parent_err_msg, node,
26402 buf_sprintf("libc headers not available; compilation does not link against libc"));
26403 }
26404 for (size_t i = 0; i < errors_len; i += 1) {
26405 Stage2ErrorMsg *clang_err = &errors_ptr[i];
26406 // Clang can emit "too many errors, stopping now", in which case `source` and `filename_ptr` are null
26407 if (clang_err->source && clang_err->filename_ptr) {
26408 ErrorMsg *err_msg = err_msg_create_with_offset(
26409 clang_err->filename_ptr ?
26410 buf_create_from_mem(clang_err->filename_ptr, clang_err->filename_len) : buf_alloc(),
26411 clang_err->line, clang_err->column, clang_err->offset, clang_err->source,
26412 buf_create_from_mem(clang_err->msg_ptr, clang_err->msg_len));
26413 err_msg_add_note(parent_err_msg, err_msg);
26414 }
26415 }
26416
26417 return ira->codegen->invalid_inst_gen;
26418 }
26419 Buf *out_zig_path = buf_create_from_mem(out_zig_path_ptr, out_zig_path_len);
2638626420
2638726421 Buf *import_code = buf_alloc();
2638826422 if ((err = file_fetch(ira->codegen, out_zig_path, import_code))) {
src/stage1/stage2.h+3-1
......@@ -165,7 +165,9 @@ ZIG_EXTERN_C const char *stage2_fetch_file(struct ZigStage1 *stage1, const char
165165 size_t *result_len);
166166
167167// ABI warning
168ZIG_EXTERN_C const char *stage2_cimport(struct ZigStage1 *stage1);
168ZIG_EXTERN_C Error stage2_cimport(struct ZigStage1 *stage1, const char *c_src_ptr, size_t c_src_len,
169 const char **out_zig_path_ptr, size_t *out_zig_path_len,
170 struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len);
169171
170172// ABI warning
171173ZIG_EXTERN_C const char *stage2_add_link_lib(struct ZigStage1 *stage1,
src/stage1/zig0.cpp+4-1
......@@ -511,7 +511,10 @@ const char *stage2_fetch_file(struct ZigStage1 *stage1, const char *path_ptr, si
511511 return buf_ptr(&contents_buf);
512512}
513513
514const char *stage2_cimport(struct ZigStage1 *stage1) {
514Error stage2_cimport(struct ZigStage1 *stage1, const char *c_src_ptr, size_t c_src_len,
515 const char **out_zig_path_ptr, size_t *out_zig_path_len,
516 struct Stage2ErrorMsg **out_errors_ptr, size_t *out_errors_len)
517{
515518 const char *msg = "stage0 called stage2_cimport";
516519 stage2_panic(msg, strlen(msg));
517520}