compiler: implement @branchHint

doc/langref.html.in

@@ -4340,6 +4340,13 @@ comptime {
       {#see_also|@sizeOf|@typeInfo#}
       {#header_close#}
 
+      {#header_open|@branchHint#}
+      <pre>{#syntax#}@branchHint(hint: BranchHint) void{#endsyntax#}</pre>
+      <p>Hints to the optimizer how likely a given branch of control flow is to be reached.</p>
+      <p>{#syntax#}BranchHint{#endsyntax#} can be found with {#syntax#}@import("std").builtin.BranchHint{#endsyntax#}.</p>
+      <p>This function is only valid as the first statement in a control flow branch, or the first statement in a function.</p>
+      {#header_close#}
+
       {#header_open|@breakpoint#}
       <pre>{#syntax#}@breakpoint() void{#endsyntax#}</pre>
       <p>
@@ -5252,15 +5259,6 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val
       </p>
       {#header_close#}
 
-      {#header_open|@setCold#}
-      <pre>{#syntax#}@setCold(comptime is_cold: bool) void{#endsyntax#}</pre>
-      <p>
-      Tells the optimizer that the current function is (or is not) rarely called.
-
-      This function is only valid within function scope.
-      </p>
-      {#header_close#}
-
       {#header_open|@setEvalBranchQuota#}
       <pre>{#syntax#}@setEvalBranchQuota(comptime new_quota: u32) void{#endsyntax#}</pre>
       <p>

doc/langref/test_functions.zig

@@ -27,9 +27,9 @@ const WINAPI: std.builtin.CallingConvention = if (native_arch == .x86) .Stdcall
 extern "kernel32" fn ExitProcess(exit_code: u32) callconv(WINAPI) noreturn;
 extern "c" fn atan2(a: f64, b: f64) f64;
 
-// The @setCold builtin tells the optimizer that a function is rarely called.
+// The @branchHint builtin can be used to tell the optimizer that a function is rarely called ("cold").
 fn abort() noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     while (true) {}
 }
 

lib/c.zig

@@ -46,7 +46,7 @@ comptime {
 // Avoid dragging in the runtime safety mechanisms into this .o file,
 // unless we're trying to test this file.
 pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     _ = error_return_trace;
     if (builtin.is_test) {
         std.debug.panic("{s}", .{msg});

lib/compiler/aro/aro/Driver/Filesystem.zig

@@ -4,7 +4,7 @@ const builtin = @import("builtin");
 const is_windows = builtin.os.tag == .windows;
 
 fn readFileFake(entries: []const Filesystem.Entry, path: []const u8, buf: []u8) ?[]const u8 {
-    @setCold(true);
+    @branchHint(.cold);
     for (entries) |entry| {
         if (mem.eql(u8, entry.path, path)) {
             const len = @min(entry.contents.len, buf.len);
@@ -16,7 +16,7 @@ fn readFileFake(entries: []const Filesystem.Entry, path: []const u8, buf: []u8)
 }
 
 fn findProgramByNameFake(entries: []const Filesystem.Entry, name: []const u8, path: ?[]const u8, buf: []u8) ?[]const u8 {
-    @setCold(true);
+    @branchHint(.cold);
     if (mem.indexOfScalar(u8, name, '/') != null) {
         @memcpy(buf[0..name.len], name);
         return buf[0..name.len];
@@ -35,7 +35,7 @@ fn findProgramByNameFake(entries: []const Filesystem.Entry, name: []const u8, pa
 }
 
 fn canExecuteFake(entries: []const Filesystem.Entry, path: []const u8) bool {
-    @setCold(true);
+    @branchHint(.cold);
     for (entries) |entry| {
         if (mem.eql(u8, entry.path, path)) {
             return entry.executable;
@@ -45,7 +45,7 @@ fn canExecuteFake(entries: []const Filesystem.Entry, path: []const u8) bool {
 }
 
 fn existsFake(entries: []const Filesystem.Entry, path: []const u8) bool {
-    @setCold(true);
+    @branchHint(.cold);
     var buf: [std.fs.max_path_bytes]u8 = undefined;
     var fib = std.heap.FixedBufferAllocator.init(&buf);
     const resolved = std.fs.path.resolvePosix(fib.allocator(), &.{path}) catch return false;

lib/compiler/aro/aro/Parser.zig

@@ -385,12 +385,12 @@ fn errExpectedToken(p: *Parser, expected: Token.Id, actual: Token.Id) Error {
 }
 
 pub fn errStr(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, str: []const u8) Compilation.Error!void {
-    @setCold(true);
+    @branchHint(.cold);
     return p.errExtra(tag, tok_i, .{ .str = str });
 }
 
 pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diagnostics.Message.Extra) Compilation.Error!void {
-    @setCold(true);
+    @branchHint(.cold);
     const tok = p.pp.tokens.get(tok_i);
     var loc = tok.loc;
     if (tok_i != 0 and tok.id == .eof) {
@@ -407,12 +407,12 @@ pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diag
 }
 
 pub fn errTok(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex) Compilation.Error!void {
-    @setCold(true);
+    @branchHint(.cold);
     return p.errExtra(tag, tok_i, .{ .none = {} });
 }
 
 pub fn err(p: *Parser, tag: Diagnostics.Tag) Compilation.Error!void {
-    @setCold(true);
+    @branchHint(.cold);
     return p.errExtra(tag, p.tok_i, .{ .none = {} });
 }
 
@@ -638,7 +638,7 @@ fn pragma(p: *Parser) Compilation.Error!bool {
 
 /// Issue errors for top-level definitions whose type was never completed.
 fn diagnoseIncompleteDefinitions(p: *Parser) !void {
-    @setCold(true);
+    @branchHint(.cold);
 
     const node_slices = p.nodes.slice();
     const tags = node_slices.items(.tag);

lib/compiler/resinator/main.zig

@@ -421,7 +421,7 @@ fn cliDiagnosticsToErrorBundle(
     gpa: std.mem.Allocator,
     diagnostics: *cli.Diagnostics,
 ) !ErrorBundle {
-    @setCold(true);
+    @branchHint(.cold);
 
     var bundle: ErrorBundle.Wip = undefined;
     try bundle.init(gpa);
@@ -468,7 +468,7 @@ fn diagnosticsToErrorBundle(
     diagnostics: *Diagnostics,
     mappings: SourceMappings,
 ) !ErrorBundle {
-    @setCold(true);
+    @branchHint(.cold);
 
     var bundle: ErrorBundle.Wip = undefined;
     try bundle.init(gpa);
@@ -559,7 +559,7 @@ fn flushErrorMessageIntoBundle(wip: *ErrorBundle.Wip, msg: ErrorBundle.ErrorMess
 }
 
 fn errorStringToErrorBundle(allocator: std.mem.Allocator, comptime format: []const u8, args: anytype) !ErrorBundle {
-    @setCold(true);
+    @branchHint(.cold);
     var bundle: ErrorBundle.Wip = undefined;
     try bundle.init(allocator);
     errdefer bundle.deinit();
@@ -574,7 +574,7 @@ fn aroDiagnosticsToErrorBundle(
     fail_msg: []const u8,
     comp: *aro.Compilation,
 ) !ErrorBundle {
-    @setCold(true);
+    @branchHint(.cold);
 
     var bundle: ErrorBundle.Wip = undefined;
     try bundle.init(gpa);

lib/compiler_rt/common.zig

@@ -72,7 +72,7 @@ pub const want_sparc_abi = builtin.cpu.arch.isSPARC();
 pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn {
     _ = error_return_trace;
     if (builtin.is_test) {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.panic("{s}", .{msg});
     } else {
         unreachable;

lib/std/Thread/Futex.zig

@@ -27,7 +27,7 @@ const atomic = std.atomic;
 /// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
 /// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
 pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
-    @setCold(true);
+    @branchHint(.cold);
 
     Impl.wait(ptr, expect, null) catch |err| switch (err) {
         error.Timeout => unreachable, // null timeout meant to wait forever
@@ -43,7 +43,7 @@ pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
 /// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
 /// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
 pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
-    @setCold(true);
+    @branchHint(.cold);
 
     // Avoid calling into the OS for no-op timeouts.
     if (timeout_ns == 0) {
@@ -56,7 +56,7 @@ pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) er
 
 /// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
 pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
-    @setCold(true);
+    @branchHint(.cold);
 
     // Avoid calling into the OS if there's nothing to wake up.
     if (max_waiters == 0) {
@@ -1048,7 +1048,7 @@ pub const Deadline = struct {
     /// - A spurious wake occurs.
     /// - The deadline expires; In which case `error.Timeout` is returned.
     pub fn wait(self: *Deadline, ptr: *const atomic.Value(u32), expect: u32) error{Timeout}!void {
-        @setCold(true);
+        @branchHint(.cold);
 
         // Check if we actually have a timeout to wait until.
         // If not just wait "forever".

lib/std/Thread/Mutex.zig

@@ -169,7 +169,7 @@ const FutexImpl = struct {
     }
 
     fn lockSlow(self: *@This()) void {
-        @setCold(true);
+        @branchHint(.cold);
 
         // Avoid doing an atomic swap below if we already know the state is contended.
         // An atomic swap unconditionally stores which marks the cache-line as modified unnecessarily.

lib/std/Thread/ResetEvent.zig

@@ -107,7 +107,7 @@ const FutexImpl = struct {
     }
 
     fn waitUntilSet(self: *Impl, timeout: ?u64) error{Timeout}!void {
-        @setCold(true);
+        @branchHint(.cold);
 
         // Try to set the state from `unset` to `waiting` to indicate
         // to the set() thread that others are blocked on the ResetEvent.

lib/std/builtin.zig

@@ -675,6 +675,25 @@ pub const ExternOptions = struct {
     is_thread_local: bool = false,
 };
 
+/// This data structure is used by the Zig language code generation and
+/// therefore must be kept in sync with the compiler implementation.
+pub const BranchHint = enum(u3) {
+    /// Equivalent to no hint given.
+    none,
+    /// This branch of control flow is more likely to be reached than its peers.
+    /// The optimizer should optimize for reaching it.
+    likely,
+    /// This branch of control flow is less likely to be reached than its peers.
+    /// The optimizer should optimize for not reaching it.
+    unlikely,
+    /// This branch of control flow is unlikely to *ever* be reached.
+    /// The optimizer may place it in a different page of memory to optimize other branches.
+    cold,
+    /// It is difficult to predict whether this branch of control flow will be reached.
+    /// The optimizer should avoid branching behavior with expensive mispredictions.
+    unpredictable,
+};
+
 /// This enum is set by the compiler and communicates which compiler backend is
 /// used to produce machine code.
 /// Think carefully before deciding to observe this value. Nearly all code should
@@ -760,7 +779,7 @@ else
 /// This function is used by the Zig language code generation and
 /// therefore must be kept in sync with the compiler implementation.
 pub fn default_panic(msg: []const u8, error_return_trace: ?*StackTrace, ret_addr: ?usize) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
 
     // For backends that cannot handle the language features depended on by the
     // default panic handler, we have a simpler panic handler:
@@ -877,27 +896,27 @@ pub fn checkNonScalarSentinel(expected: anytype, actual: @TypeOf(expected)) void
 }
 
 pub fn panicSentinelMismatch(expected: anytype, actual: @TypeOf(expected)) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     std.debug.panicExtra(null, @returnAddress(), "sentinel mismatch: expected {any}, found {any}", .{ expected, actual });
 }
 
 pub fn panicUnwrapError(st: ?*StackTrace, err: anyerror) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     std.debug.panicExtra(st, @returnAddress(), "attempt to unwrap error: {s}", .{@errorName(err)});
 }
 
 pub fn panicOutOfBounds(index: usize, len: usize) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     std.debug.panicExtra(null, @returnAddress(), "index out of bounds: index {d}, len {d}", .{ index, len });
 }
 
 pub fn panicStartGreaterThanEnd(start: usize, end: usize) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     std.debug.panicExtra(null, @returnAddress(), "start index {d} is larger than end index {d}", .{ start, end });
 }
 
 pub fn panicInactiveUnionField(active: anytype, wanted: @TypeOf(active)) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
     std.debug.panicExtra(null, @returnAddress(), "access of union field '{s}' while field '{s}' is active", .{ @tagName(wanted), @tagName(active) });
 }
 
@@ -930,7 +949,7 @@ pub const panic_messages = struct {
 };
 
 pub noinline fn returnError(st: *StackTrace) void {
-    @setCold(true);
+    @branchHint(.cold);
     @setRuntimeSafety(false);
     addErrRetTraceAddr(st, @returnAddress());
 }

lib/std/debug.zig

@@ -409,7 +409,7 @@ pub fn assertReadable(slice: []const volatile u8) void {
 }
 
 pub fn panic(comptime format: []const u8, args: anytype) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
 
     panicExtra(@errorReturnTrace(), @returnAddress(), format, args);
 }
@@ -422,7 +422,7 @@ pub fn panicExtra(
     comptime format: []const u8,
     args: anytype,
 ) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
 
     const size = 0x1000;
     const trunc_msg = "(msg truncated)";
@@ -450,7 +450,7 @@ threadlocal var panic_stage: usize = 0;
 // `panicImpl` could be useful in implementing a custom panic handler which
 // calls the default handler (on supported platforms)
 pub fn panicImpl(trace: ?*const std.builtin.StackTrace, first_trace_addr: ?usize, msg: []const u8) noreturn {
-    @setCold(true);
+    @branchHint(.cold);
 
     if (enable_segfault_handler) {
         // If a segfault happens while panicking, we want it to actually segfault, not trigger

lib/std/fmt/parse_float/convert_slow.zig

@@ -36,7 +36,7 @@ pub fn getShift(n: usize) usize {
 /// Note that this function needs a lot of stack space and is marked
 /// cold to hint against inlining into the caller.
 pub fn convertSlow(comptime T: type, s: []const u8) BiasedFp(T) {
-    @setCold(true);
+    @branchHint(.cold);
 
     const MantissaT = mantissaType(T);
     const min_exponent = -(1 << (math.floatExponentBits(T) - 1)) + 1;

lib/std/hash/xxhash.zig

@@ -593,7 +593,7 @@ pub const XxHash3 = struct {
     }
 
     fn hash3(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len > 0 and input.len < 4);
 
         const flip: [2]u32 = @bitCast(secret[0..8].*);
@@ -609,7 +609,7 @@ pub const XxHash3 = struct {
     }
 
     fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len >= 4 and input.len <= 8);
 
         const flip: [2]u64 = @bitCast(secret[8..24].*);
@@ -625,7 +625,7 @@ pub const XxHash3 = struct {
     }
 
     fn hash16(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len > 8 and input.len <= 16);
 
         const flip: [4]u64 = @bitCast(secret[24..56].*);
@@ -641,7 +641,7 @@ pub const XxHash3 = struct {
     }
 
     fn hash128(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len > 16 and input.len <= 128);
 
         var acc = XxHash64.prime_1 *% @as(u64, input.len);
@@ -657,7 +657,7 @@ pub const XxHash3 = struct {
     }
 
     fn hash240(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len > 128 and input.len <= 240);
 
         var acc = XxHash64.prime_1 *% @as(u64, input.len);
@@ -676,7 +676,7 @@ pub const XxHash3 = struct {
     }
 
     noinline fn hashLong(seed: u64, input: []const u8) u64 {
-        @setCold(true);
+        @branchHint(.cold);
         std.debug.assert(input.len >= 240);
 
         const block_count = ((input.len - 1) / @sizeOf(Block)) * @sizeOf(Block);