Need to reload caller-save live-outs on the fast-paths

This is a very rare case, but I'm not sure how we didn't get burned
by this before.  But when llvm passes a caller-save register
as a live-out, we spill it around the patchpoint.  But I guess the
fastpath would just skip past the reloading.  This commit modifies
initializePatchpoint3, which handles this, to also pass back the
continue_addr in addition to some other metadata about the spilling/
reloading it had to do.

src/asm_writing/rewriter.cpp

@@ -1961,9 +1961,9 @@ void setSlowpathFunc(uint8_t* pp_addr, void* func) {
     *(void**)&pp_addr[2] = func;
 }
 
-std::pair<uint8_t*, uint8_t*> initializePatchpoint3(void* slowpath_func, uint8_t* start_addr, uint8_t* end_addr,
-                                                    int scratch_offset, int scratch_size,
-                                                    const std::unordered_set<int>& live_outs, SpillMap& remapped) {
+PatchpointInitializationInfo initializePatchpoint3(void* slowpath_func, uint8_t* start_addr, uint8_t* end_addr,
+                                                   int scratch_offset, int scratch_size,
+                                                   const std::unordered_set<int>& live_outs, SpillMap& remapped) {
     assert(start_addr < end_addr);
 
     int est_slowpath_size = INITIAL_CALL_SIZE;
@@ -2019,9 +2019,22 @@ std::pair<uint8_t*, uint8_t*> initializePatchpoint3(void* slowpath_func, uint8_t
     // if (regs_to_spill.size())
     // assem.trap();
     assem.emitBatchPush(scratch_offset, scratch_size, regs_to_spill);
-    uint8_t* rtn = assem.emitCall(slowpath_func, assembler::R11);
+    uint8_t* slowpath_rtn_addr = assem.emitCall(slowpath_func, assembler::R11);
     assem.emitBatchPop(scratch_offset, scratch_size, regs_to_spill);
 
+    // The place we should continue if we took a fast path.
+    // If we have to reload things, make sure to set it to the beginning
+    // of the reloading section.
+    // If there's nothing to reload, as a small optimization, set it to the end of
+    // the patchpoint, past any nops.
+    // (Actually I think the calculations of the size above were exact so there should
+    // always be 0 nops, but this optimization shouldn't hurt.)
+    uint8_t* continue_addr;
+    if (regs_to_reload.empty())
+        continue_addr = end_addr;
+    else
+        continue_addr = assem.curInstPointer();
+
     for (assembler::Register r : regs_to_reload) {
         StackMap::Record::Location& l = remapped[r];
         assert(l.type == StackMap::Record::Location::LocationType::Indirect);
@@ -2033,6 +2046,6 @@ std::pair<uint8_t*, uint8_t*> initializePatchpoint3(void* slowpath_func, uint8_t
     assem.fillWithNops();
     assert(!assem.hasFailed());
 
-    return std::make_pair(slowpath_start, rtn);
+    return PatchpointInitializationInfo(slowpath_start, slowpath_rtn_addr, continue_addr);
 }
 }

src/asm_writing/rewriter.h

@@ -534,10 +534,18 @@ typedef std::map<assembler::GenericRegister, StackMap::Record::Location, GRCompa
 bool spillFrameArgumentIfNecessary(StackMap::Record::Location& l, uint8_t*& inst_addr, uint8_t* inst_end,
                                    int& scratch_offset, int& scratch_size, SpillMap& remapped);
 
-// returns (start_of_slowpath, return_addr_of_slowpath_call)
-std::pair<uint8_t*, uint8_t*> initializePatchpoint3(void* slowpath_func, uint8_t* start_addr, uint8_t* end_addr,
-                                                    int scratch_offset, int scratch_size,
-                                                    const std::unordered_set<int>& live_outs, SpillMap& remapped);
+struct PatchpointInitializationInfo {
+    uint8_t* slowpath_start;
+    uint8_t* slowpath_rtn_addr;
+    uint8_t* continue_addr;
+
+    PatchpointInitializationInfo(uint8_t* slowpath_start, uint8_t* slowpath_rtn_addr, uint8_t* continue_addr)
+        : slowpath_start(slowpath_start), slowpath_rtn_addr(slowpath_rtn_addr), continue_addr(continue_addr) {}
+};
+
+PatchpointInitializationInfo initializePatchpoint3(void* slowpath_func, uint8_t* start_addr, uint8_t* end_addr,
+                                                   int scratch_offset, int scratch_size,
+                                                   const std::unordered_set<int>& live_outs, SpillMap& remapped);
 
 template <> inline RewriterVar* RewriterVar::getAttrCast<bool, bool>(int offset, Location loc) {
     return getAttr(offset, loc, assembler::MovType::ZBL);

src/codegen/patchpoints.cpp

@@ -244,13 +244,10 @@ void processStackmap(CompiledFunction* cf, StackMap* stackmap) {
         }
 
 
+        auto initialization_info = initializePatchpoint3(slowpath_func, start_addr, end_addr, scratch_rbp_offset,
+                                                         scratch_size, live_outs, frame_remapped);
 
-        auto _p = initializePatchpoint3(slowpath_func, start_addr, end_addr, scratch_rbp_offset, scratch_size,
-                                        live_outs, frame_remapped);
-        uint8_t* slowpath_start = _p.first;
-        uint8_t* slowpath_rtn_addr = _p.second;
-
-        ASSERT(slowpath_start - start_addr >= ic->num_slots * ic->slot_size,
+        ASSERT(initialization_info.slowpath_start - start_addr >= ic->num_slots * ic->slot_size,
                "Used more slowpath space than expected; change ICSetupInfo::totalSize()?");
 
         assert(pp->numICStackmapArgs() == 0); // don't do anything with these for now
@@ -269,7 +266,8 @@ void processStackmap(CompiledFunction* cf, StackMap* stackmap) {
         int scratch_rsp_offset = scratch_rbp_offset + (stack_size - 8);
 
         std::unique_ptr<ICInfo> icinfo
-            = registerCompiledPatchpoint(start_addr, slowpath_start, end_addr, slowpath_rtn_addr, ic,
+            = registerCompiledPatchpoint(start_addr, initialization_info.slowpath_start,
+                                         initialization_info.continue_addr, initialization_info.slowpath_rtn_addr, ic,
                                          StackInfo(scratch_size, scratch_rsp_offset), std::move(live_outs));
 
         assert(cf);

src/runtime/ics.cpp

@@ -273,12 +273,13 @@ RuntimeIC::RuntimeIC(void* func_addr, int num_slots, int slot_size) : eh_frame(R
 
 
         SpillMap _spill_map;
-        std::pair<uint8_t*, uint8_t*> p
+        PatchpointInitializationInfo initialization_info
             = initializePatchpoint3(func_addr, pp_start, pp_end, 0 /* scratch_offset */, 0 /* scratch_size */,
                                     std::unordered_set<int>(), _spill_map);
         assert(_spill_map.size() == 0);
-        assert(p.first == pp_start + patchable_size);
-        assert(p.second == pp_end);
+        assert(initialization_info.slowpath_start == pp_start + patchable_size);
+        assert(initialization_info.slowpath_rtn_addr == pp_end);
+        assert(initialization_info.continue_addr == pp_end);
 
         StackInfo stack_info(SCRATCH_BYTES, 0);
         icinfo = registerCompiledPatchpoint(pp_start, pp_start + patchable_size, pp_end, pp_end, setup_info.get(),