Indirect-addressed addq

src/asm_writing/assembler.cpp

@@ -103,6 +103,42 @@ void Assembler::emitArith(Immediate imm, Register r, int opcode) {
     }
 }
 
+void Assembler::emitArith(Immediate imm, Indirect mem, int opcode) {
+    int64_t amount = imm.val;
+    assert(fitsInto<int32_t>(amount));
+    assert(0 <= opcode && opcode < 8);
+
+    int rex = REX_W;
+
+    int mem_idx = mem.base.regnum;
+    if (mem_idx >= 8) {
+        rex |= REX_B;
+        mem_idx -= 8;
+    }
+
+    // TODO: needs testing, then remove this trap
+    trap();
+
+    emitRex(rex);
+
+    bool needssib = (mem_idx == 0b100);
+    assert(!needssib && "untested");
+    int mode = getModeFromOffset(mem.offset, mem_idx);
+
+    if (-0x80 <= amount && amount < 0x80) {
+        emitByte(0x83);
+        if (needssib)
+            emitSIB(0b00, 0b100, mem_idx);
+        emitModRM(mode, opcode, mem_idx);
+        emitByte(amount);
+    } else {
+        emitByte(0x81);
+        if (needssib)
+            emitSIB(0b00, 0b100, mem_idx);
+        emitModRM(mode, opcode, mem_idx);
+        emitInt(amount, 4);
+    }
+}
 
 void Assembler::emitByte(uint8_t b) {
     if (addr >= end_addr) {
@@ -156,10 +192,12 @@ void Assembler::emitSIB(uint8_t scalebits, uint8_t index, uint8_t base) {
     emitByte((scalebits << 6) | (index << 3) | base);
 }
 
-int Assembler::getModeFromOffset(int offset) const {
-    if (offset == 0)
+int Assembler::getModeFromOffset(int offset, int reg_idx) const {
+    if (offset == 0) {
+        if (reg_idx == 0b101)
+            return 0b01;
         return 0b00;
-    else if (-0x80 <= offset && offset < 0x80)
+    } else if (-0x80 <= offset && offset < 0x80)
         return 0b01;
     else
         return 0b10;
@@ -198,7 +236,7 @@ void Assembler::movq(Immediate src, Indirect dest) {
     emitByte(0xc7);
 
     bool needssib = (dest_idx == 0b100);
-    int mode = getModeFromOffset(dest.offset);
+    int mode = getModeFromOffset(dest.offset, dest_idx);
     emitModRM(mode, 0, dest_idx);
 
     if (needssib)
@@ -258,7 +296,7 @@ void Assembler::mov(Register src, Indirect dest) {
     emitByte(0x89);
 
     bool needssib = (dest_idx == 0b100);
-    int mode = getModeFromOffset(dest.offset);
+    int mode = getModeFromOffset(dest.offset, dest_idx);
     emitModRM(mode, src_idx, dest_idx);
 
     if (needssib)
@@ -465,7 +503,7 @@ void Assembler::movsd(XMMRegister src, Indirect dest) {
     emitByte(0x11);
 
     bool needssib = (dest_idx == 0b100);
-    int mode = getModeFromOffset(dest.offset);
+    int mode = getModeFromOffset(dest.offset, dest_idx);
     emitModRM(mode, src_idx, dest_idx);
 
     if (needssib)
@@ -631,6 +669,10 @@ void Assembler::sub(Immediate imm, Register reg) {
     emitArith(imm, reg, OPCODE_SUB);
 }
 
+void Assembler::add(Immediate imm, Indirect mem) {
+    emitArith(imm, mem, OPCODE_ADD);
+}
+
 void Assembler::incl(Indirect mem) {
     int src_idx = mem.base.regnum;
 
@@ -910,7 +952,7 @@ void Assembler::lea(Indirect mem, Register reg) {
     emitByte(0x8D);
 
     bool needssib = (mem_idx == 0b100);
-    int mode = getModeFromOffset(mem.offset);
+    int mode = getModeFromOffset(mem.offset, mem_idx);
     emitModRM(mode, reg_idx, mem_idx);
 
     if (needssib)

src/asm_writing/assembler.h

@@ -88,8 +88,9 @@ private:
     void emitModRM(uint8_t mod, uint8_t reg, uint8_t rm);
     void emitSIB(uint8_t scalebits, uint8_t index, uint8_t base);
     void emitArith(Immediate imm, Register reg, int opcode);
+    void emitArith(Immediate imm, Indirect mem, int opcode);
 
-    int getModeFromOffset(int offset) const;
+    int getModeFromOffset(int offset, int reg_idx) const;
 
 public:
     Assembler(uint8_t* start, int size) : start_addr(start), end_addr(start + size), addr(start_addr), failed(false) {}
@@ -153,6 +154,7 @@ public:
     void pop(Register reg);
 
     void add(Immediate imm, Register reg);
+    void add(Immediate imm, Indirect mem);
     void sub(Immediate imm, Register reg);
 
     void incl(Indirect mem);

src/asm_writing/rewriter.cpp

@@ -512,7 +512,9 @@ void RewriterVar::xdecref() {
     }, { this }, ActionType::MUTATION);
 }
 
-void Rewriter::_incref(RewriterVar* var) {
+void Rewriter::_incref(RewriterVar* var, int num_refs) {
+    assert(num_refs > 0);
+
     // Small optimization: skip any time we want to do xincref(NULL)
     if (var->isConstant() && var->constant_value == 0) {
         assert(var->nullable);
@@ -533,7 +535,11 @@ void Rewriter::_incref(RewriterVar* var) {
     assembler->incq(assembler::Immediate(&_Py_RefTotal));
 #endif
     auto reg = var->getInReg();
-    assembler->incq(assembler::Indirect(reg, offsetof(Box, ob_refcnt)));
+
+    if (num_refs == 1)
+        assembler->incq(assembler::Indirect(reg, offsetof(Box, ob_refcnt)));
+     else
+        assembler->add(assembler::Immediate(num_refs), assembler::Indirect(reg, offsetof(Box, ob_refcnt)));
 
     // Doesn't call bumpUse, since this function is designed to be callable from other emitting functions.
     // (ie the caller should call bumpUse)
@@ -1195,12 +1201,7 @@ RewriterVar* RewriterVar::setType(RefType type) {
                 assert(num_needed_refs >= 0);
                 if (num_needed_refs > 0) {
                     if (rewriter->isDoneGuarding()) {
-                        assert(num_needed_refs == 1); // not bad just curious
-
-                        // XXX do a single add instead of multiple incs
-                        for (int i = 0; i < num_needed_refs; i++) {
-                            this->rewriter->_incref(this);
-                        }
+                        this->rewriter->_incref(this, num_needed_refs);
                     } else {
                         rewriter->pending_increfs.push_back(std::make_pair(this, num_needed_refs));
                     }

src/asm_writing/rewriter.h

@@ -565,7 +565,7 @@ protected:
     void _toBool(RewriterVar* result, RewriterVar* var, Location loc = Location::any());
 
     // These do not call bumpUse on their arguments:
-    void _incref(RewriterVar* var);
+    void _incref(RewriterVar* var, int num_refs=1);
     void _decref(RewriterVar* var);
     void _xdecref(RewriterVar* var);