byop-engine/vendor/github.com/cloudwego/iasm/x86_64/encodings.go

//
// Copyright 2024 CloudWeGo Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package x86_64

import (
    `encoding/binary`
    `math`
)

/** Operand Encoding Helpers **/

func imml(v interface{}) byte {
    return byte(toImmAny(v) & 0x0f)
}

func relv(v interface{}) int64 {
    switch r := v.(type) {
        case *Label         : return 0
        case RelativeOffset : return int64(r)
        default             : panic("invalid relative offset")
    }
}

func addr(v interface{}) interface{} {
    switch a := v.(*MemoryOperand).Addr; a.Type {
        case Memory    : return a.Memory
        case Offset    : return a.Offset
        case Reference : return a.Reference
        default        : panic("invalid memory operand type")
    }
}

func bcode(v interface{}) byte {
    if m, ok := v.(*MemoryOperand); !ok {
        panic("v is not a memory operand")
    } else if m.Broadcast == 0 {
        return 0
    } else {
        return 1
    }
}

func vcode(v interface{}) byte {
    switch r := v.(type) {
        case XMMRegister    : return byte(r)
        case YMMRegister    : return byte(r)
        case ZMMRegister    : return byte(r)
        case MaskedRegister : return vcode(r.Reg)
        default             : panic("v is not a vector register")
    }
}

func kcode(v interface{}) byte {
    switch r := v.(type) {
        case KRegister      : return byte(r)
        case XMMRegister    : return 0
        case YMMRegister    : return 0
        case ZMMRegister    : return 0
        case RegisterMask   : return byte(r.K)
        case MaskedRegister : return byte(r.Mask.K)
        case *MemoryOperand : return toKcodeMem(r)
        default             : panic("v is not a maskable operand")
    }
}

func zcode(v interface{}) byte {
    switch r := v.(type) {
        case KRegister      : return 0
        case XMMRegister    : return 0
        case YMMRegister    : return 0
        case ZMMRegister    : return 0
        case RegisterMask   : return toZcodeRegM(r)
        case MaskedRegister : return toZcodeRegM(r.Mask)
        case *MemoryOperand : return toZcodeMem(r)
        default             : panic("v is not a maskable operand")
    }
}

func lcode(v interface{}) byte {
    switch r := v.(type) {
        case Register8      : return byte(r & 0x07)
        case Register16     : return byte(r & 0x07)
        case Register32     : return byte(r & 0x07)
        case Register64     : return byte(r & 0x07)
        case KRegister      : return byte(r & 0x07)
        case MMRegister     : return byte(r & 0x07)
        case XMMRegister    : return byte(r & 0x07)
        case YMMRegister    : return byte(r & 0x07)
        case ZMMRegister    : return byte(r & 0x07)
        case MaskedRegister : return lcode(r.Reg)
        default             : panic("v is not a register")
    }
}

func hcode(v interface{}) byte {
    switch r := v.(type) {
        case Register8      : return byte(r >> 3) & 1
        case Register16     : return byte(r >> 3) & 1
        case Register32     : return byte(r >> 3) & 1
        case Register64     : return byte(r >> 3) & 1
        case KRegister      : return byte(r >> 3) & 1
        case MMRegister     : return byte(r >> 3) & 1
        case XMMRegister    : return byte(r >> 3) & 1
        case YMMRegister    : return byte(r >> 3) & 1
        case ZMMRegister    : return byte(r >> 3) & 1
        case MaskedRegister : return hcode(r.Reg)
        default             : panic("v is not a register")
    }
}

func ecode(v interface{}) byte {
    switch r := v.(type) {
        case Register8      : return byte(r >> 4) & 1
        case Register16     : return byte(r >> 4) & 1
        case Register32     : return byte(r >> 4) & 1
        case Register64     : return byte(r >> 4) & 1
        case KRegister      : return byte(r >> 4) & 1
        case MMRegister     : return byte(r >> 4) & 1
        case XMMRegister    : return byte(r >> 4) & 1
        case YMMRegister    : return byte(r >> 4) & 1
        case ZMMRegister    : return byte(r >> 4) & 1
        case MaskedRegister : return ecode(r.Reg)
        default             : panic("v is not a register")
    }
}

func hlcode(v interface{}) byte {
    switch r := v.(type) {
        case Register8      : return toHLcodeReg8(r)
        case Register16     : return byte(r & 0x0f)
        case Register32     : return byte(r & 0x0f)
        case Register64     : return byte(r & 0x0f)
        case KRegister      : return byte(r & 0x0f)
        case MMRegister     : return byte(r & 0x0f)
        case XMMRegister    : return byte(r & 0x0f)
        case YMMRegister    : return byte(r & 0x0f)
        case ZMMRegister    : return byte(r & 0x0f)
        case MaskedRegister : return hlcode(r.Reg)
        default             : panic("v is not a register")
    }
}

func ehcode(v interface{}) byte {
    switch r := v.(type) {
        case Register8      : return byte(r >> 3) & 0x03
        case Register16     : return byte(r >> 3) & 0x03
        case Register32     : return byte(r >> 3) & 0x03
        case Register64     : return byte(r >> 3) & 0x03
        case KRegister      : return byte(r >> 3) & 0x03
        case MMRegister     : return byte(r >> 3) & 0x03
        case XMMRegister    : return byte(r >> 3) & 0x03
        case YMMRegister    : return byte(r >> 3) & 0x03
        case ZMMRegister    : return byte(r >> 3) & 0x03
        case MaskedRegister : return ehcode(r.Reg)
        default             : panic("v is not a register")
    }
}

func toImmAny(v interface{}) int64 {
    if x, ok := asInt64(v); ok {
        return x
    } else {
        panic("value is not an integer")
    }
}

func toHcodeOpt(v interface{}) byte {
    if v == nil {
        return 0
    } else {
        return hcode(v)
    }
}

func toEcodeVMM(v interface{}, x byte) byte {
    switch r := v.(type) {
        case XMMRegister : return ecode(r)
        case YMMRegister : return ecode(r)
        case ZMMRegister : return ecode(r)
        default          : return x
    }
}

func toKcodeMem(v *MemoryOperand) byte {
    if !v.Masked {
        return 0
    } else {
        return byte(v.Mask.K)
    }
}

func toZcodeMem(v *MemoryOperand) byte {
    if !v.Masked || v.Mask.Z {
        return 0
    } else {
        return 1
    }
}

func toZcodeRegM(v RegisterMask) byte {
    if v.Z {
        return 1
    } else {
        return 0
    }
}

func toHLcodeReg8(v Register8) byte {
    switch v {
        case AH: fallthrough
        case BH: fallthrough
        case CH: fallthrough
        case DH: panic("ah/bh/ch/dh registers never use 4-bit encoding")
        default: return byte(v & 0x0f)
    }
}

/** Instruction Encoding Helpers **/

const (
    _N_inst = 16
)

const (
    _F_rel1 = 1 << iota
    _F_rel4
)

type _Encoding struct {
    len     int
    flags   int
    bytes   [_N_inst]byte
    encoder func(m *_Encoding, v []interface{})
}

// buf ensures len + n <= len(bytes).
func (self *_Encoding) buf(n int) []byte {
    if i := self.len; i + n > _N_inst {
        panic("instruction too long")
    } else {
        return self.bytes[i:]
    }
}

// emit encodes a single byte.
func (self *_Encoding) emit(v byte) {
    self.buf(1)[0] = v
    self.len++
}

// imm1 encodes a single byte immediate value.
func (self *_Encoding) imm1(v int64) {
    self.emit(byte(v))
}

// imm2 encodes a two-byte immediate value in little-endian.
func (self *_Encoding) imm2(v int64) {
    binary.LittleEndian.PutUint16(self.buf(2), uint16(v))
    self.len += 2
}

// imm4 encodes a 4-byte immediate value in little-endian.
func (self *_Encoding) imm4(v int64) {
    binary.LittleEndian.PutUint32(self.buf(4), uint32(v))
    self.len += 4
}

// imm8 encodes an 8-byte immediate value in little-endian.
func (self *_Encoding) imm8(v int64) {
    binary.LittleEndian.PutUint64(self.buf(8), uint64(v))
    self.len += 8
}

// vex2 encodes a 2-byte or 3-byte VEX prefix.
//
//                          2-byte VEX prefix:
// Requires: VEX.W = 0, VEX.mmmmm = 0b00001 and VEX.B = VEX.X = 0
//         +----------------+
// Byte 0: | Bits 0-7: 0xc5 |
//         +----------------+
//
//         +-----------+----------------+----------+--------------+
// Byte 1: | Bit 7: ~R | Bits 3-6 ~vvvv | Bit 2: L | Bits 0-1: pp |
//         +-----------+----------------+----------+--------------+
//
//                          3-byte VEX prefix:
//         +----------------+
// Byte 0: | Bits 0-7: 0xc4 |
//         +----------------+
//
//         +-----------+-----------+-----------+-------------------+
// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: 0b00001 |
//         +-----------+-----------+-----------+-------------------+
//
//         +----------+-----------------+----------+--------------+
// Byte 2: | Bit 7: 0 | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp |
//         +----------+-----------------+----------+--------------+
//
func (self *_Encoding) vex2(lpp byte, r byte, rm interface{}, vvvv byte) {
    var b byte
    var x byte

    /* VEX.R must be a single-bit mask */
    if r > 1 {
        panic("VEX.R must be a 1-bit mask")
    }

    /* VEX.Lpp must be a 3-bit mask */
    if lpp &^ 0b111 != 0 {
        panic("VEX.Lpp must be a 3-bit mask")
    }

    /* VEX.vvvv must be a 4-bit mask */
    if vvvv &^ 0b1111 != 0 {
        panic("VEX.vvvv must be a 4-bit mask")
    }

    /* encode the RM bits if any */
    if rm != nil {
        switch v := rm.(type) {
            case *Label         : break
            case Register       : b = hcode(v)
            case MemoryAddress  : b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index)
            case RelativeOffset : break
            default             : panic("rm is expected to be a register or a memory address")
        }
    }

    /* if VEX.B and VEX.X are zeroes, 2-byte VEX prefix can be used */
    if x == 0 && b == 0 {
        self.emit(0xc5)
        self.emit(0xf8 ^ (r << 7) ^ (vvvv << 3) ^ lpp)
    } else {
        self.emit(0xc4)
        self.emit(0xe1 ^ (r << 7) ^ (x << 6) ^ (b << 5))
        self.emit(0x78 ^ (vvvv << 3) ^ lpp)
    }
}

// vex3 encodes a 3-byte VEX or XOP prefix.
//
//                         3-byte VEX/XOP prefix
//         +-----------------------------------+
// Byte 0: | Bits 0-7: 0xc4 (VEX) / 0x8f (XOP) |
//         +-----------------------------------+
//
//         +-----------+-----------+-----------+-----------------+
// Byte 1: | Bit 7: ~R | Bit 6: ~X | Bit 5: ~B | Bits 0-4: mmmmm |
//         +-----------+-----------+-----------+-----------------+
//
//         +----------+-----------------+----------+--------------+
// Byte 2: | Bit 7: W | Bits 3-6: ~vvvv | Bit 2: L | Bits 0-1: pp |
//         +----------+-----------------+----------+--------------+
//
func (self *_Encoding) vex3(esc byte, mmmmm byte, wlpp byte, r byte, rm interface{}, vvvv byte) {
    var b byte
    var x byte

    /* VEX.R must be a single-bit mask */
    if r > 1 {
        panic("VEX.R must be a 1-bit mask")
    }

    /* VEX.vvvv must be a 4-bit mask */
    if vvvv &^ 0b1111 != 0 {
        panic("VEX.vvvv must be a 4-bit mask")
    }

    /* escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix */
    if esc != 0xc4 && esc != 0x8f {
        panic("escape must be a 3-byte VEX (0xc4) or XOP (0x8f) prefix")
    }

    /* VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7 */
    if wlpp &^ 0b10000111 != 0 {
        panic("VEX.W____Lpp is expected to have no bits set except 0, 1, 2 and 7")
    }

    /* VEX.m-mmmm is expected to be a 5-bit mask */
    if mmmmm &^ 0b11111 != 0 {
        panic("VEX.m-mmmm is expected to be a 5-bit mask")
    }

    /* encode the RM bits */
    switch v := rm.(type) {
        case *Label         : break
        case MemoryAddress  : b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index)
        case RelativeOffset : break
        default             : panic("rm is expected to be a register or a memory address")
    }

    /* encode the 3-byte VEX or XOP prefix */
    self.emit(esc)
    self.emit(0xe0 ^ (r << 7) ^ (x << 6) ^ (b << 5) ^ mmmmm)
    self.emit(0x78 ^ (vvvv << 3) ^ wlpp)
}

// evex encodes a 4-byte EVEX prefix.
func (self *_Encoding) evex(mm byte, w1pp byte, ll byte, rr byte, rm interface{}, vvvvv byte, aaa byte, zz byte, bb byte) {
    var b byte
    var x byte

    /* EVEX.b must be a single-bit mask */
    if bb > 1 {
        panic("EVEX.b must be a 1-bit mask")
    }

    /* EVEX.z must be a single-bit mask */
    if zz > 1 {
        panic("EVEX.z must be a 1-bit mask")
    }

    /* EVEX.mm must be a 2-bit mask */
    if mm &^ 0b11 != 0 {
        panic("EVEX.mm must be a 2-bit mask")
    }

    /* EVEX.L'L must be a 2-bit mask */
    if ll &^ 0b11 != 0 {
        panic("EVEX.L'L must be a 2-bit mask")
    }

    /* EVEX.R'R must be a 2-bit mask */
    if rr &^ 0b11 != 0 {
        panic("EVEX.R'R must be a 2-bit mask")
    }

    /* EVEX.aaa must be a 3-bit mask */
    if aaa &^ 0b111 != 0 {
        panic("EVEX.aaa must be a 3-bit mask")
    }

    /* EVEX.v'vvvv must be a 5-bit mask */
    if vvvvv &^ 0b11111 != 0 {
        panic("EVEX.v'vvvv must be a 5-bit mask")
    }

    /* EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7 */
    if w1pp &^ 0b10000011 != 0b100 {
        panic("EVEX.W____1pp is expected to have no bits set except 0, 1, 2, and 7")
    }

    /* extract bits from EVEX.R'R and EVEX.v'vvvv */
    r1, r0 := rr >> 1, rr & 1
    v1, v0 := vvvvv >> 4, vvvvv & 0b1111

    /* encode the RM bits if any */
    if rm != nil {
        switch m := rm.(type) {
            case *Label         : break
            case Register       : b, x = hcode(m), ecode(m)
            case MemoryAddress  : b, x, v1 = toHcodeOpt(m.Base), toHcodeOpt(m.Index), toEcodeVMM(m.Index, v1)
            case RelativeOffset : break
            default             : panic("rm is expected to be a register or a memory address")
        }
    }

    /* EVEX prefix bytes */
    p0 := (r0 << 7) | (x << 6) | (b << 5) | (r1 << 4) | mm
    p1 := (v0 << 3) | w1pp
    p2 := (zz << 7) | (ll << 5) | (b << 4) | (v1 << 3) | aaa

    /* p0: invert RXBR' (bits 4-7)
     * p1: invert vvvv  (bits 3-6)
     * p2: invert V'    (bit  3) */
    self.emit(0x62)
    self.emit(p0 ^ 0xf0)
    self.emit(p1 ^ 0x78)
    self.emit(p2 ^ 0x08)
}

// rexm encodes a mandatory REX prefix.
func (self *_Encoding) rexm(w byte, r byte, rm interface{}) {
    var b byte
    var x byte

    /* REX.R must be 0 or 1 */
    if r != 0 && r != 1 {
        panic("REX.R must be 0 or 1")
    }

    /* REX.W must be 0 or 1 */
    if w != 0 && w != 1 {
        panic("REX.W must be 0 or 1")
    }

    /* encode the RM bits */
    switch v := rm.(type) {
        case *Label         : break
        case MemoryAddress  : b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index)
        case RelativeOffset : break
        default             : panic("rm is expected to be a register or a memory address")
    }

    /* encode the REX prefix */
    self.emit(0x40 | (w << 3) | (r << 2) | (x << 1) | b)
}

// rexo encodes an optional REX prefix.
func (self *_Encoding) rexo(r byte, rm interface{}, force bool) {
    var b byte
    var x byte

    /* REX.R must be 0 or 1 */
    if r != 0 && r != 1 {
        panic("REX.R must be 0 or 1")
    }

    /* encode the RM bits */
    switch v := rm.(type) {
        case *Label         : break
        case Register       : b = hcode(v)
        case MemoryAddress  : b, x = toHcodeOpt(v.Base), toHcodeOpt(v.Index)
        case RelativeOffset : break
        default             : panic("rm is expected to be a register or a memory address")
    }

    /* if REX.R, REX.X, and REX.B are all zeroes, REX prefix can be omitted */
    if force || r != 0 || x != 0 || b != 0 {
        self.emit(0x40 | (r << 2) | (x << 1) | b)
    }
}

// mrsd encodes ModR/M, SIB and Displacement.
//
//                    ModR/M byte
// +----------------+---------------+---------------+
// | Bits 6-7: Mode | Bits 3-5: Reg | Bits 0-2: R/M |
// +----------------+---------------+---------------+
//
//                         SIB byte
// +-----------------+-----------------+----------------+
// | Bits 6-7: Scale | Bits 3-5: Index | Bits 0-2: Base |
// +-----------------+-----------------+----------------+
//
func (self *_Encoding) mrsd(reg byte, rm interface{}, disp8v int32) {
    var ok bool
    var mm MemoryAddress
    var ro RelativeOffset

    /* ModRM encodes the lower 3-bit of the register */
    if reg > 7 {
        panic("invalid register bits")
    }

    /* check the displacement scale */
    switch disp8v {
        case  1: break
        case  2: break
        case  4: break
        case  8: break
        case 16: break
        case 32: break
        case 64: break
        default: panic("invalid displacement size")
    }

    /* special case: unresolved labels, assuming a zero offset */
    if _, ok = rm.(*Label); ok {
        self.emit(0x05 | (reg << 3))
        self.imm4(0)
        return
    }

    /* special case: RIP-relative offset
     * ModRM.Mode == 0 and ModeRM.R/M == 5 indicates (rip + disp32) addressing */
    if ro, ok = rm.(RelativeOffset); ok {
        self.emit(0x05 | (reg << 3))
        self.imm4(int64(ro))
        return
    }

    /* must be a generic memory address */
    if mm, ok = rm.(MemoryAddress); !ok {
        panic("rm must be a memory address")
    }

    /* absolute addressing, encoded as disp(%rbp,%rsp,1) */
    if mm.Base == nil && mm.Index == nil {
        self.emit(0x04 | (reg << 3))
        self.emit(0x25)
        self.imm4(int64(mm.Displacement))
        return
    }

    /* no SIB byte */
    if mm.Index == nil && lcode(mm.Base) != 0b100 {
        cc := lcode(mm.Base)
        dv := mm.Displacement

        /* ModRM.Mode == 0 (no displacement) */
        if dv == 0 && mm.Base != RBP && mm.Base != R13 {
            if cc == 0b101 {
                panic("rbp/r13 is not encodable as a base register (interpreted as disp32 address)")
            } else {
                self.emit((reg << 3) | cc)
                return
            }
        }

        /* ModRM.Mode == 1 (8-bit displacement) */
        if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv % disp8v == 0 {
            self.emit(0x40 | (reg << 3) | cc)
            self.imm1(int64(dq))
            return
        }

        /* ModRM.Mode == 2 (32-bit displacement) */
        self.emit(0x80 | (reg << 3) | cc)
        self.imm4(int64(mm.Displacement))
        return
    }

    /* all encodings below use ModRM.R/M = 4 (0b100) to indicate the presence of SIB */
    if mm.Index == RSP {
        panic("rsp is not encodable as an index register (interpreted as no index)")
    }

    /* index = 4 (0b100) denotes no-index encoding */
    var scale byte
    var index byte = 0x04

    /* encode the scale byte */
    if mm.Scale != 0 {
        switch mm.Scale {
            case 1  : scale = 0
            case 2  : scale = 1
            case 4  : scale = 2
            case 8  : scale = 3
            default : panic("invalid scale value")
        }
    }

    /* encode the index byte */
    if mm.Index != nil {
        index = lcode(mm.Index)
    }

    /* SIB.Base = 5 (0b101) and ModRM.Mode = 0 indicates no-base encoding with disp32 */
    if mm.Base == nil {
        self.emit((reg << 3) | 0b100)
        self.emit((scale << 6) | (index << 3) | 0b101)
        self.imm4(int64(mm.Displacement))
        return
    }

    /* base L-code & displacement value */
    cc := lcode(mm.Base)
    dv := mm.Displacement

    /* ModRM.Mode == 0 (no displacement) */
    if dv == 0 && cc != 0b101 {
        self.emit((reg << 3) | 0b100)
        self.emit((scale << 6) | (index << 3) | cc)
        return
    }

    /* ModRM.Mode == 1 (8-bit displacement) */
    if dq := dv / disp8v; dq >= math.MinInt8 && dq <= math.MaxInt8 && dv % disp8v == 0 {
        self.emit(0x44 | (reg << 3))
        self.emit((scale << 6) | (index << 3) | cc)
        self.imm1(int64(dq))
        return
    }

    /* ModRM.Mode == 2 (32-bit displacement) */
    self.emit(0x84 | (reg << 3))
    self.emit((scale << 6) | (index << 3) | cc)
    self.imm4(int64(mm.Displacement))
}

// encode invokes the encoder to encode this instruction.
func (self *_Encoding) encode(v []interface{}) int {
    self.len = 0
    self.encoder(self, v)
    return self.len
}