OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 6
Code Issues Packages Projects Releases Wiki Activity
llvm-project/llvm/tools/llvm-exegesis/lib/X86/Target.cpp

970 lines
33 KiB
C++
Raw Blame History

//===-- Target.cpp ----------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../Target.h"
#include "../Error.h"
#include "../ParallelSnippetGenerator.h"
#include "../SerialSnippetGenerator.h"
#include "../SnippetGenerator.h"
#include "MCTargetDesc/X86BaseInfo.h"
#include "MCTargetDesc/X86MCTargetDesc.h"
#include "X86.h"
#include "X86Counter.h"
#include "X86RegisterInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Host.h"
#include <memory>
#include <string>
#include <vector>
#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
#include <immintrin.h>
#include <intrin.h>
#endif
namespace llvm {
namespace exegesis {
static cl::OptionCategory
BenchmarkOptions("llvm-exegesis benchmark x86-options");
// If a positive value is specified, we are going to use the LBR in
// latency-mode.
//
// Note:
// - A small value is preferred, but too low a value could result in
// throttling.
// - A prime number is preferred to avoid always skipping certain blocks.
//
static cl::opt<unsigned> LbrSamplingPeriod(
"x86-lbr-sample-period",
cl::desc("The sample period (nbranches/sample), used for LBR sampling"),
cl::cat(BenchmarkOptions), cl::init(0));
// FIXME: Validates that repetition-mode is loop if LBR is requested.
// Returns a non-null reason if we cannot handle the memory references in this
// instruction.
static const char *isInvalidMemoryInstr(const Instruction &Instr) {
switch (Instr.Description.TSFlags & X86II::FormMask) {
default:
return "Unknown FormMask value";
// These have no memory access.
case X86II::Pseudo:
case X86II::RawFrm:
case X86II::AddCCFrm:
case X86II::PrefixByte:
case X86II::MRMDestReg:
case X86II::MRMSrcReg:
case X86II::MRMSrcReg4VOp3:
case X86II::MRMSrcRegOp4:
case X86II::MRMSrcRegCC:
case X86II::MRMXrCC:
case X86II::MRMr0:
case X86II::MRMXr:
case X86II::MRM0r:
case X86II::MRM1r:
case X86II::MRM2r:
case X86II::MRM3r:
case X86II::MRM4r:
case X86II::MRM5r:
case X86II::MRM6r:
case X86II::MRM7r:
case X86II::MRM0X:
case X86II::MRM1X:
case X86II::MRM2X:
case X86II::MRM3X:
case X86II::MRM4X:
case X86II::MRM5X:
case X86II::MRM6X:
case X86II::MRM7X:
case X86II::MRM_C0:
case X86II::MRM_C1:
case X86II::MRM_C2:
case X86II::MRM_C3:
case X86II::MRM_C4:
case X86II::MRM_C5:
case X86II::MRM_C6:
case X86II::MRM_C7:
case X86II::MRM_C8:
case X86II::MRM_C9:
case X86II::MRM_CA:
case X86II::MRM_CB:
case X86II::MRM_CC:
case X86II::MRM_CD:
case X86II::MRM_CE:
case X86II::MRM_CF:
case X86II::MRM_D0:
case X86II::MRM_D1:
case X86II::MRM_D2:
case X86II::MRM_D3:
case X86II::MRM_D4:
case X86II::MRM_D5:
case X86II::MRM_D6:
case X86II::MRM_D7:
case X86II::MRM_D8:
case X86II::MRM_D9:
case X86II::MRM_DA:
case X86II::MRM_DB:
case X86II::MRM_DC:
case X86II::MRM_DD:
case X86II::MRM_DE:
case X86II::MRM_DF:
case X86II::MRM_E0:
case X86II::MRM_E1:
case X86II::MRM_E2:
case X86II::MRM_E3:
case X86II::MRM_E4:
case X86II::MRM_E5:
case X86II::MRM_E6:
case X86II::MRM_E7:
case X86II::MRM_E8:
case X86II::MRM_E9:
case X86II::MRM_EA:
case X86II::MRM_EB:
case X86II::MRM_EC:
case X86II::MRM_ED:
case X86II::MRM_EE:
case X86II::MRM_EF:
case X86II::MRM_F0:
case X86II::MRM_F1:
case X86II::MRM_F2:
case X86II::MRM_F3:
case X86II::MRM_F4:
case X86II::MRM_F5:
case X86II::MRM_F6:
case X86II::MRM_F7:
case X86II::MRM_F8:
case X86II::MRM_F9:
case X86II::MRM_FA:
case X86II::MRM_FB:
case X86II::MRM_FC:
case X86II::MRM_FD:
case X86II::MRM_FE:
case X86II::MRM_FF:
case X86II::RawFrmImm8:
return nullptr;
case X86II::AddRegFrm:
return (Instr.Description.Opcode == X86::POP16r ||
Instr.Description.Opcode == X86::POP32r ||
Instr.Description.Opcode == X86::PUSH16r ||
Instr.Description.Opcode == X86::PUSH32r)
? "unsupported opcode: unsupported memory access"
: nullptr;
// These access memory and are handled.
case X86II::MRMDestMem:
case X86II::MRMSrcMem:
case X86II::MRMSrcMem4VOp3:
case X86II::MRMSrcMemOp4:
case X86II::MRMSrcMemCC:
case X86II::MRMXmCC:
case X86II::MRMXm:
case X86II::MRM0m:
case X86II::MRM1m:
case X86II::MRM2m:
case X86II::MRM3m:
case X86II::MRM4m:
case X86II::MRM5m:
case X86II::MRM6m:
case X86II::MRM7m:
return nullptr;
// These access memory and are not handled yet.
case X86II::RawFrmImm16:
case X86II::RawFrmMemOffs:
case X86II::RawFrmSrc:
case X86II::RawFrmDst:
case X86II::RawFrmDstSrc:
return "unsupported opcode: non uniform memory access";
}
}
// If the opcode is invalid, returns a pointer to a character literal indicating
// the reason. nullptr indicates a valid opcode.
static const char *isInvalidOpcode(const Instruction &Instr) {
const auto OpcodeName = Instr.Name;
if ((Instr.Description.TSFlags & X86II::FormMask) == X86II::Pseudo)
return "unsupported opcode: pseudo instruction";
if ((OpcodeName.startswith("POP") && !OpcodeName.startswith("POPCNT")) ||
OpcodeName.startswith("PUSH") || OpcodeName.startswith("ADJCALLSTACK") ||
OpcodeName.startswith("LEAVE"))
return "unsupported opcode: Push/Pop/AdjCallStack/Leave";
switch (Instr.Description.Opcode) {
case X86::LFS16rm:
case X86::LFS32rm:
case X86::LFS64rm:
case X86::LGS16rm:
case X86::LGS32rm:
case X86::LGS64rm:
case X86::LSS16rm:
case X86::LSS32rm:
case X86::LSS64rm:
case X86::SYSENTER:
return "unsupported opcode";
default:
break;
}
if (const auto reason = isInvalidMemoryInstr(Instr))
return reason;
// We do not handle instructions with OPERAND_PCREL.
for (const Operand &Op : Instr.Operands)
if (Op.isExplicit() &&
Op.getExplicitOperandInfo().OperandType == MCOI::OPERAND_PCREL)
return "unsupported opcode: PC relative operand";
// We do not handle second-form X87 instructions. We only handle first-form
// ones (_Fp), see comment in X86InstrFPStack.td.
for (const Operand &Op : Instr.Operands)
if (Op.isReg() && Op.isExplicit() &&
Op.getExplicitOperandInfo().RegClass == X86::RSTRegClassID)
return "unsupported second-form X87 instruction";
return nullptr;
}
static unsigned getX86FPFlags(const Instruction &Instr) {
return Instr.Description.TSFlags & X86II::FPTypeMask;
}
// Helper to fill a memory operand with a value.
static void setMemOp(InstructionTemplate &IT, int OpIdx,
const MCOperand &OpVal) {
const auto Op = IT.getInstr().Operands[OpIdx];
assert(Op.isExplicit() && "invalid memory pattern");
IT.getValueFor(Op) = OpVal;
}
// Common (latency, uops) code for LEA templates. `GetDestReg` takes the
// addressing base and index registers and returns the LEA destination register.
static Expected<std::vector<CodeTemplate>> generateLEATemplatesCommon(
const Instruction &Instr, const BitVector &ForbiddenRegisters,
const LLVMState &State, const SnippetGenerator::Options &Opts,
std::function<void(unsigned, unsigned, BitVector &CandidateDestRegs)>
RestrictDestRegs) {
assert(Instr.Operands.size() == 6 && "invalid LEA");
assert(X86II::getMemoryOperandNo(Instr.Description.TSFlags) == 1 &&
"invalid LEA");
constexpr const int kDestOp = 0;
constexpr const int kBaseOp = 1;
constexpr const int kIndexOp = 3;
auto PossibleDestRegs =
Instr.Operands[kDestOp].getRegisterAliasing().sourceBits();
remove(PossibleDestRegs, ForbiddenRegisters);
auto PossibleBaseRegs =
Instr.Operands[kBaseOp].getRegisterAliasing().sourceBits();
remove(PossibleBaseRegs, ForbiddenRegisters);
auto PossibleIndexRegs =
Instr.Operands[kIndexOp].getRegisterAliasing().sourceBits();
remove(PossibleIndexRegs, ForbiddenRegisters);
const auto &RegInfo = State.getRegInfo();
std::vector<CodeTemplate> Result;
for (const unsigned BaseReg : PossibleBaseRegs.set_bits()) {
for (const unsigned IndexReg : PossibleIndexRegs.set_bits()) {
for (int LogScale = 0; LogScale <= 3; ++LogScale) {
// FIXME: Add an option for controlling how we explore immediates.
for (const int Disp : {0, 42}) {
InstructionTemplate IT(&Instr);
const int64_t Scale = 1ull << LogScale;
setMemOp(IT, 1, MCOperand::createReg(BaseReg));
setMemOp(IT, 2, MCOperand::createImm(Scale));
setMemOp(IT, 3, MCOperand::createReg(IndexReg));
setMemOp(IT, 4, MCOperand::createImm(Disp));
// SegmentReg must be 0 for LEA.
setMemOp(IT, 5, MCOperand::createReg(0));
// Output reg candidates are selected by the caller.
auto PossibleDestRegsNow = PossibleDestRegs;
RestrictDestRegs(BaseReg, IndexReg, PossibleDestRegsNow);
assert(PossibleDestRegsNow.set_bits().begin() !=
PossibleDestRegsNow.set_bits().end() &&
"no remaining registers");
setMemOp(
IT, 0,
MCOperand::createReg(*PossibleDestRegsNow.set_bits().begin()));
CodeTemplate CT;
CT.Instructions.push_back(std::move(IT));
CT.Config = formatv("{3}(%{0}, %{1}, {2})", RegInfo.getName(BaseReg),
RegInfo.getName(IndexReg), Scale, Disp)
.str();
Result.push_back(std::move(CT));
if (Result.size() >= Opts.MaxConfigsPerOpcode)
return std::move(Result);
}
}
}
}
return std::move(Result);
}
namespace {
class X86SerialSnippetGenerator : public SerialSnippetGenerator {
public:
using SerialSnippetGenerator::SerialSnippetGenerator;
Expected<std::vector<CodeTemplate>>
generateCodeTemplates(InstructionTemplate Variant,
const BitVector &ForbiddenRegisters) const override;
};
} // namespace
Expected<std::vector<CodeTemplate>>
X86SerialSnippetGenerator::generateCodeTemplates(
InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
const Instruction &Instr = Variant.getInstr();
if (const auto reason = isInvalidOpcode(Instr))
return make_error<Failure>(reason);
// LEA gets special attention.
const auto Opcode = Instr.Description.getOpcode();
if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
return generateLEATemplatesCommon(
Instr, ForbiddenRegisters, State, Opts,
[this](unsigned BaseReg, unsigned IndexReg,
BitVector &CandidateDestRegs) {
// We just select a destination register that aliases the base
// register.
CandidateDestRegs &=
State.getRATC().getRegister(BaseReg).aliasedBits();
});
}
if (Instr.hasMemoryOperands())
return make_error<Failure>(
"unsupported memory operand in latency measurements");
switch (getX86FPFlags(Instr)) {
case X86II::NotFP:
return SerialSnippetGenerator::generateCodeTemplates(Variant,
ForbiddenRegisters);
case X86II::ZeroArgFP:
case X86II::OneArgFP:
case X86II::SpecialFP:
case X86II::CompareFP:
case X86II::CondMovFP:
return make_error<Failure>("Unsupported x87 Instruction");
case X86II::OneArgFPRW:
case X86II::TwoArgFP:
// These are instructions like
// - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
// - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
// They are intrinsically serial and do not modify the state of the stack.
return generateSelfAliasingCodeTemplates(Variant);
default:
llvm_unreachable("Unknown FP Type!");
}
}
namespace {
class X86ParallelSnippetGenerator : public ParallelSnippetGenerator {
public:
using ParallelSnippetGenerator::ParallelSnippetGenerator;
Expected<std::vector<CodeTemplate>>
generateCodeTemplates(InstructionTemplate Variant,
const BitVector &ForbiddenRegisters) const override;
};
} // namespace
Expected<std::vector<CodeTemplate>>
X86ParallelSnippetGenerator::generateCodeTemplates(
InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const {
const Instruction &Instr = Variant.getInstr();
if (const auto reason = isInvalidOpcode(Instr))
return make_error<Failure>(reason);
// LEA gets special attention.
const auto Opcode = Instr.Description.getOpcode();
if (Opcode == X86::LEA64r || Opcode == X86::LEA64_32r) {
return generateLEATemplatesCommon(
Instr, ForbiddenRegisters, State, Opts,
[this](unsigned BaseReg, unsigned IndexReg,
BitVector &CandidateDestRegs) {
// Any destination register that is not used for addressing is fine.
remove(CandidateDestRegs,
State.getRATC().getRegister(BaseReg).aliasedBits());
remove(CandidateDestRegs,
State.getRATC().getRegister(IndexReg).aliasedBits());
});
}
switch (getX86FPFlags(Instr)) {
case X86II::NotFP:
return ParallelSnippetGenerator::generateCodeTemplates(Variant,
ForbiddenRegisters);
case X86II::ZeroArgFP:
case X86II::OneArgFP:
case X86II::SpecialFP:
return make_error<Failure>("Unsupported x87 Instruction");
case X86II::OneArgFPRW:
case X86II::TwoArgFP:
// These are instructions like
// - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
// - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
// They are intrinsically serial and do not modify the state of the stack.
// We generate the same code for latency and uops.
return generateSelfAliasingCodeTemplates(Variant);
case X86II::CompareFP:
case X86II::CondMovFP:
// We can compute uops for any FP instruction that does not grow or shrink
// the stack (either do not touch the stack or push as much as they pop).
return generateUnconstrainedCodeTemplates(
Variant, "instruction does not grow/shrink the FP stack");
default:
llvm_unreachable("Unknown FP Type!");
}
}
static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
switch (RegBitWidth) {
case 8:
return X86::MOV8ri;
case 16:
return X86::MOV16ri;
case 32:
return X86::MOV32ri;
case 64:
return X86::MOV64ri;
}
llvm_unreachable("Invalid Value Width");
}
// Generates instruction to load an immediate value into a register.
static MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
const APInt &Value) {
if (Value.getBitWidth() > RegBitWidth)
llvm_unreachable("Value must fit in the Register");
return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
.addReg(Reg)
.addImm(Value.getZExtValue());
}
// Allocates scratch memory on the stack.
static MCInst allocateStackSpace(unsigned Bytes) {
return MCInstBuilder(X86::SUB64ri8)
.addReg(X86::RSP)
.addReg(X86::RSP)
.addImm(Bytes);
}
// Fills scratch memory at offset `OffsetBytes` with value `Imm`.
static MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
uint64_t Imm) {
return MCInstBuilder(MovOpcode)
// Address = ESP
.addReg(X86::RSP) // BaseReg
.addImm(1) // ScaleAmt
.addReg(0) // IndexReg
.addImm(OffsetBytes) // Disp
.addReg(0) // Segment
// Immediate.
.addImm(Imm);
}
// Loads scratch memory into register `Reg` using opcode `RMOpcode`.
static MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
return MCInstBuilder(RMOpcode)
.addReg(Reg)
// Address = ESP
.addReg(X86::RSP) // BaseReg
.addImm(1) // ScaleAmt
.addReg(0) // IndexReg
.addImm(0) // Disp
.addReg(0); // Segment
}
// Releases scratch memory.
static MCInst releaseStackSpace(unsigned Bytes) {
return MCInstBuilder(X86::ADD64ri8)
.addReg(X86::RSP)
.addReg(X86::RSP)
.addImm(Bytes);
}
// Reserves some space on the stack, fills it with the content of the provided
// constant and provide methods to load the stack value into a register.
namespace {
struct ConstantInliner {
explicit ConstantInliner(const APInt &Constant) : Constant_(Constant) {}
std::vector<MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
unsigned Opcode);
std::vector<MCInst> loadX87STAndFinalize(unsigned Reg);
std::vector<MCInst> loadX87FPAndFinalize(unsigned Reg);
std::vector<MCInst> popFlagAndFinalize();
std::vector<MCInst> loadImplicitRegAndFinalize(unsigned Opcode,
unsigned Value);
private:
ConstantInliner &add(const MCInst &Inst) {
Instructions.push_back(Inst);
return *this;
}
void initStack(unsigned Bytes);
static constexpr const unsigned kF80Bytes = 10; // 80 bits.
APInt Constant_;
std::vector<MCInst> Instructions;
};
} // namespace
std::vector<MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
unsigned RegBitWidth,
unsigned Opcode) {
assert((RegBitWidth & 7) == 0 && "RegBitWidth must be a multiple of 8 bits");
initStack(RegBitWidth / 8);
add(loadToReg(Reg, Opcode));
add(releaseStackSpace(RegBitWidth / 8));
return std::move(Instructions);
}
std::vector<MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) {
initStack(kF80Bytes);
add(MCInstBuilder(X86::LD_F80m)
// Address = ESP
.addReg(X86::RSP) // BaseReg
.addImm(1) // ScaleAmt
.addReg(0) // IndexReg
.addImm(0) // Disp
.addReg(0)); // Segment
if (Reg != X86::ST0)
add(MCInstBuilder(X86::ST_Frr).addReg(Reg));
add(releaseStackSpace(kF80Bytes));
return std::move(Instructions);
}
std::vector<MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) {
initStack(kF80Bytes);
add(MCInstBuilder(X86::LD_Fp80m)
.addReg(Reg)
// Address = ESP
.addReg(X86::RSP) // BaseReg
.addImm(1) // ScaleAmt
.addReg(0) // IndexReg
.addImm(0) // Disp
.addReg(0)); // Segment
add(releaseStackSpace(kF80Bytes));
return std::move(Instructions);
}
std::vector<MCInst> ConstantInliner::popFlagAndFinalize() {
initStack(8);
add(MCInstBuilder(X86::POPF64));
return std::move(Instructions);
}
std::vector<MCInst>
ConstantInliner::loadImplicitRegAndFinalize(unsigned Opcode, unsigned Value) {
add(allocateStackSpace(4));
add(fillStackSpace(X86::MOV32mi, 0, Value)); // Mask all FP exceptions
add(MCInstBuilder(Opcode)
// Address = ESP
.addReg(X86::RSP) // BaseReg
.addImm(1) // ScaleAmt
.addReg(0) // IndexReg
.addImm(0) // Disp
.addReg(0)); // Segment
add(releaseStackSpace(4));
return std::move(Instructions);
}
void ConstantInliner::initStack(unsigned Bytes) {
assert(Constant_.getBitWidth() <= Bytes * 8 &&
"Value does not have the correct size");
const APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
? Constant_.sext(Bytes * 8)
: Constant_;
add(allocateStackSpace(Bytes));
size_t ByteOffset = 0;
for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
add(fillStackSpace(
X86::MOV32mi, ByteOffset,
WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
if (Bytes - ByteOffset >= 2) {
add(fillStackSpace(
X86::MOV16mi, ByteOffset,
WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
ByteOffset += 2;
}
if (Bytes - ByteOffset >= 1)
add(fillStackSpace(
X86::MOV8mi, ByteOffset,
WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
}
#include "X86GenExegesis.inc"
namespace {
class X86SavedState : public ExegesisTarget::SavedState {
public:
X86SavedState() {
#ifdef __x86_64__
# if defined(_MSC_VER)
_fxsave64(FPState);
Eflags = __readeflags();
# elif defined(__GNUC__)
__builtin_ia32_fxsave64(FPState);
Eflags = __builtin_ia32_readeflags_u64();
# endif
#else
llvm_unreachable("X86 exegesis running on non-X86 target");
#endif
}
~X86SavedState() {
// Restoring the X87 state does not flush pending exceptions, make sure
// these exceptions are flushed now.
#ifdef __x86_64__
# if defined(_MSC_VER)
_clearfp();
_fxrstor64(FPState);
__writeeflags(Eflags);
# elif defined(__GNUC__)
asm volatile("fwait");
__builtin_ia32_fxrstor64(FPState);
__builtin_ia32_writeeflags_u64(Eflags);
# endif
#else
llvm_unreachable("X86 exegesis running on non-X86 target");
#endif
}
private:
#ifdef __x86_64__
alignas(16) char FPState[512];
uint64_t Eflags;
#endif
};
class ExegesisX86Target : public ExegesisTarget {
public:
ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
Expected<std::unique_ptr<pfm::Counter>>
createCounter(StringRef CounterName, const LLVMState &State) const override {
// If LbrSamplingPeriod was provided, then ignore the
// CounterName because we only have one for LBR.
if (LbrSamplingPeriod > 0) {
// Can't use LBR without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, or without
// __linux__ (for now)
#if defined(HAVE_LIBPFM) && defined(LIBPFM_HAS_FIELD_CYCLES) && \
defined(__linux__)
return std::make_unique<X86LbrCounter>(
X86LbrPerfEvent(LbrSamplingPeriod));
#else
return llvm::make_error<llvm::StringError>(
"LBR counter requested without HAVE_LIBPFM, LIBPFM_HAS_FIELD_CYCLES, "
"or running on Linux.",
llvm::errc::invalid_argument);
#endif
}
return ExegesisTarget::createCounter(CounterName, State);
}
private:
void addTargetSpecificPasses(PassManagerBase &PM) const override;
unsigned getScratchMemoryRegister(const Triple &TT) const override;
unsigned getLoopCounterRegister(const Triple &) const override;
unsigned getMaxMemoryAccessSize() const override { return 64; }
Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,
MCOperand &AssignedValue,
const BitVector &ForbiddenRegs) const override;
void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
unsigned Offset) const override;
void decrementLoopCounterAndJump(MachineBasicBlock &MBB,
MachineBasicBlock &TargetMBB,
const MCInstrInfo &MII) const override;
std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, unsigned Reg,
const APInt &Value) const override;
ArrayRef<unsigned> getUnavailableRegisters() const override {
return makeArrayRef(kUnavailableRegisters,
sizeof(kUnavailableRegisters) /
sizeof(kUnavailableRegisters[0]));
}
bool allowAsBackToBack(const Instruction &Instr) const override {
const unsigned Opcode = Instr.Description.Opcode;
return !isInvalidOpcode(Instr) && Opcode != X86::LEA64r &&
Opcode != X86::LEA64_32r && Opcode != X86::LEA16r;
}
std::vector<InstructionTemplate>
generateInstructionVariants(const Instruction &Instr,
unsigned MaxConfigsPerOpcode) const override;
std::unique_ptr<SnippetGenerator> createSerialSnippetGenerator(
const LLVMState &State,
const SnippetGenerator::Options &Opts) const override {
return std::make_unique<X86SerialSnippetGenerator>(State, Opts);
}
std::unique_ptr<SnippetGenerator> createParallelSnippetGenerator(
const LLVMState &State,
const SnippetGenerator::Options &Opts) const override {
return std::make_unique<X86ParallelSnippetGenerator>(State, Opts);
}
bool matchesArch(Triple::ArchType Arch) const override {
return Arch == Triple::x86_64 || Arch == Triple::x86;
}
Error checkFeatureSupport() const override {
// LBR is the only feature we conditionally support now.
// So if LBR is not requested, then we should be able to run the benchmarks.
if (LbrSamplingPeriod == 0)
return Error::success();
#if defined(__linux__) && defined(HAVE_LIBPFM) && \
defined(LIBPFM_HAS_FIELD_CYCLES)
// FIXME: Fix this.
// https://bugs.llvm.org/show_bug.cgi?id=48918
// For now, only do the check if we see an Intel machine because
// the counter uses some intel-specific magic and it could
// be confuse and think an AMD machine actually has LBR support.
#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \
defined(_M_X64)
using namespace sys::detail::x86;
if (getVendorSignature() == VendorSignatures::GENUINE_INTEL)
// If the kernel supports it, the hardware still may not have it.
return X86LbrCounter::checkLbrSupport();
#else
llvm_unreachable("Running X86 exegesis on non-X86 target");
#endif
#endif
return llvm::make_error<llvm::StringError>(
"LBR not supported on this kernel and/or platform",
llvm::errc::not_supported);
}
std::unique_ptr<SavedState> withSavedState() const override {
return std::make_unique<X86SavedState>();
}
static const unsigned kUnavailableRegisters[4];
};
// We disable a few registers that cannot be encoded on instructions with a REX
// prefix.
const unsigned ExegesisX86Target::kUnavailableRegisters[4] = {X86::AH, X86::BH,
X86::CH, X86::DH};
// We're using one of R8-R15 because these registers are never hardcoded in
// instructions (e.g. MOVS writes to EDI, ESI, EDX), so they have less
// conflicts.
constexpr const unsigned kLoopCounterReg = X86::R8;
} // namespace
void ExegesisX86Target::addTargetSpecificPasses(PassManagerBase &PM) const {
// Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
PM.add(createX86FloatingPointStackifierPass());
}
unsigned ExegesisX86Target::getScratchMemoryRegister(const Triple &TT) const {
if (!TT.isArch64Bit()) {
// FIXME: This would require popping from the stack, so we would have to
// add some additional setup code.
return 0;
}
return TT.isOSWindows() ? X86::RCX : X86::RDI;
}
unsigned ExegesisX86Target::getLoopCounterRegister(const Triple &TT) const {
if (!TT.isArch64Bit()) {
return 0;
}
return kLoopCounterReg;
}
Error ExegesisX86Target::randomizeTargetMCOperand(
const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
const BitVector &ForbiddenRegs) const {
const Operand &Op = Instr.getPrimaryOperand(Var);
switch (Op.getExplicitOperandInfo().OperandType) {
case X86::OperandType::OPERAND_ROUNDING_CONTROL:
AssignedValue =
MCOperand::createImm(randomIndex(X86::STATIC_ROUNDING::TO_ZERO));
return Error::success();
default:
break;
}
return make_error<Failure>(
Twine("unimplemented operand type ")
.concat(Twine(Op.getExplicitOperandInfo().OperandType)));
}
void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
unsigned Reg,
unsigned Offset) const {
assert(!isInvalidMemoryInstr(IT.getInstr()) &&
"fillMemoryOperands requires a valid memory instruction");
int MemOpIdx = X86II::getMemoryOperandNo(IT.getInstr().Description.TSFlags);
assert(MemOpIdx >= 0 && "invalid memory operand index");
// getMemoryOperandNo() ignores tied operands, so we have to add them back.
MemOpIdx += X86II::getOperandBias(IT.getInstr().Description);
setMemOp(IT, MemOpIdx + 0, MCOperand::createReg(Reg)); // BaseReg
setMemOp(IT, MemOpIdx + 1, MCOperand::createImm(1)); // ScaleAmt
setMemOp(IT, MemOpIdx + 2, MCOperand::createReg(0)); // IndexReg
setMemOp(IT, MemOpIdx + 3, MCOperand::createImm(Offset)); // Disp
setMemOp(IT, MemOpIdx + 4, MCOperand::createReg(0)); // Segment
}
void ExegesisX86Target::decrementLoopCounterAndJump(
MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB,
const MCInstrInfo &MII) const {
BuildMI(&MBB, DebugLoc(), MII.get(X86::ADD64ri8))
.addDef(kLoopCounterReg)
.addUse(kLoopCounterReg)
.addImm(-1);
BuildMI(&MBB, DebugLoc(), MII.get(X86::JCC_1))
.addMBB(&TargetMBB)
.addImm(X86::COND_NE);
}
std::vector<MCInst> ExegesisX86Target::setRegTo(const MCSubtargetInfo &STI,
unsigned Reg,
const APInt &Value) const {
if (X86::GR8RegClass.contains(Reg))
return {loadImmediate(Reg, 8, Value)};
if (X86::GR16RegClass.contains(Reg))
return {loadImmediate(Reg, 16, Value)};
if (X86::GR32RegClass.contains(Reg))
return {loadImmediate(Reg, 32, Value)};
if (X86::GR64RegClass.contains(Reg))
return {loadImmediate(Reg, 64, Value)};
ConstantInliner CI(Value);
if (X86::VR64RegClass.contains(Reg))
return CI.loadAndFinalize(Reg, 64, X86::MMX_MOVQ64rm);
if (X86::VR128XRegClass.contains(Reg)) {
if (STI.getFeatureBits()[X86::FeatureAVX512])
return CI.loadAndFinalize(Reg, 128, X86::VMOVDQU32Z128rm);
if (STI.getFeatureBits()[X86::FeatureAVX])
return CI.loadAndFinalize(Reg, 128, X86::VMOVDQUrm);
return CI.loadAndFinalize(Reg, 128, X86::MOVDQUrm);
}
if (X86::VR256XRegClass.contains(Reg)) {
if (STI.getFeatureBits()[X86::FeatureAVX512])
return CI.loadAndFinalize(Reg, 256, X86::VMOVDQU32Z256rm);
if (STI.getFeatureBits()[X86::FeatureAVX])
return CI.loadAndFinalize(Reg, 256, X86::VMOVDQUYrm);
}
if (X86::VR512RegClass.contains(Reg))
if (STI.getFeatureBits()[X86::FeatureAVX512])
return CI.loadAndFinalize(Reg, 512, X86::VMOVDQU32Zrm);
if (X86::RSTRegClass.contains(Reg)) {
return CI.loadX87STAndFinalize(Reg);
}
if (X86::RFP32RegClass.contains(Reg) || X86::RFP64RegClass.contains(Reg) ||
X86::RFP80RegClass.contains(Reg)) {
return CI.loadX87FPAndFinalize(Reg);
}
if (Reg == X86::EFLAGS)
return CI.popFlagAndFinalize();
if (Reg == X86::MXCSR)
return CI.loadImplicitRegAndFinalize(
STI.getFeatureBits()[X86::FeatureAVX] ? X86::VLDMXCSR : X86::LDMXCSR,
0x1f80);
if (Reg == X86::FPCW)
return CI.loadImplicitRegAndFinalize(X86::FLDCW16m, 0x37f);
return {}; // Not yet implemented.
}
// Instruction can have some variable operands, and we may want to see how
// different operands affect performance. So for each operand position,
// precompute all the possible choices we might care about,
// and greedily generate all the possible combinations of choices.
std::vector<InstructionTemplate> ExegesisX86Target::generateInstructionVariants(
const Instruction &Instr, unsigned MaxConfigsPerOpcode) const {
bool Exploration = false;
SmallVector<SmallVector<MCOperand, 1>, 4> VariableChoices;
VariableChoices.resize(Instr.Variables.size());
for (auto I : llvm::zip(Instr.Variables, VariableChoices)) {
const Variable &Var = std::get<0>(I);
SmallVectorImpl<MCOperand> &Choices = std::get<1>(I);
switch (Instr.getPrimaryOperand(Var).getExplicitOperandInfo().OperandType) {
default:
// We don't wish to explicitly explore this variable.
Choices.emplace_back(); // But add invalid MCOperand to simplify logic.
continue;
case X86::OperandType::OPERAND_COND_CODE: {
Exploration = true;
auto CondCodes = enum_seq_inclusive(X86::CondCode::COND_O,
X86::CondCode::LAST_VALID_COND,
force_iteration_on_noniterable_enum);
Choices.reserve(CondCodes.size());
for (int CondCode : CondCodes)
Choices.emplace_back(MCOperand::createImm(CondCode));
break;
}
}
}
// If we don't wish to explore any variables, defer to the baseline method.
if (!Exploration)
return ExegesisTarget::generateInstructionVariants(Instr,
MaxConfigsPerOpcode);
std::vector<InstructionTemplate> Variants;
size_t NumVariants;
CombinationGenerator<MCOperand, decltype(VariableChoices)::value_type, 4> G(
VariableChoices);
// How many operand combinations can we produce, within the limit?
NumVariants = std::min(G.numCombinations(), (size_t)MaxConfigsPerOpcode);
// And actually produce all the wanted operand combinations.
Variants.reserve(NumVariants);
G.generate([&](ArrayRef<MCOperand> State) -> bool {
Variants.emplace_back(&Instr);
Variants.back().setVariableValues(State);
// Did we run out of space for variants?
return Variants.size() >= NumVariants;
});
assert(Variants.size() == NumVariants &&
Variants.size() <= MaxConfigsPerOpcode &&
"Should not produce too many variants");
return Variants;
}
static ExegesisTarget *getTheExegesisX86Target() {
static ExegesisX86Target Target;
return &Target;
}
void InitializeX86ExegesisTarget() {
ExegesisTarget::registerTarget(getTheExegesisX86Target());
}
} // namespace exegesis
} // namespace llvm
Reference in New Issue View Git Blame Copy Permalink