Skip to content

Commit

Permalink
[RISC-V] Add proposed mapping for Ztso
Browse files Browse the repository at this point in the history 
Currently LLVM emits Ztso code for fences, loads, and stores (behind an
experimental flag) [1]. This patch updates the mapping and implements
support for LR/SC and AMO ops. This updated mapping is compatible with
the RVWMO ABI present in the psABI. Additional context can be found in
the psABI pull request [2].

[1] https://reviews.llvm.org/D143076
[2] riscv-non-isa/riscv-elf-psabi-doc#391

Differential Revision: https://reviews.llvm.org/D155517
  • Loading branch information
@patrick-rivos @asb
patrick-rivos authored and asb committed Aug 10, 2023
1 parent 0696a53 commit fcad2bb
Show file tree
Hide file tree
Showing 7 changed files with 6,384 additions and 3,139 deletions.
89 changes: 61 additions & 28 deletions llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ namespace {

class RISCVExpandAtomicPseudo : public MachineFunctionPass {
public:
const RISCVSubtarget *STI;
const RISCVInstrInfo *TII;
static char ID;

Expand Down Expand Up @@ -72,7 +73,8 @@ class RISCVExpandAtomicPseudo : public MachineFunctionPass {
char RISCVExpandAtomicPseudo::ID = 0;

bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
STI = &MF.getSubtarget<RISCVSubtarget>();
TII = STI->getInstrInfo();

#ifndef NDEBUG
const unsigned OldSize = getInstSizeInBytes(MF);
Expand Down Expand Up @@ -148,24 +150,30 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
return false;
}

static unsigned getLRForRMW32(AtomicOrdering Ordering) {
static unsigned getLRForRMW32(AtomicOrdering Ordering,
const RISCVSubtarget *Subtarget) {
switch (Ordering) {
default:
llvm_unreachable("Unexpected AtomicOrdering");
case AtomicOrdering::Monotonic:
return RISCV::LR_W;
case AtomicOrdering::Acquire:
if (Subtarget->hasStdExtZtso())
return RISCV::LR_W;
return RISCV::LR_W_AQ;
case AtomicOrdering::Release:
return RISCV::LR_W;
case AtomicOrdering::AcquireRelease:
if (Subtarget->hasStdExtZtso())
return RISCV::LR_W;
return RISCV::LR_W_AQ;
case AtomicOrdering::SequentiallyConsistent:
return RISCV::LR_W_AQ_RL;
}
}

static unsigned getSCForRMW32(AtomicOrdering Ordering) {
static unsigned getSCForRMW32(AtomicOrdering Ordering,
const RISCVSubtarget *Subtarget) {
switch (Ordering) {
default:
llvm_unreachable("Unexpected AtomicOrdering");
Expand All @@ -174,32 +182,42 @@ static unsigned getSCForRMW32(AtomicOrdering Ordering) {
case AtomicOrdering::Acquire:
return RISCV::SC_W;
case AtomicOrdering::Release:
if (Subtarget->hasStdExtZtso())
return RISCV::SC_W;
return RISCV::SC_W_RL;
case AtomicOrdering::AcquireRelease:
if (Subtarget->hasStdExtZtso())
return RISCV::SC_W;
return RISCV::SC_W_RL;
case AtomicOrdering::SequentiallyConsistent:
return RISCV::SC_W_RL;
}
}

static unsigned getLRForRMW64(AtomicOrdering Ordering) {
static unsigned getLRForRMW64(AtomicOrdering Ordering,
const RISCVSubtarget *Subtarget) {
switch (Ordering) {
default:
llvm_unreachable("Unexpected AtomicOrdering");
case AtomicOrdering::Monotonic:
return RISCV::LR_D;
case AtomicOrdering::Acquire:
if (Subtarget->hasStdExtZtso())
return RISCV::LR_D;
return RISCV::LR_D_AQ;
case AtomicOrdering::Release:
return RISCV::LR_D;
case AtomicOrdering::AcquireRelease:
if (Subtarget->hasStdExtZtso())
return RISCV::LR_D;
return RISCV::LR_D_AQ;
case AtomicOrdering::SequentiallyConsistent:
return RISCV::LR_D_AQ_RL;
}
}

static unsigned getSCForRMW64(AtomicOrdering Ordering) {
static unsigned getSCForRMW64(AtomicOrdering Ordering,
const RISCVSubtarget *Subtarget) {
switch (Ordering) {
default:
llvm_unreachable("Unexpected AtomicOrdering");
Expand All @@ -208,35 +226,42 @@ static unsigned getSCForRMW64(AtomicOrdering Ordering) {
case AtomicOrdering::Acquire:
return RISCV::SC_D;
case AtomicOrdering::Release:
if (Subtarget->hasStdExtZtso())
return RISCV::SC_D;
return RISCV::SC_D_RL;
case AtomicOrdering::AcquireRelease:
if (Subtarget->hasStdExtZtso())
return RISCV::SC_D;
return RISCV::SC_D_RL;
case AtomicOrdering::SequentiallyConsistent:
return RISCV::SC_D_RL;
}
}

static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
static unsigned getLRForRMW(AtomicOrdering Ordering, int Width,
const RISCVSubtarget *Subtarget) {
if (Width == 32)
return getLRForRMW32(Ordering);
return getLRForRMW32(Ordering, Subtarget);
if (Width == 64)
return getLRForRMW64(Ordering);
return getLRForRMW64(Ordering, Subtarget);
llvm_unreachable("Unexpected LR width\n");
}

static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
static unsigned getSCForRMW(AtomicOrdering Ordering, int Width,
const RISCVSubtarget *Subtarget) {
if (Width == 32)
return getSCForRMW32(Ordering);
return getSCForRMW32(Ordering, Subtarget);
if (Width == 64)
return getSCForRMW64(Ordering);
return getSCForRMW64(Ordering, Subtarget);
llvm_unreachable("Unexpected SC width\n");
}

static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
DebugLoc DL, MachineBasicBlock *ThisMBB,
MachineBasicBlock *LoopMBB,
MachineBasicBlock *DoneMBB,
AtomicRMWInst::BinOp BinOp, int Width) {
AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
Register DestReg = MI.getOperand(0).getReg();
Register ScratchReg = MI.getOperand(1).getReg();
Register AddrReg = MI.getOperand(2).getReg();
Expand All @@ -249,7 +274,7 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
// binop scratch, dest, val
// sc.[w|d] scratch, scratch, (addr)
// bnez scratch, loop
BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)), DestReg)
.addReg(AddrReg);
switch (BinOp) {
default:
Expand All @@ -263,7 +288,7 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
.addImm(-1);
break;
}
BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)), ScratchReg)
.addReg(AddrReg)
.addReg(ScratchReg);
BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
Expand Down Expand Up @@ -294,10 +319,13 @@ static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
.addReg(ScratchReg);
}

static void doMaskedAtomicBinOpExpansion(
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
static void doMaskedAtomicBinOpExpansion(const RISCVInstrInfo *TII,
MachineInstr &MI, DebugLoc DL,
MachineBasicBlock *ThisMBB,
MachineBasicBlock *LoopMBB,
MachineBasicBlock *DoneMBB,
AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
Register DestReg = MI.getOperand(0).getReg();
Register ScratchReg = MI.getOperand(1).getReg();
Expand All @@ -315,7 +343,7 @@ static void doMaskedAtomicBinOpExpansion(
// xor scratch, destreg, scratch
// sc.w scratch, scratch, (alignedaddr)
// bnez scratch, loop
BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering, STI)), DestReg)
.addReg(AddrReg);
switch (BinOp) {
default:
Expand Down Expand Up @@ -348,7 +376,7 @@ static void doMaskedAtomicBinOpExpansion(
insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
ScratchReg);

BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering, STI)), ScratchReg)
.addReg(AddrReg)
.addReg(ScratchReg);
BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
Expand Down Expand Up @@ -380,10 +408,11 @@ bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
MBB.addSuccessor(LoopMBB);

if (!IsMasked)
doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width,
STI);
else
doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
Width);
Width, STI);

NextMBBI = MBB.end();
MI.eraseFromParent();
Expand Down Expand Up @@ -455,7 +484,7 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
// mv scratch1, destreg
// [sext scratch2 if signed min/max]
// ifnochangeneeded scratch2, incr, .looptail
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering, STI)), DestReg)
.addReg(AddrReg);
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
.addReg(DestReg)
Expand Down Expand Up @@ -507,7 +536,7 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
// .looptail:
// sc.w scratch1, scratch1, (addr)
// bnez scratch1, loop
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering, STI)), Scratch1Reg)
.addReg(AddrReg)
.addReg(Scratch1Reg);
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
Expand Down Expand Up @@ -626,7 +655,8 @@ bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
// .loophead:
// lr.[w|d] dest, (addr)
// bne dest, cmpval, done
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)),
DestReg)
.addReg(AddrReg);
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
.addReg(DestReg)
Expand All @@ -635,7 +665,8 @@ bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
// .looptail:
// sc.[w|d] scratch, newval, (addr)
// bnez scratch, loophead
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)),
ScratchReg)
.addReg(AddrReg)
.addReg(NewValReg);
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
Expand All @@ -648,7 +679,8 @@ bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
// and scratch, dest, mask
// bne scratch, cmpval, done
Register MaskReg = MI.getOperand(5).getReg();
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)),
DestReg)
.addReg(AddrReg);
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
.addReg(DestReg)
Expand All @@ -666,7 +698,8 @@ bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
// bnez scratch, loophead
insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
MaskReg, ScratchReg);
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)),
ScratchReg)
.addReg(AddrReg)
.addReg(ScratchReg);
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/RISCV/RISCVFeatures.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@ def FeatureStdExtZtso
def HasStdExtZtso : Predicate<"Subtarget->hasStdExtZTso()">,
AssemblerPredicate<(all_of FeatureStdExtZtso),
"'Ztso' (Memory Model - Total Store Order)">;
def NotHasStdExtZtso : Predicate<"!Subtarget->hasStdExtZtso()">;

def FeatureStdExtZawrs : SubtargetFeature<"zawrs", "HasStdExtZawrs", "true",
"'Zawrs' (Wait on Reservation Set)">;
Expand Down
5 changes: 4 additions & 1 deletion llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16909,8 +16909,11 @@ Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
if (Subtarget.hasStdExtZtso())
if (Subtarget.hasStdExtZtso()) {
if (isa<StoreInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
return Builder.CreateFence(Ord);
return nullptr;
}

if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
return Builder.CreateFence(AtomicOrdering::Acquire);
Expand Down
18 changes: 16 additions & 2 deletions llvm/lib/Target/RISCV/RISCVInstrInfoA.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,10 @@ let Predicates = [HasAtomicLdSt, IsRV64] in {
def : AtomicStPat<atomic_store_64, SD, GPR, i64>;
}

let Predicates = [HasStdExtA] in {

/// AMOs

multiclass AMOPat<string AtomicOp, string BaseInst, ValueType vt = XLenVT> {
let Predicates = [HasStdExtA, NotHasStdExtZtso] in {
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_monotonic"),
!cast<RVInst>(BaseInst), vt>;
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_acquire"),
Expand All @@ -149,6 +148,21 @@ multiclass AMOPat<string AtomicOp, string BaseInst, ValueType vt = XLenVT> {
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_seq_cst"),
!cast<RVInst>(BaseInst#"_AQ_RL"), vt>;
}
let Predicates = [HasStdExtA, HasStdExtZtso] in {
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_monotonic"),
!cast<RVInst>(BaseInst), vt>;
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_acquire"),
!cast<RVInst>(BaseInst), vt>;
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_release"),
!cast<RVInst>(BaseInst), vt>;
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_acq_rel"),
!cast<RVInst>(BaseInst), vt>;
def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_seq_cst"),
!cast<RVInst>(BaseInst), vt>;
}
}

let Predicates = [HasStdExtA] in {

defm : AMOPat<"atomic_swap_32", "AMOSWAP_W">;
defm : AMOPat<"atomic_load_add_32", "AMOADD_W">;
Expand Down
Loading

0 comments on commit fcad2bb

Please sign in to comment.