Skip to content

riscv64: Split clobber save/restore SP adjustment for compressed instructions #13797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rajeshgangam wants to merge 1 commit into
base: main
Choose a base branch
from
Open

riscv64: Split clobber save/restore SP adjustment for compressed instructions #13797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cranelift/codegen/src/isa/aarch64/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_clobber_save(
call_conv: isa::CallConv,
flags: &settings::Flags,
_isa_flags: &aarch64_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let (clobbered_int, clobbered_vec) = frame_layout.clobbered_callee_saves_by_class();
Expand Down Expand Up @@ -1020,6 +1021,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
fn gen_clobber_restore(
call_conv: isa::CallConv,
_flags: &settings::Flags,
_isa_flags: &aarch64_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
Expand Down
1 change: 1 addition & 0 deletions cranelift/codegen/src/isa/aarch64/inst/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3661,6 +3661,7 @@ fn emit_return_call_common_sequence<T>(
for inst in AArch64MachineDeps::gen_clobber_restore(
CallConv::Tail,
&emit_info.flags,
&emit_info.isa_flags,
state.frame_layout(),
) {
inst.emit(sink, emit_info, state);
Expand Down
2 changes: 2 additions & 0 deletions cranelift/codegen/src/isa/pulley_shared/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ where
fn gen_clobber_save(
_call_conv: isa::CallConv,
_flags: &settings::Flags,
_isa_flags: &PulleyFlags,
_frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]> {
// Note that this is intentionally empty because everything necessary
Expand All @@ -443,6 +444,7 @@ where
fn gen_clobber_restore(
_call_conv: isa::CallConv,
_flags: &settings::Flags,
_isa_flags: &PulleyFlags,
_frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]> {
// Intentionally empty as restores happen for Pulley in `gen_return`.
Expand Down
122 changes: 79 additions & 43 deletions cranelift/codegen/src/isa/riscv64/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
fn gen_clobber_save(
_call_conv: isa::CallConv,
flags: &settings::Flags,
isa_flags: &RiscvFlags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
Expand Down Expand Up @@ -479,61 +480,91 @@ impl ABIMachineSpec for Riscv64MachineDeps {
});
}

// Adjust the stack pointer downward for clobbers, the function fixed
// frame (spillslots and storage slots), and outgoing arguments.
let stack_size = frame_layout.clobber_size
+ frame_layout.fixed_frame_storage_size
+ frame_layout.outgoing_args_size;

// Store each clobbered register in order at offsets from SP,
// placing them above the fixed frame slots.
if stack_size > 0 {
insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32)));

let mut cur_offset = 0;
for reg in &frame_layout.clobbered_callee_saves {
let r_reg = reg.to_reg();
let ty = match r_reg.class() {
RegClass::Int => I64,
RegClass::Float => F64,
RegClass::Vector => I8X16,
};
cur_offset = align_to(cur_offset, ty.bytes());
insts.push(Inst::gen_store(
AMode::SPOffset(i64::from(stack_size - cur_offset - ty.bytes())),
Reg::from(reg.to_reg()),
ty,
MemFlagsData::trusted(),
));
let clobber_size = frame_layout.clobber_size;
let remaining_frame_size =
frame_layout.fixed_frame_storage_size + frame_layout.outgoing_args_size;

if flags.unwind_info() {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset: frame_layout.clobber_size - cur_offset - ty.bytes(),
reg: r_reg,
},
});
}
// When the Zca extension is available, split the SP adjustment so
// callee-save stores use small offsets that fit in c.sdsp (≤504 bytes).
// Without Zca, use a single combined decrement to avoid an extra instruction.
let split_adjustment = isa_flags.has_zca() && clobber_size > 0 && remaining_frame_size > 0;

if split_adjustment {
insts.extend(Self::gen_sp_reg_adjust(-(clobber_size as i32)));
} else {
let stack_size = clobber_size + remaining_frame_size;
if stack_size > 0 {
insts.extend(Self::gen_sp_reg_adjust(-(stack_size as i32)));
}
}

let store_base = if split_adjustment {
clobber_size
} else {
clobber_size + remaining_frame_size
};

let mut cur_offset = 0;
for reg in &frame_layout.clobbered_callee_saves {
let r_reg = reg.to_reg();
let ty = match r_reg.class() {
RegClass::Int => I64,
RegClass::Float => F64,
RegClass::Vector => I8X16,
};
cur_offset = align_to(cur_offset, ty.bytes());
insts.push(Inst::gen_store(
AMode::SPOffset(i64::from(store_base - cur_offset - ty.bytes())),
Reg::from(reg.to_reg()),
ty,
MemFlagsData::trusted(),
));

cur_offset += ty.bytes();
assert!(cur_offset <= stack_size);
if flags.unwind_info() {
insts.push(Inst::Unwind {
inst: UnwindInst::SaveReg {
clobber_offset: clobber_size - cur_offset - ty.bytes(),
reg: r_reg,
},
});
}

cur_offset += ty.bytes();
assert!(cur_offset <= clobber_size);
}

if split_adjustment {
insts.extend(Self::gen_sp_reg_adjust(-(remaining_frame_size as i32)));
}

insts
}

fn gen_clobber_restore(
_call_conv: isa::CallConv,
_flags: &settings::Flags,
isa_flags: &RiscvFlags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();

let stack_size = frame_layout.clobber_size
+ frame_layout.fixed_frame_storage_size
+ frame_layout.outgoing_args_size;
let mut cur_offset = 0;
let clobber_size = frame_layout.clobber_size;
let remaining_frame_size =
frame_layout.fixed_frame_storage_size + frame_layout.outgoing_args_size;

let split_adjustment = isa_flags.has_zca() && clobber_size > 0 && remaining_frame_size > 0;

if split_adjustment {
insts.extend(Self::gen_sp_reg_adjust(remaining_frame_size as i32));
}

let load_base = if split_adjustment {
clobber_size
} else {
clobber_size + remaining_frame_size
};

let mut cur_offset = 0;
for reg in &frame_layout.clobbered_callee_saves {
let rreg = reg.to_reg();
let ty = match rreg.class() {
Expand All @@ -544,15 +575,20 @@ impl ABIMachineSpec for Riscv64MachineDeps {
cur_offset = align_to(cur_offset, ty.bytes());
insts.push(Inst::gen_load(
reg.map(Reg::from),
AMode::SPOffset(i64::from(stack_size - cur_offset - ty.bytes())),
AMode::SPOffset(i64::from(load_base - cur_offset - ty.bytes())),
ty,
MemFlagsData::trusted(),
));
cur_offset += ty.bytes();
}

if stack_size > 0 {
insts.extend(Self::gen_sp_reg_adjust(stack_size as i32));
let restore_size = if split_adjustment {
clobber_size
} else {
clobber_size + remaining_frame_size
};
if restore_size > 0 {
insts.extend(Self::gen_sp_reg_adjust(restore_size as i32));
}

insts
Expand Down
2 changes: 2 additions & 0 deletions cranelift/codegen/src/isa/s390x/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,7 @@ impl ABIMachineSpec for S390xMachineDeps {
fn gen_clobber_save(
call_conv: isa::CallConv,
flags: &settings::Flags,
_isa_flags: &s390x_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
Expand Down Expand Up @@ -860,6 +861,7 @@ impl ABIMachineSpec for S390xMachineDeps {
fn gen_clobber_restore(
call_conv: isa::CallConv,
_flags: &settings::Flags,
_isa_flags: &s390x_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
Expand Down
2 changes: 2 additions & 0 deletions cranelift/codegen/src/isa/x64/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
fn gen_clobber_save(
_call_conv: isa::CallConv,
flags: &settings::Flags,
_isa_flags: &x64_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]> {
let mut insts = SmallVec::new();
Expand Down Expand Up @@ -764,6 +765,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
fn gen_clobber_restore(
_call_conv: isa::CallConv,
_flags: &settings::Flags,
_isa_flags: &x64_settings::Flags,
frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]> {
let mut insts = SmallVec::new();
Expand Down
9 changes: 6 additions & 3 deletions cranelift/codegen/src/isa/x64/inst/emit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1879,9 +1879,12 @@ fn emit_return_call_common_sequence<T>(

let tmp = call_info.tmp.to_writable_reg();

for inst in
X64ABIMachineSpec::gen_clobber_restore(CallConv::Tail, &info.flags, state.frame_layout())
{
for inst in X64ABIMachineSpec::gen_clobber_restore(
CallConv::Tail,
&info.flags,
&info.isa_flags,
state.frame_layout(),
) {
inst.emit(sink, info, state);
}

Expand Down
4 changes: 4 additions & 0 deletions cranelift/codegen/src/machinst/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@ pub trait ABIMachineSpec {
fn gen_clobber_save(
call_conv: isa::CallConv,
flags: &settings::Flags,
isa_flags: &Self::F,
frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]>;

Expand All @@ -552,6 +553,7 @@ pub trait ABIMachineSpec {
fn gen_clobber_restore(
call_conv: isa::CallConv,
flags: &settings::Flags,
isa_flags: &Self::F,
frame_layout: &FrameLayout,
) -> SmallVec<[Self::I; 16]>;

Expand Down Expand Up @@ -2287,6 +2289,7 @@ impl<M: ABIMachineSpec> Callee<M> {
insts.extend(M::gen_clobber_save(
self.call_conv,
&self.flags,
&self.isa_flags,
&frame_layout,
));

Expand All @@ -2306,6 +2309,7 @@ impl<M: ABIMachineSpec> Callee<M> {
insts.extend(M::gen_clobber_restore(
self.call_conv,
&self.flags,
&self.isa_flags,
&frame_layout,
));

Expand Down
Loading
Loading