andrey/rCore
1
0
Fork 0
mirror of https://github.com/rcore-os/rCore.git synced 2024-11-22 16:16:16 +04:00
Code Issues Projects Releases Wiki Activity

aarch64: use new cache & TLB maintenance operations

Browse Source
This commit is contained in:
equation314 2019-10-04 23:25:42 +08:00
parent acd7ee945a
commit 90c2cd28f8
7 changed files with 46 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
kernel/Cargo.lock generated
View File

@ -2,8 +2,8 @@
# It is not intended for manual editing.
[[package]]
name = "aarch64"
version = "2.8.0"
source = "git+https://github.com/rcore-os/aarch64#e56d6b52b8be11ce18a6395cd87c07905bdf4422"
version = "3.0.0"
source = "git+https://github.com/rcore-os/aarch64#fe633820b6866f5442be0ae6de7fd9182333a27a"
dependencies = [
"bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -65,10 +65,10 @@ dependencies = [
[[package]]
name = "bcm2837"
version = "2.4.0"
source = "git+https://github.com/rcore-os/bcm2837#960c7dd5b510bc28015c5733c3c90e3e3b0c8514"
version = "2.5.0"
source = "git+https://github.com/rcore-os/bcm2837#63468ef3465d5e104d828472686cbd779378f1df"
dependencies = [
"aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)",
"aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"volatile 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -325,11 +325,11 @@ dependencies = [
name = "rcore"
version = "0.2.0"
dependencies = [
"aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)",
"aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)",
"acpi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"aml 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)",
"bcm2837 2.4.0 (git+https://github.com/rcore-os/bcm2837)",
"bcm2837 2.5.0 (git+https://github.com/rcore-os/bcm2837)",
"bit_field 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bitmap-allocator 0.1.0 (git+https://github.com/rcore-os/bitmap-allocator)",
@ -690,14 +690,14 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)" = "<none>"
"checksum aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)" = "<none>"
"checksum acpi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c18d706bdc322dd4f8f7930a5879ad8df3d78d4452a678d5419c72f9f69acea"
"checksum aml 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b7669e841017880c2710777c46ec654272163379bbe55de6e17a2a2388d44d92"
"checksum apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)" = "<none>"
"checksum array-init 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72"
"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf"
"checksum bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a3caf393d93b2d453e80638d0674597020cef3382ada454faacd43d1a55a735a"
"checksum bcm2837 2.4.0 (git+https://github.com/rcore-os/bcm2837)" = "<none>"
"checksum bcm2837 2.5.0 (git+https://github.com/rcore-os/bcm2837)" = "<none>"
"checksum bit_field 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a165d606cf084741d4ac3a28fb6e9b1eb0bd31f6cd999098cfddb0b2ab381dc0"
"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56"
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"

4
kernel/Cargo.toml
View File

@ -85,8 +85,8 @@ aml = "0.4"
riscv = { git = "https://github.com/rcore-os/riscv", features = ["inline-asm"] }
[target.'cfg(target_arch = "aarch64")'.dependencies]
aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.8.0" }
bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "2.4.0", optional = true }
aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "3.0.0" }
bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "2.5.0", optional = true }
[target.'cfg(target_arch = "mips")'.dependencies]
mips = "^0.2.0"

6
kernel/src/arch/aarch64/board/raspi3/mailbox.rs
View File

@ -3,7 +3,7 @@
//! (ref: https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface)
use crate::memory::kernel_offset;
use aarch64::asm;
use aarch64::cache::*;
use alloc::string::String;
use bcm2837::addr::phys_to_bus;
use bcm2837::mailbox::{Mailbox, MailboxChannel};
@ -220,13 +220,13 @@ macro_rules! send_request {
{
// flush data cache around mailbox accesses
let mut mbox = MAILBOX.lock();
asm::flush_dcache_range(start, end);
DCache::<Clean, PoC>::flush_range(start, end, SY);
mbox.write(
MailboxChannel::Property,
phys_to_bus(kernel_offset(start) as u32),
);
mbox.read(MailboxChannel::Property);
asm::flush_dcache_range(start, end);
DCache::<Invalidate, PoC>::flush_range(start, end, SY);
}
match req.0.req_resp_code {

17
kernel/src/arch/aarch64/boot/mod.rs
View File

@ -3,7 +3,7 @@ use crate::memory::phys_to_virt;
use aarch64::paging::{memory_attribute::*, PageTableAttribute as Attr, PageTableFlags as EF};
use aarch64::paging::{Page, PageTable, PhysFrame, Size1GiB, Size2MiB, Size4KiB};
use aarch64::{align_down, align_up, PhysAddr, ALIGN_1GIB, ALIGN_2MIB};
use aarch64::{asm, barrier, regs::*};
use aarch64::{barrier, cache, regs::*, translation};
global_asm!(include_str!("entry.S"));
@ -96,24 +96,19 @@ extern "C" fn enable_mmu() {
// Set both TTBR0_EL1 and TTBR1_EL1
let frame_lvl4 = PhysFrame::<Size4KiB>::of_addr(page_table_lvl4 as u64);
asm::ttbr_el1_write(0, frame_lvl4);
asm::ttbr_el1_write(1, frame_lvl4);
asm::tlb_invalidate_all();
// Switch the MMU on.
//
// First, force all previous changes to be seen before the MMU is enabled.
unsafe { barrier::isb(barrier::SY) }
translation::ttbr_el1_write(0, frame_lvl4);
translation::ttbr_el1_write(1, frame_lvl4);
translation::local_invalidate_tlb_all();
// Enable the MMU and turn on data and instruction caching.
SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable);
// Force MMU init to complete before next instruction
unsafe { barrier::isb(barrier::SY) }
unsafe { barrier::isb() }
// Invalidate the local I-cache so that any instructions fetched
// speculatively from the PoC are discarded
asm::flush_icache_all();
cache::ICache::local_flush_all();
}
#[no_mangle]

7
kernel/src/arch/aarch64/cpu.rs
View File

@ -1,6 +1,6 @@
use crate::consts::SMP_CORES;
use crate::memory::{kernel_offset, phys_to_virt};
use aarch64::asm;
use aarch64::{asm, cache::*};
use core::{cmp, mem};
pub use super::board::{CPU_NUM, CPU_SPIN_TABLE};
@ -24,9 +24,10 @@ pub unsafe fn start_others() {
let release_addr = phys_to_virt(CPU_SPIN_TABLE[i]) as *mut usize;
let entry_addr = kernel_offset(slave_startup as usize);
*release_addr = entry_addr;
asm::flush_dcache_range(
DCache::<CleanAndInvalidate, PoC>::flush_area(
release_addr as usize,
release_addr as usize + mem::size_of::<usize>(),
mem::size_of::<usize>(),
SY,
);
asm::sev();
}

4
kernel/src/arch/aarch64/interrupt/context.rs
View File

@ -1,8 +1,8 @@
//! TrapFrame and context definitions for aarch64.
use aarch64::asm::{tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write_asid};
use aarch64::barrier;
use aarch64::paging::PhysFrame;
use aarch64::translation::{local_invalidate_tlb_all, ttbr_el1_read, ttbr_el1_write_asid};
use lazy_static::lazy_static;
use spin::Mutex;
@ -229,7 +229,7 @@ impl AsidAllocator {
if self.0.generation == 0 {
self.0.generation += 1;
}
tlb_invalidate_all();
local_invalidate_tlb_all();
}
self.0.value += 1;
return self.0;

29
kernel/src/arch/aarch64/paging.rs
View File

@ -1,8 +1,7 @@
//! Page table implementations for aarch64.
use crate::memory::{alloc_frame, dealloc_frame, phys_to_virt};
use aarch64::asm::{flush_dcache_range, flush_icache_all};
use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write};
use aarch64::cache::*;
use aarch64::paging::{
frame::PhysFrame as Frame,
mapper::{MappedPageTable, Mapper},
@ -10,6 +9,8 @@ use aarch64::paging::{
page_table::{PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF},
FrameAllocator, FrameDeallocator, Page as PageAllSizes, Size2MiB, Size4KiB,
};
use aarch64::translation::{invalidate_tlb_vaddr, local_invalidate_tlb_all};
use aarch64::translation::{ttbr_el1_read, ttbr_el1_write};
use aarch64::{align_down, align_up, PhysAddr, ALIGN_2MIB};
use core::mem::ManuallyDrop;
use log::*;
@ -74,10 +75,20 @@ impl PageTable for PageTableImpl {
fn flush_cache_copy_user(&mut self, start: usize, end: usize, execute: bool) {
if execute {
// clean D-cache to PoU to ensure new instructions has been written into memory
flush_dcache_range(start, end);
// invalidate I-cache to PoU to ensure old instructions has been flushed
flush_icache_all();
// clean D-cache to PoU to ensure new instructions has been written
// into memory
DCache::<Clean, PoU>::flush_range(start, end, ISH);
// invalidate I-cache to PoU to ensure old instructions has been
// flushed
if get_l1_icache_policy() == L1ICachePolicy::PIPT {
// Cortex-A57 use PIPT, address translation is transparent
ICache::<Invalidate, PoU>::flush_range(start, end, ISH);
} else {
// Cortex-A53 (raspi3) use VIPT, the effect of invalidation is
// only visible to the VA, need to invalidate the entire
// I-cache to invalidate all aliases of a PA.
ICache::flush_all();
}
}
}
}
@ -98,7 +109,7 @@ pub enum MMIOType {
// TODO: software dirty bit needs to be reconsidered
impl Entry for PageEntry {
fn update(&mut self) {
tlb_invalidate(self.1.start_address());
invalidate_tlb_vaddr(self.1.start_address());
}
fn present(&self) -> bool {
@ -244,7 +255,7 @@ impl PageTableImpl {
/// Used in `arch::memory::map_kernel()`.
pub unsafe fn activate_as_kernel(&self) {
ttbr_el1_write(1, Frame::of_addr(self.token() as u64));
tlb_invalidate_all();
local_invalidate_tlb_all();
}
/// Map physical memory [start, end)
/// to virtual space [phys_to_virt(start), phys_to_virt(end))
@ -300,7 +311,7 @@ impl PageTableExt for PageTableImpl {
}
fn flush_tlb() {
tlb_invalidate_all();
local_invalidate_tlb_all();
}
}