diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index ad56eec3..564a5092 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -2,8 +2,8 @@ # It is not intended for manual editing. [[package]] name = "aarch64" -version = "2.8.0" -source = "git+https://github.com/rcore-os/aarch64#e56d6b52b8be11ce18a6395cd87c07905bdf4422" +version = "3.0.0" +source = "git+https://github.com/rcore-os/aarch64#fe633820b6866f5442be0ae6de7fd9182333a27a" dependencies = [ "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -65,10 +65,10 @@ dependencies = [ [[package]] name = "bcm2837" -version = "2.4.0" -source = "git+https://github.com/rcore-os/bcm2837#960c7dd5b510bc28015c5733c3c90e3e3b0c8514" +version = "2.5.0" +source = "git+https://github.com/rcore-os/bcm2837#63468ef3465d5e104d828472686cbd779378f1df" dependencies = [ - "aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)", + "aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "volatile 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -325,11 +325,11 @@ dependencies = [ name = "rcore" version = "0.2.0" dependencies = [ - "aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)", + "aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)", "acpi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "aml 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)", - "bcm2837 2.4.0 (git+https://github.com/rcore-os/bcm2837)", + "bcm2837 2.5.0 (git+https://github.com/rcore-os/bcm2837)", "bit_field 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitmap-allocator 0.1.0 (git+https://github.com/rcore-os/bitmap-allocator)", @@ -690,14 +690,14 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum aarch64 2.8.0 (git+https://github.com/rcore-os/aarch64)" = "" +"checksum aarch64 3.0.0 (git+https://github.com/rcore-os/aarch64)" = "" "checksum acpi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c18d706bdc322dd4f8f7930a5879ad8df3d78d4452a678d5419c72f9f69acea" "checksum aml 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b7669e841017880c2710777c46ec654272163379bbe55de6e17a2a2388d44d92" "checksum apic 0.1.0 (git+https://github.com/rcore-os/apic-rs)" = "" "checksum array-init 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" "checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" "checksum bare-metal 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a3caf393d93b2d453e80638d0674597020cef3382ada454faacd43d1a55a735a" -"checksum bcm2837 2.4.0 (git+https://github.com/rcore-os/bcm2837)" = "" +"checksum bcm2837 2.5.0 (git+https://github.com/rcore-os/bcm2837)" = "" "checksum bit_field 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a165d606cf084741d4ac3a28fb6e9b1eb0bd31f6cd999098cfddb0b2ab381dc0" "checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index b64e19a4..dada1327 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -85,8 +85,8 @@ aml = "0.4" riscv = { git = "https://github.com/rcore-os/riscv", features = ["inline-asm"] } [target.'cfg(target_arch = "aarch64")'.dependencies] -aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "2.8.0" } -bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "2.4.0", optional = true } +aarch64 = { git = "https://github.com/rcore-os/aarch64", version = "3.0.0" } +bcm2837 = { git = "https://github.com/rcore-os/bcm2837", version = "2.5.0", optional = true } [target.'cfg(target_arch = "mips")'.dependencies] mips = "^0.2.0" diff --git a/kernel/src/arch/aarch64/board/raspi3/mailbox.rs b/kernel/src/arch/aarch64/board/raspi3/mailbox.rs index 1bacb36d..a1981f16 100644 --- a/kernel/src/arch/aarch64/board/raspi3/mailbox.rs +++ b/kernel/src/arch/aarch64/board/raspi3/mailbox.rs @@ -3,7 +3,7 @@ //! (ref: https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface) use crate::memory::kernel_offset; -use aarch64::asm; +use aarch64::cache::*; use alloc::string::String; use bcm2837::addr::phys_to_bus; use bcm2837::mailbox::{Mailbox, MailboxChannel}; @@ -220,13 +220,13 @@ macro_rules! send_request { { // flush data cache around mailbox accesses let mut mbox = MAILBOX.lock(); - asm::flush_dcache_range(start, end); + DCache::::flush_range(start, end, SY); mbox.write( MailboxChannel::Property, phys_to_bus(kernel_offset(start) as u32), ); mbox.read(MailboxChannel::Property); - asm::flush_dcache_range(start, end); + DCache::::flush_range(start, end, SY); } match req.0.req_resp_code { diff --git a/kernel/src/arch/aarch64/boot/mod.rs b/kernel/src/arch/aarch64/boot/mod.rs index 0196d86c..d14ab3cb 100644 --- a/kernel/src/arch/aarch64/boot/mod.rs +++ b/kernel/src/arch/aarch64/boot/mod.rs @@ -3,7 +3,7 @@ use crate::memory::phys_to_virt; use aarch64::paging::{memory_attribute::*, PageTableAttribute as Attr, PageTableFlags as EF}; use aarch64::paging::{Page, PageTable, PhysFrame, Size1GiB, Size2MiB, Size4KiB}; use aarch64::{align_down, align_up, PhysAddr, ALIGN_1GIB, ALIGN_2MIB}; -use aarch64::{asm, barrier, regs::*}; +use aarch64::{barrier, cache, regs::*, translation}; global_asm!(include_str!("entry.S")); @@ -96,24 +96,19 @@ extern "C" fn enable_mmu() { // Set both TTBR0_EL1 and TTBR1_EL1 let frame_lvl4 = PhysFrame::::of_addr(page_table_lvl4 as u64); - asm::ttbr_el1_write(0, frame_lvl4); - asm::ttbr_el1_write(1, frame_lvl4); - asm::tlb_invalidate_all(); - - // Switch the MMU on. - // - // First, force all previous changes to be seen before the MMU is enabled. - unsafe { barrier::isb(barrier::SY) } + translation::ttbr_el1_write(0, frame_lvl4); + translation::ttbr_el1_write(1, frame_lvl4); + translation::local_invalidate_tlb_all(); // Enable the MMU and turn on data and instruction caching. SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); // Force MMU init to complete before next instruction - unsafe { barrier::isb(barrier::SY) } + unsafe { barrier::isb() } // Invalidate the local I-cache so that any instructions fetched // speculatively from the PoC are discarded - asm::flush_icache_all(); + cache::ICache::local_flush_all(); } #[no_mangle] diff --git a/kernel/src/arch/aarch64/cpu.rs b/kernel/src/arch/aarch64/cpu.rs index 3d3bee9f..712dfa66 100644 --- a/kernel/src/arch/aarch64/cpu.rs +++ b/kernel/src/arch/aarch64/cpu.rs @@ -1,6 +1,6 @@ use crate::consts::SMP_CORES; use crate::memory::{kernel_offset, phys_to_virt}; -use aarch64::asm; +use aarch64::{asm, cache::*}; use core::{cmp, mem}; pub use super::board::{CPU_NUM, CPU_SPIN_TABLE}; @@ -24,9 +24,10 @@ pub unsafe fn start_others() { let release_addr = phys_to_virt(CPU_SPIN_TABLE[i]) as *mut usize; let entry_addr = kernel_offset(slave_startup as usize); *release_addr = entry_addr; - asm::flush_dcache_range( + DCache::::flush_area( release_addr as usize, - release_addr as usize + mem::size_of::(), + mem::size_of::(), + SY, ); asm::sev(); } diff --git a/kernel/src/arch/aarch64/interrupt/context.rs b/kernel/src/arch/aarch64/interrupt/context.rs index c2bf8f68..7b5b8d16 100644 --- a/kernel/src/arch/aarch64/interrupt/context.rs +++ b/kernel/src/arch/aarch64/interrupt/context.rs @@ -1,8 +1,8 @@ //! TrapFrame and context definitions for aarch64. -use aarch64::asm::{tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write_asid}; use aarch64::barrier; use aarch64::paging::PhysFrame; +use aarch64::translation::{local_invalidate_tlb_all, ttbr_el1_read, ttbr_el1_write_asid}; use lazy_static::lazy_static; use spin::Mutex; @@ -229,7 +229,7 @@ impl AsidAllocator { if self.0.generation == 0 { self.0.generation += 1; } - tlb_invalidate_all(); + local_invalidate_tlb_all(); } self.0.value += 1; return self.0; diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index 1d2db902..8890565f 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -1,8 +1,7 @@ //! Page table implementations for aarch64. use crate::memory::{alloc_frame, dealloc_frame, phys_to_virt}; -use aarch64::asm::{flush_dcache_range, flush_icache_all}; -use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write}; +use aarch64::cache::*; use aarch64::paging::{ frame::PhysFrame as Frame, mapper::{MappedPageTable, Mapper}, @@ -10,6 +9,8 @@ use aarch64::paging::{ page_table::{PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF}, FrameAllocator, FrameDeallocator, Page as PageAllSizes, Size2MiB, Size4KiB, }; +use aarch64::translation::{invalidate_tlb_vaddr, local_invalidate_tlb_all}; +use aarch64::translation::{ttbr_el1_read, ttbr_el1_write}; use aarch64::{align_down, align_up, PhysAddr, ALIGN_2MIB}; use core::mem::ManuallyDrop; use log::*; @@ -74,10 +75,20 @@ impl PageTable for PageTableImpl { fn flush_cache_copy_user(&mut self, start: usize, end: usize, execute: bool) { if execute { - // clean D-cache to PoU to ensure new instructions has been written into memory - flush_dcache_range(start, end); - // invalidate I-cache to PoU to ensure old instructions has been flushed - flush_icache_all(); + // clean D-cache to PoU to ensure new instructions has been written + // into memory + DCache::::flush_range(start, end, ISH); + // invalidate I-cache to PoU to ensure old instructions has been + // flushed + if get_l1_icache_policy() == L1ICachePolicy::PIPT { + // Cortex-A57 use PIPT, address translation is transparent + ICache::::flush_range(start, end, ISH); + } else { + // Cortex-A53 (raspi3) use VIPT, the effect of invalidation is + // only visible to the VA, need to invalidate the entire + // I-cache to invalidate all aliases of a PA. + ICache::flush_all(); + } } } } @@ -98,7 +109,7 @@ pub enum MMIOType { // TODO: software dirty bit needs to be reconsidered impl Entry for PageEntry { fn update(&mut self) { - tlb_invalidate(self.1.start_address()); + invalidate_tlb_vaddr(self.1.start_address()); } fn present(&self) -> bool { @@ -244,7 +255,7 @@ impl PageTableImpl { /// Used in `arch::memory::map_kernel()`. pub unsafe fn activate_as_kernel(&self) { ttbr_el1_write(1, Frame::of_addr(self.token() as u64)); - tlb_invalidate_all(); + local_invalidate_tlb_all(); } /// Map physical memory [start, end) /// to virtual space [phys_to_virt(start), phys_to_virt(end)) @@ -300,7 +311,7 @@ impl PageTableExt for PageTableImpl { } fn flush_tlb() { - tlb_invalidate_all(); + local_invalidate_tlb_all(); } }