diff --git a/crate/aarch64/Cargo.lock b/crate/aarch64/Cargo.lock new file mode 100644 index 00000000..2d0a3293 --- /dev/null +++ b/crate/aarch64/Cargo.lock @@ -0,0 +1,68 @@ +[[package]] +name = "aarch64" +version = "0.1.0" +dependencies = [ + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit_field" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cortex-a" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "os_bootinfo" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tock-registers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "usize_conversions" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ux" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" +"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum cortex-a 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2b187d0d728b4a99ba1d79f9671b976bcdd71a8a2c719585218fd2dc14a4d08c" +"checksum os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "66481dbeb5e773e7bd85b63cd6042c30786f834338288c5ec4f3742673db360a" +"checksum register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e10f31b6d2299e5620986ad9fcdd66463e125ad72af4f403f9aedf7592d5ccdb" +"checksum tock-registers 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a385d94f3f62e60445a0adb9ff8d9621faa272234530d4c0f848ec98f88e316" +"checksum usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f70329e2cbe45d6c97a5112daad40c34cd9a4e18edb5a2a18fefeb584d8d25e5" +"checksum ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "53d8df5dd8d07fedccd202de1887d94481fadaea3db70479f459e8163a1fab41" diff --git a/crate/aarch64/Cargo.toml b/crate/aarch64/Cargo.toml new file mode 100644 index 00000000..7514dd0e --- /dev/null +++ b/crate/aarch64/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "aarch64" +version = "0.1.0" +authors = ["koumingyang <1761674434@qq.com>"] + +[dependencies] +register = "0.2.0" +bit_field = "0.9.0" +bitflags = "1.0.1" +usize_conversions = "0.2.0" +os_bootinfo = "0.2.0" +bare-metal = "0.2.0" + +[dependencies.ux] +default-features = false +version = "0.1.0" \ No newline at end of file diff --git a/crate/aarch64/src/addr.rs b/crate/aarch64/src/addr.rs new file mode 100644 index 00000000..e3475675 --- /dev/null +++ b/crate/aarch64/src/addr.rs @@ -0,0 +1,418 @@ +use core::convert::{Into, TryInto}; +use core::fmt; +use core::ops::{Add, AddAssign, Sub, SubAssign}; + +use bit_field::BitField; +use usize_conversions::FromUsize; +use ux::*; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct VirtAddr(u64); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct PhysAddr(u64); + +#[derive(Debug)] +pub struct VirtAddrNotValid(u64); + +impl VirtAddr { + /// Creates a new canonical virtual address. + /// + /// This function performs sign extension of bit 47 to make the address canonical. Panics + /// if the bits in the range 48 to 64 contain data (i.e. are not null and no sign extension). + pub fn new(addr: u64) -> VirtAddr { + Self::try_new(addr).expect( + "invalid virtual address", + ) + } + + /// Tries to create a new canonical virtual address. + /// in aarch64, valid virtual address starts with 0x0000 or 0xffff. + pub fn try_new(addr: u64) -> Result { + match addr.get_bits(48..64) { + 0 | 0xffff => Ok(VirtAddr(addr)), // address is canonical + other => Err(VirtAddrNotValid(other)), + } + } + + pub fn new_unchecked(addr: u64) -> VirtAddr { + VirtAddr(addr) + } + + /// Creates a virtual address that points to `0`. + pub const fn zero() -> VirtAddr { + VirtAddr(0) + } + + /// Converts the address to an `u64`. + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Creates a virtual address from the given pointer + pub fn from_ptr(ptr: *const T) -> Self { + use usize_conversions::FromUsize; + + Self::new(u64::from_usize(ptr as usize)) + } + + /// Converts the address to a raw pointer. + #[cfg(target_pointer_width = "64")] + pub fn as_ptr(self) -> *const T { + use usize_conversions::usize_from; + + usize_from(self.as_u64()) as *const T + } + + /// Converts the address to a mutable raw pointer. + #[cfg(target_pointer_width = "64")] + pub fn as_mut_ptr(self) -> *mut T { + self.as_ptr::() as *mut T + } + + /// Aligns the virtual address upwards to the given alignment. + /// + /// See the `align_up` function for more information. + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_up(self.0, align.into())) + } + + /// Aligns the virtual address downwards to the given alignment. + /// + /// See the `align_down` function for more information. + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_down(self.0, align.into())) + } + + /// Checks whether the virtual address has the demanded alignment. + pub fn is_aligned(self, align: U) -> bool + where + U: Into, + { + self.align_down(align) == self + } + + /// Returns the 12-bit page offset of this virtual address. + pub fn page_offset(&self) -> u12 { + u12::new((self.0 & 0xfff).try_into().unwrap()) + } + + /// Returns the 9-bit level 1 page table index. + pub fn p1_index(&self) -> u9 { + u9::new(((self.0 >> 12) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 2 page table index. + pub fn p2_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 3 page table index. + pub fn p3_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9 >> 9) & 0o777).try_into().unwrap()) + } + + /// Returns the 9-bit level 4 page table index. + pub fn p4_index(&self) -> u9 { + u9::new(((self.0 >> 12 >> 9 >> 9 >> 9) & 0o777).try_into().unwrap()) + } +} + +impl fmt::Debug for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "VirtAddr({:#x})", self.0) + } +} + +impl Add for VirtAddr { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + VirtAddr::new(self.0 + rhs) + } +} + +impl AddAssign for VirtAddr { + fn add_assign(&mut self, rhs: u64) { + *self = *self + rhs; + } +} + +impl Add for VirtAddr +where + u64: FromUsize, +{ + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + self + u64::from_usize(rhs) + } +} + +impl AddAssign for VirtAddr +where + u64: FromUsize, +{ + fn add_assign(&mut self, rhs: usize) { + self.add_assign(u64::from_usize(rhs)) + } +} + +impl Sub for VirtAddr { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + VirtAddr::new(self.0.checked_sub(rhs).unwrap()) + } +} + +impl SubAssign for VirtAddr { + fn sub_assign(&mut self, rhs: u64) { + *self = *self - rhs; + } +} + +impl Sub for VirtAddr +where + u64: FromUsize, +{ + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + self - u64::from_usize(rhs) + } +} + +impl SubAssign for VirtAddr +where + u64: FromUsize, +{ + fn sub_assign(&mut self, rhs: usize) { + self.sub_assign(u64::from_usize(rhs)) + } +} + +impl Sub for VirtAddr { + type Output = u64; + fn sub(self, rhs: VirtAddr) -> Self::Output { + self.as_u64().checked_sub(rhs.as_u64()).unwrap() + } +} + +/// A passed `u64` was not a valid physical address. +/// +/// This means that bits 52 to 64 are not were not all null. +#[derive(Debug)] +pub struct PhysAddrNotValid(u64); + +impl PhysAddr { + /// Creates a new physical address. + /// + /// Panics if a bit in the range 52 to 64 is set. + pub fn new(addr: u64) -> PhysAddr { + assert_eq!( + addr.get_bits(52..64), + 0, + "physical addresses must not have any bits in the range 52 to 64 set" + ); + PhysAddr(addr) + } + + /// Tries to create a new physical address. + /// + /// Fails if any bits in the range 52 to 64 are set. + pub fn try_new(addr: u64) -> Result { + match addr.get_bits(52..64) { + 0 => Ok(PhysAddr(addr)), // address is valid + other => Err(PhysAddrNotValid(other)), + } + } + + /// Converts the address to an `u64`. + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Convenience method for checking if a physical address is null. + pub fn is_null(&self) -> bool { + self.0 == 0 + } + + /// Aligns the physical address upwards to the given alignment. + /// + /// See the `align_up` function for more information. + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_up(self.0, align.into())) + } + + /// Aligns the physical address downwards to the given alignment. + /// + /// See the `align_down` function for more information. + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_down(self.0, align.into())) + } + + /// Checks whether the physical address has the demanded alignment. + pub fn is_aligned(self, align: U) -> bool + where + U: Into, + { + self.align_down(align) == self + } +} + +impl fmt::Debug for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "PhysAddr({:#x})", self.0) + } +} + +impl fmt::Binary for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::LowerHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Octal for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::UpperHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl Add for PhysAddr { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + PhysAddr::new(self.0 + rhs) + } +} + +impl AddAssign for PhysAddr { + fn add_assign(&mut self, rhs: u64) { + *self = *self + rhs; + } +} + +impl Add for PhysAddr +where + u64: FromUsize, +{ + type Output = Self; + fn add(self, rhs: usize) -> Self::Output { + self + u64::from_usize(rhs) + } +} + +impl AddAssign for PhysAddr +where + u64: FromUsize, +{ + fn add_assign(&mut self, rhs: usize) { + self.add_assign(u64::from_usize(rhs)) + } +} + +impl Sub for PhysAddr { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + PhysAddr::new(self.0.checked_sub(rhs).unwrap()) + } +} + +impl SubAssign for PhysAddr { + fn sub_assign(&mut self, rhs: u64) { + *self = *self - rhs; + } +} + +impl Sub for PhysAddr +where + u64: FromUsize, +{ + type Output = Self; + fn sub(self, rhs: usize) -> Self::Output { + self - u64::from_usize(rhs) + } +} + +impl SubAssign for PhysAddr +where + u64: FromUsize, +{ + fn sub_assign(&mut self, rhs: usize) { + self.sub_assign(u64::from_usize(rhs)) + } +} + +impl Sub for PhysAddr { + type Output = u64; + fn sub(self, rhs: PhysAddr) -> Self::Output { + self.as_u64().checked_sub(rhs.as_u64()).unwrap() + } +} + +/// Align address downwards. +/// +/// Returns the greatest x with alignment `align` so that x <= addr. The alignment must be +/// a power of 2. +pub fn align_down(addr: u64, align: u64) -> u64 { + assert!(align.is_power_of_two(), "`align` must be a power of two"); + addr & !(align - 1) +} + +/// Align address upwards. +/// +/// Returns the smallest x with alignment `align` so that x >= addr. The alignment must be +/// a power of 2. +pub fn align_up(addr: u64, align: u64) -> u64 { + assert!(align.is_power_of_two(), "`align` must be a power of two"); + let align_mask = align - 1; + if addr & align_mask == 0 { + addr // already aligned + } else { + (addr | align_mask) + 1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_align_up() { + // align 1 + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(1234, 1), 1234); + assert_eq!(align_up(0xffffffffffffffff, 1), 0xffffffffffffffff); + // align 2 + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(1233, 2), 1234); + assert_eq!(align_up(0xfffffffffffffffe, 2), 0xfffffffffffffffe); + // address 0 + assert_eq!(align_up(0, 128), 0); + assert_eq!(align_up(0, 1), 0); + assert_eq!(align_up(0, 2), 0); + assert_eq!(align_up(0, 0x8000000000000000), 0); + } +} diff --git a/crate/aarch64/src/asm.rs b/crate/aarch64/src/asm.rs new file mode 100644 index 00000000..d97fa78a --- /dev/null +++ b/crate/aarch64/src/asm.rs @@ -0,0 +1,162 @@ +use paging::PhysFrame; +use addr::PhysAddr; +use regs::*; + +#[inline(always)] +pub fn tlb_invalidate() { + unsafe{ + asm!("dsb ishst + tlbi vmalle1is + dsb ish + tlbi vmalle1is + isb"); + } +} + +/// Returns the current stack pointer. +#[inline(always)] +pub fn sp() -> *const u8 { + let ptr: usize; + unsafe { + asm!("mov $0, sp" : "=r"(ptr)); + } + + ptr as *const u8 +} + +#[inline(always)] +pub unsafe fn get_pc() -> usize { + let pc: usize; + asm!("ADR $0, ." : "=r"(pc)); + pc +} + +/// Returns the current exception level. +/// +/// # Safety +/// This function should only be called when EL is >= 1. +#[inline(always)] +pub unsafe fn current_el() -> u8 { + let el_reg: u64; + asm!("mrs $0, CurrentEL" : "=r"(el_reg)); + ((el_reg & 0b1100) >> 2) as u8 +} + +#[inline(always)] +pub unsafe fn get_far() -> usize { + let far: usize; + asm!("mrs $0, far_el1" : "=r"(far)); + far +} + +#[inline(always)] +pub unsafe fn get_ttbr0() -> usize { + let ttbr0: usize; + asm!("mrs $0, ttbr0_el1" : "=r"(ttbr0)); + ttbr0 +} + +#[inline(always)] +pub unsafe fn get_ttbr1() -> usize { + let ttbr0: usize; + asm!("mrs $0, ttbr1_el1" : "=r"(ttbr0)); + ttbr0 +} + +/// Returns the SPSel value. +#[inline(always)] +pub fn sp_sel() -> u8 { + let ptr: u32; + unsafe { + asm!("mrs $0, SPSel" : "=r"(ptr)); + } + + (ptr & 1) as u8 +} + +/// Returns the core currently executing. +/// +/// # Safety +/// +/// This function should only be called when EL is >= 1. +pub unsafe fn affinity() -> usize { + let x: usize; + asm!("mrs $0, mpidr_el1 + and $0, $0, #3" + : "=r"(x)); + + x +} + +pub fn wfi() { + unsafe { + asm!("wfi" :::: "volatile"); + } +} + + +/// The classic no-op +#[inline] +pub fn nop() { + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { asm!("nop" :::: "volatile") }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +/// Wait For Event +#[inline] +pub fn wfe() { + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { asm!("wfe" :::: "volatile") }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +/// Exception return +/// +/// Will jump to wherever the corresponding link register points to, and +/// therefore never return. +#[inline] +pub fn eret() -> ! { + use core; + + match () { + #[cfg(target_arch = "aarch64")] + () => unsafe { + asm!("eret" :::: "volatile"); + core::intrinsics::unreachable() + }, + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } +} + +bitflags! { + /// Controls cache settings for the level 4 page table. + pub struct ttbr0_el1_Flags: u64 { + + const COMMON_NOT_PRIVATE = 1 << 0; + } +} + +pub fn ttbr0_el1_read() -> (PhysFrame, ttbr0_el1_Flags) { + let value = TTBR0_EL1.get(); + let flags = ttbr0_el1_Flags::from_bits_truncate(value); + let addr = PhysAddr::new(value & 0x_000f_ffff_ffff_f000); + let frame = PhysFrame::containing_address(addr); + (frame, flags) +} + +pub fn ttbr0_el1_write(frame: PhysFrame) { + let addr = frame.start_address(); + let value = addr.as_u64(); + TTBR0_EL1.set_baddr(value); +} diff --git a/crate/aarch64/src/barrier.rs b/crate/aarch64/src/barrier.rs new file mode 100644 index 00000000..0e48209e --- /dev/null +++ b/crate/aarch64/src/barrier.rs @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +// Borrow implementations from the pending upstream ACLE implementation until it +// is merged. Afterwards, we'll probably just reexport them, hoping that the API +// doesn't change. +// +// https://github.com/rust-lang-nursery/stdsimd/pull/557 + +mod sealed { + pub trait Dmb { + unsafe fn __dmb(&self); + } + + pub trait Dsb { + unsafe fn __dsb(&self); + } + + pub trait Isb { + unsafe fn __isb(&self); + } +} + +macro_rules! dmb_dsb { + ($A:ident) => { + impl sealed::Dmb for $A { + #[inline(always)] + unsafe fn __dmb(&self) { + asm!(concat!("DMB ", stringify!($A)) : : : "memory" : "volatile") + } + } + impl sealed::Dsb for $A { + #[inline(always)] + unsafe fn __dsb(&self) { + asm!(concat!("DSB ", stringify!($A)) : : : "memory" : "volatile") + } + } + }; +} + +pub struct SY; + +dmb_dsb!(SY); + +impl sealed::Isb for SY { + #[inline(always)] + unsafe fn __isb(&self) { + asm!("ISB SY" : : : "memory" : "volatile") + } +} + +#[inline(always)] +pub unsafe fn dmb(arg: A) +where + A: sealed::Dmb, +{ + arg.__dmb() +} + +#[inline(always)] +pub unsafe fn dsb(arg: A) +where + A: sealed::Dsb, +{ + arg.__dsb() +} + +#[inline(always)] +pub unsafe fn isb(arg: A) +where + A: sealed::Isb, +{ + arg.__isb() +} diff --git a/crate/aarch64/src/lib.rs b/crate/aarch64/src/lib.rs new file mode 100644 index 00000000..bb8bf725 --- /dev/null +++ b/crate/aarch64/src/lib.rs @@ -0,0 +1,29 @@ +#![no_std] +//#![deny(warnings)] +#![feature(asm)] +#![feature(const_fn)] +#![feature(core_intrinsics)] +#![feature(try_from)] + +extern crate bare_metal; +#[macro_use] +extern crate register; +#[macro_use] +extern crate bitflags; +extern crate bit_field; +extern crate os_bootinfo; +extern crate usize_conversions; + +/// Provides the non-standard-width integer types `u2`–`u63`. +/// +/// We use these integer types in various APIs, for example `u9` for page tables indices. +pub extern crate ux; + +pub use addr::{align_down, align_up, PhysAddr, VirtAddr}; + +pub mod asm; +pub mod addr; +pub mod paging; +pub mod barrier; +pub mod regs; + diff --git a/crate/aarch64/src/paging/frame_alloc.rs b/crate/aarch64/src/paging/frame_alloc.rs new file mode 100644 index 00000000..f3ea4c8b --- /dev/null +++ b/crate/aarch64/src/paging/frame_alloc.rs @@ -0,0 +1,15 @@ +//! Traits for abstracting away frame allocation and deallocation. + +use paging::{PageSize, PhysFrame}; + +/// A trait for types that can allocate a frame of memory. +pub trait FrameAllocator { + /// Allocate a frame of the appropriate size and return it if possible. + fn alloc(&mut self) -> Option>; +} + +/// A trait for types that can deallocate a frame of memory. +pub trait FrameDeallocator { + /// Deallocate the given frame of memory. + fn dealloc(&mut self, frame: PhysFrame); +} diff --git a/crate/aarch64/src/paging/mod.rs b/crate/aarch64/src/paging/mod.rs new file mode 100644 index 00000000..56073b23 --- /dev/null +++ b/crate/aarch64/src/paging/mod.rs @@ -0,0 +1,528 @@ +//! Abstractions for page tables and other paging related structures. +//! +//! Page tables translate virtual memory “pages” to physical memory “frames”. + +pub use self::frame_alloc::*; +pub use self::page_table::*; +#[cfg(target_arch = "aarch64")] +pub use self::recursive::*; + +use core::fmt; +use core::marker::PhantomData; +use core::ops::{Add, AddAssign, Sub, SubAssign}; +use os_bootinfo; +use ux::*; +use addr::{PhysAddr, VirtAddr}; + +mod frame_alloc; +mod page_table; +mod recursive; + +/// Trait for abstracting over the three possible page sizes on x86_64, 4KiB, 2MiB, 1GiB. +pub trait PageSize: Copy + Eq + PartialOrd + Ord { + /// The page size in bytes. + const SIZE: u64; + + /// A string representation of the page size for debug output. + const SIZE_AS_DEBUG_STR: &'static str; +} + +/// This trait is implemented for 4KiB and 2MiB pages, but not for 1GiB pages. +pub trait NotGiantPageSize: PageSize {} + +/// A standard 4KiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size4KiB {} + +/// A “huge” 2MiB page. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size2MiB {} + +/// A “giant” 1GiB page. +/// +/// (Only available on newer x86_64 CPUs.) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Size1GiB {} + +impl PageSize for Size4KiB { + const SIZE: u64 = 4096; + const SIZE_AS_DEBUG_STR: &'static str = "4KiB"; +} + +impl NotGiantPageSize for Size4KiB {} + +impl PageSize for Size2MiB { + const SIZE: u64 = Size4KiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "2MiB"; +} + +impl NotGiantPageSize for Size2MiB {} + +impl PageSize for Size1GiB { + const SIZE: u64 = Size2MiB::SIZE * 512; + const SIZE_AS_DEBUG_STR: &'static str = "1GiB"; +} + +/// A virtual memory page. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub struct Page { + start_address: VirtAddr, + size: PhantomData, +} + +impl Page { + /// Returns the page that starts at the given virtual address. + /// + /// Returns an error if the address is not correctly aligned (i.e. is not a valid page start). + pub fn from_start_address(address: VirtAddr) -> Result { + if !address.is_aligned(S::SIZE) { + return Err(()); + } + Ok(Page::containing_address(address)) + } + + /// Returns the page that contains the given virtual address. + pub fn containing_address(address: VirtAddr) -> Self { + Page { + start_address: address.align_down(S::SIZE), + size: PhantomData, + } + } + + /// Returns the start address of the page. + pub fn start_address(&self) -> VirtAddr { + self.start_address + } + + /// Returns the size the page (4KB, 2MB or 1GB). + pub const fn size(&self) -> u64 { + S::SIZE + } + + /// Returns the level 4 page table index of this page. + pub fn p4_index(&self) -> u9 { + self.start_address().p4_index() + } + + /// Returns the level 3 page table index of this page. + pub fn p3_index(&self) -> u9 { + self.start_address().p3_index() + } + + /// Returns a range of pages, exclusive `end`. + pub fn range(start: Self, end: Self) -> PageRange { + PageRange { start, end } + } + + /// Returns a range of pages, inclusive `end`. + pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { + PageRangeInclusive { start, end } + } +} + +impl Page { + /// Returns the level 2 page table index of this page. + pub fn p2_index(&self) -> u9 { + self.start_address().p2_index() + } +} + +impl Page { + /// Returns the 1GiB memory page with the specified page table indices. + pub fn from_page_table_indices_1gib(p4_index: u9, p3_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + +impl Page { + /// Returns the 2MiB memory page with the specified page table indices. + pub fn from_page_table_indices_2mib(p4_index: u9, p3_index: u9, p2_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + addr.set_bits(21..30, u64::from(p2_index)); + Page::containing_address(VirtAddr::new(addr)) + } +} + +impl Page { + /// Returns the 4KiB memory page with the specified page table indices. + pub fn from_page_table_indices(p4_index: u9, p3_index: u9, p2_index: u9, p1_index: u9) -> Self { + use bit_field::BitField; + + let mut addr = 0; + addr.set_bits(39..48, u64::from(p4_index)); + addr.set_bits(30..39, u64::from(p3_index)); + addr.set_bits(21..30, u64::from(p2_index)); + addr.set_bits(12..21, u64::from(p1_index)); + Page::containing_address(VirtAddr::new(addr)) + } + + /// Returns the level 1 page table index of this page. + pub fn p1_index(&self) -> u9 { + self.start_address().p1_index() + } +} + +impl fmt::Debug for Page { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_fmt(format_args!( + "Page[{}]({:#x})", + S::SIZE_AS_DEBUG_STR, + self.start_address().as_u64() + )) + } +} + +impl Add for Page { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + Page::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) + } +} + +impl AddAssign for Page { + fn add_assign(&mut self, rhs: u64) { + *self = self.clone() + rhs; + } +} + +impl Sub for Page { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + Page::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) + } +} + +impl SubAssign for Page { + fn sub_assign(&mut self, rhs: u64) { + *self = self.clone() - rhs; + } +} + +impl Sub for Page { + type Output = u64; + fn sub(self, rhs: Self) -> Self::Output { + (self.start_address - rhs.start_address) / S::SIZE + } +} + +/// A range of pages with exclusive upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PageRange { + /// The start of the range, inclusive. + pub start: Page, + /// The end of the range, exclusive. + pub end: Page, +} + +impl PageRange { + /// Returns wether this range contains no pages. + pub fn is_empty(&self) -> bool { + !(self.start < self.end) + } +} + +impl Iterator for PageRange { + type Item = Page; + + fn next(&mut self) -> Option { + if self.start < self.end { + let page = self.start.clone(); + self.start += 1; + Some(page) + } else { + None + } + } +} + +impl PageRange { + /// Converts the range of 2MiB pages to a range of 4KiB pages. + pub fn as_4kib_page_range(self) -> PageRange { + PageRange { + start: Page::containing_address(self.start.start_address()), + end: Page::containing_address(self.end.start_address()), + } + } +} + +impl fmt::Debug for PageRange { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PageRange") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// A range of pages with inclusive upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PageRangeInclusive { + /// The start of the range, inclusive. + pub start: Page, + /// The end of the range, inclusive. + pub end: Page, +} + +impl PageRangeInclusive { + /// Returns wether this range contains no pages. + pub fn is_empty(&self) -> bool { + !(self.start <= self.end) + } +} + +impl Iterator for PageRangeInclusive { + type Item = Page; + + fn next(&mut self) -> Option { + if self.start <= self.end { + let page = self.start.clone(); + self.start += 1; + Some(page) + } else { + None + } + } +} + +impl fmt::Debug for PageRangeInclusive { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PageRangeInclusive") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// A physical memory frame. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub struct PhysFrame { + start_address: PhysAddr, + size: PhantomData, +} + +impl PhysFrame { + /// Returns the frame that starts at the given virtual address. + /// + /// Returns an error if the address is not correctly aligned (i.e. is not a valid frame start). + pub fn from_start_address(address: PhysAddr) -> Result { + if !address.is_aligned(S::SIZE) { + return Err(()); + } + Ok(PhysFrame::containing_address(address)) + } + + /// Returns the frame that contains the given physical address. + pub fn containing_address(address: PhysAddr) -> Self { + PhysFrame { + start_address: address.align_down(S::SIZE), + size: PhantomData, + } + } + + /// Returns the start address of the frame. + pub fn start_address(&self) -> PhysAddr { + self.start_address + } + + /// Returns the size the frame (4KB, 2MB or 1GB). + pub fn size(&self) -> u64 { + S::SIZE + } + + /// Returns a range of frames, exclusive `end`. + pub fn range(start: PhysFrame, end: PhysFrame) -> PhysFrameRange { + PhysFrameRange { start, end } + } + + /// Returns a range of frames, inclusive `end`. + pub fn range_inclusive(start: PhysFrame, end: PhysFrame) -> PhysFrameRangeInclusive { + PhysFrameRangeInclusive { start, end } + } +} + +impl fmt::Debug for PhysFrame { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_fmt(format_args!( + "PhysFrame[{}]({:#x})", + S::SIZE_AS_DEBUG_STR, + self.start_address().as_u64() + )) + } +} + +impl Add for PhysFrame { + type Output = Self; + fn add(self, rhs: u64) -> Self::Output { + PhysFrame::containing_address(self.start_address() + rhs * u64::from(S::SIZE)) + } +} + +impl AddAssign for PhysFrame { + fn add_assign(&mut self, rhs: u64) { + *self = self.clone() + rhs; + } +} + +impl Sub for PhysFrame { + type Output = Self; + fn sub(self, rhs: u64) -> Self::Output { + PhysFrame::containing_address(self.start_address() - rhs * u64::from(S::SIZE)) + } +} + +impl SubAssign for PhysFrame { + fn sub_assign(&mut self, rhs: u64) { + *self = self.clone() - rhs; + } +} + +impl Sub> for PhysFrame { + type Output = u64; + fn sub(self, rhs: PhysFrame) -> Self::Output { + (self.start_address - rhs.start_address) / S::SIZE + } +} + +/// An range of physical memory frames, exclusive the upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PhysFrameRange { + /// The start of the range, inclusive. + pub start: PhysFrame, + /// The end of the range, exclusive. + pub end: PhysFrame, +} + +impl PhysFrameRange { + /// Returns whether the range contains no frames. + pub fn is_empty(&self) -> bool { + !(self.start < self.end) + } +} + +impl Iterator for PhysFrameRange { + type Item = PhysFrame; + + fn next(&mut self) -> Option { + if self.start < self.end { + let frame = self.start.clone(); + self.start += 1; + Some(frame) + } else { + None + } + } +} + +impl From for PhysFrameRange { + fn from(range: os_bootinfo::FrameRange) -> Self { + PhysFrameRange { + start: PhysFrame::from_start_address(PhysAddr::new(range.start_addr())).unwrap(), + end: PhysFrame::from_start_address(PhysAddr::new(range.end_addr())).unwrap(), + } + } +} + +impl Into for PhysFrameRange { + fn into(self) -> os_bootinfo::FrameRange { + os_bootinfo::FrameRange::new( + self.start.start_address().as_u64(), + self.end.start_address().as_u64(), + ) + } +} + +impl fmt::Debug for PhysFrameRange { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PhysFrameRange") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +/// An range of physical memory frames, inclusive the upper bound. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct PhysFrameRangeInclusive { + /// The start of the range, inclusive. + pub start: PhysFrame, + /// The start of the range, exclusive. + pub end: PhysFrame, +} + +impl PhysFrameRangeInclusive { + /// Returns whether the range contains no frames. + pub fn is_empty(&self) -> bool { + !(self.start <= self.end) + } +} + +impl Iterator for PhysFrameRangeInclusive { + type Item = PhysFrame; + + fn next(&mut self) -> Option { + if self.start <= self.end { + let frame = self.start.clone(); + self.start += 1; + Some(frame) + } else { + None + } + } +} + +impl fmt::Debug for PhysFrameRangeInclusive { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("PhysFrameRangeInclusive") + .field("start", &self.start) + .field("end", &self.end) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + pub fn test_page_ranges() { + let page_size = Size4KiB::SIZE; + let number = 1000; + + let start_addr = VirtAddr::new(0xdeadbeaf); + let start: Page = Page::containing_address(start_addr); + let end = start.clone() + number; + + let mut range = Page::range(start.clone(), end.clone()); + for i in 0..number { + assert_eq!( + range.next(), + Some(Page::containing_address(start_addr + page_size * i)) + ); + } + assert_eq!(range.next(), None); + + let mut range_inclusive = Page::range_inclusive(start, end); + for i in 0..=number { + assert_eq!( + range_inclusive.next(), + Some(Page::containing_address(start_addr + page_size * i)) + ); + } + assert_eq!(range_inclusive.next(), None); + } +} diff --git a/crate/aarch64/src/paging/page_table.rs b/crate/aarch64/src/paging/page_table.rs new file mode 100644 index 00000000..b0747681 --- /dev/null +++ b/crate/aarch64/src/paging/page_table.rs @@ -0,0 +1,185 @@ +use core::fmt; +use core::ops::{Index, IndexMut}; + +use super::{PageSize, PhysFrame, Size4KiB}; +use addr::PhysAddr; + +use usize_conversions::usize_from; +use ux::*; + +/// The error returned by the `PageTableEntry::frame` method. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum FrameError { + /// The entry does not have the `PRESENT` flag set, so it isn't currently mapped to a frame. + FrameNotPresent, + /// The entry does have the `HUGE_PAGE` flag set. The `frame` method has a standard 4KiB frame + /// as return type, so a huge frame can't be returned. + HugeFrame, +} + +/// A 64-bit page table entry. +#[derive(Clone)] +#[repr(transparent)] +pub struct PageTableEntry { + entry: u64, +} + +impl PageTableEntry { + /// Returns whether this entry is zero. + pub fn is_unused(&self) -> bool { + self.entry == 0 + } + + /// Sets this entry to zero. + pub fn set_unused(&mut self) { + self.entry = 0; + } + + /// Returns the flags of this entry. + pub fn flags(&self) -> PageTableFlags { + PageTableFlags::from_bits_truncate(self.entry) + } + + /// Returns the physical address mapped by this entry, might be zero. + pub fn addr(&self) -> PhysAddr { + PhysAddr::new(self.entry & 0x000fffff_fffff000) + } + + /// Returns the physical frame mapped by this entry. + /// + /// Returns the following errors: + /// + /// - `FrameError::FrameNotPresent` if the entry doesn't have the `PRESENT` flag set. + /// - `FrameError::HugeFrame` if the entry has the `HUGE_PAGE` flag set (for huge pages the + /// `addr` function must be used) + pub fn frame(&self) -> Result { + if !self.flags().contains(PageTableFlags::PRESENT) { + Err(FrameError::FrameNotPresent) + } else if self.flags().contains(PageTableFlags::HUGE_PAGE) { + Err(FrameError::HugeFrame) + } else { + Ok(PhysFrame::containing_address(self.addr())) + } + } + + /// Map the entry to the specified physical address with the specified flags. + pub fn set_addr(&mut self, addr: PhysAddr, flags: PageTableFlags) { + assert!(addr.is_aligned(Size4KiB::SIZE)); + self.entry = (addr.as_u64()) | flags.bits(); + } + + /// Map the entry to the specified physical frame with the specified flags. + pub fn set_frame(&mut self, frame: PhysFrame, flags: PageTableFlags) { + assert!(!flags.contains(PageTableFlags::HUGE_PAGE)); + self.set_addr(frame.start_address(), flags) + } + + /// Sets the flags of this entry. + pub fn set_flags(&mut self, flags: PageTableFlags) { + self.entry = self.addr().as_u64() | flags.bits(); + } +} + +impl fmt::Debug for PageTableEntry { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut f = f.debug_struct("PageTableEntry"); + f.field("addr", &self.addr()); + f.field("flags", &self.flags()); + f.finish() + } +} + +bitflags! { + /// Possible flags for a page table entry. + pub struct PageTableFlags: u64 { + const ALL = 0xffffffff_ffffffff; + const TYPE_MASK = 3 << 0; + const TYPE_FAULT = 0 << 0; + const TYPE_PAGE = 3 << 0; + const TABLE_BIT = 1 << 1; + + const PRESENT = 1 << 0; + const USER_ACCESSIBLE = 1 << 6; /* AP[1] */ + const RDONLY = 1 << 7; /* AP[2] */ + const SHARED = 3 << 8; /* SH[1:0], inner shareable */ + const BIT_8 = 1 << 8; + const BIT_9 = 1 << 9; + /* + pub const ATTRIB_SH_NON_SHAREABLE: usize = 0x0 << 8; + pub const ATTRIB_SH_OUTER_SHAREABLE: usize = 0x2 << 8; + pub const ATTRIB_SH_INNER_SHAREABLE: usize = 0x3 << 8; + */ + + const ACCESSED = 1 << 10; /* AF, Access Flag */ + const NONE_GLOBAL = 1 << 11; /* None Global */ + const GLOBAL = (!(1 << 11)); + const DBM = 1 << 51; /* Dirty Bit Management */ + const WRITE = 1 << 51; /* DBM */ + const CONT = 1 << 52; /* Contiguous range */ + const PXN = 1 << 53; /* Privileged XN */ + const UXN = 1 << 54; /* User XN */ + const HYP_XN = 1 << 54; /* HYP XN */ + const DIRTY = 1 << 55; + const SWAPPED = 1 << 56; + const HUGE_PAGE = 1 << 57; + const PROT_NONE = 1 << 58; + + } +} + +/// The number of entries in a page table. +const ENTRY_COUNT: usize = 512; + +/// Represents a page table. +/// +/// Always page-sized. +/// +/// This struct implements the `Index` and `IndexMut` traits, so the entries can be accessed +/// through index operations. For example, `page_table[15]` returns the 15th page table entry. +#[repr(transparent)] +pub struct PageTable { + entries: [PageTableEntry; ENTRY_COUNT], +} + +impl PageTable { + /// Clears all entries. + pub fn zero(&mut self) { + for entry in self.entries.iter_mut() { + entry.set_unused(); + } + } +} + +impl Index for PageTable { + type Output = PageTableEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl Index for PageTable { + type Output = PageTableEntry; + + fn index(&self, index: u9) -> &Self::Output { + &self.entries[usize_from(u16::from(index))] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: u9) -> &mut Self::Output { + &mut self.entries[usize_from(u16::from(index))] + } +} + +impl fmt::Debug for PageTable { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.entries[..].fmt(f) + } +} diff --git a/crate/aarch64/src/paging/recursive.rs b/crate/aarch64/src/paging/recursive.rs new file mode 100644 index 00000000..0cbc6d09 --- /dev/null +++ b/crate/aarch64/src/paging/recursive.rs @@ -0,0 +1,623 @@ +#![cfg(target_arch = "aarch64")] + +use asm::tlb_invalidate; +use paging::{ + frame_alloc::FrameAllocator, + page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags}, + NotGiantPageSize, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, +}; +use paging::page_table::PageTableFlags as Flags; +use asm::ttbr0_el1_read; +use ux::u9; +use addr::{PhysAddr, VirtAddr}; + +/// This type represents a page whose mapping has changed in the page table. +/// +/// The old mapping might be still cached in the translation lookaside buffer (TLB), so it needs +/// to be flushed from the TLB before it's accessed. This type is returned from function that +/// change the mapping of a page to ensure that the TLB flush is not forgotten. +#[derive(Debug)] +#[must_use = "Page Table changes must be flushed or ignored."] +pub struct MapperFlush(Page); + +impl MapperFlush { + /// Create a new flush promise + fn new(page: Page) -> Self { + MapperFlush(page) + } + + /// Flush the page from the TLB to ensure that the newest mapping is used. + pub fn flush(self) { + tlb_invalidate(); + } + + /// Don't flush the TLB and silence the “must be used” warning. + pub fn ignore(self) {} +} + +/// A trait for common page table operations. +pub trait Mapper { + /// Creates a new mapping in the page table. + /// + /// This function might need additional physical frames to create new page tables. These + /// frames are allocated from the `allocator` argument. At most three frames are required. + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator; + + /// Removes a mapping from the page table and returns the frame that used to be mapped. + /// + /// Note that no page tables or pages are deallocated. + fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + + /// Updates the flags of an existing mapping. + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError>; + + /// Return the frame that the specified page is mapped to. + fn translate_page(&self, page: Page) -> Option>; + + /// Maps the given frame to the virtual page with the same address. + fn identity_map( + &mut self, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + S: PageSize, + Self: Mapper, + { + let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); + self.map_to(page, frame, flags, allocator) + } +} + +/// A recursive page table is a last level page table with an entry mapped to the table itself. +/// +/// This recursive mapping allows accessing all page tables in the hierarchy: +/// +/// - To access the level 4 page table, we “loop“ (i.e. follow the recursively mapped entry) four +/// times. +/// - To access a level 3 page table, we “loop” three times and then use the level 4 index. +/// - To access a level 2 page table, we “loop” two times, then use the level 4 index, then the +/// level 3 index. +/// - To access a level 1 page table, we “loop” once, then use the level 4 index, then the +/// level 3 index, then the level 2 index. +/// +/// This struct implements the `Mapper` trait. +#[derive(Debug)] +pub struct RecursivePageTable<'a> { + p4: &'a mut PageTable, + recursive_index: u9, +} + +/// An error indicating that the given page table is not recursively mapped. +/// +/// Returned from `RecursivePageTable::new`. +#[derive(Debug)] +pub struct NotRecursivelyMapped; + +/// This error is returned from `map_to` and similar methods. +#[derive(Debug)] +pub enum MapToError { + /// An additional frame was needed for the mapping process, but the frame allocator + /// returned `None`. + FrameAllocationFailed, + /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the + /// given page is part of an already mapped huge page. + ParentEntryHugePage, + /// The given page is already mapped to a physical frame. + PageAlreadyMapped, +} + +/// An error indicating that an `unmap` call failed. +#[derive(Debug)] +pub enum UnmapError { + /// An upper level page table entry has the `HUGE_PAGE` flag set, which means that the + /// given page is part of a huge page and can't be freed individually. + ParentEntryHugePage, + /// The given page is not mapped to a physical frame. + PageNotMapped, + /// The page table entry for the given page points to an invalid physical address. + InvalidFrameAddress(PhysAddr), +} + +/// An error indicating that an `update_flags` call failed. +#[derive(Debug)] +pub enum FlagUpdateError { + /// The given page is not mapped to a physical frame. + PageNotMapped, +} + +impl<'a> RecursivePageTable<'a> { + /// Creates a new RecursivePageTable from the passed level 4 PageTable. + /// + /// The page table must be recursively mapped, that means: + /// + /// - The page table must have one recursive entry, i.e. an entry that points to the table + /// itself. + /// - The reference must use that “loop”, i.e. be of the form `0o_xxx_xxx_xxx_xxx_0000` + /// where `xxx` is the recursive entry. + /// - The page table must be active, i.e. the CR3 register must contain its physical address. + /// + /// Otherwise `Err(NotRecursivelyMapped)` is returned. + pub fn new(table: &'a mut PageTable) -> Result { + let page = Page::containing_address(VirtAddr::new(table as *const _ as u64)); + let recursive_index = page.p4_index(); + + if page.p3_index() != recursive_index + || page.p2_index() != recursive_index + || page.p1_index() != recursive_index + { + return Err(NotRecursivelyMapped); + } + if Ok(ttbr0_el1_read().0) != table[recursive_index].frame() { + return Err(NotRecursivelyMapped); + } + + Ok(RecursivePageTable { + p4: table, + recursive_index, + }) + } + + /// Creates a new RecursivePageTable without performing any checks. + /// + /// The `recursive_index` parameter must be the index of the recursively mapped entry. + pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: u9) -> Self { + RecursivePageTable { + p4: table, + recursive_index, + } + } + + /// Internal helper function to create the page table of the next level if needed. + /// + /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed, + /// and the entry is updated to that address. If the passed entry is already mapped, the next + /// table is returned directly. + /// + /// The `next_page_table` page must be the page of the next page table in the hierarchy. + /// + /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator + /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set + /// in the passed entry. + unsafe fn create_next_table<'b, A>( + entry: &'b mut PageTableEntry, + next_table_page: Page, + allocator: &mut A, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator, + { + /// This inner function is used to limit the scope of `unsafe`. + /// + /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe. + fn inner<'b, A>( + entry: &'b mut PageTableEntry, + next_table_page: Page, + allocator: &mut A, + ) -> Result<&'b mut PageTable, MapToError> + where + A: FrameAllocator, + { + + let created; + + if entry.is_unused() { + if let Some(frame) = allocator.alloc() { + entry.set_frame(frame, Flags::PRESENT | Flags::WRITE); + created = true; + } else { + return Err(MapToError::FrameAllocationFailed); + } + } else { + created = false; + } + if entry.flags().contains(Flags::HUGE_PAGE) { + return Err(MapToError::ParentEntryHugePage); + } + + let page_table_ptr = next_table_page.start_address().as_mut_ptr(); + let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) }; + if created { + page_table.zero(); + } + Ok(page_table) + } + + inner(entry, next_table_page, allocator) + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + if !p3[page.p3_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &mut p3[page.p3_index()]; + let flags = p3_entry.flags(); + + if !flags.contains(PageTableFlags::PRESENT) { + return Err(UnmapError::PageNotMapped); + } + if !flags.contains(PageTableFlags::HUGE_PAGE) { + return Err(UnmapError::ParentEntryHugePage); + } + + let frame = PhysFrame::from_start_address(p3_entry.addr()) + .map_err(|()| UnmapError::InvalidFrameAddress(p3_entry.addr()))?; + + p3_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p3_entry.addr()).ok() + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + let p2_page = p2_page(page, self.recursive_index); + let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; + + if !p2[page.p2_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + p3_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &mut p2[page.p2_index()]; + let flags = p2_entry.flags(); + + if !flags.contains(PageTableFlags::PRESENT) { + return Err(UnmapError::PageNotMapped); + } + if !flags.contains(PageTableFlags::HUGE_PAGE) { + return Err(UnmapError::ParentEntryHugePage); + } + + let frame = PhysFrame::from_start_address(p2_entry.addr()) + .map_err(|()| UnmapError::InvalidFrameAddress(p2_entry.addr()))?; + + p2_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + + if p2[page.p2_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + + if p2_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p2_entry.addr()).ok() + } +} + +impl<'a> Mapper for RecursivePageTable<'a> { + fn map_to( + &mut self, + page: Page, + frame: PhysFrame, + flags: PageTableFlags, + allocator: &mut A, + ) -> Result, MapToError> + where + A: FrameAllocator, + { + let p4 = &mut self.p4; + + let p3_page = p3_page(page, self.recursive_index); + let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? }; + + let p2_page = p2_page(page, self.recursive_index); + let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? }; + + let p1_page = p1_page(page, self.recursive_index); + let p1 = unsafe { Self::create_next_table(&mut p2[page.p2_index()], p1_page, allocator)? }; + + if !p1[page.p1_index()].is_unused() { + return Err(MapToError::PageAlreadyMapped); + } + p1[page.p1_index()].set_frame(frame, flags); + + Ok(MapperFlush::new(page)) + } + + fn unmap( + &mut self, + page: Page, + ) -> Result<(PhysFrame, MapperFlush), UnmapError> { + let p4 = &mut self.p4; + let p4_entry = &p4[page.p4_index()]; + p4_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + p3_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + p2_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + let p1_entry = &mut p1[page.p1_index()]; + + let frame = p1_entry.frame().map_err(|err| match err { + FrameError::FrameNotPresent => UnmapError::PageNotMapped, + FrameError::HugeFrame => UnmapError::ParentEntryHugePage, + })?; + + p1_entry.set_unused(); + Ok((frame, MapperFlush::new(page))) + } + + fn update_flags( + &mut self, + page: Page, + flags: PageTableFlags, + ) -> Result, FlagUpdateError> { + let p4 = &mut self.p4; + + if p4[page.p4_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) }; + + if p3[page.p3_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) }; + + if p2[page.p2_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) }; + + if p1[page.p1_index()].is_unused() { + return Err(FlagUpdateError::PageNotMapped); + } + + p1[page.p1_index()].set_flags(flags); + + Ok(MapperFlush::new(page)) + } + + fn translate_page(&self, page: Page) -> Option> { + let p4 = &self.p4; + + if p4[page.p4_index()].is_unused() { + return None; + } + + let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) }; + let p3_entry = &p3[page.p3_index()]; + + if p3_entry.is_unused() { + return None; + } + + let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) }; + let p2_entry = &p2[page.p2_index()]; + + if p2_entry.is_unused() { + return None; + } + + let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) }; + let p1_entry = &p1[page.p1_index()]; + + if p1_entry.is_unused() { + return None; + } + + PhysFrame::from_start_address(p1_entry.addr()).ok() + } +} + +fn p3_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p3_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p3_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + recursive_index, + recursive_index, + page.p4_index(), + ) +} + +fn p2_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p2_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p2_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + recursive_index, + page.p4_index(), + page.p3_index(), + ) +} + +fn p1_ptr(page: Page, recursive_index: u9) -> *mut PageTable { + p1_page(page, recursive_index).start_address().as_mut_ptr() +} + +fn p1_page(page: Page, recursive_index: u9) -> Page { + Page::from_page_table_indices( + recursive_index, + page.p4_index(), + page.p3_index(), + page.p2_index(), + ) +} diff --git a/crate/aarch64/src/regs/cntfrq_el0.rs b/crate/aarch64/src/regs/cntfrq_el0.rs new file mode 100644 index 00000000..df56ac9f --- /dev/null +++ b/crate/aarch64/src/regs/cntfrq_el0.rs @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Frequency register - EL0 +//! +//! This register is provided so that software can discover the frequency of the +//! system counter. It must be programmed with this value as part of system +//! initialization. The value of the register is not interpreted by hardware. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u32, "CNTFRQ_EL0"); +} + +pub static CNTFRQ_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cnthctl_el2.rs b/crate/aarch64/src/regs/cnthctl_el2.rs new file mode 100644 index 00000000..f5e3c2cd --- /dev/null +++ b/crate/aarch64/src/regs/cnthctl_el2.rs @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Hypervisor Control register - EL2 +//! +//! Controls the generation of an event stream from the physical counter, and +//! access from Non-secure EL1 to the physical counter and the Non-secure EL1 +//! physical timer. + +use register::cpu::RegisterReadWrite; + +// When HCR_EL2.E2H == 0: +// TODO: Figure out how we can differentiate depending on HCR_EL2.E2H state +// +// For now, implement the HCR_EL2.E2H == 0 version +register_bitfields! {u32, + CNTHCTL_EL2 [ + /// Traps Non-secure EL0 and EL1 accesses to the physical timer + /// registers to EL2. + /// + /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the + /// CNTP_CTL_EL0, CNTP_CVAL_EL0, and CNTP_TVAL_EL0 are trapped to EL2, + /// unless it is trapped by CNTKCTL_EL1.EL0PTEN. + /// + /// From AArch32 state: Non-secure EL0 and EL1 accesses to the + /// CNTP_CTL, CNTP_CVAL, and CNTP_TVAL are trapped to EL2, unless it + /// is trapped by CNTKCTL_EL1.EL0PTEN or CNTKCTL.PL0PTEN. + /// + /// 1 This control does not cause any instructions to be trapped. + /// + /// If EL3 is implemented and EL2 is not implemented, behavior is as if + /// this bit is 1 other than for the purpose of a direct read. + EL1PCEN OFFSET(1) NUMBITS(1) [], + + /// Traps Non-secure EL0 and EL1 accesses to the physical counter + /// register to EL2. + /// + /// 0 From AArch64 state: Non-secure EL0 and EL1 accesses to the + /// CNTPCT_EL0 are trapped to EL2, unless it is trapped by + /// CNTKCTL_EL1.EL0PCTEN. + /// + /// From AArch32 state: Non-secure EL0 and EL1 accesses to the CNTPCT + /// are trapped to EL2, unless it is trapped by CNTKCTL_EL1.EL0PCTEN + /// or CNTKCTL.PL0PCTEN. + /// + /// 1 This control does not cause any instructions to be trapped. + /// + /// If EL3 is implemented and EL2 is not implemented, behavior is as if + /// this bit is 1 other than for the purpose of a direct read. + EL1PCTEN OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTHCTL_EL2"); + sys_coproc_write_raw!(u32, "CNTHCTL_EL2"); +} + +#[allow(non_upper_case_globals)] +pub static CNTHCTL_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_ctl_el0.rs b/crate/aarch64/src/regs/cntp_ctl_el0.rs new file mode 100644 index 00000000..76991ebd --- /dev/null +++ b/crate/aarch64/src/regs/cntp_ctl_el0.rs @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Timer Control register - EL0 +//! +//! Control register for the EL1 physical timer. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + CNTP_CTL_EL0 [ + /// The status of the timer. This bit indicates whether the timer + /// condition is met: + /// + /// 0 Timer condition is not met. + /// 1 Timer condition is met. + /// + /// When the value of the ENABLE bit is 1, ISTATUS indicates whether the + /// timer condition is met. ISTATUS takes no account of the value of the + /// IMASK bit. If the value of ISTATUS is 1 and the value of IMASK is 0 + /// then the timer interrupt is asserted. + /// + /// When the value of the ENABLE bit is 0, the ISTATUS field is UNKNOWN. + /// + /// This bit is read-only. + ISTATUS OFFSET(2) NUMBITS(1) [], + + /// Timer interrupt mask bit. Permitted values are: + /// + /// 0 Timer interrupt is not masked by the IMASK bit. + /// 1 Timer interrupt is masked by the IMASK bit. + IMASK OFFSET(1) NUMBITS(1) [], + + /// Enables the timer. Permitted values are: + /// + /// 0 Timer disabled. + /// 1 Timer enabled. + ENABLE OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTP_CTL_EL0"); + sys_coproc_write_raw!(u32, "CNTP_CTL_EL0"); +} + +pub static CNTP_CTL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntp_tval_el0.rs b/crate/aarch64/src/regs/cntp_tval_el0.rs new file mode 100644 index 00000000..bdf5f6a7 --- /dev/null +++ b/crate/aarch64/src/regs/cntp_tval_el0.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Timer TimerValue register - EL0 +//! +//! Holds the timer value for the EL1 physical timer. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "CNTP_TVAL_EL0"); + sys_coproc_write_raw!(u32, "CNTP_TVAL_EL0"); +} + +pub static CNTP_TVAL_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntpct_el0.rs b/crate/aarch64/src/regs/cntpct_el0.rs new file mode 100644 index 00000000..b381d991 --- /dev/null +++ b/crate/aarch64/src/regs/cntpct_el0.rs @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Physical Count register - EL0 +//! +//! Holds the 64-bit physical count value. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "CNTPCT_EL0"); +} + +pub static CNTPCT_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/cntvoff_el2.rs b/crate/aarch64/src/regs/cntvoff_el2.rs new file mode 100644 index 00000000..aff70749 --- /dev/null +++ b/crate/aarch64/src/regs/cntvoff_el2.rs @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Counter-timer Virtual Offset register - EL2 +//! +//! Holds the 64-bit virtual offset. This is the offset between the physical +//! count value visible in CNTPCT_EL0 and the virtual count value visible in +//! CNTVCT_EL0. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "CNTVOFF_EL2"); + sys_coproc_write_raw!(u64, "CNTVOFF_EL2"); +} + +pub static CNTVOFF_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/currentel.rs b/crate/aarch64/src/regs/currentel.rs new file mode 100644 index 00000000..91b8e0a6 --- /dev/null +++ b/crate/aarch64/src/regs/currentel.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Current Exception Level +//! +//! Holds the current Exception level. + +use register::cpu::RegisterReadOnly; + +register_bitfields! {u32, + CurrentEL [ + /// Current Exception level. Possible values of this field are: + /// + /// 00 EL0 + /// 01 EL1 + /// 10 EL2 + /// 11 EL3 + /// + /// When the HCR_EL2.NV bit is 1, Non-secure EL1 read accesses to the + /// CurrentEL register return the value of 0x2 in this field. + /// + /// This field resets to a value that is architecturally UNKNOWN. + EL OFFSET(2) NUMBITS(2) [ + EL0 = 0, + EL1 = 1, + EL2 = 2, + EL3 = 3 + ] + ] +} + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u32, "CurrentEL"); +} + +#[allow(non_upper_case_globals)] +pub static CurrentEL: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/daif.rs b/crate/aarch64/src/regs/daif.rs new file mode 100644 index 00000000..bf810a2b --- /dev/null +++ b/crate/aarch64/src/regs/daif.rs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Interrupt Mask Bits +//! +//! Allows access to the interrupt mask bits. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + DAIF [ + /// Process state D mask. The possible values of this bit are: + /// + /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are not masked. + /// + /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are masked. + /// + /// When the target Exception level of the debug exception is higher + /// than the current Exception level, the exception is not masked by + /// this bit. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1. + D OFFSET(9) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// SError interrupt mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + A OFFSET(8) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// IRQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + I OFFSET(7) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// FIQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1 . + F OFFSET(6) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ] + ] +} + + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "DAIF"); + sys_coproc_write_raw!(u32, "DAIF"); +} + +pub static DAIF: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/elr_el2.rs b/crate/aarch64/src/regs/elr_el2.rs new file mode 100644 index 00000000..0786fbb1 --- /dev/null +++ b/crate/aarch64/src/regs/elr_el2.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Exception Link Register - EL2 +//! +//! When taking an exception to EL2, holds the address to return to. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "ELR_EL2"); + sys_coproc_write_raw!(u64, "ELR_EL2"); +} + +pub static ELR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/hcr_el2.rs b/crate/aarch64/src/regs/hcr_el2.rs new file mode 100644 index 00000000..683bbef6 --- /dev/null +++ b/crate/aarch64/src/regs/hcr_el2.rs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Hypervisor Configuration Register - EL2 +//! +//! Provides configuration controls for virtualization, including defining +//! whether various Non-secure operations are trapped to EL2. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + HCR_EL2 [ + /// Execution state control for lower Exception levels: + /// + /// 0 Lower levels are all AArch32. + /// 1 The Execution state for EL1 is AArch64. The Execution state for + /// EL0 is determined by the current value of PSTATE.nRW when + /// executing at EL0. + /// + /// If all lower Exception levels cannot use AArch32 then this bit is + /// RAO/WI. + /// + /// In an implementation that includes EL3, when SCR_EL3.NS==0, the PE + /// behaves as if this bit has the same value as the SCR_EL3.RW bit for + /// all purposes other than a direct read or write access of HCR_EL2. + /// + /// The RW bit is permitted to be cached in a TLB. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this field behaves as 1 for all purposes other than a + /// direct read of the value of this bit. + RW OFFSET(31) NUMBITS(1) [ + AllLowerELsAreAarch32 = 0, + EL1IsAarch64 = 1 + ], + + /// Default Cacheability. + /// + /// 0 This control has no effect on the Non-secure EL1&0 translation + /// regime. + /// + /// 1 In Non-secure state: + /// - When EL1 is using AArch64, the PE behaves as if the value of + /// the SCTLR_EL1.M field is 0 for all purposes other than + /// returning the value of a direct read of SCTLR_EL1. + /// + /// - When EL1 is using AArch32, the PE behaves as if the value of + /// the SCTLR.M field is 0 for all purposes other than returning + /// the value of a direct read of SCTLR. + /// + /// - The PE behaves as if the value of the HCR_EL2.VM field is 1 + /// for all purposes other than returning the value of a direct + /// read of HCR_EL2. + /// + /// - The memory type produced by stage 1 of the EL1&0 translation + /// regime is Normal Non-Shareable, Inner Write-Back Read-Allocate + /// Write-Allocate, Outer Write-Back Read-Allocate Write-Allocate. + /// + /// This field has no effect on the EL2, EL2&0, and EL3 translation + /// regimes. + /// + /// This field is permitted to be cached in a TLB. + /// + /// In an implementation that includes EL3, when the value of SCR_EL3.NS + /// is 0 the PE behaves as if this field is 0 for all purposes other + /// than a direct read or write access of HCR_EL2. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this field behaves as 0 for all purposes other than a + /// direct read of the value of this field. + DC OFFSET(12) NUMBITS(1) [], + + /// Set/Way Invalidation Override. Causes Non-secure EL1 execution of + /// the data cache invalidate by set/way instructions to perform a data + /// cache clean and invalidate by set/way: + /// + /// 0 This control has no effect on the operation of data cache + /// invalidate by set/way instructions. + /// + /// 1 Data cache invalidate by set/way instructions perform a data cache + /// clean and invalidate by set/way. + /// + /// When the value of this bit is 1: + /// + /// AArch32: DCISW performs the same invalidation as a DCCISW + /// instruction. + /// + /// AArch64: DC ISW performs the same invalidation as a DC CISW + /// instruction. + /// + /// This bit can be implemented as RES 1. + /// + /// In an implementation that includes EL3, when the value of SCR_EL3.NS + /// is 0 the PE behaves as if this field is 0 for all purposes other + /// than a direct read or write access of HCR_EL2. + /// + /// When HCR_EL2.TGE is 1, the PE ignores the value of this field for + /// all purposes other than a direct read of this field. + SWIO OFFSET(1) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "HCR_EL2"); + sys_coproc_write_raw!(u64, "HCR_EL2"); +} + +pub static HCR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs b/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs new file mode 100644 index 00000000..f75813ca --- /dev/null +++ b/crate/aarch64/src/regs/id_aa64mmfr0_el1.rs @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! AArch64 Memory Model Feature Register 0 - EL1 +//! +//! Provides information about the implemented memory model and memory +//! management support in AArch64 state. + +use register::cpu::RegisterReadOnly; + +register_bitfields! {u64, + ID_AA64MMFR0_EL1 [ + /// Support for 4KiB memory translation granule size. Defined values + /// are: + /// + /// 0000 4KiB granule supported. + /// 1111 4KiB granule not supported. + /// + /// All other values are reserved. + TGran4 OFFSET(28) NUMBITS(4) [ + Supported = 0b0000, + NotSupported = 0b1111 + ], + + /// Support for 64KiB memory translation granule size. Defined values + /// are: + /// + /// 0000 64KiB granule supported. + /// 1111 64KiB granule not supported. + /// + /// All other values are reserved. + TGran64 OFFSET(24) NUMBITS(4) [ + Supported = 0b0000, + NotSupported = 0b1111 + ], + + /// Physical Address range supported. Defined values are: + /// + /// 0000 32 bits, 4GiB. + /// 0001 36 bits, 64GiB. + /// 0010 40 bits, 1TiB. + /// 0011 42 bits, 4TiB. + /// 0100 44 bits, 16TiB. + /// 0101 48 bits, 256TiB. + /// 0110 52 bits, 4PiB. + /// + /// All other values are reserved. + /// + /// The value 0110 is permitted only if the implementation includes + /// ARMv8.2-LPA, otherwise it is reserved. + PARange OFFSET(0) NUMBITS(4) [ + Bits_32 = 0b0000, + Bits_36 = 0b0001, + Bits_40 = 0b0010, + Bits_42 = 0b0011, + Bits_44 = 0b0100, + Bits_48 = 0b0101, + Bits_52 = 0b0110 + ] + ] +} + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "ID_AA64MMFR0_EL1"); +} + +pub static ID_AA64MMFR0_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/macros.rs b/crate/aarch64/src/regs/macros.rs new file mode 100644 index 00000000..bd4439c5 --- /dev/null +++ b/crate/aarch64/src/regs/macros.rs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +macro_rules! __read_raw { + ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { + /// Reads the raw bits of the CPU register. + #[inline] + fn get(&self) -> $width { + match () { + #[cfg(target_arch = "aarch64")] + () => { + let reg; + unsafe { + asm!(concat!($asm_instr, " $0, ", $asm_reg_name) : "=r"(reg) ::: "volatile"); + } + reg + } + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } + } + }; +} + +macro_rules! __write_raw { + ($width:ty, $asm_instr:tt, $asm_reg_name:tt) => { + /// Writes raw bits to the CPU register. + #[cfg_attr(not(target_arch = "aarch64"), allow(unused_variables))] + #[inline] + fn set(&self, value: $width) { + match () { + #[cfg(target_arch = "aarch64")] + () => { + unsafe { + asm!(concat!($asm_instr, " ", $asm_reg_name, ", $0") :: "r"(value) :: "volatile") + } + } + + #[cfg(not(target_arch = "aarch64"))] + () => unimplemented!(), + } + } + }; +} + +/// Raw read from system coprocessor registers +macro_rules! sys_coproc_read_raw { + ($width:ty, $asm_reg_name:tt) => { + __read_raw!($width, "mrs", $asm_reg_name); + }; +} + +/// Raw write to system coprocessor registers +macro_rules! sys_coproc_write_raw { + ($width:ty, $asm_reg_name:tt) => { + __write_raw!($width, "msr", $asm_reg_name); + }; +} + +/// Raw read from (ordinary) registers +macro_rules! read_raw { + ($width:ty, $asm_reg_name:tt) => { + __read_raw!($width, "mov", $asm_reg_name); + }; +} +/// Raw write to (ordinary) registers +macro_rules! write_raw { + ($width:ty, $asm_reg_name:tt) => { + __write_raw!($width, "mov", $asm_reg_name); + }; +} diff --git a/crate/aarch64/src/regs/mair_el1.rs b/crate/aarch64/src/regs/mair_el1.rs new file mode 100644 index 00000000..2c7c7da4 --- /dev/null +++ b/crate/aarch64/src/regs/mair_el1.rs @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Memory Attribute Indirection Register - EL1 +//! +//! Provides the memory attribute encodings corresponding to the possible +//! AttrIndx values in a Long-descriptor format translation table entry for +//! stage 1 translations at EL1. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + MAIR_EL1 [ + // TODO: Macrofy this + + /// Attribute 7 + Attr7_HIGH OFFSET(60) NUMBITS(4) [], + Attr7_LOW_DEVICE OFFSET(56) NUMBITS(4) [], + Attr7_LOW_MEMORY OFFSET(56) NUMBITS(4) [], + + /// Attribute 6 + Attr6_HIGH OFFSET(52) NUMBITS(4) [], + Attr6_LOW_DEVICE OFFSET(48) NUMBITS(4) [], + Attr6_LOW_MEMORY OFFSET(48) NUMBITS(4) [], + + /// Attribute 5 + Attr5_HIGH OFFSET(44) NUMBITS(4) [], + Attr5_LOW_DEVICE OFFSET(40) NUMBITS(4) [], + Attr5_LOW_MEMORY OFFSET(40) NUMBITS(4) [], + + /// Attribute 4 + Attr4_HIGH OFFSET(36) NUMBITS(4) [], + Attr4_LOW_DEVICE OFFSET(32) NUMBITS(4) [], + Attr4_LOW_MEMORY OFFSET(32) NUMBITS(4) [], + + /// Attribute 3 + Attr3_HIGH OFFSET(28) NUMBITS(4) [], + Attr3_LOW_DEVICE OFFSET(24) NUMBITS(4) [], + Attr3_LOW_MEMORY OFFSET(24) NUMBITS(4) [], + + /// Attribute 2 + Attr2_HIGH OFFSET(20) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr2_LOW_DEVICE OFFSET(16) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr2_LOW_MEMORY OFFSET(16) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + + /// Attribute 1 + Attr1_HIGH OFFSET(12) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr1_LOW_DEVICE OFFSET(8) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr1_LOW_MEMORY OFFSET(8) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + + /// Attribute 0 + Attr0_HIGH OFFSET(4) NUMBITS(4) [ + Device = 0b0000, + Memory_OuterNonCacheable = 0b0100, + Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ], + Attr0_LOW_DEVICE OFFSET(0) NUMBITS(4) [ + Device_nGnRE = 0b0100 + ], + Attr0_LOW_MEMORY OFFSET(0) NUMBITS(4) [ + InnerNonCacheable = 0b0100, + InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc = 0b1111 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "MAIR_EL1"); + sys_coproc_write_raw!(u64, "MAIR_EL1"); +} + +pub static MAIR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/mod.rs b/crate/aarch64/src/regs/mod.rs new file mode 100644 index 00000000..f6a0e3d8 --- /dev/null +++ b/crate/aarch64/src/regs/mod.rs @@ -0,0 +1,51 @@ +//! Processor core registers + +#[macro_use] +mod macros; + +mod cntfrq_el0; +mod cnthctl_el2; +mod cntp_ctl_el0; +mod cntp_tval_el0; +mod cntpct_el0; +mod cntvoff_el2; +mod currentel; +mod daif; +mod elr_el2; +mod hcr_el2; +mod id_aa64mmfr0_el1; +mod mair_el1; +mod mpidr_el1; +mod sctlr_el1; +mod sp; +mod sp_el0; +mod sp_el1; +mod spsel; +mod spsr_el2; +mod tcr_el1; +mod ttbr0_el1; + +// Export only the R/W traits and the static reg definitions +pub use register::cpu::*; + +pub use self::cntfrq_el0::CNTFRQ_EL0; +pub use self::cnthctl_el2::CNTHCTL_EL2; +pub use self::cntp_ctl_el0::CNTP_CTL_EL0; +pub use self::cntp_tval_el0::CNTP_TVAL_EL0; +pub use self::cntpct_el0::CNTPCT_EL0; +pub use self::cntvoff_el2::CNTVOFF_EL2; +pub use self::currentel::CurrentEL; +pub use self::daif::DAIF; +pub use self::elr_el2::ELR_EL2; +pub use self::hcr_el2::HCR_EL2; +pub use self::id_aa64mmfr0_el1::ID_AA64MMFR0_EL1; +pub use self::mair_el1::MAIR_EL1; +pub use self::mpidr_el1::MPIDR_EL1; +pub use self::sctlr_el1::SCTLR_EL1; +pub use self::sp::SP; +pub use self::sp_el0::SP_EL0; +pub use self::sp_el1::SP_EL1; +pub use self::spsel::SPSel; +pub use self::spsr_el2::SPSR_EL2; +pub use self::tcr_el1::TCR_EL1; +pub use self::ttbr0_el1::TTBR0_EL1; diff --git a/crate/aarch64/src/regs/mpidr_el1.rs b/crate/aarch64/src/regs/mpidr_el1.rs new file mode 100644 index 00000000..6fbfea00 --- /dev/null +++ b/crate/aarch64/src/regs/mpidr_el1.rs @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Multiprocessor Affinity Register - EL1 +//! +//! In a multiprocessor system, provides an additional PE +//! identification mechanism for scheduling purposes. + +use register::cpu::RegisterReadOnly; + +pub struct Reg; + +impl RegisterReadOnly for Reg { + sys_coproc_read_raw!(u64, "MPIDR_EL1"); +} + +pub static MPIDR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sctlr_el1.rs b/crate/aarch64/src/regs/sctlr_el1.rs new file mode 100644 index 00000000..1f463b42 --- /dev/null +++ b/crate/aarch64/src/regs/sctlr_el1.rs @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! System Control Register - EL1 +//! +//! Provides top level control of the system, including its memory system, at +//! EL1 and EL0. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SCTLR_EL1 [ + /// Instruction access Cacheability control, for accesses at EL0 and + /// EL1: + /// + /// 0 All instruction access to Normal memory from EL0 and EL1 are + /// Non-cacheable for all levels of instruction and unified cache. + /// + /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage + /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, + /// Inner Non-cacheable, Outer Non-cacheable memory. + /// + /// 1 This control has no effect on the Cacheability of instruction + /// access to Normal memory from EL0 and EL1. + /// + /// If the value of SCTLR_EL1.M is 0, instruction accesses from stage + /// 1 of the EL1&0 translation regime are to Normal, Outer Shareable, + /// Inner Write-Through, Outer Write-Through memory. + /// + /// When the value of the HCR_EL2.DC bit is 1, then instruction access + /// to Normal memory from EL0 and EL1 are Cacheable regardless of the + /// value of the SCTLR_EL1.I bit. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this bit has no effect on the PE. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 0. + I OFFSET(12) NUMBITS(1) [ + NonCacheable = 0, + Cacheable = 1 + ], + + /// Cacheability control, for data accesses. + /// + /// 0 All data access to Normal memory from EL0 and EL1, and all Normal + /// memory accesses to the EL1&0 stage 1 translation tables, are + /// Non-cacheable for all levels of data and unified cache. + /// + /// 1 This control has no effect on the Cacheability of: + /// - Data access to Normal memory from EL0 and EL1. + /// - Normal memory accesses to the EL1&0 stage 1 translation + /// tables. + /// + /// When the value of the HCR_EL2.DC bit is 1, the PE ignores + /// SCLTR.C. This means that Non-secure EL0 and Non-secure EL1 data + /// accesses to Normal memory are Cacheable. + /// + /// When ARMv8.1-VHE is implemented, and the value of HCR_EL2.{E2H, TGE} + /// is {1, 1}, this bit has no effect on the PE. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 0. + C OFFSET(2) NUMBITS(1) [ + NonCacheable = 0, + Cacheable = 1 + ], + + /// MMU enable for EL1 and EL0 stage 1 address translation. Possible + /// values of this bit are: + /// + /// 0 EL1 and EL0 stage 1 address translation disabled. + /// See the SCTLR_EL1.I field for the behavior of instruction accesses + /// to Normal memory. + /// 1 EL1 and EL0 stage 1 address translation enabled. + M OFFSET(0) NUMBITS(1) [ + Disable = 0, + Enable = 1 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SCTLR_EL1"); + sys_coproc_write_raw!(u32, "SCTLR_EL1"); +} + +pub static SCTLR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp.rs b/crate/aarch64/src/regs/sp.rs new file mode 100644 index 00000000..f9f578be --- /dev/null +++ b/crate/aarch64/src/regs/sp.rs @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + read_raw!(u64, "sp"); + write_raw!(u64, "sp"); +} + +pub static SP: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el0.rs b/crate/aarch64/src/regs/sp_el0.rs new file mode 100644 index 00000000..aa82fdbc --- /dev/null +++ b/crate/aarch64/src/regs/sp_el0.rs @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer - EL0 +//! +//! Holds the stack pointer associated with EL0. At higher Exception levels, +//! this is used as the current stack pointer when the value of SPSel.SP is 0. + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "SP_EL0"); + sys_coproc_write_raw!(u64, "SP_EL0"); +} + +pub static SP_EL0: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/sp_el1.rs b/crate/aarch64/src/regs/sp_el1.rs new file mode 100644 index 00000000..4357412b --- /dev/null +++ b/crate/aarch64/src/regs/sp_el1.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! The stack pointer - EL1 +//! +//! Holds the stack pointer associated with EL1. When executing at EL1, the +//! value of SPSel.SP determines the current stack pointer: +//! +//! SPSel.SP | current stack pointer +//! -------------------------------- +//! 0 | SP_EL0 +//! 1 | SP_EL1 + +use register::cpu::RegisterReadWrite; + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "SP_EL1"); + sys_coproc_write_raw!(u64, "SP_EL1"); +} + +pub static SP_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsel.rs b/crate/aarch64/src/regs/spsel.rs new file mode 100644 index 00000000..91e3694f --- /dev/null +++ b/crate/aarch64/src/regs/spsel.rs @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Stack Pointer Select +//! +//! Allows the Stack Pointer to be selected between SP_EL0 and SP_ELx. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SPSel [ + /// Stack pointer to use. Possible values of this bit are: + /// + /// 0 Use SP_EL0 at all Exception levels. + /// 1 Use SP_ELx for Exception level ELx. + /// + /// When this register has an architecturally-defined reset value, this + /// field resets to 1. + SP OFFSET(0) NUMBITS(1) [ + EL0 = 0, + ELx = 1 + ] + ] +} + + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SPSEL"); + sys_coproc_write_raw!(u32, "SPSEL"); +} + +#[allow(non_upper_case_globals)] +pub static SPSel: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/spsr_el2.rs b/crate/aarch64/src/regs/spsr_el2.rs new file mode 100644 index 00000000..56078a45 --- /dev/null +++ b/crate/aarch64/src/regs/spsr_el2.rs @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Saved Program Status Register - EL2 +//! +//! Holds the saved process state when an exception is taken to EL2. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u32, + SPSR_EL2 [ + /// Process state D mask. The possible values of this bit are: + /// + /// 0 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are not masked. + /// + /// 1 Watchpoint, Breakpoint, and Software Step exceptions targeted at + /// the current Exception level are masked. + /// + /// When the target Exception level of the debug exception is higher + /// than the current Exception level, the exception is not masked by + /// this bit. + D OFFSET(9) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// SError interrupt mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + A OFFSET(8) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// IRQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + I OFFSET(7) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// FIQ mask bit. The possible values of this bit are: + /// + /// 0 Exception not masked. + /// 1 Exception masked. + F OFFSET(6) NUMBITS(1) [ + Unmasked = 0, + Masked = 1 + ], + + /// AArch64 state (Exception level and selected SP) that an exception + /// was taken from. The possible values are: + /// + /// M[3:0] | State + /// -------------- + /// 0b0000 | EL0t + /// 0b0100 | EL1t + /// 0b0101 | EL1h + /// 0b1000 | EL2t + /// 0b1001 | EL2h + /// + /// Other values are reserved, and returning to an Exception level that + /// is using AArch64 with a reserved value in this field is treated as + /// an illegal exception return. + /// + /// The bits in this field are interpreted as follows: + /// - M[3:2] holds the Exception Level. + /// - M[1] is unused and is RES 0 for all non-reserved values. + /// - M[0] is used to select the SP: + /// - 0 means the SP is always SP0. + /// - 1 means the exception SP is determined by the EL. + M OFFSET(0) NUMBITS(4) [ + EL0t = 0b0000, + EL1t = 0b0100, + EL1h = 0b0101, + EL2t = 0b1000, + EL2h = 0b1001 + ] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u32, "SPSR_EL2"); + sys_coproc_write_raw!(u32, "SPSR_EL2"); +} + +pub static SPSR_EL2: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/tcr_el1.rs b/crate/aarch64/src/regs/tcr_el1.rs new file mode 100644 index 00000000..bcd04258 --- /dev/null +++ b/crate/aarch64/src/regs/tcr_el1.rs @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Translation Control Register - EL1 +//! +//! The control register for stage 1 of the EL1&0 translation regime. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + TCR_EL1 [ + /// Top Byte ignored - indicates whether the top byte of an address is + /// used for address match for the TTBR0_EL1 region, or ignored and used + /// for tagged addresses. Defined values are: + /// + /// 0 Top Byte used in the address calculation. + /// 1 Top Byte ignored in the address calculation. + TBI0 OFFSET(37) NUMBITS(1) [ + Used = 0, + Ignored = 1 + ], + + /// Intermediate Physical Address Size. + /// + /// 000 32 bits, 4GiB. + /// 001 36 bits, 64GiB. + /// 010 40 bits, 1TiB. + /// 011 42 bits, 4TiB. + /// 100 44 bits, 16TiB. + /// 101 48 bits, 256TiB. + /// 110 52 bits, 4PiB + /// + /// Other values are reserved. + /// + /// The reserved values behave in the same way as the 101 or 110 + /// encoding, but software must not rely on this property as the + /// behavior of the reserved values might change in a future revision of + /// the architecture. + /// + /// The value 110 is permitted only if ARMv8.2-LPA is implemented and + /// the translation granule size is 64KiB. + /// + /// In an implementation that supports 52-bit PAs, if the value of this + /// field is not 110 , then bits[51:48] of every translation table base + /// address for the stage of translation controlled by TCR_EL1 are 0000 + /// . + IPS OFFSET(32) NUMBITS(3) [ + Bits_32 = 0b000, + Bits_36 = 0b001, + Bits_40 = 0b010, + Bits_42 = 0b011, + Bits_44 = 0b100, + Bits_48 = 0b101, + Bits_52 = 0b110 + ], + + /// Granule size for the TTBR0_EL1. + /// + /// 00 4KiB + /// 01 64KiB + /// 10 16KiB + /// + /// Other values are reserved. + /// + /// If the value is programmed to either a reserved value, or a size + /// that has not been implemented, then the hardware will treat the + /// field as if it has been programmed to an IMPLEMENTATION DEFINED + /// choice of the sizes that has been implemented for all purposes other + /// than the value read back from this register. + /// + /// It is IMPLEMENTATION DEFINED whether the value read back is the + /// value programmed or the value that corresponds to the size chosen. + TG0 OFFSET(14) NUMBITS(2) [ + KiB_4 = 0b00, + KiB_16 = 0b10, + KiB_64 = 0b01 + ], + + /// Shareability attribute for memory associated with translation table + /// walks using TTBR0_EL1. + /// + /// 00 Non-shareable + /// 10 Outer Shareable + /// 11 Inner Shareable + /// + /// Other values are reserved. + SH0 OFFSET(12) NUMBITS(2) [ + None = 0b00, + Outer = 0b10, + Inner = 0b11 + ], + + /// Outer cacheability attribute for memory associated with translation + /// table walks using TTBR0_EL1. + /// + /// 00 Normal memory, Outer Non-cacheable + /// + /// 01 Normal memory, Outer Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Outer Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Outer Write-Back Read-Allocate No Write-Allocate + /// Cacheable + ORGN0 OFFSET(10) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Inner cacheability attribute for memory associated with translation + /// table walks using TTBR0_EL1. + /// + /// 00 Normal memory, Inner Non-cacheable + /// + /// 01 Normal memory, Inner Write-Back Read-Allocate Write-Allocate + /// Cacheable + /// + /// 10 Normal memory, Inner Write-Through Read-Allocate No + /// Write-Allocate Cacheable + /// + /// 11 Normal memory, Inner Write-Back Read-Allocate No Write-Allocate + /// Cacheable + IRGN0 OFFSET(8) NUMBITS(2) [ + NonCacheable = 0b00, + WriteBack_ReadAlloc_WriteAlloc_Cacheable = 0b01, + WriteThrough_ReadAlloc_NoWriteAlloc_Cacheable = 0b10, + WriteBack_ReadAlloc_NoWriteAlloc_Cacheable = 0b11 + ], + + /// Translation table walk disable for translations using + /// TTBR0_EL1. This bit controls whether a translation table walk is + /// performed on a TLB miss, for an address that is translated using + /// TTBR0_EL1. The encoding of this bit is: + /// + /// 0 Perform translation table walks using TTBR0_EL1. + /// + /// 1 A TLB miss on an address that is translated using TTBR0_EL1 + /// generates a Translation fault. No translation table walk is + /// performed. + EPD0 OFFSET(7) NUMBITS(1) [ + EnableTTBR0Walks = 0, + DisableTTBR0Walks = 1 + ], + + /// The size offset of the memory region addressed by TTBR0_EL1. The + /// region size is 2^(64-T0SZ) bytes. + /// + /// The maximum and minimum possible values for T0SZ depend on the level + /// of translation table and the memory translation granule size, as + /// described in the AArch64 Virtual Memory System Architecture chapter. + T0SZ OFFSET(0) NUMBITS(6) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "TCR_EL1"); + sys_coproc_write_raw!(u64, "TCR_EL1"); +} + +pub static TCR_EL1: Reg = Reg {}; diff --git a/crate/aarch64/src/regs/ttbr0_el1.rs b/crate/aarch64/src/regs/ttbr0_el1.rs new file mode 100644 index 00000000..c1112565 --- /dev/null +++ b/crate/aarch64/src/regs/ttbr0_el1.rs @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 by the author(s) + * + * ============================================================================= + * + * Licensed under either of + * - Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + * - MIT License (http://opensource.org/licenses/MIT) + * at your option. + * + * ============================================================================= + * + * Author(s): + * - Andre Richter + */ + +//! Translation Table Base Register 0 - EL1 +//! +//! Holds the base address of the translation table for the initial lookup for +//! stage 1 of the translation of an address from the lower VA range in the +//! EL1&0 translation regime, and other information for this translation regime. + +use register::cpu::RegisterReadWrite; + +register_bitfields! {u64, + TTBR0_EL1 [ + /// An ASID for the translation table base address. The TCR_EL1.A1 field + /// selects either TTBR0_EL1.ASID or TTBR1_EL1.ASID. + /// + /// If the implementation has only 8 bits of ASID, then the upper 8 bits + /// of this field are RES 0. + ASID OFFSET(48) NUMBITS(16) [], + + /// Translation table base address + BADDR OFFSET(1) NUMBITS(47) [], + + /// Common not Private + CnP OFFSET(0) NUMBITS(1) [] + ] +} + +pub struct Reg; + +impl RegisterReadWrite for Reg { + sys_coproc_read_raw!(u64, "TTBR0_EL1"); + sys_coproc_write_raw!(u64, "TTBR0_EL1"); +} + +impl Reg { + #[inline] + pub fn set_baddr(&self, addr: u64) { + self.write(TTBR0_EL1::BADDR.val(addr >> 1)); + } +} + +pub static TTBR0_EL1: Reg = Reg {}; diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index e1b7d050..05f07102 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -1,3 +1,16 @@ +[[package]] +name = "aarch64" +version = "0.1.0" +dependencies = [ + "bare-metal 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "os_bootinfo 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "usize_conversions 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "atags" version = "0.1.0" @@ -243,6 +256,7 @@ dependencies = [ name = "ucore" version = "0.1.0" dependencies = [ + "aarch64 0.1.0", "atags 0.1.0", "bbl 0.1.0", "bcm2837 0.1.0", diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index e2bba971..c80b6ef7 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -45,6 +45,7 @@ bbl = { path = "../crate/bbl" } [target.'cfg(target_arch = "aarch64")'.dependencies] cortex-a = "2.2.1" +aarch64 = { path = "../crate/aarch64" } atags = { path = "../crate/atags" } bcm2837 = { path = "../crate/bcm2837", features = ["use_generic_timer"] } diff --git a/kernel/src/arch/aarch64/memory.rs b/kernel/src/arch/aarch64/memory.rs index bf553546..32990d2f 100644 --- a/kernel/src/arch/aarch64/memory.rs +++ b/kernel/src/arch/aarch64/memory.rs @@ -1,22 +1,78 @@ //! Memory initialization for aarch64. +use bit_allocator::BitAlloc; use ucore_memory::PAGE_SIZE; +use memory::{FRAME_ALLOCATOR, init_heap}; use super::atags::atags::Atags; -use super::super::HEAP_ALLOCATOR; +//use super::super::HEAP_ALLOCATOR; +use aarch64::{barrier, regs::*}; +use core::ops::Range; /// Memory initialization. pub fn init() { - let (start, end) = memory_map().expect("failed to find memory map"); + /*let (start, end) = memory_map().expect("failed to find memory map"); unsafe { HEAP_ALLOCATOR.lock().init(start, end - start); - } - info!("memory: init end"); + }*/ + + init_frame_allocator(); + init_heap(); + init_mmu(); } extern "C" { static _end: u8; } + +fn init_frame_allocator() { + let mut ba = FRAME_ALLOCATOR.lock(); + let (start, end) = memory_map().expect("failed to find memory map"); + ba.insert(to_range(start, end)); + info!("FrameAllocator init end"); + + fn to_range(start: usize, end: usize) -> Range { + let page_start = start / PAGE_SIZE; + let page_end = (end - 1) / PAGE_SIZE + 1; + page_start..page_end + } +} + +fn init_mmu() { + // device. + MAIR_EL1.write( + // Attribute 1 + MAIR_EL1::Attr1_HIGH::Device + + MAIR_EL1::Attr1_LOW_DEVICE::Device_nGnRE + // Attribute 0 + + MAIR_EL1::Attr0_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr0_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc, + ); + // Configure various settings of stage 1 of the EL1 translation regime. + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_4 // 4 KiB granule + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(34), // Start walks at level 2 + ); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + unsafe { barrier::isb(barrier::SY); } + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction + unsafe { barrier::isb(barrier::SY); } +} + /// Returns the (start address, end address) of the available memory on this /// system if it can be determined. If it cannot, `None` is returned. /// @@ -33,3 +89,4 @@ pub fn memory_map() -> Option<(usize, usize)> { None } + diff --git a/kernel/src/arch/aarch64/paging.rs b/kernel/src/arch/aarch64/paging.rs index e9cc43e8..b468f8ba 100644 --- a/kernel/src/arch/aarch64/paging.rs +++ b/kernel/src/arch/aarch64/paging.rs @@ -1,218 +1,233 @@ //! Page table implementations for aarch64. - -use ucore_memory::memory_set::*; -use ucore_memory::paging::*; - -type VirtAddr = usize; -type PhysAddr = usize; - -use alloc::alloc::{alloc, Layout}; +use bit_allocator::{BitAlloc}; +// Depends on kernel use memory::{active_table, alloc_frame, alloc_stack, dealloc_frame}; +use ucore_memory::memory_set::*; +use ucore_memory::PAGE_SIZE; +use ucore_memory::paging::*; +use aarch64::asm::{tlb_invalidate, ttbr0_el1_read, ttbr0_el1_write}; +use aarch64::{PhysAddr, VirtAddr}; +use aarch64::paging::{Mapper, PageTable as Aarch64PageTable, PageTableEntry, PageTableFlags as EF, RecursivePageTable}; +use aarch64::paging::{FrameAllocator, FrameDeallocator, Page, PageRange, PhysFrame as Frame, Size4KiB}; +use aarch64::{regs::*}; -/// TODO -pub struct ActivePageTable { - // TODO +pub trait PageExt { + fn of_addr(address: usize) -> Self; + fn range_of(begin: usize, end: usize) -> PageRange; } -impl ActivePageTable { - /// TODO - pub unsafe fn new() -> Self { - unimplemented!() +impl PageExt for Page { + fn of_addr(address: usize) -> Self { + Page::containing_address(VirtAddr::new(address as u64)) + } + fn range_of(begin: usize, end: usize) -> PageRange { + Page::range(Page::of_addr(begin), Page::of_addr(end - 1) + 1) } } +pub trait FrameExt { + fn of_addr(address: usize) -> Self; +} + +impl FrameExt for Frame { + fn of_addr(address: usize) -> Self { + Frame::containing_address(PhysAddr::new(address as u64)) + } +} + +pub struct ActivePageTable(RecursivePageTable<'static>); + +pub struct PageEntry(PageTableEntry); + impl PageTable for ActivePageTable { type Entry = PageEntry; - fn map(&mut self, addr: VirtAddr, target: PhysAddr) -> &mut Self::Entry { - unimplemented!() - } - fn unmap(&mut self, addr: VirtAddr) { - unimplemented!() + fn map(&mut self, addr: usize, target: usize) -> &mut PageEntry { + let flags = EF::PRESENT | EF::WRITE | EF::UXN; + self.0.map_to(Page::of_addr(addr), Frame::of_addr(target), flags, &mut FrameAllocatorForAarch64) + .unwrap().flush(); + self.get_entry(addr) } - fn get_entry(&mut self, addr: VirtAddr) -> &mut Self::Entry { - unimplemented!() + fn unmap(&mut self, addr: usize) { + let (frame, flush) = self.0.unmap(Page::of_addr(addr)).unwrap(); + flush.flush(); } - // For testing with mock - fn get_page_slice_mut<'a, 'b>(&'a mut self, addr: VirtAddr) -> &'b mut [u8] { - unimplemented!() + fn get_entry(&mut self, addr: usize) -> &mut PageEntry { + let entry_addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + unsafe { &mut *(entry_addr as *mut PageEntry) } } - fn read(&mut self, addr: VirtAddr) -> u8 { - unimplemented!() + fn get_page_slice_mut<'a, 'b>(&'a mut self, addr: usize) -> &'b mut [u8] { + use core::slice; + unsafe { slice::from_raw_parts_mut((addr & !0xfffusize) as *mut u8, PAGE_SIZE) } } - fn write(&mut self, addr: VirtAddr, data: u8) { - unimplemented!() + fn read(&mut self, addr: usize) -> u8 { + unsafe { *(addr as *const u8) } + } + + fn write(&mut self, addr: usize, data: u8) { + unsafe { *(addr as *mut u8) = data; } } } -/// TODO -pub struct PageEntry { - // TODO +impl ActivePageTable { + pub unsafe fn new() -> Self { + ActivePageTable(RecursivePageTable::new(&mut *(0xffffffff_fffff000 as *mut _)).unwrap()) + } + fn with_temporary_map(&mut self, frame: &Frame, f: impl FnOnce(&mut ActivePageTable, &mut Aarch64PageTable)) { + // Create a temporary page + let page = Page::of_addr(0xcafebabe); + assert!(self.0.translate_page(page).is_none(), "temporary page is already mapped"); + // Map it to table + self.map(page.start_address().as_u64() as usize, frame.start_address().as_u64() as usize); + // Call f + let table = unsafe { &mut *page.start_address().as_mut_ptr() }; + f(self, table); + // Unmap the page + self.unmap(0xcafebabe); + } } impl Entry for PageEntry { - /// IMPORTANT! - /// This must be called after any change to ensure it become effective. - /// Usually this will make a flush to TLB/MMU. fn update(&mut self) { - unimplemented!() + tlb_invalidate(); } - /// Will be set when accessed - fn accessed(&self) -> bool { - unimplemented!() - } + fn present(&self) -> bool { self.0.flags().contains(EF::PRESENT) } + fn accessed(&self) -> bool { self.0.flags().contains(EF::ACCESSED) } + fn writable(&self) -> bool { self.0.flags().contains(EF::WRITE) } + fn dirty(&self) -> bool { self.hw_dirty() && self.sw_dirty() } - /// Will be set when written - fn dirty(&self) -> bool { - unimplemented!() + fn clear_accessed(&mut self) { self.as_flags().remove(EF::ACCESSED); } + fn clear_dirty(&mut self) + { + self.as_flags().remove(EF::DIRTY); + self.as_flags().insert(EF::RDONLY); } - - /// Will PageFault when try to write page where writable=0 - fn writable(&self) -> bool { - unimplemented!() + fn set_writable(&mut self, value: bool) + { + self.as_flags().set(EF::RDONLY, !value); + self.as_flags().set(EF::WRITE, value); } - - /// Will PageFault when try to access page where present=0 - fn present(&self) -> bool { - unimplemented!() + fn set_present(&mut self, value: bool) { self.as_flags().set(EF::PRESENT, value); } + fn target(&self) -> usize { self.0.addr().as_u64() as usize } + fn set_target(&mut self, target: usize) { + let flags = self.0.flags(); + self.0.set_addr(PhysAddr::new(target as u64), flags); } - - fn clear_accessed(&mut self) { - unimplemented!() - } - - fn clear_dirty(&mut self) { - unimplemented!() - } - - fn set_writable(&mut self, value: bool) { - unimplemented!() - } - - fn set_present(&mut self, value: bool) { - unimplemented!() - } - - fn target(&self) -> PhysAddr { - unimplemented!() - } - - fn set_target(&mut self, target: PhysAddr) { - unimplemented!() - } - - // For Copy-on-write extension - fn writable_shared(&self) -> bool { - unimplemented!() - } - - fn readonly_shared(&self) -> bool { - unimplemented!() - } - + fn writable_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } + fn readonly_shared(&self) -> bool { self.0.flags().contains(EF::BIT_9) } fn set_shared(&mut self, writable: bool) { - unimplemented!() + let flags = self.as_flags(); + flags.set(EF::BIT_8, writable); + flags.set(EF::BIT_9, writable); } - - fn clear_shared(&mut self) { - unimplemented!() - } - - // For Swap extension - fn swapped(&self) -> bool { - unimplemented!() - } - - fn set_swapped(&mut self, value: bool) { - unimplemented!() - } - - fn user(&self) -> bool { - unimplemented!() - } - + fn clear_shared(&mut self) { self.as_flags().remove(EF::BIT_8 | EF::BIT_9); } + fn user(&self) -> bool { self.0.flags().contains(EF::USER_ACCESSIBLE) } + fn swapped(&self) -> bool { self.0.flags().contains(EF::SWAPPED) } + fn set_swapped(&mut self, value: bool) { self.as_flags().set(EF::SWAPPED, value); } fn set_user(&mut self, value: bool) { - unimplemented!() + self.as_flags().set(EF::USER_ACCESSIBLE, value); + if value { + let mut addr = self as *const _ as usize; + for _ in 0..3 { + // Upper level entry + addr = ((addr >> 9) & 0o777_777_777_7770) | 0xffffff80_00000000; + // set USER_ACCESSIBLE + unsafe { (*(addr as *mut EF)).insert(EF::USER_ACCESSIBLE) }; + } + } } + fn execute(&self) -> bool { !self.0.flags().contains(EF::UXN) } + fn set_execute(&mut self, value: bool) { self.as_flags().set(EF::UXN, !value); } +} - fn execute(&self) -> bool { - unimplemented!() - } - - fn set_execute(&mut self, value: bool) { - unimplemented!() +impl PageEntry { + fn read_only(&self) -> bool { self.0.flags().contains(EF::RDONLY) } + fn hw_dirty(&self) -> bool { self.writable() && !self.read_only() } + fn sw_dirty(&self) -> bool { self.0.flags().contains(EF::DIRTY) } + fn as_flags(&mut self) -> &mut EF { + unsafe { &mut *(self as *mut _ as *mut EF) } } } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct MockFrame(PhysAddr); - -impl MockFrame { - pub fn of_addr(addr: PhysAddr) -> Self { - MockFrame(addr) - } - pub fn start_address(&self) -> PhysAddr { - unimplemented!() - } - pub fn p2_index(&self) -> usize { - unimplemented!() - } - pub fn p1_index(&self) -> usize { - unimplemented!() - } - pub fn number(&self) -> usize { - unimplemented!() - } -} - -/// TODO +#[derive(Debug)] pub struct InactivePageTable0 { - p4_frame: MockFrame, + p4_frame: Frame, } -/// TODO impl InactivePageTable for InactivePageTable0 { type Active = ActivePageTable; fn new() -> Self { - unsafe { - let layout = Layout::new::(); - let ptr = alloc(layout); - let frame = MockFrame::of_addr(*ptr as usize); - InactivePageTable0 { p4_frame: frame } - } + let mut pt = Self::new_bare(); + pt.map_kernel(); + pt } fn new_bare() -> Self { - unimplemented!() + let frame = Self::alloc_frame().map(|target| Frame::of_addr(target)) + .expect("failed to allocate frame"); + active_table().with_temporary_map(&frame, |_, table: &mut Aarch64PageTable| { + table.zero(); + // set up recursive mapping for the table + table[511].set_frame(frame.clone(), EF::PRESENT | EF::WRITE); + }); + InactivePageTable0 { p4_frame: frame } } fn edit(&mut self, f: impl FnOnce(&mut Self::Active)) { - unimplemented!() + active_table().with_temporary_map(&ttbr0_el1_read().0, |active_table, p4_table: &mut Aarch64PageTable| { + let backup = p4_table[0o777].clone(); + + // overwrite recursive mapping + p4_table[0o777].set_frame(self.p4_frame.clone(), EF::PRESENT | EF::WRITE); + tlb_invalidate(); + + // execute f in the new context + f(active_table); + + // restore recursive mapping to original p4 table + p4_table[0o777] = backup; + tlb_invalidate(); + }); } unsafe fn activate(&self) { - unimplemented!() + let old_frame = ttbr0_el1_read().0; + let new_frame = self.p4_frame.clone(); + debug!("switch table {:?} -> {:?}", old_frame, new_frame); + if old_frame != new_frame { + ttbr0_el1_write(new_frame); + } } unsafe fn with(&self, f: impl FnOnce()) { - unimplemented!() + let old_frame = ttbr0_el1_read().0; + let new_frame = self.p4_frame.clone(); + debug!("switch table {:?} -> {:?}", old_frame, new_frame); + if old_frame != new_frame { + ttbr0_el1_write(new_frame); + } + f(); + debug!("switch table {:?} -> {:?}", new_frame, old_frame); + if old_frame != new_frame { + ttbr0_el1_write(old_frame); + } } fn token(&self) -> usize { - 0 + self.p4_frame.start_address().as_u64() as usize // as CR3 } - fn alloc_frame() -> Option { + fn alloc_frame() -> Option { alloc_frame() } - fn dealloc_frame(target: PhysAddr) { + fn dealloc_frame(target: usize) { dealloc_frame(target) } @@ -220,3 +235,37 @@ impl InactivePageTable for InactivePageTable0 { alloc_stack() } } + +impl InactivePageTable0 { + fn map_kernel(&mut self) { + let mut table = unsafe { &mut *(0xffffffff_fffff000 as *mut Aarch64PageTable) }; + // Kernel at 0xffff_ff00_0000_0000 + // Kernel stack at 0x0000_57ac_0000_0000 (defined in bootloader crate) + let e0 = table[0].clone(); + self.edit(|_| { + table[0].set_addr(e0.addr(), e0.flags() & EF::GLOBAL); + //table[175].set_addr(estack.addr(), estack.flags() & EF::GLOBAL); + }); + } +} + +impl Drop for InactivePageTable0 { + fn drop(&mut self) { + info!("PageTable dropping: {:?}", self); + Self::dealloc_frame(self.p4_frame.start_address().as_u64() as usize); + } +} + +struct FrameAllocatorForAarch64; + +impl FrameAllocator for FrameAllocatorForAarch64 { + fn alloc(&mut self) -> Option { + alloc_frame().map(|addr| Frame::of_addr(addr)) + } +} + +impl FrameDeallocator for FrameAllocatorForAarch64 { + fn dealloc(&mut self, frame: Frame) { + dealloc_frame(frame.start_address().as_u64() as usize); + } +} \ No newline at end of file diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index 4474c9c8..063b1d01 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -33,6 +33,8 @@ extern crate volatile; #[cfg(target_arch = "x86_64")] extern crate x86_64; extern crate xmas_elf; +#[cfg(target_arch = "aarch64")] +extern crate aarch64; use linked_list_allocator::LockedHeap;