Flush I/D cache after copying user code in page fault handler

* Now SMP works in real machine with no bugs!!!

crate/memory/src/memory_set/handler/delay.rs

@@ -42,6 +42,7 @@ impl<T: FrameAllocator> MemoryHandler for Delay<T> {
             let entry = pt.map(addr, target);
             attr.apply(entry);
             pt.get_page_slice_mut(addr).copy_from_slice(data);
+            pt.flush_cache_copy_user(addr, addr + data.len(), attr.execute);
         } else {
             // delay map
             self.map(pt, addr, attr);
@@ -60,9 +61,11 @@ impl<T: FrameAllocator> MemoryHandler for Delay<T> {
         entry.update();
         //init with zero for delay mmap mode
         let data = pt.get_page_slice_mut(addr);
+        let len = data.len();
         for x in data {
             *x = 0;
         }
+        pt.flush_cache_copy_user(addr, addr + len, false);
         true
     }
 }

crate/memory/src/memory_set/handler/file.rs

@@ -51,6 +51,7 @@ impl<F: Read, T: FrameAllocator> MemoryHandler for File<F, T> {
             let entry = pt.map(addr, target);
             attr.apply(entry);
             pt.get_page_slice_mut(addr).copy_from_slice(data);
+            pt.flush_cache_copy_user(addr, addr + data.len(), attr.execute);
         } else {
             // delay map
             self.map(pt, addr, attr);
@@ -63,18 +64,20 @@ impl<F: Read, T: FrameAllocator> MemoryHandler for File<F, T> {
         if entry.present() {
             return false;
         }
+        let execute = entry.execute();
         let frame = self.allocator.alloc().expect("failed to alloc frame");
         entry.set_target(frame);
         entry.set_present(true);
         entry.update();
 
-        self.fill_data(pt, addr);
+        let read_size = self.fill_data(pt, addr);
+        pt.flush_cache_copy_user(addr, addr + read_size, execute);
         true
     }
 }
 
 impl<F: Read, T: FrameAllocator> File<F, T> {
-    fn fill_data(&self, pt: &mut dyn PageTable, addr: VirtAddr) {
+    fn fill_data(&self, pt: &mut dyn PageTable, addr: VirtAddr) -> usize {
         let data = pt.get_page_slice_mut(addr);
         let file_offset = addr + self.file_start - self.mem_start;
         let read_size = (self.file_end as isize - file_offset as isize)
@@ -84,6 +87,7 @@ impl<F: Read, T: FrameAllocator> File<F, T> {
         if read_size != PAGE_SIZE {
             data[read_size..].iter_mut().for_each(|x| *x = 0);
         }
+        read_size
     }
 }
 

crate/memory/src/paging/mod.rs

@@ -26,6 +26,9 @@ pub trait PageTable {
     /// Get a mutable reference of the content of a page of virtual address `addr`
     fn get_page_slice_mut<'a>(&mut self, addr: VirtAddr) -> &'a mut [u8];
 
+    /// When copied user data (in page fault handler)，maybe need to flush I/D cache.
+    fn flush_cache_copy_user(&mut self, start: VirtAddr, end: VirtAddr, execute: bool);
+
     /// Read data from virtual address `addr`
     /// Used for testing with mock
     fn read(&mut self, _addr: VirtAddr) -> u8 {

kernel/src/arch/aarch64/cpu.rs

@@ -1,6 +1,7 @@
 use crate::consts::SMP_CORES;
 use crate::memory::{kernel_offset, phys_to_virt};
 use aarch64::asm;
+use core::{cmp, mem};
 
 pub use super::board::{CPU_NUM, CPU_SPIN_TABLE};
 
@@ -16,14 +17,17 @@ pub unsafe fn start_others() {
     extern "C" {
         fn slave_startup();
     }
-    for i in 0..core::cmp::min(CPU_NUM, *SMP_CORES) {
+    for i in 0..cmp::min(CPU_NUM, *SMP_CORES) {
         if i == 0 {
             continue;
         }
         let release_addr = phys_to_virt(CPU_SPIN_TABLE[i]) as *mut usize;
         let entry_addr = kernel_offset(slave_startup as usize);
         *release_addr = entry_addr;
-        asm::flush_dcache_line(release_addr as usize);
+        asm::flush_dcache_range(
+            release_addr as usize,
+            release_addr as usize + mem::size_of::<usize>(),
+        );
         asm::sev();
     }
 }

kernel/src/arch/aarch64/paging.rs

@@ -1,6 +1,7 @@
 //! Page table implementations for aarch64.
 
 use crate::memory::{alloc_frame, dealloc_frame, phys_to_virt};
+use aarch64::asm::{flush_dcache_range, flush_icache_all};
 use aarch64::asm::{tlb_invalidate, tlb_invalidate_all, ttbr_el1_read, ttbr_el1_write};
 use aarch64::paging::{
     frame::PhysFrame as Frame,
@@ -70,6 +71,15 @@ impl PageTable for PageTableImpl {
         let vaddr = phys_to_virt(frame.start_address().as_u64() as usize);
         unsafe { core::slice::from_raw_parts_mut(vaddr as *mut u8, 0x1000) }
     }
+
+    fn flush_cache_copy_user(&mut self, start: usize, end: usize, execute: bool) {
+        if execute {
+            // clean D-cache to PoU to ensure new instructions has been written into memory
+            flush_dcache_range(start, end);
+            // invalidate I-cache to PoU to ensure old instructions has been flushed
+            flush_icache_all();
+        }
+    }
 }
 
 fn frame_to_page_table(frame: Frame) -> *mut Aarch64PageTable {

kernel/src/arch/mipsel/paging.rs

@@ -58,6 +58,8 @@ impl PageTable for PageTableImpl {
         let vaddr = frame.to_kernel_unmapped().as_usize();
         unsafe { core::slice::from_raw_parts_mut(vaddr as *mut u8, 0x1000) }
     }
+
+    fn flush_cache_copy_user(&mut self, _start: usize, _end: usize, _execute: bool) {}
 }
 
 extern "C" {

kernel/src/arch/riscv32/paging.rs

@@ -66,6 +66,8 @@ impl PageTable for PageTableImpl {
         let vaddr = frame.start_address().as_usize() + PHYSICAL_MEMORY_OFFSET;
         unsafe { core::slice::from_raw_parts_mut(vaddr as *mut u8, 0x1000) }
     }
+
+    fn flush_cache_copy_user(&mut self, _start: usize, _end: usize, _execute: bool) {}
 }
 
 /// implementation for the Entry trait in /crate/memory/src/paging/mod.rs

kernel/src/arch/x86_64/paging.rs

@@ -93,6 +93,8 @@ impl PageTable for PageTableImpl {
         let vaddr = phys_to_virt(frame.start_address().as_u64() as usize);
         unsafe { core::slice::from_raw_parts_mut(vaddr as *mut u8, 0x1000) }
     }
+
+    fn flush_cache_copy_user(&mut self, _start: usize, _end: usize, _execute: bool) {}
 }
 
 fn frame_to_page_table(frame: Frame) -> *mut x86PageTable {