Use underlying arena for memory operations in binary translation

lib/libriscv/memory.cpp

@@ -184,11 +184,13 @@ namespace riscv
 			}
 		}
 
-		if (attr.read && !attr.write && m_ropages.end == 0) {
-			// If the serialization fails, we will fallback to memcpy
-			// with set_page_attr, like normal.
-			if (serialize_pages(m_ropages, hdr->p_vaddr, src, len, attr))
-				return;
+		if constexpr (!riscv::binary_translation_enabled) {
+			if (attr.read && !attr.write && m_ropages.end == 0) {
+				// If the serialization fails, we will fallback to memcpy
+				// with set_page_attr, like normal.
+				if (serialize_pages(m_ropages, hdr->p_vaddr, src, len, attr))
+					return;
+			}
 		}
 
 		// Load into virtual memory

lib/libriscv/memory_inline_pages.hpp

@@ -41,8 +41,10 @@ inline const Page& Memory<W>::get_page(const address_t address) const
 template <int W>
 inline const Page& Memory<W>::get_exec_pageno(const address_t pageno) const
 {
-	if (m_ropages.contains(pageno)) {
-		return m_ropages.pages[pageno - m_ropages.begin];
+	if constexpr (!riscv::binary_translation_enabled) {
+		if (m_ropages.contains(pageno)) {
+			return m_ropages.pages[pageno - m_ropages.begin];
+		}
 	}
 
 	auto it = m_pages.find(pageno);
@@ -60,8 +62,10 @@ inline const Page& Memory<W>::get_pageno(const address_t pageno) const
 		return it->second;
 	}
 
-	if (m_ropages.contains(pageno)) {
-		return m_ropages.pages[pageno - m_ropages.begin];
+	if constexpr (!riscv::binary_translation_enabled) {
+		if (m_ropages.contains(pageno)) {
+			return m_ropages.pages[pageno - m_ropages.begin];
+		}
 	}
 
 	return m_page_readf_handler(*this, pageno);

lib/libriscv/memory_rw.cpp

@@ -25,8 +25,10 @@ namespace riscv
 			}
 		} else {
 
-			if (UNLIKELY(m_ropages.contains(pageno))) {
-				this->protection_fault(pageno * Page::size());
+			if constexpr (!riscv::binary_translation_enabled) {
+				if (UNLIKELY(m_ropages.contains(pageno))) {
+					this->protection_fault(pageno * Page::size());
+				}
 			}
 
 			// Handler must produce a new page, or throw
@@ -66,8 +68,12 @@ namespace riscv
 	}
 
 	template <int W>
-	const Page& Memory<W>::default_page_read(const Memory<W>&, address_t)
+	const Page& Memory<W>::default_page_read(const Memory<W>& mem, address_t pageno)
 	{
+		// This is a copy-on-write zeroed area, but we must respect the underlying arena
+		if (pageno < mem.m_arena_pages) {
+			return const_cast<Memory<W>&> (mem).create_writable_pageno(pageno);
+		}
 		return Page::cow_page();
 	}
 

lib/libriscv/page.hpp

@@ -132,6 +132,18 @@ struct Page
 		attr.is_cow = false;
 		attr.non_owning = false;
 	}
+	void write_to_another(PageData* other)
+	{
+		if (this->m_page != nullptr)
+		{
+			other->buffer8 = m_page->buffer8;
+			if (attr.non_owning) m_page.release();
+		}
+		m_page.reset(other);
+		attr.write = true;
+		attr.is_cow = false;
+		attr.non_owning = true;
+	}
 
 	// Loan a page from somewhere else, that will not be
 	// deleted here. There is no ref-counting mechanism, and

lib/libriscv/tr_api.cpp

@@ -57,16 +57,17 @@ typedef struct {
 	fp64reg fr[32];
 } CPU;
 
-#define PUREFUNC __attribute__((pure))
 #define PAGENO(x) ((addr_t)(x) >> 12)
 #define PAGEOFF(x) ((addr_t)(x) & 0xFFF)
 
-typedef PUREFUNC const char* (*mem_ld_t) (const CPU*, addr_t);
-typedef PUREFUNC char* (*mem_st_t) (const CPU*, addr_t);
+typedef const char* (*mem_ld_t) (const CPU*, addr_t);
+typedef char* (*mem_st_t) (const CPU*, addr_t);
 
 static struct CallbackTable {
 	mem_ld_t mem_ld;
 	mem_st_t mem_st;
+	void (*vec_load)(const CPU*, int, addr_t);
+	void (*vec_store)(const CPU*, addr_t, int);
 	int  (*syscall)(CPU*, addr_t);
 	void (*ebreak)(CPU*);
 	void (*system)(CPU*, uint32_t);
@@ -76,6 +77,7 @@ static struct CallbackTable {
 	double (*sqrtf64)(double);
 } api;
 static char* arena_base;
+static addr_t arena_size;
 static uint64_t* cur_insn;
 static uint64_t* max_insn;
 
@@ -110,10 +112,11 @@ static inline uint64_t MUL128(
 	return (middle << 32) | (uint32_t)p00;
 }
 
-extern void init(struct CallbackTable* table, char* abase, uint64_t* cur_icount, uint64_t* max_icount)
+extern void init(struct CallbackTable* table, char* abase, uint64_t asize, uint64_t* cur_icount, uint64_t* max_icount)
 {
 	api = *table;
 	arena_base = abase;
+	arena_size = asize;
 	cur_insn = cur_icount;
 	max_insn = max_icount;
 };

lib/libriscv/tr_api.hpp

@@ -6,6 +6,8 @@ namespace riscv {
 	struct CallbackTable {
 		const void* (*mem_read)(CPU<W>&, address_type<W> addr);
 		void* (*mem_write) (CPU<W>&, address_type<W> addr);
+		void (*vec_load)(CPU<W>&, int vd, address_type<W> addr);
+		void (*vec_store) (CPU<W>&, address_type<W> addr, int vd);
 		int  (*syscall)(CPU<W>&, address_type<W>);
 		void (*ebreak)(CPU<W>&);
 		void (*system)(CPU<W>&, uint32_t);

lib/libriscv/tr_emit.cpp

@@ -3,6 +3,9 @@
 #include "rv32i_instr.hpp"
 #include "rvfd.hpp"
 #include "tr_types.hpp"
+#ifdef RISCV_EXT_VECTOR
+#include "rvv.hpp"
+#endif
 
 #define PCRELA(x) ((address_t) (tinfo.basepc + index() * 4 + (x)))
 #define PCRELS(x) std::to_string(PCRELA(x)) + "UL"
@@ -164,7 +167,16 @@ struct Emitter
 		}
 
 		const auto address = from_reg(reg) + " + " + from_imm(imm);
-		add_code("const char* " + data + " = api.mem_ld(cpu, PAGENO(" + address + "));");
+		if (cpu.machine().memory.uses_memory_arena()) {
+			add_code(
+				"const char* " + data + ";",
+				"if (" + address + " < arena_size)",
+				data + " = &arena_base[(" + address + ") & ~0xFFFLL];",
+				"else",
+				data + " = api.mem_ld(cpu, PAGENO(" + address + "));");
+		} else {
+			add_code("const char* " + data + " = api.mem_ld(cpu, PAGENO(" + address + "));");
+		}
 		return cast + "*(" + type + "*)&" + data + "[PAGEOFF(" + address + ")]";
 	}
 	void memory_store(std::string type, int reg, int32_t imm, std::string value)
@@ -183,7 +195,16 @@ struct Emitter
 		}
 
 		const auto address = from_reg(reg) + " + " + from_imm(imm);
-		add_code("char* " + data + " = api.mem_st(cpu, PAGENO(" + address + "));");
+		if (cpu.machine().memory.uses_memory_arena()) {
+			add_code(
+				"char* " + data + ";",
+				"if (" + address + " < arena_size)",
+				data + " = &arena_base[(" + address + ") & ~0xFFFLL];",
+				"else",
+				data + " = api.mem_st(cpu, PAGENO(" + address + "));");
+		} else {
+			add_code("char* " + data + " = api.mem_st(cpu, PAGENO(" + address + "));");
+		}
 		add_code(
 			"*(" + type + "*)&" + data + "[PAGEOFF(" + address + ")] = " + value + ";"
 		);
@@ -776,6 +797,13 @@ void Emitter<W>::emit()
 			case 0x3: // FLD
 				code += "load_dbl(&" + from_fpreg(fi.Itype.rd) + ", " + this->memory_load<uint64_t>("uint64_t", fi.Itype.rs1, fi.Itype.signed_imm()) + ");\n";
 				break;
+#ifdef RISCV_EXT_VECTOR
+			case 0x6: { // VLE32
+				const rv32v_instruction vi { instr };
+				code += "api.vec_load(cpu, " + std::to_string(vi.VLS.vd) + ", " + from_reg(vi.VLS.rs1) + ");\n";
+				break;
+			}
+#endif
 			default:
 				code += "api.execute(cpu, " + std::to_string(instr.whole) + ");\n";
 				break;
@@ -790,6 +818,13 @@ void Emitter<W>::emit()
 			case 0x3: // FSD
 				this->memory_store("int64_t", fi.Stype.rs1, fi.Stype.signed_imm(), from_fpreg(fi.Stype.rs2) + ".i64");
 				break;
+#ifdef RISCV_EXT_VECTOR
+			case 0x6: { // VSE32
+				const rv32v_instruction vi { instr };
+				code += "api.vec_store(cpu, " + from_reg(vi.VLS.rs1) + ", " + std::to_string(vi.VLS.vd) + ");\n";
+				break;
+			}
+#endif
 			default:
 				code += "api.execute(cpu, " + std::to_string(instr.whole) + ");\n";
 				break;

lib/libriscv/tr_translate.cpp

@@ -321,14 +321,30 @@ void CPU<W>::activate_dylib(DecodedExecuteSegment<W>& exec, void* dylib) const
 		return;
 	}
 
-	auto func = (void (*)(const CallbackTable<W>&, void*, uint64_t*, uint64_t*)) ptr;
+	auto func = (void (*)(const CallbackTable<W>&, void*, uint64_t, uint64_t*, uint64_t*)) ptr;
 	func(CallbackTable<W>{
 		.mem_read = [] (CPU<W>& cpu, address_type<W> addr) -> const void* {
 			return cpu.machine().memory.cached_readable_page(addr << 12, 1).buffer8.data();
 		},
 		.mem_write = [] (CPU<W>& cpu, address_type<W> addr) -> void* {
 			return cpu.machine().memory.cached_writable_page(addr << 12).buffer8.data();
 		},
+		.vec_load = [] (CPU<W>& cpu, int vd, address_type<W> addr) {
+#ifdef RISCV_EXT_VECTOR
+			auto& rvv = cpu.registers().rvv();
+			rvv.get(vd) = cpu.machine().memory.template read<VectorLane> (addr);
+#else
+			(void)cpu; (void)addr; (void)vd;
+#endif
+		},
+		.vec_store = [] (CPU<W>& cpu, address_type<W> addr, int vd) {
+#ifdef RISCV_EXT_VECTOR
+			auto& rvv = cpu.registers().rvv();
+			cpu.machine().memory.template write<VectorLane> (addr, rvv.get(vd));
+#else
+			(void)cpu; (void)addr; (void)vd;
+#endif
+		},
 		.syscall = [] (CPU<W>& cpu, address_type<W> n) -> int {
 			auto old_pc = cpu.pc();
 			cpu.machine().system_call(n);
@@ -361,6 +377,7 @@ void CPU<W>::activate_dylib(DecodedExecuteSegment<W>& exec, void* dylib) const
 		},
 	},
 	m_machine.memory.memory_arena_ptr(),
+	m_machine.memory.memory_arena_size(),
 	&m_machine.get_counters().first,
 	&m_machine.get_counters().second);