From: kettenis <kettenis@openbsd.org>
Date: Wed, 27 Mar 2024 23:10:18 +0000 (+0000)
Subject: The RISC-V architecture specification says that memory read/writes are
X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=a3fd9ccd2992e33e45c4ee35fb816cb267314f0a;p=openbsd

The RISC-V architecture specification says that memory read/writes are
not ordered with respect to mmio read/writes.  This appears to happen on
T-Head C920 cores as I'm seeing interrupts being enabled before the lock
is released in mtx_leave() despite program order releasing the lock
before enabling interrupts.  This is fixed by adding the necessary fences
in more or less the same places where Linux uses them.

ok patrick@, jca@
---

diff --git a/sys/arch/riscv64/include/bus.h b/sys/arch/riscv64/include/bus.h
index 14a324c04d3..9e6449976ab 100644
--- a/sys/arch/riscv64/include/bus.h
+++ b/sys/arch/riscv64/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.5 2022/12/29 11:35:01 kettenis Exp $	*/
+/*	$OpenBSD: bus.h,v 1.6 2024/03/27 23:10:18 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB Sweden.  All rights reserved.
@@ -265,8 +265,10 @@ bus_space_copy_1(void *v, bus_space_handle_t h1, bus_size_t o1,
 	char *s = (char *)(h1 + o1);
 	char *d = (char *)(h2 + o2);
 
+	__asm volatile ("fence w,o" ::: "memory");
 	while (c--)
 		*d++ = *s++;
+	__asm volatile ("fence io,iwr" ::: "memory");
 }
 
 
@@ -277,8 +279,10 @@ bus_space_copy_2(void *v, bus_space_handle_t h1, bus_size_t o1,
 	short *s = (short *)(h1 + o1);
 	short *d = (short *)(h2 + o2);
 
+	__asm volatile ("fence w,o" ::: "memory");
 	while (c--)
 		*d++ = *s++;
+	__asm volatile ("fence io,iwr" ::: "memory");
 }
 
 static __inline void
@@ -288,8 +292,10 @@ bus_space_copy_4(void *v, bus_space_handle_t h1, bus_size_t o1,
 	int *s = (int *)(h1 + o1);
 	int *d = (int *)(h2 + o2);
 
+	__asm volatile ("fence w,o" ::: "memory");
 	while (c--)
 		*d++ = *s++;
+	__asm volatile ("fence io,iwr" ::: "memory");
 }
 
 static __inline void
@@ -299,8 +305,10 @@ bus_space_copy_8(void *v, bus_space_handle_t h1, bus_size_t o1,
 	int64_t *s = (int64_t *)(h1 + o1);
 	int64_t *d = (int64_t *)(h2 + o2);
 
+	__asm volatile ("fence w,o" ::: "memory");
 	while (c--)
 		*d++ = *s++;
+	__asm volatile ("fence io,iwr" ::: "memory");
 }
 
 /*----------------------------------------------------------------------------*/
diff --git a/sys/arch/riscv64/riscv64/bus_space.c b/sys/arch/riscv64/riscv64/bus_space.c
index d20ffb18460..4884d68fba9 100644
--- a/sys/arch/riscv64/riscv64/bus_space.c
+++ b/sys/arch/riscv64/riscv64/bus_space.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_space.c,v 1.2 2021/05/12 01:20:52 jsg Exp $	*/
+/*	$OpenBSD: bus_space.c,v 1.3 2024/03/27 23:10:18 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -64,53 +64,69 @@ bus_space_t *fdt_cons_bs_tag = &riscv64_bs_tag;
 uint8_t
 generic_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-	return *(volatile uint8_t *)(h + o);
+	uint8_t val = *(volatile uint8_t *)(h + o);
+	__asm volatile ("fence i,ir" ::: "memory");
+	return val;
 }
 
 uint16_t
 generic_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-	return *(volatile uint16_t *)(h + o);
+	uint16_t val = *(volatile uint16_t *)(h + o);
+	__asm volatile ("fence i,ir" ::: "memory");
+	return val;
 }
 
 uint32_t
 generic_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-	return *(volatile uint32_t *)(h + o);
+	uint32_t val = *(volatile uint32_t *)(h + o);
+	__asm volatile ("fence i,ir" ::: "memory");
+	return val;
 }
 
 uint64_t
 generic_space_read_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
 {
-	return *(volatile uint64_t *)(h + o);
+	uint64_t val = *(volatile uint64_t *)(h + o);
+	__asm volatile ("fence i,ir" ::: "memory");
+	return val;
 }
 
 void
 generic_space_write_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint8_t v)
 {
+	__asm volatile ("fence w,o" ::: "memory");
 	*(volatile uint8_t *)(h + o) = v;
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint16_t v)
 {
+	__asm volatile ("fence w,o" ::: "memory");
 	*(volatile uint16_t *)(h + o) = v;
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint32_t v)
 {
+	__asm volatile ("fence w,o" ::: "memory");
 	*(volatile uint32_t *)(h + o) = v;
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
 generic_space_write_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
     uint64_t v)
 {
+	__asm volatile ("fence w,o" ::: "memory");
 	*(volatile uint64_t *)(h + o) = v;
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -123,6 +139,7 @@ generic_space_read_raw_2(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
 		*(uint16_t *)buf = *addr;
 		buf += 2;
 	}
+	__asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -130,11 +147,13 @@ generic_space_write_raw_2(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
 	volatile uint16_t *addr = (volatile uint16_t *)(h + o);
+	__asm volatile ("fence w,o" ::: "memory");
 	len >>= 1;
 	while (len-- != 0) {
 		*addr = *(uint16_t *)buf;
 		buf += 2;
 	}
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -147,6 +166,7 @@ generic_space_read_raw_4(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
 		*(uint32_t *)buf = *addr;
 		buf += 4;
 	}
+	__asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -154,11 +174,13 @@ generic_space_write_raw_4(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
 	volatile uint32_t *addr = (volatile uint32_t *)(h + o);
+	__asm volatile ("fence w,o" ::: "memory");
 	len >>= 2;
 	while (len-- != 0) {
 		*addr = *(uint32_t *)buf;
 		buf += 4;
 	}
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 void
@@ -171,6 +193,7 @@ generic_space_read_raw_8(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
 		*(uint64_t *)buf = *addr;
 		buf += 8;
 	}
+	__asm volatile ("fence i,ir" ::: "memory");
 }
 
 void
@@ -178,11 +201,13 @@ generic_space_write_raw_8(bus_space_tag_t t, bus_space_handle_t h, bus_addr_t o,
     const uint8_t *buf, bus_size_t len)
 {
 	volatile uint64_t *addr = (volatile uint64_t *)(h + o);
+	__asm volatile ("fence w,o" ::: "memory");
 	len >>= 3;
 	while (len-- != 0) {
 		*addr = *(uint64_t *)buf;
 		buf += 8;
 	}
+	__asm volatile ("fence o,w" ::: "memory");
 }
 
 int