Rewrite cache management functions

ee/kernel/Makefile

@@ -79,7 +79,10 @@ IOPCONTROL_OBJS = SifIopReboot.o SifIopReset.o SifIopIsAlive.o SifIopSync.o __io
 
 GLUE_OBJS = DIntr.o EIntr.o EnableIntc.o DisableIntc.o EnableDmac.o DisableDmac.o
 GLUE_OBJS += iEnableIntc.o iDisableIntc.o iEnableDmac.o iDisableDmac.o
-GLUE_OBJS += SyncDCache.o iSyncDCache.o InvalidDCache.o iInvalidDCache.o
+
+### Cache
+CACHE_OBJS = SyncDCache.o iSyncDCache.o InvalidDCache.o iInvalidDCache.o
+CACHE_OBJS += sceSifWriteBackDCache.o
 
 ### SIO objects
 
@@ -218,14 +221,13 @@ KERNEL_OBJS = ResetEE.o SetGsCrt.o $(EXEC_SYSCALLS) \
 	SetPgifHandler.o SetVSyncFlag.o SetSyscall.o sceSifDmaStat.o isceSifDmaStat.o \
 	sceSifSetDma.o isceSifSetDma.o sceSifSetDChain.o isceSifSetDChain.o sceSifSetReg.o \
 	sceSifGetReg.o $(EXECOSD_SYSCALL) Deci2Call.o PSMode.o MachineType.o GetMemorySize.o _GetGsDxDyOffset.o \
-	_InitTLB.o SetMemoryMode.o GetMemoryMode.o \
-	sceSifWriteBackDCache.o _SyncDCache.o _InvalidDCache.o __errno.o errno.o \
-	strncpy.o strlen.o memcpy.o memset.o __syscall.o GPfuncs.o _print.o
+	_InitTLB.o SetMemoryMode.o GetMemoryMode.o __errno.o errno.o strncpy.o strlen.o memcpy.o \
+	memset.o __syscall.o GPfuncs.o _print.o
 
 EE_OBJS = $(KERNEL_OBJS) $(SIFCMD_OBJS) $(SIFRPC_OBJS) $(FILEIO_OBJS) \
 	$(LOADFILE_OBJS) $(IOPHEAP_OBJS) $(IOPCONTROL_OBJS) $(ROM0_OBJS) $(CONFIG_OBJS) \
 	$(GLUE_OBJS) $(SIO_OBJS) $(TIMER_OBJS) $(TIMER_ALARM_OBJS) $(DELAYTHREAD_OBJS) $(GETKERNEL_OBJS) \
-	$(INITSYS_OBJS) $(KERNEL_UTIL_OBJS) deci2.o tty.o erl-support.o
+	$(CACHE_OBJS) $(INITSYS_OBJS) $(KERNEL_UTIL_OBJS) deci2.o tty.o erl-support.o
 
 
 EE_OBJS += $(THREAD_OBJS) $(LIBOSD_OBJS) $(TLBFUNC_OBJS) $(ALARM_OBJS) $(EXIT_OBJS) $(SETUP_OBJS) setup_syscalls.o debug.o
@@ -295,6 +297,10 @@ $(GLUE_OBJS:%=$(EE_OBJS_DIR)%): $(EE_SRC_DIR)glue.c
 	$(DIR_GUARD)
 	$(EE_C_COMPILE) -DF_$(*:$(EE_OBJS_DIR)%=%) $< -c -o $@
 
+$(CACHE_OBJS:%=$(EE_OBJS_DIR)%): $(EE_SRC_DIR)cache.c
+	$(DIR_GUARD)
+	$(EE_C_COMPILE) -DF_$(*:$(EE_OBJS_DIR)%=%) $< -c -o $@
+
 $(THREAD_OBJS:%=$(EE_OBJS_DIR)%): $(EE_SRC_DIR)thread.c
 	$(DIR_GUARD)
 	$(EE_C_COMPILE) -DF_$(*:$(EE_OBJS_DIR)%=%) $< -c -o $@

ee/kernel/include/cache.h

@@ -0,0 +1,116 @@
+#ifndef CACHE_H_
+#define CACHE_H_
+
+#define CACHE_LINE_SIZE 0x40
+#define CACHE_LINE_MASK (~(CACHE_LINE_SIZE - 1))
+
+#define OP_IHIN   0x0b /* Instruction cache: Hit INvalidate.  */
+#define OP_DXWBIN 0x14 /* Data cache: indeX WriteBack INvalidate.  */
+#define OP_DXIN   0x16 /* Data cache: indeX INvalidate.  */
+#define OP_DHWBIN 0x18 /* Data cache: Hit WriteBack INvalidate.  */
+#define OP_DHIN   0x1a /* Data cache: Hit INvalidate.  */
+
+#define DCACHE_OP_LINE(op, line) \
+    __asm__ volatile(            \
+        ".set push          \n"  \
+        ".set noreorder     \n"  \
+        "sync.l             \n"  \
+        "cache %0, 0(%1)    \n"  \
+        "sync.l             \n"  \
+        ".set pop           \n"  \
+        :                        \
+        : "i"(op), "r"(line));
+
+#define ICACHE_OP_LINE(op, line) \
+    __asm__ volatile(            \
+        ".set push          \n"  \
+        ".set noreorder     \n"  \
+        "sync.p             \n"  \
+        "cache %0, 0(%1)    \n"  \
+        "sync.p             \n"  \
+        ".set pop           \n"  \
+        :                        \
+        : "i"(op), "r"(line));
+
+/* Single line operations */
+static inline void dcache_writeback_line(unsigned addr)
+{
+    DCACHE_OP_LINE(OP_DHWBIN, addr);
+}
+
+static inline void dcache_invalid_line(unsigned addr)
+{
+    DCACHE_OP_LINE(OP_DHIN, addr);
+}
+
+static inline void icache_invalid_line(unsigned addr)
+{
+    ICACHE_OP_LINE(OP_IHIN, addr);
+}
+
+/*
+ * The standard SyncDCache/InvalidDCache functions have a slightly awkward
+ * API in that the range is inclusive of both the start and end, requiring
+ * you to remember to substract from the end to avoid affecting unrelated
+ * cache lines.
+ *
+ *
+ * These functions are exclusive of the end and can therefore be used
+ * with sizeof in less error-prone way.
+ * e.g. dcache_writeback_range(&mystruct, &mystruct + sizeof(mystruct));
+ */
+
+/*
+ * Write back data cache lines corresponding to range [start, end)
+ */
+static inline void dcache_writeback_range(unsigned start, unsigned end)
+{
+    start = start & CACHE_LINE_MASK;
+    end   = (end - 1) & CACHE_LINE_MASK;
+
+    while (1) {
+        dcache_writeback_line(start);
+        if (start == end) {
+            break;
+        }
+        start += CACHE_LINE_SIZE;
+    }
+}
+
+/*
+ * Invalidate data cache lines corresponding to range [start, end)
+ */
+static inline void dcache_invalid_range(unsigned start, unsigned end)
+{
+    start = start & CACHE_LINE_MASK;
+    end   = (end - 1) & CACHE_LINE_MASK;
+
+    while (1) {
+        dcache_invalid_line(start);
+        if (start == end) {
+            break;
+        }
+        start += CACHE_LINE_SIZE;
+    }
+}
+
+/*
+ * Invalidate instruction cache lines corresponding to range [start, end)
+ * This also automatically invalidates the relevant BTAC entries.
+ */
+static inline void icache_invalid_range(unsigned start, unsigned end)
+{
+    start = start & CACHE_LINE_MASK;
+    end   = (end - 1) & CACHE_LINE_MASK;
+
+    while (1) {
+        icache_invalid_line(start);
+        if (start == end) {
+            break;
+        }
+        start += CACHE_LINE_SIZE;
+    }
+}
+
+
+#endif // CACHE_H_

ee/kernel/include/kernel.h

@@ -538,9 +538,6 @@ extern int SetMemoryMode(int mode); // Arbitrarily named.
 /* (DESR kernels only) Get the value set by SetMemoryMode. */
 extern int GetMemoryMode(void); // Arbitrarily named.
 
-extern void _SyncDCache(void *start, void *end);
-extern void _InvalidDCache(void *start, void *end);
-
 extern void *GetSyscallHandler(int syscall_no);
 extern void *GetExceptionHandler(int except_no);
 extern void *GetInterruptHandler(int intr_no);

ee/kernel/src/cache.c

@@ -0,0 +1,48 @@
+#include "cache.h"
+
+#include "kernel.h"
+
+/*
+ * Write back data cache lines corresponding to range [start, end)
+ */
+#ifdef F_sceSifWriteBackDCache
+void sceSifWriteBackDCache(void *ptr, int size)
+{
+    dcache_writeback_range((u32)ptr, (u32)ptr + size);
+}
+#endif
+
+/*
+ * These functions affect range [start, end]
+ * (Inclusive on both sides)
+ *
+ * e.g. SyncDCache(0x0, 0x40) would affect two cache lines.
+ */
+
+#ifdef F_SyncDCache
+void SyncDCache(void *start, void *end)
+{
+    dcache_writeback_range((u32)start, (u32)end + 1);
+}
+#endif
+
+#ifdef F_iSyncDCache
+void iSyncDCache(void *start, void *end)
+{
+    dcache_writeback_range((u32)start, (u32)end + 1);
+}
+#endif
+
+#ifdef F_InvalidDCache
+void InvalidDCache(void *start, void *end)
+{
+    dcache_invalid_range((u32)start, (u32)end + 1);
+}
+#endif
+
+#ifdef F_iInvalidDCache
+void iInvalidDCache(void *start, void *end)
+{
+    dcache_invalid_range((u32)start, (u32)end + 1);
+}
+#endif

ee/kernel/src/glue.c

@@ -246,55 +246,3 @@ int iReleaseAlarm(int alarm_id)
     return res;
 }
 #endif
-
-#ifdef F_SyncDCache
-void SyncDCache(void *start, void *end)
-{
-    int eie;
-
-    asm volatile("mfc0\t%0, $12"
-                 : "=r"(eie));
-    eie &= 0x10000;
-
-    if (eie)
-        DI();
-
-    _SyncDCache((void *)((u32)start & 0xffffffc0), (void *)((u32)end & 0xffffffc0));
-
-    if (eie)
-        EI();
-}
-#endif
-
-#ifdef F_iSyncDCache
-void iSyncDCache(void *start, void *end)
-{
-    _SyncDCache((void *)((u32)start & 0xffffffc0), (void *)((u32)end & 0xffffffc0));
-}
-#endif
-
-#ifdef F_InvalidDCache
-void InvalidDCache(void *start, void *end)
-{
-    int eie;
-
-    asm volatile("mfc0\t%0, $12"
-                 : "=r"(eie));
-    eie &= 0x10000;
-
-    if (eie)
-        DI();
-
-    _InvalidDCache((void *)((u32)start & 0xffffffc0), (void *)((u32)end & 0xffffffc0));
-
-    if (eie)
-        EI();
-}
-#endif
-
-#ifdef F_iInvalidDCache
-void iInvalidDCache(void *start, void *end)
-{
-    _InvalidDCache((void *)((u32)start & 0xffffffc0), (void *)((u32)end & 0xffffffc0));
-}
-#endif