Fixed buffer overflow after gfxSetScreenFormat (#318)

diff --git a/libctru/source/gfx.c b/libctru/source/gfx.c
index 6fa27fb3f5fe87e5930e82d35d7e61cdff25f6bb..e8adbb3ff0a139c3fd928ce0d8a1c0107e1e0ee2 100644 (file)
--- a/libctru/source/gfx.c
+++ b/libctru/source/gfx.c
@@ -16,6 +16,8 @@ u8* gfxSharedMemory;
 u8* gfxTopLeftFramebuffers[2];
 u8* gfxTopRightFramebuffers[2];
 u8* gfxBottomFramebuffers[2];
+u32 gfxTopFramebufferMaxSize;
+u32 gfxBottomFramebufferMaxSize;
 
 static bool enable3d;
 static u8 currentBuffer[2];
@@ -28,6 +30,9 @@ static GSPGPU_FramebufferFormats botFormat = GSP_BGR8_OES;
 
 static GSPGPU_FramebufferInfo* const framebufferInfoSt[] = { &topFramebufferInfo, &bottomFramebufferInfo };
 
+static void (*screenFree)(void *) = NULL;
+static void *(*screenAlloc)(size_t) = NULL;
+
 void gfxSet3D(bool enable)
 {
        enable3d=enable;
@@ -38,39 +43,64 @@ bool gfxIs3D(void)
        return enable3d;
 }
 
+u32 __get_bytes_per_pixel(GSPGPU_FramebufferFormats format) {
+       switch(format) {
+       case GSP_RGBA8_OES:
+               return 4;
+       case GSP_BGR8_OES:
+               return 3;
+       case GSP_RGB565_OES:
+       case GSP_RGB5_A1_OES:
+       case GSP_RGBA4_OES:
+               return 2;
+       }
+
+       return 3;
+}
+
 void gfxSetScreenFormat(gfxScreen_t screen, GSPGPU_FramebufferFormats format) {
-    if(screen==GFX_TOP)
-        topFormat = format;
-    else
-        botFormat = format;
+       if (screenAlloc == NULL || screenFree == NULL) return;
+       if(screen==GFX_TOP)
+       {
+               u32 topSize = 400 * 240 * __get_bytes_per_pixel(format);
+               if (gfxTopFramebufferMaxSize < topSize)
+               {
+                       screenFree(gfxTopLeftFramebuffers[0]);
+                       screenFree(gfxTopLeftFramebuffers[1]);
+                       screenFree(gfxTopRightFramebuffers[0]);
+                       screenFree(gfxTopRightFramebuffers[1]);
+                       gfxTopLeftFramebuffers[0]=screenAlloc(topSize);
+                       gfxTopLeftFramebuffers[1]=screenAlloc(topSize);
+                       gfxTopRightFramebuffers[0]=screenAlloc(topSize);
+                       gfxTopRightFramebuffers[1]=screenAlloc(topSize);
+                       gfxTopFramebufferMaxSize = topSize;
+               }
+               topFormat = format;
+       }else{
+               u32 bottomSize = 320 * 240 * __get_bytes_per_pixel(format);
+               if (gfxBottomFramebufferMaxSize < bottomSize)
+               {
+                       screenFree(gfxBottomFramebuffers[0]);
+                       screenFree(gfxBottomFramebuffers[1]);
+                       gfxBottomFramebuffers[0]=screenAlloc(bottomSize);
+                       gfxBottomFramebuffers[1]=screenAlloc(bottomSize);
+                       gfxBottomFramebufferMaxSize = bottomSize;
+               }
+               botFormat = format;
+       }
 }
 
 GSPGPU_FramebufferFormats gfxGetScreenFormat(gfxScreen_t screen) {
-    if(screen==GFX_TOP)
-        return topFormat;
-    else
-        return botFormat;
+       if(screen==GFX_TOP)
+               return topFormat;
+       else
+               return botFormat;
 }
 
 void gfxSetDoubleBuffering(gfxScreen_t screen, bool doubleBuffering) {
        doubleBuf[screen] = doubleBuffering ? 1 : 0; // make sure they're the integer values '1' and '0'
 }
 
-u32 __get_bytes_per_pixel(GSPGPU_FramebufferFormats format) {
-    switch(format) {
-    case GSP_RGBA8_OES:
-        return 4;
-    case GSP_BGR8_OES:
-        return 3;
-    case GSP_RGB565_OES:
-    case GSP_RGB5_A1_OES:
-    case GSP_RGBA4_OES:
-        return 2;
-    }
-
-    return 3;
-}
-
 void gfxSetFramebufferInfo(gfxScreen_t screen, u8 id)
 {
        if(screen==GFX_TOP)
@@ -105,12 +135,8 @@ void gfxWriteFramebufferInfo(gfxScreen_t screen)
        framebufferInfoHeader[0x1]=1;
 }
 
-static void (*screenFree)(void *) = NULL;
-
 void gfxInit(GSPGPU_FramebufferFormats topFormat, GSPGPU_FramebufferFormats bottomFormat, bool vrambuffers)
 {
-       void *(*screenAlloc)(size_t);
-
        if (vrambuffers)
        {
                screenAlloc=vramAlloc;
@@ -150,6 +176,8 @@ void gfxInit(GSPGPU_FramebufferFormats topFormat, GSPGPU_FramebufferFormats bott
        gfxBottomFramebuffers[1]=screenAlloc(bottomSize);
        gfxTopRightFramebuffers[0]=screenAlloc(topSize);
        gfxTopRightFramebuffers[1]=screenAlloc(topSize);
+       gfxTopFramebufferMaxSize = topSize;
+       gfxBottomFramebufferMaxSize = bottomSize;
 
        enable3d=false;