fix(SEC-2): replace input-sized C VLAs with heap allocation at three sites

iosMath/lib/MTMathAtomFactory.m

@@ -141,7 +141,8 @@ + (MTMathList *)mathListForCharacters:(NSString *)chars
 {
     NSParameterAssert(chars);
     NSInteger len = chars.length;
-    unichar buff[len];
+    unichar *buff = malloc(sizeof(unichar) * (size_t)len);
+    NSAssert(len == 0 || buff != NULL, @"Failed to allocate buff");
     [chars getCharacters:buff range:NSMakeRange(0, len)];
     MTMathList* list = [[MTMathList alloc] init];
     for (NSInteger i = 0; i < len; i++) {
@@ -150,6 +151,7 @@ + (MTMathList *)mathListForCharacters:(NSString *)chars
             [list addAtom:atom];
         }
     }
+    free(buff);
     return list;
 }
 

iosMath/render/internal/MTTypesetter.m

@@ -424,15 +424,31 @@ static UTF32Char styleCharacter(unichar ch, MTFontStyle fontStyle)
 }
 
 static NSString* changeFont(NSString* str, MTFontStyle fontStyle) {
-    NSMutableString* retval = [NSMutableString stringWithCapacity:str.length];
-    unichar charBuffer[str.length];
-    [str getCharacters:charBuffer range:NSMakeRange(0, str.length)];
-    for (int i = 0; i < str.length; ++i) {
-        unichar ch = charBuffer[i];
-        UTF32Char unicode = styleCharacter(ch, fontStyle);
-        unicode = NSSwapHostIntToLittle(unicode);
-        NSString* charStr = [[NSString alloc] initWithBytes:&unicode length:sizeof(unicode) encoding:NSUTF32LittleEndianStringEncoding];
-        [retval appendString:charStr];
+    NSUInteger length = str.length;
+    NSMutableString* retval = [NSMutableString stringWithCapacity:length];
+    // Hot path: almost every nucleus is a single character (length == 1).
+    // Use a fixed-size stack buffer for the common small case to avoid a
+    // malloc/free per call.  Inputs longer than 256 unichars still fall back
+    // to the heap so the SEC-2 fix (no unbounded VLA) remains intact.
+    unichar stackBuf[256];
+    unichar *charBuffer = (length <= 256) ? stackBuf : malloc(sizeof(unichar) * (size_t)length);
+    NSCAssert(length == 0 || charBuffer != NULL, @"Failed to allocate charBuffer");
+    // Wrap in @try/@finally so charBuffer is released on all exit paths,
+    // including the IllegalCharacter / Invalid style exceptions that
+    // styleCharacter can @throw from inside the loop.
+    @try {
+        [str getCharacters:charBuffer range:NSMakeRange(0, length)];
+        for (NSUInteger i = 0; i < length; ++i) {
+            unichar ch = charBuffer[i];
+            UTF32Char unicode = styleCharacter(ch, fontStyle);
+            unicode = NSSwapHostIntToLittle(unicode);
+            NSString* charStr = [[NSString alloc] initWithBytes:&unicode length:sizeof(unicode) encoding:NSUTF32LittleEndianStringEncoding];
+            [retval appendString:charStr];
+        }
+    } @finally {
+        if (charBuffer != stackBuf) {
+            free(charBuffer);
+        }
     }
     return retval;
 }
@@ -2127,24 +2143,29 @@ - (MTDisplay*) makeTable:(MTMathTable*) table
         return [[MTMathListDisplay alloc] initWithDisplays:[NSArray array] range:table.indexRange];
     }
 
-    CGFloat columnWidths[numColumns];
-    // NOTE: Using memset to initialize columnWidths array avoids
-    // Xcode Analyze "Assigned value is garbage or undefined".
-    // https://stackoverflow.com/questions/21191194/analyzer-warning-assigned-value-is-garbage-or-undefined
-    memset(columnWidths, 0, sizeof(columnWidths));
-    NSArray<NSArray<MTDisplay*>*>* displays = [self typesetCells:table columnWidths:columnWidths];
-
-    // Position all the columns in each row
-    NSMutableArray<MTDisplay*>* rowDisplays = [NSMutableArray arrayWithCapacity:table.cells.count];
-    for (NSArray<MTDisplay*>* row in displays) {
-        MTMathListDisplay* rowDisplay = [self makeRowWithColumns:row forTable:table columnWidths:columnWidths];
-        [rowDisplays addObject:rowDisplay];
+    CGFloat *columnWidths = calloc(numColumns, sizeof(CGFloat));
+    NSAssert(columnWidths != NULL, @"Failed to allocate columnWidths");
+    // Wrap in @try/@finally so columnWidths is released on all exit paths.
+    // typesetCells:/makeRowWithColumns: eventually call changeFont, which can
+    // @throw IllegalCharacter / Invalid style; those would otherwise leak the buffer.
+    MTMathListDisplay* tableDisplay = nil;
+    @try {
+        NSArray<NSArray<MTDisplay*>*>* displays = [self typesetCells:table columnWidths:columnWidths];
+
+        // Position all the columns in each row
+        NSMutableArray<MTDisplay*>* rowDisplays = [NSMutableArray arrayWithCapacity:table.cells.count];
+        for (NSArray<MTDisplay*>* row in displays) {
+            MTMathListDisplay* rowDisplay = [self makeRowWithColumns:row forTable:table columnWidths:columnWidths];
+            [rowDisplays addObject:rowDisplay];
+        }
+
+        // Position all the rows
+        [self positionRows:rowDisplays forTable:table];
+        tableDisplay = [[MTMathListDisplay alloc] initWithDisplays:rowDisplays range:table.indexRange];
+        tableDisplay.position = _currentPosition;
+    } @finally {
+        free(columnWidths);
     }
-
-    // Position all the rows
-    [self positionRows:rowDisplays forTable:table];
-    MTMathListDisplay* tableDisplay = [[MTMathListDisplay alloc] initWithDisplays:rowDisplays range:table.indexRange];
-    tableDisplay.position = _currentPosition;
     return tableDisplay;
 }
 

iosMathTests/MTTypesetterTest.m

@@ -2725,4 +2725,59 @@ - (void)testOversetRowRendersAtScriptScriptWhenNestedInSuperscript
     XCTAssertLessThan(nestedStack.over.ascent, baselineStack.over.ascent);
 }
 
+// SEC-2 regression tests: heap-allocate input-sized buffers (VLA → malloc/free)
+// These verify that the three fixed sites correctly handle larger inputs without
+// crashing or producing wrong results.
+
+- (void)testMathListForCharactersLargeInput_SEC2
+{
+    // Site 1: +[MTMathAtomFactory mathListForCharacters:]
+    // Build a 10,000-character digit string. The old VLA would put 20 KB on the
+    // stack; with the heap fix it should succeed and return exactly 10,000 atoms.
+    NSMutableString* digits = [NSMutableString stringWithCapacity:10000];
+    for (int i = 0; i < 10000; i++) {
+        [digits appendString:@"1"];
+    }
+    MTMathList* list = [MTMathAtomFactory mathListForCharacters:digits];
+    XCTAssertNotNil(list, @"mathListForCharacters: should not return nil for a 10k-digit string");
+    XCTAssertEqual(list.atoms.count, (NSUInteger)10000, @"Each character should produce exactly one atom");
+}
+
+- (void)testChangeFontLargeNucleus_SEC2
+{
+    // Site 2: changeFont() in MTTypesetter (exercised via rendering a long
+    // variable/number run). Build a math list with a single ordinary atom whose
+    // nucleus is 10,000 'x' characters. The typesetter calls changeFont on it
+    // which would stack-overflow with a VLA; with the heap fix it should
+    // produce a non-nil display.
+    NSMutableString* longNucleus = [NSMutableString stringWithCapacity:10000];
+    for (int i = 0; i < 10000; i++) {
+        [longNucleus appendString:@"x"];
+    }
+    MTMathAtom* atom = [MTMathAtom atomWithType:kMTMathAtomVariable value:longNucleus];
+    MTMathList* list = [[MTMathList alloc] init];
+    [list addAtom:atom];
+    MTMathListDisplay* display = [MTTypesetter createLineForMathList:list font:self.font style:kMTLineStyleDisplay];
+    XCTAssertNotNil(display, @"Rendering a 10k-char nucleus should produce a display (not crash)");
+    XCTAssertGreaterThan(display.ascent, 0, @"Display should have positive ascent");
+}
+
+- (void)testMathTableManyColumns_SEC2
+{
+    // Site 3: -[MTTypesetter makeTable:] columnWidths VLA.
+    // Build a table with 500 columns (all empty cells). The old VLA would put
+    // 500*8 = 4 KB on the stack; with the heap fix it should succeed and return
+    // a non-nil display.
+    MTMathTable* table = [[MTMathTable alloc] init];
+    NSUInteger numCols = 500;
+    for (NSUInteger col = 0; col < numCols; col++) {
+        MTMathList* cell = [[MTMathList alloc] init];
+        [table setCell:cell forRow:0 column:col];
+    }
+    MTMathList* mathList = [[MTMathList alloc] init];
+    [mathList addAtom:table];
+    MTMathListDisplay* display = [MTTypesetter createLineForMathList:mathList font:self.font style:kMTLineStyleDisplay];
+    XCTAssertNotNil(display, @"Rendering a 500-column table should produce a display");
+}
+
 @end