diff --git a/src/gfx_layout.cpp b/src/gfx_layout.cpp
--- a/src/gfx_layout.cpp
+++ b/src/gfx_layout.cpp
@@ -297,10 +297,10 @@ ptrdiff_t Layouter::GetCharAtPosition(in
 Font *Layouter::GetFont(FontSize size, TextColour colour)
 {
 	FontColourMap::iterator it = fonts[size].find(colour);
-	if (it != fonts[size].end()) return it->second;
+	if (it != fonts[size].end()) return it->second.get();
 
-	fonts[size][colour] = new Font(size, colour);
-	return fonts[size][colour];
+	fonts[size][colour] = std::make_unique<Font>(size, colour);
+	return fonts[size][colour].get();
 }
 
 /**
@@ -309,9 +309,6 @@ Font *Layouter::GetFont(FontSize size, T
  */
 void Layouter::ResetFontCache(FontSize size)
 {
-	for (auto &pair : fonts[size]) {
-		delete pair.second;
-	}
 	fonts[size].clear();
 
 	/* We must reset the linecache since it references the just freed fonts */
diff --git a/src/gfx_layout.h b/src/gfx_layout.h
--- a/src/gfx_layout.h
+++ b/src/gfx_layout.h
@@ -80,7 +80,7 @@ public:
 	Font(FontSize size, TextColour colour);
 };
 
-/** Mapping from index to font. */
+/** Mapping from index to font. The pointer is owned by FontColourMap. */
 using FontMap = std::map<int, Font *>;
 
 /**
@@ -169,7 +169,7 @@ private:
 
 	static LineCacheItem &GetCachedParagraphLayout(std::string_view str, const FontState &state);
 
-	using FontColourMap = std::map<TextColour, Font *>;
+	using FontColourMap = std::map<TextColour, std::unique_ptr<Font>>;
 	static FontColourMap fonts[FS_END];
 public:
 	static Font *GetFont(FontSize size, TextColour colour);