diff --git a/src/video/sdl_v.cpp b/src/video/sdl_v.cpp
--- a/src/video/sdl_v.cpp
+++ b/src/video/sdl_v.cpp
@@ -730,7 +730,13 @@ void VideoDriver_SDL::MainLoop()
 		}
 
 		if (cur_ticks >= next_game_tick || (_fast_forward && !_pause_mode)) {
-			next_game_tick = cur_ticks + std::chrono::milliseconds(MILLISECONDS_PER_TICK);
+			if (_fast_forward && !_pause_mode) {
+				next_game_tick = cur_ticks + std::chrono::milliseconds(MILLISECONDS_PER_TICK);
+			} else {
+				next_game_tick += std::chrono::milliseconds(MILLISECONDS_PER_TICK);
+				/* Avoid next_game_tick getting behind more and more if it cannot keep up. */
+				if (next_game_tick < cur_ticks - std::chrono::milliseconds(ALLOWED_DRIFT * MILLISECONDS_PER_TICK)) next_game_tick = cur_ticks;
+			}
 
 			/* The gameloop is the part that can run asynchronously. The rest
 			 * except sleeping can't. */
@@ -740,7 +746,9 @@ void VideoDriver_SDL::MainLoop()
 		}
 
 		if (cur_ticks >= next_draw_tick) {
-			next_draw_tick = cur_ticks + std::chrono::milliseconds(MILLISECONDS_PER_TICK);
+			next_draw_tick += std::chrono::milliseconds(MILLISECONDS_PER_TICK);
+			/* Avoid next_draw_tick getting behind more and more if it cannot keep up. */
+			if (next_draw_tick < cur_ticks - std::chrono::microseconds(ALLOWED_DRIFT * MILLISECONDS_PER_TICK)) next_draw_tick = cur_ticks;
 
 			bool old_ctrl_pressed = _ctrl_pressed;