diff --git a/network_client.c b/network_client.c
--- a/network_client.c
+++ b/network_client.c
@@ -508,12 +508,6 @@ DEF_CLIENT_RECEIVE_COMMAND(PACKET_SERVER
 			_local_player = _network_playas - 1;
 			DeleteWindowById(WC_NETWORK_STATUS_WINDOW, 0);
 		}
-
-		// Remeber the player
-		if (_local_player != OWNER_SPECTATOR)
-			_network_playas = _local_player + 1;
-		else
-			_network_playas = OWNER_SPECTATOR;
 	}
 
 	return NETWORK_RECV_STATUS_OKAY;
diff --git a/players.c b/players.c
--- a/players.c
+++ b/players.c
@@ -653,8 +653,11 @@ int32 CmdPlayerCtrl(int x, int y, uint32
 
 		if (p != NULL) {
 			if (_local_player == OWNER_SPECTATOR) {
-				_local_player = p->index;
-				MarkWholeScreenDirty();
+				/* Check if we do not want to be a spectator in network */
+				if (!_networking || _network_server || _network_playas != OWNER_SPECTATOR) {
+					_local_player = p->index;
+					MarkWholeScreenDirty();
+				}
 			}
 #ifdef ENABLE_NETWORK
 			if (_network_server) {