cpp/openttd-patchpack/source Changeset - r23730:3b11f535de42

Changeset - r23730:3b11f535de42

Parent rev.

Child rev.

[Not reviewed]

master

0 14 0

Niels Martin Hansen - 5 years ago 2019-04-15 17:49:30
nielsm@indvikleren.dk

Change: Limit memory allocations for each Squirrel instance

This can avoid out-of-memory situations due to single scripts using up the entire address space.
Instead, scripts that go above the maximum are killed.
The maximum is default 1 GB per script, but can be configured by a setting.

14 files changed with 221 insertions and 5 deletions:

src/3rdparty/squirrel/squirrel/sqmem.cpp

src/ai/ai_instance.cpp

src/ai/ai_scanner.cpp

src/lang/english.txt

src/saveload/saveload.h

src/script/api/script_object.cpp

src/script/api/script_object.hpp

src/script/script_instance.cpp

src/script/script_instance.hpp

src/script/squirrel.cpp

147

src/script/squirrel.hpp

src/settings_gui.cpp

src/settings_type.h

src/table/settings.ini

0 comments (0 inline, 0 general)

src/3rdparty/squirrel/squirrel/sqmem.cpp

➞

Show inline comments

@@ @@ -9,8 +9,10 @@ @@
 #include "../../../core/alloc_func.hpp"
 #include "../../../safeguards.h"
 void *sq_vm_malloc(SQUnsignedInteger size){	return MallocT<char>((size_t)size); }
 #ifdef SQUIRREL_DEFAULT_ALLOCATOR
 void *sq_vm_malloc(SQUnsignedInteger size) { return MallocT<char>((size_t)size); }
 void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size){ return ReallocT<char>(static_cast<char*>(p), (size_t)size); }
+void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size) { return ReallocT<char>(static_cast<char*>(p), (size_t)size); }
 void sq_vm_free(void *p, SQUnsignedInteger size){	free(p); }
 void sq_vm_free(void *p, SQUnsignedInteger size) { free(p); }
 #endif

src/ai/ai_instance.cpp

➞

Show inline comments

@@ @@ -230,6 +230,7 @@ void AIInstance::Died() @@
 void AIInstance::LoadDummyScript()
+{
 	ScriptAllocatorScope alloc_scope(this->engine);
 	extern void Script_CreateDummy(HSQUIRRELVM vm, StringID string, const char *type);
 	Script_CreateDummy(this->engine->GetVM(), STR_ERROR_AI_NO_AI_FOUND, "AI");
+}

src/ai/ai_scanner.cpp

➞

Show inline comments

@@ @@ -31,6 +31,8 @@ void AIScannerInfo::Initialize() @@
+{
 	ScriptScanner::Initialize("AIScanner");
 	ScriptAllocatorScope alloc_scope(this->engine);
 	/* Create the dummy AI */
 	free(this->main_script);
 	this->main_script = stredup("%_dummy");

src/lang/english.txt

➞

Show inline comments

@@ @@ -1488,6 +1488,9 @@ STR_CONFIG_SETTING_AI_IN_MULTIPLAYER @@
 STR_CONFIG_SETTING_AI_IN_MULTIPLAYER_HELPTEXT                   :Allow AI computer players to participate in multiplayer games
 STR_CONFIG_SETTING_SCRIPT_MAX_OPCODES                           :#opcodes before scripts are suspended: {STRING2}
 STR_CONFIG_SETTING_SCRIPT_MAX_OPCODES_HELPTEXT                  :Maximum number of computation steps that a script can take in one turn
 STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY                            :Max memory usage per script: {STRING2}
 STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_HELPTEXT                   :How much memory a single script may consume before it's forcibly terminated. This may need to be increased for large maps.
 STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_VALUE                      :{COMMA} MiB
 STR_CONFIG_SETTING_SERVINT_ISPERCENT                            :Service intervals are in percents: {STRING2}
 STR_CONFIG_SETTING_SERVINT_ISPERCENT_HELPTEXT                   :Choose whether servicing of vehicles is triggered by the time passed since last service or by reliability dropping by a certain percentage of the maximum reliability

src/saveload/saveload.h

➞

Show inline comments

@@ @@ -300,6 +300,8 @@ enum SaveLoadVersion : uint16 { @@
 	SLV_TREES_WATER_CLASS,                  ///< 213  PR#7405 WaterClass update for tree tiles.
 	SLV_ROAD_TYPES,                         ///< 214  PR#6811 NewGRF road types.
 	SLV_SCRIPT_MEMLIMIT,                    ///< 215  PR#7516 Limit on AI/GS memory consumption.
 	SL_MAX_VERSION,                         ///< Highest possible saveload version
 };

src/script/api/script_object.cpp

➞

Show inline comments

@@ @@ -38,7 +38,7 @@ static ScriptStorage *GetStorage() @@
 /* static */ ScriptInstance *ScriptObject::ActiveInstance::active = nullptr;
 ScriptObject::ActiveInstance::ActiveInstance(ScriptInstance *instance)
+ScriptObject::ActiveInstance::ActiveInstance(ScriptInstance *instance) : alc_scope(instance->engine)
+{
 	this->last_active = ScriptObject::ActiveInstance::active;
 	ScriptObject::ActiveInstance::active = instance;

src/script/api/script_object.hpp

➞

Show inline comments

@@ @@ -18,6 +18,7 @@ @@
 #include "script_types.hpp"
 #include "../script_suspend.hpp"
 #include "../squirrel.hpp"
 /**
  * The callback function for Mode-classes.
@@ @@ -48,6 +49,7 @@ protected: @@
 		~ActiveInstance();
 	private:
 		ScriptInstance *last_active;    ///< The active instance before we go instantiated.
 		ScriptAllocatorScope alc_scope; ///< Keep the correct allocator for the script instance activated
 		static ScriptInstance *active;  ///< The global current active instance.
 	};

src/script/script_instance.cpp

➞

Show inline comments

@@ @@ -153,6 +153,8 @@ void ScriptInstance::Died() @@
 	DEBUG(script, 0, "The script died unexpectedly.");
 	this->is_dead = true;
 	this->last_allocated_memory = this->GetAllocatedMemory(); // Update cache
 	if (this->instance != nullptr) this->engine->ReleaseObject(this->instance);
 	delete this->engine;
 	this->instance = nullptr;
@@ @@ -698,3 +700,9 @@ void ScriptInstance::InsertEvent(class S @@
 	ScriptEventController::InsertEvent(event);
+}
 size_t ScriptInstance::GetAllocatedMemory() const
+{
 	if (this->engine == nullptr) return this->last_allocated_memory;
 	return this->engine->GetAllocatedMemory();
+}

src/script/script_instance.hpp

➞

Show inline comments

@@ @@ -198,6 +198,8 @@ public: @@
 	 */
 	bool IsSleeping() { return this->suspend != 0; }
 	size_t GetAllocatedMemory() const;
 protected:
 	class Squirrel *engine;               ///< A wrapper around the squirrel vm.
 	const char *versionAPI;               ///< Current API used by this script.
@@ @@ -241,6 +243,7 @@ private: @@
 	int suspend;                          ///< The amount of ticks to suspend this script before it's allowed to continue.
 	bool is_paused;                       ///< Is the script paused? (a paused script will not be executed until unpaused)
 	Script_SuspendCallbackProc *callback; ///< Callback that should be called in the next tick the script runs.
 	size_t last_allocated_memory;         ///< Last known allocated memory value (for display for crashed scripts)
 	/**
 	 * Call the script Load function if it exists and data was loaded

src/script/squirrel.cpp

➞

Show inline comments

@@ @@ -10,17 +10,127 @@ @@
 /** @file squirrel.cpp the implementation of the Squirrel class. It handles all Squirrel-stuff and gives a nice API back to work with. */
 #include <stdarg.h>
 #include <map>
 #include "../stdafx.h"
 #include "../debug.h"
 #include "squirrel_std.hpp"
 #include "../fileio_func.h"
 #include "../string_func.h"
 #include "script_fatalerror.hpp"
 #include "../settings_type.h"
 #include <sqstdaux.h>
 #include <../squirrel/sqpcheader.h>
 #include <../squirrel/sqvm.h>
 #include "../core/alloc_func.hpp"
 #include "../safeguards.h"
 /*
  * If changing the call paths into the scripting engine, define this symbol to enable full debugging of allocations.
  * This lets you track whether the allocator context is being switched correctly in all call paths.
 #define SCRIPT_DEBUG_ALLOCATIONS
 */
 struct ScriptAllocator {
 	size_t allocated_size;   ///< Sum of allocated data size
 	size_t allocation_limit; ///< Maximum this allocator may use before allocations fail
 	static const size_t SAFE_LIMIT = 0x8000000; ///< 128 MiB, a safe choice for almost any situation
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 	std::map<void *, size_t> allocations;
 #endif
 	void CheckLimit() const
+	{
 		if (this->allocated_size > this->allocation_limit) throw Script_FatalError("Maximum memory allocation exceeded");
+	}
 	void *Malloc(SQUnsignedInteger size)
+	{
 		void *p = MallocT<char>(size);
 		this->allocated_size += size;
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 		assert(p != nullptr);
 		assert(this->allocations.find(p) == this->allocations.end());
 		this->allocations[p] = size;
 #endif
 		return p;
+	}
 	void *Realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size)
+	{
 		if (p == nullptr) {
 			return this->Malloc(size);
+		}
 		if (size == 0) {
 			this->Free(p, oldsize);
 			return nullptr;
+		}
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 		assert(this->allocations[p] == oldsize);
 		this->allocations.erase(p);
 #endif
 		void *new_p = ReallocT<char>(static_cast<char *>(p), size);
 		this->allocated_size -= oldsize;
 		this->allocated_size += size;
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 		assert(new_p != nullptr);
 		assert(this->allocations.find(p) == this->allocations.end());
 		this->allocations[new_p] = size;
 #endif
 		return new_p;
+	}
 	void Free(void *p, SQUnsignedInteger size)
+	{
 		if (p == nullptr) return;
 		free(p);
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 		assert(this->allocations.at(p) == size);
 		this->allocations.erase(p);
 #endif
+	}
 	ScriptAllocator()
+	{
 		this->allocated_size = 0;
 		this->allocation_limit = static_cast<size_t>(_settings_game.script.script_max_memory_megabytes) << 20;
 		if (this->allocation_limit == 0) this->allocation_limit = SAFE_LIMIT; // in case the setting is somehow zero
+	}
 	~ScriptAllocator()
+	{
 #ifdef SCRIPT_DEBUG_ALLOCATIONS
 		assert(this->allocations.size() == 0);
 #endif
+	}
 };
 ScriptAllocator *_squirrel_allocator = nullptr;
 /* See 3rdparty/squirrel/squirrel/sqmem.cpp for the default allocator implementation, which this overrides */
 #ifndef SQUIRREL_DEFAULT_ALLOCATOR
 void *sq_vm_malloc(SQUnsignedInteger size) { return _squirrel_allocator->Malloc(size); }
 void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size) { return _squirrel_allocator->Realloc(p, oldsize, size); }
 void sq_vm_free(void *p, SQUnsignedInteger size) { _squirrel_allocator->Free(p, size); }
 #endif
 size_t Squirrel::GetAllocatedMemory() const noexcept
+{
 	assert(this->allocator != nullptr);
 	return this->allocator->allocated_size;
+}
 void Squirrel::CompileError(HSQUIRRELVM vm, const SQChar *desc, const SQChar *source, SQInteger line, SQInteger column)
+{
 	SQChar buf[1024];
@@ @@ -115,6 +225,8 @@ void Squirrel::PrintFunc(HSQUIRRELVM vm, @@
 void Squirrel::AddMethod(const char *method_name, SQFUNCTION proc, uint nparam, const char *params, void *userdata, int size)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushstring(this->vm, method_name, -1);
 	if (size != 0) {
@@ @@ -130,6 +242,8 @@ void Squirrel::AddMethod(const char *met @@
 void Squirrel::AddConst(const char *var_name, int value)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushstring(this->vm, var_name, -1);
 	sq_pushinteger(this->vm, value);
 	sq_newslot(this->vm, -3, SQTrue);
@@ @@ -137,6 +251,8 @@ void Squirrel::AddConst(const char *var_ @@
 void Squirrel::AddConst(const char *var_name, bool value)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushstring(this->vm, var_name, -1);
 	sq_pushbool(this->vm, value);
 	sq_newslot(this->vm, -3, SQTrue);
@@ @@ -144,6 +260,8 @@ void Squirrel::AddConst(const char *var_ @@
 void Squirrel::AddClassBegin(const char *class_name)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushroottable(this->vm);
 	sq_pushstring(this->vm, class_name, -1);
 	sq_newclass(this->vm, SQFalse);
@@ @@ -151,6 +269,8 @@ void Squirrel::AddClassBegin(const char @@
 void Squirrel::AddClassBegin(const char *class_name, const char *parent_class)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushroottable(this->vm);
 	sq_pushstring(this->vm, class_name, -1);
 	sq_pushstring(this->vm, parent_class, -1);
@@ @@ -164,6 +284,8 @@ void Squirrel::AddClassBegin(const char @@
 void Squirrel::AddClassEnd()
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_newslot(vm, -3, SQFalse);
 	sq_pop(vm, 1);
+}
@@ @@ -171,6 +293,8 @@ void Squirrel::AddClassEnd() @@
 bool Squirrel::MethodExists(HSQOBJECT instance, const char *method_name)
+{
 	assert(!this->crashed);
 	ScriptAllocatorScope alloc_scope(this);
 	int top = sq_gettop(this->vm);
 	/* Go to the instance-root */
 	sq_pushobject(this->vm, instance);
@@ @@ -187,6 +311,8 @@ bool Squirrel::MethodExists(HSQOBJECT in @@
 bool Squirrel::Resume(int suspend)
+{
 	assert(!this->crashed);
 	ScriptAllocatorScope alloc_scope(this);
 	/* Did we use more operations than we should have in the
 	 * previous tick? If so, subtract that from the current run. */
 	if (this->overdrawn_ops > 0 && suspend > 0) {
@@ @@ -200,23 +326,29 @@ bool Squirrel::Resume(int suspend) @@
 	this->crashed = !sq_resumecatch(this->vm, suspend);
 	this->overdrawn_ops = -this->vm->_ops_till_suspend;
 	this->allocator->CheckLimit();
 	return this->vm->_suspended != 0;
+}
 void Squirrel::ResumeError()
+{
 	assert(!this->crashed);
 	ScriptAllocatorScope alloc_scope(this);
 	sq_resumeerror(this->vm);
+}
 void Squirrel::CollectGarbage()
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_collectgarbage(this->vm);
+}
 bool Squirrel::CallMethod(HSQOBJECT instance, const char *method_name, HSQOBJECT *ret, int suspend)
+{
 	assert(!this->crashed);
 	ScriptAllocatorScope alloc_scope(this);
 	this->allocator->CheckLimit();
 	/* Store the stack-location for the return value. We need to
 	 * restore this after saving or the stack will be corrupted
 	 * if we're in the middle of a DoCommand. */
@@ @@ -325,17 +457,20 @@ bool Squirrel::CallBoolMethod(HSQOBJECT @@
 bool Squirrel::CreateClassInstance(const char *class_name, void *real_instance, HSQOBJECT *instance)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	return Squirrel::CreateClassInstanceVM(this->vm, class_name, real_instance, instance, nullptr);
+}
 Squirrel::Squirrel(const char *APIName) :
 	APIName(APIName)
+	APIName(APIName), allocator(new ScriptAllocator())
+{
 	this->Initialize();
+}
 void Squirrel::Initialize()
+{
 	ScriptAllocatorScope alloc_scope(this);
 	this->global_pointer = nullptr;
 	this->print_func = nullptr;
 	this->crashed = false;
@@ @@ -432,6 +567,8 @@ static SQInteger _io_file_read(SQUserPoi @@
 SQRESULT Squirrel::LoadFile(HSQUIRRELVM vm, const char *filename, SQBool printerror)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	FILE *file;
 	size_t size;
 	if (strncmp(this->GetAPIName(), "AI", 2) == 0) {
@@ @@ -517,6 +654,8 @@ SQRESULT Squirrel::LoadFile(HSQUIRRELVM @@
 bool Squirrel::LoadScript(HSQUIRRELVM vm, const char *script, bool in_root)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	/* Make sure we are always in the root-table */
 	if (in_root) sq_pushroottable(vm);
@@ @@ -549,6 +688,8 @@ Squirrel::~Squirrel() @@
 void Squirrel::Uninitialize()
+{
 	ScriptAllocatorScope alloc_scope(this);
 	/* Clean up the stuff */
 	sq_pop(this->vm, 1);
 	sq_close(this->vm);
@@ @@ -562,6 +703,8 @@ void Squirrel::Reset() @@
 void Squirrel::InsertResult(bool result)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushbool(this->vm, result);
 	if (this->IsSuspended()) { // Called before resuming a suspended script?
 		vm->GetAt(vm->_stackbase + vm->_suspended_target) = vm->GetUp(-1);
@@ @@ -571,6 +714,8 @@ void Squirrel::InsertResult(bool result) @@
 void Squirrel::InsertResult(int result)
+{
 	ScriptAllocatorScope alloc_scope(this);
 	sq_pushinteger(this->vm, result);
 	if (this->IsSuspended()) { // Called before resuming a suspended script?
 		vm->GetAt(vm->_stackbase + vm->_suspended_target) = vm->GetUp(-1);
@@ @@ -600,6 +745,7 @@ void Squirrel::CrashOccurred() @@
 bool Squirrel::CanSuspend()
+{
 	ScriptAllocatorScope alloc_scope(this);
 	return sq_can_suspend(this->vm);
+}

src/script/squirrel.hpp

➞

Show inline comments

@@ @@ -20,7 +20,11 @@ enum ScriptType { @@
 	ST_GS, ///< The script is for Game scripts.
 };
 struct ScriptAllocator;
 class Squirrel {
 	friend class ScriptAllocatorScope;
 private:
 	typedef void (SQPrintFunc)(bool error_msg, const SQChar *message);
@@ @@ -30,6 +34,7 @@ private: @@
 	bool crashed;            ///< True if the squirrel script made an error.
 	int overdrawn_ops;       ///< The amount of operations we have overdrawn.
 	const char *APIName;     ///< Name of the API used for this squirrel.
 	std::unique_ptr<ScriptAllocator> allocator; ///< Allocator object used by this script.
 	/**
 	 * The internal RunError handler. It looks up the real error and calls RunError with it.
@@ @@ -272,6 +277,31 @@ public: @@
 	 * Completely reset the engine; start from scratch.
 	 */
 	void Reset();
 	/**
 	 * Get number of bytes allocated by this VM.
 	 */
 	size_t GetAllocatedMemory() const noexcept;
 };
 extern ScriptAllocator *_squirrel_allocator;
 class ScriptAllocatorScope {
 	ScriptAllocator *old_allocator;
 public:
 	ScriptAllocatorScope(const Squirrel *engine)
+	{
 		this->old_allocator = _squirrel_allocator;
 		/* This may get called with a nullptr engine, in case of a crashed script */
 		_squirrel_allocator = engine != nullptr ? engine->allocator.get() : nullptr;
+	}
 	~ScriptAllocatorScope()
+	{
 		_squirrel_allocator = this->old_allocator;
+	}
 };
 #endif /* SQUIRREL_HPP */

src/settings_gui.cpp

➞

Show inline comments

@@ @@ -1757,6 +1757,7 @@ static SettingsContainer &GetSettingsTre @@
+			{
 				npc->Add(new SettingEntry("script.settings_profile"));
 				npc->Add(new SettingEntry("script.script_max_opcode_till_suspend"));
 				npc->Add(new SettingEntry("script.script_max_memory_megabytes"));
 				npc->Add(new SettingEntry("difficulty.competitor_speed"));
 				npc->Add(new SettingEntry("ai.ai_in_multiplayer"));
 				npc->Add(new SettingEntry("ai.ai_disable_veh_train"));

src/settings_type.h

➞

Show inline comments

@@ @@ -339,6 +339,7 @@ struct AISettings { @@
 struct ScriptSettings {
 	uint8  settings_profile;                 ///< difficulty profile to set initial settings of scripts, esp. random AIs
 	uint32 script_max_opcode_till_suspend;   ///< max opcode calls till scripts will suspend
 	uint32 script_max_memory_megabytes;      ///< limit on memory a single script instance may have allocated
 };
 /** Settings related to the new pathfinder. */

src/table/settings.ini

➞

Show inline comments

@@ @@ -1555,6 +1555,21 @@ strhelp = STR_CONFIG_SETTING_SCRIPT_MAX @@
 strval   = STR_JUST_COMMA
 cat      = SC_EXPERT
 [SDT_VAR]
 base     = GameSettings
 var      = script.script_max_memory_megabytes
 type     = SLE_UINT32
 from     = SLV_SCRIPT_MEMLIMIT
 guiflags = SGF_NEWGAME_ONLY
 def      = 1024
 min      = 8
 max      = 8192
 interval = 8
 str      = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY
 strhelp  = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_HELPTEXT
 strval   = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_VALUE
 cat      = SC_EXPERT
 ##
 [SDT_VAR]
 base     = GameSettings

0 comments (0 inline, 0 general)