diff --git a/src/saveload/saveload.cpp b/src/saveload/saveload.cpp --- a/src/saveload/saveload.cpp +++ b/src/saveload/saveload.cpp @@ -52,6 +52,7 @@ #include "table/strings.h" +#include "saveload_buffer.h" #include "saveload_internal.h" #include "saveload_filter.h" @@ -59,6 +60,8 @@ extern const SaveLoadVersion SAVEGAME_VERSION = (SaveLoadVersion)(SL_MAX_VERSION - 1); ///< Current savegame version of OpenTTD. +const SaveLoadVersion SAVEGAME_VERSION_EXT = (SaveLoadVersion)(0x8000); ///< Savegame extension indicator mask + SavegameType _savegame_type; ///< type of savegame we are loading FileToSaveLoad _file_to_saveload; ///< File to save or load in the openttd loop. @@ -68,6 +71,11 @@ byte _sl_minor_version; ///< the m std::string _savegame_format; ///< how to compress savegames bool _do_autosave; ///< are we doing an autosave at the moment? +extern bool _sl_is_ext_version; +extern bool _sl_might_be_legacy_patchpack_save; +extern SaveLoadVersion _sl_is_faked_ext; +extern std::string _sl_xv_version_label; + /** What are we currently doing? */ enum SaveLoadAction { SLA_LOAD, ///< loading @@ -83,113 +91,6 @@ enum NeedLength { NL_CALCLENGTH = 2, ///< need to calculate the length }; -/** Save in chunks of 128 KiB. */ -static const size_t MEMORY_CHUNK_SIZE = 128 * 1024; - -/** A buffer for reading (and buffering) savegame data. */ -struct ReadBuffer { - byte buf[MEMORY_CHUNK_SIZE]; ///< Buffer we're going to read from. - byte *bufp; ///< Location we're at reading the buffer. - byte *bufe; ///< End of the buffer we can read from. - LoadFilter *reader; ///< The filter used to actually read. - size_t read; ///< The amount of read bytes so far from the filter. - - /** - * Initialise our variables. - * @param reader The filter to actually read data. - */ - ReadBuffer(LoadFilter *reader) : bufp(nullptr), bufe(nullptr), reader(reader), read(0) - { - } - - inline byte ReadByte() - { - if (this->bufp == this->bufe) { - size_t len = this->reader->Read(this->buf, lengthof(this->buf)); - if (len == 0) SlErrorCorrupt("Unexpected end of chunk"); - - this->read += len; - this->bufp = this->buf; - this->bufe = this->buf + len; - } - - return *this->bufp++; - } - - /** - * Get the size of the memory dump made so far. - * @return The size. - */ - size_t GetSize() const - { - return this->read - (this->bufe - this->bufp); - } -}; - - -/** Container for dumping the savegame (quickly) to memory. */ -struct MemoryDumper { - std::vector blocks; ///< Buffer with blocks of allocated memory. - byte *buf; ///< Buffer we're going to write to. - byte *bufe; ///< End of the buffer we write to. - - /** Initialise our variables. */ - MemoryDumper() : buf(nullptr), bufe(nullptr) - { - } - - ~MemoryDumper() - { - for (auto p : this->blocks) { - free(p); - } - } - - /** - * Write a single byte into the dumper. - * @param b The byte to write. - */ - inline void WriteByte(byte b) - { - /* Are we at the end of this chunk? */ - if (this->buf == this->bufe) { - this->buf = CallocT(MEMORY_CHUNK_SIZE); - this->blocks.push_back(this->buf); - this->bufe = this->buf + MEMORY_CHUNK_SIZE; - } - - *this->buf++ = b; - } - - /** - * Flush this dumper into a writer. - * @param writer The filter we want to use. - */ - void Flush(SaveFilter *writer) - { - uint i = 0; - size_t t = this->GetSize(); - - while (t > 0) { - size_t to_write = std::min(MEMORY_CHUNK_SIZE, t); - - writer->Write(this->blocks[i++], to_write); - t -= to_write; - } - - writer->Finish(); - } - - /** - * Get the size of the memory dump made so far. - * @return The size. - */ - size_t GetSize() const - { - return this->blocks.size() * MEMORY_CHUNK_SIZE - (this->bufe - this->buf); - } -}; - /** The saveload struct, containing reader-writer functions, buffer, version, etc. */ struct SaveLoadParams { SaveLoadAction action; ///< are we doing a save or a load atm. @@ -216,9 +117,127 @@ struct SaveLoadParams { static SaveLoadParams _sl; ///< Parameters used for/at saveload. +void ReadBuffer::SkipBytesSlowPath(size_t bytes) +{ + bytes -= (this->bufe - this->bufp); + while (true) { + size_t len = this->reader->Read(this->buf, lengthof(this->buf)); + if (len == 0) SlErrorCorrupt("Unexpected end of chunk"); + this->read += len; + if (len >= bytes) { + this->bufp = this->buf + bytes; + this->bufe = this->buf + len; + return; + } + else { + bytes -= len; + } + } +} + +void ReadBuffer::AcquireBytes() +{ + size_t remainder = this->bufe - this->bufp; + if (remainder) { + memmove(this->buf, this->bufp, remainder); + } + size_t len = this->reader->Read(this->buf + remainder, lengthof(this->buf) - remainder); + if (len == 0) SlErrorCorrupt("Unexpected end of chunk"); + + this->read += len; + this->bufp = this->buf; + this->bufe = this->buf + remainder + len; +} + +void MemoryDumper::FinaliseBlock() +{ + assert(this->saved_buf == nullptr); + if (!this->blocks.empty()) { + size_t s = MEMORY_CHUNK_SIZE - (this->bufe - this->buf); + this->blocks.back().size = s; + this->completed_block_bytes += s; + } + this->buf = this->bufe = nullptr; +} + +void MemoryDumper::AllocateBuffer() +{ + if (this->saved_buf) { + const size_t offset = this->buf - this->autolen_buf; + const size_t size = (this->autolen_buf_end - this->autolen_buf) * 2; + this->autolen_buf = ReallocT(this->autolen_buf, size); + this->autolen_buf_end = this->autolen_buf + size; + this->buf = this->autolen_buf + offset; + this->bufe = this->autolen_buf_end; + return; + } + this->FinaliseBlock(); + this->buf = MallocT(MEMORY_CHUNK_SIZE); + this->blocks.emplace_back(this->buf); + this->bufe = this->buf + MEMORY_CHUNK_SIZE; +} + +/** + * Flush this dumper into a writer. + * @param writer The filter we want to use. + */ +void MemoryDumper::Flush(SaveFilter* writer) +{ + this->FinaliseBlock(); + + size_t block_count = this->blocks.size(); + for (size_t i = 0; i < block_count; i++) { + writer->Write(this->blocks[i].data, this->blocks[i].size); + } + + writer->Finish(); +} + +void MemoryDumper::StartAutoLength() +{ + assert(this->saved_buf == nullptr); + + this->saved_buf = this->buf; + this->saved_bufe = this->bufe; + this->buf = this->autolen_buf; + this->bufe = this->autolen_buf_end; +} + +std::pair MemoryDumper::StopAutoLength() +{ + assert(this->saved_buf != nullptr); + auto res = std::make_pair(this->autolen_buf, this->buf - this->autolen_buf); + + this->buf = this->saved_buf; + this->bufe = this->saved_bufe; + this->saved_buf = this->saved_bufe = nullptr; + return res; +} + +/** + * Get the size of the memory dump made so far. + * @return The size. + */ +size_t MemoryDumper::GetSize() const +{ + assert(this->saved_buf == nullptr); + return this->completed_block_bytes + (this->bufe ? (MEMORY_CHUNK_SIZE - (this->bufe - this->buf)) : 0); +} + +ReadBuffer* ReadBuffer::GetCurrent() +{ + return _sl.reader; +} + +MemoryDumper* MemoryDumper::GetCurrent() +{ + return _sl.dumper; +} + static const std::vector &ChunkHandlers() { /* These define the chunks */ + extern const ChunkHandlerTable _version_ext_chunk_handlers; extern const ChunkHandlerTable _gamelog_chunk_handlers; extern const ChunkHandlerTable _map_chunk_handlers; extern const ChunkHandlerTable _misc_chunk_handlers; @@ -255,6 +274,7 @@ static const std::vector> 24, ch.id >> 16, ch.id >> 8, ch.id); @@ -435,41 +456,60 @@ void SlWriteByte(byte b) _sl.dumper->WriteByte(b); } -static inline int SlReadUint16() +int SlReadUint16() { - int x = SlReadByte() << 8; - return x | SlReadByte(); + _sl.reader->CheckBytes(2); + return _sl.reader->RawReadUint16(); } -static inline uint32 SlReadUint32() +uint32 SlReadUint32() { - uint32 x = SlReadUint16() << 16; - return x | SlReadUint16(); + _sl.reader->CheckBytes(4); + return _sl.reader->RawReadUint32(); } -static inline uint64 SlReadUint64() +uint64 SlReadUint64() { - uint32 x = SlReadUint32(); - uint32 y = SlReadUint32(); - return (uint64)x << 32 | y; + _sl.reader->CheckBytes(8); + return _sl.reader->RawReadUint64(); +} + +void SlWriteUint16(uint16 v) +{ + _sl.dumper->CheckBytes(2); + _sl.dumper->RawWriteUint16(v); } -static inline void SlWriteUint16(uint16 v) +void SlWriteUint32(uint32 v) { - SlWriteByte(GB(v, 8, 8)); - SlWriteByte(GB(v, 0, 8)); + _sl.dumper->CheckBytes(4); + _sl.dumper->RawWriteUint32(v); } -static inline void SlWriteUint32(uint32 v) +void SlWriteUint64(uint64 x) { - SlWriteUint16(GB(v, 16, 16)); - SlWriteUint16(GB(v, 0, 16)); + _sl.dumper->CheckBytes(8); + _sl.dumper->RawWriteUint64(x); } -static inline void SlWriteUint64(uint64 x) +/** + * Returns number of bytes read so far + * May only be called during a load/load check action + */ +size_t SlGetBytesRead() { - SlWriteUint32((uint32)(x >> 32)); - SlWriteUint32((uint32)x); + assert(_sl.action == SLA_LOAD || _sl.action == SLA_LOAD_CHECK); + return _sl.reader->GetSize(); +} + +/** + * Returns number of bytes written so far + * May only be called during a save action + */ +size_t SlGetBytesWritten() +{ + assert(_sl.action == SLA_SAVE); + return _sl.dumper->GetSize(); } /** @@ -1060,6 +1100,9 @@ static void SlStdString(void *ptr, VarTy case SLA_LOAD_CHECK: case SLA_LOAD: { size_t len = SlReadArrayLength(); + if (len > 65535) { + SlErrorCorrupt("String too long"); + } if (GetVarMemType(conv) == SLE_VAR_NULL) { SlSkipBytes(len); return; @@ -1554,7 +1597,7 @@ static void SlVector(void *vector, VarTy /** Are we going to save this object or not? */ static inline bool SlIsObjectValidInSavegame(const SaveLoad &sld) { - return (_sl_version >= sld.version_from && _sl_version < sld.version_to); + return sld.ext_feature_test.IsFeaturePresent(_sl_version, sld.version_from, sld.version_to); } /** @@ -2048,7 +2091,7 @@ std::vector SlCompatTableHeade /* In old savegames there can be data we no longer care for. We * skip this by simply reading the amount of bytes indicated and * send those to /dev/null. */ - saveloads.push_back({"", SL_NULL, SLE_FILE_U8 | SLE_VAR_NULL, slc.length, slc.version_from, slc.version_to, 0, nullptr, 0, nullptr}); + saveloads.push_back({"", SL_NULL, SLE_FILE_U8 | SLE_VAR_NULL, slc.length, slc.version_from, slc.version_to, 0, nullptr, 0, nullptr, slc.ext_feature_test}); } else { auto sld_it = key_lookup.find(slc.name); /* If this branch triggers, it means that an entry in the @@ -2061,6 +2104,7 @@ std::vector SlCompatTableHeade SlErrorCorrupt("Internal error with savegame compatibility"); } for (auto &sld : sld_it->second) { + if (!slc.ext_feature_test.IsFeaturePresent(_sl_version, SL_MIN_VERSION, SL_MAX_VERSION)) continue; saveloads.push_back(*sld); } } @@ -2186,7 +2230,7 @@ static void SlLoadChunk(const ChunkHandl * If the chunkhandler is nullptr, the chunk is skipped. * @param ch The chunkhandler that will be used for the operation */ -static void SlLoadCheckChunk(const ChunkHandler &ch) +static void SlLoadCheckChunk(const ChunkHandler* ch) { byte m = SlReadByte(); size_t len; @@ -2206,11 +2250,25 @@ static void SlLoadCheckChunk(const Chunk case CH_TABLE: case CH_ARRAY: _sl.array_index = 0; - ch.LoadCheck(); + if (ch) + { + ch->LoadCheck(); + } + else + { + SlSkipArray(); + } break; case CH_SPARSE_TABLE: case CH_SPARSE_ARRAY: - ch.LoadCheck(); + if (ch) + { + ch->LoadCheck(); + } + else + { + SlSkipArray(); + } break; case CH_RIFF: /* Read length */ @@ -2218,7 +2276,13 @@ static void SlLoadCheckChunk(const Chunk len += SlReadUint16(); _sl.obj_len = len; endoffs = _sl.reader->GetSize() + len; - ch.LoadCheck(len); + if (ch) { + ch->LoadCheck(len); + } + else + { + SlSkipBytes(len); + } if (_sl.reader->GetSize() != endoffs) SlErrorCorrupt("Invalid chunk size"); break; default: @@ -2230,6 +2294,25 @@ static void SlLoadCheckChunk(const Chunk } /** + * Load a chunk of data for checking savegames. + * @param ch The chunkhandler that will be used for the operation + */ +static void SlLoadCheckChunk(const ChunkHandler &ch) +{ + SlLoadCheckChunk(&ch); +} + +/** + * Skip unwanted chunk + * @param id Chunk ID + */ +static void SlLoadSkipChunk(uint32 id) +{ + Debug(sl, 1, "Discarding chunk {:c}{:c}{:c}{:c}", id >> 24, id >> 16, id >> 8, id); + SlLoadCheckChunk(nullptr); +} + +/** * Save a chunk of data (eg. vehicles, stations, etc.). Each chunk is * prefixed by an ID identifying it, followed by data, and terminator where appropriate * @param ch The chunkhandler that will be used for the operation @@ -2302,8 +2385,17 @@ static void SlLoadChunks() Debug(sl, 2, "Loading chunk {:c}{:c}{:c}{:c}", id >> 24, id >> 16, id >> 8, id); ch = SlFindChunkHandler(id); - if (ch == nullptr) SlErrorCorrupt("Unknown chunk type"); - SlLoadChunk(*ch); + if (ch == nullptr) { + if (SlXvIsChunkDiscardable(id)) { + SlLoadSkipChunk(id); + } + else { + SlErrorCorrupt("Unknown chunk type"); + } + } + else { + SlLoadChunk(*ch); + } } } @@ -2317,8 +2409,17 @@ static void SlLoadCheckChunks() Debug(sl, 2, "Loading chunk {:c}{:c}{:c}{:c}", id >> 24, id >> 16, id >> 8, id); ch = SlFindChunkHandler(id); - if (ch == nullptr) SlErrorCorrupt("Unknown chunk type"); - SlLoadCheckChunk(*ch); + if (ch == nullptr) { + if (SlXvIsChunkDiscardable(id)) { + SlLoadSkipChunk(id); + } + else { + SlErrorCorruptFmt("Unknown chunk type %c%c%c%c", id >> 24, id >> 16, id >> 8, id); + } + } + else { + SlLoadCheckChunk(*ch); + } } } @@ -2976,7 +3077,7 @@ static SaveOrLoadResult SaveFileToDisk(b const SaveLoadFormat *fmt = GetSavegameFormat(_savegame_format, &compression); /* We have written our stuff to memory, now write it to file! */ - uint32 hdr[2] = { fmt->tag, TO_BE32(SAVEGAME_VERSION << 16) }; + uint32 hdr[2] = { fmt->tag, TO_BE32((uint32)(SAVEGAME_VERSION | SAVEGAME_VERSION_EXT) << 16) }; _sl.sf->Write((byte*)hdr, sizeof(hdr)); _sl.sf = fmt->init_write(_sl.sf, compression); @@ -3035,6 +3136,7 @@ static SaveOrLoadResult DoSave(SaveFilte _sl.sf = writer; _sl_version = SAVEGAME_VERSION; + SlXvSetCurrentState(); SaveViewportBeforeSaveGame(); SlSaveChunks(); @@ -3087,6 +3189,8 @@ static SaveOrLoadResult DoLoad(LoadFilte _load_check_data.checkable = true; } + SlXvResetState(); + uint32 hdr[2]; if (_sl.lf->Read((byte*)hdr, sizeof(hdr)) != sizeof(hdr)) SlError(STR_GAME_SAVELOAD_ERROR_FILE_NOT_READABLE); @@ -3099,6 +3203,7 @@ static SaveOrLoadResult DoLoad(LoadFilte _sl.lf->Reset(); _sl_version = SL_MIN_VERSION; _sl_minor_version = 0; + SlXvResetState(); /* Try to find the LZO savegame format; it uses 'OTTD' as tag. */ fmt = _saveload_formats; @@ -3121,11 +3226,18 @@ static SaveOrLoadResult DoLoad(LoadFilte * Therefore it is loaded, but never saved (or, it saves a 0 in any scenario). */ _sl_minor_version = (TO_BE32(hdr[1]) >> 8) & 0xFF; - Debug(sl, 1, "Loading savegame version {}", _sl_version); + if (_sl_version & SAVEGAME_VERSION_EXT) { + _sl_version = (SaveLoadVersion)(_sl_version & ~SAVEGAME_VERSION_EXT); + _sl_is_ext_version = true; + } else if (_settings_client.gui.load_legacy_patchpack_savedata ) { + SlXvCheckSpecialSavegameVersionsA(); + } + + Debug(sl, 1, "Loading savegame version {}{}", _sl_version, _sl_is_ext_version ? " (extended)" : ""); /* Is the version higher than the current? */ if (_sl_version > SAVEGAME_VERSION) SlError(STR_GAME_SAVELOAD_ERROR_TOO_NEW_SAVEGAME); - if (_sl_version >= SLV_START_PATCHPACKS && _sl_version <= SLV_END_PATCHPACKS) SlError(STR_GAME_SAVELOAD_ERROR_PATCHPACK); + if (_sl_version >= SLV_START_PATCHPACKS && _sl_version < SLV_END_PATCHPACKS && !_sl_might_be_legacy_patchpack_save) SlError(STR_GAME_SAVELOAD_ERROR_PATCHPACK); break; } @@ -3182,6 +3294,8 @@ static SaveOrLoadResult DoLoad(LoadFilte if (load_check) { /* Load chunks into _load_check_data. * No pools are loaded. References are not possible, and thus do not need resolving. */ + _load_check_data.save_version = _sl_is_faked_ext != SL_MIN_VERSION ? _sl_is_faked_ext : _sl_version; + _load_check_data.save_ext_type = _sl_is_ext_version ? PSXT_EXTENDED : PSXT_NONE; SlLoadCheckChunks(); } else { /* Load chunks and resolve references */ @@ -3207,6 +3321,7 @@ static SaveOrLoadResult DoLoad(LoadFilte } GamelogStopAction(); + SlXvSetCurrentState(); } return SL_OK; @@ -3263,12 +3378,14 @@ SaveOrLoadResult SaveOrLoad(const std::s if (!LoadOldSaveGame(filename)) return SL_REINIT; _sl_version = SL_MIN_VERSION; _sl_minor_version = 0; + SlXvResetState(); GamelogStartAction(GLAT_LOAD); if (!AfterLoadGame()) { GamelogStopAction(); return SL_REINIT; } GamelogStopAction(); + SlXvSetCurrentState(); return SL_OK; }