Changeset - r27221:3735cc2cef6b
[Not reviewed]
master
0 17 5
Patric Stout - 12 months ago 2023-04-30 18:37:40
truebrain@openttd.org
Feature: drop ICU-lx in favour of directly interfacing with harfbuzz

This means we have RTL support again with ICU 58+. It makes use of:
- ICU for bidi-itemization
- ICU for script-itemization
- OpenTTD for style-itemization
- harfbuzz for shaping
22 files changed with 1018 insertions and 190 deletions:
0 comments (0 inline, 0 general)
.github/workflows/ci-build.yml
Show inline comments
 
@@ -110,12 +110,13 @@ jobs:
 

	
 
        echo "::group::Install dependencies"
 
        sudo apt-get install -y --no-install-recommends \
 
          liballegro4-dev \
 
          libcurl4-openssl-dev \
 
          libfontconfig-dev \
 
          libharfbuzz-dev \
 
          libicu-dev \
 
          liblzma-dev \
 
          liblzo2-dev \
 
          ${{ matrix.libraries }} \
 
          zlib1g-dev \
 
          # EOF
.github/workflows/release-linux.yml
Show inline comments
 
@@ -31,21 +31,26 @@ jobs:
 
        restore-keys: |
 
          ubuntu-20.04-vcpkg-release
 

	
 
    - name: Install dependencies
 
      run: |
 
        echo "::group::Install system dependencies"
 
        # ICU is used as vcpkg fails to install ICU. Other dependencies
 
        # are needed either for vcpkg or for the packages installed with
 
        # vcpkg.
 
        # perl-IPC-Cmd, wget, and zip are needed to run vcpkg.
 
        # autoconf-archive is needed to build ICU.
 
        yum install -y \
 
          libicu-devel \
 
          autoconf-archive \
 
          perl-IPC-Cmd \
 
          wget \
 
          zip \
 
          # EOF
 

	
 
        # aclocal looks first in /usr/local/share/aclocal, and if that doesn't
 
        # exist only looks in /usr/share/aclocal. We have files in both that
 
        # are important. So copy the latter to the first, and we are good to
 
        # go.
 
        cp /usr/share/aclocal/* /usr/local/share/aclocal/
 
        echo "::endgroup::"
 

	
 
        # We use vcpkg for our dependencies, to get more up-to-date version.
 
        echo "::group::Install vcpkg and dependencies"
 

	
 
        # We do a little dance to make sure we copy the cached install folder
 
@@ -66,12 +71,14 @@ jobs:
 
          ln -sf $(pwd)/installed/x64-linux/tools/python3/python3.[0-9][0-9] /usr/bin/python3
 

	
 
          ./vcpkg install \
 
            curl[http2] \
 
            fontconfig \
 
            freetype \
 
            harfbuzz \
 
            icu \
 
            liblzma \
 
            libpng \
 
            lzo \
 
            sdl2 \
 
            zlib \
 
            # EOF
CMakeLists.txt
Show inline comments
 
@@ -140,13 +140,14 @@ if(NOT OPTION_DEDICATED)
 
            find_package(SDL2)
 
            if(NOT SDL2_FOUND)
 
                find_package(SDL)
 
            endif()
 
            find_package(Fluidsynth)
 
            find_package(Fontconfig)
 
            find_package(ICU OPTIONAL_COMPONENTS i18n lx)
 
            find_package(Harfbuzz)
 
            find_package(ICU OPTIONAL_COMPONENTS i18n)
 
        endif()
 
    endif()
 
endif()
 
if(APPLE)
 
    find_package(Iconv)
 

	
 
@@ -175,12 +176,18 @@ check_ipo_supported(RESULT IPO_FOUND)
 
show_options()
 

	
 
if(UNIX AND NOT APPLE AND NOT OPTION_DEDICATED)
 
    if(NOT SDL_FOUND AND NOT SDL2_FOUND AND NOT ALLEGRO_FOUND)
 
        message(FATAL_ERROR "SDL, SDL2 or Allegro is required for this platform")
 
    endif()
 
    if(HARFBUZZ_FOUND AND NOT ICU_i18n_FOUND)
 
        message(WARNING "HarfBuzz depends on ICU i18n to function; HarfBuzz will be disabled")
 
    endif()
 
    if(NOT HARFBUZZ_FOUND)
 
        message(WARNING "Without HarfBuzz and ICU i18n the game will not be able to render right-to-left languages correctly")
 
    endif()
 
endif()
 
if(APPLE)
 
    if(NOT AUDIOTOOLBOX_LIBRARY)
 
        message(FATAL_ERROR "AudioToolbox is required for this platform")
 
    endif()
 
    if(NOT AUDIOUNIT_LIBRARY)
 
@@ -286,13 +293,13 @@ if(NOT OPTION_DEDICATED)
 
    link_package(Fluidsynth)
 
    link_package(SDL)
 
    link_package(SDL2 TARGET SDL2::SDL2)
 
    link_package(Allegro)
 
    link_package(FREETYPE TARGET Freetype::Freetype)
 
    link_package(Fontconfig TARGET Fontconfig::Fontconfig)
 
    link_package(ICU_lx)
 
    link_package(Harfbuzz TARGET harfbuzz::harfbuzz)
 
    link_package(ICU_i18n)
 

	
 
    if(SDL2_FOUND AND OPENGL_FOUND AND UNIX)
 
        # SDL2 dynamically loads OpenGL if needed, so do not link to OpenGL when
 
        # on Linux. For Windows, we need to link to OpenGL as we also have a win32
 
        # driver using it.
COMPILING.md
Show inline comments
 
@@ -13,12 +13,13 @@ OpenTTD makes use of the following exter
 
For Linux, the following additional libraries are used:
 

	
 
- (encouraged) libcurl: content downloads
 
- libSDL2: hardware access (video, sound, mouse)
 
- libfreetype: loading generic fonts and rendering them
 
- libfontconfig: searching for fonts, resolving font names to actual fonts
 
- harfbuzz: handling of right-to-left scripts (e.g. Arabic and Persian) (required libicu)
 
- libicu: handling of right-to-left scripts (e.g. Arabic and Persian) and
 
   natural sorting of strings
 

	
 
If you are building a dedicated-server only, you don't need the last four.
 

	
 
OpenTTD does not require any of the libraries to be present, but without
Doxyfile.in
Show inline comments
 
@@ -289,14 +289,14 @@ PREDEFINED             = WITH_ZLIB \
 
                         WITH_LZO \
 
                         WITH_LIBLZMA \
 
                         WITH_SDL \
 
                         WITH_PNG \
 
                         WITH_FONTCONFIG \
 
                         WITH_FREETYPE \
 
                         WITH_HARFBUZZ \
 
                         WITH_ICU_I18N \
 
                         WITH_ICU_LX \
 
                         UNICODE \
 
                         _UNICODE \
 
                         _GNU_SOURCE \
 
                         FINAL=
 
EXPAND_AS_DEFINED      =
 
SKIP_FUNCTION_MACROS   = YES
README.md
Show inline comments
 
@@ -180,9 +180,12 @@ The exact licensing terms can be found i
 
The fmt implementation in `src/3rdparty/fmt` is licensed under the MIT license.
 
See `src/3rdparty/fmt/LICENSE.rst` for the complete license text.
 

	
 
The catch2 implementation in `src/3rdparty/catch2` is licensed under the Boost Software License, Version 1.0.
 
See `src/3rdparty/catch2/LICENSE.txt` for the complete license text.
 

	
 
The icu scriptrun implementation in `src/3rdparty/icu` is licensed under the Unicode license.
 
See `src/3rdparty/icu/LICENSE` for the complete license text.
 

	
 
## 4.0 Credits
 

	
 
See [CREDITS.md](./CREDITS.md)
cmake/FindHarfbuzz.cmake
Show inline comments
 
new file 100644
 
#[=======================================================================[.rst:
 
FindHarfBuzz
 
-------
 

	
 
Finds the harfbuzz library.
 

	
 
Result Variables
 
^^^^^^^^^^^^^^^^
 

	
 
This will define the following variables:
 

	
 
``Harfbuzz_FOUND``
 
  True if the system has the harfbuzz library.
 
``Harfbuzz_INCLUDE_DIRS``
 
  Include directories needed to use harfbuzz.
 
``Harfbuzz_LIBRARIES``
 
  Libraries needed to link to harfbuzz.
 
``Harfbuzz_VERSION``
 
  The version of the harfbuzz library which was found.
 

	
 
Cache Variables
 
^^^^^^^^^^^^^^^
 

	
 
The following cache variables may also be set:
 

	
 
``Harfbuzz_INCLUDE_DIR``
 
  The directory containing ``hb.h``.
 
``Harfbuzz_LIBRARY``
 
  The path to the harfbuzz library.
 

	
 
#]=======================================================================]
 

	
 
find_package(PkgConfig QUIET)
 
pkg_check_modules(PC_Harfbuzz QUIET harfbuzz)
 

	
 
find_path(Harfbuzz_INCLUDE_DIR
 
    NAMES hb.h
 
    PATHS ${PC_Harfbuzz_INCLUDE_DIRS}
 
)
 

	
 
find_library(Harfbuzz_LIBRARY
 
    NAMES harfbuzz
 
    PATHS ${PC_Harfbuzz_LIBRARY_DIRS}
 
)
 

	
 
set(Harfbuzz_VERSION ${PC_Harfbuzz_VERSION})
 

	
 
include(FindPackageHandleStandardArgs)
 
find_package_handle_standard_args(Harfbuzz
 
    FOUND_VAR Harfbuzz_FOUND
 
    REQUIRED_VARS
 
        Harfbuzz_LIBRARY
 
        Harfbuzz_INCLUDE_DIR
 
    VERSION_VAR Harfbuzz_VERSION
 
)
 

	
 
if(Harfbuzz_FOUND)
 
    set(Harfbuzz_LIBRARIES ${Harfbuzz_LIBRARY})
 
    set(Harfbuzz_INCLUDE_DIRS ${Harfbuzz_INCLUDE_DIR})
 
endif()
 

	
 
mark_as_advanced(
 
    Harfbuzz_INCLUDE_DIR
 
    Harfbuzz_LIBRARY
 
)
cmake/FindICU.cmake
Show inline comments
 
@@ -6,13 +6,13 @@
 
#[=======================================================================[.rst:
 
FindICU
 
-------
 

	
 
Finds components of the ICU library.
 

	
 
Accepted components are: uc, i18n, le, lx, io
 
Accepted components are: uc, i18n, le, lx, io, data
 

	
 
Result Variables
 
^^^^^^^^^^^^^^^^
 

	
 
This will define the following variables:
 

	
 
@@ -28,13 +28,13 @@ This will define the following variables
 
  Libraries needed to link to the <c> component of ICU library.
 

	
 
#]=======================================================================]
 

	
 
find_package(PkgConfig QUIET)
 

	
 
set(ICU_KNOWN_COMPONENTS "uc" "i18n" "le" "lx" "io")
 
set(ICU_KNOWN_COMPONENTS "uc" "i18n" "le" "lx" "io" "data")
 

	
 
foreach(MOD_NAME IN LISTS ICU_FIND_COMPONENTS)
 
    if(NOT MOD_NAME IN_LIST ICU_KNOWN_COMPONENTS)
 
        message(FATAL_ERROR "Unknown ICU component: ${MOD_NAME}")
 
    endif()
 
    pkg_check_modules(PC_ICU_${MOD_NAME} QUIET icu-${MOD_NAME})
src/3rdparty/CMakeLists.txt
Show inline comments
 
add_subdirectory(catch2)
 
add_subdirectory(fmt)
 
add_subdirectory(icu)
 
add_subdirectory(md5)
 
add_subdirectory(squirrel)
 
add_subdirectory(opengl)
 
add_subdirectory(os2)
src/3rdparty/icu/CMakeLists.txt
Show inline comments
 
new file 100644
 
add_files(
 
    scriptrun.cpp
 
    scriptrun.h
 
    CONDITION ICU_i18n_FOUND
 
)
src/3rdparty/icu/LICENSE
Show inline comments
 
new file 100644
 
UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
 

	
 
See Terms of Use <https://www.unicode.org/copyright.html>
 
for definitions of Unicode Inc.’s Data Files and Software.
 

	
 
NOTICE TO USER: Carefully read the following legal agreement.
 
BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
 
DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
 
YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
 
TERMS AND CONDITIONS OF THIS AGREEMENT.
 
IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
 
THE DATA FILES OR SOFTWARE.
 

	
 
COPYRIGHT AND PERMISSION NOTICE
 

	
 
Copyright © 1991-2023 Unicode, Inc. All rights reserved.
 
Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
 

	
 
Permission is hereby granted, free of charge, to any person obtaining
 
a copy of the Unicode data files and any associated documentation
 
(the "Data Files") or Unicode software and any associated documentation
 
(the "Software") to deal in the Data Files or Software
 
without restriction, including without limitation the rights to use,
 
copy, modify, merge, publish, distribute, and/or sell copies of
 
the Data Files or Software, and to permit persons to whom the Data Files
 
or Software are furnished to do so, provided that either
 
(a) this copyright and permission notice appear with all copies
 
of the Data Files or Software, or
 
(b) this copyright and permission notice appear in associated
 
Documentation.
 

	
 
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
 
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
 
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 
NONINFRINGEMENT OF THIRD PARTY RIGHTS.
 
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
 
NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
 
DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 
DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
 

	
 
Except as contained in this notice, the name of a copyright holder
 
shall not be used in advertising or otherwise to promote the sale,
 
use or other dealings in these Data Files or Software without prior
 
written authorization of the copyright holder.
src/3rdparty/icu/scriptrun.cpp
Show inline comments
 
new file 100644
 
// © 2016 and later: Unicode, Inc. and others.
 
// License & terms of use: http://www.unicode.org/copyright.html
 
/*
 
 *******************************************************************************
 
 *
 
 *   Copyright (C) 1999-2016, International Business Machines
 
 *   Corporation and others.  All Rights Reserved.
 
 *
 
 *******************************************************************************
 
 *   file name:  scrptrun.cpp
 
 *
 
 *   created on: 10/17/2001
 
 *   created by: Eric R. Mader
 
 */
 

	
 
#include <unicode/utypes.h>
 
#include <unicode/uscript.h>
 

	
 
#include "scriptrun.h"
 

	
 
// Copied from cmemory.h
 
#define UPRV_LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
 

	
 
U_NAMESPACE_BEGIN
 

	
 
const char ScriptRun::fgClassID=0;
 

	
 
UChar32 ScriptRun::pairedChars[] = {
 
    0x0028, 0x0029, // ascii paired punctuation
 
    0x003c, 0x003e,
 
    0x005b, 0x005d,
 
    0x007b, 0x007d,
 
    0x00ab, 0x00bb, // guillemets
 
    0x2018, 0x2019, // general punctuation
 
    0x201c, 0x201d,
 
    0x2039, 0x203a,
 
    0x3008, 0x3009, // chinese paired punctuation
 
    0x300a, 0x300b,
 
    0x300c, 0x300d,
 
    0x300e, 0x300f,
 
    0x3010, 0x3011,
 
    0x3014, 0x3015,
 
    0x3016, 0x3017,
 
    0x3018, 0x3019,
 
    0x301a, 0x301b
 
};
 

	
 
const int32_t ScriptRun::pairedCharCount = UPRV_LENGTHOF(pairedChars);
 
const int32_t ScriptRun::pairedCharPower = 1 << highBit(pairedCharCount);
 
const int32_t ScriptRun::pairedCharExtra = pairedCharCount - pairedCharPower;
 

	
 
int8_t ScriptRun::highBit(int32_t value)
 
{
 
    if (value <= 0) {
 
        return -32;
 
    }
 

	
 
    int8_t bit = 0;
 

	
 
    if (value >= 1 << 16) {
 
        value >>= 16;
 
        bit += 16;
 
    }
 

	
 
    if (value >= 1 << 8) {
 
        value >>= 8;
 
        bit += 8;
 
    }
 

	
 
    if (value >= 1 << 4) {
 
        value >>= 4;
 
        bit += 4;
 
    }
 

	
 
    if (value >= 1 << 2) {
 
        value >>= 2;
 
        bit += 2;
 
    }
 

	
 
    if (value >= 1 << 1) {
 
        value >>= 1;
 
        bit += 1;
 
    }
 

	
 
    return bit;
 
}
 

	
 
int32_t ScriptRun::getPairIndex(UChar32 ch)
 
{
 
    int32_t probe = pairedCharPower;
 
    int32_t index = 0;
 

	
 
    if (ch >= pairedChars[pairedCharExtra]) {
 
        index = pairedCharExtra;
 
    }
 

	
 
    while (probe > (1 << 0)) {
 
        probe >>= 1;
 

	
 
        if (ch >= pairedChars[index + probe]) {
 
            index += probe;
 
        }
 
    }
 

	
 
    if (pairedChars[index] != ch) {
 
        index = -1;
 
    }
 

	
 
    return index;
 
}
 

	
 
UBool ScriptRun::sameScript(int32_t scriptOne, int32_t scriptTwo)
 
{
 
    return scriptOne <= USCRIPT_INHERITED || scriptTwo <= USCRIPT_INHERITED || scriptOne == scriptTwo;
 
}
 

	
 
UBool ScriptRun::next()
 
{
 
    int32_t startSP  = parenSP;  // used to find the first new open character
 
    UErrorCode error = U_ZERO_ERROR;
 

	
 
    // if we've fallen off the end of the text, we're done
 
    if (scriptEnd >= charLimit) {
 
        return false;
 
    }
 

	
 
    scriptCode = USCRIPT_COMMON;
 

	
 
    for (scriptStart = scriptEnd; scriptEnd < charLimit; scriptEnd += 1) {
 
        char16_t   high = charArray[scriptEnd];
 
        UChar32 ch   = high;
 

	
 
        // if the character is a high surrogate and it's not the last one
 
        // in the text, see if it's followed by a low surrogate
 
        if (high >= 0xD800 && high <= 0xDBFF && scriptEnd < charLimit - 1)
 
        {
 
            char16_t low = charArray[scriptEnd + 1];
 

	
 
            // if it is followed by a low surrogate,
 
            // consume it and form the full character
 
            if (low >= 0xDC00 && low <= 0xDFFF) {
 
                ch = (high - 0xD800) * 0x0400 + low - 0xDC00 + 0x10000;
 
                scriptEnd += 1;
 
            }
 
        }
 

	
 
        UScriptCode sc = uscript_getScript(ch, &error);
 
        int32_t pairIndex = getPairIndex(ch);
 

	
 
        // Paired character handling:
 
        //
 
        // if it's an open character, push it onto the stack.
 
        // if it's a close character, find the matching open on the
 
        // stack, and use that script code. Any non-matching open
 
        // characters above it on the stack will be poped.
 
        if (pairIndex >= 0) {
 
            if ((pairIndex & 1) == 0) {
 
                parenStack[++parenSP].pairIndex = pairIndex;
 
                parenStack[parenSP].scriptCode  = scriptCode;
 
            } else if (parenSP >= 0) {
 
                int32_t pi = pairIndex & ~1;
 

	
 
                while (parenSP >= 0 && parenStack[parenSP].pairIndex != pi) {
 
                    parenSP -= 1;
 
                }
 

	
 
                if (parenSP < startSP) {
 
                    startSP = parenSP;
 
                }
 

	
 
                if (parenSP >= 0) {
 
                    sc = parenStack[parenSP].scriptCode;
 
                }
 
            }
 
        }
 

	
 
        if (sameScript(scriptCode, sc)) {
 
            if (scriptCode <= USCRIPT_INHERITED && sc > USCRIPT_INHERITED) {
 
                scriptCode = sc;
 

	
 
                // now that we have a final script code, fix any open
 
                // characters we pushed before we knew the script code.
 
                while (startSP < parenSP) {
 
                    parenStack[++startSP].scriptCode = scriptCode;
 
                }
 
            }
 

	
 
            // if this character is a close paired character,
 
            // pop it from the stack
 
            if (pairIndex >= 0 && (pairIndex & 1) != 0 && parenSP >= 0) {
 
                parenSP -= 1;
 
                startSP -= 1;
 
            }
 
        } else {
 
            // if the run broke on a surrogate pair,
 
            // end it before the high surrogate
 
            if (ch >= 0x10000) {
 
                scriptEnd -= 1;
 
            }
 

	
 
            break;
 
        }
 
    }
 

	
 
    return true;
 
}
 

	
 
U_NAMESPACE_END
src/3rdparty/icu/scriptrun.h
Show inline comments
 
new file 100644
 
// © 2016 and later: Unicode, Inc. and others.
 
// License & terms of use: http://www.unicode.org/copyright.html
 
/*
 
 *******************************************************************************
 
 *
 
 *   Copyright (C) 1999-2003, International Business Machines
 
 *   Corporation and others.  All Rights Reserved.
 
 *
 
 *******************************************************************************
 
 *   file name:  scrptrun.h
 
 *
 
 *   created on: 10/17/2001
 
 *   created by: Eric R. Mader
 
 */
 

	
 
#ifndef __SCRPTRUN_H
 
#define __SCRPTRUN_H
 

	
 
#include <unicode/utypes.h>
 
#include <unicode/uobject.h>
 
#include <unicode/uscript.h>
 

	
 
U_NAMESPACE_BEGIN
 

	
 
struct ScriptRecord
 
{
 
    UChar32 startChar;
 
    UChar32 endChar;
 
    UScriptCode scriptCode;
 
};
 

	
 
struct ParenStackEntry
 
{
 
    int32_t pairIndex;
 
    UScriptCode scriptCode;
 
};
 

	
 
class ScriptRun : public UObject {
 
public:
 
    ScriptRun();
 

	
 
    ScriptRun(const char16_t *chars, int32_t length);
 

	
 
    ScriptRun(const char16_t *chars, int32_t start, int32_t length);
 

	
 
    void reset();
 

	
 
    void reset(int32_t start, int32_t count);
 

	
 
    void reset(const char16_t *chars, int32_t start, int32_t length);
 

	
 
    int32_t getScriptStart();
 

	
 
    int32_t getScriptEnd();
 

	
 
    UScriptCode getScriptCode();
 

	
 
    UBool next();
 

	
 
    /**
 
     * ICU "poor man's RTTI", returns a UClassID for the actual class.
 
     *
 
     * @stable ICU 2.2
 
     */
 
    virtual inline UClassID getDynamicClassID() const override { return getStaticClassID(); }
 

	
 
    /**
 
     * ICU "poor man's RTTI", returns a UClassID for this class.
 
     *
 
     * @stable ICU 2.2
 
     */
 
    static inline UClassID getStaticClassID() { return (UClassID)const_cast<char *>(&fgClassID); }
 

	
 
private:
 

	
 
    static UBool sameScript(int32_t scriptOne, int32_t scriptTwo);
 

	
 
    int32_t charStart;
 
    int32_t charLimit;
 
    const char16_t *charArray;
 

	
 
    int32_t scriptStart;
 
    int32_t scriptEnd;
 
    UScriptCode scriptCode;
 

	
 
    ParenStackEntry parenStack[128];
 
    int32_t parenSP;
 

	
 
    static int8_t highBit(int32_t value);
 
    static int32_t getPairIndex(UChar32 ch);
 

	
 
    static UChar32 pairedChars[];
 
    static const int32_t pairedCharCount;
 
    static const int32_t pairedCharPower;
 
    static const int32_t pairedCharExtra;
 

	
 
    /**
 
     * The address of this static class variable serves as this class's ID
 
     * for ICU "poor man's RTTI".
 
     */
 
    static const char fgClassID;
 
};
 

	
 
inline ScriptRun::ScriptRun()
 
{
 
    reset(nullptr, 0, 0);
 
}
 

	
 
inline ScriptRun::ScriptRun(const char16_t *chars, int32_t length)
 
{
 
    reset(chars, 0, length);
 
}
 

	
 
inline ScriptRun::ScriptRun(const char16_t *chars, int32_t start, int32_t length)
 
{
 
    reset(chars, start, length);
 
}
 

	
 
inline int32_t ScriptRun::getScriptStart()
 
{
 
    return scriptStart;
 
}
 

	
 
inline int32_t ScriptRun::getScriptEnd()
 
{
 
    return scriptEnd;
 
}
 

	
 
inline UScriptCode ScriptRun::getScriptCode()
 
{
 
    return scriptCode;
 
}
 

	
 
inline void ScriptRun::reset()
 
{
 
    scriptStart = charStart;
 
    scriptEnd   = charStart;
 
    scriptCode  = USCRIPT_INVALID_CODE;
 
    parenSP     = -1;
 
}
 

	
 
inline void ScriptRun::reset(int32_t start, int32_t length)
 
{
 
    charStart = start;
 
    charLimit = start + length;
 

	
 
    reset();
 
}
 

	
 
inline void ScriptRun::reset(const char16_t *chars, int32_t start, int32_t length)
 
{
 
    charArray = chars;
 

	
 
    reset(start, length);
 
}
 

	
 
U_NAMESPACE_END
 

	
 
#endif
src/CMakeLists.txt
Show inline comments
 
@@ -33,13 +33,13 @@ add_files(
 
    CONDITION SSE_FOUND
 
)
 

	
 
add_files(
 
    gfx_layout_icu.cpp
 
    gfx_layout_icu.h
 
    CONDITION ICU_lx_FOUND
 
    CONDITION ICU_i18n_FOUND AND HARFBUZZ_FOUND
 
)
 

	
 
add_files(
 
    aircraft.h
 
    aircraft_cmd.cpp
 
    aircraft_cmd.h
src/crashlog.cpp
Show inline comments
 
@@ -47,15 +47,18 @@
 
#	include <png.h>
 
#endif /* WITH_PNG */
 
#ifdef WITH_FREETYPE
 
#	include <ft2build.h>
 
#	include FT_FREETYPE_H
 
#endif /* WITH_FREETYPE */
 
#if defined(WITH_ICU_LX) || defined(WITH_ICU_I18N)
 
#ifdef WITH_HARFBUZZ
 
#	include <hb.h>
 
#endif /* WITH_HARFBUZZ */
 
#ifdef WITH_ICU_I18N
 
#	include <unicode/uversion.h>
 
#endif /* WITH_ICU_LX || WITH_ICU_I18N */
 
#endif /* WITH_ICU_I18N */
 
#ifdef WITH_LIBLZMA
 
#	include <lzma.h>
 
#endif
 
#ifdef WITH_LZO
 
#include <lzo/lzo1x.h>
 
#endif
 
@@ -235,25 +238,24 @@ char *CrashLog::LogLibraries(char *buffe
 
	FT_Init_FreeType(&library);
 
	FT_Library_Version(library, &major, &minor, &patch);
 
	FT_Done_FreeType(library);
 
	buffer += seprintf(buffer, last, " FreeType:   %d.%d.%d\n", major, minor, patch);
 
#endif /* WITH_FREETYPE */
 

	
 
#if defined(WITH_ICU_LX) || defined(WITH_ICU_I18N)
 
#if defined(WITH_HARFBUZZ)
 
	buffer += seprintf(buffer, last, " HarfBuzz:   %s\n", hb_version_string());
 
#endif /* WITH_HARFBUZZ */
 

	
 
#if defined(WITH_ICU_I18N)
 
	/* 4 times 0-255, separated by dots (.) and a trailing '\0' */
 
	char buf[4 * 3 + 3 + 1];
 
	UVersionInfo ver;
 
	u_getVersion(ver);
 
	u_versionToString(ver, buf);
 
#ifdef WITH_ICU_I18N
 
	buffer += seprintf(buffer, last, " ICU i18n:   %s\n", buf);
 
#endif
 
#ifdef WITH_ICU_LX
 
	buffer += seprintf(buffer, last, " ICU lx:     %s\n", buf);
 
#endif
 
#endif /* WITH_ICU_LX || WITH_ICU_I18N */
 
#endif /* WITH_ICU_I18N */
 

	
 
#ifdef WITH_LIBLZMA
 
	buffer += seprintf(buffer, last, " LZMA:       %s\n", lzma_version_string());
 
#endif
 

	
 
#ifdef WITH_LZO
src/fontcache/freetypefontcache.cpp
Show inline comments
 
@@ -31,22 +31,23 @@
 
/** Font cache for fonts that are based on a freetype font. */
 
class FreeTypeFontCache : public TrueTypeFontCache {
 
private:
 
	FT_Face face;  ///< The font face associated with this font.
 

	
 
	void SetFontSize(FontSize fs, FT_Face face, int pixels);
 
	virtual const void *InternalGetFontTable(uint32 tag, size_t &length);
 
	virtual const Sprite *InternalGetGlyph(GlyphID key, bool aa);
 
	const void *InternalGetFontTable(uint32 tag, size_t &length) override;
 
	const Sprite *InternalGetGlyph(GlyphID key, bool aa) override;
 

	
 
public:
 
	FreeTypeFontCache(FontSize fs, FT_Face face, int pixels);
 
	~FreeTypeFontCache();
 
	virtual void ClearFontCache();
 
	virtual GlyphID MapCharToGlyph(WChar key);
 
	virtual const char *GetFontName() { return face->family_name; }
 
	virtual bool IsBuiltInFont() { return false; }
 
	void ClearFontCache() override;
 
	GlyphID MapCharToGlyph(WChar key) override;
 
	const char *GetFontName() override { return face->family_name; }
 
	bool IsBuiltInFont() override { return false; }
 
	const void *GetOSHandle() override { return &face; }
 
};
 

	
 
FT_Library _library = nullptr;
 

	
 

	
 
/**
src/gfx_layout.cpp
Show inline comments
 
@@ -7,23 +7,21 @@
 

	
 
/** @file gfx_layout.cpp Handling of laying out text. */
 

	
 
#include "stdafx.h"
 
#include "gfx_layout.h"
 
#include "string_func.h"
 
#include "strings_func.h"
 
#include "zoom_func.h"
 
#include "debug.h"
 

	
 
#include "table/control_codes.h"
 

	
 
#include "gfx_layout_fallback.h"
 

	
 
#ifdef WITH_ICU_LX
 
#if defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)
 
#include "gfx_layout_icu.h"
 
#endif /* WITH_ICU_LX */
 
#endif /* WITH_ICU_I18N && WITH_HARFBUZZ */
 

	
 
#ifdef WITH_UNISCRIBE
 
#include "os/windows/string_uniscribe.h"
 
#endif /* WITH_UNISCRIBE */
 

	
 
#ifdef WITH_COCOA
 
@@ -92,14 +90,14 @@ static inline void GetLayouter(Layouter:
 
		} else if (c >= SCC_FIRST_FONT && c <= SCC_LAST_FONT) {
 
			state.SetFontSize((FontSize)(c - SCC_FIRST_FONT));
 
		} else {
 
			/* Filter out non printable characters */
 
			if (!IsPrintable(c)) continue;
 
			/* Filter out text direction characters that shouldn't be drawn, and
 
			 * will not be handled in the fallback non ICU case because they are
 
			 * mostly needed for RTL languages which need more ICU support. */
 
			 * will not be handled in the fallback case because they are mostly
 
			 * needed for RTL languages which need more proper shaping support. */
 
			if (!T::SUPPORTS_RTL && IsTextDirectionChar(c)) continue;
 
			buff += T::AppendToBuffer(buff, buffer_last, c);
 
			continue;
 
		}
 

	
 
		if (!fontMapping.Contains(buff - buff_begin)) {
 
@@ -145,27 +143,23 @@ Layouter::Layouter(const char *str, int 
 
			str = lineend + 1;
 
			state = line.state_after;
 
			line.layout->Reflow();
 
		} else {
 
			/* Line is new, layout it */
 
			FontState old_state = state;
 
#if defined(WITH_ICU_LX) || defined(WITH_UNISCRIBE) || defined(WITH_COCOA)
 
#if (defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)) || defined(WITH_UNISCRIBE) || defined(WITH_COCOA)
 
			const char *old_str = str;
 
#endif
 

	
 
#ifdef WITH_ICU_LX
 
			GetLayouter<ICUParagraphLayoutFactory>(line, str, state);
 
#if defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)
 
			if (line.layout == nullptr) {
 
				static bool warned = false;
 
				if (!warned) {
 
					Debug(misc, 0, "ICU layouter bailed on the font. Falling back to the fallback layouter");
 
					warned = true;
 
				GetLayouter<ICUParagraphLayoutFactory>(line, str, state);
 
				if (line.layout == nullptr) {
 
					state = old_state;
 
					str = old_str;
 
				}
 

	
 
				state = old_state;
 
				str = old_str;
 
			}
 
#endif
 

	
 
#ifdef WITH_UNISCRIBE
 
			if (line.layout == nullptr) {
 
				GetLayouter<UniscribeParagraphLayoutFactory>(line, str, state);
src/gfx_layout.h
Show inline comments
 
@@ -18,19 +18,12 @@
 
#include <string>
 
#include <stack>
 
#include <string_view>
 
#include <type_traits>
 
#include <vector>
 

	
 
#ifdef WITH_ICU_LX
 
#include "layout/ParagraphLayout.h"
 
#define ICU_FONTINSTANCE : public icu::LEFontInstance
 
#else /* WITH_ICU_LX */
 
#define ICU_FONTINSTANCE
 
#endif /* WITH_ICU_LX */
 

	
 
/**
 
 * Text drawing parameters, which can change while drawing a line, but are kept between multiple parts
 
 * of the same text, e.g. on line breaks.
 
 */
 
struct FontState {
 
	FontSize fontsize;       ///< Current font size.
 
@@ -80,36 +73,18 @@ struct FontState {
 
	}
 
};
 

	
 
/**
 
 * Container with information about a font.
 
 */
 
class Font ICU_FONTINSTANCE {
 
class Font {
 
public:
 
	FontCache *fc;     ///< The font we are using.
 
	TextColour colour; ///< The colour this font has to be.
 

	
 
	Font(FontSize size, TextColour colour);
 

	
 
#ifdef WITH_ICU_LX
 
	/* Implementation details of LEFontInstance */
 

	
 
	le_int32 getUnitsPerEM() const;
 
	le_int32 getAscent() const;
 
	le_int32 getDescent() const;
 
	le_int32 getLeading() const;
 
	float getXPixelsPerEm() const;
 
	float getYPixelsPerEm() const;
 
	float getScaleFactorX() const;
 
	float getScaleFactorY() const;
 
	const void *getFontTable(LETag tableTag) const;
 
	const void *getFontTable(LETag tableTag, size_t &length) const;
 
	LEGlyphID mapCharToGlyph(LEUnicode32 ch) const;
 
	void getGlyphAdvance(LEGlyphID glyph, LEPoint &advance) const;
 
	le_bool getGlyphPoint(LEGlyphID glyph, le_int32 pointNumber, LEPoint &point) const;
 
#endif /* WITH_ICU_LX */
 
};
 

	
 
/** Mapping from index to font. */
 
typedef SmallMap<int, Font *> FontMap;
 

	
 
/**
 
@@ -180,13 +155,13 @@ class Layouter : public std::vector<std:
 
		}
 
	};
 
public:
 
	/** Item in the linecache */
 
	struct LineCacheItem {
 
		/* Stuff that cannot be freed until the ParagraphLayout is freed */
 
		void *buffer;              ///< Accessed by both ICU's and our ParagraphLayout::nextLine.
 
		void *buffer;              ///< Accessed by our ParagraphLayout::nextLine.
 
		FontMap runs;              ///< Accessed by our ParagraphLayout::nextLine.
 

	
 
		FontState state_after;     ///< Font state after the line.
 
		ParagraphLayouter *layout; ///< Layout of the line.
 

	
 
		LineCacheItem() : buffer(nullptr), layout(nullptr) {}
src/gfx_layout_fallback.cpp
Show inline comments
 
@@ -30,14 +30,13 @@
 
 * font. The sub strings are then already fully laid out, and only
 
 * need actual drawing.
 
 *
 
 * The positions in a visual run are sequential pairs of X,Y of the
 
 * begin of each of the glyphs plus an extra pair to mark the end.
 
 *
 
 * @note This variant does not handle left-to-right properly. This
 
 *       is supported in the one ParagraphLayout coming from ICU.
 
 * @note This variant does not handle right-to-left properly.
 
 */
 
class FallbackParagraphLayout : public ParagraphLayouter {
 
public:
 
	/** Visual run contains data about the bit of text with the same font. */
 
	class FallbackVisualRun : public ParagraphLayouter::VisualRun {
 
		Font *font;       ///< The font used to layout these.
src/gfx_layout_icu.cpp
Show inline comments
 
@@ -2,176 +2,529 @@
 
 * This file is part of OpenTTD.
 
 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
 
 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
 
 */
 

	
 
/** @file gfx_layout_icu.cpp Handling of laying out text with ICU. */
 
/** @file gfx_layout_icu.cpp Handling of laying out with ICU / Harfbuzz. */
 

	
 
#include "stdafx.h"
 

	
 
#include "gfx_layout_icu.h"
 

	
 
#include <unicode/ustring.h>
 
#include "debug.h"
 
#include "strings_func.h"
 
#include "language.h"
 
#include "table/control_codes.h"
 
#include "zoom_func.h"
 

	
 
#include "3rdparty/icu/scriptrun.h"
 

	
 
#include <unicode/ubidi.h>
 
#include <unicode/brkiter.h>
 

	
 
#include <hb.h>
 
#include <hb-ft.h>
 

	
 
#include <deque>
 

	
 
#include "safeguards.h"
 

	
 
/* Implementation details of LEFontInstance */
 

	
 
le_int32 Font::getUnitsPerEM() const
 
{
 
	return this->fc->GetUnitsPerEM();
 
}
 

	
 
le_int32 Font::getAscent() const
 
{
 
	return this->fc->GetAscender();
 
}
 

	
 
le_int32 Font::getDescent() const
 
{
 
	return -this->fc->GetDescender();
 
}
 

	
 
le_int32 Font::getLeading() const
 
{
 
	return this->fc->GetHeight();
 
}
 

	
 
float Font::getXPixelsPerEm() const
 
{
 
	return (float)this->fc->GetHeight();
 
}
 

	
 
float Font::getYPixelsPerEm() const
 
{
 
	return (float)this->fc->GetHeight();
 
}
 
/** harfbuzz doesn't use floats, so we need a value to scale position with to get sub-pixel precision. */
 
constexpr float FONT_SCALE = 64.0;
 

	
 
float Font::getScaleFactorX() const
 
{
 
	return 1.0f;
 
}
 

	
 
float Font::getScaleFactorY() const
 
{
 
	return 1.0f;
 
}
 

	
 
const void *Font::getFontTable(LETag tableTag) const
 
{
 
	size_t length;
 
	return this->getFontTable(tableTag, length);
 
}
 
/**
 
 * Helper class to store the information of all the runs of a paragraph in.
 
 *
 
 * During itemization, more and more information is filled in.
 
 */
 
class ICURun {
 
public:
 
	int start; ///< Start of the run in the buffer.
 
	int length; ///< Length of the run in the buffer.
 
	UBiDiLevel level; ///< Embedding level of the run.
 
	UScriptCode script; ///< Script of the run.
 
	Font *font; ///< Font of the run.
 

	
 
const void *Font::getFontTable(LETag tableTag, size_t &length) const
 
{
 
	return this->fc->GetFontTable(tableTag, length);
 
}
 

	
 
LEGlyphID Font::mapCharToGlyph(LEUnicode32 ch) const
 
{
 
	if (IsTextDirectionChar(ch)) return 0;
 
	return this->fc->MapCharToGlyph(ch);
 
}
 
	std::vector<GlyphID> glyphs; ///< The glyphs of the run. Valid after Shape() is called.
 
	std::vector<int> advance; ///< The advance (width) of the glyphs. Valid after Shape() is called.
 
	std::vector<int> glyph_to_char; ///< The mapping from glyphs to characters. Valid after Shape() is called.
 
	std::vector<float> positions; ///< The positions of the glyphs. Valid after Shape() is called.
 
	int total_advance; ///< The total advance of the run. Valid after Shape() is called.
 

	
 
void Font::getGlyphAdvance(LEGlyphID glyph, LEPoint &advance) const
 
{
 
	advance.fX = glyph == 0xFFFF ? 0 : this->fc->GetGlyphWidth(glyph);
 
	advance.fY = 0;
 
}
 
	ICURun(int start, int length, UBiDiLevel level, UScriptCode script = USCRIPT_UNKNOWN, Font *font = nullptr) : start(start), length(length), level(level), script(script), font(font) {}
 

	
 
le_bool Font::getGlyphPoint(LEGlyphID glyph, le_int32 pointNumber, LEPoint &point) const
 
{
 
	return false;
 
}
 
	void Shape(UChar *buff, size_t length);
 
};
 

	
 
/**
 
 * Wrapper for doing layouts with ICU.
 
 */
 
class ICUParagraphLayout : public ParagraphLayouter {
 
	icu::ParagraphLayout *p; ///< The actual ICU paragraph layout.
 
public:
 
	/** Visual run contains data about the bit of text with the same font. */
 
	class ICUVisualRun : public ParagraphLayouter::VisualRun {
 
		const icu::ParagraphLayout::VisualRun *vr; ///< The actual ICU vr.
 
	private:
 
		std::vector<GlyphID> glyphs;
 
		std::vector<float> positions;
 
		std::vector<int> glyph_to_char;
 

	
 
		int total_advance;
 
		const Font *font;
 

	
 
	public:
 
		ICUVisualRun(const icu::ParagraphLayout::VisualRun *vr) : vr(vr) { }
 
		ICUVisualRun(const ICURun &run, int x);
 

	
 
		const Font *GetFont() const override          { return (const Font*)vr->getFont(); }
 
		int GetGlyphCount() const override            { return vr->getGlyphCount(); }
 
		const GlyphID *GetGlyphs() const override     { return vr->getGlyphs(); }
 
		const float *GetPositions() const override    { return vr->getPositions(); }
 
		int GetLeading() const override               { return vr->getLeading(); }
 
		const int *GetGlyphToCharMap() const override { return vr->getGlyphToCharMap(); }
 
		const GlyphID *GetGlyphs() const override { return this->glyphs.data(); }
 
		const float *GetPositions() const override { return this->positions.data(); }
 
		const int *GetGlyphToCharMap() const override { return this->glyph_to_char.data(); }
 

	
 
		const Font *GetFont() const override { return this->font; }
 
		int GetLeading() const override { return this->font->fc->GetHeight(); }
 
		int GetGlyphCount() const override { return this->glyphs.size(); }
 
		int GetAdvance() const { return this->total_advance; }
 
	};
 

	
 
	/** A single line worth of VisualRuns. */
 
	class ICULine : public std::vector<ICUVisualRun>, public ParagraphLayouter::Line {
 
		icu::ParagraphLayout::Line *l; ///< The actual ICU line.
 

	
 
	public:
 
		ICULine(icu::ParagraphLayout::Line *l) : l(l)
 
		{
 
			for (int i = 0; i < l->countRuns(); i++) {
 
				this->emplace_back(l->getVisualRun(i));
 
			}
 
		}
 
		~ICULine() override { delete l; }
 

	
 
		int GetLeading() const override { return l->getLeading(); }
 
		int GetWidth() const override   { return l->getWidth(); }
 
		int CountRuns() const override  { return l->countRuns(); }
 
		const ParagraphLayouter::VisualRun &GetVisualRun(int run) const override { return this->at(run); }
 
		int GetLeading() const override;
 
		int GetWidth() const override;
 
		int CountRuns() const override { return (uint)this->size();  }
 
		const VisualRun &GetVisualRun(int run) const override { return this->at(run); }
 

	
 
		int GetInternalCharLength(WChar c) const override
 
		{
 
			/* ICU uses UTF-16 internally which means we need to account for surrogate pairs. */
 
			return Utf8CharLen(c) < 4 ? 1 : 2;
 
			return c >= 0x010000U ? 2 : 1;
 
		}
 
	};
 

	
 
	ICUParagraphLayout(icu::ParagraphLayout *p) : p(p) { }
 
	~ICUParagraphLayout() override { delete p; }
 
	void Reflow() override  { p->reflow(); }
 
private:
 
	std::vector<ICURun> runs;
 
	UChar *buff;
 
	size_t buff_length;
 
	std::vector<ICURun>::iterator current_run;
 
	int partial_offset;
 

	
 
	std::unique_ptr<const Line> NextLine(int max_width) override
 
public:
 
	ICUParagraphLayout(std::vector<ICURun> &runs, UChar *buff, size_t buff_length) : runs(runs), buff(buff), buff_length(buff_length)
 
	{
 
		icu::ParagraphLayout::Line *l = p->nextLine(max_width);
 
		return std::unique_ptr<const Line>(l == nullptr ? nullptr : new ICULine(l));
 
		this->Reflow();
 
	}
 

	
 
	~ICUParagraphLayout() override { }
 

	
 
	void Reflow() override
 
	{
 
		this->current_run = this->runs.begin();
 
		this->partial_offset = 0;
 
	}
 

	
 
	std::unique_ptr<const Line> NextLine(int max_width) override;
 
};
 

	
 
/* static */ ParagraphLayouter *ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &fontMapping)
 
/**
 
 * Constructor for a new ICUVisualRun.
 
 *
 
 * It bases all information on the ICURun, which should already be shaped.
 
 *
 
 * @param run The ICURun to base the visual run on.
 
 * @param x The offset of the run on the line.
 
 */
 
ICUParagraphLayout::ICUVisualRun::ICUVisualRun(const ICURun &run, int x) :
 
	glyphs(run.glyphs), glyph_to_char(run.glyph_to_char), total_advance(run.total_advance), font(run.font)
 
{
 
	int32 length = buff_end - buff;
 
	/* If there are no positions, the ICURun was not Shaped; that should never happen. */
 
	assert(run.positions.size() != 0);
 
	this->positions.reserve(run.positions.size());
 

	
 
	/* "positions" is an array of x/y. So we need to alternate. */
 
	bool is_x = true;
 
	for (auto &position : run.positions) {
 
		if (is_x) {
 
			this->positions.push_back(position + x);
 
		} else {
 
			this->positions.push_back(position);
 
		}
 

	
 
		is_x = !is_x;
 
	}
 
}
 

	
 
/**
 
 * Shape a single run.
 
 *
 
 * @param buff The buffer of which a partial (depending on start/length of the run) will be shaped.
 
 * @param length The length of the buffer.
 
 */
 
void ICURun::Shape(UChar *buff, size_t buff_length) {
 
	auto hbfont = hb_ft_font_create_referenced(*(static_cast<const FT_Face *>(font->fc->GetOSHandle())));
 
	hb_font_set_scale(hbfont, this->font->fc->GetFontSize() * FONT_SCALE, this->font->fc->GetFontSize() * FONT_SCALE);
 

	
 
	/* ICU buffer is in UTF-16. */
 
	auto hbbuf = hb_buffer_create();
 
	hb_buffer_add_utf16(hbbuf, reinterpret_cast<uint16 *>(buff), buff_length, this->start, this->length);
 

	
 
	/* Set all the properties of this segment. */
 
	hb_buffer_set_direction(hbbuf, (this->level & 1) == 1 ? HB_DIRECTION_RTL : HB_DIRECTION_LTR);
 
	hb_buffer_set_script(hbbuf, hb_script_from_string(uscript_getShortName(this->script), -1));
 
	hb_buffer_set_language(hbbuf, hb_language_from_string(_current_language->isocode, -1));
 
	hb_buffer_set_cluster_level(hbbuf, HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES);
 

	
 
	/* Shape the segment. */
 
	hb_shape(hbfont, hbbuf, nullptr, 0);
 

	
 
	unsigned int glyph_count;
 
	auto glyph_info = hb_buffer_get_glyph_infos(hbbuf, &glyph_count);
 
	auto glyph_pos = hb_buffer_get_glyph_positions(hbbuf, &glyph_count);
 

	
 
	/* Make sure any former run is lost. */
 
	this->glyphs.clear();
 
	this->glyph_to_char.clear();
 
	this->positions.clear();
 
	this->advance.clear();
 

	
 
	/* Reserve space, as we already know the size. */
 
	this->glyphs.reserve(glyph_count);
 
	this->glyph_to_char.reserve(glyph_count);
 
	this->positions.reserve(glyph_count * 2 + 2);
 
	this->advance.reserve(glyph_count);
 

	
 
	/* Prepare the glyphs/position. ICUVisualRun will give the position an offset if needed. */
 
	hb_position_t advance = 0;
 
	for (unsigned int i = 0; i < glyph_count; i++) {
 
		int x_advance;
 

	
 
		if (buff[glyph_info[i].cluster] >= SCC_SPRITE_START && buff[glyph_info[i].cluster] <= SCC_SPRITE_END) {
 
			auto glyph = this->font->fc->MapCharToGlyph(buff[glyph_info[i].cluster]);
 

	
 
			this->glyphs.push_back(glyph);
 
			this->positions.push_back(advance);
 
			this->positions.push_back((this->font->fc->GetHeight() - ScaleSpriteTrad(FontCache::GetDefaultFontHeight(this->font->fc->GetSize()))) / 2); // Align sprite font to centre
 
			x_advance = this->font->fc->GetGlyphWidth(glyph);
 
		} else {
 
			this->glyphs.push_back(glyph_info[i].codepoint);
 
			this->positions.push_back(glyph_pos[i].x_offset / FONT_SCALE + advance);
 
			this->positions.push_back(glyph_pos[i].y_offset / FONT_SCALE);
 
			x_advance = glyph_pos[i].x_advance / FONT_SCALE;
 
		}
 

	
 
	if (length == 0) {
 
		/* ICU's ParagraphLayout cannot handle empty strings, so fake one. */
 
		buff[0] = ' ';
 
		length = 1;
 
		fontMapping.back().first++;
 
		this->glyph_to_char.push_back(glyph_info[i].cluster);
 
		this->advance.push_back(x_advance);
 
		advance += x_advance;
 
	}
 

	
 
	/* Position has one more element to close off the array. */
 
	this->positions.push_back(advance);
 
	this->positions.push_back(0);
 

	
 
	/* Track the total advancement we made. */
 
	this->total_advance = advance;
 

	
 
	hb_buffer_destroy(hbbuf);
 
	hb_font_destroy(hbfont);
 
}
 

	
 
/**
 
 * Get the height of the line.
 
 * @return The maximum height of the line.
 
 */
 
int ICUParagraphLayout::ICULine::GetLeading() const
 
{
 
	int leading = 0;
 
	for (const auto &run : *this) {
 
		leading = std::max(leading, run.GetLeading());
 
	}
 

	
 
	return leading;
 
}
 

	
 
/**
 
 * Get the width of this line.
 
 * @return The width of the line.
 
 */
 
int ICUParagraphLayout::ICULine::GetWidth() const
 
{
 
	int length = 0;
 
	for (const auto &run : *this) {
 
		length += run.GetAdvance();
 
	}
 

	
 
	return length;
 
}
 

	
 
/**
 
 * Itemize the string into runs per embedding level.
 
 *
 
 * Later on, based on the levels, we can deduce the order of a subset of runs.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeBidi(UChar *buff, size_t length)
 
{
 
	auto ubidi = ubidi_open();
 

	
 
	auto parLevel = _current_text_dir == TD_RTL ? UBIDI_RTL : UBIDI_LTR;
 

	
 
	UErrorCode err = U_ZERO_ERROR;
 
	ubidi_setPara(ubidi, buff, length, parLevel, nullptr, &err);
 
	if (U_FAILURE(err)) {
 
		Debug(fontcache, 0, "Failed to set paragraph: %s", u_errorName(err));
 
		ubidi_close(ubidi);
 
		return std::vector<ICURun>();
 
	}
 

	
 
	int32_t count = ubidi_countRuns(ubidi, &err);
 
	if (U_FAILURE(err)) {
 
		Debug(fontcache, 0, "Failed to count runs: %s", u_errorName(err));
 
		ubidi_close(ubidi);
 
		return std::vector<ICURun>();
 
	}
 

	
 
	std::vector<ICURun> runs;
 
	runs.reserve(count);
 

	
 
	/* Find the breakpoints for the logical runs. So we get runs that say "from START to END". */
 
	int32_t logical_pos = 0;
 
	while (static_cast<size_t>(logical_pos) < length) {
 
		auto start_pos = logical_pos;
 

	
 
		/* Fetch the embedding level, so we can order bidi correctly later on. */
 
		UBiDiLevel level;
 
		ubidi_getLogicalRun(ubidi, start_pos, &logical_pos, &level);
 

	
 
		runs.emplace_back(ICURun(start_pos, logical_pos - start_pos, level));
 
	}
 

	
 
	/* Fill ICU's FontRuns with the right data. */
 
	icu::FontRuns runs(fontMapping.size());
 
	for (auto &pair : fontMapping) {
 
		runs.add(pair.second, pair.first);
 
	assert(static_cast<size_t>(count) == runs.size());
 

	
 
	ubidi_close(ubidi);
 
	return runs;
 
}
 

	
 
/**
 
 * Itemize the string into runs per script, based on the previous created runs.
 
 *
 
 * Basically, this always returns the same or more runs than given.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @param runs_current The current runs.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeScript(UChar *buff, size_t length, std::vector<ICURun> &runs_current)
 
{
 
	std::vector<ICURun> runs;
 
	icu::ScriptRun script_itemizer(buff, length);
 

	
 
	int cur_pos = 0;
 
	auto cur_run = runs_current.begin();
 
	while (true) {
 
		while (cur_pos < script_itemizer.getScriptEnd() && cur_run != runs_current.end()) {
 
			int stop_pos = std::min(script_itemizer.getScriptEnd(), cur_run->start + cur_run->length);
 
			assert(stop_pos - cur_pos > 0);
 

	
 
			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, script_itemizer.getScriptCode()));
 

	
 
			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
 
			cur_pos = stop_pos;
 
		}
 

	
 
		if (!script_itemizer.next()) break;
 
	}
 

	
 
	return runs;
 
}
 

	
 
/**
 
 * Itemize the string into runs per style, based on the previous created runs.
 
 *
 
 * Basically, this always returns the same or more runs than given.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @param runs_current The current runs.
 
 * @param font_mapping The font mapping.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeStyle(UChar *buff, size_t length, std::vector<ICURun> &runs_current, FontMap &font_mapping)
 
{
 
	std::vector<ICURun> runs;
 

	
 
	int cur_pos = 0;
 
	auto cur_run = runs_current.begin();
 
	for (auto const &font_map : font_mapping) {
 
		while (cur_pos < font_map.first && cur_run != runs_current.end()) {
 
			int stop_pos = std::min(font_map.first, cur_run->start + cur_run->length);
 
			assert(stop_pos - cur_pos > 0);
 

	
 
			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, cur_run->script, font_map.second));
 

	
 
			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
 
			cur_pos = stop_pos;
 
		}
 
	}
 

	
 
	return runs;
 
}
 

	
 
/* static */ ParagraphLayouter *ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &font_mapping)
 
{
 
	size_t length = buff_end - buff;
 
	/* Can't layout an empty string. */
 
	if (length == 0) return nullptr;
 

	
 
	/* Can't layout our in-built sprite fonts. */
 
	for (auto const &pair : font_mapping) {
 
		if (pair.second->fc->IsBuiltInFont()) return nullptr;
 
	}
 

	
 
	auto runs = ItemizeBidi(buff, length);
 
	runs = ItemizeScript(buff, length, runs);
 
	runs = ItemizeStyle(buff, length, runs, font_mapping);
 

	
 
	if (runs.size() == 0) return nullptr;
 

	
 
	for (auto &run : runs) {
 
		run.Shape(buff, length);
 
	}
 

	
 
	LEErrorCode status = LE_NO_ERROR;
 
	/* ParagraphLayout does not copy "buff", so it must stay valid.
 
		* "runs" is copied according to the ICU source, but the documentation does not specify anything, so this might break somewhen. */
 
	icu::ParagraphLayout *p = new icu::ParagraphLayout(buff, length, &runs, nullptr, nullptr, nullptr, _current_text_dir == TD_RTL ? 1 : 0, false, status);
 
	if (status != LE_NO_ERROR) {
 
		delete p;
 
		return nullptr;
 
	return new ICUParagraphLayout(runs, buff, length);
 
}
 

	
 
std::unique_ptr<const ICUParagraphLayout::Line> ICUParagraphLayout::NextLine(int max_width)
 
{
 
	std::vector<ICURun>::iterator start_run = this->current_run;
 
	std::vector<ICURun>::iterator last_run = this->current_run;
 

	
 
	if (start_run == this->runs.end()) return nullptr;
 

	
 
	int cur_width = 0;
 

	
 
	/* Add remaining width of the first run if it is a broken run. */
 
	if (this->partial_offset > 0) {
 
		if ((start_run->level & 1) == 0) {
 
			for (size_t i = this->partial_offset; i < start_run->advance.size(); i++) {
 
				cur_width += start_run->advance[i];
 
			}
 
		} else {
 
			for (int i = 0; i < this->partial_offset; i++) {
 
				cur_width += start_run->advance[i];
 
			}
 
		}
 
		last_run++;
 
	}
 

	
 
	/* Gather runs until the line is full. */
 
	while (last_run != this->runs.end() && cur_width < max_width) {
 
		cur_width += last_run->total_advance;
 
		last_run++;
 
	}
 

	
 
	return new ICUParagraphLayout(p);
 
	/* If the text does not fit into the available width, find a suitable breaking point. */
 
	int new_partial_length = 0;
 
	if (cur_width > max_width) {
 
		auto locale = icu::Locale(_current_language->isocode);
 

	
 
		/* Create a break-iterator to find a good place to break lines. */
 
		UErrorCode err = U_ZERO_ERROR;
 
		auto break_iterator = icu::BreakIterator::createLineInstance(locale, err);
 
		break_iterator->setText(icu::UnicodeString(this->buff, this->buff_length));
 

	
 
		auto overflow_run = last_run - 1;
 

	
 
		/* Find the last glyph that fits. */
 
		size_t index;
 
		if ((overflow_run->level & 1) == 0) {
 
			/* LTR */
 
			for (index = overflow_run->glyphs.size(); index > 0; index--) {
 
				cur_width -= overflow_run->advance[index - 1];
 
				if (cur_width <= max_width) break;
 
			}
 
			index--;
 
		} else {
 
			/* RTL */
 
			for (index = 0; index < overflow_run->glyphs.size(); index++) {
 
				cur_width -= overflow_run->advance[index];
 
				if (cur_width <= max_width) break;
 
			}
 
		}
 

	
 
		/* Find the character that matches; this is the start of the cluster. */
 
		auto char_pos = overflow_run->glyph_to_char[index];
 

	
 
		/* See if there is a good breakpoint inside this run. */
 
		int32_t break_pos = break_iterator->preceding(char_pos + 1);
 
		if (break_pos != icu::BreakIterator::DONE && break_pos > overflow_run->start + this->partial_offset) {
 
			/* There is a line-break inside this run that is suitable. */
 
			new_partial_length = break_pos - overflow_run->start - this->partial_offset;
 
		} else if (overflow_run != start_run) {
 
			/* There is no suitable line-break in this run, but it is also not
 
			 * the only run on this line. So we remove the run. */
 
			last_run--;
 
		} else {
 
			/* There is no suitable line-break and this is the only run on the
 
			 * line. So we break at the cluster. This is not pretty, but the
 
			 * best we can do. */
 
			new_partial_length = char_pos - this->partial_offset;
 
		}
 
	}
 

	
 
	/* Reorder the runs on this line for display. */
 
	std::vector<UBiDiLevel> bidi_level;
 
	for (auto run = start_run; run != last_run; run++) {
 
		bidi_level.push_back(run->level);
 
	}
 
	std::vector<int32_t> vis_to_log(bidi_level.size());
 
	ubidi_reorderVisual(bidi_level.data(), bidi_level.size(), vis_to_log.data());
 

	
 
	/* Create line. */
 
	std::unique_ptr<ICULine> line(new ICULine());
 

	
 
	int cur_pos = 0;
 
	for (auto &i : vis_to_log) {
 
		auto i_run = start_run + i;
 
		/* Copy the ICURun here, so we can modify it in case of a partial. */
 
		ICURun run = *i_run;
 

	
 
		if (i_run == last_run - 1 && new_partial_length > 0) {
 
			if (i_run == start_run && this->partial_offset > 0) {
 
				assert(run.length > this->partial_offset);
 
				run.start += this->partial_offset;
 
				run.length -= this->partial_offset;
 
			}
 

	
 
			assert(run.length > new_partial_length);
 
			run.length = new_partial_length;
 

	
 
			run.Shape(this->buff, this->buff_length);
 
		} else if (i_run == start_run && this->partial_offset > 0) {
 
			assert(run.length > this->partial_offset);
 

	
 
			run.start += this->partial_offset;
 
			run.length -= this->partial_offset;
 

	
 
			run.Shape(this->buff, this->buff_length);
 
		}
 

	
 
		auto total_advance = run.total_advance;
 
		line->emplace_back(std::move(run), cur_pos);
 
		cur_pos += total_advance;
 
	}
 

	
 
	if (new_partial_length > 0) {
 
		this->current_run = last_run - 1;
 
		this->partial_offset += new_partial_length;
 
	} else {
 
		this->current_run = last_run;
 
		this->partial_offset = 0;
 
	}
 

	
 
	return line;
 
}
 

	
 
/* static */ size_t ICUParagraphLayoutFactory::AppendToBuffer(UChar *buff, const UChar *buffer_last, WChar c)
 
{
 
	/* Transform from UTF-32 to internal ICU format of UTF-16. */
 
	int32 length = 0;
src/gfx_layout_icu.h
Show inline comments
 
@@ -8,12 +8,13 @@
 
/** @file gfx_layout_icu.h Functions related to laying out the texts with ICU. */
 

	
 
#ifndef GFX_LAYOUT_ICU_H
 
#define GFX_LAYOUT_ICU_H
 

	
 
#include "gfx_layout.h"
 

	
 
#include <unicode/ustring.h>
 

	
 
/**
 
 * Helper class to construct a new #ICUParagraphLayout.
 
 */
 
class ICUParagraphLayoutFactory {
src/strings.cpp
Show inline comments
 
@@ -2249,13 +2249,13 @@ void CheckForMissingGlyphs(bool base_fon
 
		return;
 
	}
 

	
 
	/* Update the font with cache */
 
	LoadStringWidthTable(searcher->Monospace());
 

	
 
#if !defined(WITH_ICU_LX) && !defined(WITH_UNISCRIBE) && !defined(WITH_COCOA)
 
#if !(defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)) && !defined(WITH_UNISCRIBE) && !defined(WITH_COCOA)
 
	/*
 
	 * For right-to-left languages we need the ICU library. If
 
	 * we do not have support for that library we warn the user
 
	 * about it with a message. As we do not want the string to
 
	 * be translated by the translators, we 'force' it into the
 
	 * binary and 'load' it via a BindCString. To do this
 
@@ -2264,13 +2264,13 @@ void CheckForMissingGlyphs(bool base_fon
 
	 * 'character' might change in the future, so for safety
 
	 * we just Utf8 Encode it into the string, which takes
 
	 * exactly three characters, so it replaces the "XXX" with
 
	 * the colour marker.
 
	 */
 
	if (_current_text_dir != TD_LTR) {
 
		static std::string err_str("XXXThis version of OpenTTD does not support right-to-left languages. Recompile with icu enabled.");
 
		static std::string err_str("XXXThis version of OpenTTD does not support right-to-left languages. Recompile with ICU + Harfbuzz enabled.");
 
		Utf8Encode(err_str.data(), SCC_YELLOW);
 
		SetDParamStr(0, err_str);
 
		ShowErrorMessage(STR_JUST_RAW_STRING, INVALID_STRING_ID, WL_ERROR);
 
	}
 
#endif /* !WITH_ICU_LX */
 
#endif /* !(WITH_ICU_I18N && WITH_HARFBUZZ) && !WITH_UNISCRIBE && !WITH_COCOA */
 
}
0 comments (0 inline, 0 general)