Changeset - r27221:3735cc2cef6b
[Not reviewed]
master
0 17 5
Patric Stout - 12 months ago 2023-04-30 18:37:40
truebrain@openttd.org
Feature: drop ICU-lx in favour of directly interfacing with harfbuzz

This means we have RTL support again with ICU 58+. It makes use of:
- ICU for bidi-itemization
- ICU for script-itemization
- OpenTTD for style-itemization
- harfbuzz for shaping
22 files changed with 1018 insertions and 190 deletions:
0 comments (0 inline, 0 general)
.github/workflows/ci-build.yml
Show inline comments
 
@@ -113,6 +113,7 @@ jobs:
 
          liballegro4-dev \
 
          libcurl4-openssl-dev \
 
          libfontconfig-dev \
 
          libharfbuzz-dev \
 
          libicu-dev \
 
          liblzma-dev \
 
          liblzo2-dev \
.github/workflows/release-linux.yml
Show inline comments
 
@@ -34,15 +34,20 @@ jobs:
 
    - name: Install dependencies
 
      run: |
 
        echo "::group::Install system dependencies"
 
        # ICU is used as vcpkg fails to install ICU. Other dependencies
 
        # are needed either for vcpkg or for the packages installed with
 
        # vcpkg.
 
        # perl-IPC-Cmd, wget, and zip are needed to run vcpkg.
 
        # autoconf-archive is needed to build ICU.
 
        yum install -y \
 
          libicu-devel \
 
          autoconf-archive \
 
          perl-IPC-Cmd \
 
          wget \
 
          zip \
 
          # EOF
 

	
 
        # aclocal looks first in /usr/local/share/aclocal, and if that doesn't
 
        # exist only looks in /usr/share/aclocal. We have files in both that
 
        # are important. So copy the latter to the first, and we are good to
 
        # go.
 
        cp /usr/share/aclocal/* /usr/local/share/aclocal/
 
        echo "::endgroup::"
 

	
 
        # We use vcpkg for our dependencies, to get more up-to-date version.
 
@@ -69,6 +74,8 @@ jobs:
 
            curl[http2] \
 
            fontconfig \
 
            freetype \
 
            harfbuzz \
 
            icu \
 
            liblzma \
 
            libpng \
 
            lzo \
CMakeLists.txt
Show inline comments
 
@@ -143,7 +143,8 @@ if(NOT OPTION_DEDICATED)
 
            endif()
 
            find_package(Fluidsynth)
 
            find_package(Fontconfig)
 
            find_package(ICU OPTIONAL_COMPONENTS i18n lx)
 
            find_package(Harfbuzz)
 
            find_package(ICU OPTIONAL_COMPONENTS i18n)
 
        endif()
 
    endif()
 
endif()
 
@@ -178,6 +179,12 @@ if(UNIX AND NOT APPLE AND NOT OPTION_DED
 
    if(NOT SDL_FOUND AND NOT SDL2_FOUND AND NOT ALLEGRO_FOUND)
 
        message(FATAL_ERROR "SDL, SDL2 or Allegro is required for this platform")
 
    endif()
 
    if(HARFBUZZ_FOUND AND NOT ICU_i18n_FOUND)
 
        message(WARNING "HarfBuzz depends on ICU i18n to function; HarfBuzz will be disabled")
 
    endif()
 
    if(NOT HARFBUZZ_FOUND)
 
        message(WARNING "Without HarfBuzz and ICU i18n the game will not be able to render right-to-left languages correctly")
 
    endif()
 
endif()
 
if(APPLE)
 
    if(NOT AUDIOTOOLBOX_LIBRARY)
 
@@ -289,7 +296,7 @@ if(NOT OPTION_DEDICATED)
 
    link_package(Allegro)
 
    link_package(FREETYPE TARGET Freetype::Freetype)
 
    link_package(Fontconfig TARGET Fontconfig::Fontconfig)
 
    link_package(ICU_lx)
 
    link_package(Harfbuzz TARGET harfbuzz::harfbuzz)
 
    link_package(ICU_i18n)
 

	
 
    if(SDL2_FOUND AND OPENGL_FOUND AND UNIX)
COMPILING.md
Show inline comments
 
@@ -16,6 +16,7 @@ For Linux, the following additional libr
 
- libSDL2: hardware access (video, sound, mouse)
 
- libfreetype: loading generic fonts and rendering them
 
- libfontconfig: searching for fonts, resolving font names to actual fonts
 
- harfbuzz: handling of right-to-left scripts (e.g. Arabic and Persian) (required libicu)
 
- libicu: handling of right-to-left scripts (e.g. Arabic and Persian) and
 
   natural sorting of strings
 

	
Doxyfile.in
Show inline comments
 
@@ -292,8 +292,8 @@ PREDEFINED             = WITH_ZLIB \
 
                         WITH_PNG \
 
                         WITH_FONTCONFIG \
 
                         WITH_FREETYPE \
 
                         WITH_HARFBUZZ \
 
                         WITH_ICU_I18N \
 
                         WITH_ICU_LX \
 
                         UNICODE \
 
                         _UNICODE \
 
                         _GNU_SOURCE \
README.md
Show inline comments
 
@@ -183,6 +183,9 @@ See `src/3rdparty/fmt/LICENSE.rst` for t
 
The catch2 implementation in `src/3rdparty/catch2` is licensed under the Boost Software License, Version 1.0.
 
See `src/3rdparty/catch2/LICENSE.txt` for the complete license text.
 

	
 
The icu scriptrun implementation in `src/3rdparty/icu` is licensed under the Unicode license.
 
See `src/3rdparty/icu/LICENSE` for the complete license text.
 

	
 
## 4.0 Credits
 

	
 
See [CREDITS.md](./CREDITS.md)
cmake/FindHarfbuzz.cmake
Show inline comments
 
new file 100644
 
#[=======================================================================[.rst:
 
FindHarfBuzz
 
-------
 

	
 
Finds the harfbuzz library.
 

	
 
Result Variables
 
^^^^^^^^^^^^^^^^
 

	
 
This will define the following variables:
 

	
 
``Harfbuzz_FOUND``
 
  True if the system has the harfbuzz library.
 
``Harfbuzz_INCLUDE_DIRS``
 
  Include directories needed to use harfbuzz.
 
``Harfbuzz_LIBRARIES``
 
  Libraries needed to link to harfbuzz.
 
``Harfbuzz_VERSION``
 
  The version of the harfbuzz library which was found.
 

	
 
Cache Variables
 
^^^^^^^^^^^^^^^
 

	
 
The following cache variables may also be set:
 

	
 
``Harfbuzz_INCLUDE_DIR``
 
  The directory containing ``hb.h``.
 
``Harfbuzz_LIBRARY``
 
  The path to the harfbuzz library.
 

	
 
#]=======================================================================]
 

	
 
find_package(PkgConfig QUIET)
 
pkg_check_modules(PC_Harfbuzz QUIET harfbuzz)
 

	
 
find_path(Harfbuzz_INCLUDE_DIR
 
    NAMES hb.h
 
    PATHS ${PC_Harfbuzz_INCLUDE_DIRS}
 
)
 

	
 
find_library(Harfbuzz_LIBRARY
 
    NAMES harfbuzz
 
    PATHS ${PC_Harfbuzz_LIBRARY_DIRS}
 
)
 

	
 
set(Harfbuzz_VERSION ${PC_Harfbuzz_VERSION})
 

	
 
include(FindPackageHandleStandardArgs)
 
find_package_handle_standard_args(Harfbuzz
 
    FOUND_VAR Harfbuzz_FOUND
 
    REQUIRED_VARS
 
        Harfbuzz_LIBRARY
 
        Harfbuzz_INCLUDE_DIR
 
    VERSION_VAR Harfbuzz_VERSION
 
)
 

	
 
if(Harfbuzz_FOUND)
 
    set(Harfbuzz_LIBRARIES ${Harfbuzz_LIBRARY})
 
    set(Harfbuzz_INCLUDE_DIRS ${Harfbuzz_INCLUDE_DIR})
 
endif()
 

	
 
mark_as_advanced(
 
    Harfbuzz_INCLUDE_DIR
 
    Harfbuzz_LIBRARY
 
)
cmake/FindICU.cmake
Show inline comments
 
@@ -9,7 +9,7 @@ FindICU
 

	
 
Finds components of the ICU library.
 

	
 
Accepted components are: uc, i18n, le, lx, io
 
Accepted components are: uc, i18n, le, lx, io, data
 

	
 
Result Variables
 
^^^^^^^^^^^^^^^^
 
@@ -31,7 +31,7 @@ This will define the following variables
 

	
 
find_package(PkgConfig QUIET)
 

	
 
set(ICU_KNOWN_COMPONENTS "uc" "i18n" "le" "lx" "io")
 
set(ICU_KNOWN_COMPONENTS "uc" "i18n" "le" "lx" "io" "data")
 

	
 
foreach(MOD_NAME IN LISTS ICU_FIND_COMPONENTS)
 
    if(NOT MOD_NAME IN_LIST ICU_KNOWN_COMPONENTS)
src/3rdparty/CMakeLists.txt
Show inline comments
 
add_subdirectory(catch2)
 
add_subdirectory(fmt)
 
add_subdirectory(icu)
 
add_subdirectory(md5)
 
add_subdirectory(squirrel)
 
add_subdirectory(opengl)
src/3rdparty/icu/CMakeLists.txt
Show inline comments
 
new file 100644
 
add_files(
 
    scriptrun.cpp
 
    scriptrun.h
 
    CONDITION ICU_i18n_FOUND
 
)
src/3rdparty/icu/LICENSE
Show inline comments
 
new file 100644
 
UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
 

	
 
See Terms of Use <https://www.unicode.org/copyright.html>
 
for definitions of Unicode Inc.’s Data Files and Software.
 

	
 
NOTICE TO USER: Carefully read the following legal agreement.
 
BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
 
DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
 
YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
 
TERMS AND CONDITIONS OF THIS AGREEMENT.
 
IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
 
THE DATA FILES OR SOFTWARE.
 

	
 
COPYRIGHT AND PERMISSION NOTICE
 

	
 
Copyright © 1991-2023 Unicode, Inc. All rights reserved.
 
Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
 

	
 
Permission is hereby granted, free of charge, to any person obtaining
 
a copy of the Unicode data files and any associated documentation
 
(the "Data Files") or Unicode software and any associated documentation
 
(the "Software") to deal in the Data Files or Software
 
without restriction, including without limitation the rights to use,
 
copy, modify, merge, publish, distribute, and/or sell copies of
 
the Data Files or Software, and to permit persons to whom the Data Files
 
or Software are furnished to do so, provided that either
 
(a) this copyright and permission notice appear with all copies
 
of the Data Files or Software, or
 
(b) this copyright and permission notice appear in associated
 
Documentation.
 

	
 
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
 
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
 
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 
NONINFRINGEMENT OF THIRD PARTY RIGHTS.
 
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
 
NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
 
DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 
DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
 

	
 
Except as contained in this notice, the name of a copyright holder
 
shall not be used in advertising or otherwise to promote the sale,
 
use or other dealings in these Data Files or Software without prior
 
written authorization of the copyright holder.
src/3rdparty/icu/scriptrun.cpp
Show inline comments
 
new file 100644
 
// © 2016 and later: Unicode, Inc. and others.
 
// License & terms of use: http://www.unicode.org/copyright.html
 
/*
 
 *******************************************************************************
 
 *
 
 *   Copyright (C) 1999-2016, International Business Machines
 
 *   Corporation and others.  All Rights Reserved.
 
 *
 
 *******************************************************************************
 
 *   file name:  scrptrun.cpp
 
 *
 
 *   created on: 10/17/2001
 
 *   created by: Eric R. Mader
 
 */
 

	
 
#include <unicode/utypes.h>
 
#include <unicode/uscript.h>
 

	
 
#include "scriptrun.h"
 

	
 
// Copied from cmemory.h
 
#define UPRV_LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
 

	
 
U_NAMESPACE_BEGIN
 

	
 
const char ScriptRun::fgClassID=0;
 

	
 
UChar32 ScriptRun::pairedChars[] = {
 
    0x0028, 0x0029, // ascii paired punctuation
 
    0x003c, 0x003e,
 
    0x005b, 0x005d,
 
    0x007b, 0x007d,
 
    0x00ab, 0x00bb, // guillemets
 
    0x2018, 0x2019, // general punctuation
 
    0x201c, 0x201d,
 
    0x2039, 0x203a,
 
    0x3008, 0x3009, // chinese paired punctuation
 
    0x300a, 0x300b,
 
    0x300c, 0x300d,
 
    0x300e, 0x300f,
 
    0x3010, 0x3011,
 
    0x3014, 0x3015,
 
    0x3016, 0x3017,
 
    0x3018, 0x3019,
 
    0x301a, 0x301b
 
};
 

	
 
const int32_t ScriptRun::pairedCharCount = UPRV_LENGTHOF(pairedChars);
 
const int32_t ScriptRun::pairedCharPower = 1 << highBit(pairedCharCount);
 
const int32_t ScriptRun::pairedCharExtra = pairedCharCount - pairedCharPower;
 

	
 
int8_t ScriptRun::highBit(int32_t value)
 
{
 
    if (value <= 0) {
 
        return -32;
 
    }
 

	
 
    int8_t bit = 0;
 

	
 
    if (value >= 1 << 16) {
 
        value >>= 16;
 
        bit += 16;
 
    }
 

	
 
    if (value >= 1 << 8) {
 
        value >>= 8;
 
        bit += 8;
 
    }
 

	
 
    if (value >= 1 << 4) {
 
        value >>= 4;
 
        bit += 4;
 
    }
 

	
 
    if (value >= 1 << 2) {
 
        value >>= 2;
 
        bit += 2;
 
    }
 

	
 
    if (value >= 1 << 1) {
 
        value >>= 1;
 
        bit += 1;
 
    }
 

	
 
    return bit;
 
}
 

	
 
int32_t ScriptRun::getPairIndex(UChar32 ch)
 
{
 
    int32_t probe = pairedCharPower;
 
    int32_t index = 0;
 

	
 
    if (ch >= pairedChars[pairedCharExtra]) {
 
        index = pairedCharExtra;
 
    }
 

	
 
    while (probe > (1 << 0)) {
 
        probe >>= 1;
 

	
 
        if (ch >= pairedChars[index + probe]) {
 
            index += probe;
 
        }
 
    }
 

	
 
    if (pairedChars[index] != ch) {
 
        index = -1;
 
    }
 

	
 
    return index;
 
}
 

	
 
UBool ScriptRun::sameScript(int32_t scriptOne, int32_t scriptTwo)
 
{
 
    return scriptOne <= USCRIPT_INHERITED || scriptTwo <= USCRIPT_INHERITED || scriptOne == scriptTwo;
 
}
 

	
 
UBool ScriptRun::next()
 
{
 
    int32_t startSP  = parenSP;  // used to find the first new open character
 
    UErrorCode error = U_ZERO_ERROR;
 

	
 
    // if we've fallen off the end of the text, we're done
 
    if (scriptEnd >= charLimit) {
 
        return false;
 
    }
 

	
 
    scriptCode = USCRIPT_COMMON;
 

	
 
    for (scriptStart = scriptEnd; scriptEnd < charLimit; scriptEnd += 1) {
 
        char16_t   high = charArray[scriptEnd];
 
        UChar32 ch   = high;
 

	
 
        // if the character is a high surrogate and it's not the last one
 
        // in the text, see if it's followed by a low surrogate
 
        if (high >= 0xD800 && high <= 0xDBFF && scriptEnd < charLimit - 1)
 
        {
 
            char16_t low = charArray[scriptEnd + 1];
 

	
 
            // if it is followed by a low surrogate,
 
            // consume it and form the full character
 
            if (low >= 0xDC00 && low <= 0xDFFF) {
 
                ch = (high - 0xD800) * 0x0400 + low - 0xDC00 + 0x10000;
 
                scriptEnd += 1;
 
            }
 
        }
 

	
 
        UScriptCode sc = uscript_getScript(ch, &error);
 
        int32_t pairIndex = getPairIndex(ch);
 

	
 
        // Paired character handling:
 
        //
 
        // if it's an open character, push it onto the stack.
 
        // if it's a close character, find the matching open on the
 
        // stack, and use that script code. Any non-matching open
 
        // characters above it on the stack will be poped.
 
        if (pairIndex >= 0) {
 
            if ((pairIndex & 1) == 0) {
 
                parenStack[++parenSP].pairIndex = pairIndex;
 
                parenStack[parenSP].scriptCode  = scriptCode;
 
            } else if (parenSP >= 0) {
 
                int32_t pi = pairIndex & ~1;
 

	
 
                while (parenSP >= 0 && parenStack[parenSP].pairIndex != pi) {
 
                    parenSP -= 1;
 
                }
 

	
 
                if (parenSP < startSP) {
 
                    startSP = parenSP;
 
                }
 

	
 
                if (parenSP >= 0) {
 
                    sc = parenStack[parenSP].scriptCode;
 
                }
 
            }
 
        }
 

	
 
        if (sameScript(scriptCode, sc)) {
 
            if (scriptCode <= USCRIPT_INHERITED && sc > USCRIPT_INHERITED) {
 
                scriptCode = sc;
 

	
 
                // now that we have a final script code, fix any open
 
                // characters we pushed before we knew the script code.
 
                while (startSP < parenSP) {
 
                    parenStack[++startSP].scriptCode = scriptCode;
 
                }
 
            }
 

	
 
            // if this character is a close paired character,
 
            // pop it from the stack
 
            if (pairIndex >= 0 && (pairIndex & 1) != 0 && parenSP >= 0) {
 
                parenSP -= 1;
 
                startSP -= 1;
 
            }
 
        } else {
 
            // if the run broke on a surrogate pair,
 
            // end it before the high surrogate
 
            if (ch >= 0x10000) {
 
                scriptEnd -= 1;
 
            }
 

	
 
            break;
 
        }
 
    }
 

	
 
    return true;
 
}
 

	
 
U_NAMESPACE_END
src/3rdparty/icu/scriptrun.h
Show inline comments
 
new file 100644
 
// © 2016 and later: Unicode, Inc. and others.
 
// License & terms of use: http://www.unicode.org/copyright.html
 
/*
 
 *******************************************************************************
 
 *
 
 *   Copyright (C) 1999-2003, International Business Machines
 
 *   Corporation and others.  All Rights Reserved.
 
 *
 
 *******************************************************************************
 
 *   file name:  scrptrun.h
 
 *
 
 *   created on: 10/17/2001
 
 *   created by: Eric R. Mader
 
 */
 

	
 
#ifndef __SCRPTRUN_H
 
#define __SCRPTRUN_H
 

	
 
#include <unicode/utypes.h>
 
#include <unicode/uobject.h>
 
#include <unicode/uscript.h>
 

	
 
U_NAMESPACE_BEGIN
 

	
 
struct ScriptRecord
 
{
 
    UChar32 startChar;
 
    UChar32 endChar;
 
    UScriptCode scriptCode;
 
};
 

	
 
struct ParenStackEntry
 
{
 
    int32_t pairIndex;
 
    UScriptCode scriptCode;
 
};
 

	
 
class ScriptRun : public UObject {
 
public:
 
    ScriptRun();
 

	
 
    ScriptRun(const char16_t *chars, int32_t length);
 

	
 
    ScriptRun(const char16_t *chars, int32_t start, int32_t length);
 

	
 
    void reset();
 

	
 
    void reset(int32_t start, int32_t count);
 

	
 
    void reset(const char16_t *chars, int32_t start, int32_t length);
 

	
 
    int32_t getScriptStart();
 

	
 
    int32_t getScriptEnd();
 

	
 
    UScriptCode getScriptCode();
 

	
 
    UBool next();
 

	
 
    /**
 
     * ICU "poor man's RTTI", returns a UClassID for the actual class.
 
     *
 
     * @stable ICU 2.2
 
     */
 
    virtual inline UClassID getDynamicClassID() const override { return getStaticClassID(); }
 

	
 
    /**
 
     * ICU "poor man's RTTI", returns a UClassID for this class.
 
     *
 
     * @stable ICU 2.2
 
     */
 
    static inline UClassID getStaticClassID() { return (UClassID)const_cast<char *>(&fgClassID); }
 

	
 
private:
 

	
 
    static UBool sameScript(int32_t scriptOne, int32_t scriptTwo);
 

	
 
    int32_t charStart;
 
    int32_t charLimit;
 
    const char16_t *charArray;
 

	
 
    int32_t scriptStart;
 
    int32_t scriptEnd;
 
    UScriptCode scriptCode;
 

	
 
    ParenStackEntry parenStack[128];
 
    int32_t parenSP;
 

	
 
    static int8_t highBit(int32_t value);
 
    static int32_t getPairIndex(UChar32 ch);
 

	
 
    static UChar32 pairedChars[];
 
    static const int32_t pairedCharCount;
 
    static const int32_t pairedCharPower;
 
    static const int32_t pairedCharExtra;
 

	
 
    /**
 
     * The address of this static class variable serves as this class's ID
 
     * for ICU "poor man's RTTI".
 
     */
 
    static const char fgClassID;
 
};
 

	
 
inline ScriptRun::ScriptRun()
 
{
 
    reset(nullptr, 0, 0);
 
}
 

	
 
inline ScriptRun::ScriptRun(const char16_t *chars, int32_t length)
 
{
 
    reset(chars, 0, length);
 
}
 

	
 
inline ScriptRun::ScriptRun(const char16_t *chars, int32_t start, int32_t length)
 
{
 
    reset(chars, start, length);
 
}
 

	
 
inline int32_t ScriptRun::getScriptStart()
 
{
 
    return scriptStart;
 
}
 

	
 
inline int32_t ScriptRun::getScriptEnd()
 
{
 
    return scriptEnd;
 
}
 

	
 
inline UScriptCode ScriptRun::getScriptCode()
 
{
 
    return scriptCode;
 
}
 

	
 
inline void ScriptRun::reset()
 
{
 
    scriptStart = charStart;
 
    scriptEnd   = charStart;
 
    scriptCode  = USCRIPT_INVALID_CODE;
 
    parenSP     = -1;
 
}
 

	
 
inline void ScriptRun::reset(int32_t start, int32_t length)
 
{
 
    charStart = start;
 
    charLimit = start + length;
 

	
 
    reset();
 
}
 

	
 
inline void ScriptRun::reset(const char16_t *chars, int32_t start, int32_t length)
 
{
 
    charArray = chars;
 

	
 
    reset(start, length);
 
}
 

	
 
U_NAMESPACE_END
 

	
 
#endif
src/CMakeLists.txt
Show inline comments
 
@@ -36,7 +36,7 @@ add_files(
 
add_files(
 
    gfx_layout_icu.cpp
 
    gfx_layout_icu.h
 
    CONDITION ICU_lx_FOUND
 
    CONDITION ICU_i18n_FOUND AND HARFBUZZ_FOUND
 
)
 

	
 
add_files(
src/crashlog.cpp
Show inline comments
 
@@ -50,9 +50,12 @@
 
#	include <ft2build.h>
 
#	include FT_FREETYPE_H
 
#endif /* WITH_FREETYPE */
 
#if defined(WITH_ICU_LX) || defined(WITH_ICU_I18N)
 
#ifdef WITH_HARFBUZZ
 
#	include <hb.h>
 
#endif /* WITH_HARFBUZZ */
 
#ifdef WITH_ICU_I18N
 
#	include <unicode/uversion.h>
 
#endif /* WITH_ICU_LX || WITH_ICU_I18N */
 
#endif /* WITH_ICU_I18N */
 
#ifdef WITH_LIBLZMA
 
#	include <lzma.h>
 
#endif
 
@@ -238,19 +241,18 @@ char *CrashLog::LogLibraries(char *buffe
 
	buffer += seprintf(buffer, last, " FreeType:   %d.%d.%d\n", major, minor, patch);
 
#endif /* WITH_FREETYPE */
 

	
 
#if defined(WITH_ICU_LX) || defined(WITH_ICU_I18N)
 
#if defined(WITH_HARFBUZZ)
 
	buffer += seprintf(buffer, last, " HarfBuzz:   %s\n", hb_version_string());
 
#endif /* WITH_HARFBUZZ */
 

	
 
#if defined(WITH_ICU_I18N)
 
	/* 4 times 0-255, separated by dots (.) and a trailing '\0' */
 
	char buf[4 * 3 + 3 + 1];
 
	UVersionInfo ver;
 
	u_getVersion(ver);
 
	u_versionToString(ver, buf);
 
#ifdef WITH_ICU_I18N
 
	buffer += seprintf(buffer, last, " ICU i18n:   %s\n", buf);
 
#endif
 
#ifdef WITH_ICU_LX
 
	buffer += seprintf(buffer, last, " ICU lx:     %s\n", buf);
 
#endif
 
#endif /* WITH_ICU_LX || WITH_ICU_I18N */
 
#endif /* WITH_ICU_I18N */
 

	
 
#ifdef WITH_LIBLZMA
 
	buffer += seprintf(buffer, last, " LZMA:       %s\n", lzma_version_string());
src/fontcache/freetypefontcache.cpp
Show inline comments
 
@@ -34,16 +34,17 @@ private:
 
	FT_Face face;  ///< The font face associated with this font.
 

	
 
	void SetFontSize(FontSize fs, FT_Face face, int pixels);
 
	virtual const void *InternalGetFontTable(uint32 tag, size_t &length);
 
	virtual const Sprite *InternalGetGlyph(GlyphID key, bool aa);
 
	const void *InternalGetFontTable(uint32 tag, size_t &length) override;
 
	const Sprite *InternalGetGlyph(GlyphID key, bool aa) override;
 

	
 
public:
 
	FreeTypeFontCache(FontSize fs, FT_Face face, int pixels);
 
	~FreeTypeFontCache();
 
	virtual void ClearFontCache();
 
	virtual GlyphID MapCharToGlyph(WChar key);
 
	virtual const char *GetFontName() { return face->family_name; }
 
	virtual bool IsBuiltInFont() { return false; }
 
	void ClearFontCache() override;
 
	GlyphID MapCharToGlyph(WChar key) override;
 
	const char *GetFontName() override { return face->family_name; }
 
	bool IsBuiltInFont() override { return false; }
 
	const void *GetOSHandle() override { return &face; }
 
};
 

	
 
FT_Library _library = nullptr;
src/gfx_layout.cpp
Show inline comments
 
@@ -10,17 +10,15 @@
 
#include "stdafx.h"
 
#include "gfx_layout.h"
 
#include "string_func.h"
 
#include "strings_func.h"
 
#include "zoom_func.h"
 
#include "debug.h"
 

	
 
#include "table/control_codes.h"
 

	
 
#include "gfx_layout_fallback.h"
 

	
 
#ifdef WITH_ICU_LX
 
#if defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)
 
#include "gfx_layout_icu.h"
 
#endif /* WITH_ICU_LX */
 
#endif /* WITH_ICU_I18N && WITH_HARFBUZZ */
 

	
 
#ifdef WITH_UNISCRIBE
 
#include "os/windows/string_uniscribe.h"
 
@@ -95,8 +93,8 @@ static inline void GetLayouter(Layouter:
 
			/* Filter out non printable characters */
 
			if (!IsPrintable(c)) continue;
 
			/* Filter out text direction characters that shouldn't be drawn, and
 
			 * will not be handled in the fallback non ICU case because they are
 
			 * mostly needed for RTL languages which need more ICU support. */
 
			 * will not be handled in the fallback case because they are mostly
 
			 * needed for RTL languages which need more proper shaping support. */
 
			if (!T::SUPPORTS_RTL && IsTextDirectionChar(c)) continue;
 
			buff += T::AppendToBuffer(buff, buffer_last, c);
 
			continue;
 
@@ -148,21 +146,17 @@ Layouter::Layouter(const char *str, int 
 
		} else {
 
			/* Line is new, layout it */
 
			FontState old_state = state;
 
#if defined(WITH_ICU_LX) || defined(WITH_UNISCRIBE) || defined(WITH_COCOA)
 
#if (defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)) || defined(WITH_UNISCRIBE) || defined(WITH_COCOA)
 
			const char *old_str = str;
 
#endif
 

	
 
#ifdef WITH_ICU_LX
 
			GetLayouter<ICUParagraphLayoutFactory>(line, str, state);
 
#if defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)
 
			if (line.layout == nullptr) {
 
				static bool warned = false;
 
				if (!warned) {
 
					Debug(misc, 0, "ICU layouter bailed on the font. Falling back to the fallback layouter");
 
					warned = true;
 
				GetLayouter<ICUParagraphLayoutFactory>(line, str, state);
 
				if (line.layout == nullptr) {
 
					state = old_state;
 
					str = old_str;
 
				}
 

	
 
				state = old_state;
 
				str = old_str;
 
			}
 
#endif
 

	
src/gfx_layout.h
Show inline comments
 
@@ -21,13 +21,6 @@
 
#include <type_traits>
 
#include <vector>
 

	
 
#ifdef WITH_ICU_LX
 
#include "layout/ParagraphLayout.h"
 
#define ICU_FONTINSTANCE : public icu::LEFontInstance
 
#else /* WITH_ICU_LX */
 
#define ICU_FONTINSTANCE
 
#endif /* WITH_ICU_LX */
 

	
 
/**
 
 * Text drawing parameters, which can change while drawing a line, but are kept between multiple parts
 
 * of the same text, e.g. on line breaks.
 
@@ -83,30 +76,12 @@ struct FontState {
 
/**
 
 * Container with information about a font.
 
 */
 
class Font ICU_FONTINSTANCE {
 
class Font {
 
public:
 
	FontCache *fc;     ///< The font we are using.
 
	TextColour colour; ///< The colour this font has to be.
 

	
 
	Font(FontSize size, TextColour colour);
 

	
 
#ifdef WITH_ICU_LX
 
	/* Implementation details of LEFontInstance */
 

	
 
	le_int32 getUnitsPerEM() const;
 
	le_int32 getAscent() const;
 
	le_int32 getDescent() const;
 
	le_int32 getLeading() const;
 
	float getXPixelsPerEm() const;
 
	float getYPixelsPerEm() const;
 
	float getScaleFactorX() const;
 
	float getScaleFactorY() const;
 
	const void *getFontTable(LETag tableTag) const;
 
	const void *getFontTable(LETag tableTag, size_t &length) const;
 
	LEGlyphID mapCharToGlyph(LEUnicode32 ch) const;
 
	void getGlyphAdvance(LEGlyphID glyph, LEPoint &advance) const;
 
	le_bool getGlyphPoint(LEGlyphID glyph, le_int32 pointNumber, LEPoint &point) const;
 
#endif /* WITH_ICU_LX */
 
};
 

	
 
/** Mapping from index to font. */
 
@@ -183,7 +158,7 @@ public:
 
	/** Item in the linecache */
 
	struct LineCacheItem {
 
		/* Stuff that cannot be freed until the ParagraphLayout is freed */
 
		void *buffer;              ///< Accessed by both ICU's and our ParagraphLayout::nextLine.
 
		void *buffer;              ///< Accessed by our ParagraphLayout::nextLine.
 
		FontMap runs;              ///< Accessed by our ParagraphLayout::nextLine.
 

	
 
		FontState state_after;     ///< Font state after the line.
src/gfx_layout_fallback.cpp
Show inline comments
 
@@ -33,8 +33,7 @@
 
 * The positions in a visual run are sequential pairs of X,Y of the
 
 * begin of each of the glyphs plus an extra pair to mark the end.
 
 *
 
 * @note This variant does not handle left-to-right properly. This
 
 *       is supported in the one ParagraphLayout coming from ICU.
 
 * @note This variant does not handle right-to-left properly.
 
 */
 
class FallbackParagraphLayout : public ParagraphLayouter {
 
public:
src/gfx_layout_icu.cpp
Show inline comments
 
@@ -5,170 +5,523 @@
 
 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
 
 */
 

	
 
/** @file gfx_layout_icu.cpp Handling of laying out text with ICU. */
 
/** @file gfx_layout_icu.cpp Handling of laying out with ICU / Harfbuzz. */
 

	
 
#include "stdafx.h"
 

	
 
#include "gfx_layout_icu.h"
 

	
 
#include <unicode/ustring.h>
 
#include "debug.h"
 
#include "strings_func.h"
 
#include "language.h"
 
#include "table/control_codes.h"
 
#include "zoom_func.h"
 

	
 
#include "3rdparty/icu/scriptrun.h"
 

	
 
#include <unicode/ubidi.h>
 
#include <unicode/brkiter.h>
 

	
 
#include <hb.h>
 
#include <hb-ft.h>
 

	
 
#include <deque>
 

	
 
#include "safeguards.h"
 

	
 
/* Implementation details of LEFontInstance */
 

	
 
le_int32 Font::getUnitsPerEM() const
 
{
 
	return this->fc->GetUnitsPerEM();
 
}
 

	
 
le_int32 Font::getAscent() const
 
{
 
	return this->fc->GetAscender();
 
}
 

	
 
le_int32 Font::getDescent() const
 
{
 
	return -this->fc->GetDescender();
 
}
 

	
 
le_int32 Font::getLeading() const
 
{
 
	return this->fc->GetHeight();
 
}
 

	
 
float Font::getXPixelsPerEm() const
 
{
 
	return (float)this->fc->GetHeight();
 
}
 

	
 
float Font::getYPixelsPerEm() const
 
{
 
	return (float)this->fc->GetHeight();
 
}
 
/** harfbuzz doesn't use floats, so we need a value to scale position with to get sub-pixel precision. */
 
constexpr float FONT_SCALE = 64.0;
 

	
 
float Font::getScaleFactorX() const
 
{
 
	return 1.0f;
 
}
 

	
 
float Font::getScaleFactorY() const
 
{
 
	return 1.0f;
 
}
 

	
 
const void *Font::getFontTable(LETag tableTag) const
 
{
 
	size_t length;
 
	return this->getFontTable(tableTag, length);
 
}
 
/**
 
 * Helper class to store the information of all the runs of a paragraph in.
 
 *
 
 * During itemization, more and more information is filled in.
 
 */
 
class ICURun {
 
public:
 
	int start; ///< Start of the run in the buffer.
 
	int length; ///< Length of the run in the buffer.
 
	UBiDiLevel level; ///< Embedding level of the run.
 
	UScriptCode script; ///< Script of the run.
 
	Font *font; ///< Font of the run.
 

	
 
const void *Font::getFontTable(LETag tableTag, size_t &length) const
 
{
 
	return this->fc->GetFontTable(tableTag, length);
 
}
 

	
 
LEGlyphID Font::mapCharToGlyph(LEUnicode32 ch) const
 
{
 
	if (IsTextDirectionChar(ch)) return 0;
 
	return this->fc->MapCharToGlyph(ch);
 
}
 
	std::vector<GlyphID> glyphs; ///< The glyphs of the run. Valid after Shape() is called.
 
	std::vector<int> advance; ///< The advance (width) of the glyphs. Valid after Shape() is called.
 
	std::vector<int> glyph_to_char; ///< The mapping from glyphs to characters. Valid after Shape() is called.
 
	std::vector<float> positions; ///< The positions of the glyphs. Valid after Shape() is called.
 
	int total_advance; ///< The total advance of the run. Valid after Shape() is called.
 

	
 
void Font::getGlyphAdvance(LEGlyphID glyph, LEPoint &advance) const
 
{
 
	advance.fX = glyph == 0xFFFF ? 0 : this->fc->GetGlyphWidth(glyph);
 
	advance.fY = 0;
 
}
 
	ICURun(int start, int length, UBiDiLevel level, UScriptCode script = USCRIPT_UNKNOWN, Font *font = nullptr) : start(start), length(length), level(level), script(script), font(font) {}
 

	
 
le_bool Font::getGlyphPoint(LEGlyphID glyph, le_int32 pointNumber, LEPoint &point) const
 
{
 
	return false;
 
}
 
	void Shape(UChar *buff, size_t length);
 
};
 

	
 
/**
 
 * Wrapper for doing layouts with ICU.
 
 */
 
class ICUParagraphLayout : public ParagraphLayouter {
 
	icu::ParagraphLayout *p; ///< The actual ICU paragraph layout.
 
public:
 
	/** Visual run contains data about the bit of text with the same font. */
 
	class ICUVisualRun : public ParagraphLayouter::VisualRun {
 
		const icu::ParagraphLayout::VisualRun *vr; ///< The actual ICU vr.
 
	private:
 
		std::vector<GlyphID> glyphs;
 
		std::vector<float> positions;
 
		std::vector<int> glyph_to_char;
 

	
 
		int total_advance;
 
		const Font *font;
 

	
 
	public:
 
		ICUVisualRun(const icu::ParagraphLayout::VisualRun *vr) : vr(vr) { }
 
		ICUVisualRun(const ICURun &run, int x);
 

	
 
		const Font *GetFont() const override          { return (const Font*)vr->getFont(); }
 
		int GetGlyphCount() const override            { return vr->getGlyphCount(); }
 
		const GlyphID *GetGlyphs() const override     { return vr->getGlyphs(); }
 
		const float *GetPositions() const override    { return vr->getPositions(); }
 
		int GetLeading() const override               { return vr->getLeading(); }
 
		const int *GetGlyphToCharMap() const override { return vr->getGlyphToCharMap(); }
 
		const GlyphID *GetGlyphs() const override { return this->glyphs.data(); }
 
		const float *GetPositions() const override { return this->positions.data(); }
 
		const int *GetGlyphToCharMap() const override { return this->glyph_to_char.data(); }
 

	
 
		const Font *GetFont() const override { return this->font; }
 
		int GetLeading() const override { return this->font->fc->GetHeight(); }
 
		int GetGlyphCount() const override { return this->glyphs.size(); }
 
		int GetAdvance() const { return this->total_advance; }
 
	};
 

	
 
	/** A single line worth of VisualRuns. */
 
	class ICULine : public std::vector<ICUVisualRun>, public ParagraphLayouter::Line {
 
		icu::ParagraphLayout::Line *l; ///< The actual ICU line.
 

	
 
	public:
 
		ICULine(icu::ParagraphLayout::Line *l) : l(l)
 
		{
 
			for (int i = 0; i < l->countRuns(); i++) {
 
				this->emplace_back(l->getVisualRun(i));
 
			}
 
		}
 
		~ICULine() override { delete l; }
 

	
 
		int GetLeading() const override { return l->getLeading(); }
 
		int GetWidth() const override   { return l->getWidth(); }
 
		int CountRuns() const override  { return l->countRuns(); }
 
		const ParagraphLayouter::VisualRun &GetVisualRun(int run) const override { return this->at(run); }
 
		int GetLeading() const override;
 
		int GetWidth() const override;
 
		int CountRuns() const override { return (uint)this->size();  }
 
		const VisualRun &GetVisualRun(int run) const override { return this->at(run); }
 

	
 
		int GetInternalCharLength(WChar c) const override
 
		{
 
			/* ICU uses UTF-16 internally which means we need to account for surrogate pairs. */
 
			return Utf8CharLen(c) < 4 ? 1 : 2;
 
			return c >= 0x010000U ? 2 : 1;
 
		}
 
	};
 

	
 
	ICUParagraphLayout(icu::ParagraphLayout *p) : p(p) { }
 
	~ICUParagraphLayout() override { delete p; }
 
	void Reflow() override  { p->reflow(); }
 
private:
 
	std::vector<ICURun> runs;
 
	UChar *buff;
 
	size_t buff_length;
 
	std::vector<ICURun>::iterator current_run;
 
	int partial_offset;
 

	
 
	std::unique_ptr<const Line> NextLine(int max_width) override
 
public:
 
	ICUParagraphLayout(std::vector<ICURun> &runs, UChar *buff, size_t buff_length) : runs(runs), buff(buff), buff_length(buff_length)
 
	{
 
		icu::ParagraphLayout::Line *l = p->nextLine(max_width);
 
		return std::unique_ptr<const Line>(l == nullptr ? nullptr : new ICULine(l));
 
		this->Reflow();
 
	}
 

	
 
	~ICUParagraphLayout() override { }
 

	
 
	void Reflow() override
 
	{
 
		this->current_run = this->runs.begin();
 
		this->partial_offset = 0;
 
	}
 

	
 
	std::unique_ptr<const Line> NextLine(int max_width) override;
 
};
 

	
 
/* static */ ParagraphLayouter *ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &fontMapping)
 
/**
 
 * Constructor for a new ICUVisualRun.
 
 *
 
 * It bases all information on the ICURun, which should already be shaped.
 
 *
 
 * @param run The ICURun to base the visual run on.
 
 * @param x The offset of the run on the line.
 
 */
 
ICUParagraphLayout::ICUVisualRun::ICUVisualRun(const ICURun &run, int x) :
 
	glyphs(run.glyphs), glyph_to_char(run.glyph_to_char), total_advance(run.total_advance), font(run.font)
 
{
 
	int32 length = buff_end - buff;
 
	/* If there are no positions, the ICURun was not Shaped; that should never happen. */
 
	assert(run.positions.size() != 0);
 
	this->positions.reserve(run.positions.size());
 

	
 
	/* "positions" is an array of x/y. So we need to alternate. */
 
	bool is_x = true;
 
	for (auto &position : run.positions) {
 
		if (is_x) {
 
			this->positions.push_back(position + x);
 
		} else {
 
			this->positions.push_back(position);
 
		}
 

	
 
		is_x = !is_x;
 
	}
 
}
 

	
 
/**
 
 * Shape a single run.
 
 *
 
 * @param buff The buffer of which a partial (depending on start/length of the run) will be shaped.
 
 * @param length The length of the buffer.
 
 */
 
void ICURun::Shape(UChar *buff, size_t buff_length) {
 
	auto hbfont = hb_ft_font_create_referenced(*(static_cast<const FT_Face *>(font->fc->GetOSHandle())));
 
	hb_font_set_scale(hbfont, this->font->fc->GetFontSize() * FONT_SCALE, this->font->fc->GetFontSize() * FONT_SCALE);
 

	
 
	/* ICU buffer is in UTF-16. */
 
	auto hbbuf = hb_buffer_create();
 
	hb_buffer_add_utf16(hbbuf, reinterpret_cast<uint16 *>(buff), buff_length, this->start, this->length);
 

	
 
	/* Set all the properties of this segment. */
 
	hb_buffer_set_direction(hbbuf, (this->level & 1) == 1 ? HB_DIRECTION_RTL : HB_DIRECTION_LTR);
 
	hb_buffer_set_script(hbbuf, hb_script_from_string(uscript_getShortName(this->script), -1));
 
	hb_buffer_set_language(hbbuf, hb_language_from_string(_current_language->isocode, -1));
 
	hb_buffer_set_cluster_level(hbbuf, HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES);
 

	
 
	/* Shape the segment. */
 
	hb_shape(hbfont, hbbuf, nullptr, 0);
 

	
 
	unsigned int glyph_count;
 
	auto glyph_info = hb_buffer_get_glyph_infos(hbbuf, &glyph_count);
 
	auto glyph_pos = hb_buffer_get_glyph_positions(hbbuf, &glyph_count);
 

	
 
	/* Make sure any former run is lost. */
 
	this->glyphs.clear();
 
	this->glyph_to_char.clear();
 
	this->positions.clear();
 
	this->advance.clear();
 

	
 
	/* Reserve space, as we already know the size. */
 
	this->glyphs.reserve(glyph_count);
 
	this->glyph_to_char.reserve(glyph_count);
 
	this->positions.reserve(glyph_count * 2 + 2);
 
	this->advance.reserve(glyph_count);
 

	
 
	/* Prepare the glyphs/position. ICUVisualRun will give the position an offset if needed. */
 
	hb_position_t advance = 0;
 
	for (unsigned int i = 0; i < glyph_count; i++) {
 
		int x_advance;
 

	
 
		if (buff[glyph_info[i].cluster] >= SCC_SPRITE_START && buff[glyph_info[i].cluster] <= SCC_SPRITE_END) {
 
			auto glyph = this->font->fc->MapCharToGlyph(buff[glyph_info[i].cluster]);
 

	
 
			this->glyphs.push_back(glyph);
 
			this->positions.push_back(advance);
 
			this->positions.push_back((this->font->fc->GetHeight() - ScaleSpriteTrad(FontCache::GetDefaultFontHeight(this->font->fc->GetSize()))) / 2); // Align sprite font to centre
 
			x_advance = this->font->fc->GetGlyphWidth(glyph);
 
		} else {
 
			this->glyphs.push_back(glyph_info[i].codepoint);
 
			this->positions.push_back(glyph_pos[i].x_offset / FONT_SCALE + advance);
 
			this->positions.push_back(glyph_pos[i].y_offset / FONT_SCALE);
 
			x_advance = glyph_pos[i].x_advance / FONT_SCALE;
 
		}
 

	
 
	if (length == 0) {
 
		/* ICU's ParagraphLayout cannot handle empty strings, so fake one. */
 
		buff[0] = ' ';
 
		length = 1;
 
		fontMapping.back().first++;
 
		this->glyph_to_char.push_back(glyph_info[i].cluster);
 
		this->advance.push_back(x_advance);
 
		advance += x_advance;
 
	}
 

	
 
	/* Position has one more element to close off the array. */
 
	this->positions.push_back(advance);
 
	this->positions.push_back(0);
 

	
 
	/* Track the total advancement we made. */
 
	this->total_advance = advance;
 

	
 
	hb_buffer_destroy(hbbuf);
 
	hb_font_destroy(hbfont);
 
}
 

	
 
/**
 
 * Get the height of the line.
 
 * @return The maximum height of the line.
 
 */
 
int ICUParagraphLayout::ICULine::GetLeading() const
 
{
 
	int leading = 0;
 
	for (const auto &run : *this) {
 
		leading = std::max(leading, run.GetLeading());
 
	}
 

	
 
	return leading;
 
}
 

	
 
/**
 
 * Get the width of this line.
 
 * @return The width of the line.
 
 */
 
int ICUParagraphLayout::ICULine::GetWidth() const
 
{
 
	int length = 0;
 
	for (const auto &run : *this) {
 
		length += run.GetAdvance();
 
	}
 

	
 
	return length;
 
}
 

	
 
/**
 
 * Itemize the string into runs per embedding level.
 
 *
 
 * Later on, based on the levels, we can deduce the order of a subset of runs.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeBidi(UChar *buff, size_t length)
 
{
 
	auto ubidi = ubidi_open();
 

	
 
	auto parLevel = _current_text_dir == TD_RTL ? UBIDI_RTL : UBIDI_LTR;
 

	
 
	UErrorCode err = U_ZERO_ERROR;
 
	ubidi_setPara(ubidi, buff, length, parLevel, nullptr, &err);
 
	if (U_FAILURE(err)) {
 
		Debug(fontcache, 0, "Failed to set paragraph: %s", u_errorName(err));
 
		ubidi_close(ubidi);
 
		return std::vector<ICURun>();
 
	}
 

	
 
	int32_t count = ubidi_countRuns(ubidi, &err);
 
	if (U_FAILURE(err)) {
 
		Debug(fontcache, 0, "Failed to count runs: %s", u_errorName(err));
 
		ubidi_close(ubidi);
 
		return std::vector<ICURun>();
 
	}
 

	
 
	std::vector<ICURun> runs;
 
	runs.reserve(count);
 

	
 
	/* Find the breakpoints for the logical runs. So we get runs that say "from START to END". */
 
	int32_t logical_pos = 0;
 
	while (static_cast<size_t>(logical_pos) < length) {
 
		auto start_pos = logical_pos;
 

	
 
		/* Fetch the embedding level, so we can order bidi correctly later on. */
 
		UBiDiLevel level;
 
		ubidi_getLogicalRun(ubidi, start_pos, &logical_pos, &level);
 

	
 
		runs.emplace_back(ICURun(start_pos, logical_pos - start_pos, level));
 
	}
 

	
 
	/* Fill ICU's FontRuns with the right data. */
 
	icu::FontRuns runs(fontMapping.size());
 
	for (auto &pair : fontMapping) {
 
		runs.add(pair.second, pair.first);
 
	assert(static_cast<size_t>(count) == runs.size());
 

	
 
	ubidi_close(ubidi);
 
	return runs;
 
}
 

	
 
/**
 
 * Itemize the string into runs per script, based on the previous created runs.
 
 *
 
 * Basically, this always returns the same or more runs than given.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @param runs_current The current runs.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeScript(UChar *buff, size_t length, std::vector<ICURun> &runs_current)
 
{
 
	std::vector<ICURun> runs;
 
	icu::ScriptRun script_itemizer(buff, length);
 

	
 
	int cur_pos = 0;
 
	auto cur_run = runs_current.begin();
 
	while (true) {
 
		while (cur_pos < script_itemizer.getScriptEnd() && cur_run != runs_current.end()) {
 
			int stop_pos = std::min(script_itemizer.getScriptEnd(), cur_run->start + cur_run->length);
 
			assert(stop_pos - cur_pos > 0);
 

	
 
			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, script_itemizer.getScriptCode()));
 

	
 
			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
 
			cur_pos = stop_pos;
 
		}
 

	
 
		if (!script_itemizer.next()) break;
 
	}
 

	
 
	return runs;
 
}
 

	
 
/**
 
 * Itemize the string into runs per style, based on the previous created runs.
 
 *
 
 * Basically, this always returns the same or more runs than given.
 
 *
 
 * @param buff The string to itemize.
 
 * @param length The length of the string.
 
 * @param runs_current The current runs.
 
 * @param font_mapping The font mapping.
 
 * @return The runs.
 
 */
 
std::vector<ICURun> ItemizeStyle(UChar *buff, size_t length, std::vector<ICURun> &runs_current, FontMap &font_mapping)
 
{
 
	std::vector<ICURun> runs;
 

	
 
	int cur_pos = 0;
 
	auto cur_run = runs_current.begin();
 
	for (auto const &font_map : font_mapping) {
 
		while (cur_pos < font_map.first && cur_run != runs_current.end()) {
 
			int stop_pos = std::min(font_map.first, cur_run->start + cur_run->length);
 
			assert(stop_pos - cur_pos > 0);
 

	
 
			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, cur_run->script, font_map.second));
 

	
 
			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
 
			cur_pos = stop_pos;
 
		}
 
	}
 

	
 
	return runs;
 
}
 

	
 
/* static */ ParagraphLayouter *ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &font_mapping)
 
{
 
	size_t length = buff_end - buff;
 
	/* Can't layout an empty string. */
 
	if (length == 0) return nullptr;
 

	
 
	/* Can't layout our in-built sprite fonts. */
 
	for (auto const &pair : font_mapping) {
 
		if (pair.second->fc->IsBuiltInFont()) return nullptr;
 
	}
 

	
 
	auto runs = ItemizeBidi(buff, length);
 
	runs = ItemizeScript(buff, length, runs);
 
	runs = ItemizeStyle(buff, length, runs, font_mapping);
 

	
 
	if (runs.size() == 0) return nullptr;
 

	
 
	for (auto &run : runs) {
 
		run.Shape(buff, length);
 
	}
 

	
 
	LEErrorCode status = LE_NO_ERROR;
 
	/* ParagraphLayout does not copy "buff", so it must stay valid.
 
		* "runs" is copied according to the ICU source, but the documentation does not specify anything, so this might break somewhen. */
 
	icu::ParagraphLayout *p = new icu::ParagraphLayout(buff, length, &runs, nullptr, nullptr, nullptr, _current_text_dir == TD_RTL ? 1 : 0, false, status);
 
	if (status != LE_NO_ERROR) {
 
		delete p;
 
		return nullptr;
 
	return new ICUParagraphLayout(runs, buff, length);
 
}
 

	
 
std::unique_ptr<const ICUParagraphLayout::Line> ICUParagraphLayout::NextLine(int max_width)
 
{
 
	std::vector<ICURun>::iterator start_run = this->current_run;
 
	std::vector<ICURun>::iterator last_run = this->current_run;
 

	
 
	if (start_run == this->runs.end()) return nullptr;
 

	
 
	int cur_width = 0;
 

	
 
	/* Add remaining width of the first run if it is a broken run. */
 
	if (this->partial_offset > 0) {
 
		if ((start_run->level & 1) == 0) {
 
			for (size_t i = this->partial_offset; i < start_run->advance.size(); i++) {
 
				cur_width += start_run->advance[i];
 
			}
 
		} else {
 
			for (int i = 0; i < this->partial_offset; i++) {
 
				cur_width += start_run->advance[i];
 
			}
 
		}
 
		last_run++;
 
	}
 

	
 
	/* Gather runs until the line is full. */
 
	while (last_run != this->runs.end() && cur_width < max_width) {
 
		cur_width += last_run->total_advance;
 
		last_run++;
 
	}
 

	
 
	return new ICUParagraphLayout(p);
 
	/* If the text does not fit into the available width, find a suitable breaking point. */
 
	int new_partial_length = 0;
 
	if (cur_width > max_width) {
 
		auto locale = icu::Locale(_current_language->isocode);
 

	
 
		/* Create a break-iterator to find a good place to break lines. */
 
		UErrorCode err = U_ZERO_ERROR;
 
		auto break_iterator = icu::BreakIterator::createLineInstance(locale, err);
 
		break_iterator->setText(icu::UnicodeString(this->buff, this->buff_length));
 

	
 
		auto overflow_run = last_run - 1;
 

	
 
		/* Find the last glyph that fits. */
 
		size_t index;
 
		if ((overflow_run->level & 1) == 0) {
 
			/* LTR */
 
			for (index = overflow_run->glyphs.size(); index > 0; index--) {
 
				cur_width -= overflow_run->advance[index - 1];
 
				if (cur_width <= max_width) break;
 
			}
 
			index--;
 
		} else {
 
			/* RTL */
 
			for (index = 0; index < overflow_run->glyphs.size(); index++) {
 
				cur_width -= overflow_run->advance[index];
 
				if (cur_width <= max_width) break;
 
			}
 
		}
 

	
 
		/* Find the character that matches; this is the start of the cluster. */
 
		auto char_pos = overflow_run->glyph_to_char[index];
 

	
 
		/* See if there is a good breakpoint inside this run. */
 
		int32_t break_pos = break_iterator->preceding(char_pos + 1);
 
		if (break_pos != icu::BreakIterator::DONE && break_pos > overflow_run->start + this->partial_offset) {
 
			/* There is a line-break inside this run that is suitable. */
 
			new_partial_length = break_pos - overflow_run->start - this->partial_offset;
 
		} else if (overflow_run != start_run) {
 
			/* There is no suitable line-break in this run, but it is also not
 
			 * the only run on this line. So we remove the run. */
 
			last_run--;
 
		} else {
 
			/* There is no suitable line-break and this is the only run on the
 
			 * line. So we break at the cluster. This is not pretty, but the
 
			 * best we can do. */
 
			new_partial_length = char_pos - this->partial_offset;
 
		}
 
	}
 

	
 
	/* Reorder the runs on this line for display. */
 
	std::vector<UBiDiLevel> bidi_level;
 
	for (auto run = start_run; run != last_run; run++) {
 
		bidi_level.push_back(run->level);
 
	}
 
	std::vector<int32_t> vis_to_log(bidi_level.size());
 
	ubidi_reorderVisual(bidi_level.data(), bidi_level.size(), vis_to_log.data());
 

	
 
	/* Create line. */
 
	std::unique_ptr<ICULine> line(new ICULine());
 

	
 
	int cur_pos = 0;
 
	for (auto &i : vis_to_log) {
 
		auto i_run = start_run + i;
 
		/* Copy the ICURun here, so we can modify it in case of a partial. */
 
		ICURun run = *i_run;
 

	
 
		if (i_run == last_run - 1 && new_partial_length > 0) {
 
			if (i_run == start_run && this->partial_offset > 0) {
 
				assert(run.length > this->partial_offset);
 
				run.start += this->partial_offset;
 
				run.length -= this->partial_offset;
 
			}
 

	
 
			assert(run.length > new_partial_length);
 
			run.length = new_partial_length;
 

	
 
			run.Shape(this->buff, this->buff_length);
 
		} else if (i_run == start_run && this->partial_offset > 0) {
 
			assert(run.length > this->partial_offset);
 

	
 
			run.start += this->partial_offset;
 
			run.length -= this->partial_offset;
 

	
 
			run.Shape(this->buff, this->buff_length);
 
		}
 

	
 
		auto total_advance = run.total_advance;
 
		line->emplace_back(std::move(run), cur_pos);
 
		cur_pos += total_advance;
 
	}
 

	
 
	if (new_partial_length > 0) {
 
		this->current_run = last_run - 1;
 
		this->partial_offset += new_partial_length;
 
	} else {
 
		this->current_run = last_run;
 
		this->partial_offset = 0;
 
	}
 

	
 
	return line;
 
}
 

	
 
/* static */ size_t ICUParagraphLayoutFactory::AppendToBuffer(UChar *buff, const UChar *buffer_last, WChar c)
src/gfx_layout_icu.h
Show inline comments
 
@@ -11,6 +11,7 @@
 
#define GFX_LAYOUT_ICU_H
 

	
 
#include "gfx_layout.h"
 

	
 
#include <unicode/ustring.h>
 

	
 
/**
src/strings.cpp
Show inline comments
 
@@ -2252,7 +2252,7 @@ void CheckForMissingGlyphs(bool base_fon
 
	/* Update the font with cache */
 
	LoadStringWidthTable(searcher->Monospace());
 

	
 
#if !defined(WITH_ICU_LX) && !defined(WITH_UNISCRIBE) && !defined(WITH_COCOA)
 
#if !(defined(WITH_ICU_I18N) && defined(WITH_HARFBUZZ)) && !defined(WITH_UNISCRIBE) && !defined(WITH_COCOA)
 
	/*
 
	 * For right-to-left languages we need the ICU library. If
 
	 * we do not have support for that library we warn the user
 
@@ -2267,10 +2267,10 @@ void CheckForMissingGlyphs(bool base_fon
 
	 * the colour marker.
 
	 */
 
	if (_current_text_dir != TD_LTR) {
 
		static std::string err_str("XXXThis version of OpenTTD does not support right-to-left languages. Recompile with icu enabled.");
 
		static std::string err_str("XXXThis version of OpenTTD does not support right-to-left languages. Recompile with ICU + Harfbuzz enabled.");
 
		Utf8Encode(err_str.data(), SCC_YELLOW);
 
		SetDParamStr(0, err_str);
 
		ShowErrorMessage(STR_JUST_RAW_STRING, INVALID_STRING_ID, WL_ERROR);
 
	}
 
#endif /* !WITH_ICU_LX */
 
#endif /* !(WITH_ICU_I18N && WITH_HARFBUZZ) && !WITH_UNISCRIBE && !WITH_COCOA */
 
}
0 comments (0 inline, 0 general)