Files @ r28616:1df48a6fd940
Branch filter:

Location: cpp/openttd-patchpack/source/src/gfx_layout_icu.cpp

glx22
Fix 48b6b18: Increase MAX_VALUATE_OPS to match the previous limit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
/*
 * This file is part of OpenTTD.
 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
 */

/** @file gfx_layout_icu.cpp Handling of laying out with ICU / Harfbuzz. */

#include "stdafx.h"
#include "gfx_layout_icu.h"

#include "debug.h"
#include "strings_func.h"
#include "language.h"
#include "table/control_codes.h"
#include "zoom_func.h"

#include "3rdparty/icu/scriptrun.h"

#include <unicode/ubidi.h>
#include <unicode/brkiter.h>

#include <hb.h>
#include <hb-ft.h>

#include "safeguards.h"

/** HarfBuzz FreeType integration sets the font scaling, which is always in 1/64th of a pixel. */
constexpr float FONT_SCALE = 64.0;

/**
 * Helper class to store the information of all the runs of a paragraph in.
 *
 * During itemization, more and more information is filled in.
 */
class ICURun {
public:
	int start; ///< Start of the run in the buffer.
	int length; ///< Length of the run in the buffer.
	UBiDiLevel level; ///< Embedding level of the run.
	UScriptCode script; ///< Script of the run.
	Font *font; ///< Font of the run.

	std::vector<GlyphID> glyphs; ///< The glyphs of the run. Valid after Shape() is called.
	std::vector<int> advance; ///< The advance (width) of the glyphs. Valid after Shape() is called.
	std::vector<int> glyph_to_char; ///< The mapping from glyphs to characters. Valid after Shape() is called.
	std::vector<Point> positions; ///< The positions of the glyphs. Valid after Shape() is called.
	int total_advance = 0; ///< The total advance of the run. Valid after Shape() is called.

	ICURun(int start, int length, UBiDiLevel level, UScriptCode script = USCRIPT_UNKNOWN, Font *font = nullptr) : start(start), length(length), level(level), script(script), font(font) {}

	void Shape(UChar *buff, size_t length);
};

/**
 * Wrapper for doing layouts with ICU.
 */
class ICUParagraphLayout : public ParagraphLayouter {
public:
	/** Visual run contains data about the bit of text with the same font. */
	class ICUVisualRun : public ParagraphLayouter::VisualRun {
	private:
		std::vector<GlyphID> glyphs;
		std::vector<Point> positions;
		std::vector<int> glyph_to_char;

		int total_advance;
		const Font *font;

	public:
		ICUVisualRun(const ICURun &run, int x);

		const std::vector<GlyphID> &GetGlyphs() const override { return this->glyphs; }
		const std::vector<Point> &GetPositions() const override { return this->positions; }
		const std::vector<int> &GetGlyphToCharMap() const override { return this->glyph_to_char; }

		const Font *GetFont() const override { return this->font; }
		int GetLeading() const override { return this->font->fc->GetHeight(); }
		int GetGlyphCount() const override { return this->glyphs.size(); }
		int GetAdvance() const { return this->total_advance; }
	};

	/** A single line worth of VisualRuns. */
	class ICULine : public std::vector<ICUVisualRun>, public ParagraphLayouter::Line {
	public:
		int GetLeading() const override;
		int GetWidth() const override;
		int CountRuns() const override { return (uint)this->size();  }
		const VisualRun &GetVisualRun(int run) const override { return this->at(run); }

		int GetInternalCharLength(char32_t c) const override
		{
			/* ICU uses UTF-16 internally which means we need to account for surrogate pairs. */
			return c >= 0x010000U ? 2 : 1;
		}
	};

private:
	std::vector<ICURun> runs;
	UChar *buff;
	size_t buff_length;
	std::vector<ICURun>::iterator current_run;
	int partial_offset;

public:
	ICUParagraphLayout(std::vector<ICURun> &runs, UChar *buff, size_t buff_length) : runs(runs), buff(buff), buff_length(buff_length)
	{
		this->Reflow();
	}

	~ICUParagraphLayout() override { }

	void Reflow() override
	{
		this->current_run = this->runs.begin();
		this->partial_offset = 0;
	}

	std::unique_ptr<const Line> NextLine(int max_width) override;
};

/**
 * Constructor for a new ICUVisualRun.
 *
 * It bases all information on the ICURun, which should already be shaped.
 *
 * @param run The ICURun to base the visual run on.
 * @param x The offset of the run on the line.
 */
ICUParagraphLayout::ICUVisualRun::ICUVisualRun(const ICURun &run, int x) :
	glyphs(run.glyphs), glyph_to_char(run.glyph_to_char), total_advance(run.total_advance), font(run.font)
{
	/* If there are no positions, the ICURun was not Shaped; that should never happen. */
	assert(!run.positions.empty());
	this->positions.reserve(run.positions.size());

	/* Copy positions, moving x coordinate by x offset. */
	for (const Point &pt : run.positions) {
		this->positions.emplace_back(pt.x + x, pt.y);
	}
}

/**
 * Shape a single run.
 *
 * @param buff The buffer of which a partial (depending on start/length of the run) will be shaped.
 * @param length The length of the buffer.
 */
void ICURun::Shape(UChar *buff, size_t buff_length)
{
	auto hbfont = hb_ft_font_create_referenced(*(static_cast<const FT_Face *>(font->fc->GetOSHandle())));
	/* Match the flags with how we render the glyphs. */
	hb_ft_font_set_load_flags(hbfont, GetFontAAState(this->font->fc->GetSize()) ? FT_LOAD_TARGET_NORMAL : FT_LOAD_TARGET_MONO);

	/* ICU buffer is in UTF-16. */
	auto hbbuf = hb_buffer_create();
	hb_buffer_add_utf16(hbbuf, reinterpret_cast<uint16_t *>(buff), buff_length, this->start, this->length);

	/* Set all the properties of this segment. */
	hb_buffer_set_direction(hbbuf, (this->level & 1) == 1 ? HB_DIRECTION_RTL : HB_DIRECTION_LTR);
	hb_buffer_set_script(hbbuf, hb_script_from_string(uscript_getShortName(this->script), -1));
	hb_buffer_set_language(hbbuf, hb_language_from_string(_current_language->isocode, -1));
	hb_buffer_set_cluster_level(hbbuf, HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES);

	/* Shape the segment. */
	hb_shape(hbfont, hbbuf, nullptr, 0);

	unsigned int glyph_count;
	auto glyph_info = hb_buffer_get_glyph_infos(hbbuf, &glyph_count);
	auto glyph_pos = hb_buffer_get_glyph_positions(hbbuf, &glyph_count);

	/* Make sure any former run is lost. */
	this->glyphs.clear();
	this->glyph_to_char.clear();
	this->positions.clear();
	this->advance.clear();

	/* Reserve space, as we already know the size. */
	this->glyphs.reserve(glyph_count);
	this->glyph_to_char.reserve(glyph_count);
	this->positions.reserve(glyph_count + 1);
	this->advance.reserve(glyph_count);

	/* Prepare the glyphs/position. ICUVisualRun will give the position an offset if needed. */
	hb_position_t advance = 0;
	for (unsigned int i = 0; i < glyph_count; i++) {
		int x_advance;

		if (buff[glyph_info[i].cluster] >= SCC_SPRITE_START && buff[glyph_info[i].cluster] <= SCC_SPRITE_END && glyph_info[i].codepoint == 0) {
			auto glyph = this->font->fc->MapCharToGlyph(buff[glyph_info[i].cluster]);

			this->glyphs.push_back(glyph);
			this->positions.emplace_back(advance, (this->font->fc->GetHeight() - ScaleSpriteTrad(FontCache::GetDefaultFontHeight(this->font->fc->GetSize()))) / 2); // Align sprite font to centre
			x_advance = this->font->fc->GetGlyphWidth(glyph);
		} else {
			this->glyphs.push_back(glyph_info[i].codepoint);
			this->positions.emplace_back(glyph_pos[i].x_offset / FONT_SCALE + advance, glyph_pos[i].y_offset / FONT_SCALE);
			x_advance = glyph_pos[i].x_advance / FONT_SCALE;
		}

		this->glyph_to_char.push_back(glyph_info[i].cluster);
		this->advance.push_back(x_advance);
		advance += x_advance;
	}

	/* End-of-run position. */
	this->positions.emplace_back(advance, 0);

	/* Track the total advancement we made. */
	this->total_advance = advance;

	hb_buffer_destroy(hbbuf);
	hb_font_destroy(hbfont);
}

/**
 * Get the height of the line.
 * @return The maximum height of the line.
 */
int ICUParagraphLayout::ICULine::GetLeading() const
{
	int leading = 0;
	for (const auto &run : *this) {
		leading = std::max(leading, run.GetLeading());
	}

	return leading;
}

/**
 * Get the width of this line.
 * @return The width of the line.
 */
int ICUParagraphLayout::ICULine::GetWidth() const
{
	int length = 0;
	for (const auto &run : *this) {
		length += run.GetAdvance();
	}

	return length;
}

/**
 * Itemize the string into runs per embedding level.
 *
 * Later on, based on the levels, we can deduce the order of a subset of runs.
 *
 * @param buff The string to itemize.
 * @param length The length of the string.
 * @return The runs.
 */
std::vector<ICURun> ItemizeBidi(UChar *buff, size_t length)
{
	auto ubidi = ubidi_open();

	auto parLevel = _current_text_dir == TD_RTL ? UBIDI_RTL : UBIDI_LTR;

	UErrorCode err = U_ZERO_ERROR;
	ubidi_setPara(ubidi, buff, length, parLevel, nullptr, &err);
	if (U_FAILURE(err)) {
		Debug(fontcache, 0, "Failed to set paragraph: %s", u_errorName(err));
		ubidi_close(ubidi);
		return std::vector<ICURun>();
	}

	int32_t count = ubidi_countRuns(ubidi, &err);
	if (U_FAILURE(err)) {
		Debug(fontcache, 0, "Failed to count runs: %s", u_errorName(err));
		ubidi_close(ubidi);
		return std::vector<ICURun>();
	}

	std::vector<ICURun> runs;
	runs.reserve(count);

	/* Find the breakpoints for the logical runs. So we get runs that say "from START to END". */
	int32_t logical_pos = 0;
	while (static_cast<size_t>(logical_pos) < length) {
		auto start_pos = logical_pos;

		/* Fetch the embedding level, so we can order bidi correctly later on. */
		UBiDiLevel level;
		ubidi_getLogicalRun(ubidi, start_pos, &logical_pos, &level);

		runs.emplace_back(ICURun(start_pos, logical_pos - start_pos, level));
	}

	assert(static_cast<size_t>(count) == runs.size());

	ubidi_close(ubidi);
	return runs;
}

/**
 * Itemize the string into runs per script, based on the previous created runs.
 *
 * Basically, this always returns the same or more runs than given.
 *
 * @param buff The string to itemize.
 * @param length The length of the string.
 * @param runs_current The current runs.
 * @return The runs.
 */
std::vector<ICURun> ItemizeScript(UChar *buff, size_t length, std::vector<ICURun> &runs_current)
{
	std::vector<ICURun> runs;
	icu::ScriptRun script_itemizer(buff, length);

	int cur_pos = 0;
	auto cur_run = runs_current.begin();
	while (true) {
		while (cur_pos < script_itemizer.getScriptEnd() && cur_run != runs_current.end()) {
			int stop_pos = std::min(script_itemizer.getScriptEnd(), cur_run->start + cur_run->length);
			assert(stop_pos - cur_pos > 0);

			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, script_itemizer.getScriptCode()));

			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
			cur_pos = stop_pos;
		}

		if (!script_itemizer.next()) break;
	}

	return runs;
}

/**
 * Itemize the string into runs per style, based on the previous created runs.
 *
 * Basically, this always returns the same or more runs than given.
 *
 * @param runs_current The current runs.
 * @param font_mapping The font mapping.
 * @return The runs.
 */
std::vector<ICURun> ItemizeStyle(std::vector<ICURun> &runs_current, FontMap &font_mapping)
{
	std::vector<ICURun> runs;

	int cur_pos = 0;
	auto cur_run = runs_current.begin();
	for (auto const &font_map : font_mapping) {
		while (cur_pos < font_map.first && cur_run != runs_current.end()) {
			int stop_pos = std::min(font_map.first, cur_run->start + cur_run->length);
			assert(stop_pos - cur_pos > 0);

			runs.push_back(ICURun(cur_pos, stop_pos - cur_pos, cur_run->level, cur_run->script, font_map.second));

			if (stop_pos == cur_run->start + cur_run->length) cur_run++;
			cur_pos = stop_pos;
		}
	}

	return runs;
}

/* static */ ParagraphLayouter *ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &font_mapping)
{
	size_t length = buff_end - buff;
	/* Can't layout an empty string. */
	if (length == 0) return nullptr;

	/* Can't layout our in-built sprite fonts. */
	for (auto const &pair : font_mapping) {
		if (pair.second->fc->IsBuiltInFont()) return nullptr;
	}

	auto runs = ItemizeBidi(buff, length);
	runs = ItemizeScript(buff, length, runs);
	runs = ItemizeStyle(runs, font_mapping);

	if (runs.empty()) return nullptr;

	for (auto &run : runs) {
		run.Shape(buff, length);
	}

	return new ICUParagraphLayout(runs, buff, length);
}

/* static */ std::unique_ptr<icu::BreakIterator> ICUParagraphLayoutFactory::break_iterator;

/**
 * Initialize data needed for the ICU layouter.
 */
/* static */ void ICUParagraphLayoutFactory::InitializeLayouter()
{
	auto locale = icu::Locale(_current_language->isocode);
	UErrorCode status = U_ZERO_ERROR;
	ICUParagraphLayoutFactory::break_iterator.reset(icu::BreakIterator::createLineInstance(locale, status));
	assert(U_SUCCESS(status));
}

/**
 * Get a thread-safe line break iterator.
 * @returns unique_ptr managed BreakIterator instance.
 */
/* static */ std::unique_ptr<icu::BreakIterator> ICUParagraphLayoutFactory::GetBreakIterator()
{
	assert(ICUParagraphLayoutFactory::break_iterator != nullptr);

	return std::unique_ptr<icu::BreakIterator>(ICUParagraphLayoutFactory::break_iterator->clone());
}

std::unique_ptr<const ICUParagraphLayout::Line> ICUParagraphLayout::NextLine(int max_width)
{
	std::vector<ICURun>::iterator start_run = this->current_run;
	std::vector<ICURun>::iterator last_run = this->current_run;

	if (start_run == this->runs.end()) return nullptr;

	int cur_width = 0;

	/* Add remaining width of the first run if it is a broken run. */
	if (this->partial_offset > 0) {
		if ((start_run->level & 1) == 0) {
			for (size_t i = this->partial_offset; i < start_run->advance.size(); i++) {
				cur_width += start_run->advance[i];
			}
		} else {
			for (int i = 0; i < this->partial_offset; i++) {
				cur_width += start_run->advance[i];
			}
		}
		last_run++;
	}

	/* Gather runs until the line is full. */
	while (last_run != this->runs.end() && cur_width < max_width) {
		cur_width += last_run->total_advance;
		last_run++;
	}

	/* If the text does not fit into the available width, find a suitable breaking point. */
	int new_partial_length = 0;
	if (cur_width > max_width) {
		/* Create a break-iterator to find a good place to break lines. */
		auto break_iterator = ICUParagraphLayoutFactory::GetBreakIterator();
		break_iterator->setText(icu::UnicodeString(this->buff, this->buff_length));

		auto overflow_run = last_run - 1;

		/* Find the last glyph that fits. */
		size_t index;
		if ((overflow_run->level & 1) == 0) {
			/* LTR */
			for (index = overflow_run->glyphs.size(); index > 0; index--) {
				cur_width -= overflow_run->advance[index - 1];
				if (cur_width <= max_width) break;
			}
			index--;
		} else {
			/* RTL */
			for (index = 0; index < overflow_run->glyphs.size(); index++) {
				cur_width -= overflow_run->advance[index];
				if (cur_width <= max_width) break;
			}
		}

		/* Find the character that matches; this is the start of the cluster. */
		auto char_pos = overflow_run->glyph_to_char[index];

		/* See if there is a good breakpoint inside this run. */
		int32_t break_pos = break_iterator->preceding(char_pos + 1);
		if (break_pos != icu::BreakIterator::DONE && break_pos > overflow_run->start + this->partial_offset) {
			/* There is a line-break inside this run that is suitable. */
			new_partial_length = break_pos - overflow_run->start - this->partial_offset;
		} else if (overflow_run != start_run) {
			/* There is no suitable line-break in this run, but it is also not
			 * the only run on this line. So we remove the run. */
			last_run--;
		} else {
			/* There is no suitable line-break and this is the only run on the
			 * line. So we break at the cluster. This is not pretty, but the
			 * best we can do. */
			new_partial_length = char_pos - overflow_run->start - this->partial_offset;
		}
	}

	/* Reorder the runs on this line for display. */
	std::vector<UBiDiLevel> bidi_level;
	for (auto run = start_run; run != last_run; run++) {
		bidi_level.push_back(run->level);
	}
	std::vector<int32_t> vis_to_log(bidi_level.size());
	ubidi_reorderVisual(bidi_level.data(), bidi_level.size(), vis_to_log.data());

	/* Create line. */
	std::unique_ptr<ICULine> line(new ICULine());

	int cur_pos = 0;
	for (auto &i : vis_to_log) {
		auto i_run = start_run + i;
		/* Copy the ICURun here, so we can modify it in case of a partial. */
		ICURun run = *i_run;

		if (i_run == last_run - 1 && new_partial_length > 0) {
			if (i_run == start_run && this->partial_offset > 0) {
				assert(run.length > this->partial_offset);
				run.start += this->partial_offset;
				run.length -= this->partial_offset;
			}

			assert(run.length > new_partial_length);
			run.length = new_partial_length;

			run.Shape(this->buff, this->buff_length);
		} else if (i_run == start_run && this->partial_offset > 0) {
			assert(run.length > this->partial_offset);

			run.start += this->partial_offset;
			run.length -= this->partial_offset;

			run.Shape(this->buff, this->buff_length);
		}

		auto total_advance = run.total_advance;
		line->emplace_back(std::move(run), cur_pos);
		cur_pos += total_advance;
	}

	if (new_partial_length > 0) {
		this->current_run = last_run - 1;
		this->partial_offset += new_partial_length;
	} else {
		this->current_run = last_run;
		this->partial_offset = 0;
	}

	return line;
}

/* static */ size_t ICUParagraphLayoutFactory::AppendToBuffer(UChar *buff, const UChar *buffer_last, char32_t c)
{
	assert(buff < buffer_last);
	/* Transform from UTF-32 to internal ICU format of UTF-16. */
	int32_t length = 0;
	UErrorCode err = U_ZERO_ERROR;
	u_strFromUTF32(buff, buffer_last - buff, &length, (UChar32*)&c, 1, &err);
	return length;
}