coa/aowow
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Search/Fixup

Browse source 
* do not use stopwords in fulltext search
   apparentyl there is a bug in innodb where including stopwords in a search causes the lookup to fail entirely.
   e.g. innodb_ft_enable_stopword must be disabled when the index is edited (rows are added/removed)
 * don't create a MATCH AGAINST search from empty search strings after sanitization
 * drop fulltext indizes for locale zhCN
   logographic languages need special treatment, which handling may differ by db provider
 * use LIKE search by default for locale zhCN. Added config option to use fulltext if supported by db.
This commit is contained in:
Sarjuuk 2026-01-17 19:08:50 +01:00
parent 7616ec25fc
commit 6df9145446
14 changed files with 156 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -54,6 +54,8 @@ audio processing may require [lame](https://sourceforge.net/projects/lame/files/
Ensure that the account you are going to use has **full** access on the database AoWoW is going to occupy and ideally only **read** access on the world database you are going to reference.
Import files 01 - 03 from `setup/sql/` in order into the AoWoW database `mysql -p {your-db-here} < setup/sql/01-db_structure.sql`, etc.
**Optional**: If you are using MySQL ≥ 8.4.0 and want to support fulltext search for locale zhCN, additionally import `setup/sql/04-db_optional_mysql_only.sql`. Enables this in settings after AoWoW has been set up.
#### 3. Server created files
See to it, that the web server is able to write the following directories and their children. If they are missing, the setup will create them with appropriate permissions
* `cache/`

21
includes/cfg.class.php
View file

@ -473,6 +473,27 @@ class Cfg
return true;
}
private static function logographic_ft_search(int|string $value, ?string &$msg = '') : bool
{
if (!$value)
return true;
$ok = true;
foreach (['?_spell', '?_items', '?_objects', '?_creature', '?_quests'] as $tbl)
{
if (DB::Aowow()->selectRow('SHOW INDEX FROM ?# WHERE `column_name` = ? AND `index_type` = "FULLTEXT"', $tbl, 'name_loc4'))
continue;
$ok = false;
$msg .= "\nNo fulltext index found on col: 'name_loc4'; tbl: '".$tbl."'.";
}
if (!$ok)
$msg .= "\nCannot enable option.\n";
return $ok;
}
}
?>

13
includes/components/filter.class.php
View file

@ -64,6 +64,7 @@ abstract class Filter
protected const PATTERN_CRV = '/[\p{C};:%\\\\]/ui';
protected const PATTERN_INT = '/\D/';
public const PATTERN_PARAM = '/^[\p{L}\p{Sm} \d\p{P}]+$/ui';
public const PATTERN_FT = '/[^[:alpha:] \d_-]/iu'; // +-*<>@()~" have special meaning; ' seems to fuck up the search; other irregular cases?
protected const ENUM_FACTION = array( 469, 1037, 1106, 529, 1012, 87, 21, 910, 609, 942, 909, 530, 69, 577, 930, 1068, 1104, 729, 369, 92,
54, 946, 67, 1052, 749, 47, 989, 1090, 1098, 978, 1011, 93, 1015, 1038, 76, 470, 349, 1031, 1077, 809,
@ -571,6 +572,10 @@ abstract class Filter
if (!$string && $this->values['na'])
$string = $this->values['na'];
// always allow sub 3 chars for logographic locales
if (Lang::getLocale()->isLogographic())
$shortStr = true;
$qry = [];
foreach ($fields as $f)
{
@ -614,7 +619,11 @@ abstract class Filter
if (!$string && $this->values['na'])
$string = $this->values['na'];
$string = preg_replace('/[^[:alpha:] \d_-]/iu', ' ', $string);
// always allow sub 3 chars for logographic locales
if (Lang::getLocale()->isLogographic() && !Cfg::get('LOGOGRAPHIC_FT_SEARCH'))
return $this->tokenizeString($fields, $string, $exact, $shortStr);
$string = trim(preg_replace(self::PATTERN_FT, ' ', $string));
if (!$string)
return [];
@ -635,7 +644,7 @@ abstract class Filter
// single cnd?
if (!$qry)
{
trigger_error('Filter::tokenizeString - could not tokenize string: '.$string, E_USER_NOTICE);
trigger_error('Filter::buildMatchLookup - could build MATCH AGAINST from: '.$string, E_USER_NOTICE);
$this->error = true;
}
else if (count($qry) > 1)

135
includes/components/search.class.php
View file

@ -79,6 +79,7 @@ class Search
private array $resultStore = [];
private array $included = [];
private array $excluded = [];
private array $fulltext = [];
private array $cndBase = ['AND'];
private bool $idSearch = false;
@ -109,25 +110,32 @@ class Search
foreach (explode(' ', $this->query) as $raw)
{
// ivalid chars for both LIKE and MATCH
$clean = str_replace(['\\', '%'], '', $raw);
if ($clean === '')
continue;
if ($clean[0] == '-')
$ex = ($clean[0] == '-');
if ($ex)
{
if (mb_strlen($clean) < 4 && !Lang::getLocale()->isLogographic())
$this->invalid[] = mb_substr($raw, 1);
else
$this->excluded[] = mb_substr(str_replace('_', '\\_', $clean), 1);
$clean = mb_substr($clean, 1);
$raw = mb_substr($raw, 1);
}
else
if (mb_strlen($clean) < 3 && !Lang::getLocale()->isLogographic())
{
if (mb_strlen($clean) < 3 && !Lang::getLocale()->isLogographic())
$this->invalid[] = $raw;
else
$this->included[] = str_replace('_', '\\_', $clean);
$this->invalid[] = $raw;
continue;
}
$this->{$ex ? 'excluded' : 'included'}[] = str_replace('_', '\\_', $clean);
// note: a fulltext search purely with exclude tokens will return no result
if (($tokens = trim(preg_replace(Filter::PATTERN_FT, ' ', $clean))) !== '')
foreach (array_filter(explode(' ', $tokens)) as $t)
if (mb_strlen($t) > 2)
$this->fulltext[] = ($ex ? '-' : '+') . $t . '*';
}
}
@ -176,21 +184,19 @@ class Search
if ($this->idSearch && $this->included)
return ['id', $this->included];
if (!$this->included)
if (Lang::getLocale()->isLogographic() && !Cfg::get('LOGOGRAPHIC_FT_SEARCH'))
return $this->createLikeLookup($fields);
if (!$this->fulltext)
return [];
// default to name-field
if (!$fields)
$fields[] = 'name_loc'.Lang::getLocale()->value;
$match = array_merge(
array_map(fn($x) => '+'.preg_replace('/[^[:alpha:] \d_-]/iu', ' ', $x).'*', $this->included),
array_map(fn($x) => '-'.preg_replace('/[^[:alpha:] \d_-]/iu', ' ', $x).'*', $this->excluded)
);
$qry = [];
foreach ($fields as $f)
$qry[] = [$f, $match, 'MATCH'];
$qry[] = [$f, $this->fulltext, 'MATCH'];
// single cnd?
if (count($qry) > 1)
@ -495,6 +501,8 @@ class Search
{
$miscData = ['calcTotal' => true];
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
if ($this->moduleMask & self::TYPE_JSON)
{
@ -574,11 +582,15 @@ class Search
private function _searchAbility() : ?array // 7 Abilities (Player + Pet) $moduleMask & 0x0000080
{
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, array( // hmm, inclued classMounts..?
['s.typeCat', [7, -2, -3, -4]],
[['s.cuFlags', (SPELL_CU_TRIGGERED | SPELL_CU_TALENT), '&'], 0],
[['s.attributes0', 0x80, '&'], 0],
$this->createMatchLookup()
$lookup
));
$abilities = new SpellList($cnd, ['calcTotal' => true]);
@ -638,10 +650,11 @@ class Search
private function _searchTalent() : ?array // 8 Talents (Player + Pet) $moduleMask & 0x0000100
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', [-7, -2]],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', [-7, -2]], $lookup]);
$talents = new SpellList($cnd, ['calcTotal' => true]);
$data = $talents->getListviewData();
@ -697,10 +710,11 @@ class Search
private function _searchGlyph() : ?array // 9 Glyphs $moduleMask & 0x0000200
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -13],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', -13], $lookup]);
$glyphs = new SpellList($cnd, ['calcTotal' => true]);
$data = $glyphs->getListviewData();
@ -751,10 +765,11 @@ class Search
private function _searchProficiency() : ?array // 10 Proficiencies $moduleMask & 0x0000400
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -11],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', -11], $lookup]);
$prof = new SpellList($cnd, ['calcTotal' => true]);
$data = $prof->getListviewData();
@ -805,10 +820,11 @@ class Search
private function _searchProfession() : ?array // 11 Professions (Primary + Secondary) $moduleMask & 0x0000800
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', [9, 11]],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', [9, 11]], $lookup]);
$prof = new SpellList($cnd, ['calcTotal' => true]);
$data = $prof->getListviewData();
@ -859,10 +875,11 @@ class Search
private function _searchCompanion() : ?array // 12 Companions $moduleMask & 0x0001000
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -6],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', -6], $lookup]);
$vPets = new SpellList($cnd, ['calcTotal' => true]);
$data = $vPets->getListviewData();
@ -913,10 +930,11 @@ class Search
private function _searchMount() : ?array // 13 Mounts $moduleMask & 0x0002000
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -5],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', -5], $lookup]);
$mounts = new SpellList($cnd, ['calcTotal' => true]);
$data = $mounts->getListviewData();
@ -966,10 +984,14 @@ class Search
private function _searchCreature() : ?array // 14 NPCs $moduleMask & 0x0004000
{
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, array(
[['flagsExtra', 0x80], 0], // exclude trigger creatures
[['cuFlags', NPC_CU_DIFFICULTY_DUMMY, '&'], 0], // exclude difficulty entries
$this->createMatchLookup()
$lookup
));
$npcs = new CreatureList($cnd, ['calcTotal' => true]);
@ -1019,9 +1041,13 @@ class Search
private function _searchQuest() : ?array // 15 Quests $moduleMask & 0x0008000
{
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, array(
[['flags', CUSTOM_UNAVAILABLE | CUSTOM_DISABLED, '&'], 0],
$this->createMatchLookup()
$lookup
));
$quests = new QuestList($cnd, ['calcTotal' => true]);
@ -1218,7 +1244,11 @@ class Search
private function _searchObject() : ?array // 19 Objects $moduleMask & 0x0080000
{
$cnd = array_merge($this->cndBase, [$this->createMatchLookup()]);
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [$lookup]);
$objects = new GameObjectList($cnd, ['calcTotal' => true]);
$data = $objects->getListviewData();
@ -1378,10 +1408,11 @@ class Search
private function _searchCreatureAbility() : ?array // 23 NPCAbilities $moduleMask & 0x0800000
{
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -8],
$this->createMatchLookup()
));
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, [['s.typeCat', -8], $lookup]);
$npcAbilities = new SpellList($cnd, ['calcTotal' => true]);
$data = $npcAbilities->getListviewData();
@ -1433,6 +1464,10 @@ class Search
private function _searchSpell() : ?array // 24 Spells (Misc + GM + triggered abilities) $moduleMask & 0x1000000
{
$lookup = $this->createMatchLookup();
if (!$lookup)
return null;
$cnd = array_merge($this->cndBase, array(
['s.typeCat', -8, '!'],
[
@ -1441,7 +1476,7 @@ class Search
['s.cuFlags', SPELL_CU_TRIGGERED, '&'],
['s.attributes0', 0x80, '&']
],
$this->createMatchLookup()
$lookup
));
$misc = new SpellList($cnd, ['calcTotal' => true]);

5
setup/sql/01-db_structure.sql
View file

@ -581,7 +581,6 @@ CREATE TABLE `aowow_creature` (
FULLTEXT `idx_name0` (`name_loc0`),
FULLTEXT `idx_name2` (`name_loc2`),
FULLTEXT `idx_name3` (`name_loc3`),
FULLTEXT `idx_name4` (`name_loc4`),
FULLTEXT `idx_name6` (`name_loc6`),
FULLTEXT `idx_name8` (`name_loc8`),
KEY `idx_spell1` (`spell1`),
@ -1473,7 +1472,6 @@ CREATE TABLE `aowow_items` (
FULLTEXT `idx_name0` (`name_loc0`),
FULLTEXT `idx_name2` (`name_loc2`),
FULLTEXT `idx_name3` (`name_loc3`),
FULLTEXT `idx_name4` (`name_loc4`),
FULLTEXT `idx_name6` (`name_loc6`),
FULLTEXT `idx_name8` (`name_loc8`),
KEY `idx_itemset` (`itemset`)
@ -1668,7 +1666,6 @@ CREATE TABLE `aowow_objects` (
FULLTEXT `idx_name0` (`name_loc0`),
FULLTEXT `idx_name2` (`name_loc2`),
FULLTEXT `idx_name3` (`name_loc3`),
FULLTEXT `idx_name4` (`name_loc4`),
FULLTEXT `idx_name6` (`name_loc6`),
FULLTEXT `idx_name8` (`name_loc8`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
@ -2272,7 +2269,6 @@ CREATE TABLE `aowow_quests` (
FULLTEXT `idx_name0` (`name_loc0`),
FULLTEXT `idx_name2` (`name_loc2`),
FULLTEXT `idx_name3` (`name_loc3`),
FULLTEXT `idx_name4` (`name_loc4`),
FULLTEXT `idx_name6` (`name_loc6`),
FULLTEXT `idx_name8` (`name_loc8`),
KEY `idx_sourcespell` (`sourceSpellId`),
@ -2881,7 +2877,6 @@ CREATE TABLE `aowow_spell` (
FULLTEXT `idx_name0` (`name_loc0`),
FULLTEXT `idx_name2` (`name_loc2`),
FULLTEXT `idx_name3` (`name_loc3`),
FULLTEXT `idx_name4` (`name_loc4`),
FULLTEXT `idx_name6` (`name_loc6`),
FULLTEXT `idx_name8` (`name_loc8`),
KEY `idx_spellfamily` (`spellFamilyId`),

4
setup/sql/02-db_initial_data.sql
View file

File diff suppressed because one or more lines are too long

5
setup/sql/04-db_optional_mysql_only.sql Normal file
View file

@ -0,0 +1,5 @@
ALTER TABLE `aowow_creature` ADD FULLTEXT `idx_name4` (`name_loc4`) WITH PARSER ngram;
ALTER TABLE `aowow_items` ADD FULLTEXT `idx_name4` (`name_loc4`) WITH PARSER ngram;
ALTER TABLE `aowow_objects` ADD FULLTEXT `idx_name4` (`name_loc4`) WITH PARSER ngram;
ALTER TABLE `aowow_quests` ADD FULLTEXT `idx_name4` (`name_loc4`) WITH PARSER ngram;
ALTER TABLE `aowow_spell` ADD FULLTEXT `idx_name4` (`name_loc4`) WITH PARSER ngram;

16
setup/sql/updates/1768672799_01.sql Normal file
View file

@ -0,0 +1,16 @@
ALTER TABLE `aowow_creature` DROP INDEX `idx_name4`;
ALTER TABLE `aowow_items` DROP INDEX `idx_name4`;
ALTER TABLE `aowow_objects` DROP INDEX `idx_name4`;
ALTER TABLE `aowow_quests` DROP INDEX `idx_name4`;
ALTER TABLE `aowow_spell` DROP INDEX `idx_name4`;
SET SESSION innodb_ft_enable_stopword = OFF;
OPTIMIZE TABLE `aowow_spell`;
OPTIMIZE TABLE `aowow_quests`;
OPTIMIZE TABLE `aowow_creature`;
OPTIMIZE TABLE `aowow_items`;
OPTIMIZE TABLE `aowow_objects`;
REPLACE INTO `aowow_config` VALUES
('logographic_ft_search', '0', '0', 1, 0x484, 'enables fulltext search for logographic languages (CN, KR, TW). The database MUST support this (i.e. MySQL implements ngram)');

4
setup/tools/sqlgen/creature.ss.php
View file

@ -102,8 +102,10 @@ CLISetup::registerSetup("sql", new class extends SetupScript
{ WHERE ct.entry IN (?a) }
LIMIT ?d, ?d';
$i = 0;
DB::Aowow()->query('TRUNCATE ?_creature');
DB::Aowow()->query('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($npcs = DB::World()->select($baseQuery, NPC_CU_INSTANCE_BOSS, $ids ?: DBSIMPLE_SKIP, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))
{
CLI::write(' * batch #' . ++$i . ' (' . count($npcs) . ')', CLI::LOG_BLANK, true, true);

4
setup/tools/sqlgen/items.ss.php
View file

@ -130,8 +130,10 @@ CLISetup::registerSetup("sql", new class extends SetupScript
{ WHERE it.entry IN (?a) }
LIMIT ?d, ?d';
$i = 0;
DB::Aowow()->query('TRUNCATE ?_items');
DB::Aowow()->query('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($items = DB::World()->select($baseQuery, $ids ?: DBSIMPLE_SKIP, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))
{
CLI::write(' * batch #' . ++$i . ' (' . count($items) . ')', CLI::LOG_BLANK, true, true);

4
setup/tools/sqlgen/objects.ss.php
View file

@ -67,8 +67,10 @@ CLISetup::registerSetup("sql", new class extends SetupScript
GROUP BY go.entry
LIMIT ?d, ?d';
$i = 0;
DB::Aowow()->query('TRUNCATE ?_objects');
DB::Aowow()->query('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($objects = DB::World()->select($baseQuery, $ids ?: DBSIMPLE_SKIP, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))
{
CLI::write(' * batch #' . ++$i . ' (' . count($objects) . ')', CLI::LOG_BLANK, true, true);

4
setup/tools/sqlgen/quests.ss.php
View file

@ -116,8 +116,10 @@ CLISetup::registerSetup("sql", new class extends SetupScript
{ WHERE q.Id IN (?a) }
LIMIT ?d, ?d';
$i = 0;
DB::Aowow()->query('TRUNCATE ?_quests');
DB::Aowow()->query('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($quests = DB::World()->select($baseQuery, $ids ?: DBSIMPLE_SKIP, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))
{
CLI::write(' * batch #' . ++$i . ' (' . count($quests) . ')', CLI::LOG_BLANK, true, true);

1
setup/tools/sqlgen/spell.ss.php
View file

@ -199,6 +199,7 @@ CLISetup::registerSetup("sql", new class extends SetupScript
DB::Aowow()->query('TRUNCATE ?_spell');
DB::Aowow()->query('SET SESSION innodb_ft_enable_stopword = OFF');
// merge serverside spells into aowow_spell
$lastMax = 0;

2
template/pages/admin/siteconfig.tpl.php
View file

@ -18,7 +18,7 @@
var a = $WH.ce('a');
a.style.opacity = 0;
a.className = errTxt ? 'icon-report' : 'icon-tick';
g_addTooltip(a, errTxt || 'success', 'q');
g_addTooltip(a, errTxt ? '<pre>' + errTxt + '</pre>' : 'success', 'q');
a.onclick = fadeout.bind(a);
setTimeout(function () { $(a).animate({ opacity: '1.0' }, 250); }, 50);
setTimeout(fadeout.bind(a), 10000);