coa/aowow
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Search/Indexing

Browse source 
* move fulltext indizes for tables /w ~10k+ rows to separate tables
   > sounds have ~12k but names are effectively incompatible with FTI
 * normalize searchable strings to catch edgecases
   > preparse spell descriptions + buffs
   > move effect text cols of items to new table
 * also fix extended search of creatures, spells & quests
This commit is contained in:
Sarjuuk 2026-03-03 14:58:19 +01:00
parent 2161a7b846
commit 5a230daad6
28 changed files with 648 additions and 165 deletions
Download patch file Download diff file Expand all files Collapse all files

123
setup/sql/01-db_structure.sql
View file

@ -578,11 +578,6 @@ CREATE TABLE `aowow_creature` (
KEY `idx_skinloot` (`skinLootId`),
KEY `idx_trainer` (`trainerType`),
KEY `idx_trainerrequirement` (`trainerRequirement`),
FULLTEXT `idx_ft_name0` (`name_loc0`),
FULLTEXT `idx_ft_name2` (`name_loc2`),
FULLTEXT `idx_ft_name3` (`name_loc3`),
FULLTEXT `idx_ft_name6` (`name_loc6`),
FULLTEXT `idx_ft_name8` (`name_loc8`),
KEY `idx_name0` (`name_loc0`),
KEY `idx_name2` (`name_loc2`),
KEY `idx_name3` (`name_loc3`),
@ -600,6 +595,24 @@ CREATE TABLE `aowow_creature` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_creature_search`
--
DROP TABLE IF EXISTS `aowow_creature_search`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `aowow_creature_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(100) DEFAULT NULL,
`nSubname` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`,`locale`),
FULLTEXT KEY `idx_ft_na` (`nName`),
FULLTEXT KEY `idx_ft_na_ex` (`nName`,`nSubname`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_creature_sounds`
--
@ -1460,12 +1473,6 @@ CREATE TABLE `aowow_items` (
`sheatheSoundId` smallint(5) unsigned NOT NULL DEFAULT 0,
`unsheatheSoundId` smallint(5) unsigned NOT NULL DEFAULT 0,
`flagsCustom` int(10) unsigned NOT NULL DEFAULT 0,
`effects_loc0` text DEFAULT NULL,
`effects_loc2` text DEFAULT NULL,
`effects_loc3` text DEFAULT NULL,
`effects_loc4` text DEFAULT NULL,
`effects_loc6` text DEFAULT NULL,
`effects_loc8` text DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `items_index` (`class`),
KEY `idx_model` (`displayId`),
@ -1482,11 +1489,6 @@ CREATE TABLE `aowow_items` (
KEY `idx_trigger4` (`spellTrigger4`),
KEY `idx_trigger5` (`spellTrigger5`),
KEY `idx_reqskill` (`requiredSkill`),
FULLTEXT `idx_ft_name0` (`name_loc0`),
FULLTEXT `idx_ft_name2` (`name_loc2`),
FULLTEXT `idx_ft_name3` (`name_loc3`),
FULLTEXT `idx_ft_name6` (`name_loc6`),
FULLTEXT `idx_ft_name8` (`name_loc8`),
KEY `idx_name0` (`name_loc0`),
KEY `idx_name2` (`name_loc2`),
KEY `idx_name3` (`name_loc3`),
@ -1497,6 +1499,26 @@ CREATE TABLE `aowow_items` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_items_search`
--
DROP TABLE IF EXISTS `aowow_items_search`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `aowow_items_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(127) DEFAULT NULL,
`nDescription` varchar(255) DEFAULT NULL,
`nEffects` text DEFAULT NULL,
PRIMARY KEY (`id`,`locale`),
FULLTEXT KEY `idx_ft_na` (`nName`),
FULLTEXT KEY `idx_ft_description` (`nDescription`),
FULLTEXT KEY `idx_ft_effects` (`nEffects`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_items_sounds`
--
@ -1682,11 +1704,6 @@ CREATE TABLE `aowow_objects` (
KEY `idx_onsuccessspell` (`onSuccessSpell`),
KEY `idx_auraspell` (`auraSpell`),
KEY `idx_triggeredspell` (`triggeredSpell`),
FULLTEXT `idx_ft_name0` (`name_loc0`),
FULLTEXT `idx_ft_name2` (`name_loc2`),
FULLTEXT `idx_ft_name3` (`name_loc3`),
FULLTEXT `idx_ft_name6` (`name_loc6`),
FULLTEXT `idx_ft_name8` (`name_loc8`),
KEY `idx_name0` (`name_loc0`),
KEY `idx_name2` (`name_loc2`),
KEY `idx_name3` (`name_loc3`),
@ -1696,6 +1713,22 @@ CREATE TABLE `aowow_objects` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_objects_search`
--
DROP TABLE IF EXISTS `aowow_objects_search`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `aowow_objects_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(127) DEFAULT NULL,
PRIMARY KEY (`id`,`locale`),
FULLTEXT KEY `idx_ft_na` (`nName`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_pet`
--
@ -2291,11 +2324,6 @@ CREATE TABLE `aowow_quests` (
`objectiveText4_loc8` text DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `nextQuestIdChain` (`nextQuestIdChain`),
FULLTEXT `idx_ft_name0` (`name_loc0`),
FULLTEXT `idx_ft_name2` (`name_loc2`),
FULLTEXT `idx_ft_name3` (`name_loc3`),
FULLTEXT `idx_ft_name6` (`name_loc6`),
FULLTEXT `idx_ft_name8` (`name_loc8`),
KEY `idx_name0` (`name_loc0`),
KEY `idx_name2` (`name_loc2`),
KEY `idx_name3` (`name_loc3`),
@ -2331,6 +2359,25 @@ CREATE TABLE `aowow_quests` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_quests_search`
--
DROP TABLE IF EXISTS `aowow_quests_search`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `aowow_quests_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(100) DEFAULT NULL,
`nObjectives` text DEFAULT NULL,
`nDetails` text DEFAULT NULL,
PRIMARY KEY (`id`,`locale`),
FULLTEXT KEY `idx_ft_na` (`nName`),
FULLTEXT KEY `idx_ft_na_ex` (`nName`,`nObjectives`,`nDetails`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_quests_startend`
--
@ -2905,11 +2952,6 @@ CREATE TABLE `aowow_spell` (
KEY `effect3AuraId` (`effect3AuraId`),
KEY `idx_skill1` (`skillLine1`),
KEY `idx_skill2` (`skillLine2OrMask`),
FULLTEXT `idx_ft_name0` (`name_loc0`),
FULLTEXT `idx_ft_name2` (`name_loc2`),
FULLTEXT `idx_ft_name3` (`name_loc3`),
FULLTEXT `idx_ft_name6` (`name_loc6`),
FULLTEXT `idx_ft_name8` (`name_loc8`),
KEY `idx_name0` (`name_loc0`),
KEY `idx_name2` (`name_loc2`),
KEY `idx_name3` (`name_loc3`),
@ -2926,6 +2968,25 @@ CREATE TABLE `aowow_spell` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_spell_search`
--
DROP TABLE IF EXISTS `aowow_spell_search`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE `aowow_spell_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(185) DEFAULT NULL,
`nDescription` text DEFAULT NULL,
`nBuff` text DEFAULT NULL,
PRIMARY KEY (`id`,`locale`),
FULLTEXT KEY `idx_ft_na` (`nName`),
FULLTEXT KEY `idx_ft_na_ex` (`nName`,`nDescription`,`nBuff`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `aowow_spell_sounds`
--

2
setup/sql/02-db_initial_data.sql
View file

@ -71,7 +71,7 @@ UNLOCK TABLES;
LOCK TABLES `aowow_dbversion` WRITE;
/*!40000 ALTER TABLE `aowow_dbversion` DISABLE KEYS */;
INSERT INTO `aowow_dbversion` VALUES (1770889049,0,NULL,NULL);
INSERT INTO `aowow_dbversion` VALUES (1772564119,0,NULL,NULL);
/*!40000 ALTER TABLE `aowow_dbversion` ENABLE KEYS */;
UNLOCK TABLES;

56
setup/sql/updates/1772564118_01.sql Normal file
View file

@ -0,0 +1,56 @@
DROP TABLE IF EXISTS `aowow_quests_search`;
CREATE TABLE `aowow_quests_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(100) DEFAULT NULL,
`nObjectives` text DEFAULT NULL,
`nDetails` text DEFAULT NULL,
PRIMARY KEY (`id`, `locale`),
FULLTEXT `idx_ft_na` (`nName`),
FULLTEXT `idx_ft_na_ex` (`nName`, `nObjectives`, `nDetails`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
DROP TABLE IF EXISTS `aowow_objects_search`;
CREATE TABLE `aowow_objects_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(127) DEFAULT NULL,
PRIMARY KEY (`id`, `locale`),
FULLTEXT `idx_ft_na` (`nName`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
DROP TABLE IF EXISTS `aowow_items_search`;
CREATE TABLE `aowow_items_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(127) DEFAULT NULL,
`nDescription` varchar(255) DEFAULT NULL,
`nEffects` text DEFAULT NULL,
PRIMARY KEY (`id`, `locale`),
FULLTEXT `idx_ft_na` (`nName`),
FULLTEXT `idx_ft_description` (`nDescription`),
FULLTEXT `idx_ft_effects` (`nEffects`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
DROP TABLE IF EXISTS `aowow_creature_search`;
CREATE TABLE `aowow_creature_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(100) DEFAULT NULL,
`nSubname` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`, `locale`),
FULLTEXT `idx_ft_na` (`nName`),
FULLTEXT `idx_ft_na_ex` (`nName`, `nSubname`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
DROP TABLE IF EXISTS `aowow_spell_search`;
CREATE TABLE `aowow_spell_search` (
`id` mediumint(8) unsigned NOT NULL,
`locale` tinyint(3) unsigned NOT NULL,
`nName` varchar(185) DEFAULT NULL,
`nDescription` text DEFAULT NULL,
`nBuff` text DEFAULT NULL,
PRIMARY KEY (`id`, `locale`),
FULLTEXT `idx_ft_na` (`nName`),
FULLTEXT `idx_ft_na_ex` (`nName`, `nDescription`, `nBuff`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

40
setup/sql/updates/1772564118_02.sql Normal file
View file

@ -0,0 +1,40 @@
ALTER TABLE `aowow_creature`
DROP INDEX `idx_ft_name0`,
DROP INDEX `idx_ft_name2`,
DROP INDEX `idx_ft_name3`,
DROP INDEX `idx_ft_name6`,
DROP INDEX `idx_ft_name8`;
ALTER TABLE `aowow_objects`
DROP INDEX `idx_ft_name0`,
DROP INDEX `idx_ft_name2`,
DROP INDEX `idx_ft_name3`,
DROP INDEX `idx_ft_name6`,
DROP INDEX `idx_ft_name8`;
ALTER TABLE `aowow_quests`
DROP INDEX `idx_ft_name0`,
DROP INDEX `idx_ft_name2`,
DROP INDEX `idx_ft_name3`,
DROP INDEX `idx_ft_name6`,
DROP INDEX `idx_ft_name8`;
ALTER TABLE `aowow_spell`
DROP INDEX `idx_ft_name0`,
DROP INDEX `idx_ft_name2`,
DROP INDEX `idx_ft_name3`,
DROP INDEX `idx_ft_name6`,
DROP INDEX `idx_ft_name8`;
ALTER TABLE `aowow_items`
DROP COLUMN `effects_loc0`,
DROP COLUMN `effects_loc2`,
DROP COLUMN `effects_loc3`,
DROP COLUMN `effects_loc4`,
DROP COLUMN `effects_loc6`,
DROP COLUMN `effects_loc8`,
DROP INDEX `idx_ft_name0`,
DROP INDEX `idx_ft_name2`,
DROP INDEX `idx_ft_name3`,
DROP INDEX `idx_ft_name6`,
DROP INDEX `idx_ft_name8`;

1
setup/tools/sqlgen/creature.ss.php
View file

@ -102,7 +102,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
LIMIT %i, %i';
DB::Aowow()->qry('TRUNCATE ::creature');
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($npcs = DB::World()->selectAssoc($baseQuery, NPC_CU_INSTANCE_BOSS, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))

55
setup/tools/sqlgen/items.ss.php
View file

@ -118,8 +118,7 @@ CLISetup::registerSetup("sql", new class extends SetupScript
0 AS dropDownSoundId,
0 AS sheatheSoundId,
0 AS unsheatheSoundId,
flagsCustom,
null AS effects_loc0, null AS effects_loc2, null AS effects_loc3, null AS effects_loc4, null AS effects_loc6, null AS effects_loc8
flagsCustom
FROM item_template it
LEFT JOIN item_template_locale itl2 ON it.entry = itl2.ID AND itl2.locale = "frFR"
LEFT JOIN item_template_locale itl3 ON it.entry = itl3.ID AND itl3.locale = "deDE"
@ -131,7 +130,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
LIMIT %i, %i';
DB::Aowow()->qry('TRUNCATE ::items');
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($items = DB::World()->selectAssoc($baseQuery, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))
@ -265,57 +263,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
DB::Aowow()->qry('UPDATE ::items SET `cuFlags` = `cuFlags` | %i WHERE `class`= %i AND `slotBak` IN %in AND `subClass` NOT IN %in', CUSTOM_EXCLUDE_FOR_LISTVIEW, ITEM_CLASS_WEAPON, $slots, $subclasses);
CLI::write('[items] - collecting spell descriptions');
CLI::write(' * fetching', tmpRow: true);
$itemSpellData = DB::Aowow()->selectAssoc(
'SELECT `id` AS "0", `spellId1` AS "1" FROM ::items WHERE `spellId1` > 0 UNION
SELECT `id` AS "0", `spellId2` AS "1" FROM ::items WHERE `spellId2` > 0 UNION
SELECT `id` AS "0", `spellId3` AS "1" FROM ::items WHERE `spellId3` > 0 UNION
SELECT `id` AS "0", `spellId4` AS "1" FROM ::items WHERE `spellId4` > 0 UNION
SELECT `id` AS "0", `spellId5` AS "1" FROM ::items WHERE `spellId5` > 0'
);
$itemSpells = new SpellList(array(['id', array_column($itemSpellData, 1)]), ['interactive' => SpellList::INTERACTIVE_NONE]);
$items = DB::Aowow()->selectAssoc('SELECT `id` AS ARRAY_KEY, `spellId1`, `spellId2`, `spellId3`, `spellId4`, `spellId5` FROM ::items WHERE `id` IN %in', array_column($itemSpellData, 0));
if (!$itemSpells->error)
{
$i = 0;
$total = count($items) * count(CLISetup::$locales);
$update = [];
foreach (CLISetup::$locales as $locId => $loc)
{
Lang::load($loc);
foreach ($items as $itemId => $spells)
{
CLI::write(' * applying ' . str_pad(++$i, strlen($total), pad_type: STR_PAD_LEFT) . ' / ' . $total . str_pad('('.number_format(100 * $i / $total, 1).'%)', 8, pad_type: STR_PAD_LEFT), tmpRow: true);
foreach ($spells as $spellId)
{
if (!$itemSpells->getEntry($spellId))
continue;
if ($_ = $itemSpells->parseText('description'))
{
if (!isset($update[$itemId]['effects_loc'.$locId]))
$update[$itemId]['effects_loc'.$locId] = '';
$update[$itemId]['effects_loc'.$locId] .= $_[0]."\n";
}
}
}
}
$i = 0;
$total = count($update);
foreach ($update as $itemId => $upd)
{
CLI::write(' * writing ' . str_pad(++$i, strlen($total), pad_type: STR_PAD_LEFT) . ' / ' . $total . str_pad('('.number_format(100 * $i / $total, 1).'%)', 8, pad_type: STR_PAD_LEFT), tmpRow: true);
DB::Aowow()->qry('UPDATE ::items SET %a WHERE `id` = %i', $upd, $itemId);
}
}
$this->reapplyCCFlags('items', Type::ITEM);
return true;

1
setup/tools/sqlgen/objects.ss.php
View file

@ -67,7 +67,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
LIMIT %i, %i';
DB::Aowow()->qry('TRUNCATE ::objects');
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($objects = DB::World()->selectAssoc($baseQuery, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))

1
setup/tools/sqlgen/quests.ss.php
View file

@ -116,7 +116,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
LIMIT %i, %i';
DB::Aowow()->qry('TRUNCATE ::quests');
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
$i = 0;
while ($quests = DB::World()->selectAssoc($baseQuery, CLISetup::SQL_BATCH * $i, CLISetup::SQL_BATCH))

348
setup/tools/sqlgen/search.ss.php Normal file
View file

@ -0,0 +1,348 @@
<?php
namespace Aowow;
if (!defined('AOWOW_REVISION'))
die('illegal access');
if (!CLI)
die('not in cli mode');
CLISetup::registerSetup("sql", new class extends SetupScript
{
use TrCustomData;
protected $info = array(
'search' => [[ ], CLISetup::ARGV_PARAM, 'Normalize strings from creatures, items, objects, quests & spells for fulltext search.'],
/* 1 */ 'creature' => [['1'], CLISetup::ARGV_OPTIONAL, '...only for creatures.'],
/* 2 */ 'item' => [['2'], CLISetup::ARGV_OPTIONAL, '...only for items.'],
/* 4 */ 'object' => [['3'], CLISetup::ARGV_OPTIONAL, '...only for objects.'],
/* 8 */ 'spell' => [['4'], CLISetup::ARGV_OPTIONAL, '...only for spells.'],
/*16 */ 'quest' => [['5'], CLISetup::ARGV_OPTIONAL, '...only for quests.']
);
protected $setupAfter = [['creature', 'items', 'objects', 'spell', 'quests'], []];
private const /* int */ OPT_NPCS = (1 << 0);
private const /* int */ OPT_ITEMS = (1 << 1);
private const /* int */ OPT_OBJECTS = (1 << 2);
private const /* int */ OPT_SPELLS = (1 << 3);
private const /* int */ OPT_QUESTS = (1 << 4);
private array $spells = [];
private array $locales = [];
public function generate() : bool
{
// find out what to do
$opts = array_slice(array_keys($this->info), 1);
$getO = CLISetup::getOpt(...$opts);
$mask = null;
// todo: have an extra search table with ngram fulltext indices
$this->locales = array_filter(CLISetup::$locales, fn($x) => !$x->isLogographic());
if ($getO['creature'])
$mask |= self::OPT_NPCS;
if ($getO['item'])
$mask |= self::OPT_ITEMS;
if ($getO['object'])
$mask |= self::OPT_OBJECTS;
if ($getO['spell'])
$mask |= self::OPT_SPELLS;
if ($getO['quest'])
$mask |= self::OPT_QUESTS;
$mask ??= (self::OPT_NPCS | self::OPT_ITEMS | self::OPT_OBJECTS | self::OPT_SPELLS | self::OPT_QUESTS);
// do what needs doing
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
if ($mask & self::OPT_NPCS)
$this->normalizeCreatures();
if ($mask & self::OPT_ITEMS)
$this->normalizeItems();
if ($mask & self::OPT_OBJECTS)
$this->normalizeObjects();
if ($mask & self::OPT_SPELLS)
$this->normalizeSpells();
if ($mask & self::OPT_QUESTS)
$this->normalizeQuests();
return true;
}
private function normalizeQuests() : void
{
CLI::write(' - creating indices for quest names, descriptions & details...');
DB::Aowow()->qry('TRUNCATE ::quests_search');
CLI::write(' * fetching', tmpRow: true);
$rows = DB::Aowow()->selectAssoc(
'SELECT `id`, 0 AS "locale", `name_loc0` AS "name", `objectives_loc0` AS "objectives", `details_loc0` AS "details" FROM ::quests UNION
SELECT `id`, 2 AS "locale", `name_loc2` AS "name", `objectives_loc2` AS "objectives", `details_loc2` AS "details" FROM ::quests UNION
SELECT `id`, 3 AS "locale", `name_loc3` AS "name", `objectives_loc3` AS "objectives", `details_loc3` AS "details" FROM ::quests UNION
SELECT `id`, 6 AS "locale", `name_loc6` AS "name", `objectives_loc6` AS "objectives", `details_loc6` AS "details" FROM ::quests UNION
SELECT `id`, 8 AS "locale", `name_loc8` AS "name", `objectives_loc8` AS "objectives", `details_loc8` AS "details" FROM ::quests'
);
CLI::write(' * normalizing', tmpRow: true);
array_walk($rows, self::normalizeRow(...), ['name', 'objectives', 'details']);
$rows = array_filter($rows, fn($x) => $x['name'] !== null);
$n = ceil(count($rows) / CLISetup::SQL_BATCH);
for ($i = 0; $i < $n; $i++)
{
$sub = array_slice($rows, $i * CLISetup::SQL_BATCH, CLISetup::SQL_BATCH);
CLI::write(' * inserting batch '.($i + 1).' / '.$n, tmpRow: true);
DB::Aowow()->qry('INSERT INTO ::quests_search %m', array(
'id' => array_column($sub, 'id'),
'locale' => array_column($sub, 'locale'),
'nName' => array_column($sub, 'name'),
'nObjectives' => array_column($sub, 'objectives'),
'nDetails' => array_column($sub, 'details')
));
}
}
private function normalizeObjects() : void
{
CLI::write(' - creating indices for object names...');
DB::Aowow()->qry('TRUNCATE ::objects_search');
CLI::write(' * fetching', tmpRow: true);
$rows = DB::Aowow()->selectAssoc(
'SELECT `id`, 0 AS "locale", `name_loc0` AS "name" FROM ::objects UNION
SELECT `id`, 2 AS "locale", `name_loc2` AS "name" FROM ::objects UNION
SELECT `id`, 3 AS "locale", `name_loc3` AS "name" FROM ::objects UNION
SELECT `id`, 6 AS "locale", `name_loc6` AS "name" FROM ::objects UNION
SELECT `id`, 8 AS "locale", `name_loc8` AS "name" FROM ::objects'
);
CLI::write(' * normalizing', tmpRow: true);
array_walk($rows, self::normalizeRow(...), ['name']);
$rows = array_filter($rows, fn($x) => $x['name'] !== null);
CLI::write(' * inserting', tmpRow: true);
DB::Aowow()->qry('INSERT INTO ::objects_search %m', array(
'id' => array_column($rows, 'id'),
'locale' => array_column($rows, 'locale'),
'nName' => array_column($rows, 'name')
));
}
private function normalizeCreatures() : void
{
CLI::write(' - creating indices for creature names & subnames...');
DB::Aowow()->qry('TRUNCATE ::creature_search');
CLI::write(' * fetching', tmpRow: true);
$rows = DB::Aowow()->selectAssoc(
'SELECT `id`, 0 AS "locale", `name_loc0` AS "name", `subname_loc0` AS "subname" FROM ::creature UNION
SELECT `id`, 2 AS "locale", `name_loc2` AS "name", `subname_loc2` AS "subname" FROM ::creature UNION
SELECT `id`, 3 AS "locale", `name_loc3` AS "name", `subname_loc3` AS "subname" FROM ::creature UNION
SELECT `id`, 6 AS "locale", `name_loc6` AS "name", `subname_loc6` AS "subname" FROM ::creature UNION
SELECT `id`, 8 AS "locale", `name_loc8` AS "name", `subname_loc8` AS "subname" FROM ::creature'
);
CLI::write(' * normalizing', tmpRow: true);
array_walk($rows, self::normalizeRow(...), ['name', 'subname']);
$rows = array_filter($rows, fn($x) => $x['name'] !== null && $x['subname'] !== null);
CLI::write(' * inserting', tmpRow: true);
DB::Aowow()->qry('INSERT INTO ::creature_search %m', array(
'id' => array_column($rows, 'id'),
'locale' => array_column($rows, 'locale'),
'nName' => array_column($rows, 'name'),
'nSubname' => array_column($rows, 'subname')
));
}
private function normalizeItems() : void
{
CLI::write(' - creating indices for item names, descriptions & spells...');
DB::Aowow()->qry('TRUNCATE ::items_search');
CLI::write(' * fetching', tmpRow: true);
$result = [];
$items = DB::Aowow()->selectAssoc('SELECT `id` AS ARRAY_KEY, `name_loc0`, `name_loc2`, `name_loc3`, `name_loc6`, `name_loc8`, `description_loc0`, `description_loc2`, `description_loc3`, `description_loc6`, `description_loc8`, `spellId1`, `spellId2`, `spellId3`, `spellId4`, `spellId5` FROM ::items');
$spells = new SpellList(array(['id',
array_filter(array_column($items, 'spellId1')),
array_filter(array_column($items, 'spellId2')),
array_filter(array_column($items, 'spellId3')),
array_filter(array_column($items, 'spellId4')),
array_filter(array_column($items, 'spellId5'))
]), ['interactive' => SpellList::INTERACTIVE_NONE]);
$n = count($items) * count($this->locales);
$j = 0;
foreach ($this->locales as $locId => $loc)
{
Lang::load($loc);
foreach ($items as $id => $item)
{
CLI::write(' * normalizing '.++$j.' / '.$n.' ('.sprintf('%.2f%%', $j * 100 / $n).')', tmpRow: true);
$name = $desc = $effects = null;
// ui escape sequences not in default 335a, but undestood by client and may be custom
if ($_ = Util::localizedString($item, 'name', true))
$name = self::normalize(Lang::unescapeUISequences($_, Lang::FMT_RAW));
if ($_ = Util::localizedString($item, 'description', true))
$desc = self::normalize($_);
for ($i = 1; $i < 6; $i++)
{
$sId = $item['spellId'.$i];
if (!$sId)
continue;
if ($spells->getEntry($sId))
if ($_ = $spells->parseText('description')[0])
$effects .= str_replace('<br />', ' ', $_);
}
if (($effects = self::normalize($effects)) || $name || $desc)
{
$result['id'][] = $id;
$result['locale'][] = $locId;
$result['nName'][] = $name;
$result['nDescription'][] = $desc;
$result['nEffects'][] = $effects;
}
}
}
$n = ceil(count(current($result)) / CLISetup::SQL_BATCH);
for ($i = 0; $i < $n; $i++)
{
CLI::write(' * inserting batch '.($i + 1).' / '.$n, tmpRow: true);
DB::Aowow()->qry('INSERT INTO ::items_search %m', array_map(fn($x) => array_slice($x, $i * CLISetup::SQL_BATCH, CLISetup::SQL_BATCH), $result));
}
}
private function normalizeSpells() : void
{
CLI::write(' - creating indices for spell names, descriptions & buffs...');
DB::Aowow()->qry('TRUNCATE ::spell_search');
CLI::write(' * fetching', tmpRow: true);
$splBuf = [];
$result = [];
$spells = DB::Aowow()->selectAssoc('SELECT `id` AS ARRAY_KEY, `name_loc0`, `name_loc2`, `name_loc3`, `name_loc6`, `name_loc8`, `buff_loc0`, `buff_loc2`, `buff_loc3`, `buff_loc6`, `buff_loc8`, `description_loc0`, `description_loc2`, `description_loc3`, `description_loc6`, `description_loc8` FROM ::spell');
$n = count($spells) * count($this->locales);
$j = 0;
foreach ($this->locales as $locId => $loc)
{
Lang::load($loc);
foreach ($spells as $id => $spell)
{
CLI::write(' * normalizing '.++$j.' / '.$n.' ('.sprintf('%.2f%%', $j * 100 / $n).')', tmpRow: true);
$name = $desc = $buff = null;
// initializing a Spell Object and parsing the tooltip is a lot of effort.
// so don't do that unless we really really have to
if (strpos($spell['description_loc'.$locId], '$') || strpos($spell['buff_loc'.$locId], '$'))
{
$splBuf[$id] ??= new SpellList(array(['id', $id]), ['interactive' => SpellList::INTERACTIVE_NONE]);
if ($_ = $splBuf[$id]->parseText('description')[0])
$desc = self::normalize(str_replace('<br />', ' ', $_));
if ($_ = $splBuf[$id]->parseText('buff')[0])
$buff = self::normalize(str_replace('<br />', ' ', $_));
}
if ($_ = $desc ?: Util::localizedString($spell, 'description', true))
$desc = self::normalize($_);
if ($_ = $buff ?: Util::localizedString($spell, 'buff', true))
$buff = self::normalize($_);
if ($_ = Util::localizedString($spell, 'name', true))
$name = self::normalize($_);
if ($buff || $name || $desc)
{
$result['id'][] = $id;
$result['locale'][] = $locId;
$result['nName'][] = $name;
$result['nDescription'][] = $desc;
$result['nBuff'][] = $buff;
}
}
}
$n = ceil(count(current($result)) / CLISetup::SQL_BATCH);
for ($i = 0; $i < $n; $i++)
{
CLI::write(' * inserting batch '.($i + 1).' / '.$n, tmpRow: true);
DB::Aowow()->qry('INSERT INTO ::spell_search %m', array_map(fn($x) => array_slice($x, $i * CLISetup::SQL_BATCH, CLISetup::SQL_BATCH), $result));
}
}
private static function normalizeRow(array &$row, int $idx, array $keys) : void
{
foreach ($keys as $key)
$row[$key] = self::normalize($row[$key] ?? '');
}
// e.g. "Zul'Aman O'Reilly" => "Zul Aman ZulAman OReilly Reilly"
private static function normalize(?string $words) : ?string
{
if (!$words)
return null;
$words = array_filter(explode(' ', $words), fn($x) => mb_strlen($x) > 2);
$result = [];
foreach ($words as $word)
{
if (($new = trim(preg_replace(Filter::PATTERN_FT, ' ', $word, count: $n))) && $n)
{
if (!strpos($new, ' ')) // caught trailing dots or something
{
$result[] = $new;
continue;
}
if ($splitWords = array_filter(explode(' ', $new), fn($x) => mb_strlen($x) > 2))
$result = array_merge($result, $splitWords);
$result[] = str_replace(' ', '', $new);
continue;
}
$result[] = $word;
}
return $result ? implode(' ', array_unique($result)) : null;
}
});
?>

1
setup/tools/sqlgen/spell.ss.php
View file

@ -199,7 +199,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
DB::Aowow()->qry('TRUNCATE ::spell');
DB::Aowow()->qry('SET SESSION innodb_ft_enable_stopword = OFF');
// merge serverside spells into aowow_spell
$lastMax = 0;