Setup/DBCReader

* port extended client file handling from other branch
 * class DBC -> DBCReader now initializes a DBCFile
   which itself is a BinaryFile
 * update DBCReader to use the new DB wrappers multi-insert feature

aowow

@@ -9,6 +9,9 @@ if (PHP_SAPI === 'cli' && getcwd().DIRECTORY_SEPARATOR.'aowow' != __FILE__)
 require_once 'includes/kernel.php';
 require_once 'includes/setup/cli.class.php';
 require_once 'includes/setup/timer.class.php';
+require_once 'includes/setup/datatypes/primitives.php';
+require_once 'includes/setup/files/binaryfile.class.php';
+require_once 'includes/setup/files/dbcfile.class.php';
 
 require_once 'setup/setup.php';
 

includes/setup/datatypes/primitives.php

@@ -0,0 +1,104 @@
+<?php
+
+namespace Aowow;
+
+if (!defined('AOWOW_REVISION'))
+    die('illegal access');
+
+
+abstract class Primitive
+{
+    public const /* int */    SIZE     = 1;
+    public const /* string */ PACK_FMT = 'x';
+
+    protected string $data;
+
+    public function __construct(BinaryFile|string $data)
+    {
+        if (is_string($data))
+            $this->data = $data;
+        else
+            $this->data = $data->read(static::SIZE);
+    }
+
+    public function pack() : string
+    {
+        return $this->data;
+    }
+
+    public function unpack() : mixed
+    {
+        return current(unpack(static::PACK_FMT, $this->data));
+    }
+
+    public function __debugInfo() : array
+    {
+        return [$this->unpack()];
+    }
+}
+
+class Char extends Primitive
+{
+    public const /* int */    SIZE     = 1;
+    public const /* string */ PACK_FMT = 'C';
+
+    public function unpack() : string
+    {
+        return chr(parent::unpack());
+    }
+}
+
+class Boolean extends Primitive
+{
+    public const /* int */    SIZE     = 1;
+    public const /* string */ PACK_FMT = 'C';
+
+    public function unpack() : string
+    {
+        return !!(parent::unpack());
+    }
+}
+
+class UInt8 extends Primitive
+{
+    public const /* int */    SIZE     = 1;
+    public const /* string */ PACK_FMT = 'C';
+}
+
+class Int8 extends Primitive
+{
+    public const /* int */    SIZE     = 1;
+    public const /* string */ PACK_FMT = 'c';
+}
+
+class UInt16 extends Primitive
+{
+    public const /* int */    SIZE     = 2;
+    public const /* string */ PACK_FMT = 'v';
+}
+
+class Int16 extends Primitive
+{
+    public const /* int */    SIZE     = 2;
+    public const /* string */ PACK_FMT = 's';
+}
+
+class UInt32 extends Primitive
+{
+    public const /* int */    SIZE     = 4;
+    public const /* string */ PACK_FMT = 'V';
+}
+
+class Int32 extends Primitive
+{
+    public const /* int */    SIZE     = 4;
+    public const /* string */ PACK_FMT = 'l';
+}
+
+class Double extends Primitive
+{
+    public const /* int */    SIZE     = 4;
+    public const /* string */ PACK_FMT = 'f';
+}
+
+?>

includes/setup/files/binaryfile.class.php

@@ -0,0 +1,202 @@
+<?php
+
+namespace Aowow;
+
+if (!defined('AOWOW_REVISION'))
+    die('illegal access');
+
+
+class BinaryFile
+{
+    private /*res*/ $handle = null;
+    private string  $data   = '';
+    private int     $pos    = 0;
+
+    protected int $filesize     = 0;
+
+    public string $error = '';
+
+    public function __construct(string $file, private bool $inRAM = true)
+    {
+        if (!file_exists($file))
+        {
+            $this->error = 'file '.$file.' not found';
+            return;
+        }
+
+        if (!$this->handle = fopen($file, 'rb'))
+        {
+            $this->error = 'failed to open file '.$file;
+            return;
+        }
+
+        $this->filesize = filesize($file);
+
+        if ($inRAM)
+            $this->data = file_get_contents($file);
+    }
+
+    public function __destruct()
+    {
+        $this->close();
+    }
+
+
+    /**********************/
+    /* direct file access */
+    /**********************/
+
+    public function read(int $bytes) : ?string
+    {
+        if ($this->error || !is_resource($this->handle) || $bytes < 0)
+            return null;
+
+        $start = $this->pos;
+        $this->pos += $bytes;
+
+        if ($this->inRAM)
+            return substr($this->data, $start, $bytes);
+        else
+            return fread($this->handle, $bytes);
+    }
+
+    public function readOffset(int $bytes, int $offset, bool $jumpBack = true) : ?string
+    {
+        if ($this->error || !is_resource($this->handle))
+            return null;
+
+        if ($jumpBack)
+            $curPos = $this->inRAM ? $this->pos : ftell($this->handle);
+
+        $this->seek($offset);
+
+        $str = $this->read($bytes);
+
+        if ($jumpBack)
+            $this->seek($curPos);
+
+        return $str;
+    }
+
+    public function seek(int $pos) : int
+    {
+        if (!is_resource($this->handle))
+            return 0;
+
+        if ($pos < 0)
+            $pos = 0;
+        if ($pos > $this->filesize)
+            $pos = $this->filesize;
+
+        $this->pos = $pos;
+
+        if (!$this->inRAM)
+            fseek($this->handle, $pos, SEEK_SET);
+
+        return $pos;
+    }
+
+    public function ffwd(int $bytes) : int
+    {
+        if (!is_resource($this->handle))
+            return 0;
+
+        $curPos = $this->inRAM ? $this->pos : ftell($this->handle);
+
+        if ($curPos + $bytes < 0)
+            $bytes -= $curPos;
+        if ($curPos + $bytes > $this->filesize)
+            $bytes -= $this->filesize;
+
+        $this->pos += $bytes;
+
+        if ($this->inRAM)
+            return $this->pos;
+
+        fseek($this->handle, $bytes, SEEK_CUR);
+        return ftell($this->handle);
+    }
+
+    public function close() : void
+    {
+        if (is_resource($this->handle))
+            fclose($this->handle);
+    }
+
+    public function tell() : int
+    {
+        if (!is_resource($this->handle))
+            return 0;
+
+        return $this->inRAM ? $this->pos : ftell($this->handle);
+    }
+
+    /******************/
+    /* read Primitive */
+    /******************/
+
+    public function readInt8() : ?Int8
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Int8($this);
+    }
+
+    public function readInt16() : ?Int16
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Int16($this);
+    }
+
+    public function readInt32() : ?Int32
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Int32($this);
+    }
+
+    public function readUInt8() : ?UInt8
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new UInt8($this);
+    }
+
+    public function readUInt16() : ?UInt16
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new UInt16($this);
+    }
+
+    public function readUInt32() : ?UInt32
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new UInt32($this);
+    }
+
+    public function readFloat() : ?Double
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Double($this);
+    }
+
+    public function readChar() : ?Char
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Char($this);
+    }
+
+    public function readBool() : ?Boolean
+    {
+        if (!is_resource($this->handle))
+            return null;
+        return new Boolean($this);
+    }
+}
+
+?>

includes/setup/files/dbcfile.class.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace Aowow;
+
+if (!defined('AOWOW_REVISION'))
+    die('illegal access');
+
+
+class DBCFile extends BinaryFile
+{
+    private const /* string */ MAGIC       = 'WDBC';
+    private const /* int    */ HEADER_SIZE = 16;
+
+    private readonly int $stringSize;
+    private readonly int $stringOffset;
+
+    public readonly int $nCols;
+    public readonly int $nRows;
+    public readonly int $recordSize;
+
+    public function __construct(string $file)
+    {
+        parent::__construct($file);
+
+        if ($this->filesize < strlen(self::MAGIC) + self::HEADER_SIZE)
+        {
+            $this->error = 'file '.$file.' too small for a dbc';
+            $this->close();
+            return;
+        }
+
+        if ($this->read(4) != self::MAGIC)
+        {
+            $this->error = 'file '.$file.' has incorrect magic bytes';
+            $this->close();
+            return;
+        }
+
+        [, $this->nRows, $this->nCols, $this->recordSize, $this->stringSize] = unpack(UInt32::PACK_FMT.'4', $this->read(self::HEADER_SIZE));
+        $this->stringOffset = strlen(self::MAGIC) + self::HEADER_SIZE + $this->recordSize * $this->nRows;
+
+        if ($this->stringOffset + $this->stringSize != $this->filesize)
+        {
+            $this->error = 'file '.$file.' has unexpected size - expected: '.($this->stringOffset + $this->stringSize).' has: '.$this->filesize;
+            $this->close();
+            return;
+        }
+    }
+
+    public function readRecord(string $colFmt = "V*") : array
+    {
+        return unpack($colFmt, $this->read($this->recordSize));
+    }
+
+    public function readString() : ?string
+    {
+        $x = $this->readUInt32();
+        if (is_null($x))
+            return null;
+
+        return $this->getStringFromBlock($x->unpack());
+    }
+
+    public function getStringFromBlock(int $offset) : ?string
+    {
+        $curPos = $this->tell();
+
+        $this->seek($this->stringOffset + $offset);
+
+        // apparently it is more efficient to read more than one byte at once..?
+        $str = '';
+        while (($pos = strpos($str, "\0")) === false)
+            $str .= $this->read(255);
+
+        $str = substr($str, 0, $pos);
+
+        $this->seek($curPos);
+
+        return $pos ? $str : null;
+    }
+}
+
+?>

setup/setup.php

@@ -12,7 +12,7 @@ if (!CLI)
 require_once 'setup/tools/setupScript.class.php';
 require_once 'setup/tools/utilityScript.class.php';
 require_once 'setup/tools/CLISetup.class.php';
-require_once 'setup/tools/dbc.class.php';
+require_once 'setup/tools/dbcreader.class.php';
 require_once 'setup/tools/imagecreatefromblp.func.php';
 
 CLISetup::init();

setup/tools/CLISetup.class.php

@@ -654,7 +654,7 @@ class CLISetup
         if (DB::Aowow()->selectCell('SHOW TABLES LIKE %s', 'dbc_'.$name) && DB::Aowow()->selectCell('SELECT count(1) FROM %n', 'dbc_'.$name))
             return true;
 
-        $dbc = new DBC($name, ['temporary' => self::getOpt('delete')]);
+        $dbc = new DBCReader($name, ['temporary' => self::getOpt('delete')]);
         if ($dbc->error)
         {
             CLI::write('CLISetup::loadDBC() - required DBC '.$name.'.dbc not found!', CLI::LOG_ERROR);

setup/tools/clisetup/dbc.us.php

@@ -40,7 +40,7 @@ CLISetup::registerUtility(new class extends UtilityScript
             if ($args[0])
                 $opts['tableName'] = $args[0];
 
-            $dbc = new DBC(strtolower($n), $opts, $args[1] ?: DBC::DEFAULT_WOW_BUILD);
+            $dbc = new DBCReader(strtolower($n), $opts, $args[1] ?: DBCReader::DEFAULT_WOW_BUILD);
             if ($dbc->error)
             {
                 CLI::write('[dbc] required DBC '.CLI::bold($n).'.dbc not found!', CLI::LOG_ERROR);
@@ -84,7 +84,7 @@ CLISetup::registerUtility(new class extends UtilityScript
         CLI::write();
         CLI::write('  Known DBC files:', -1, false);
 
-        $defs   = DBC::getDefinitions();
+        $defs   = DBCReader::getDefinitions();
         $letter = '';
         $buff   = [];
 

setup/tools/dbc.class.php

@@ -1,512 +0,0 @@
-<?php
-
-namespace Aowow;
-
-/*
-    DBC::read - PHP function for loading DBC file into array
-    This file is a part of AoWoW project.
-    Copyright (C) 2009-2010  Mix <ru-mangos.ru>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-if (!defined('AOWOW_REVISION'))
-    die('illegal access');
-
-if (!CLI)
-    die('not in cli mode');
-
-
-class DBC
-{
-    private $isGameTable = false;
-    private $localized   = false;
-    private $tempTable   = true;
-    private $tableName   = '';
-
-    private $dataBuffer  = [];
-    private $bufferSize  = 500;
-
-    private static $structs = [];
-
-    private $fileRefs    = [];
-    private $curFile     = '';
-
-    public $error  = true;
-    public $fields = [];
-    public $format = [];
-    public $file   = '';
-
-    private $macro = array(
-        'LOC'   => 'sxsssxsxsxxxxxxxx',                     // pre 4.x locale block (in use)
-        'X_LOC' => 'xxxxxxxxxxxxxxxxx'                      // pre 4.x locale block (unused)
-    );
-
-    private $unpackFmt = array(                             // Supported format characters:
-        'x' => 'x/x/x/x',                                   // x - not used/unknown, 4 bytes
-        'X' => 'x',                                         // X - not used/unknown, 1 byte
-        's' => 'V',                                         // s - string block index, 4 bytes
-        'S' => 'V',                                         // S - string block index, 4 bytes - localized; autofill
-        'f' => 'f',                                         // f - float, 4 bytes (rounded to 4 digits after comma)
-        'i' => 'l',                                         // i - signed int, 4 bytes
-        'I' => 'l',                                         // I - signed int, 4 bytes, sql index
-        'u' => 'V',                                         // u - unsigned int, 4 bytes
-        'U' => 'V',                                         // U - unsigned int, 4 bytes, sql index
-        'b' => 'C',                                         // b - unsigned char, 1 byte
-        'd' => 'x4',                                        // d - ordered by this field, not included in array
-        'n' => 'V'                                          // n - unsigned int, 4 bytes, sql primary key
-    );
-
-    public  const DEFAULT_WOW_BUILD = '12340';
-    private const INI_FILE_PATH     = 'setup/tools/dbc/%s.ini';
-
-    public function __construct($file, $opts = [], string $wowBuild = self::DEFAULT_WOW_BUILD)
-    {
-        self::loadStructs($wowBuild);
-
-        $file = strtolower($file);
-        if (empty(self::$structs[$file]))
-        {
-            CLI::write('no structure known for '.$file.'.dbc, build '.$wowBuild, CLI::LOG_ERROR);
-            return;
-        }
-
-        foreach (self::$structs[$file] as $name => $type)
-        {
-            // resolove locale macro
-            if (isset($this->macro[$type]))
-            {
-                $this->localized = true;
-                for ($i = 0; $i < strlen($this->macro[$type]); $i++)
-                    $this->format[$name.'_loc'.$i] = $this->macro[$type][$i];
-            }
-            else
-            {
-                $this->format[$name] = $type;
-                if ($type == 'S')
-                    $this->localized = true;
-            }
-        }
-
-        $this->file = $file;
-
-        if (is_bool($opts['temporary']))
-            $this->tempTable = $opts['temporary'];
-
-        if (!empty($opts['tableName']))
-            $this->tableName = $opts['tableName'];
-        else
-            $this->tableName = 'dbc_'.$file;
-
-        // gameTable-DBCs don't have an index and are accessed through value order
-        // allas, you cannot do this with mysql, so we add a 'virtual' index
-        $this->isGameTable = array_values($this->format) == ['f'] && substr($file, 0, 2) == 'gt';
-
-        $foundMask = 0x0;
-        foreach (Locale::cases() as $loc)
-        {
-            if (!in_array($loc, CLISetup::$locales))
-                continue;
-
-            if ($foundMask & (1 << $loc->value))
-                continue;
-
-            foreach ($loc->gameDirs() as $dir)
-            {
-                $fullPath = CLI::nicePath($this->file.'.dbc', CLISetup::$srcDir, $dir, 'DBFilesClient');
-                if (!CLISetup::fileExists($fullPath))
-                    continue;
-
-                $this->curFile = $fullPath;
-                if ($this->validateFile($loc))
-                {
-                    $foundMask |= (1 << $loc->value);
-                    break;
-                }
-            }
-        }
-
-        if (!$this->fileRefs)
-        {
-            CLI::write('no suitable files found for '.$file.'.dbc, aborting.', CLI::LOG_ERROR);
-            return;
-        }
-
-        // check if DBCs are identical
-        $headers = array_column($this->fileRefs, 2);
-        $x = array_unique(array_column($headers, 'recordCount'));
-        if (count($x) != 1)
-        {
-            CLI::write('some DBCs have different record counts ('.implode(', ', $x).' respectively). cannot merge!', CLI::LOG_ERROR);
-            return;
-        }
-        $x = array_unique(array_column($headers, 'fieldCount'));
-        if (count($x) != 1)
-        {
-            CLI::write('some DBCs have differenct field counts ('.implode(', ', $x).' respectively). cannot merge!', CLI::LOG_ERROR);
-            return;
-        }
-        $x = array_unique(array_column($headers, 'recordSize'));
-        if (count($x) != 1)
-        {
-            CLI::write('some DBCs have differenct record sizes ('.implode(', ', $x).' respectively). cannot merge!', CLI::LOG_ERROR);
-            return;
-        }
-
-        $this->error = false;
-    }
-
-    public function readFile()
-    {
-        if (!$this->file || $this->error)
-            return [];
-
-        $this->createTable();
-
-        if ($this->localized)
-            CLI::write(' - DBC: reading and merging '.$this->file.'.dbc for locales '.Lang::concat(array_keys($this->fileRefs), callback: fn($x) => CLI::bold(Locale::from($x)->name)));
-        else
-            CLI::write(' - DBC: reading '.$this->file.'.dbc');
-
-        if (!$this->read())
-        {
-            CLI::write(' - DBC::read() returned with error', CLI::LOG_ERROR);
-            return false;
-        }
-
-        return true;
-    }
-
-    public function getTableName() : string
-    {
-        return $this->tableName;
-    }
-
-    public static function getDefinitions() : array
-    {
-        if (empty(self::$structs))
-            self::loadStructs();
-
-        return array_keys(self::$structs);
-    }
-
-    private static function loadStructs(string $wowBuild = self::DEFAULT_WOW_BUILD) : void
-    {
-        $structFile = sprintf(self::INI_FILE_PATH, $wowBuild);
-
-        if (!file_exists($structFile))
-        {
-            CLI::write('no structure file found for wow build '.$wowBuild, CLI::LOG_ERROR);
-            return;
-        }
-
-        self::$structs = parse_ini_file($structFile, true);
-    }
-
-    private function endClean()
-    {
-        foreach ($this->fileRefs as &$ref)
-            fclose($ref[0]);
-
-        $this->dataBuffer = null;
-    }
-
-    private function readHeader(&$handle = null) : array
-    {
-        if (!is_resource($handle))
-            $handle = fopen($this->curFile, 'rb');
-
-        if (!$handle)
-            return [];
-
-        if (fread($handle, 4) != 'WDBC')
-        {
-            CLI::write('file '.$this->curFile.' has incorrect magic bytes', CLI::LOG_ERROR);
-            fclose($handle);
-            return [];
-        }
-
-        return unpack('VrecordCount/VfieldCount/VrecordSize/VstringSize', fread($handle, 16));
-    }
-
-    private function validateFile(Locale $loc) : bool
-    {
-        $filesize = filesize($this->curFile);
-        if ($filesize < 20)
-        {
-            CLI::write('file '.$this->curFile.' is too small for a DBC file', CLI::LOG_ERROR);
-            return false;
-        }
-
-        $header = $this->readHeader($handle);
-        if (!$header)
-        {
-            CLI::write('cannot open file '.$this->curFile, CLI::LOG_ERROR);
-            return false;
-        }
-
-        // Different debug checks to be sure, that file was opened correctly
-        $debugStr = '(recordCount='.$header['recordCount'].
-                    ' fieldCount=' .$header['fieldCount'] .
-                    ' recordSize=' .$header['recordSize'] .
-                    ' stringSize=' .$header['stringSize'] .')';
-
-        if ($header['recordCount'] * $header['recordSize'] + $header['stringSize'] + 20 != $filesize)
-        {
-            CLI::write('file '.$this->curFile.' has incorrect size '.$filesize.': '.$debugStr, CLI::LOG_ERROR);
-            fclose($handle);
-            return false;
-        }
-
-        if ($header['fieldCount'] != count($this->format))
-        {
-            CLI::write('incorrect format ('.implode('', $this->format).') specified for file '.$this->curFile.' fieldCount='.$header['fieldCount'], CLI::LOG_ERROR);
-            fclose($handle);
-            return false;
-        }
-
-        $this->fileRefs[$loc->value] = [$handle, $this->curFile, $header];
-
-        return true;
-    }
-
-    private function createTable()
-    {
-        if ($this->error)
-            return;
-
-        $pKey    = '';
-        $query   = 'CREATE '.($this->tempTable ? 'TEMPORARY' : '').' TABLE `'.$this->tableName.'` (';
-        $indizes = [];
-
-        if ($this->isGameTable)
-        {
-            $query .= '`idx` INT SIGNED NOT NULL, ';
-            $pKey   = 'idx';
-        }
-
-        foreach ($this->format as $name => $type)
-        {
-            switch ($type)
-            {
-                case 'f':
-                    $query .= '`'.$name.'` FLOAT NOT NULL, ';
-                    break;
-                case 'S':
-                    for ($l = 0; $l < strlen($this->macro['LOC']); $l++)
-                        if ($this->macro['LOC'][$l] == 's')
-                            $query .= '`'.$name.'_loc'.$l.'` TEXT NULL, ';
-
-                    break;
-                case 's':
-                    $query .= '`'.$name.'` TEXT NULL, ';
-                    break;
-                case 'b':
-                    $query .= '`'.$name.'` TINYINT UNSIGNED NOT NULL, ';
-                    break;
-                case 'I':
-                    $indizes[] = $name;
-                case 'i':
-                case 'n':
-                    $query .= '`'.$name.'` INT SIGNED NOT NULL, ';
-                    break;
-                case 'U':
-                    $indizes[] = $name;
-                case 'u':
-                    $query .= '`'.$name.'` INT UNSIGNED NOT NULL, ';
-                    break;
-                default:                                    // 'x', 'X', 'd'
-                    continue 2;
-            }
-
-            if ($type == 'n')
-                $pKey = $name;
-        }
-
-        foreach ($indizes as $i)
-            $query .= 'KEY `idx_'.$i.'` (`'.$i.'`), ';
-
-        if ($pKey)
-            $query .= 'PRIMARY KEY (`'.$pKey.'`) ';
-        else
-            $query = substr($query, 0, -2);
-
-        $query .=  ') COLLATE=\'utf8mb4_unicode_ci\' ENGINE=InnoDB';
-
-        DB::Aowow()->qry('DROP TABLE IF EXISTS %n', $this->tableName);
-        DB::Aowow()->qry($query);
-    }
-
-    private function writeToDB()
-    {
-        if (!$this->dataBuffer || $this->error)
-            return;
-
-        $cols = [];
-        foreach ($this->format as $n => $type)
-        {
-            switch ($type)
-            {
-                case 'x':
-                case 'X':
-                case 'd':
-                    continue 2;
-                case 'S':
-                    for ($l = 0; $l < strlen($this->macro['LOC']); $l++)
-                        if ($this->macro['LOC'][$l] == 's')
-                         $cols[] = $n.'_loc'.$l;
-                    break;
-                default:
-                    $cols[] = $n;
-            }
-        }
-
-        if ($this->isGameTable)
-            array_unshift($cols, 'idx');
-
-        foreach ($this->dataBuffer as $row)
-            DB::Aowow()->qry('INSERT INTO %n %v', $this->tableName, array_combine($cols, $row));
-
-        $this->dataBuffer = [];
-    }
-
-    private function read()
-    {
-        // Check that record size also matches
-        $itr       = 0;
-        $recSize   = 0;
-        $unpackStr = '';
-        foreach ($this->format as $ch)
-        {
-            if ($ch == 'X' || $ch == 'b')
-                $recSize += 1;
-            else
-                $recSize += 4;
-
-            if (!isset($this->unpackFmt[$ch]))
-            {
-                CLI::write('unknown format parameter \''.$ch.'\' in format string', CLI::LOG_ERROR);
-                return false;
-            }
-
-            $unpackStr .= '/'.$this->unpackFmt[$ch];
-
-            if ($ch != 'X' && $ch != 'x')
-                $unpackStr .= 'f'.$itr;                     // output can't have numeric key as it gets interpreted as repeat factor here
-
-            $itr++;
-        }
-
-        $unpackStr = substr($unpackStr, 1);
-
-        // Optimizing unpack string: 'x/x/x/x/x/x' => 'x6'
-        while (preg_match('/(x\/)+x/', $unpackStr, $r))
-            $unpackStr = substr_replace($unpackStr, 'x'.((strlen($r[0]) + 1) / 2), strpos($unpackStr, $r[0]), strlen($r[0]));
-
-        // we asserted all DBCs to be identical in structure. pick first header for checks
-        $header = reset($this->fileRefs)[2];
-
-        if ($recSize != $header['recordSize'])
-        {
-            CLI::write('format string size ('.$recSize.') for file '.$this->file.' does not match actual size ('.$header['recordSize'].')', CLI::LOG_ERROR);
-            return false;
-        }
-
-        // And, finally, extract the records
-        $strBlock = 4 + 16 + $header['recordSize'] * $header['recordCount'];
-
-        for ($i = 0; $i < $header['recordCount']; $i++)
-        {
-            $row = [];
-            $idx = $i;
-
-            // add 'virtual' enumerator for gt*-dbcs
-            if ($this->isGameTable)
-                $row[-1] = $i;
-
-            foreach ($this->fileRefs as $locId => [$handle, $fullPath, $header])
-            {
-                $rec = unpack($unpackStr, fread($handle, $header['recordSize']));
-
-                $offset = 0;
-                foreach (array_values($this->format) as $j => $type)
-                {
-                    if (!isset($rec['f'.$j]))
-                        continue;
-
-                    $outIdx = $j + $offset;
-
-                    if (!empty($row[$outIdx]) && $type != 'S')
-                        continue;
-
-                    switch ($type)
-                    {
-                        case 'S':                           // localized String - autofill
-                            $offset = substr_count($this->macro['LOC'], 's');
-
-                            for ($k = 0; $k < strlen($this->macro['LOC']); $k++)
-                            {
-                                if ($this->macro['LOC'][$k] != 's')
-                                    continue;
-
-                                if (!isset($row[$j + $k]))  // prep locale fields
-                                    $row[$j + $k] = null;
-                            }
-
-                            // provide outIdx for passthrough
-                            $outIdx = $j + $locId;
-                        case 's':
-                            $curPos = ftell($handle);
-                            fseek($handle, $strBlock + $rec['f'.$j]);
-
-                            $str = $chr = '';
-                            do
-                            {
-                                $str .= $chr;
-                                $chr = fread($handle, 1);
-                            }
-                            while ($chr != "\000");
-
-                            fseek($handle, $curPos);
-                            $row[$outIdx] = $str;
-                            break;
-                        case 'f':
-                            $row[$outIdx] = round($rec['f'.$j], 8);
-                            break;
-                        case 'n':                               // DO NOT BREAK!
-                            $idx = $rec['f'.$j];
-                        default:                                // nothing special .. 'i', 'u' and the likes
-                            $row[$outIdx] = $rec['f'.$j];
-                    }
-                }
-
-                if (!$this->localized)                          // one match is enough
-                    break;
-            }
-
-            $this->dataBuffer[$idx] = array_values($row);
-
-            if (count($this->dataBuffer) >= $this->bufferSize)
-                $this->writeToDB();
-        }
-
-        $this->writeToDB();
-
-        $this->endClean();
-
-        return true;
-    }
-}
-
-?>

setup/tools/dbcreader.class.php

@@ -0,0 +1,362 @@
+<?php
+
+namespace Aowow;
+
+/*
+    DBCReader::read - PHP function for loading DBC file into array
+    This file is a part of AoWoW project.
+    Copyright (C) 2009-2010  Mix <ru-mangos.ru>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+if (!defined('AOWOW_REVISION'))
+    die('illegal access');
+
+if (!CLI)
+    die('not in cli mode');
+
+
+class DBCReader
+{
+    private const /* string */ INI_FILE_PATH   = 'setup/tools/dbc/%s.ini';
+    private const /* int    */ MAX_INSERT_ROWS = 500;
+
+    public  const /* string */ DEFAULT_WOW_BUILD = '12340';
+
+    private bool   $isGameTable = false;
+    private bool   $isLocalized = false;
+    private bool   $isTempTable = true;
+    private string $tableName   = '';
+    private array  $dataBuffer  = [];
+    private array  $fileRefs    = [];
+    private array  $format      = [];
+    private string $recordFmt   = '';
+    private array  $macro       = array(
+        'LOC'   => 'sxsssxsxsxxxxxxxx',                     // pre 4.x locale block (in use)
+        'X_LOC' => 'xxxxxxxxxxxxxxxxx'                      // pre 4.x locale block (unused)
+    );
+    private array  $unpackFmt   = array(                    // Supported format characters:
+        'x' => Primitive::PACK_FMT.'4',                     // x - not used/unknown, 4 bytes
+        'X' => Primitive::PACK_FMT,                         // X - not used/unknown, 1 byte
+        's' => UInt32::PACK_FMT,                            // s - string block index, 4 bytes
+        'S' => UInt32::PACK_FMT,                            // S - string block index, 4 bytes - localized; autofill
+        'f' => Double::PACK_FMT,                            // f - float, 4 bytes (rounded to 4 digits after comma)
+        'i' => Int32::PACK_FMT,                             // i - signed int, 4 bytes
+        'I' => Int32::PACK_FMT,                             // I - signed int, 4 bytes, sql index
+        'u' => UInt32::PACK_FMT,                            // u - unsigned int, 4 bytes
+        'U' => UInt32::PACK_FMT,                            // U - unsigned int, 4 bytes, sql index
+        'b' => UInt8::PACK_FMT,                             // b - unsigned char, 1 byte
+        'd' => Primitive::PACK_FMT.'4',                     // d - ordered by this field, not included in array
+        'n' => UInt32::PACK_FMT                             // n - unsigned int, 4 bytes, sql primary key
+    );
+
+    private static array $structs = [];
+
+    public bool $error = true;
+
+    public function __construct(public string $file, array $opts = [], string $wowBuild = self::DEFAULT_WOW_BUILD)
+    {
+        self::loadStructs($wowBuild);
+
+        $this->file = strtolower($this->file);
+        if (empty(self::$structs[$this->file]))
+        {
+            CLI::write('no structure known for '.$this->file.'.dbc, build '.$wowBuild, CLI::LOG_ERROR);
+            return;
+        }
+
+        foreach (self::$structs[$this->file] as $name => $type)
+        {
+            // resolove locale macro
+            if (isset($this->macro[$type]))
+            {
+                $this->isLocalized = true;
+                for ($i = 0; $i < strlen($this->macro[$type]); $i++)
+                {
+                    $this->format[$name.'_loc'.$i] = $this->macro[$type][$i];
+                    $this->recordFmt .= '/'.$this->unpackFmt[$this->macro[$type][$i]].$name.'_loc'.$i;
+                }
+            }
+            else if (!isset($this->unpackFmt[$type]))
+            {
+                CLI::write('unknown format parameter '.CLI::bold($type).' at for field '.CLI::bold($name).' in format string', CLI::LOG_ERROR);
+                return;
+            }
+            else
+            {
+                $this->format[$name] = $type;
+                $this->recordFmt .= '/'.$this->unpackFmt[$type];
+                if ($type !== 'x' && $type !== 'X')
+                    $this->recordFmt .= $name;
+
+                if ($type === 'S')
+                    $this->isLocalized = true;
+            }
+        }
+
+        // Optimizing unpack string: 'x/x/x/x/x/x' => 'x6'
+        $this->recordFmt = preg_replace_callback('/x(\/x)+/i', fn($m) => 'x'.((strlen($m[0]) + 1) / 2), substr($this->recordFmt, 1));
+
+        if (is_bool($opts['temporary']))
+            $this->isTempTable = $opts['temporary'];
+
+        if (!empty($opts['tableName']))
+            $this->tableName = $opts['tableName'];
+        else
+            $this->tableName = 'dbc_'.$this->file;
+
+        // gameTable-DBCs don't have an index and are accessed through value order
+        // allas, you cannot do this with mysql, so we add a 'virtual' index
+        $this->isGameTable = array_values($this->format) == ['f'] && substr($this->file, 0, 2) == 'gt';
+
+        $foundMask = 0x0;
+        foreach (Locale::cases() as $loc)
+        {
+            if (!in_array($loc, CLISetup::$locales))
+                continue;
+
+            if ($foundMask & (1 << $loc->value))
+                continue;
+
+            foreach ($loc->gameDirs() as $dir)
+            {
+                $fullPath = CLI::nicePath($this->file.'.dbc', CLISetup::$srcDir, $dir, 'DBFilesClient');
+                if (!CLISetup::fileExists($fullPath))
+                    continue;
+
+                $dbcFile = new DBCFile($fullPath);
+                if ($dbcFile->error)
+                {
+                    CLI::write($dbcFile->error, CLI::LOG_ERROR);
+                    unset($dbcFile);
+                    continue;
+                }
+
+                if ($dbcFile->nCols != count($this->format))
+                {
+                    CLI::write('incorrect format specified for file '.$this->file.' - expected fields: '.count($this->format).' read fields: '.$dbcFile->nCols, CLI::LOG_ERROR);
+                    unset($dbcFile);
+                    continue;
+                }
+
+                $recSize = 0;
+                foreach ($this->format as $ch)
+                    $recSize += ($ch == 'X' || $ch == 'b') ? 1 : 4;
+
+                if ($recSize != $dbcFile->recordSize)
+                {
+                    CLI::write('format string size ('.$recSize.') for file '.$this->file.' does not match actual size ('.$dbcFile->recordSize.')', CLI::LOG_ERROR);
+                    unset($dbcFile);
+                    continue;
+                }
+
+                $this->fileRefs[$loc->value] = $dbcFile;
+                $foundMask |= (1 << $loc->value);
+            }
+        }
+
+        if (!$this->fileRefs)
+        {
+            CLI::write('no suitable files found for '.$this->file.'.dbc, aborting.', CLI::LOG_ERROR);
+            return;
+        }
+
+        // check if DBCs are identical
+
+        $tests = ['nRows' => null, 'nCols' => null, 'recordSize' => null];
+        foreach ($this->fileRefs as $fileRef)
+        {
+            foreach ($tests as $field => $val)
+            {
+                if ($val === null)
+                    $tests[$field] = $fileRef->{$field};
+                else if ($val != $fileRef->{$field})
+                {
+                    CLI::write('some DBCs have different '.$field.': '.CLI::bold($val).' <> '.CLI::bold($fileRef->{$field}).' respectively. cannot merge!', CLI::LOG_ERROR);
+                    return;
+                }
+            }
+        }
+
+        $this->error = false;
+    }
+
+    public function readFile() : bool
+    {
+        if (!$this->file || $this->error)
+            return false;
+
+        $this->createTable();
+
+        if ($this->isLocalized)
+            CLI::write(' - DBC: reading and merging '.$this->file.'.dbc for locales '.Lang::concat(array_keys($this->fileRefs), callback: fn($x) => CLI::bold(Locale::from($x)->name)));
+        else
+            CLI::write(' - DBC: reading '.$this->file.'.dbc');
+
+        $this->read();
+
+        return true;
+    }
+
+    public function getTableName() : string
+    {
+        return $this->tableName;
+    }
+
+    public static function getDefinitions() : array
+    {
+        if (empty(self::$structs))
+            self::loadStructs();
+
+        return array_keys(self::$structs);
+    }
+
+    private static function loadStructs(string $wowBuild = self::DEFAULT_WOW_BUILD) : void
+    {
+        $structFile = sprintf(self::INI_FILE_PATH, $wowBuild);
+
+        if (!file_exists($structFile))
+        {
+            CLI::write('no structure file found for wow build '.$wowBuild, CLI::LOG_ERROR);
+            return;
+        }
+
+        self::$structs = parse_ini_file($structFile, true);
+    }
+
+    private function endClean() : void
+    {
+        unset($this->fileRefs, $this->dataBuffer);
+    }
+
+    private function createTable() : void
+    {
+        if ($this->error)
+            return;
+
+        $pKey    = '';
+        $query   = 'CREATE '.($this->isTempTable ? 'TEMPORARY' : '').' TABLE `'.$this->tableName.'` (';
+        $indizes = [];
+
+        if ($this->isGameTable)
+        {
+            $query .= '`idx` INT SIGNED NOT NULL, ';
+            $pKey   = 'idx';
+        }
+
+        foreach ($this->format as $name => $type)
+        {
+            $query .= match($type)
+            {
+                'f'           => '`'.$name.'` FLOAT NOT NULL, ',
+                's'           => '`'.$name.'` TEXT NULL, ',
+                'b'           => '`'.$name.'` TINYINT UNSIGNED NOT NULL, ',
+                'i', 'I', 'n' => '`'.$name.'` INT SIGNED NOT NULL, ',
+                'u', 'U'      => '`'.$name.'` INT SIGNED NOT NULL, ',
+                'S'           => (function ($n) {
+                                     $buf = '';
+                                     for ($l = 0; $l < strlen($this->macro['LOC']); $l++)
+                                         if ($this->macro['LOC'][$l] == 's')
+                                             $buf .= '`'.$n.'_loc'.$l.'` TEXT NULL, ';
+                                     return $buf;
+                                 })($name),
+                default       => ''                         // 'x', 'X', 'd'
+            };
+
+            if ($this->isGameTable)
+                continue;
+
+            if ($type == 'I' || $type == 'U')
+                $indizes[] = $name;
+            if ($type == 'n')
+                $pKey = $name;
+        }
+
+        foreach ($indizes as $i)
+            $query .= 'KEY `idx_'.$i.'` (`'.$i.'`), ';
+
+        if ($pKey)
+            $query .= 'PRIMARY KEY (`'.$pKey.'`) ';
+        else
+            $query = substr($query, 0, -2);
+
+        $query .=  ') COLLATE=\'utf8mb4_unicode_ci\' ENGINE=InnoDB';
+
+        DB::Aowow()->qry('DROP TABLE IF EXISTS %n', $this->tableName);
+        DB::Aowow()->qry($query);
+    }
+
+    private function writeToDB() : void
+    {
+        if (!$this->dataBuffer || $this->error)
+            return;
+
+        DB::Aowow()->qry('INSERT INTO %n %m', $this->tableName, $this->dataBuffer);
+
+        $this->dataBuffer = [];
+    }
+
+    private function read() : void
+    {
+        $nRows = reset($this->fileRefs)->nRows;             // set to actual value once we have a file handle
+
+        for ($i = 0; $i < $nRows; $i++)
+        {
+            // add 'virtual' enumerator for gt*-dbcs
+            if ($this->isGameTable)
+                $this->dataBuffer['idx'][$i] = $i;
+
+            foreach ($this->fileRefs as $locId => $dbcFile)
+            {
+                // note that the file pointer is already on the first record as the DBCFile reads its own header
+                $row = $dbcFile->readRecord($this->recordFmt);
+
+                foreach ($row as $name => $value)
+                {
+                    $type = $this->format[$name];
+
+                    // handle locale fields for post 3.3.5a DBCs
+                    if ($type === 'S')
+                    {
+                        for ($k = 0; $k < strlen($this->macro['LOC']); $k++)
+                            if ($this->macro['LOC'][$k] === 's')
+                                $this->dataBuffer[$name.'_loc'.$k][$i] ??= null;
+
+                        $this->dataBuffer[$name.'_loc'.$locId][$i] ??= $dbcFile->getStringFromBlock($value);
+                    }
+                    if (empty($this->dataBuffer[$name][$i]))
+                    {
+                        if ($type == 's')
+                            $this->dataBuffer[$name][$i] ??= $dbcFile->getStringFromBlock($value);
+                        else
+                            $this->dataBuffer[$name][$i] = $value;
+                    }
+                }
+
+                if (!$this->isLocalized)                    // one match is enough
+                    break;
+            }
+
+            if (count(current($this->dataBuffer)) >= self::MAX_INSERT_ROWS)
+                $this->writeToDB();
+        }
+
+        $this->writeToDB();
+
+        $this->endClean();
+    }
+}
+
+?>

setup/tools/setupScript.class.php

@@ -30,7 +30,7 @@ trait TrDBCcopy
 
         CLI::write('[sql] copying '.$this->dbcSourceFiles[0].'.dbc into aowow_'.$this->command);
 
-        $dbc = new DBC($this->dbcSourceFiles[0], ['temporary' => false, 'tableName' => 'aowow_'.$this->command]);
+        $dbc = new DBCReader($this->dbcSourceFiles[0], ['temporary' => false, 'tableName' => 'aowow_'.$this->command]);
         if ($dbc->error)
             return false;
 

setup/tools/sqlgen/sounds.ss.php

@@ -62,10 +62,10 @@ CLISetup::registerSetup("sql", new class extends SetupScript
         // .mp3 => audio/mpeg
 
         $query =
-           'SELECT `id` AS `id`, `type` AS `cat`, `name`, 0 AS `cuFlags`,
+           'SELECT `id` AS "id", `type` AS "cat", `name`, 0 AS "cuFlags",
-                   `file1` AS `soundFile1`, `file2` AS `soundFile2`, `file3` AS `soundFile3`, `file4` AS `soundFile4`, `file5`  AS `soundFile5`,
+                   `file1` AS "soundFile1", `file2` AS "soundFile2", `file3` AS "soundFile3", `file4` AS "soundFile4", `file5`  AS "soundFile5",
-                   `file6` AS `soundFile6`, `file7` AS `soundFile7`, `file8` AS `soundFile8`, `file9` AS `soundFile9`, `file10` AS `soundFile10`,
+                   `file6` AS "soundFile6", `file7` AS "soundFile7", `file8` AS "soundFile8", `file9` AS "soundFile9", `file10` AS "soundFile10",
-                   `path`, `flags`
+                   IFNULL(`path`, "") AS "path", `flags`
             FROM   dbc_soundentries
             LIMIT  %i, %i';
 
@@ -95,6 +95,9 @@ CLISetup::registerSetup("sql", new class extends SetupScript
                 $hasDupes = false;
                 for ($i = 1; $i < 11; $i++)
                 {
+                    if (!$s['soundFile'.$i])
+                        continue;
+
                     $nicePath = CLI::nicePath($s['soundFile'.$i], $s['path']);
                     if ($s['soundFile'.$i] && array_key_exists($nicePath, $soundIndex))
                     {
@@ -134,9 +137,6 @@ CLISetup::registerSetup("sql", new class extends SetupScript
                         CLI::write('[sound] Group '.str_pad('['.$s['id'].']', 7).' '.CLI::bold($s['name']).' has invalid sound file '.CLI::bold($s['soundFile'.$i]).' on index '.$i.'! Skipping...', CLI::LOG_WARN);
                         $s['soundFile'.$i] = null;
                     }
-                    // empty case
-                    else
-                        $s['soundFile'.$i] = null;
                 }
 
                 if (!$fileSets && !$hasDupes)