master
ブランチ master
LittleBeast / src / Std / Lib / Db / Convert / DumpSource.php
19 KB、457行 生 Blame 履歴
1<?php
2/*************************************************************
3# 076 License
4
5Copyright (c) テクニカル諏訪子
6
7Permission is hereby granted to any person obtaining a copy of the software
8Little Beast (the "Software") to use, modify, merge, copy, publish, distribute,
9sublicense, and/or sell copies of the Software, subject to the following conditions:
10
11 1. **Origin Attribution**:
12 - You must not misrepresent the origin of the Software; you must not claim
13 you created the original Software.
14 - If the Software is used in a product, you must either:
15 a. Provide clear attribution in the product's documentation, user interface,
16 or other visible areas, **OR**
17 b. Pay the original developers a fee they specify in writing.
18 2. **Usage Restriction**:
19 - The Software, or any derivative works, dependencies, or libraries
20 incorporating it, must not be used for censorship or to suppress freedom of
21 speech, expression, or creativity. Prohibited uses include, but are not
22 limited to:
23 - Censorship of so-called "hate speech", visuals, non-mainstream opinions,
24 ideas, or objective reality.
25 - Tools or systems designed to restrict access to information or
26 artistic works.
27 3. **Notice Preservation**:
28 - This license and the above copyright notice must remain intact in all copies
29 of the source code.
30
31THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
32INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
33PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
34HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
35CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
36OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
37 */
38namespace Std\Lib\Db\Convert;
39
40use Std\Lib\Db\Convert;
41use Std\Lib\Db\DbException;
42use Std\Lib\Db\Sqlite\Blob;
43
44/**
45 * SQLのダンプ(mysqldump、pg_dump(平文、COPY・INSERT)、sqlite3の.dump)から読む
46 *
47 * ファイルを2回読む(1回目はテーブルの定義、2回目は行)。CREATE TABLE、ALTER TABLE … ADD CONSTRAINT(主キー・一意)・
48 * ADD PRIMARY KEY・ADD KEY・MODIFY、CREATE INDEX、シーケンス(nextval、OWNED BY、IDENTITY)、INSERT、COPYを使い、
49 * 他の文(SET、ビュー、関数、権限等)は無視する。
50 */
51final class DumpSource implements Source {
52 public string $dialect;
53 private array $warnings = [];
54 private ?array $tables = null;
55
56 /**
57 * @param string $path ダンプのファイル(gzipで圧縮したファイルも可)
58 * @param string|null $dialect mysql、pgsql、sqlite(nullなら最初の部分から推測)
59 */
60 public function __construct(private string $path, ?string $dialect = null) {
61 if (!is_file($path) || !is_readable($path)) throw new DbException("ダンプのファイルを読めません:{$path}", 0, 'HY000');
62 $head = $this->head(65536);
63 if (str_starts_with($head, 'PGDMP')) throw new DbException("pg_dumpのカスタム・ディレクトリの形式には未対応です(プレーンの形式、-Fpで作って下さい):{$path}", 0, 'HY000');
64 if (str_starts_with($head, "SQLite format 3\0")) throw new DbException("ダンプではなく、SQLiteのデータベースのファイルです:{$path}", 0, 'HY000');
65 $this->dialect = $dialect ?? self::detect($head);
66 }
67
68 private function open() {
69 // zlibが有れば、gzopenは圧縮していないファイルもそのまま読める
70 $gz = function_exists('gzopen');
71 if (!$gz && str_ends_with(strtolower($this->path), '.gz')) throw new DbException("圧縮したダンプにはzlibが必要です:{$this->path}", 0, 'HY000');
72 $fh = $gz ? @gzopen($this->path, 'rb') : @fopen($this->path, 'rb');
73 if ($fh === false) throw new DbException("ダンプのファイルを開けません:{$this->path}", 0, 'HY000');
74 return [$fh, $gz];
75 }
76
77 private function head(int $n): string {
78 [$fh, $gz] = $this->open();
79 $s = $gz ? gzread($fh, $n) : fread($fh, $n);
80 $gz ? gzclose($fh) : fclose($fh);
81 return (string)$s;
82 }
83
84 /** ダンプの方言を推測する */
85 public static function detect(string $head): string {
86 if (preg_match('/^-- LittleBeast.*((mysql|pgsql|sqlite))/m', $head, $m)) return $m[1];
87 if (preg_match('/PostgreSQL database dump|^COPY .* FROM stdin;|SET search_path|pg_catalog\./mi', $head)) return 'pgsql';
88 if (preg_match('/MySQL dump|MariaDB dump|^\/\*!\d{5}|ENGINE=|`[^`]+`/mi', $head)) return 'mysql';
89 return 'sqlite';
90 }
91
92 public function warnings(): array {
93 return array_values(array_unique($this->warnings));
94 }
95
96 /**
97 * 文を順番に(COPYのデータはcopy)
98 *
99 * @return \Generator ['sql' => 文] 又は ['copy' => COPYの文, 'lines' => データの行の配列]
100 */
101 public function statements(): \Generator {
102 [$fh, $gz] = $this->open();
103 $read = $gz ? fn() => gzgets($fh) : fn() => fgets($fh);
104 $delim = ';';
105 $buf = '';
106 $state = '';
107 try {
108 $first = true;
109 while (($line = $read()) !== false) {
110 if ($first) {
111 // UTF-8のBOM
112 if (str_starts_with($line, "\xEF\xBB\xBF")) $line = substr($line, 3);
113 $first = false;
114 }
115 // COPY … FROM stdin;の後のデータ
116 if ($state === 'copy') {
117 if (rtrim($line, "\r\n") === '\\.') {
118 if ($copyLines) yield ['copy' => $copySql, 'lines' => $copyLines];
119 $copyLines = [];
120 $state = '';
121 continue;
122 }
123 $copyLines[] = rtrim($line, "\r\n");
124 if (count($copyLines) >= 500) {
125 yield ['copy' => $copySql, 'lines' => $copyLines];
126 $copyLines = [];
127 }
128 continue;
129 }
130 if ($state === '' && trim($buf) === '' && preg_match('/^\s*DELIMITER\s+(\S+)\s*$/i', $line, $m)) {
131 $delim = $m[1];
132 continue;
133 }
134 $i = 0;
135 $len = strlen($line);
136 while ($i < $len) {
137 if ($state === "'" || $state === '"' || $state === '`') {
138 $bs = $this->dialect === 'mysql' && $state !== '`' || ($state === "'" && !empty($estring));
139 $j = $i;
140 while (true) {
141 $k = strcspn($line, $state.($bs ? '\\' : ''), $j);
142 $j += $k;
143 if ($j >= $len) break;
144 if ($line[$j] === '\\') {
145 $j += 2;
146 continue;
147 }
148 // 「''」
149 if (($line[$j + 1] ?? '') === $state) {
150 $j += 2;
151 continue;
152 }
153 $state = '';
154 $j++;
155 break;
156 }
157 $buf .= substr($line, $i, min($j, $len) - $i);
158 $i = min($j, $len);
159 continue;
160 }
161 if ($state === '*/') {
162 $j = strpos($line, '*/', $i);
163 if ($j === false) {
164 $buf .= substr($line, $i);
165 break;
166 }
167 $buf .= substr($line, $i, $j + 2 - $i);
168 $i = $j + 2;
169 $state = '';
170 continue;
171 }
172 if (str_starts_with($state, '$')) {
173 $j = strpos($line, $state, $i);
174 if ($j === false) {
175 $buf .= substr($line, $i);
176 break;
177 }
178 $buf .= substr($line, $i, $j + strlen($state) - $i);
179 $i = $j + strlen($state);
180 $state = '';
181 continue;
182 }
183 $k = strcspn($line, "'\"`-#/\$".$delim[0], $i);
184 $buf .= substr($line, $i, $k);
185 $i += $k;
186 if ($i >= $len) break;
187 $c = $line[$i];
188 if ($c === $delim[0] && substr($line, $i, strlen($delim)) === $delim) {
189 $sql = trim($buf);
190 $buf = '';
191 $i += strlen($delim);
192 if ($sql !== '') {
193 if (preg_match('/^COPY\s.*\sFROM\s+stdin/is', $sql)) {
194 $state = 'copy';
195 $copySql = $sql;
196 $copyLines = [];
197 break;
198 }
199 yield ['sql' => $sql];
200 }
201 continue;
202 }
203 if ($c === "'" || $c === '"' || $c === '`') {
204 if ($c === '`' && $this->dialect !== 'mysql' && $this->dialect !== 'sqlite') {
205 $buf .= $c;
206 $i++;
207 continue;
208 }
209 $estring = $c === "'" && $i > 0 && ($line[$i - 1] === 'E' || $line[$i - 1] === 'e') && !ctype_alnum($line[$i - 2] ?? ' ');
210 $state = $c;
211 $buf .= $c;
212 $i++;
213 continue;
214 }
215 if ($c === '-' && ($line[$i + 1] ?? '') === '-' || ($c === '#' && $this->dialect === 'mysql')) {
216 // 行の最後までのコメント
217 break;
218 }
219 if ($c === '/' && ($line[$i + 1] ?? '') === '*') {
220 $state = '*/';
221 $buf .= '/*';
222 $i += 2;
223 continue;
224 }
225 if ($c === '$' && $this->dialect === 'pgsql' && preg_match('/\G\$([A-Za-z_][A-Za-z0-9_]*)?\$/', $line, $m, 0, $i)) {
226 $state = $m[0];
227 $buf .= $m[0];
228 $i += strlen($m[0]);
229 continue;
230 }
231 $buf .= $c;
232 $i++;
233 }
234 if ($state === '' && $buf !== '' && !str_ends_with($buf, "\n")) $buf .= "\n";
235 }
236 if ($state !== '') throw new DbException("ダンプのファイルが途中で終わっています({$state}):{$this->path}", 0, 'HY000');
237 if (trim($buf) !== '') yield ['sql' => trim($buf)];
238 } finally {
239 $gz ? gzclose($fh) : fclose($fh);
240 }
241 }
242
243 public function tables(): array {
244 if ($this->tables !== null) return array_values($this->tables);
245 $tables = [];
246 $seqOwner = [];
247 foreach ($this->statements() as $st) {
248 if (!isset($st['sql'])) continue;
249 $sql = $st['sql'];
250 // MySQLの「/*!40101 … */」の中の文
251 if (preg_match('/^\/\*!\d*\s*(.*)\*\/$/s', $sql, $m)) $sql = trim($m[1]);
252 $head = strtoupper(substr(ltrim($sql), 0, 40));
253 if (str_starts_with($head, 'CREATE') && preg_match('/^CREATE\s+(?:(?:TEMP|TEMPORARY|UNLOGGED|GLOBAL|LOCAL)\s+)*TABLE\b/i', $sql)) {
254 if (preg_match('/\bAS\s+SELECT\b/i', $sql) && !str_contains($sql, '(')) continue;
255 $t = Sql::createTable($sql, $this->dialect, $this->warnings);
256 if ($t === null) continue;
257 if (in_array(strtolower($t['name']), ['sqlite_sequence', 'sqlite_stat1', 'sqlite_stat4'], true)) continue;
258 $tables[strtolower($t['name'])] = $t;
259 } else if (str_starts_with($head, 'CREATE') && preg_match('/^CREATE\s+(UNIQUE\s+)?INDEX\b/i', $sql)) {
260 $this->createIndex($sql, $tables);
261 } else if (str_starts_with($head, 'ALTER TABLE')) {
262 $this->alterTable($sql, $tables);
263 } else if (preg_match('/^ALTER\s+SEQUENCE\s+(\S+)\s+OWNED\s+BY\s+(\S+)\s*$/i', $sql, $m)) {
264 $parts = explode('.', str_replace('"', '', $m[2]));
265 $col = array_pop($parts);
266 $tbl = array_pop($parts);
267 $seqOwner[strtolower(self::lastPart($m[1]))] = [strtolower((string)$tbl), $col];
268 } else if (preg_match('/^CREATE\s+(OR\s+REPLACE\s+)?(VIEW|TRIGGER|FUNCTION|PROCEDURE|MATERIALIZED)/i', $sql, $m)) {
269 $this->warnings[] = strtoupper($m[2]).'は変換しません';
270 }
271 }
272 // nextval('seq')のDEFAULTはalterTable()でautoincにする。OWNED BYだけの場合
273 foreach ($seqOwner as [$tbl, $col]) {
274 if (!isset($tables[$tbl])) continue;
275 foreach ($tables[$tbl]['columns'] as $i => $c) {
276 if (strcasecmp($c['name'], $col) === 0 && $c['type'] === 'int' && !empty($this->nextval[$tbl][strtolower($col)])) $tables[$tbl]['columns'][$i]['autoinc'] = true;
277 }
278 }
279 $this->tables = $tables;
280 return array_values($tables);
281 }
282
283 private array $nextval = [];
284
285 private static function lastPart(string $name): string {
286 $p = explode('.', str_replace(['"', '`'], '', $name));
287 return end($p);
288 }
289
290 private function createIndex(string $sql, array &$tables): void {
291 $tok = Sql::tokens($sql, $this->dialect);
292 $p = 1;
293 $unique = false;
294 if (($tok[$p]['u'] ?? '') === 'UNIQUE') {
295 $unique = true;
296 $p++;
297 }
298 $p++;
299 if (($tok[$p]['u'] ?? '') === 'CONCURRENTLY') $p++;
300 if (($tok[$p]['u'] ?? '') === 'IF') $p += 3;
301 $name = '';
302 if (($tok[$p]['u'] ?? '') !== 'ON') $name = Sql::name($tok, $p);
303 $p++;
304 if (($tok[$p]['u'] ?? '') === 'ONLY') $p++;
305 $table = strtolower(Sql::name($tok, $p));
306 if (($tok[$p]['u'] ?? '') === 'USING') $p += 2;
307 if (!isset($tables[$table])) return;
308 $cols = Sql::columnList($tok, $p);
309 if (in_array(null, $cols, true) || ($tok[$p]['u'] ?? '') === 'WHERE') {
310 $this->warnings[] = "式・部分のインデックスは変換しません:{$name}";
311 return;
312 }
313 $tables[$table]['indexes'][] = ['name' => $name, 'columns' => $cols, 'unique' => $unique];
314 }
315
316 private function alterTable(string $sql, array &$tables): void {
317 $tok = Sql::tokens($sql, $this->dialect);
318 $p = 2;
319 if (($tok[$p]['u'] ?? '') === 'IF') $p += 2;
320 if (($tok[$p]['u'] ?? '') === 'ONLY') $p++;
321 $table = strtolower(Sql::name($tok, $p));
322 if (!isset($tables[$table])) return;
323 $n = count($tok);
324 // 「,」で区切った複数の操作
325 $ops = [];
326 $cur = [];
327 $depth = 0;
328 for (; $p < $n; $p++) {
329 if ($tok[$p]['t'] === 'op' && $tok[$p]['v'] === '(') $depth++;
330 if ($tok[$p]['t'] === 'op' && $tok[$p]['v'] === ')') $depth--;
331 if ($depth === 0 && $tok[$p]['t'] === 'op' && $tok[$p]['v'] === ',') {
332 $ops[] = $cur;
333 $cur = [];
334 continue;
335 }
336 $cur[] = $tok[$p];
337 }
338 if ($cur) $ops[] = $cur;
339 foreach ($ops as $op) {
340 $u = array_map(fn($t) => $t['u'] ?? '', $op);
341 if (($u[0] ?? '') === 'ADD') {
342 $k = 1;
343 $cname = '';
344 if (($u[$k] ?? '') === 'CONSTRAINT') {
345 $cname = (string)($op[$k + 1]['v'] ?? '');
346 $k += 2;
347 }
348 if (($u[$k] ?? '') === 'PRIMARY') {
349 while ($k < count($op) && !($op[$k]['t'] === 'op' && $op[$k]['v'] === '(')) $k++;
350 $tables[$table]['pk'] = array_values(array_filter(Sql::columnList($op, $k)));
351 } else if (in_array($u[$k] ?? '', ['UNIQUE', 'KEY', 'INDEX'], true)) {
352 $unique = $u[$k] === 'UNIQUE';
353 $k++;
354 if (in_array($u[$k] ?? '', ['KEY', 'INDEX'], true)) $k++;
355 if (!($op[$k]['t'] === 'op' && $op[$k]['v'] === '(')) $cname = (string)$op[$k++]['v'];
356 while ($k < count($op) && !($op[$k]['t'] === 'op' && $op[$k]['v'] === '(')) $k++;
357 $cols = Sql::columnList($op, $k);
358 if (!in_array(null, $cols, true)) $tables[$table]['indexes'][] = ['name' => $cname, 'columns' => $cols, 'unique' => $unique];
359 } else if (($u[$k] ?? '') === 'FOREIGN') {
360 $this->warnings[] = "外部キーは変換しません:{$tables[$table]['name']}";
361 }
362 } else if (($u[0] ?? '') === 'ALTER') {
363 // ALTER COLUMN x SET DEFAULT nextval(…)、ADD GENERATED … AS IDENTITY
364 $k = ($u[1] ?? '') === 'COLUMN' ? 2 : 1;
365 $col = (string)($op[$k]['v'] ?? '');
366 $rest = strtoupper(Sql::text(array_slice($op, $k + 1)));
367 if (str_contains($rest, 'NEXTVAL') || str_contains($rest, 'IDENTITY')) {
368 foreach ($tables[$table]['columns'] as $i => $c) {
369 if (strcasecmp($c['name'], $col) === 0) {
370 $tables[$table]['columns'][$i]['autoinc'] = true;
371 $this->nextval[$table][strtolower($col)] = true;
372 }
373 }
374 }
375 } else if (($u[0] ?? '') === 'MODIFY' || ($u[0] ?? '') === 'CHANGE') {
376 // MODIFY [COLUMN] col 型 …(phpMyAdminのAUTO_INCREMENT)
377 $k = ($u[1] ?? '') === 'COLUMN' ? 2 : 1;
378 if ($u[0] === 'CHANGE') $k++;
379 $def = Sql::columnDef(array_slice($op, $k), $this->dialect, $table, $this->warnings);
380 if ($def === null) continue;
381 foreach ($tables[$table]['columns'] as $i => $c) {
382 if (strcasecmp($c['name'], $def['name']) === 0) {
383 unset($def['pk'], $def['unique']);
384 $tables[$table]['columns'][$i] = $def + $c;
385 }
386 }
387 }
388 }
389 }
390
391 public function rows(array $tables, callable $sink): void {
392 $defs = [];
393 foreach ($this->tables() as $t) $defs[strtolower($t['name'])] = $t;
394 $want = array_flip($tables);
395 foreach ($this->statements() as $st) {
396 if (isset($st['copy'])) {
397 if (!preg_match('/^COPY\s+(?:ONLY\s+)?(\S+)\s*(?:\(([^)]*)\))?\s+FROM/is', $st['copy'], $m)) continue;
398 $name = strtolower(self::lastPart($m[1]));
399 if (!isset($want[$name], $defs[$name])) continue;
400 $cols = isset($m[2]) && trim($m[2]) !== '' ? array_map(fn($c) => trim($c, " \t\n\"`"), explode(',', $m[2])) : null;
401 $rows = [];
402 foreach ($st['lines'] as $line) $rows[] = Sql::copyLine($line);
403 $sink($defs[$name]['name'], $this->arrange($defs[$name], $cols, $rows, true));
404 continue;
405 }
406 $sql = $st['sql'];
407 if (!preg_match('/^(INSERT|REPLACE)\b/i', $sql)) continue;
408 $ins = Sql::insert($sql, $this->dialect);
409 if ($ins === null) continue;
410 $name = strtolower($ins['table']);
411 if (!isset($want[$name], $defs[$name])) continue;
412 if (!empty($ins['select'])) {
413 $this->warnings[] = "INSERT … SELECTは変換しません:{$ins['table']}";
414 continue;
415 }
416 $sink($defs[$name]['name'], $this->arrange($defs[$name], $ins['columns'], $ins['rows'], false));
417 }
418 }
419
420 /** 列の順番にする(COPYの文字列は型に合わせる) */
421 private function arrange(array $t, ?array $cols, array $rows, bool $copy): array {
422 $map = null;
423 if ($cols !== null) {
424 $idx = [];
425 foreach ($t['columns'] as $i => $c) $idx[strtolower($c['name'])] = $i;
426 $map = [];
427 foreach ($cols as $k => $n) {
428 if (!isset($idx[strtolower($n)])) throw new DbException("ダンプの列がありません:{$t['name']}.{$n}", 0, 'HY000');
429 $map[$k] = $idx[strtolower($n)];
430 }
431 }
432 $nc = count($t['columns']);
433 $out = [];
434 foreach ($rows as $r) {
435 $row = array_fill(0, $nc, null);
436 foreach ($r as $k => $v) {
437 $i = $map === null ? $k : ($map[$k] ?? null);
438 if ($i === null || $i >= $nc) continue;
439 $c = $t['columns'][$i];
440 // PostgreSQLの配列({a,"b c"})はJSONの配列
441 if ($v !== null && !empty($c['array']) && $this->dialect === 'pgsql' && is_string($v) && str_starts_with(ltrim($v), '{') || (is_string($v) && str_starts_with($v, '[') && str_contains($v, ']={'))) {
442 $arr = \Std\Lib\Db\Pgsql::parseArray($v);
443 if ($arr !== null) $v = $arr;
444 }
445 // PostgreSQLのbytea(\x…)、COPYのbool(t、f)
446 if ($v !== null && !$v instanceof Blob && $c['type'] === 'blob') {
447 $s = (string)$v;
448 if ($this->dialect === 'pgsql' && str_starts_with($s, '\\x')) $v = new Blob((string)hex2bin(substr($s, 2)));
449 else if ($this->dialect === 'pgsql' && $copy && str_contains($s, '\\')) $v = new Blob(stripcslashes($s));
450 }
451 $row[$i] = $v;
452 }
453 $out[] = $row;
454 }
455 return $out;
456 }
457}