| 1 | <?php |
| 2 | /************************************************************* |
| 3 | # 076 License |
| 4 | |
| 5 | Copyright (c) テクニカル諏訪子 |
| 6 | |
| 7 | Permission is hereby granted to any person obtaining a copy of the software |
| 8 | Little Beast (the "Software") to use, modify, merge, copy, publish, distribute, |
| 9 | sublicense, and/or sell copies of the Software, subject to the following conditions: |
| 10 | |
| 11 | 1. **Origin Attribution**: |
| 12 | - You must not misrepresent the origin of the Software; you must not claim |
| 13 | you created the original Software. |
| 14 | - If the Software is used in a product, you must either: |
| 15 | a. Provide clear attribution in the product's documentation, user interface, |
| 16 | or other visible areas, **OR** |
| 17 | b. Pay the original developers a fee they specify in writing. |
| 18 | 2. **Usage Restriction**: |
| 19 | - The Software, or any derivative works, dependencies, or libraries |
| 20 | incorporating it, must not be used for censorship or to suppress freedom of |
| 21 | speech, expression, or creativity. Prohibited uses include, but are not |
| 22 | limited to: |
| 23 | - Censorship of so-called "hate speech", visuals, non-mainstream opinions, |
| 24 | ideas, or objective reality. |
| 25 | - Tools or systems designed to restrict access to information or |
| 26 | artistic works. |
| 27 | 3. **Notice Preservation**: |
| 28 | - This license and the above copyright notice must remain intact in all copies |
| 29 | of the source code. |
| 30 | |
| 31 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, |
| 32 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A |
| 33 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT |
| 34 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF |
| 35 | CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE |
| 36 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 37 | */ |
| 38 | namespace Std\Lib\Db\Convert; |
| 39 | |
| 40 | use Std\Lib\Db\DbException; |
| 41 | use Std\Lib\Db\Sqlite\Blob; |
| 42 | |
| 43 | /** |
| 44 | * LittleBeastのJSONのデータ（data/）から読む（Storesを参照） |
| 45 | * |
| 46 | * データのディレクトリが無い・空なら、テーブルも無い。SQLiteのファイル（data/site.sqlite等）は飛ばす。 |
| 47 | */ |
| 48 | final class JsonSource implements Source { |
| 49 | private array $warnings = []; |
| 50 | private ?array $files = null; |
| 51 | private ?array $data = null; |
| 52 | |
| 53 | /** @param string $dir データのディレクトリ（普通はROOT/data） */ |
| 54 | public function __construct(private string $dir) { |
| 55 | $this->dir = rtrim($dir, '/\\'); |
| 56 | } |
| 57 | |
| 58 | public function warnings(): array { |
| 59 | return array_values(array_unique($this->warnings)); |
| 60 | } |
| 61 | |
| 62 | /** データのファイル（相対パス => 絶対パス） */ |
| 63 | private function files(): array { |
| 64 | if ($this->files !== null) return $this->files; |
| 65 | $out = []; |
| 66 | if (is_dir($this->dir)) { |
| 67 | $it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($this->dir, \FilesystemIterator::SKIP_DOTS)); |
| 68 | foreach ($it as $f) { |
| 69 | if (!$f->isFile()) continue; |
| 70 | $rel = str_replace('\\', '/', substr($f->getPathname(), strlen($this->dir) + 1)); |
| 71 | $out[$rel] = $f->getPathname(); |
| 72 | } |
| 73 | } |
| 74 | ksort($out, SORT_STRING); |
| 75 | return $this->files = $out; |
| 76 | } |
| 77 | |
| 78 | /** 全部の行を作る（データは小さいので、メモリーに） */ |
| 79 | private function load(): array { |
| 80 | if ($this->data !== null) return $this->data; |
| 81 | $d = array_fill_keys(array_keys(Stores::tables()), []); |
| 82 | $generic = []; |
| 83 | foreach ($this->files() as $rel => $abs) { |
| 84 | foreach (Stores::SKIP as $re) { |
| 85 | if (preg_match($re, $rel)) { |
| 86 | if (str_starts_with($rel, 'db/') && str_ends_with($rel, '.json')) $generic[$rel] = $abs; |
| 87 | continue 2; |
| 88 | } |
| 89 | } |
| 90 | $text = (string)@file_get_contents($abs); |
| 91 | // SQLiteのデータベース（例：data/site.sqlite）とジャーナル |
| 92 | if (str_starts_with($text, "SQLite format 3\0") || preg_match('/-(journal|wal|shm)$/', $rel)) { |
| 93 | $this->warnings[] = "SQLiteのファイルは変換しません：{$rel}"; |
| 94 | continue; |
| 95 | } |
| 96 | if (!$this->store($rel, $text, $d)) $d['lb_files'][] = [$rel, new Blob($text)]; |
| 97 | } |
| 98 | $this->data = ['stores' => $d, 'generic' => $generic]; |
| 99 | return $this->data; |
| 100 | } |
| 101 | |
| 102 | /** |
| 103 | * ファイルを決まった形のテーブルにする |
| 104 | * |
| 105 | * @return bool falseならlb_files |
| 106 | */ |
| 107 | private function store(string $rel, string $text, array &$d): bool { |
| 108 | $isJson = str_ends_with($rel, '.json'); |
| 109 | $json = $isJson ? json_decode($text, true) : null; |
| 110 | if ($isJson && json_last_error() !== JSON_ERROR_NONE) return false; |
| 111 | if (preg_match('#^user/(\d+)\.([A-Za-z0-9]+)\.json$#', $rel, $m)) { |
| 112 | if (!is_array($json)) return false; |
| 113 | $role = $json['role'] ?? null; |
| 114 | $d['lb_users'][] = [(int)$m[1], $m[2], isset($json['email']) && is_string($json['email']) ? $json['email'] : null, is_int($role) ? $role : null, $text]; |
| 115 | return true; |
| 116 | } |
| 117 | if ($rel === 'comments/ap.json') { |
| 118 | if (!is_array($json)) return false; |
| 119 | foreach ($json as $k => $v) $d['lb_comment_ap'][] = [(string)$k, self::encode($v)]; |
| 120 | return true; |
| 121 | } |
| 122 | if (preg_match('#^comments/(.+)\.json$#', $rel, $m) && Stores::safeThread($m[1])) { |
| 123 | if (!is_array($json) || (isset($json['comments']) && !is_array($json['comments']))) return false; |
| 124 | $comments = $json['comments'] ?? []; |
| 125 | unset($json['comments']); |
| 126 | $d['lb_comment_threads'][] = [$m[1], self::encode((object)$json)]; |
| 127 | $seq = 0; |
| 128 | foreach ($comments as $id => $c) $d['lb_comments'][] = [$m[1], (string)$id, $seq++, self::encode($c)]; |
| 129 | return true; |
| 130 | } |
| 131 | if (preg_match('#^notifications/([^/]+)\.json$#', $rel, $m) && Stores::safeName($m[1])) { |
| 132 | if (!is_array($json) || !array_is_list($json)) return false; |
| 133 | foreach ($json as $i => $n) $d['lb_notifications'][] = [$m[1], $i, self::encode($n)]; |
| 134 | return true; |
| 135 | } |
| 136 | if ($rel === 'cron/cron.json') { |
| 137 | if (!is_array($json) || !array_is_list($json)) return false; |
| 138 | foreach ($json as $i => $j) $d['lb_cron'][] = [$i, self::encode($j)]; |
| 139 | return true; |
| 140 | } |
| 141 | if ($rel === 'followers.txt' || $rel === 'following.txt') { |
| 142 | $t = $rel === 'followers.txt' ? 'lb_followers' : 'lb_following'; |
| 143 | $lines = array_values(array_filter(array_map('trim', explode("\n", $text)), fn($l) => $l !== '')); |
| 144 | foreach ($lines as $i => $l) $d[$t][] = [$i, $l]; |
| 145 | return true; |
| 146 | } |
| 147 | if (preg_match('#^ap/notes/([^/]+)\.json$#', $rel, $m) && Stores::safeName($m[1])) { |
| 148 | if (!is_array($json)) return false; |
| 149 | $d['lb_ap_notes'][] = [$m[1], $text]; |
| 150 | return true; |
| 151 | } |
| 152 | return false; |
| 153 | } |
| 154 | |
| 155 | public static function encode(mixed $v): string { |
| 156 | return (string)json_encode($v, Stores::JSON_FLAGS); |
| 157 | } |
| 158 | |
| 159 | public function tables(): array { |
| 160 | $data = $this->load(); |
| 161 | $out = []; |
| 162 | foreach (Stores::tables() as $name => $t) { |
| 163 | if (!empty($data['stores'][$name])) $out[] = $t; |
| 164 | } |
| 165 | foreach ($data['generic'] as $rel => $abs) { |
| 166 | $g = $this->generic($abs); |
| 167 | if ($g === null) { |
| 168 | $this->warnings[] = "読めないテーブルのファイル：{$rel}"; |
| 169 | continue; |
| 170 | } |
| 171 | $out[] = $g['table']; |
| 172 | } |
| 173 | return $out; |
| 174 | } |
| 175 | |
| 176 | /** data/db/{テーブル}.json */ |
| 177 | private function generic(string $abs): ?array { |
| 178 | static $cache = []; |
| 179 | if (isset($cache[$abs])) return $cache[$abs]; |
| 180 | $j = json_decode((string)@file_get_contents($abs), true); |
| 181 | if (!is_array($j) || !isset($j['table'], $j['columns']) || !is_array($j['columns'])) return null; |
| 182 | $t = ['name' => (string)$j['table'], 'columns' => $j['columns'], 'pk' => $j['pk'] ?? [], 'indexes' => $j['indexes'] ?? []]; |
| 183 | foreach ($t['columns'] as $i => $c) { |
| 184 | if (isset($c['default']['value']) && is_array($c['default']['value'])) $t['columns'][$i]['default']['value'] = self::decodeValue($c['default']['value']); |
| 185 | } |
| 186 | return $cache[$abs] = ['table' => $t, 'rows' => $j['rows'] ?? []]; |
| 187 | } |
| 188 | |
| 189 | /** JSONの値（{"$blob": base64}、{"$float": "NaN"}） */ |
| 190 | public static function decodeValue(mixed $v): mixed { |
| 191 | if (is_array($v)) { |
| 192 | if (isset($v['$blob'])) return new Blob((string)base64_decode((string)$v['$blob'])); |
| 193 | if (isset($v['$float'])) return match ($v['$float']) { 'NaN' => NAN, 'Infinity' => INF, default => -INF }; |
| 194 | return json_encode($v, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); |
| 195 | } |
| 196 | return $v; |
| 197 | } |
| 198 | |
| 199 | public function maxLengths(array $t): array { |
| 200 | $out = []; |
| 201 | $this->rows([strtolower($t['name'])], function(string $name, array $rows) use ($t, &$out): void { |
| 202 | foreach ($t['columns'] as $i => $c) { |
| 203 | foreach ($rows as $r) { |
| 204 | $v = $r[$i] ?? null; |
| 205 | $len = $v instanceof Blob ? strlen($v->data) : (is_string($v) ? mb_strlen($v, 'UTF-8') : 0); |
| 206 | $out[$c['name']] = max($out[$c['name']] ?? 0, $len); |
| 207 | } |
| 208 | } |
| 209 | }); |
| 210 | return $out; |
| 211 | } |
| 212 | |
| 213 | public function rows(array $tables, callable $sink): void { |
| 214 | $data = $this->load(); |
| 215 | foreach ($data['stores'] as $name => $rows) { |
| 216 | if (!in_array($name, $tables, true) || empty($rows)) continue; |
| 217 | foreach (array_chunk($rows, 500) as $chunk) $sink($name, $chunk); |
| 218 | } |
| 219 | foreach ($data['generic'] as $abs) { |
| 220 | $g = $this->generic($abs); |
| 221 | if ($g === null || !in_array(strtolower($g['table']['name']), $tables, true)) continue; |
| 222 | foreach (array_chunk($g['rows'], 500) as $chunk) { |
| 223 | $sink($g['table']['name'], array_map(fn($r) => array_map([self::class, 'decodeValue'], (array)$r), $chunk)); |
| 224 | } |
| 225 | } |
| 226 | } |
| 227 | } |