suwako / ShaderPlayground

master
ブランチ master
ShaderPlayground / deps / luajit / include / lj_jit.h
15 KB、419行 生 Blame 履歴
1/*
2** Common definitions for the JIT compiler.
3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6#ifndef _LJ_JIT_H
7#define _LJ_JIT_H
8
9#include "lj_obj.h"
10#include "lj_ir.h"
11
12/* JIT engine flags. */
13#define JIT_F_ON 0x00000001
14
15/* CPU-specific JIT engine flags. */
16#if LJ_TARGET_X86ORX64
17#define JIT_F_CMOV 0x00000010
18#define JIT_F_SSE2 0x00000020
19#define JIT_F_SSE3 0x00000040
20#define JIT_F_SSE4_1 0x00000080
21#define JIT_F_P4 0x00000100
22#define JIT_F_PREFER_IMUL 0x00000200
23#define JIT_F_SPLIT_XMM 0x00000400
24#define JIT_F_LEA_AGU 0x00000800
25
26/* Names for the CPU-specific flags. Must match the order above. */
27#define JIT_F_CPU_FIRST JIT_F_CMOV
28#define JIT_F_CPUSTRING "\4CMOV\4SSE2\4SSE3\6SSE4.1\2P4\3AMD\2K8\4ATOM"
29#elif LJ_TARGET_ARM
30#define JIT_F_ARMV6_ 0x00000010
31#define JIT_F_ARMV6T2_ 0x00000020
32#define JIT_F_ARMV7 0x00000040
33#define JIT_F_VFPV2 0x00000080
34#define JIT_F_VFPV3 0x00000100
35
36#define JIT_F_ARMV6 (JIT_F_ARMV6_|JIT_F_ARMV6T2_|JIT_F_ARMV7)
37#define JIT_F_ARMV6T2 (JIT_F_ARMV6T2_|JIT_F_ARMV7)
38#define JIT_F_VFP (JIT_F_VFPV2|JIT_F_VFPV3)
39
40/* Names for the CPU-specific flags. Must match the order above. */
41#define JIT_F_CPU_FIRST JIT_F_ARMV6_
42#define JIT_F_CPUSTRING "\5ARMv6\7ARMv6T2\5ARMv7\5VFPv2\5VFPv3"
43#elif LJ_TARGET_PPC
44#define JIT_F_SQRT 0x00000010
45#define JIT_F_ROUND 0x00000020
46
47/* Names for the CPU-specific flags. Must match the order above. */
48#define JIT_F_CPU_FIRST JIT_F_SQRT
49#define JIT_F_CPUSTRING "\4SQRT\5ROUND"
50#elif LJ_TARGET_MIPS
51#define JIT_F_MIPS32R2 0x00000010
52
53/* Names for the CPU-specific flags. Must match the order above. */
54#define JIT_F_CPU_FIRST JIT_F_MIPS32R2
55#define JIT_F_CPUSTRING "\010MIPS32R2"
56#else
57#define JIT_F_CPU_FIRST 0
58#define JIT_F_CPUSTRING ""
59#endif
60
61/* Optimization flags. */
62#define JIT_F_OPT_MASK 0x0fff0000
63
64#define JIT_F_OPT_FOLD 0x00010000
65#define JIT_F_OPT_CSE 0x00020000
66#define JIT_F_OPT_DCE 0x00040000
67#define JIT_F_OPT_FWD 0x00080000
68#define JIT_F_OPT_DSE 0x00100000
69#define JIT_F_OPT_NARROW 0x00200000
70#define JIT_F_OPT_LOOP 0x00400000
71#define JIT_F_OPT_ABC 0x00800000
72#define JIT_F_OPT_SINK 0x01000000
73#define JIT_F_OPT_FUSE 0x02000000
74
75/* Optimizations names for -O. Must match the order above. */
76#define JIT_F_OPT_FIRST JIT_F_OPT_FOLD
77#define JIT_F_OPTSTRING \
78 "\4fold\3cse\3dce\3fwd\3dse\6narrow\4loop\3abc\4sink\4fuse"
79
80/* Optimization levels set a fixed combination of flags. */
81#define JIT_F_OPT_0 0
82#define JIT_F_OPT_1 (JIT_F_OPT_FOLD|JIT_F_OPT_CSE|JIT_F_OPT_DCE)
83#define JIT_F_OPT_2 (JIT_F_OPT_1|JIT_F_OPT_NARROW|JIT_F_OPT_LOOP)
84#define JIT_F_OPT_3 (JIT_F_OPT_2|\
85 JIT_F_OPT_FWD|JIT_F_OPT_DSE|JIT_F_OPT_ABC|JIT_F_OPT_SINK|JIT_F_OPT_FUSE)
86#define JIT_F_OPT_DEFAULT JIT_F_OPT_3
87
88#if LJ_TARGET_WINDOWS || LJ_64
89/* See: https://devblogs.microsoft.com/oldnewthing/20031008-00/?p=42223 */
90#define JIT_P_sizemcode_DEFAULT 64
91#else
92/* Could go as low as 4K, but the mmap() overhead would be rather high. */
93#define JIT_P_sizemcode_DEFAULT 32
94#endif
95
96/* Optimization parameters and their defaults. Length is a char in octal! */
97#define JIT_PARAMDEF(_) \
98 _(\010, maxtrace, 1000) /* Max. # of traces in cache. */ \
99 _(\011, maxrecord, 4000) /* Max. # of recorded IR instructions. */ \
100 _(\012, maxirconst, 500) /* Max. # of IR constants of a trace. */ \
101 _(\007, maxside, 100) /* Max. # of side traces of a root trace. */ \
102 _(\007, maxsnap, 500) /* Max. # of snapshots for a trace. */ \
103 \
104 _(\007, hotloop, 56) /* # of iter. to detect a hot loop/call. */ \
105 _(\007, hotexit, 10) /* # of taken exits to start a side trace. */ \
106 _(\007, tryside, 4) /* # of attempts to compile a side trace. */ \
107 \
108 _(\012, instunroll, 4) /* Max. unroll for instable loops. */ \
109 _(\012, loopunroll, 15) /* Max. unroll for loop ops in side traces. */ \
110 _(\012, callunroll, 3) /* Max. unroll for recursive calls. */ \
111 _(\011, recunroll, 2) /* Min. unroll for true recursion. */ \
112 \
113 /* Size of each machine code area (in KBytes). */ \
114 _(\011, sizemcode, JIT_P_sizemcode_DEFAULT) \
115 /* Max. total size of all machine code areas (in KBytes). */ \
116 _(\010, maxmcode, 512) \
117 /* End of list. */
118
119enum {
120#define JIT_PARAMENUM(len, name, value) JIT_P_##name,
121JIT_PARAMDEF(JIT_PARAMENUM)
122#undef JIT_PARAMENUM
123 JIT_P__MAX
124};
125
126#define JIT_PARAMSTR(len, name, value) #len #name
127#define JIT_P_STRING JIT_PARAMDEF(JIT_PARAMSTR)
128
129/* Trace compiler state. */
130typedef enum {
131 LJ_TRACE_IDLE, /* Trace compiler idle. */
132 LJ_TRACE_ACTIVE = 0x10,
133 LJ_TRACE_RECORD, /* Bytecode recording active. */
134 LJ_TRACE_START, /* New trace started. */
135 LJ_TRACE_END, /* End of trace. */
136 LJ_TRACE_ASM, /* Assemble trace. */
137 LJ_TRACE_ERR /* Trace aborted with error. */
138} TraceState;
139
140/* Post-processing action. */
141typedef enum {
142 LJ_POST_NONE, /* No action. */
143 LJ_POST_FIXCOMP, /* Fixup comparison and emit pending guard. */
144 LJ_POST_FIXGUARD, /* Fixup and emit pending guard. */
145 LJ_POST_FIXGUARDSNAP, /* Fixup and emit pending guard and snapshot. */
146 LJ_POST_FIXBOOL, /* Fixup boolean result. */
147 LJ_POST_FIXCONST, /* Fixup constant results. */
148 LJ_POST_FFRETRY /* Suppress recording of retried fast functions. */
149} PostProc;
150
151/* Machine code type. */
152#if LJ_TARGET_X86ORX64
153typedef uint8_t MCode;
154#else
155typedef uint32_t MCode;
156#endif
157
158/* Linked list of MCode areas. */
159typedef struct MCLink {
160 MCode *next; /* Next area. */
161 size_t size; /* Size of current area. */
162} MCLink;
163
164/* Stack snapshot header. */
165typedef struct SnapShot {
166 uint32_t mapofs; /* Offset into snapshot map. */
167 IRRef1 ref; /* First IR ref for this snapshot. */
168 uint8_t nslots; /* Number of valid slots. */
169 uint8_t topslot; /* Maximum frame extent. */
170 uint8_t nent; /* Number of compressed entries. */
171 uint8_t count; /* Count of taken exits for this snapshot. */
172} SnapShot;
173
174#define SNAPCOUNT_DONE 255 /* Already compiled and linked a side trace. */
175
176/* Compressed snapshot entry. */
177typedef uint32_t SnapEntry;
178
179#define SNAP_FRAME 0x010000 /* Frame slot. */
180#define SNAP_CONT 0x020000 /* Continuation slot. */
181#define SNAP_NORESTORE 0x040000 /* No need to restore slot. */
182#define SNAP_SOFTFPNUM 0x080000 /* Soft-float number. */
183LJ_STATIC_ASSERT(SNAP_FRAME == TREF_FRAME);
184LJ_STATIC_ASSERT(SNAP_CONT == TREF_CONT);
185
186#define SNAP(slot, flags, ref) (((SnapEntry)(slot) << 24) + (flags) + (ref))
187#define SNAP_TR(slot, tr) \
188 (((SnapEntry)(slot) << 24) + ((tr) & (TREF_CONT|TREF_FRAME|TREF_REFMASK)))
189#define SNAP_MKPC(pc) ((SnapEntry)u32ptr(pc))
190#define SNAP_MKFTSZ(ftsz) ((SnapEntry)(ftsz))
191#define snap_ref(sn) ((sn) & 0xffff)
192#define snap_slot(sn) ((BCReg)((sn) >> 24))
193#define snap_isframe(sn) ((sn) & SNAP_FRAME)
194#define snap_pc(sn) ((const BCIns *)(uintptr_t)(sn))
195#define snap_setref(sn, ref) (((sn) & (0xffff0000&~SNAP_NORESTORE)) | (ref))
196
197/* Snapshot and exit numbers. */
198typedef uint32_t SnapNo;
199typedef uint32_t ExitNo;
200
201/* Trace number. */
202typedef uint32_t TraceNo; /* Used to pass around trace numbers. */
203typedef uint16_t TraceNo1; /* Stored trace number. */
204
205/* Type of link. ORDER LJ_TRLINK */
206typedef enum {
207 LJ_TRLINK_NONE, /* Incomplete trace. No link, yet. */
208 LJ_TRLINK_ROOT, /* Link to other root trace. */
209 LJ_TRLINK_LOOP, /* Loop to same trace. */
210 LJ_TRLINK_TAILREC, /* Tail-recursion. */
211 LJ_TRLINK_UPREC, /* Up-recursion. */
212 LJ_TRLINK_DOWNREC, /* Down-recursion. */
213 LJ_TRLINK_INTERP, /* Fallback to interpreter. */
214 LJ_TRLINK_RETURN /* Return to interpreter. */
215} TraceLink;
216
217/* Trace object. */
218typedef struct GCtrace {
219 GCHeader;
220 uint16_t nsnap; /* Number of snapshots. */
221 IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */
222 GCRef gclist;
223 IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
224 IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */
225 uint32_t nsnapmap; /* Number of snapshot map elements. */
226 SnapShot *snap; /* Snapshot array. */
227 SnapEntry *snapmap; /* Snapshot map. */
228 GCRef startpt; /* Starting prototype. */
229 MRef startpc; /* Bytecode PC of starting instruction. */
230 BCIns startins; /* Original bytecode of starting instruction. */
231 MSize szmcode; /* Size of machine code. */
232 MCode *mcode; /* Start of machine code. */
233 MSize mcloop; /* Offset of loop start in machine code. */
234 uint16_t nchild; /* Number of child traces (root trace only). */
235 uint16_t spadjust; /* Stack pointer adjustment (offset in bytes). */
236 TraceNo1 traceno; /* Trace number. */
237 TraceNo1 link; /* Linked trace (or self for loops). */
238 TraceNo1 root; /* Root trace of side trace (or 0 for root traces). */
239 TraceNo1 nextroot; /* Next root trace for same prototype. */
240 TraceNo1 nextside; /* Next side trace of same root trace. */
241 uint8_t sinktags; /* Trace has SINK tags. */
242 uint8_t topslot; /* Top stack slot already checked to be allocated. */
243 uint8_t linktype; /* Type of link. */
244 uint8_t unused1;
245#ifdef LUAJIT_USE_GDBJIT
246 void *gdbjit_entry; /* GDB JIT entry. */
247#endif
248} GCtrace;
249
250#define gco2trace(o) check_exp((o)->gch.gct == ~LJ_TTRACE, (GCtrace *)(o))
251#define traceref(J, n) \
252 check_exp((n)>0 && (MSize)(n)<J->sizetrace, (GCtrace *)gcref(J->trace[(n)]))
253
254LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtrace, gclist));
255
256static LJ_AINLINE MSize snap_nextofs(GCtrace *T, SnapShot *snap)
257{
258 if (snap+1 == &T->snap[T->nsnap])
259 return T->nsnapmap;
260 else
261 return (snap+1)->mapofs;
262}
263
264/* Round-robin penalty cache for bytecodes leading to aborted traces. */
265typedef struct HotPenalty {
266 MRef pc; /* Starting bytecode PC. */
267 uint16_t val; /* Penalty value, i.e. hotcount start. */
268 uint16_t reason; /* Abort reason (really TraceErr). */
269} HotPenalty;
270
271#define PENALTY_SLOTS 64 /* Penalty cache slot. Must be a power of 2. */
272#define PENALTY_MIN (36*2) /* Minimum penalty value. */
273#define PENALTY_MAX 60000 /* Maximum penalty value. */
274#define PENALTY_RNDBITS 4 /* # of random bits to add to penalty value. */
275
276/* Round-robin backpropagation cache for narrowing conversions. */
277typedef struct BPropEntry {
278 IRRef1 key; /* Key: original reference. */
279 IRRef1 val; /* Value: reference after conversion. */
280 IRRef mode; /* Mode for this entry (currently IRCONV_*). */
281} BPropEntry;
282
283/* Number of slots for the backpropagation cache. Must be a power of 2. */
284#define BPROP_SLOTS 16
285
286/* Scalar evolution analysis cache. */
287typedef struct ScEvEntry {
288 MRef pc; /* Bytecode PC of FORI. */
289 IRRef1 idx; /* Index reference. */
290 IRRef1 start; /* Constant start reference. */
291 IRRef1 stop; /* Constant stop reference. */
292 IRRef1 step; /* Constant step reference. */
293 IRType1 t; /* Scalar type. */
294 uint8_t dir; /* Direction. 1: +, 0: -. */
295} ScEvEntry;
296
297/* 128 bit SIMD constants. */
298enum {
299 LJ_KSIMD_ABS,
300 LJ_KSIMD_NEG,
301 LJ_KSIMD__MAX
302};
303
304/* Get 16 byte aligned pointer to SIMD constant. */
305#define LJ_KSIMD(J, n) \
306 ((TValue *)(((intptr_t)&J->ksimd[2*(n)] + 15) & ~(intptr_t)15))
307
308/* Set/reset flag to activate the SPLIT pass for the current trace. */
309#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
310#define lj_needsplit(J) (J->needsplit = 1)
311#define lj_resetsplit(J) (J->needsplit = 0)
312#else
313#define lj_needsplit(J) UNUSED(J)
314#define lj_resetsplit(J) UNUSED(J)
315#endif
316
317/* Fold state is used to fold instructions on-the-fly. */
318typedef struct FoldState {
319 IRIns ins; /* Currently emitted instruction. */
320 IRIns left; /* Instruction referenced by left operand. */
321 IRIns right; /* Instruction referenced by right operand. */
322} FoldState;
323
324/* JIT compiler state. */
325typedef struct jit_State {
326 GCtrace cur; /* Current trace. */
327
328 lua_State *L; /* Current Lua state. */
329 const BCIns *pc; /* Current PC. */
330 GCfunc *fn; /* Current function. */
331 GCproto *pt; /* Current prototype. */
332 TRef *base; /* Current frame base, points into J->slots. */
333
334 uint32_t flags; /* JIT engine flags. */
335 BCReg maxslot; /* Relative to baseslot. */
336 BCReg baseslot; /* Current frame base, offset into J->slots. */
337
338 uint8_t mergesnap; /* Allowed to merge with next snapshot. */
339 uint8_t needsnap; /* Need snapshot before recording next bytecode. */
340 IRType1 guardemit; /* Accumulated IRT_GUARD for emitted instructions. */
341 uint8_t bcskip; /* Number of bytecode instructions to skip. */
342
343 FoldState fold; /* Fold state. */
344
345 const BCIns *bc_min; /* Start of allowed bytecode range for root trace. */
346 MSize bc_extent; /* Extent of the range. */
347
348 TraceState state; /* Trace compiler state. */
349
350 int32_t instunroll; /* Unroll counter for instable loops. */
351 int32_t loopunroll; /* Unroll counter for loop ops in side traces. */
352 int32_t tailcalled; /* Number of successive tailcalls. */
353 int32_t framedepth; /* Current frame depth. */
354 int32_t retdepth; /* Return frame depth (count of RETF). */
355
356 MRef k64; /* Pointer to chained array of 64 bit constants. */
357 TValue ksimd[LJ_KSIMD__MAX*2+1]; /* 16 byte aligned SIMD constants. */
358
359 IRIns *irbuf; /* Temp. IR instruction buffer. Biased with REF_BIAS. */
360 IRRef irtoplim; /* Upper limit of instuction buffer (biased). */
361 IRRef irbotlim; /* Lower limit of instuction buffer (biased). */
362 IRRef loopref; /* Last loop reference or ref of final LOOP (or 0). */
363
364 MSize sizesnap; /* Size of temp. snapshot buffer. */
365 SnapShot *snapbuf; /* Temp. snapshot buffer. */
366 SnapEntry *snapmapbuf; /* Temp. snapshot map buffer. */
367 MSize sizesnapmap; /* Size of temp. snapshot map buffer. */
368
369 PostProc postproc; /* Required post-processing after execution. */
370#if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
371 int needsplit; /* Need SPLIT pass. */
372#endif
373
374 GCRef *trace; /* Array of traces. */
375 TraceNo freetrace; /* Start of scan for next free trace. */
376 MSize sizetrace; /* Size of trace array. */
377
378 IRRef1 chain[IR__MAX]; /* IR instruction skip-list chain anchors. */
379 TRef slot[LJ_MAX_JSLOTS+LJ_STACK_EXTRA]; /* Stack slot map. */
380
381 int32_t param[JIT_P__MAX]; /* JIT engine parameters. */
382
383 MCode *exitstubgroup[LJ_MAX_EXITSTUBGR]; /* Exit stub group addresses. */
384
385 HotPenalty penalty[PENALTY_SLOTS]; /* Penalty slots. */
386 uint32_t penaltyslot; /* Round-robin index into penalty slots. */
387 uint32_t prngstate; /* PRNG state. */
388
389 BPropEntry bpropcache[BPROP_SLOTS]; /* Backpropagation cache slots. */
390 uint32_t bpropslot; /* Round-robin index into bpropcache slots. */
391
392 ScEvEntry scev; /* Scalar evolution analysis cache slots. */
393
394 const BCIns *startpc; /* Bytecode PC of starting instruction. */
395 TraceNo parent; /* Parent of current side trace (0 for root traces). */
396 ExitNo exitno; /* Exit number in parent of current side trace. */
397
398 BCIns *patchpc; /* PC for pending re-patch. */
399 BCIns patchins; /* Instruction for pending re-patch. */
400
401 int mcprot; /* Protection of current mcode area. */
402 MCode *mcarea; /* Base of current mcode area. */
403 MCode *mctop; /* Top of current mcode area. */
404 MCode *mcbot; /* Bottom of current mcode area. */
405 size_t szmcarea; /* Size of current mcode area. */
406 size_t szallmcarea; /* Total size of all allocated mcode areas. */
407
408 TValue errinfo; /* Additional info element for trace errors. */
409} jit_State;
410
411/* Trivial PRNG e.g. used for penalty randomization. */
412static LJ_AINLINE uint32_t LJ_PRNG_BITS(jit_State *J, int bits)
413{
414 /* Yes, this LCG is very weak, but that doesn't matter for our use case. */
415 J->prngstate = J->prngstate * 1103515245 + 12345;
416 return J->prngstate >> (32-bits);
417}
418
419#endif