suwako / ShaderPlayground

master
ブランチ master
ShaderPlayground / deps / luajit / include / lj_asm_ppc.h
69.4 KB、2175行 生 Blame 履歴
1/*
2** PPC IR assembler (SSA IR -> machine code).
3** Copyright (C) 2005-2026 Mike Pall. See Copyright Notice in luajit.h
4*/
5
6/* -- Register allocator extensions --------------------------------------- */
7
8/* Allocate a register with a hint. */
9static Reg ra_hintalloc(ASMState *as, IRRef ref, Reg hint, RegSet allow)
10{
11 Reg r = IR(ref)->r;
12 if (ra_noreg(r)) {
13 if (!ra_hashint(r) && !iscrossref(as, ref))
14 ra_sethint(IR(ref)->r, hint); /* Propagate register hint. */
15 r = ra_allocref(as, ref, allow);
16 }
17 ra_noweak(as, r);
18 return r;
19}
20
21/* Allocate two source registers for three-operand instructions. */
22static Reg ra_alloc2(ASMState *as, IRIns *ir, RegSet allow)
23{
24 IRIns *irl = IR(ir->op1), *irr = IR(ir->op2);
25 Reg left = irl->r, right = irr->r;
26 if (ra_hasreg(left)) {
27 ra_noweak(as, left);
28 if (ra_noreg(right))
29 right = ra_allocref(as, ir->op2, rset_exclude(allow, left));
30 else
31 ra_noweak(as, right);
32 } else if (ra_hasreg(right)) {
33 ra_noweak(as, right);
34 left = ra_allocref(as, ir->op1, rset_exclude(allow, right));
35 } else if (ra_hashint(right)) {
36 right = ra_allocref(as, ir->op2, allow);
37 left = ra_alloc1(as, ir->op1, rset_exclude(allow, right));
38 } else {
39 left = ra_allocref(as, ir->op1, allow);
40 right = ra_alloc1(as, ir->op2, rset_exclude(allow, left));
41 }
42 return left | (right << 8);
43}
44
45/* -- Guard handling ------------------------------------------------------ */
46
47/* Setup exit stubs after the end of each trace. */
48static void asm_exitstub_setup(ASMState *as, ExitNo nexits)
49{
50 ExitNo i;
51 MCode *mxp = as->mctop;
52 if (mxp - (nexits + 3 + MCLIM_REDZONE) < as->mclim)
53 asm_mclimit(as);
54 /* 1: mflr r0; bl ->vm_exit_handler; li r0, traceno; bl <1; bl <1; ... */
55 for (i = nexits-1; (int32_t)i >= 0; i--)
56 *--mxp = PPCI_BL|(((-3-i)&0x00ffffffu)<<2);
57 *--mxp = PPCI_LI|PPCF_T(RID_TMP)|as->T->traceno; /* Read by exit handler. */
58 mxp--;
59 *mxp = PPCI_BL|((((MCode *)(void *)lj_vm_exit_handler-mxp)&0x00ffffffu)<<2);
60 *--mxp = PPCI_MFLR|PPCF_T(RID_TMP);
61 as->mctop = mxp;
62}
63
64static MCode *asm_exitstub_addr(ASMState *as, ExitNo exitno)
65{
66 /* Keep this in-sync with exitstub_trace_addr(). */
67 return as->mctop + exitno + 3;
68}
69
70/* Emit conditional branch to exit for guard. */
71static void asm_guardcc(ASMState *as, PPCCC cc)
72{
73 MCode *target = asm_exitstub_addr(as, as->snapno);
74 MCode *p = as->mcp;
75 if (LJ_UNLIKELY(p == as->invmcp)) {
76 as->loopinv = 1;
77 *p = PPCI_B | (((target-p) & 0x00ffffffu) << 2);
78 emit_condbranch(as, PPCI_BC, cc^4, p);
79 return;
80 }
81 emit_condbranch(as, PPCI_BC, cc, target);
82}
83
84/* -- Operand fusion ------------------------------------------------------ */
85
86/* Limit linear search to this distance. Avoids O(n^2) behavior. */
87#define CONFLICT_SEARCH_LIM 31
88
89/* Check if there's no conflicting instruction between curins and ref. */
90static int noconflict(ASMState *as, IRRef ref, IROp conflict)
91{
92 IRIns *ir = as->ir;
93 IRRef i = as->curins;
94 if (i > ref + CONFLICT_SEARCH_LIM)
95 return 0; /* Give up, ref is too far away. */
96 while (--i > ref)
97 if (ir[i].o == conflict)
98 return 0; /* Conflict found. */
99 return 1; /* Ok, no conflict. */
100}
101
102/* Fuse the array base of colocated arrays. */
103static int32_t asm_fuseabase(ASMState *as, IRRef ref)
104{
105 IRIns *ir = IR(ref);
106 if (ir->o == IR_TNEW && ir->op1 <= LJ_MAX_COLOSIZE &&
107 !neverfuse(as) && noconflict(as, ref, IR_NEWREF))
108 return (int32_t)sizeof(GCtab);
109 return 0;
110}
111
112/* Indicates load/store indexed is ok. */
113#define AHUREF_LSX ((int32_t)0x80000000)
114
115/* Fuse array/hash/upvalue reference into register+offset operand. */
116static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
117{
118 IRIns *ir = IR(ref);
119 if (ra_noreg(ir->r)) {
120 if (ir->o == IR_AREF) {
121 if (mayfuse(as, ref)) {
122 if (irref_isk(ir->op2)) {
123 IRRef tab = IR(ir->op1)->op1;
124 int32_t ofs = asm_fuseabase(as, tab);
125 IRRef refa = ofs ? tab : ir->op1;
126 ofs += 8*IR(ir->op2)->i;
127 if (checki16(ofs)) {
128 *ofsp = ofs;
129 return ra_alloc1(as, refa, allow);
130 }
131 }
132 if (*ofsp == AHUREF_LSX) {
133 Reg base = ra_alloc1(as, ir->op1, allow);
134 Reg idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
135 return base | (idx << 8);
136 }
137 }
138 } else if (ir->o == IR_HREFK) {
139 if (mayfuse(as, ref)) {
140 int32_t ofs = (int32_t)(IR(ir->op2)->op2 * sizeof(Node));
141 if (checki16(ofs)) {
142 *ofsp = ofs;
143 return ra_alloc1(as, ir->op1, allow);
144 }
145 }
146 } else if (ir->o == IR_UREFC) {
147 if (irref_isk(ir->op1)) {
148 GCfunc *fn = ir_kfunc(IR(ir->op1));
149 int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv);
150 int32_t jgl = (intptr_t)J2G(as->J);
151 if ((uint32_t)(ofs-jgl) < 65536) {
152 *ofsp = ofs-jgl-32768;
153 return RID_JGL;
154 } else {
155 *ofsp = (int16_t)ofs;
156 return ra_allock(as, ofs-(int16_t)ofs, allow);
157 }
158 }
159 }
160 }
161 *ofsp = 0;
162 return ra_alloc1(as, ref, allow);
163}
164
165/* Fuse XLOAD/XSTORE reference into load/store operand. */
166static void asm_fusexref(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
167 RegSet allow, int32_t ofs)
168{
169 IRIns *ir = IR(ref);
170 Reg base;
171 if (ra_noreg(ir->r) && canfuse(as, ir)) {
172 if (ir->o == IR_ADD) {
173 int32_t ofs2;
174 if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) {
175 ofs = ofs2;
176 ref = ir->op1;
177 } else if (ofs == 0) {
178 Reg right, left = ra_alloc2(as, ir, allow);
179 right = (left >> 8); left &= 255;
180 emit_fab(as, PPCI_LWZX | ((pi >> 20) & 0x780), rt, left, right);
181 return;
182 }
183 } else if (ir->o == IR_STRREF) {
184 lua_assert(ofs == 0);
185 ofs = (int32_t)sizeof(GCstr);
186 if (irref_isk(ir->op2)) {
187 ofs += IR(ir->op2)->i;
188 ref = ir->op1;
189 } else if (irref_isk(ir->op1)) {
190 ofs += IR(ir->op1)->i;
191 ref = ir->op2;
192 } else {
193 /* NYI: Fuse ADD with constant. */
194 Reg tmp, right, left = ra_alloc2(as, ir, allow);
195 right = (left >> 8); left &= 255;
196 tmp = ra_scratch(as, rset_exclude(rset_exclude(allow, left), right));
197 emit_fai(as, pi, rt, tmp, ofs);
198 emit_tab(as, PPCI_ADD, tmp, left, right);
199 return;
200 }
201 if (!checki16(ofs)) {
202 Reg left = ra_alloc1(as, ref, allow);
203 Reg right = ra_allock(as, ofs, rset_exclude(allow, left));
204 emit_fab(as, PPCI_LWZX | ((pi >> 20) & 0x780), rt, left, right);
205 return;
206 }
207 }
208 }
209 base = ra_alloc1(as, ref, allow);
210 emit_fai(as, pi, rt, base, ofs);
211}
212
213/* Fuse XLOAD/XSTORE reference into indexed-only load/store operand. */
214static void asm_fusexrefx(ASMState *as, PPCIns pi, Reg rt, IRRef ref,
215 RegSet allow)
216{
217 IRIns *ira = IR(ref);
218 Reg right, left;
219 if (canfuse(as, ira) && ira->o == IR_ADD && ra_noreg(ira->r)) {
220 left = ra_alloc2(as, ira, allow);
221 right = (left >> 8); left &= 255;
222 } else {
223 right = ra_alloc1(as, ref, allow);
224 left = RID_R0;
225 }
226 emit_tab(as, pi, rt, left, right);
227}
228
229/* Fuse to multiply-add/sub instruction. */
230static int asm_fusemadd(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pir)
231{
232 IRRef lref = ir->op1, rref = ir->op2;
233 IRIns *irm;
234 if (lref != rref &&
235 ((mayfuse(as, lref) && (irm = IR(lref), irm->o == IR_MUL) &&
236 ra_noreg(irm->r)) ||
237 (mayfuse(as, rref) && (irm = IR(rref), irm->o == IR_MUL) &&
238 (rref = lref, pi = pir, ra_noreg(irm->r))))) {
239 Reg dest = ra_dest(as, ir, RSET_FPR);
240 Reg add = ra_alloc1(as, rref, RSET_FPR);
241 Reg right, left = ra_alloc2(as, irm, rset_exclude(RSET_FPR, add));
242 right = (left >> 8); left &= 255;
243 emit_facb(as, pi, dest, left, right, add);
244 return 1;
245 }
246 return 0;
247}
248
249/* -- Calls --------------------------------------------------------------- */
250
251/* Generate a call to a C function. */
252static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
253{
254 uint32_t n, nargs = CCI_NARGS(ci);
255 int32_t ofs = 8;
256 Reg gpr = REGARG_FIRSTGPR, fpr = REGARG_FIRSTFPR;
257 if ((void *)ci->func)
258 emit_call(as, (void *)ci->func);
259 for (n = 0; n < nargs; n++) { /* Setup args. */
260 IRRef ref = args[n];
261 if (ref) {
262 IRIns *ir = IR(ref);
263 if (irt_isfp(ir->t)) {
264 if (fpr <= REGARG_LASTFPR) {
265 lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */
266 ra_leftov(as, fpr, ref);
267 fpr++;
268 } else {
269 Reg r = ra_alloc1(as, ref, RSET_FPR);
270 if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
271 emit_spstore(as, ir, r, ofs);
272 ofs += irt_isnum(ir->t) ? 8 : 4;
273 }
274 } else {
275 if (gpr <= REGARG_LASTGPR) {
276 lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */
277 ra_leftov(as, gpr, ref);
278 gpr++;
279 } else {
280 Reg r = ra_alloc1(as, ref, RSET_GPR);
281 emit_spstore(as, ir, r, ofs);
282 ofs += 4;
283 }
284 }
285 } else {
286 if (gpr <= REGARG_LASTGPR)
287 gpr++;
288 else
289 ofs += 4;
290 }
291 checkmclim(as);
292 }
293 if ((ci->flags & CCI_VARARG)) /* Vararg calls need to know about FPR use. */
294 emit_tab(as, fpr == REGARG_FIRSTFPR ? PPCI_CRXOR : PPCI_CREQV, 6, 6, 6);
295}
296
297/* Setup result reg/sp for call. Evict scratch regs. */
298static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
299{
300 RegSet drop = RSET_SCRATCH;
301 int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
302 if ((ci->flags & CCI_NOFPRCLOBBER))
303 drop &= ~RSET_FPR;
304 if (ra_hasreg(ir->r))
305 rset_clear(drop, ir->r); /* Dest reg handled below. */
306 if (hiop && ra_hasreg((ir+1)->r))
307 rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
308 ra_evictset(as, drop); /* Evictions must be performed first. */
309 if (ra_used(ir)) {
310 lua_assert(!irt_ispri(ir->t));
311 if (irt_isfp(ir->t)) {
312 if ((ci->flags & CCI_CASTU64)) {
313 /* Use spill slot or temp slots. */
314 int32_t ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
315 Reg dest = ir->r;
316 if (ra_hasreg(dest)) {
317 ra_free(as, dest);
318 ra_modified(as, dest);
319 emit_fai(as, PPCI_LFD, dest, RID_SP, ofs);
320 }
321 emit_tai(as, PPCI_STW, RID_RETHI, RID_SP, ofs);
322 emit_tai(as, PPCI_STW, RID_RETLO, RID_SP, ofs+4);
323 } else {
324 ra_destreg(as, ir, RID_FPRET);
325 }
326 } else if (hiop) {
327 ra_destpair(as, ir);
328 } else {
329 ra_destreg(as, ir, RID_RET);
330 }
331 }
332}
333
334static void asm_call(ASMState *as, IRIns *ir)
335{
336 IRRef args[CCI_NARGS_MAX];
337 const CCallInfo *ci = &lj_ir_callinfo[ir->op2];
338 asm_collectargs(as, ir, ci, args);
339 asm_setupresult(as, ir, ci);
340 asm_gencall(as, ci, args);
341}
342
343static void asm_callx(ASMState *as, IRIns *ir)
344{
345 IRRef args[CCI_NARGS_MAX*2];
346 CCallInfo ci;
347 IRRef func;
348 IRIns *irf;
349 ci.flags = asm_callx_flags(as, ir);
350 asm_collectargs(as, ir, &ci, args);
351 asm_setupresult(as, ir, &ci);
352 func = ir->op2; irf = IR(func);
353 if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
354 if (irref_isk(func)) { /* Call to constant address. */
355 ci.func = (ASMFunction)(void *)(irf->i);
356 } else { /* Need a non-argument register for indirect calls. */
357 RegSet allow = RSET_GPR & ~RSET_RANGE(RID_R0, REGARG_LASTGPR+1);
358 Reg freg = ra_alloc1(as, func, allow);
359 *--as->mcp = PPCI_BCTRL;
360 *--as->mcp = PPCI_MTCTR | PPCF_T(freg);
361 ci.func = (ASMFunction)(void *)0;
362 }
363 asm_gencall(as, &ci, args);
364}
365
366static void asm_callid(ASMState *as, IRIns *ir, IRCallID id)
367{
368 const CCallInfo *ci = &lj_ir_callinfo[id];
369 IRRef args[2];
370 args[0] = ir->op1;
371 args[1] = ir->op2;
372 asm_setupresult(as, ir, ci);
373 asm_gencall(as, ci, args);
374}
375
376/* -- Returns ------------------------------------------------------------- */
377
378/* Return to lower frame. Guard that it goes to the right spot. */
379static void asm_retf(ASMState *as, IRIns *ir)
380{
381 Reg base = ra_alloc1(as, REF_BASE, RSET_GPR);
382 void *pc = ir_kptr(IR(ir->op2));
383 int32_t delta = 1+bc_a(*((const BCIns *)pc - 1));
384 as->topslot -= (BCReg)delta;
385 if ((int32_t)as->topslot < 0) as->topslot = 0;
386 irt_setmark(IR(REF_BASE)->t); /* Children must not coalesce with BASE reg. */
387 emit_setgl(as, base, jit_base);
388 emit_addptr(as, base, -8*delta);
389 asm_guardcc(as, CC_NE);
390 emit_ab(as, PPCI_CMPW, RID_TMP,
391 ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base)));
392 emit_tai(as, PPCI_LWZ, RID_TMP, base, -8);
393}
394
395/* -- Type conversions ---------------------------------------------------- */
396
397static void asm_tointg(ASMState *as, IRIns *ir, Reg left)
398{
399 RegSet allow = RSET_FPR;
400 Reg tmp = ra_scratch(as, rset_clear(allow, left));
401 Reg fbias = ra_scratch(as, rset_clear(allow, tmp));
402 Reg dest = ra_dest(as, ir, RSET_GPR);
403 Reg hibias = ra_allock(as, 0x43300000, rset_exclude(RSET_GPR, dest));
404 asm_guardcc(as, CC_NE);
405 emit_fab(as, PPCI_FCMPU, 0, tmp, left);
406 emit_fab(as, PPCI_FSUB, tmp, tmp, fbias);
407 emit_fai(as, PPCI_LFD, tmp, RID_SP, SPOFS_TMP);
408 emit_tai(as, PPCI_STW, RID_TMP, RID_SP, SPOFS_TMPLO);
409 emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
410 emit_asi(as, PPCI_XORIS, RID_TMP, dest, 0x8000);
411 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
412 emit_lsptr(as, PPCI_LFS, (fbias & 31),
413 (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
414 RSET_GPR);
415 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
416 emit_fb(as, PPCI_FCTIWZ, tmp, left);
417}
418
419static void asm_tobit(ASMState *as, IRIns *ir)
420{
421 RegSet allow = RSET_FPR;
422 Reg dest = ra_dest(as, ir, RSET_GPR);
423 Reg left = ra_alloc1(as, ir->op1, allow);
424 Reg right = ra_alloc1(as, ir->op2, rset_clear(allow, left));
425 Reg tmp = ra_scratch(as, rset_clear(allow, right));
426 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
427 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
428 emit_fab(as, PPCI_FADD, tmp, left, right);
429}
430
431static void asm_conv(ASMState *as, IRIns *ir)
432{
433 IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
434 int stfp = (st == IRT_NUM || st == IRT_FLOAT);
435 IRRef lref = ir->op1;
436 lua_assert(irt_type(ir->t) != st);
437 lua_assert(!(irt_isint64(ir->t) ||
438 (st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
439 if (irt_isfp(ir->t)) {
440 Reg dest = ra_dest(as, ir, RSET_FPR);
441 if (stfp) { /* FP to FP conversion. */
442 if (st == IRT_NUM) /* double -> float conversion. */
443 emit_fb(as, PPCI_FRSP, dest, ra_alloc1(as, lref, RSET_FPR));
444 else /* float -> double conversion is a no-op on PPC. */
445 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
446 } else { /* Integer to FP conversion. */
447 /* IRT_INT: Flip hibit, bias with 2^52, subtract 2^52+2^31. */
448 /* IRT_U32: Bias with 2^52, subtract 2^52. */
449 RegSet allow = RSET_GPR;
450 Reg left = ra_alloc1(as, lref, allow);
451 Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, left));
452 Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
453 const float *kbias;
454 if (irt_isfloat(ir->t)) emit_fb(as, PPCI_FRSP, dest, dest);
455 emit_fab(as, PPCI_FSUB, dest, dest, fbias);
456 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
457 kbias = (const float *)lj_ir_k64_find(as->J, U64x(59800004,59800000));
458 if (st == IRT_U32) kbias++;
459 emit_lsptr(as, PPCI_LFS, (fbias & 31), (void *)kbias,
460 rset_clear(allow, hibias));
461 emit_tai(as, PPCI_STW, st == IRT_U32 ? left : RID_TMP,
462 RID_SP, SPOFS_TMPLO);
463 emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
464 if (st != IRT_U32) emit_asi(as, PPCI_XORIS, RID_TMP, left, 0x8000);
465 }
466 } else if (stfp) { /* FP to integer conversion. */
467 if (irt_isguard(ir->t)) {
468 /* Checked conversions are only supported from number to int. */
469 lua_assert(irt_isint(ir->t) && st == IRT_NUM);
470 asm_tointg(as, ir, ra_alloc1(as, lref, RSET_FPR));
471 } else {
472 Reg dest = ra_dest(as, ir, RSET_GPR);
473 Reg left = ra_alloc1(as, lref, RSET_FPR);
474 Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
475 if (irt_isu32(ir->t)) {
476 /* Convert both x and x-2^31 to int and merge results. */
477 Reg tmpi = ra_scratch(as, rset_exclude(RSET_GPR, dest));
478 emit_asb(as, PPCI_OR, dest, dest, tmpi); /* Select with mask idiom. */
479 emit_asb(as, PPCI_AND, tmpi, tmpi, RID_TMP);
480 emit_asb(as, PPCI_ANDC, dest, dest, RID_TMP);
481 emit_tai(as, PPCI_LWZ, tmpi, RID_SP, SPOFS_TMPLO); /* tmp = (int)(x) */
482 emit_tai(as, PPCI_ADDIS, dest, dest, 0x8000); /* dest += 2^31 */
483 emit_asb(as, PPCI_SRAWI, RID_TMP, dest, 31); /* mask = -(dest < 0) */
484 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
485 emit_tai(as, PPCI_LWZ, dest,
486 RID_SP, SPOFS_TMPLO); /* dest = (int)(x-2^31) */
487 emit_fb(as, PPCI_FCTIWZ, tmp, left);
488 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
489 emit_fb(as, PPCI_FCTIWZ, tmp, tmp);
490 emit_fab(as, PPCI_FSUB, tmp, left, tmp);
491 emit_lsptr(as, PPCI_LFS, (tmp & 31),
492 (void *)lj_ir_k64_find(as->J, U64x(4f000000,00000000)),
493 RSET_GPR);
494 } else {
495 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
496 emit_fai(as, PPCI_STFD, tmp, RID_SP, SPOFS_TMP);
497 emit_fb(as, PPCI_FCTIWZ, tmp, left);
498 }
499 }
500 } else {
501 Reg dest = ra_dest(as, ir, RSET_GPR);
502 if (st >= IRT_I8 && st <= IRT_U16) { /* Extend to 32 bit integer. */
503 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
504 lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
505 if ((ir->op2 & IRCONV_SEXT))
506 emit_as(as, st == IRT_I8 ? PPCI_EXTSB : PPCI_EXTSH, dest, left);
507 else
508 emit_rot(as, PPCI_RLWINM, dest, left, 0, st == IRT_U8 ? 24 : 16, 31);
509 } else { /* 32/64 bit integer conversions. */
510 /* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
511 ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
512 }
513 }
514}
515
516#if LJ_HASFFI
517static void asm_conv64(ASMState *as, IRIns *ir)
518{
519 IRType st = (IRType)((ir-1)->op2 & IRCONV_SRCMASK);
520 IRType dt = (((ir-1)->op2 & IRCONV_DSTMASK) >> IRCONV_DSH);
521 IRCallID id;
522 const CCallInfo *ci;
523 IRRef args[2];
524 args[0] = ir->op1;
525 args[1] = (ir-1)->op1;
526 if (st == IRT_NUM || st == IRT_FLOAT) {
527 id = IRCALL_fp64_d2l + ((st == IRT_FLOAT) ? 2 : 0) + (dt - IRT_I64);
528 ir--;
529 } else {
530 id = IRCALL_fp64_l2d + ((dt == IRT_FLOAT) ? 2 : 0) + (st - IRT_I64);
531 }
532 ci = &lj_ir_callinfo[id];
533 asm_setupresult(as, ir, ci);
534 asm_gencall(as, ci, args);
535}
536#endif
537
538static void asm_strto(ASMState *as, IRIns *ir)
539{
540 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_strscan_num];
541 IRRef args[2];
542 int32_t ofs;
543 RegSet drop = RSET_SCRATCH;
544 if (ra_hasreg(ir->r)) rset_set(drop, ir->r); /* Spill dest reg (if any). */
545 ra_evictset(as, drop);
546 asm_guardcc(as, CC_EQ);
547 emit_ai(as, PPCI_CMPWI, RID_RET, 0); /* Test return status. */
548 args[0] = ir->op1; /* GCstr *str */
549 args[1] = ASMREF_TMP1; /* TValue *n */
550 asm_gencall(as, ci, args);
551 /* Store the result to the spill slot or temp slots. */
552 ofs = ir->s ? sps_scale(ir->s) : SPOFS_TMP;
553 emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_SP, ofs);
554}
555
556/* Get pointer to TValue. */
557static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
558{
559 IRIns *ir = IR(ref);
560 if (irt_isnum(ir->t)) {
561 if (irref_isk(ref)) /* Use the number constant itself as a TValue. */
562 ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
563 else /* Otherwise force a spill and use the spill slot. */
564 emit_tai(as, PPCI_ADDI, dest, RID_SP, ra_spill(as, ir));
565 } else {
566 /* Otherwise use g->tmptv to hold the TValue. */
567 RegSet allow = rset_exclude(RSET_GPR, dest);
568 Reg type;
569 emit_tai(as, PPCI_ADDI, dest, RID_JGL, offsetof(global_State, tmptv)-32768);
570 if (!irt_ispri(ir->t)) {
571 Reg src = ra_alloc1(as, ref, allow);
572 emit_setgl(as, src, tmptv.gcr);
573 }
574 type = ra_allock(as, irt_toitype(ir->t), allow);
575 emit_setgl(as, type, tmptv.it);
576 }
577}
578
579static void asm_tostr(ASMState *as, IRIns *ir)
580{
581 IRRef args[2];
582 args[0] = ASMREF_L;
583 as->gcsteps++;
584 if (irt_isnum(IR(ir->op1)->t) || (ir+1)->o == IR_HIOP) {
585 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromnum];
586 args[1] = ASMREF_TMP1; /* const lua_Number * */
587 asm_setupresult(as, ir, ci); /* GCstr * */
588 asm_gencall(as, ci, args);
589 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op1);
590 } else {
591 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_str_fromint];
592 args[1] = ir->op1; /* int32_t k */
593 asm_setupresult(as, ir, ci); /* GCstr * */
594 asm_gencall(as, ci, args);
595 }
596}
597
598/* -- Memory references --------------------------------------------------- */
599
600static void asm_aref(ASMState *as, IRIns *ir)
601{
602 Reg dest = ra_dest(as, ir, RSET_GPR);
603 Reg idx, base;
604 if (irref_isk(ir->op2)) {
605 IRRef tab = IR(ir->op1)->op1;
606 int32_t ofs = asm_fuseabase(as, tab);
607 IRRef refa = ofs ? tab : ir->op1;
608 ofs += 8*IR(ir->op2)->i;
609 if (checki16(ofs)) {
610 base = ra_alloc1(as, refa, RSET_GPR);
611 emit_tai(as, PPCI_ADDI, dest, base, ofs);
612 return;
613 }
614 }
615 base = ra_alloc1(as, ir->op1, RSET_GPR);
616 idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
617 emit_tab(as, PPCI_ADD, dest, RID_TMP, base);
618 emit_slwi(as, RID_TMP, idx, 3);
619}
620
621/* Inlined hash lookup. Specialized for key type and for const keys.
622** The equivalent C code is:
623** Node *n = hashkey(t, key);
624** do {
625** if (lj_obj_equal(&n->key, key)) return &n->val;
626** } while ((n = nextnode(n)));
627** return niltv(L);
628*/
629static void asm_href(ASMState *as, IRIns *ir, IROp merge)
630{
631 RegSet allow = RSET_GPR;
632 int destused = ra_used(ir);
633 Reg dest = ra_dest(as, ir, allow);
634 Reg tab = ra_alloc1(as, ir->op1, rset_clear(allow, dest));
635 Reg key = RID_NONE, tmp1 = RID_TMP, tmp2;
636 Reg tisnum = RID_NONE, tmpnum = RID_NONE;
637 IRRef refkey = ir->op2;
638 IRIns *irkey = IR(refkey);
639 IRType1 kt = irkey->t;
640 uint32_t khash;
641 MCLabel l_end, l_loop, l_next;
642
643 rset_clear(allow, tab);
644 if (irt_isnum(kt)) {
645 key = ra_alloc1(as, refkey, RSET_FPR);
646 tmpnum = ra_scratch(as, rset_exclude(RSET_FPR, key));
647 tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow);
648 rset_clear(allow, tisnum);
649 } else if (!irt_ispri(kt)) {
650 key = ra_alloc1(as, refkey, allow);
651 rset_clear(allow, key);
652 }
653 tmp2 = ra_scratch(as, allow);
654 rset_clear(allow, tmp2);
655
656 /* Key not found in chain: jump to exit (if merged) or load niltv. */
657 l_end = emit_label(as);
658 as->invmcp = NULL;
659 if (merge == IR_NE)
660 asm_guardcc(as, CC_EQ);
661 else if (destused)
662 emit_loada(as, dest, niltvg(J2G(as->J)));
663
664 /* Follow hash chain until the end. */
665 l_loop = --as->mcp;
666 emit_ai(as, PPCI_CMPWI, dest, 0);
667 emit_tai(as, PPCI_LWZ, dest, dest, (int32_t)offsetof(Node, next));
668 l_next = emit_label(as);
669
670 /* Type and value comparison. */
671 if (merge == IR_EQ)
672 asm_guardcc(as, CC_EQ);
673 else
674 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
675 if (irt_isnum(kt)) {
676 emit_fab(as, PPCI_FCMPU, 0, tmpnum, key);
677 emit_condbranch(as, PPCI_BC, CC_GE, l_next);
678 emit_ab(as, PPCI_CMPLW, tmp1, tisnum);
679 emit_fai(as, PPCI_LFD, tmpnum, dest, (int32_t)offsetof(Node, key.n));
680 } else {
681 if (!irt_ispri(kt)) {
682 emit_ab(as, PPCI_CMPW, tmp2, key);
683 emit_condbranch(as, PPCI_BC, CC_NE, l_next);
684 }
685 emit_ai(as, PPCI_CMPWI, tmp1, irt_toitype(irkey->t));
686 if (!irt_ispri(kt))
687 emit_tai(as, PPCI_LWZ, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
688 }
689 emit_tai(as, PPCI_LWZ, tmp1, dest, (int32_t)offsetof(Node, key.it));
690 *l_loop = PPCI_BC | PPCF_Y | PPCF_CC(CC_NE) |
691 (((char *)as->mcp-(char *)l_loop) & 0xffffu);
692
693 /* Load main position relative to tab->node into dest. */
694 khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
695 if (khash == 0) {
696 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
697 } else {
698 Reg tmphash = tmp1;
699 if (irref_isk(refkey))
700 tmphash = ra_allock(as, khash, allow);
701 emit_tab(as, PPCI_ADD, dest, dest, tmp1);
702 emit_tai(as, PPCI_MULLI, tmp1, tmp1, sizeof(Node));
703 emit_asb(as, PPCI_AND, tmp1, tmp2, tmphash);
704 emit_tai(as, PPCI_LWZ, dest, tab, (int32_t)offsetof(GCtab, node));
705 emit_tai(as, PPCI_LWZ, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
706 if (irref_isk(refkey)) {
707 /* Nothing to do. */
708 } else if (irt_isstr(kt)) {
709 emit_tai(as, PPCI_LWZ, tmp1, key, (int32_t)offsetof(GCstr, hash));
710 } else { /* Must match with hash*() in lj_tab.c. */
711 emit_tab(as, PPCI_SUBF, tmp1, tmp2, tmp1);
712 emit_rotlwi(as, tmp2, tmp2, HASH_ROT3);
713 emit_asb(as, PPCI_XOR, tmp1, tmp1, tmp2);
714 emit_rotlwi(as, tmp1, tmp1, (HASH_ROT2+HASH_ROT1)&31);
715 emit_tab(as, PPCI_SUBF, tmp2, dest, tmp2);
716 if (irt_isnum(kt)) {
717 int32_t ofs = ra_spill(as, irkey);
718 emit_asb(as, PPCI_XOR, tmp2, tmp2, tmp1);
719 emit_rotlwi(as, dest, tmp1, HASH_ROT1);
720 emit_tab(as, PPCI_ADD, tmp1, tmp1, tmp1);
721 emit_tai(as, PPCI_LWZ, tmp2, RID_SP, ofs+4);
722 emit_tai(as, PPCI_LWZ, tmp1, RID_SP, ofs);
723 } else {
724 emit_asb(as, PPCI_XOR, tmp2, key, tmp1);
725 emit_rotlwi(as, dest, tmp1, HASH_ROT1);
726 emit_tai(as, PPCI_ADDI, tmp1, tmp2, HASH_BIAS);
727 emit_tai(as, PPCI_ADDIS, tmp2, key, (HASH_BIAS + 32768)>>16);
728 }
729 }
730 }
731}
732
733static void asm_hrefk(ASMState *as, IRIns *ir)
734{
735 IRIns *kslot = IR(ir->op2);
736 IRIns *irkey = IR(kslot->op1);
737 int32_t ofs = (int32_t)(kslot->op2 * sizeof(Node));
738 int32_t kofs = ofs + (int32_t)offsetof(Node, key);
739 Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
740 Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
741 Reg key = RID_NONE, type = RID_TMP, idx = node;
742 RegSet allow = rset_exclude(RSET_GPR, node);
743 lua_assert(ofs % sizeof(Node) == 0);
744 if (ofs > 32736) {
745 idx = dest;
746 rset_clear(allow, dest);
747 kofs = (int32_t)offsetof(Node, key);
748 } else if (ra_hasreg(dest)) {
749 emit_tai(as, PPCI_ADDI, dest, node, ofs);
750 }
751 asm_guardcc(as, CC_NE);
752 if (!irt_ispri(irkey->t)) {
753 key = ra_scratch(as, allow);
754 rset_clear(allow, key);
755 }
756 rset_clear(allow, type);
757 if (irt_isnum(irkey->t)) {
758 emit_cmpi(as, key, (int32_t)ir_knum(irkey)->u32.lo);
759 asm_guardcc(as, CC_NE);
760 emit_cmpi(as, type, (int32_t)ir_knum(irkey)->u32.hi);
761 } else {
762 if (ra_hasreg(key)) {
763 emit_cmpi(as, key, irkey->i); /* May use RID_TMP, i.e. type. */
764 asm_guardcc(as, CC_NE);
765 }
766 emit_ai(as, PPCI_CMPWI, type, irt_toitype(irkey->t));
767 }
768 if (ra_hasreg(key)) emit_tai(as, PPCI_LWZ, key, idx, kofs+4);
769 emit_tai(as, PPCI_LWZ, type, idx, kofs);
770 if (ofs > 32736) {
771 emit_tai(as, PPCI_ADDIS, dest, dest, (ofs + 32768) >> 16);
772 emit_tai(as, PPCI_ADDI, dest, node, ofs);
773 }
774}
775
776static void asm_newref(ASMState *as, IRIns *ir)
777{
778 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_tab_newkey];
779 IRRef args[3];
780 if (ir->r == RID_SINK)
781 return;
782 args[0] = ASMREF_L; /* lua_State *L */
783 args[1] = ir->op1; /* GCtab *t */
784 args[2] = ASMREF_TMP1; /* cTValue *key */
785 asm_setupresult(as, ir, ci); /* TValue * */
786 asm_gencall(as, ci, args);
787 asm_tvptr(as, ra_releasetmp(as, ASMREF_TMP1), ir->op2);
788}
789
790static void asm_uref(ASMState *as, IRIns *ir)
791{
792 Reg dest = ra_dest(as, ir, RSET_GPR);
793 if (irref_isk(ir->op1)) {
794 GCfunc *fn = ir_kfunc(IR(ir->op1));
795 MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
796 emit_lsptr(as, PPCI_LWZ, dest, v, RSET_GPR);
797 } else {
798 Reg uv = ra_scratch(as, RSET_GPR);
799 Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
800 if (ir->o == IR_UREFC) {
801 asm_guardcc(as, CC_NE);
802 emit_ai(as, PPCI_CMPWI, RID_TMP, 1);
803 emit_tai(as, PPCI_ADDI, dest, uv, (int32_t)offsetof(GCupval, tv));
804 emit_tai(as, PPCI_LBZ, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
805 } else {
806 emit_tai(as, PPCI_LWZ, dest, uv, (int32_t)offsetof(GCupval, v));
807 }
808 emit_tai(as, PPCI_LWZ, uv, func,
809 (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
810 }
811}
812
813static void asm_fref(ASMState *as, IRIns *ir)
814{
815 UNUSED(as); UNUSED(ir);
816 lua_assert(!ra_used(ir));
817}
818
819static void asm_strref(ASMState *as, IRIns *ir)
820{
821 Reg dest = ra_dest(as, ir, RSET_GPR);
822 IRRef ref = ir->op2, refk = ir->op1;
823 int32_t ofs = (int32_t)sizeof(GCstr);
824 Reg r;
825 if (irref_isk(ref)) {
826 IRRef tmp = refk; refk = ref; ref = tmp;
827 } else if (!irref_isk(refk)) {
828 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
829 IRIns *irr = IR(ir->op2);
830 if (ra_hasreg(irr->r)) {
831 ra_noweak(as, irr->r);
832 right = irr->r;
833 } else if (mayfuse(as, irr->op2) &&
834 irr->o == IR_ADD && irref_isk(irr->op2) &&
835 checki16(ofs + IR(irr->op2)->i)) {
836 ofs += IR(irr->op2)->i;
837 right = ra_alloc1(as, irr->op1, rset_exclude(RSET_GPR, left));
838 } else {
839 right = ra_allocref(as, ir->op2, rset_exclude(RSET_GPR, left));
840 }
841 emit_tai(as, PPCI_ADDI, dest, dest, ofs);
842 emit_tab(as, PPCI_ADD, dest, left, right);
843 return;
844 }
845 r = ra_alloc1(as, ref, RSET_GPR);
846 ofs += IR(refk)->i;
847 if (checki16(ofs))
848 emit_tai(as, PPCI_ADDI, dest, r, ofs);
849 else
850 emit_tab(as, PPCI_ADD, dest, r,
851 ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
852}
853
854/* -- Loads and stores ---------------------------------------------------- */
855
856static PPCIns asm_fxloadins(IRIns *ir)
857{
858 switch (irt_type(ir->t)) {
859 case IRT_I8: return PPCI_LBZ; /* Needs sign-extension. */
860 case IRT_U8: return PPCI_LBZ;
861 case IRT_I16: return PPCI_LHA;
862 case IRT_U16: return PPCI_LHZ;
863 case IRT_NUM: return PPCI_LFD;
864 case IRT_FLOAT: return PPCI_LFS;
865 default: return PPCI_LWZ;
866 }
867}
868
869static PPCIns asm_fxstoreins(IRIns *ir)
870{
871 switch (irt_type(ir->t)) {
872 case IRT_I8: case IRT_U8: return PPCI_STB;
873 case IRT_I16: case IRT_U16: return PPCI_STH;
874 case IRT_NUM: return PPCI_STFD;
875 case IRT_FLOAT: return PPCI_STFS;
876 default: return PPCI_STW;
877 }
878}
879
880static void asm_fload(ASMState *as, IRIns *ir)
881{
882 Reg dest = ra_dest(as, ir, RSET_GPR);
883 Reg idx = ra_alloc1(as, ir->op1, RSET_GPR);
884 PPCIns pi = asm_fxloadins(ir);
885 int32_t ofs;
886 if (ir->op2 == IRFL_TAB_ARRAY) {
887 ofs = asm_fuseabase(as, ir->op1);
888 if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
889 emit_tai(as, PPCI_ADDI, dest, idx, ofs);
890 return;
891 }
892 }
893 ofs = field_ofs[ir->op2];
894 lua_assert(!irt_isi8(ir->t));
895 emit_tai(as, pi, dest, idx, ofs);
896}
897
898static void asm_fstore(ASMState *as, IRIns *ir)
899{
900 if (ir->r != RID_SINK) {
901 Reg src = ra_alloc1(as, ir->op2, RSET_GPR);
902 IRIns *irf = IR(ir->op1);
903 Reg idx = ra_alloc1(as, irf->op1, rset_exclude(RSET_GPR, src));
904 int32_t ofs = field_ofs[irf->op2];
905 PPCIns pi = asm_fxstoreins(ir);
906 emit_tai(as, pi, src, idx, ofs);
907 }
908}
909
910static void asm_xload(ASMState *as, IRIns *ir)
911{
912 Reg dest = ra_dest(as, ir, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
913 lua_assert(!(ir->op2 & IRXLOAD_UNALIGNED));
914 if (irt_isi8(ir->t))
915 emit_as(as, PPCI_EXTSB, dest, dest);
916 asm_fusexref(as, asm_fxloadins(ir), dest, ir->op1, RSET_GPR, 0);
917}
918
919static void asm_xstore(ASMState *as, IRIns *ir, int32_t ofs)
920{
921 IRIns *irb;
922 if (ir->r == RID_SINK)
923 return;
924 if (ofs == 0 && mayfuse(as, ir->op2) && (irb = IR(ir->op2))->o == IR_BSWAP &&
925 ra_noreg(irb->r) && (irt_isint(ir->t) || irt_isu32(ir->t))) {
926 /* Fuse BSWAP with XSTORE to stwbrx. */
927 Reg src = ra_alloc1(as, irb->op1, RSET_GPR);
928 asm_fusexrefx(as, PPCI_STWBRX, src, ir->op1, rset_exclude(RSET_GPR, src));
929 } else {
930 Reg src = ra_alloc1(as, ir->op2, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
931 asm_fusexref(as, asm_fxstoreins(ir), src, ir->op1,
932 rset_exclude(RSET_GPR, src), ofs);
933 }
934}
935
936static void asm_ahuvload(ASMState *as, IRIns *ir)
937{
938 IRType1 t = ir->t;
939 Reg dest = RID_NONE, type = RID_TMP, tmp = RID_TMP, idx;
940 RegSet allow = RSET_GPR;
941 int32_t ofs = AHUREF_LSX;
942 if (ra_used(ir)) {
943 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
944 if (!irt_isnum(t)) ofs = 0;
945 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
946 rset_clear(allow, dest);
947 }
948 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
949 if (irt_isnum(t)) {
950 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, rset_exclude(allow, idx));
951 asm_guardcc(as, CC_GE);
952 emit_ab(as, PPCI_CMPLW, type, tisnum);
953 if (ra_hasreg(dest)) {
954 if (ofs == AHUREF_LSX) {
955 tmp = ra_scratch(as, rset_exclude(rset_exclude(RSET_GPR,
956 (idx&255)), (idx>>8)));
957 emit_fab(as, PPCI_LFDX, dest, (idx&255), tmp);
958 } else {
959 emit_fai(as, PPCI_LFD, dest, idx, ofs);
960 }
961 }
962 } else {
963 asm_guardcc(as, CC_NE);
964 emit_ai(as, PPCI_CMPWI, type, irt_toitype(t));
965 if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, idx, ofs+4);
966 }
967 if (ofs == AHUREF_LSX) {
968 emit_tab(as, PPCI_LWZX, type, (idx&255), tmp);
969 emit_slwi(as, tmp, (idx>>8), 3);
970 } else {
971 emit_tai(as, PPCI_LWZ, type, idx, ofs);
972 }
973}
974
975static void asm_ahustore(ASMState *as, IRIns *ir)
976{
977 RegSet allow = RSET_GPR;
978 Reg idx, src = RID_NONE, type = RID_NONE;
979 int32_t ofs = AHUREF_LSX;
980 if (ir->r == RID_SINK)
981 return;
982 if (irt_isnum(ir->t)) {
983 src = ra_alloc1(as, ir->op2, RSET_FPR);
984 } else {
985 if (!irt_ispri(ir->t)) {
986 src = ra_alloc1(as, ir->op2, allow);
987 rset_clear(allow, src);
988 ofs = 0;
989 }
990 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
991 rset_clear(allow, type);
992 }
993 idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
994 if (irt_isnum(ir->t)) {
995 if (ofs == AHUREF_LSX) {
996 emit_fab(as, PPCI_STFDX, src, (idx&255), RID_TMP);
997 emit_slwi(as, RID_TMP, (idx>>8), 3);
998 } else {
999 emit_fai(as, PPCI_STFD, src, idx, ofs);
1000 }
1001 } else {
1002 if (ra_hasreg(src))
1003 emit_tai(as, PPCI_STW, src, idx, ofs+4);
1004 if (ofs == AHUREF_LSX) {
1005 emit_tab(as, PPCI_STWX, type, (idx&255), RID_TMP);
1006 emit_slwi(as, RID_TMP, (idx>>8), 3);
1007 } else {
1008 emit_tai(as, PPCI_STW, type, idx, ofs);
1009 }
1010 }
1011}
1012
1013static void asm_sload(ASMState *as, IRIns *ir)
1014{
1015 int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 0 : 4);
1016 IRType1 t = ir->t;
1017 Reg dest = RID_NONE, type = RID_NONE, base;
1018 RegSet allow = RSET_GPR;
1019 lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */
1020 lua_assert(irt_isguard(t) || !(ir->op2 & IRSLOAD_TYPECHECK));
1021 lua_assert(LJ_DUALNUM ||
1022 !irt_isint(t) || (ir->op2 & (IRSLOAD_CONVERT|IRSLOAD_FRAME)));
1023 if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
1024 dest = ra_scratch(as, RSET_FPR);
1025 asm_tointg(as, ir, dest);
1026 t.irt = IRT_NUM; /* Continue with a regular number type check. */
1027 } else if (ra_used(ir)) {
1028 lua_assert(irt_isnum(t) || irt_isint(t) || irt_isaddr(t));
1029 dest = ra_dest(as, ir, irt_isnum(t) ? RSET_FPR : RSET_GPR);
1030 rset_clear(allow, dest);
1031 base = ra_alloc1(as, REF_BASE, allow);
1032 rset_clear(allow, base);
1033 if ((ir->op2 & IRSLOAD_CONVERT)) {
1034 if (irt_isint(t)) {
1035 emit_tai(as, PPCI_LWZ, dest, RID_SP, SPOFS_TMPLO);
1036 dest = ra_scratch(as, RSET_FPR);
1037 emit_fai(as, PPCI_STFD, dest, RID_SP, SPOFS_TMP);
1038 emit_fb(as, PPCI_FCTIWZ, dest, dest);
1039 t.irt = IRT_NUM; /* Check for original type. */
1040 } else {
1041 Reg tmp = ra_scratch(as, allow);
1042 Reg hibias = ra_allock(as, 0x43300000, rset_clear(allow, tmp));
1043 Reg fbias = ra_scratch(as, rset_exclude(RSET_FPR, dest));
1044 emit_fab(as, PPCI_FSUB, dest, dest, fbias);
1045 emit_fai(as, PPCI_LFD, dest, RID_SP, SPOFS_TMP);
1046 emit_lsptr(as, PPCI_LFS, (fbias & 31),
1047 (void *)lj_ir_k64_find(as->J, U64x(59800004,59800000)),
1048 rset_clear(allow, hibias));
1049 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPLO);
1050 emit_tai(as, PPCI_STW, hibias, RID_SP, SPOFS_TMPHI);
1051 emit_asi(as, PPCI_XORIS, tmp, tmp, 0x8000);
1052 dest = tmp;
1053 t.irt = IRT_INT; /* Check for original type. */
1054 }
1055 }
1056 goto dotypecheck;
1057 }
1058 base = ra_alloc1(as, REF_BASE, allow);
1059 rset_clear(allow, base);
1060dotypecheck:
1061 if (irt_isnum(t)) {
1062 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1063 Reg tisnum = ra_allock(as, (int32_t)LJ_TISNUM, allow);
1064 asm_guardcc(as, CC_GE);
1065 emit_ab(as, PPCI_CMPLW, RID_TMP, tisnum);
1066 type = RID_TMP;
1067 }
1068 if (ra_hasreg(dest)) emit_fai(as, PPCI_LFD, dest, base, ofs-4);
1069 } else {
1070 if ((ir->op2 & IRSLOAD_TYPECHECK)) {
1071 asm_guardcc(as, CC_NE);
1072 emit_ai(as, PPCI_CMPWI, RID_TMP, irt_toitype(t));
1073 type = RID_TMP;
1074 }
1075 if (ra_hasreg(dest)) emit_tai(as, PPCI_LWZ, dest, base, ofs);
1076 }
1077 if (ra_hasreg(type)) emit_tai(as, PPCI_LWZ, type, base, ofs-4);
1078}
1079
1080/* -- Allocations --------------------------------------------------------- */
1081
1082#if LJ_HASFFI
1083static void asm_cnew(ASMState *as, IRIns *ir)
1084{
1085 CTState *cts = ctype_ctsG(J2G(as->J));
1086 CTypeID ctypeid = (CTypeID)IR(ir->op1)->i;
1087 CTSize sz = (ir->o == IR_CNEWI || ir->op2 == REF_NIL) ?
1088 lj_ctype_size(cts, ctypeid) : (CTSize)IR(ir->op2)->i;
1089 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_mem_newgco];
1090 IRRef args[2];
1091 RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
1092 RegSet drop = RSET_SCRATCH;
1093 lua_assert(sz != CTSIZE_INVALID);
1094
1095 args[0] = ASMREF_L; /* lua_State *L */
1096 args[1] = ASMREF_TMP1; /* MSize size */
1097 as->gcsteps++;
1098
1099 if (ra_hasreg(ir->r))
1100 rset_clear(drop, ir->r); /* Dest reg handled below. */
1101 ra_evictset(as, drop);
1102 if (ra_used(ir))
1103 ra_destreg(as, ir, RID_RET); /* GCcdata * */
1104
1105 /* Initialize immutable cdata object. */
1106 if (ir->o == IR_CNEWI) {
1107 int32_t ofs = sizeof(GCcdata);
1108 lua_assert(sz == 4 || sz == 8);
1109 if (sz == 8) {
1110 ofs += 4;
1111 lua_assert((ir+1)->o == IR_HIOP);
1112 }
1113 for (;;) {
1114 Reg r = ra_alloc1(as, ir->op2, allow);
1115 emit_tai(as, PPCI_STW, r, RID_RET, ofs);
1116 rset_clear(allow, r);
1117 if (ofs == sizeof(GCcdata)) break;
1118 ofs -= 4; ir++;
1119 }
1120 }
1121 /* Initialize gct and ctypeid. lj_mem_newgco() already sets marked. */
1122 emit_tai(as, PPCI_STB, RID_RET+1, RID_RET, offsetof(GCcdata, gct));
1123 emit_tai(as, PPCI_STH, RID_TMP, RID_RET, offsetof(GCcdata, ctypeid));
1124 emit_ti(as, PPCI_LI, RID_RET+1, ~LJ_TCDATA);
1125 emit_ti(as, PPCI_LI, RID_TMP, ctypeid); /* Lower 16 bit used. Sign-ext ok. */
1126 asm_gencall(as, ci, args);
1127 ra_allockreg(as, (int32_t)(sz+sizeof(GCcdata)),
1128 ra_releasetmp(as, ASMREF_TMP1));
1129}
1130#else
1131#define asm_cnew(as, ir) ((void)0)
1132#endif
1133
1134/* -- Write barriers ------------------------------------------------------ */
1135
1136static void asm_tbar(ASMState *as, IRIns *ir)
1137{
1138 Reg tab = ra_alloc1(as, ir->op1, RSET_GPR);
1139 Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
1140 Reg link = RID_TMP;
1141 MCLabel l_end = emit_label(as);
1142 emit_tai(as, PPCI_STW, link, tab, (int32_t)offsetof(GCtab, gclist));
1143 emit_tai(as, PPCI_STB, mark, tab, (int32_t)offsetof(GCtab, marked));
1144 emit_setgl(as, tab, gc.grayagain);
1145 lua_assert(LJ_GC_BLACK == 0x04);
1146 emit_rot(as, PPCI_RLWINM, mark, mark, 0, 30, 28); /* Clear black bit. */
1147 emit_getgl(as, link, gc.grayagain);
1148 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
1149 emit_asi(as, PPCI_ANDIDOT, RID_TMP, mark, LJ_GC_BLACK);
1150 emit_tai(as, PPCI_LBZ, mark, tab, (int32_t)offsetof(GCtab, marked));
1151}
1152
1153static void asm_obar(ASMState *as, IRIns *ir)
1154{
1155 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_barrieruv];
1156 IRRef args[2];
1157 MCLabel l_end;
1158 Reg obj, val, tmp;
1159 /* No need for other object barriers (yet). */
1160 lua_assert(IR(ir->op1)->o == IR_UREFC);
1161 ra_evictset(as, RSET_SCRATCH);
1162 l_end = emit_label(as);
1163 args[0] = ASMREF_TMP1; /* global_State *g */
1164 args[1] = ir->op1; /* TValue *tv */
1165 asm_gencall(as, ci, args);
1166 emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1167 obj = IR(ir->op1)->r;
1168 tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
1169 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_EQ, l_end);
1170 emit_asi(as, PPCI_ANDIDOT, tmp, tmp, LJ_GC_BLACK);
1171 emit_condbranch(as, PPCI_BC, CC_EQ, l_end);
1172 emit_asi(as, PPCI_ANDIDOT, RID_TMP, RID_TMP, LJ_GC_WHITES);
1173 val = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, obj));
1174 emit_tai(as, PPCI_LBZ, tmp, obj,
1175 (int32_t)offsetof(GCupval, marked)-(int32_t)offsetof(GCupval, tv));
1176 emit_tai(as, PPCI_LBZ, RID_TMP, val, (int32_t)offsetof(GChead, marked));
1177}
1178
1179/* -- Arithmetic and logic operations ------------------------------------- */
1180
1181static void asm_fparith(ASMState *as, IRIns *ir, PPCIns pi)
1182{
1183 Reg dest = ra_dest(as, ir, RSET_FPR);
1184 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1185 right = (left >> 8); left &= 255;
1186 if (pi == PPCI_FMUL)
1187 emit_fac(as, pi, dest, left, right);
1188 else
1189 emit_fab(as, pi, dest, left, right);
1190}
1191
1192static void asm_fpunary(ASMState *as, IRIns *ir, PPCIns pi)
1193{
1194 Reg dest = ra_dest(as, ir, RSET_FPR);
1195 Reg left = ra_hintalloc(as, ir->op1, dest, RSET_FPR);
1196 emit_fb(as, pi, dest, left);
1197}
1198
1199static int asm_fpjoin_pow(ASMState *as, IRIns *ir)
1200{
1201 IRIns *irp = IR(ir->op1);
1202 if (irp == ir-1 && irp->o == IR_MUL && !ra_used(irp)) {
1203 IRIns *irpp = IR(irp->op1);
1204 if (irpp == ir-2 && irpp->o == IR_FPMATH &&
1205 irpp->op2 == IRFPM_LOG2 && !ra_used(irpp)) {
1206 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_pow];
1207 IRRef args[2];
1208 args[0] = irpp->op1;
1209 args[1] = irp->op2;
1210 asm_setupresult(as, ir, ci);
1211 asm_gencall(as, ci, args);
1212 return 1;
1213 }
1214 }
1215 return 0;
1216}
1217
1218static void asm_add(ASMState *as, IRIns *ir)
1219{
1220 if (irt_isnum(ir->t)) {
1221 if (!asm_fusemadd(as, ir, PPCI_FMADD, PPCI_FMADD))
1222 asm_fparith(as, ir, PPCI_FADD);
1223 } else {
1224 Reg dest = ra_dest(as, ir, RSET_GPR);
1225 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1226 PPCIns pi;
1227 if (irref_isk(ir->op2)) {
1228 int32_t k = IR(ir->op2)->i;
1229 if (checki16(k)) {
1230 pi = PPCI_ADDI;
1231 /* May fail due to spills/restores above, but simplifies the logic. */
1232 if (as->flagmcp == as->mcp) {
1233 as->flagmcp = NULL;
1234 as->mcp++;
1235 pi = PPCI_ADDICDOT;
1236 }
1237 emit_tai(as, pi, dest, left, k);
1238 return;
1239 } else if ((k & 0xffff) == 0) {
1240 emit_tai(as, PPCI_ADDIS, dest, left, (k >> 16));
1241 return;
1242 } else if (!as->sectref) {
1243 emit_tai(as, PPCI_ADDIS, dest, dest, (k + 32768) >> 16);
1244 emit_tai(as, PPCI_ADDI, dest, left, k);
1245 return;
1246 }
1247 }
1248 pi = PPCI_ADD;
1249 /* May fail due to spills/restores above, but simplifies the logic. */
1250 if (as->flagmcp == as->mcp) {
1251 as->flagmcp = NULL;
1252 as->mcp++;
1253 pi |= PPCF_DOT;
1254 }
1255 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1256 emit_tab(as, pi, dest, left, right);
1257 }
1258}
1259
1260static void asm_sub(ASMState *as, IRIns *ir)
1261{
1262 if (irt_isnum(ir->t)) {
1263 if (!asm_fusemadd(as, ir, PPCI_FMSUB, PPCI_FNMSUB))
1264 asm_fparith(as, ir, PPCI_FSUB);
1265 } else {
1266 PPCIns pi = PPCI_SUBF;
1267 Reg dest = ra_dest(as, ir, RSET_GPR);
1268 Reg left, right;
1269 if (irref_isk(ir->op1)) {
1270 int32_t k = IR(ir->op1)->i;
1271 if (checki16(k)) {
1272 right = ra_alloc1(as, ir->op2, RSET_GPR);
1273 emit_tai(as, PPCI_SUBFIC, dest, right, k);
1274 return;
1275 }
1276 }
1277 /* May fail due to spills/restores above, but simplifies the logic. */
1278 if (as->flagmcp == as->mcp) {
1279 as->flagmcp = NULL;
1280 as->mcp++;
1281 pi |= PPCF_DOT;
1282 }
1283 left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1284 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1285 emit_tab(as, pi, dest, right, left); /* Subtract right _from_ left. */
1286 }
1287}
1288
1289static void asm_mul(ASMState *as, IRIns *ir)
1290{
1291 if (irt_isnum(ir->t)) {
1292 asm_fparith(as, ir, PPCI_FMUL);
1293 } else {
1294 PPCIns pi = PPCI_MULLW;
1295 Reg dest = ra_dest(as, ir, RSET_GPR);
1296 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1297 if (irref_isk(ir->op2)) {
1298 int32_t k = IR(ir->op2)->i;
1299 if (checki16(k)) {
1300 emit_tai(as, PPCI_MULLI, dest, left, k);
1301 return;
1302 }
1303 }
1304 /* May fail due to spills/restores above, but simplifies the logic. */
1305 if (as->flagmcp == as->mcp) {
1306 as->flagmcp = NULL;
1307 as->mcp++;
1308 pi |= PPCF_DOT;
1309 }
1310 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1311 emit_tab(as, pi, dest, left, right);
1312 }
1313}
1314
1315static void asm_neg(ASMState *as, IRIns *ir)
1316{
1317 if (irt_isnum(ir->t)) {
1318 asm_fpunary(as, ir, PPCI_FNEG);
1319 } else {
1320 Reg dest, left;
1321 PPCIns pi = PPCI_NEG;
1322 if (as->flagmcp == as->mcp) {
1323 as->flagmcp = NULL;
1324 as->mcp++;
1325 pi |= PPCF_DOT;
1326 }
1327 dest = ra_dest(as, ir, RSET_GPR);
1328 left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1329 emit_tab(as, pi, dest, left, 0);
1330 }
1331}
1332
1333static void asm_arithov(ASMState *as, IRIns *ir, PPCIns pi)
1334{
1335 Reg dest, left, right;
1336 if (as->flagmcp == as->mcp) {
1337 as->flagmcp = NULL;
1338 as->mcp++;
1339 }
1340 asm_guardcc(as, CC_SO);
1341 dest = ra_dest(as, ir, RSET_GPR);
1342 left = ra_alloc2(as, ir, RSET_GPR);
1343 right = (left >> 8); left &= 255;
1344 if (pi == PPCI_SUBFO) { Reg tmp = left; left = right; right = tmp; }
1345 emit_tab(as, pi|PPCF_DOT, dest, left, right);
1346}
1347
1348#if LJ_HASFFI
1349static void asm_add64(ASMState *as, IRIns *ir)
1350{
1351 Reg dest = ra_dest(as, ir, RSET_GPR);
1352 Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
1353 PPCIns pi = PPCI_ADDE;
1354 if (irref_isk(ir->op2)) {
1355 int32_t k = IR(ir->op2)->i;
1356 if (k == 0)
1357 pi = PPCI_ADDZE;
1358 else if (k == -1)
1359 pi = PPCI_ADDME;
1360 else
1361 goto needright;
1362 right = 0;
1363 } else {
1364 needright:
1365 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1366 }
1367 emit_tab(as, pi, dest, left, right);
1368 ir--;
1369 dest = ra_dest(as, ir, RSET_GPR);
1370 left = ra_alloc1(as, ir->op1, RSET_GPR);
1371 if (irref_isk(ir->op2)) {
1372 int32_t k = IR(ir->op2)->i;
1373 if (checki16(k)) {
1374 emit_tai(as, PPCI_ADDIC, dest, left, k);
1375 return;
1376 }
1377 }
1378 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1379 emit_tab(as, PPCI_ADDC, dest, left, right);
1380}
1381
1382static void asm_sub64(ASMState *as, IRIns *ir)
1383{
1384 Reg dest = ra_dest(as, ir, RSET_GPR);
1385 Reg left, right = ra_alloc1(as, ir->op2, RSET_GPR);
1386 PPCIns pi = PPCI_SUBFE;
1387 if (irref_isk(ir->op1)) {
1388 int32_t k = IR(ir->op1)->i;
1389 if (k == 0)
1390 pi = PPCI_SUBFZE;
1391 else if (k == -1)
1392 pi = PPCI_SUBFME;
1393 else
1394 goto needleft;
1395 left = 0;
1396 } else {
1397 needleft:
1398 left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, right));
1399 }
1400 emit_tab(as, pi, dest, right, left); /* Subtract right _from_ left. */
1401 ir--;
1402 dest = ra_dest(as, ir, RSET_GPR);
1403 right = ra_alloc1(as, ir->op2, RSET_GPR);
1404 if (irref_isk(ir->op1)) {
1405 int32_t k = IR(ir->op1)->i;
1406 if (checki16(k)) {
1407 emit_tai(as, PPCI_SUBFIC, dest, right, k);
1408 return;
1409 }
1410 }
1411 left = ra_alloc1(as, ir->op1, rset_exclude(RSET_GPR, right));
1412 emit_tab(as, PPCI_SUBFC, dest, right, left);
1413}
1414
1415static void asm_neg64(ASMState *as, IRIns *ir)
1416{
1417 Reg dest = ra_dest(as, ir, RSET_GPR);
1418 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1419 emit_tab(as, PPCI_SUBFZE, dest, left, 0);
1420 ir--;
1421 dest = ra_dest(as, ir, RSET_GPR);
1422 left = ra_alloc1(as, ir->op1, RSET_GPR);
1423 emit_tai(as, PPCI_SUBFIC, dest, left, 0);
1424}
1425#endif
1426
1427static void asm_bitnot(ASMState *as, IRIns *ir)
1428{
1429 Reg dest, left, right;
1430 PPCIns pi = PPCI_NOR;
1431 if (as->flagmcp == as->mcp) {
1432 as->flagmcp = NULL;
1433 as->mcp++;
1434 pi |= PPCF_DOT;
1435 }
1436 dest = ra_dest(as, ir, RSET_GPR);
1437 if (mayfuse(as, ir->op1)) {
1438 IRIns *irl = IR(ir->op1);
1439 if (irl->o == IR_BAND)
1440 pi ^= (PPCI_NOR ^ PPCI_NAND);
1441 else if (irl->o == IR_BXOR)
1442 pi ^= (PPCI_NOR ^ PPCI_EQV);
1443 else if (irl->o != IR_BOR)
1444 goto nofuse;
1445 left = ra_hintalloc(as, irl->op1, dest, RSET_GPR);
1446 right = ra_alloc1(as, irl->op2, rset_exclude(RSET_GPR, left));
1447 } else {
1448nofuse:
1449 left = right = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1450 }
1451 emit_asb(as, pi, dest, left, right);
1452}
1453
1454static void asm_bitswap(ASMState *as, IRIns *ir)
1455{
1456 Reg dest = ra_dest(as, ir, RSET_GPR);
1457 IRIns *irx;
1458 if (mayfuse(as, ir->op1) && (irx = IR(ir->op1))->o == IR_XLOAD &&
1459 ra_noreg(irx->r) && (irt_isint(irx->t) || irt_isu32(irx->t))) {
1460 /* Fuse BSWAP with XLOAD to lwbrx. */
1461 asm_fusexrefx(as, PPCI_LWBRX, dest, irx->op1, RSET_GPR);
1462 } else {
1463 Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
1464 Reg tmp = dest;
1465 if (tmp == left) {
1466 tmp = RID_TMP;
1467 emit_mr(as, dest, RID_TMP);
1468 }
1469 emit_rot(as, PPCI_RLWIMI, tmp, left, 24, 16, 23);
1470 emit_rot(as, PPCI_RLWIMI, tmp, left, 24, 0, 7);
1471 emit_rotlwi(as, tmp, left, 8);
1472 }
1473}
1474
1475static void asm_bitop(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1476{
1477 Reg dest = ra_dest(as, ir, RSET_GPR);
1478 Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
1479 if (irref_isk(ir->op2)) {
1480 int32_t k = IR(ir->op2)->i;
1481 Reg tmp = left;
1482 if ((checku16(k) || (k & 0xffff) == 0) || (tmp = dest, !as->sectref)) {
1483 if (!checku16(k)) {
1484 emit_asi(as, pik ^ (PPCI_ORI ^ PPCI_ORIS), dest, tmp, (k >> 16));
1485 if ((k & 0xffff) == 0) return;
1486 }
1487 emit_asi(as, pik, dest, left, k);
1488 return;
1489 }
1490 }
1491 /* May fail due to spills/restores above, but simplifies the logic. */
1492 if (as->flagmcp == as->mcp) {
1493 as->flagmcp = NULL;
1494 as->mcp++;
1495 pi |= PPCF_DOT;
1496 }
1497 right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1498 emit_asb(as, pi, dest, left, right);
1499}
1500
1501/* Fuse BAND with contiguous bitmask and a shift to rlwinm. */
1502static void asm_fuseandsh(ASMState *as, PPCIns pi, int32_t mask, IRRef ref)
1503{
1504 IRIns *ir;
1505 Reg left;
1506 if (mayfuse(as, ref) && (ir = IR(ref), ra_noreg(ir->r)) &&
1507 irref_isk(ir->op2) && ir->o >= IR_BSHL && ir->o <= IR_BROR) {
1508 int32_t sh = (IR(ir->op2)->i & 31);
1509 switch (ir->o) {
1510 case IR_BSHL:
1511 if ((mask & ((1u<<sh)-1))) goto nofuse;
1512 break;
1513 case IR_BSHR:
1514 if ((mask & ~((~0u)>>sh))) goto nofuse;
1515 sh = ((32-sh)&31);
1516 break;
1517 case IR_BROL:
1518 break;
1519 default:
1520 goto nofuse;
1521 }
1522 left = ra_alloc1(as, ir->op1, RSET_GPR);
1523 *--as->mcp = pi | PPCF_T(left) | PPCF_B(sh);
1524 return;
1525 }
1526nofuse:
1527 left = ra_alloc1(as, ref, RSET_GPR);
1528 *--as->mcp = pi | PPCF_T(left);
1529}
1530
1531static void asm_bitand(ASMState *as, IRIns *ir)
1532{
1533 Reg dest, left, right;
1534 IRRef lref = ir->op1;
1535 PPCIns dot = 0;
1536 IRRef op2;
1537 if (as->flagmcp == as->mcp) {
1538 as->flagmcp = NULL;
1539 as->mcp++;
1540 dot = PPCF_DOT;
1541 }
1542 dest = ra_dest(as, ir, RSET_GPR);
1543 if (irref_isk(ir->op2)) {
1544 int32_t k = IR(ir->op2)->i;
1545 if (k) {
1546 /* First check for a contiguous bitmask as used by rlwinm. */
1547 uint32_t s1 = lj_ffs((uint32_t)k);
1548 uint32_t k1 = ((uint32_t)k >> s1);
1549 if ((k1 & (k1+1)) == 0) {
1550 asm_fuseandsh(as, PPCI_RLWINM|dot | PPCF_A(dest) |
1551 PPCF_MB(31-lj_fls((uint32_t)k)) | PPCF_ME(31-s1),
1552 k, lref);
1553 return;
1554 }
1555 if (~(uint32_t)k) {
1556 uint32_t s2 = lj_ffs(~(uint32_t)k);
1557 uint32_t k2 = (~(uint32_t)k >> s2);
1558 if ((k2 & (k2+1)) == 0) {
1559 asm_fuseandsh(as, PPCI_RLWINM|dot | PPCF_A(dest) |
1560 PPCF_MB(32-s2) | PPCF_ME(30-lj_fls(~(uint32_t)k)),
1561 k, lref);
1562 return;
1563 }
1564 }
1565 }
1566 if (checku16(k)) {
1567 left = ra_alloc1(as, lref, RSET_GPR);
1568 emit_asi(as, PPCI_ANDIDOT, dest, left, k);
1569 return;
1570 } else if ((k & 0xffff) == 0) {
1571 left = ra_alloc1(as, lref, RSET_GPR);
1572 emit_asi(as, PPCI_ANDISDOT, dest, left, (k >> 16));
1573 return;
1574 }
1575 }
1576 op2 = ir->op2;
1577 if (mayfuse(as, op2) && IR(op2)->o == IR_BNOT && ra_noreg(IR(op2)->r)) {
1578 dot ^= (PPCI_AND ^ PPCI_ANDC);
1579 op2 = IR(op2)->op1;
1580 }
1581 left = ra_hintalloc(as, lref, dest, RSET_GPR);
1582 right = ra_alloc1(as, op2, rset_exclude(RSET_GPR, left));
1583 emit_asb(as, PPCI_AND ^ dot, dest, left, right);
1584}
1585
1586static void asm_bitshift(ASMState *as, IRIns *ir, PPCIns pi, PPCIns pik)
1587{
1588 Reg dest, left;
1589 Reg dot = 0;
1590 if (as->flagmcp == as->mcp) {
1591 as->flagmcp = NULL;
1592 as->mcp++;
1593 dot = PPCF_DOT;
1594 }
1595 dest = ra_dest(as, ir, RSET_GPR);
1596 left = ra_alloc1(as, ir->op1, RSET_GPR);
1597 if (irref_isk(ir->op2)) { /* Constant shifts. */
1598 int32_t shift = (IR(ir->op2)->i & 31);
1599 if (pik == 0) /* SLWI */
1600 emit_rot(as, PPCI_RLWINM|dot, dest, left, shift, 0, 31-shift);
1601 else if (pik == 1) /* SRWI */
1602 emit_rot(as, PPCI_RLWINM|dot, dest, left, (32-shift)&31, shift, 31);
1603 else
1604 emit_asb(as, pik|dot, dest, left, shift);
1605 } else {
1606 Reg right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
1607 emit_asb(as, pi|dot, dest, left, right);
1608 }
1609}
1610
1611static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
1612{
1613 if (irt_isnum(ir->t)) {
1614 Reg dest = ra_dest(as, ir, RSET_FPR);
1615 Reg tmp = dest;
1616 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1617 right = (left >> 8); left &= 255;
1618 if (tmp == left || tmp == right)
1619 tmp = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_FPR,
1620 dest), left), right));
1621 emit_facb(as, PPCI_FSEL, dest, tmp,
1622 ismax ? left : right, ismax ? right : left);
1623 emit_fab(as, PPCI_FSUB, tmp, left, right);
1624 } else {
1625 Reg dest = ra_dest(as, ir, RSET_GPR);
1626 Reg tmp1 = RID_TMP, tmp2 = dest;
1627 Reg right, left = ra_alloc2(as, ir, RSET_GPR);
1628 right = (left >> 8); left &= 255;
1629 if (tmp2 == left || tmp2 == right)
1630 tmp2 = ra_scratch(as, rset_exclude(rset_exclude(rset_exclude(RSET_GPR,
1631 dest), left), right));
1632 emit_tab(as, PPCI_ADD, dest, tmp2, right);
1633 emit_asb(as, ismax ? PPCI_ANDC : PPCI_AND, tmp2, tmp2, tmp1);
1634 emit_tab(as, PPCI_SUBFE, tmp1, tmp1, tmp1);
1635 emit_tab(as, PPCI_SUBFC, tmp2, tmp2, tmp1);
1636 emit_asi(as, PPCI_XORIS, tmp2, right, 0x8000);
1637 emit_asi(as, PPCI_XORIS, tmp1, left, 0x8000);
1638 }
1639}
1640
1641/* -- Comparisons --------------------------------------------------------- */
1642
1643#define CC_UNSIGNED 0x08 /* Unsigned integer comparison. */
1644#define CC_TWO 0x80 /* Check two flags for FP comparison. */
1645
1646/* Map of comparisons to flags. ORDER IR. */
1647static const uint8_t asm_compmap[IR_ABC+1] = {
1648 /* op int cc FP cc */
1649 /* LT */ CC_GE + (CC_GE<<4),
1650 /* GE */ CC_LT + (CC_LE<<4) + CC_TWO,
1651 /* LE */ CC_GT + (CC_GE<<4) + CC_TWO,
1652 /* GT */ CC_LE + (CC_LE<<4),
1653 /* ULT */ CC_GE + CC_UNSIGNED + (CC_GT<<4) + CC_TWO,
1654 /* UGE */ CC_LT + CC_UNSIGNED + (CC_LT<<4),
1655 /* ULE */ CC_GT + CC_UNSIGNED + (CC_GT<<4),
1656 /* UGT */ CC_LE + CC_UNSIGNED + (CC_LT<<4) + CC_TWO,
1657 /* EQ */ CC_NE + (CC_NE<<4),
1658 /* NE */ CC_EQ + (CC_EQ<<4),
1659 /* ABC */ CC_LE + CC_UNSIGNED + (CC_LT<<4) + CC_TWO /* Same as UGT. */
1660};
1661
1662static void asm_intcomp_(ASMState *as, IRRef lref, IRRef rref, Reg cr, PPCCC cc)
1663{
1664 Reg right, left = ra_alloc1(as, lref, RSET_GPR);
1665 if (irref_isk(rref)) {
1666 int32_t k = IR(rref)->i;
1667 if ((cc & CC_UNSIGNED) == 0) { /* Signed comparison with constant. */
1668 if (checki16(k)) {
1669 emit_tai(as, PPCI_CMPWI, cr, left, k);
1670 /* Signed comparison with zero and referencing previous ins? */
1671 if (k == 0 && lref == as->curins-1)
1672 as->flagmcp = as->mcp; /* Allow elimination of the compare. */
1673 return;
1674 } else if ((cc & 3) == (CC_EQ & 3)) { /* Use CMPLWI for EQ or NE. */
1675 if (checku16(k)) {
1676 emit_tai(as, PPCI_CMPLWI, cr, left, k);
1677 return;
1678 } else if (!as->sectref && ra_noreg(IR(rref)->r)) {
1679 emit_tai(as, PPCI_CMPLWI, cr, RID_TMP, k);
1680 emit_asi(as, PPCI_XORIS, RID_TMP, left, (k >> 16));
1681 return;
1682 }
1683 }
1684 } else { /* Unsigned comparison with constant. */
1685 if (checku16(k)) {
1686 emit_tai(as, PPCI_CMPLWI, cr, left, k);
1687 return;
1688 }
1689 }
1690 }
1691 right = ra_alloc1(as, rref, rset_exclude(RSET_GPR, left));
1692 emit_tab(as, (cc & CC_UNSIGNED) ? PPCI_CMPLW : PPCI_CMPW, cr, left, right);
1693}
1694
1695static void asm_comp(ASMState *as, IRIns *ir)
1696{
1697 PPCCC cc = asm_compmap[ir->o];
1698 if (irt_isnum(ir->t)) {
1699 Reg right, left = ra_alloc2(as, ir, RSET_FPR);
1700 right = (left >> 8); left &= 255;
1701 asm_guardcc(as, (cc >> 4));
1702 if ((cc & CC_TWO))
1703 emit_tab(as, PPCI_CROR, ((cc>>4)&3), ((cc>>4)&3), (CC_EQ&3));
1704 emit_fab(as, PPCI_FCMPU, 0, left, right);
1705 } else {
1706 IRRef lref = ir->op1, rref = ir->op2;
1707 if (irref_isk(lref) && !irref_isk(rref)) {
1708 /* Swap constants to the right (only for ABC). */
1709 IRRef tmp = lref; lref = rref; rref = tmp;
1710 if ((cc & 2) == 0) cc ^= 1; /* LT <-> GT, LE <-> GE */
1711 }
1712 asm_guardcc(as, cc);
1713 asm_intcomp_(as, lref, rref, 0, cc);
1714 }
1715}
1716
1717#if LJ_HASFFI
1718/* 64 bit integer comparisons. */
1719static void asm_comp64(ASMState *as, IRIns *ir)
1720{
1721 PPCCC cc = asm_compmap[(ir-1)->o];
1722 if ((cc&3) == (CC_EQ&3)) {
1723 asm_guardcc(as, cc);
1724 emit_tab(as, (cc&4) ? PPCI_CRAND : PPCI_CROR,
1725 (CC_EQ&3), (CC_EQ&3), 4+(CC_EQ&3));
1726 } else {
1727 asm_guardcc(as, CC_EQ);
1728 emit_tab(as, PPCI_CROR, (CC_EQ&3), (CC_EQ&3), ((cc^~(cc>>2))&1));
1729 emit_tab(as, (cc&4) ? PPCI_CRAND : PPCI_CRANDC,
1730 (CC_EQ&3), (CC_EQ&3), 4+(cc&3));
1731 }
1732 /* Loword comparison sets cr1 and is unsigned, except for equality. */
1733 asm_intcomp_(as, (ir-1)->op1, (ir-1)->op2, 4,
1734 cc | ((cc&3) == (CC_EQ&3) ? 0 : CC_UNSIGNED));
1735 /* Hiword comparison sets cr0. */
1736 asm_intcomp_(as, ir->op1, ir->op2, 0, cc);
1737 as->flagmcp = NULL; /* Doesn't work here. */
1738}
1739#endif
1740
1741/* -- Support for 64 bit ops in 32 bit mode ------------------------------- */
1742
1743/* Hiword op of a split 64 bit op. Previous op must be the loword op. */
1744static void asm_hiop(ASMState *as, IRIns *ir)
1745{
1746#if LJ_HASFFI
1747 /* HIOP is marked as a store because it needs its own DCE logic. */
1748 int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
1749 if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
1750 if ((ir-1)->o == IR_CONV) { /* Conversions to/from 64 bit. */
1751 as->curins--; /* Always skip the CONV. */
1752 if (usehi || uselo)
1753 asm_conv64(as, ir);
1754 return;
1755 } else if ((ir-1)->o <= IR_NE) { /* 64 bit integer comparisons. ORDER IR. */
1756 as->curins--; /* Always skip the loword comparison. */
1757 asm_comp64(as, ir);
1758 return;
1759 } else if ((ir-1)->o == IR_XSTORE) {
1760 as->curins--; /* Handle both stores here. */
1761 if ((ir-1)->r != RID_SINK) {
1762 asm_xstore(as, ir, 0);
1763 asm_xstore(as, ir-1, 4);
1764 }
1765 return;
1766 }
1767 if (!usehi) return; /* Skip unused hiword op for all remaining ops. */
1768 switch ((ir-1)->o) {
1769 case IR_ADD: as->curins--; asm_add64(as, ir); break;
1770 case IR_SUB: as->curins--; asm_sub64(as, ir); break;
1771 case IR_NEG: as->curins--; asm_neg64(as, ir); break;
1772 case IR_CALLN:
1773 case IR_CALLXS:
1774 if (!uselo)
1775 ra_allocref(as, ir->op1, RID2RSET(RID_RETLO)); /* Mark lo op as used. */
1776 break;
1777 case IR_CNEWI:
1778 /* Nothing to do here. Handled by lo op itself. */
1779 break;
1780 default: lua_assert(0); break;
1781 }
1782#else
1783 UNUSED(as); UNUSED(ir); lua_assert(0); /* Unused without FFI. */
1784#endif
1785}
1786
1787/* -- Stack handling ------------------------------------------------------ */
1788
1789/* Check Lua stack size for overflow. Use exit handler as fallback. */
1790static void asm_stack_check(ASMState *as, BCReg topslot,
1791 IRIns *irp, RegSet allow, ExitNo exitno)
1792{
1793 /* Try to get an unused temp. register, otherwise spill/restore RID_RET*. */
1794 Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
1795 rset_clear(allow, pbase);
1796 tmp = allow ? rset_pickbot(allow) :
1797 (pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
1798 emit_condbranch(as, PPCI_BC, CC_LT, asm_exitstub_addr(as, exitno));
1799 if (allow == RSET_EMPTY) /* Restore temp. register. */
1800 emit_tai(as, PPCI_LWZ, tmp, RID_SP, SPOFS_TMPW);
1801 else
1802 ra_modified(as, tmp);
1803 emit_ai(as, PPCI_CMPLWI, RID_TMP, (int32_t)(8*topslot));
1804 emit_tab(as, PPCI_SUBF, RID_TMP, pbase, tmp);
1805 emit_tai(as, PPCI_LWZ, tmp, tmp, offsetof(lua_State, maxstack));
1806 if (pbase == RID_TMP)
1807 emit_getgl(as, RID_TMP, jit_base);
1808 emit_getgl(as, tmp, jit_L);
1809 if (allow == RSET_EMPTY) /* Spill temp. register. */
1810 emit_tai(as, PPCI_STW, tmp, RID_SP, SPOFS_TMPW);
1811}
1812
1813/* Restore Lua stack from on-trace state. */
1814static void asm_stack_restore(ASMState *as, SnapShot *snap)
1815{
1816 SnapEntry *map = &as->T->snapmap[snap->mapofs];
1817 SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
1818 MSize n, nent = snap->nent;
1819 /* Store the value of all modified slots to the Lua stack. */
1820 for (n = 0; n < nent; n++) {
1821 SnapEntry sn = map[n];
1822 BCReg s = snap_slot(sn);
1823 int32_t ofs = 8*((int32_t)s-1);
1824 IRRef ref = snap_ref(sn);
1825 IRIns *ir = IR(ref);
1826 if ((sn & SNAP_NORESTORE))
1827 continue;
1828 if (irt_isnum(ir->t)) {
1829 Reg src = ra_alloc1(as, ref, RSET_FPR);
1830 emit_fai(as, PPCI_STFD, src, RID_BASE, ofs);
1831 } else {
1832 Reg type;
1833 RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
1834 lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
1835 if (!irt_ispri(ir->t)) {
1836 Reg src = ra_alloc1(as, ref, allow);
1837 rset_clear(allow, src);
1838 emit_tai(as, PPCI_STW, src, RID_BASE, ofs+4);
1839 }
1840 if ((sn & (SNAP_CONT|SNAP_FRAME))) {
1841 if (s == 0) continue; /* Do not overwrite link to previous frame. */
1842 type = ra_allock(as, (int32_t)(*flinks--), allow);
1843 } else {
1844 type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
1845 }
1846 emit_tai(as, PPCI_STW, type, RID_BASE, ofs);
1847 }
1848 checkmclim(as);
1849 }
1850 lua_assert(map + nent == flinks);
1851}
1852
1853/* -- GC handling --------------------------------------------------------- */
1854
1855/* Marker to prevent patching the GC check exit. */
1856#define PPC_NOPATCH_GC_CHECK PPCI_ORIS
1857
1858/* Check GC threshold and do one or more GC steps. */
1859static void asm_gc_check(ASMState *as)
1860{
1861 const CCallInfo *ci = &lj_ir_callinfo[IRCALL_lj_gc_step_jit];
1862 IRRef args[2];
1863 MCLabel l_end;
1864 Reg tmp;
1865 ra_evictset(as, RSET_SCRATCH);
1866 l_end = emit_label(as);
1867 /* Exit trace if in GCSatomic or GCSfinalize. Avoids syncing GC objects. */
1868 asm_guardcc(as, CC_NE); /* Assumes asm_snap_prep() already done. */
1869 *--as->mcp = PPC_NOPATCH_GC_CHECK;
1870 emit_ai(as, PPCI_CMPWI, RID_RET, 0);
1871 args[0] = ASMREF_TMP1; /* global_State *g */
1872 args[1] = ASMREF_TMP2; /* MSize steps */
1873 asm_gencall(as, ci, args);
1874 emit_tai(as, PPCI_ADDI, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
1875 tmp = ra_releasetmp(as, ASMREF_TMP2);
1876 emit_loadi(as, tmp, as->gcsteps);
1877 /* Jump around GC step if GC total < GC threshold. */
1878 emit_condbranch(as, PPCI_BC|PPCF_Y, CC_LT, l_end);
1879 emit_ab(as, PPCI_CMPLW, RID_TMP, tmp);
1880 emit_getgl(as, tmp, gc.threshold);
1881 emit_getgl(as, RID_TMP, gc.total);
1882 as->gcsteps = 0;
1883 checkmclim(as);
1884}
1885
1886/* -- Loop handling ------------------------------------------------------- */
1887
1888/* Fixup the loop branch. */
1889static void asm_loop_fixup(ASMState *as)
1890{
1891 MCode *p = as->mctop;
1892 MCode *target = as->mcp;
1893 if (as->loopinv) { /* Inverted loop branch? */
1894 /* asm_guardcc already inverted the cond branch and patched the final b. */
1895 p[-2] = (p[-2] & (0xffff0000u & ~PPCF_Y)) | (((target-p+2) & 0x3fffu) << 2);
1896 } else {
1897 p[-1] = PPCI_B|(((target-p+1)&0x00ffffffu)<<2);
1898 }
1899}
1900
1901/* -- Head of trace ------------------------------------------------------- */
1902
1903/* Coalesce BASE register for a root trace. */
1904static void asm_head_root_base(ASMState *as)
1905{
1906 IRIns *ir = IR(REF_BASE);
1907 Reg r = ir->r;
1908 if (ra_hasreg(r)) {
1909 ra_free(as, r);
1910 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1911 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1912 if (r != RID_BASE)
1913 emit_mr(as, r, RID_BASE);
1914 }
1915}
1916
1917/* Coalesce BASE register for a side trace. */
1918static Reg asm_head_side_base(ASMState *as, IRIns *irp)
1919{
1920 IRIns *ir = IR(REF_BASE);
1921 Reg r = ir->r;
1922 if (ra_hasreg(r)) {
1923 ra_free(as, r);
1924 if (rset_test(as->modset, r) || irt_ismarked(ir->t))
1925 ir->r = RID_INIT; /* No inheritance for modified BASE register. */
1926 if (irp->r == r) {
1927 return r; /* Same BASE register already coalesced. */
1928 } else if (ra_hasreg(irp->r) && rset_test(as->freeset, irp->r)) {
1929 emit_mr(as, r, irp->r); /* Move from coalesced parent reg. */
1930 return irp->r;
1931 } else {
1932 emit_getgl(as, r, jit_base); /* Otherwise reload BASE. */
1933 }
1934 }
1935 return RID_NONE;
1936}
1937
1938/* -- Tail of trace ------------------------------------------------------- */
1939
1940/* Fixup the tail code. */
1941static void asm_tail_fixup(ASMState *as, TraceNo lnk)
1942{
1943 MCode *p = as->mctop;
1944 MCode *target;
1945 int32_t spadj = as->T->spadjust;
1946 if (spadj == 0) {
1947 *--p = PPCI_NOP;
1948 *--p = PPCI_NOP;
1949 as->mctop = p;
1950 } else {
1951 /* Patch stack adjustment. */
1952 lua_assert(checki16(CFRAME_SIZE+spadj));
1953 p[-3] = PPCI_ADDI | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | (CFRAME_SIZE+spadj);
1954 p[-2] = PPCI_STWU | PPCF_T(RID_TMP) | PPCF_A(RID_SP) | spadj;
1955 }
1956 /* Patch exit branch. */
1957 target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp;
1958 p[-1] = PPCI_B|(((target-p+1)&0x00ffffffu)<<2);
1959}
1960
1961/* Prepare tail of code. */
1962static void asm_tail_prep(ASMState *as)
1963{
1964 MCode *p = as->mctop - 1; /* Leave room for exit branch. */
1965 if (as->loopref) {
1966 as->invmcp = as->mcp = p;
1967 } else {
1968 as->mcp = p-2; /* Leave room for stack pointer adjustment. */
1969 as->invmcp = NULL;
1970 }
1971}
1972
1973/* -- Instruction dispatch ------------------------------------------------ */
1974
1975/* Assemble a single instruction. */
1976static void asm_ir(ASMState *as, IRIns *ir)
1977{
1978 switch ((IROp)ir->o) {
1979 /* Miscellaneous ops. */
1980 case IR_LOOP: asm_loop(as); break;
1981 case IR_NOP: case IR_XBAR: lua_assert(!ra_used(ir)); break;
1982 case IR_USE:
1983 ra_alloc1(as, ir->op1, irt_isfp(ir->t) ? RSET_FPR : RSET_GPR); break;
1984 case IR_PHI: asm_phi(as, ir); break;
1985 case IR_HIOP: asm_hiop(as, ir); break;
1986 case IR_GCSTEP: asm_gcstep(as, ir); break;
1987
1988 /* Guarded assertions. */
1989 case IR_EQ: case IR_NE:
1990 if ((ir-1)->o == IR_HREF && ir->op1 == as->curins-1) {
1991 as->curins--;
1992 asm_href(as, ir-1, (IROp)ir->o);
1993 break;
1994 }
1995 /* fallthrough */
1996 case IR_LT: case IR_GE: case IR_LE: case IR_GT:
1997 case IR_ULT: case IR_UGE: case IR_ULE: case IR_UGT:
1998 case IR_ABC:
1999 asm_comp(as, ir);
2000 break;
2001
2002 case IR_RETF: asm_retf(as, ir); break;
2003
2004 /* Bit ops. */
2005 case IR_BNOT: asm_bitnot(as, ir); break;
2006 case IR_BSWAP: asm_bitswap(as, ir); break;
2007
2008 case IR_BAND: asm_bitand(as, ir); break;
2009 case IR_BOR: asm_bitop(as, ir, PPCI_OR, PPCI_ORI); break;
2010 case IR_BXOR: asm_bitop(as, ir, PPCI_XOR, PPCI_XORI); break;
2011
2012 case IR_BSHL: asm_bitshift(as, ir, PPCI_SLW, 0); break;
2013 case IR_BSHR: asm_bitshift(as, ir, PPCI_SRW, 1); break;
2014 case IR_BSAR: asm_bitshift(as, ir, PPCI_SRAW, PPCI_SRAWI); break;
2015 case IR_BROL: asm_bitshift(as, ir, PPCI_RLWNM|PPCF_MB(0)|PPCF_ME(31),
2016 PPCI_RLWINM|PPCF_MB(0)|PPCF_ME(31)); break;
2017 case IR_BROR: lua_assert(0); break;
2018
2019 /* Arithmetic ops. */
2020 case IR_ADD: asm_add(as, ir); break;
2021 case IR_SUB: asm_sub(as, ir); break;
2022 case IR_MUL: asm_mul(as, ir); break;
2023 case IR_DIV: asm_fparith(as, ir, PPCI_FDIV); break;
2024 case IR_MOD: asm_callid(as, ir, IRCALL_lj_vm_modi); break;
2025 case IR_POW: asm_callid(as, ir, IRCALL_lj_vm_powi); break;
2026 case IR_NEG: asm_neg(as, ir); break;
2027
2028 case IR_ABS: asm_fpunary(as, ir, PPCI_FABS); break;
2029 case IR_ATAN2: asm_callid(as, ir, IRCALL_atan2); break;
2030 case IR_LDEXP: asm_callid(as, ir, IRCALL_ldexp); break;
2031 case IR_MIN: asm_min_max(as, ir, 0); break;
2032 case IR_MAX: asm_min_max(as, ir, 1); break;
2033 case IR_FPMATH:
2034 if (ir->op2 == IRFPM_EXP2 && asm_fpjoin_pow(as, ir))
2035 break;
2036 if (ir->op2 == IRFPM_SQRT && (as->flags & JIT_F_SQRT))
2037 asm_fpunary(as, ir, PPCI_FSQRT);
2038 else
2039 asm_callid(as, ir, IRCALL_lj_vm_floor + ir->op2);
2040 break;
2041
2042 /* Overflow-checking arithmetic ops. */
2043 case IR_ADDOV: asm_arithov(as, ir, PPCI_ADDO); break;
2044 case IR_SUBOV: asm_arithov(as, ir, PPCI_SUBFO); break;
2045 case IR_MULOV: asm_arithov(as, ir, PPCI_MULLWO); break;
2046
2047 /* Memory references. */
2048 case IR_AREF: asm_aref(as, ir); break;
2049 case IR_HREF: asm_href(as, ir, 0); break;
2050 case IR_HREFK: asm_hrefk(as, ir); break;
2051 case IR_NEWREF: asm_newref(as, ir); break;
2052 case IR_UREFO: case IR_UREFC: asm_uref(as, ir); break;
2053 case IR_FREF: asm_fref(as, ir); break;
2054 case IR_STRREF: asm_strref(as, ir); break;
2055
2056 /* Loads and stores. */
2057 case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
2058 asm_ahuvload(as, ir);
2059 break;
2060 case IR_FLOAD: asm_fload(as, ir); break;
2061 case IR_XLOAD: asm_xload(as, ir); break;
2062 case IR_SLOAD: asm_sload(as, ir); break;
2063
2064 case IR_ASTORE: case IR_HSTORE: case IR_USTORE: asm_ahustore(as, ir); break;
2065 case IR_FSTORE: asm_fstore(as, ir); break;
2066 case IR_XSTORE: asm_xstore(as, ir, 0); break;
2067
2068 /* Allocations. */
2069 case IR_SNEW: case IR_XSNEW: asm_snew(as, ir); break;
2070 case IR_TNEW: asm_tnew(as, ir); break;
2071 case IR_TDUP: asm_tdup(as, ir); break;
2072 case IR_CNEW: case IR_CNEWI: asm_cnew(as, ir); break;
2073
2074 /* Write barriers. */
2075 case IR_TBAR: asm_tbar(as, ir); break;
2076 case IR_OBAR: asm_obar(as, ir); break;
2077
2078 /* Type conversions. */
2079 case IR_CONV: asm_conv(as, ir); break;
2080 case IR_TOBIT: asm_tobit(as, ir); break;
2081 case IR_TOSTR: asm_tostr(as, ir); break;
2082 case IR_STRTO: asm_strto(as, ir); break;
2083
2084 /* Calls. */
2085 case IR_CALLN: case IR_CALLL: case IR_CALLS: asm_call(as, ir); break;
2086 case IR_CALLXS: asm_callx(as, ir); break;
2087 case IR_CARG: break;
2088
2089 default:
2090 setintV(&as->J->errinfo, ir->o);
2091 lj_trace_err_info(as->J, LJ_TRERR_NYIIR);
2092 break;
2093 }
2094}
2095
2096/* -- Trace setup --------------------------------------------------------- */
2097
2098/* Ensure there are enough stack slots for call arguments. */
2099static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
2100{
2101 IRRef args[CCI_NARGS_MAX*2];
2102 uint32_t i, nargs = (int)CCI_NARGS(ci);
2103 int nslots = 2, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
2104 asm_collectargs(as, ir, ci, args);
2105 for (i = 0; i < nargs; i++)
2106 if (args[i] && irt_isfp(IR(args[i])->t)) {
2107 if (nfpr > 0) nfpr--; else nslots = (nslots+3) & ~1;
2108 } else {
2109 if (ngpr > 0) ngpr--; else nslots++;
2110 }
2111 if (nslots > as->evenspill) /* Leave room for args in stack slots. */
2112 as->evenspill = nslots;
2113 return irt_isfp(ir->t) ? REGSP_HINT(RID_FPRET) : REGSP_HINT(RID_RET);
2114}
2115
2116static void asm_setup_target(ASMState *as)
2117{
2118 asm_exitstub_setup(as, as->T->nsnap + (as->parent ? 1 : 0));
2119}
2120
2121/* -- Trace patching ------------------------------------------------------ */
2122
2123/* Patch exit jumps of existing machine code to a new target. */
2124void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
2125{
2126 MCode *p = T->mcode;
2127 MCode *pe = (MCode *)((char *)p + T->szmcode);
2128 MCode *px = exitstub_trace_addr(T, exitno);
2129 MCode *cstart = NULL;
2130 MCode *mcarea = lj_mcode_patch(J, p, 0);
2131 int clearso = 0, patchlong = 1;
2132 for (; p < pe; p++) {
2133 /* Look for exitstub branch, try to replace with branch to target. */
2134 uint32_t ins = *p;
2135 if ((ins & 0xfc000000u) == 0x40000000u &&
2136 ((ins ^ ((char *)px-(char *)p)) & 0xffffu) == 0) {
2137 ptrdiff_t delta = (char *)target - (char *)p;
2138 if (((ins >> 16) & 3) == (CC_SO&3)) {
2139 clearso = sizeof(MCode);
2140 delta -= sizeof(MCode);
2141 }
2142 /* Many, but not all short-range branches can be patched directly. */
2143 if (p[-1] == PPC_NOPATCH_GC_CHECK) {
2144 patchlong = 0;
2145 } else if (((delta + 0x8000) >> 16) == 0) {
2146 *p = (ins & 0xffdf0000u) | ((uint32_t)delta & 0xffffu) |
2147 ((delta & 0x8000) * (PPCF_Y/0x8000));
2148 if (!cstart) cstart = p;
2149 }
2150 } else if ((ins & 0xfc000000u) == PPCI_B &&
2151 ((ins ^ ((char *)px-(char *)p)) & 0x03ffffffu) == 0) {
2152 ptrdiff_t delta = (char *)target - (char *)p;
2153 lua_assert(((delta + 0x02000000) >> 26) == 0);
2154 *p = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2155 if (!cstart) cstart = p;
2156 }
2157 }
2158 /* Always patch long-range branch in exit stub itself. Except, if we can't. */
2159 if (patchlong) {
2160 ptrdiff_t delta = (char *)target - (char *)px - clearso;
2161 lua_assert(((delta + 0x02000000) >> 26) == 0);
2162 *px = PPCI_B | ((uint32_t)delta & 0x03ffffffu);
2163 }
2164 if (!cstart) cstart = px;
2165 lj_mcode_sync(cstart, px+1);
2166 if (clearso) { /* Extend the current trace. Ugly workaround. */
2167 MCode *pp = J->cur.mcode;
2168 J->cur.szmcode += sizeof(MCode);
2169 *--pp = PPCI_MCRXR; /* Clear SO flag. */
2170 J->cur.mcode = pp;
2171 lj_mcode_sync(pp, pp+1);
2172 }
2173 lj_mcode_patch(J, mcarea, 1);
2174}
2175