arm64-gen.c 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847
  1. /*
  2. * A64 code generator for TCC
  3. *
  4. * Copyright (c) 2014-2015 Edmund Grimley Evans
  5. *
  6. * Copying and distribution of this file, with or without modification,
  7. * are permitted in any medium without royalty provided the copyright
  8. * notice and this notice are preserved. This file is offered as-is,
  9. * without any warranty.
  10. */
  11. #ifdef TARGET_DEFS_ONLY
  12. // Number of registers available to allocator:
  13. #define NB_REGS 28 // x0-x18, x30, v0-v7
  14. #define TREG_R(x) (x) // x = 0..18
  15. #define TREG_R30 19
  16. #define TREG_F(x) (x + 20) // x = 0..7
  17. // Register classes sorted from more general to more precise:
  18. #define RC_INT (1 << 0)
  19. #define RC_FLOAT (1 << 1)
  20. #define RC_R(x) (1 << (2 + (x))) // x = 0..18
  21. #define RC_R30 (1 << 21)
  22. #define RC_F(x) (1 << (22 + (x))) // x = 0..7
  23. #define RC_IRET (RC_R(0)) // int return register class
  24. #define RC_FRET (RC_F(0)) // float return register class
  25. #define REG_IRET (TREG_R(0)) // int return register number
  26. #define REG_FRET (TREG_F(0)) // float return register number
  27. #define PTR_SIZE 8
  28. #define LDOUBLE_SIZE 16
  29. #define LDOUBLE_ALIGN 16
  30. #define MAX_ALIGN 16
  31. #define CHAR_IS_UNSIGNED
  32. /******************************************************/
  33. #else /* ! TARGET_DEFS_ONLY */
  34. /******************************************************/
  35. #include "tcc.h"
  36. #include <assert.h>
  37. ST_DATA const int reg_classes[NB_REGS] = {
  38. RC_INT | RC_R(0),
  39. RC_INT | RC_R(1),
  40. RC_INT | RC_R(2),
  41. RC_INT | RC_R(3),
  42. RC_INT | RC_R(4),
  43. RC_INT | RC_R(5),
  44. RC_INT | RC_R(6),
  45. RC_INT | RC_R(7),
  46. RC_INT | RC_R(8),
  47. RC_INT | RC_R(9),
  48. RC_INT | RC_R(10),
  49. RC_INT | RC_R(11),
  50. RC_INT | RC_R(12),
  51. RC_INT | RC_R(13),
  52. RC_INT | RC_R(14),
  53. RC_INT | RC_R(15),
  54. RC_INT | RC_R(16),
  55. RC_INT | RC_R(17),
  56. RC_INT | RC_R(18),
  57. RC_R30, // not in RC_INT as we make special use of x30
  58. RC_FLOAT | RC_F(0),
  59. RC_FLOAT | RC_F(1),
  60. RC_FLOAT | RC_F(2),
  61. RC_FLOAT | RC_F(3),
  62. RC_FLOAT | RC_F(4),
  63. RC_FLOAT | RC_F(5),
  64. RC_FLOAT | RC_F(6),
  65. RC_FLOAT | RC_F(7)
  66. };
  67. #define IS_FREG(x) ((x) >= TREG_F(0))
  68. static uint32_t intr(int r)
  69. {
  70. assert(TREG_R(0) <= r && r <= TREG_R30);
  71. return r < TREG_R30 ? r : 30;
  72. }
  73. static uint32_t fltr(int r)
  74. {
  75. assert(TREG_F(0) <= r && r <= TREG_F(7));
  76. return r - TREG_F(0);
  77. }
  78. // Add an instruction to text section:
  79. ST_FUNC void o(unsigned int c)
  80. {
  81. int ind1 = ind + 4;
  82. if (nocode_wanted)
  83. return;
  84. if (ind1 > cur_text_section->data_allocated)
  85. section_realloc(cur_text_section, ind1);
  86. write32le(cur_text_section->data + ind, c);
  87. ind = ind1;
  88. }
  89. static int arm64_encode_bimm64(uint64_t x)
  90. {
  91. int neg = x & 1;
  92. int rep, pos, len;
  93. if (neg)
  94. x = ~x;
  95. if (!x)
  96. return -1;
  97. if (x >> 2 == (x & (((uint64_t)1 << (64 - 2)) - 1)))
  98. rep = 2, x &= ((uint64_t)1 << 2) - 1;
  99. else if (x >> 4 == (x & (((uint64_t)1 << (64 - 4)) - 1)))
  100. rep = 4, x &= ((uint64_t)1 << 4) - 1;
  101. else if (x >> 8 == (x & (((uint64_t)1 << (64 - 8)) - 1)))
  102. rep = 8, x &= ((uint64_t)1 << 8) - 1;
  103. else if (x >> 16 == (x & (((uint64_t)1 << (64 - 16)) - 1)))
  104. rep = 16, x &= ((uint64_t)1 << 16) - 1;
  105. else if (x >> 32 == (x & (((uint64_t)1 << (64 - 32)) - 1)))
  106. rep = 32, x &= ((uint64_t)1 << 32) - 1;
  107. else
  108. rep = 64;
  109. pos = 0;
  110. if (!(x & (((uint64_t)1 << 32) - 1))) x >>= 32, pos += 32;
  111. if (!(x & (((uint64_t)1 << 16) - 1))) x >>= 16, pos += 16;
  112. if (!(x & (((uint64_t)1 << 8) - 1))) x >>= 8, pos += 8;
  113. if (!(x & (((uint64_t)1 << 4) - 1))) x >>= 4, pos += 4;
  114. if (!(x & (((uint64_t)1 << 2) - 1))) x >>= 2, pos += 2;
  115. if (!(x & (((uint64_t)1 << 1) - 1))) x >>= 1, pos += 1;
  116. len = 0;
  117. if (!(~x & (((uint64_t)1 << 32) - 1))) x >>= 32, len += 32;
  118. if (!(~x & (((uint64_t)1 << 16) - 1))) x >>= 16, len += 16;
  119. if (!(~x & (((uint64_t)1 << 8) - 1))) x >>= 8, len += 8;
  120. if (!(~x & (((uint64_t)1 << 4) - 1))) x >>= 4, len += 4;
  121. if (!(~x & (((uint64_t)1 << 2) - 1))) x >>= 2, len += 2;
  122. if (!(~x & (((uint64_t)1 << 1) - 1))) x >>= 1, len += 1;
  123. if (x)
  124. return -1;
  125. if (neg) {
  126. pos = (pos + len) & (rep - 1);
  127. len = rep - len;
  128. }
  129. return ((0x1000 & rep << 6) | (((rep - 1) ^ 31) << 1 & 63) |
  130. ((rep - pos) & (rep - 1)) << 6 | (len - 1));
  131. }
  132. static uint32_t arm64_movi(int r, uint64_t x)
  133. {
  134. uint64_t m = 0xffff;
  135. int e;
  136. if (!(x & ~m))
  137. return 0x52800000 | r | x << 5; // movz w(r),#(x)
  138. if (!(x & ~(m << 16)))
  139. return 0x52a00000 | r | x >> 11; // movz w(r),#(x >> 16),lsl #16
  140. if (!(x & ~(m << 32)))
  141. return 0xd2c00000 | r | x >> 27; // movz x(r),#(x >> 32),lsl #32
  142. if (!(x & ~(m << 48)))
  143. return 0xd2e00000 | r | x >> 43; // movz x(r),#(x >> 48),lsl #48
  144. if ((x & ~m) == m << 16)
  145. return (0x12800000 | r |
  146. (~x << 5 & 0x1fffe0)); // movn w(r),#(~x)
  147. if ((x & ~(m << 16)) == m)
  148. return (0x12a00000 | r |
  149. (~x >> 11 & 0x1fffe0)); // movn w(r),#(~x >> 16),lsl #16
  150. if (!~(x | m))
  151. return (0x92800000 | r |
  152. (~x << 5 & 0x1fffe0)); // movn x(r),#(~x)
  153. if (!~(x | m << 16))
  154. return (0x92a00000 | r |
  155. (~x >> 11 & 0x1fffe0)); // movn x(r),#(~x >> 16),lsl #16
  156. if (!~(x | m << 32))
  157. return (0x92c00000 | r |
  158. (~x >> 27 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
  159. if (!~(x | m << 48))
  160. return (0x92e00000 | r |
  161. (~x >> 43 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32
  162. if (!(x >> 32) && (e = arm64_encode_bimm64(x | x << 32)) >= 0)
  163. return 0x320003e0 | r | (uint32_t)e << 10; // movi w(r),#(x)
  164. if ((e = arm64_encode_bimm64(x)) >= 0)
  165. return 0xb20003e0 | r | (uint32_t)e << 10; // movi x(r),#(x)
  166. return 0;
  167. }
  168. static void arm64_movimm(int r, uint64_t x)
  169. {
  170. uint32_t i;
  171. if ((i = arm64_movi(r, x)))
  172. o(i); // a single MOV
  173. else {
  174. // MOVZ/MOVN and 1-3 MOVKs
  175. int z = 0, m = 0;
  176. uint32_t mov1 = 0xd2800000; // movz
  177. uint64_t x1 = x;
  178. for (i = 0; i < 64; i += 16) {
  179. z += !(x >> i & 0xffff);
  180. m += !(~x >> i & 0xffff);
  181. }
  182. if (m > z) {
  183. x1 = ~x;
  184. mov1 = 0x92800000; // movn
  185. }
  186. for (i = 0; i < 64; i += 16)
  187. if (x1 >> i & 0xffff) {
  188. o(mov1 | r | (x1 >> i & 0xffff) << 5 | i << 17);
  189. // movz/movn x(r),#(*),lsl #(i)
  190. break;
  191. }
  192. for (i += 16; i < 64; i += 16)
  193. if (x1 >> i & 0xffff)
  194. o(0xf2800000 | r | (x >> i & 0xffff) << 5 | i << 17);
  195. // movk x(r),#(*),lsl #(i)
  196. }
  197. }
  198. // Patch all branches in list pointed to by t to branch to a:
  199. ST_FUNC void gsym_addr(int t_, int a_)
  200. {
  201. uint32_t t = t_;
  202. uint32_t a = a_;
  203. while (t) {
  204. unsigned char *ptr = cur_text_section->data + t;
  205. uint32_t next = read32le(ptr);
  206. if (a - t + 0x8000000 >= 0x10000000)
  207. tcc_error("branch out of range");
  208. write32le(ptr, (a - t == 4 ? 0xd503201f : // nop
  209. 0x14000000 | ((a - t) >> 2 & 0x3ffffff))); // b
  210. t = next;
  211. }
  212. }
  213. // Patch all branches in list pointed to by t to branch to current location:
  214. ST_FUNC void gsym(int t)
  215. {
  216. gsym_addr(t, ind);
  217. }
  218. static int arm64_type_size(int t)
  219. {
  220. switch (t & VT_BTYPE) {
  221. case VT_INT: return 2;
  222. case VT_BYTE: return 0;
  223. case VT_SHORT: return 1;
  224. case VT_PTR: return 3;
  225. case VT_FUNC: return 3;
  226. case VT_FLOAT: return 2;
  227. case VT_DOUBLE: return 3;
  228. case VT_LDOUBLE: return 4;
  229. case VT_BOOL: return 0;
  230. case VT_LLONG: return 3;
  231. }
  232. assert(0);
  233. return 0;
  234. }
  235. static void arm64_spoff(int reg, uint64_t off)
  236. {
  237. uint32_t sub = off >> 63;
  238. if (sub)
  239. off = -off;
  240. if (off < 4096)
  241. o(0x910003e0 | sub << 30 | reg | off << 10);
  242. // (add|sub) x(reg),sp,#(off)
  243. else {
  244. arm64_movimm(30, off); // use x30 for offset
  245. o(0x8b3e63e0 | sub << 30 | reg); // (add|sub) x(reg),sp,x30
  246. }
  247. }
  248. static void arm64_ldrx(int sg, int sz_, int dst, int bas, uint64_t off)
  249. {
  250. uint32_t sz = sz_;
  251. if (sz >= 2)
  252. sg = 0;
  253. if (!(off & ~((uint32_t)0xfff << sz)))
  254. o(0x39400000 | dst | bas << 5 | off << (10 - sz) |
  255. (uint32_t)!!sg << 23 | sz << 30); // ldr(*) x(dst),[x(bas),#(off)]
  256. else if (off < 256 || -off <= 256)
  257. o(0x38400000 | dst | bas << 5 | (off & 511) << 12 |
  258. (uint32_t)!!sg << 23 | sz << 30); // ldur(*) x(dst),[x(bas),#(off)]
  259. else {
  260. arm64_movimm(30, off); // use x30 for offset
  261. o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 |
  262. (uint32_t)(!!sg + 1) << 22 | sz << 30); // ldr(*) x(dst),[x(bas),x30]
  263. }
  264. }
  265. static void arm64_ldrv(int sz_, int dst, int bas, uint64_t off)
  266. {
  267. uint32_t sz = sz_;
  268. if (!(off & ~((uint32_t)0xfff << sz)))
  269. o(0x3d400000 | dst | bas << 5 | off << (10 - sz) |
  270. (sz & 4) << 21 | (sz & 3) << 30); // ldr (s|d|q)(dst),[x(bas),#(off)]
  271. else if (off < 256 || -off <= 256)
  272. o(0x3c400000 | dst | bas << 5 | (off & 511) << 12 |
  273. (sz & 4) << 21 | (sz & 3) << 30); // ldur (s|d|q)(dst),[x(bas),#(off)]
  274. else {
  275. arm64_movimm(30, off); // use x30 for offset
  276. o(0x3c606800 | dst | bas << 5 | (uint32_t)30 << 16 |
  277. sz << 30 | (sz & 4) << 21); // ldr (s|d|q)(dst),[x(bas),x30]
  278. }
  279. }
  280. static void arm64_ldrs(int reg_, int size)
  281. {
  282. uint32_t reg = reg_;
  283. // Use x30 for intermediate value in some cases.
  284. switch (size) {
  285. default: assert(0); break;
  286. case 1:
  287. arm64_ldrx(0, 0, reg, reg, 0);
  288. break;
  289. case 2:
  290. arm64_ldrx(0, 1, reg, reg, 0);
  291. break;
  292. case 3:
  293. arm64_ldrx(0, 1, 30, reg, 0);
  294. arm64_ldrx(0, 0, reg, reg, 2);
  295. o(0x2a0043c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #16
  296. break;
  297. case 4:
  298. arm64_ldrx(0, 2, reg, reg, 0);
  299. break;
  300. case 5:
  301. arm64_ldrx(0, 2, 30, reg, 0);
  302. arm64_ldrx(0, 0, reg, reg, 4);
  303. o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
  304. break;
  305. case 6:
  306. arm64_ldrx(0, 2, 30, reg, 0);
  307. arm64_ldrx(0, 1, reg, reg, 4);
  308. o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
  309. break;
  310. case 7:
  311. arm64_ldrx(0, 2, 30, reg, 0);
  312. arm64_ldrx(0, 2, reg, reg, 3);
  313. o(0x53087c00 | reg | reg << 5); // lsr w(reg), w(reg), #8
  314. o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32
  315. break;
  316. case 8:
  317. arm64_ldrx(0, 3, reg, reg, 0);
  318. break;
  319. case 9:
  320. arm64_ldrx(0, 0, reg + 1, reg, 8);
  321. arm64_ldrx(0, 3, reg, reg, 0);
  322. break;
  323. case 10:
  324. arm64_ldrx(0, 1, reg + 1, reg, 8);
  325. arm64_ldrx(0, 3, reg, reg, 0);
  326. break;
  327. case 11:
  328. arm64_ldrx(0, 2, reg + 1, reg, 7);
  329. o(0x53087c00 | (reg+1) | (reg+1) << 5); // lsr w(reg+1), w(reg+1), #8
  330. arm64_ldrx(0, 3, reg, reg, 0);
  331. break;
  332. case 12:
  333. arm64_ldrx(0, 2, reg + 1, reg, 8);
  334. arm64_ldrx(0, 3, reg, reg, 0);
  335. break;
  336. case 13:
  337. arm64_ldrx(0, 3, reg + 1, reg, 5);
  338. o(0xd358fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #24
  339. arm64_ldrx(0, 3, reg, reg, 0);
  340. break;
  341. case 14:
  342. arm64_ldrx(0, 3, reg + 1, reg, 6);
  343. o(0xd350fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #16
  344. arm64_ldrx(0, 3, reg, reg, 0);
  345. break;
  346. case 15:
  347. arm64_ldrx(0, 3, reg + 1, reg, 7);
  348. o(0xd348fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #8
  349. arm64_ldrx(0, 3, reg, reg, 0);
  350. break;
  351. case 16:
  352. o(0xa9400000 | reg | (reg+1) << 10 | reg << 5);
  353. // ldp x(reg),x(reg+1),[x(reg)]
  354. break;
  355. }
  356. }
  357. static void arm64_strx(int sz_, int dst, int bas, uint64_t off)
  358. {
  359. uint32_t sz = sz_;
  360. if (!(off & ~((uint32_t)0xfff << sz)))
  361. o(0x39000000 | dst | bas << 5 | off << (10 - sz) | sz << 30);
  362. // str(*) x(dst),[x(bas],#(off)]
  363. else if (off < 256 || -off <= 256)
  364. o(0x38000000 | dst | bas << 5 | (off & 511) << 12 | sz << 30);
  365. // stur(*) x(dst),[x(bas],#(off)]
  366. else {
  367. arm64_movimm(30, off); // use x30 for offset
  368. o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30);
  369. // str(*) x(dst),[x(bas),x30]
  370. }
  371. }
  372. static void arm64_strv(int sz_, int dst, int bas, uint64_t off)
  373. {
  374. uint32_t sz = sz_;
  375. if (!(off & ~((uint32_t)0xfff << sz)))
  376. o(0x3d000000 | dst | bas << 5 | off << (10 - sz) |
  377. (sz & 4) << 21 | (sz & 3) << 30); // str (s|d|q)(dst),[x(bas),#(off)]
  378. else if (off < 256 || -off <= 256)
  379. o(0x3c000000 | dst | bas << 5 | (off & 511) << 12 |
  380. (sz & 4) << 21 | (sz & 3) << 30); // stur (s|d|q)(dst),[x(bas),#(off)]
  381. else {
  382. arm64_movimm(30, off); // use x30 for offset
  383. o(0x3c206800 | dst | bas << 5 | (uint32_t)30 << 16 |
  384. sz << 30 | (sz & 4) << 21); // str (s|d|q)(dst),[x(bas),x30]
  385. }
  386. }
  387. static void arm64_sym(int r, Sym *sym, unsigned long addend)
  388. {
  389. // Currently TCC's linker does not generate COPY relocations for
  390. // STT_OBJECTs when tcc is invoked with "-run". This typically
  391. // results in "R_AARCH64_ADR_PREL_PG_HI21 relocation failed" when
  392. // a program refers to stdin. A workaround is to avoid that
  393. // relocation and use only relocations with unlimited range.
  394. int avoid_adrp = 1;
  395. if (avoid_adrp || sym->a.weak) {
  396. // (GCC uses a R_AARCH64_ABS64 in this case.)
  397. greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G0_NC, addend);
  398. o(0xd2800000 | r); // mov x(rt),#0,lsl #0
  399. greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G1_NC, addend);
  400. o(0xf2a00000 | r); // movk x(rt),#0,lsl #16
  401. greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G2_NC, addend);
  402. o(0xf2c00000 | r); // movk x(rt),#0,lsl #32
  403. greloca(cur_text_section, sym, ind, R_AARCH64_MOVW_UABS_G3, addend);
  404. o(0xf2e00000 | r); // movk x(rt),#0,lsl #48
  405. }
  406. else {
  407. greloca(cur_text_section, sym, ind, R_AARCH64_ADR_PREL_PG_HI21, addend);
  408. o(0x90000000 | r);
  409. greloca(cur_text_section, sym, ind, R_AARCH64_ADD_ABS_LO12_NC, addend);
  410. o(0x91000000 | r | r << 5);
  411. }
  412. }
  413. ST_FUNC void load(int r, SValue *sv)
  414. {
  415. int svtt = sv->type.t;
  416. int svr = sv->r & ~VT_LVAL_TYPE;
  417. int svrv = svr & VT_VALMASK;
  418. uint64_t svcul = (uint32_t)sv->c.i;
  419. svcul = svcul >> 31 & 1 ? svcul - ((uint64_t)1 << 32) : svcul;
  420. if (svr == (VT_LOCAL | VT_LVAL)) {
  421. if (IS_FREG(r))
  422. arm64_ldrv(arm64_type_size(svtt), fltr(r), 29, svcul);
  423. else
  424. arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
  425. intr(r), 29, svcul);
  426. return;
  427. }
  428. if ((svr & ~VT_VALMASK) == VT_LVAL && svrv < VT_CONST) {
  429. if (IS_FREG(r))
  430. arm64_ldrv(arm64_type_size(svtt), fltr(r), intr(svrv), 0);
  431. else
  432. arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
  433. intr(r), intr(svrv), 0);
  434. return;
  435. }
  436. if (svr == (VT_CONST | VT_LVAL | VT_SYM)) {
  437. arm64_sym(30, sv->sym, svcul); // use x30 for address
  438. if (IS_FREG(r))
  439. arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, 0);
  440. else
  441. arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
  442. intr(r), 30, 0);
  443. return;
  444. }
  445. if (svr == (VT_CONST | VT_SYM)) {
  446. arm64_sym(intr(r), sv->sym, svcul);
  447. return;
  448. }
  449. if (svr == VT_CONST) {
  450. if ((svtt & VT_BTYPE) != VT_VOID)
  451. arm64_movimm(intr(r), arm64_type_size(svtt) == 3 ?
  452. sv->c.i : (uint32_t)svcul);
  453. return;
  454. }
  455. if (svr < VT_CONST) {
  456. if (IS_FREG(r) && IS_FREG(svr))
  457. if (svtt == VT_LDOUBLE)
  458. o(0x4ea01c00 | fltr(r) | fltr(svr) << 5);
  459. // mov v(r).16b,v(svr).16b
  460. else
  461. o(0x1e604000 | fltr(r) | fltr(svr) << 5); // fmov d(r),d(svr)
  462. else if (!IS_FREG(r) && !IS_FREG(svr))
  463. o(0xaa0003e0 | intr(r) | intr(svr) << 16); // mov x(r),x(svr)
  464. else
  465. assert(0);
  466. return;
  467. }
  468. if (svr == VT_LOCAL) {
  469. if (-svcul < 0x1000)
  470. o(0xd10003a0 | intr(r) | -svcul << 10); // sub x(r),x29,#...
  471. else {
  472. arm64_movimm(30, -svcul); // use x30 for offset
  473. o(0xcb0003a0 | intr(r) | (uint32_t)30 << 16); // sub x(r),x29,x30
  474. }
  475. return;
  476. }
  477. if (svr == VT_JMP || svr == VT_JMPI) {
  478. int t = (svr == VT_JMPI);
  479. arm64_movimm(intr(r), t);
  480. o(0x14000002); // b .+8
  481. gsym(svcul);
  482. arm64_movimm(intr(r), t ^ 1);
  483. return;
  484. }
  485. if (svr == (VT_LLOCAL | VT_LVAL)) {
  486. arm64_ldrx(0, 3, 30, 29, svcul); // use x30 for offset
  487. if (IS_FREG(r))
  488. arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, 0);
  489. else
  490. arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt),
  491. intr(r), 30, 0);
  492. return;
  493. }
  494. printf("load(%x, (%x, %x, %llx))\n", r, svtt, sv->r, (long long)svcul);
  495. assert(0);
  496. }
  497. ST_FUNC void store(int r, SValue *sv)
  498. {
  499. int svtt = sv->type.t;
  500. int svr = sv->r & ~VT_LVAL_TYPE;
  501. int svrv = svr & VT_VALMASK;
  502. uint64_t svcul = (uint32_t)sv->c.i;
  503. svcul = svcul >> 31 & 1 ? svcul - ((uint64_t)1 << 32) : svcul;
  504. if (svr == (VT_LOCAL | VT_LVAL)) {
  505. if (IS_FREG(r))
  506. arm64_strv(arm64_type_size(svtt), fltr(r), 29, svcul);
  507. else
  508. arm64_strx(arm64_type_size(svtt), intr(r), 29, svcul);
  509. return;
  510. }
  511. if ((svr & ~VT_VALMASK) == VT_LVAL && svrv < VT_CONST) {
  512. if (IS_FREG(r))
  513. arm64_strv(arm64_type_size(svtt), fltr(r), intr(svrv), 0);
  514. else
  515. arm64_strx(arm64_type_size(svtt), intr(r), intr(svrv), 0);
  516. return;
  517. }
  518. if (svr == (VT_CONST | VT_LVAL | VT_SYM)) {
  519. arm64_sym(30, sv->sym, svcul); // use x30 for address
  520. if (IS_FREG(r))
  521. arm64_strv(arm64_type_size(svtt), fltr(r), 30, 0);
  522. else
  523. arm64_strx(arm64_type_size(svtt), intr(r), 30, 0);
  524. return;
  525. }
  526. printf("store(%x, (%x, %x, %llx))\n", r, svtt, sv->r, (long long)svcul);
  527. assert(0);
  528. }
  529. static void arm64_gen_bl_or_b(int b)
  530. {
  531. if ((vtop->r & (VT_VALMASK | VT_LVAL)) == VT_CONST && (vtop->r & VT_SYM)) {
  532. assert(!b);
  533. greloca(cur_text_section, vtop->sym, ind, R_AARCH64_CALL26, 0);
  534. o(0x94000000); // bl .
  535. }
  536. else
  537. o(0xd61f0000 | (uint32_t)!b << 21 | intr(gv(RC_R30)) << 5); // br/blr
  538. }
  539. static int arm64_hfa_aux(CType *type, int *fsize, int num)
  540. {
  541. if (is_float(type->t)) {
  542. int a, n = type_size(type, &a);
  543. if (num >= 4 || (*fsize && *fsize != n))
  544. return -1;
  545. *fsize = n;
  546. return num + 1;
  547. }
  548. else if ((type->t & VT_BTYPE) == VT_STRUCT) {
  549. int is_struct = 0; // rather than union
  550. Sym *field;
  551. for (field = type->ref->next; field; field = field->next)
  552. if (field->c) {
  553. is_struct = 1;
  554. break;
  555. }
  556. if (is_struct) {
  557. int num0 = num;
  558. for (field = type->ref->next; field; field = field->next) {
  559. if (field->c != (num - num0) * *fsize)
  560. return -1;
  561. num = arm64_hfa_aux(&field->type, fsize, num);
  562. if (num == -1)
  563. return -1;
  564. }
  565. if (type->ref->c != (num - num0) * *fsize)
  566. return -1;
  567. return num;
  568. }
  569. else { // union
  570. int num0 = num;
  571. for (field = type->ref->next; field; field = field->next) {
  572. int num1 = arm64_hfa_aux(&field->type, fsize, num0);
  573. if (num1 == -1)
  574. return -1;
  575. num = num1 < num ? num : num1;
  576. }
  577. if (type->ref->c != (num - num0) * *fsize)
  578. return -1;
  579. return num;
  580. }
  581. }
  582. else if (type->t & VT_ARRAY) {
  583. int num1;
  584. if (!type->ref->c)
  585. return num;
  586. num1 = arm64_hfa_aux(&type->ref->type, fsize, num);
  587. if (num1 == -1 || (num1 != num && type->ref->c > 4))
  588. return -1;
  589. num1 = num + type->ref->c * (num1 - num);
  590. if (num1 > 4)
  591. return -1;
  592. return num1;
  593. }
  594. return -1;
  595. }
  596. static int arm64_hfa(CType *type, int *fsize)
  597. {
  598. if ((type->t & VT_BTYPE) == VT_STRUCT || (type->t & VT_ARRAY)) {
  599. int sz = 0;
  600. int n = arm64_hfa_aux(type, &sz, 0);
  601. if (0 < n && n <= 4) {
  602. if (fsize)
  603. *fsize = sz;
  604. return n;
  605. }
  606. }
  607. return 0;
  608. }
  609. static unsigned long arm64_pcs_aux(int n, CType **type, unsigned long *a)
  610. {
  611. int nx = 0; // next integer register
  612. int nv = 0; // next vector register
  613. unsigned long ns = 32; // next stack offset
  614. int i;
  615. for (i = 0; i < n; i++) {
  616. int hfa = arm64_hfa(type[i], 0);
  617. int size, align;
  618. if ((type[i]->t & VT_ARRAY) ||
  619. (type[i]->t & VT_BTYPE) == VT_FUNC)
  620. size = align = 8;
  621. else
  622. size = type_size(type[i], &align);
  623. if (hfa)
  624. // B.2
  625. ;
  626. else if (size > 16) {
  627. // B.3: replace with pointer
  628. if (nx < 8)
  629. a[i] = nx++ << 1 | 1;
  630. else {
  631. ns = (ns + 7) & ~7;
  632. a[i] = ns | 1;
  633. ns += 8;
  634. }
  635. continue;
  636. }
  637. else if ((type[i]->t & VT_BTYPE) == VT_STRUCT)
  638. // B.4
  639. size = (size + 7) & ~7;
  640. // C.1
  641. if (is_float(type[i]->t) && nv < 8) {
  642. a[i] = 16 + (nv++ << 1);
  643. continue;
  644. }
  645. // C.2
  646. if (hfa && nv + hfa <= 8) {
  647. a[i] = 16 + (nv << 1);
  648. nv += hfa;
  649. continue;
  650. }
  651. // C.3
  652. if (hfa) {
  653. nv = 8;
  654. size = (size + 7) & ~7;
  655. }
  656. // C.4
  657. if (hfa || (type[i]->t & VT_BTYPE) == VT_LDOUBLE) {
  658. ns = (ns + 7) & ~7;
  659. ns = (ns + align - 1) & -align;
  660. }
  661. // C.5
  662. if ((type[i]->t & VT_BTYPE) == VT_FLOAT)
  663. size = 8;
  664. // C.6
  665. if (hfa || is_float(type[i]->t)) {
  666. a[i] = ns;
  667. ns += size;
  668. continue;
  669. }
  670. // C.7
  671. if ((type[i]->t & VT_BTYPE) != VT_STRUCT && size <= 8 && nx < 8) {
  672. a[i] = nx++ << 1;
  673. continue;
  674. }
  675. // C.8
  676. if (align == 16)
  677. nx = (nx + 1) & ~1;
  678. // C.9
  679. if ((type[i]->t & VT_BTYPE) != VT_STRUCT && size == 16 && nx < 7) {
  680. a[i] = nx << 1;
  681. nx += 2;
  682. continue;
  683. }
  684. // C.10
  685. if ((type[i]->t & VT_BTYPE) == VT_STRUCT && size <= (8 - nx) * 8) {
  686. a[i] = nx << 1;
  687. nx += (size + 7) >> 3;
  688. continue;
  689. }
  690. // C.11
  691. nx = 8;
  692. // C.12
  693. ns = (ns + 7) & ~7;
  694. ns = (ns + align - 1) & -align;
  695. // C.13
  696. if ((type[i]->t & VT_BTYPE) == VT_STRUCT) {
  697. a[i] = ns;
  698. ns += size;
  699. continue;
  700. }
  701. // C.14
  702. if (size < 8)
  703. size = 8;
  704. // C.15
  705. a[i] = ns;
  706. ns += size;
  707. }
  708. return ns - 32;
  709. }
  710. static unsigned long arm64_pcs(int n, CType **type, unsigned long *a)
  711. {
  712. unsigned long stack;
  713. // Return type:
  714. if ((type[0]->t & VT_BTYPE) == VT_VOID)
  715. a[0] = -1;
  716. else {
  717. arm64_pcs_aux(1, type, a);
  718. assert(a[0] == 0 || a[0] == 1 || a[0] == 16);
  719. }
  720. // Argument types:
  721. stack = arm64_pcs_aux(n, type + 1, a + 1);
  722. if (0) {
  723. int i;
  724. for (i = 0; i <= n; i++) {
  725. if (!i)
  726. printf("arm64_pcs return: ");
  727. else
  728. printf("arm64_pcs arg %d: ", i);
  729. if (a[i] == (unsigned long)-1)
  730. printf("void\n");
  731. else if (a[i] == 1 && !i)
  732. printf("X8 pointer\n");
  733. else if (a[i] < 16)
  734. printf("X%lu%s\n", a[i] / 2, a[i] & 1 ? " pointer" : "");
  735. else if (a[i] < 32)
  736. printf("V%lu\n", a[i] / 2 - 8);
  737. else
  738. printf("stack %lu%s\n",
  739. (a[i] - 32) & ~1, a[i] & 1 ? " pointer" : "");
  740. }
  741. }
  742. return stack;
  743. }
  744. ST_FUNC void gfunc_call(int nb_args)
  745. {
  746. CType *return_type;
  747. CType **t;
  748. unsigned long *a, *a1;
  749. unsigned long stack;
  750. int i;
  751. return_type = &vtop[-nb_args].type.ref->type;
  752. if ((return_type->t & VT_BTYPE) == VT_STRUCT)
  753. --nb_args;
  754. t = tcc_malloc((nb_args + 1) * sizeof(*t));
  755. a = tcc_malloc((nb_args + 1) * sizeof(*a));
  756. a1 = tcc_malloc((nb_args + 1) * sizeof(*a1));
  757. t[0] = return_type;
  758. for (i = 0; i < nb_args; i++)
  759. t[nb_args - i] = &vtop[-i].type;
  760. stack = arm64_pcs(nb_args, t, a);
  761. // Allocate space for structs replaced by pointer:
  762. for (i = nb_args; i; i--)
  763. if (a[i] & 1) {
  764. SValue *arg = &vtop[i - nb_args];
  765. int align, size = type_size(&arg->type, &align);
  766. assert((arg->type.t & VT_BTYPE) == VT_STRUCT);
  767. stack = (stack + align - 1) & -align;
  768. a1[i] = stack;
  769. stack += size;
  770. }
  771. stack = (stack + 15) >> 4 << 4;
  772. assert(stack < 0x1000);
  773. if (stack)
  774. o(0xd10003ff | stack << 10); // sub sp,sp,#(n)
  775. // First pass: set all values on stack
  776. for (i = nb_args; i; i--) {
  777. vpushv(vtop - nb_args + i);
  778. if (a[i] & 1) {
  779. // struct replaced by pointer
  780. int r = get_reg(RC_INT);
  781. arm64_spoff(intr(r), a1[i]);
  782. vset(&vtop->type, r | VT_LVAL, 0);
  783. vswap();
  784. vstore();
  785. if (a[i] >= 32) {
  786. // pointer on stack
  787. r = get_reg(RC_INT);
  788. arm64_spoff(intr(r), a1[i]);
  789. arm64_strx(3, intr(r), 31, (a[i] - 32) >> 1 << 1);
  790. }
  791. }
  792. else if (a[i] >= 32) {
  793. // value on stack
  794. if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
  795. int r = get_reg(RC_INT);
  796. arm64_spoff(intr(r), a[i] - 32);
  797. vset(&vtop->type, r | VT_LVAL, 0);
  798. vswap();
  799. vstore();
  800. }
  801. else if (is_float(vtop->type.t)) {
  802. gv(RC_FLOAT);
  803. arm64_strv(arm64_type_size(vtop[0].type.t),
  804. fltr(vtop[0].r), 31, a[i] - 32);
  805. }
  806. else {
  807. gv(RC_INT);
  808. arm64_strx(arm64_type_size(vtop[0].type.t),
  809. intr(vtop[0].r), 31, a[i] - 32);
  810. }
  811. }
  812. --vtop;
  813. }
  814. // Second pass: assign values to registers
  815. for (i = nb_args; i; i--, vtop--) {
  816. if (a[i] < 16 && !(a[i] & 1)) {
  817. // value in general-purpose registers
  818. if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
  819. int align, size = type_size(&vtop->type, &align);
  820. vtop->type.t = VT_PTR;
  821. gaddrof();
  822. gv(RC_R(a[i] / 2));
  823. arm64_ldrs(a[i] / 2, size);
  824. }
  825. else
  826. gv(RC_R(a[i] / 2));
  827. }
  828. else if (a[i] < 16)
  829. // struct replaced by pointer in register
  830. arm64_spoff(a[i] / 2, a1[i]);
  831. else if (a[i] < 32) {
  832. // value in floating-point registers
  833. if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
  834. uint32_t j, sz, n = arm64_hfa(&vtop->type, &sz);
  835. vtop->type.t = VT_PTR;
  836. gaddrof();
  837. gv(RC_R30);
  838. for (j = 0; j < n; j++)
  839. o(0x3d4003c0 |
  840. (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
  841. (a[i] / 2 - 8 + j) |
  842. j << 10); // ldr ([sdq])(*),[x30,#(j * sz)]
  843. }
  844. else
  845. gv(RC_F(a[i] / 2 - 8));
  846. }
  847. }
  848. if ((return_type->t & VT_BTYPE) == VT_STRUCT) {
  849. if (a[0] == 1) {
  850. // indirect return: set x8 and discard the stack value
  851. gv(RC_R(8));
  852. --vtop;
  853. }
  854. else
  855. // return in registers: keep the address for after the call
  856. vswap();
  857. }
  858. save_regs(0);
  859. arm64_gen_bl_or_b(0);
  860. --vtop;
  861. if (stack)
  862. o(0x910003ff | stack << 10); // add sp,sp,#(n)
  863. {
  864. int rt = return_type->t;
  865. int bt = rt & VT_BTYPE;
  866. if (bt == VT_BYTE || bt == VT_SHORT)
  867. // Promote small integers:
  868. o(0x13001c00 | (bt == VT_SHORT) << 13 |
  869. (uint32_t)!!(rt & VT_UNSIGNED) << 30); // [su]xt[bh] w0,w0
  870. else if (bt == VT_STRUCT && !(a[0] & 1)) {
  871. // A struct was returned in registers, so write it out:
  872. gv(RC_R(8));
  873. --vtop;
  874. if (a[0] == 0) {
  875. int align, size = type_size(return_type, &align);
  876. assert(size <= 16);
  877. if (size > 8)
  878. o(0xa9000500); // stp x0,x1,[x8]
  879. else if (size)
  880. arm64_strx(size > 4 ? 3 : size > 2 ? 2 : size > 1, 0, 8, 0);
  881. }
  882. else if (a[0] == 16) {
  883. uint32_t j, sz, n = arm64_hfa(return_type, &sz);
  884. for (j = 0; j < n; j++)
  885. o(0x3d000100 |
  886. (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
  887. (a[i] / 2 - 8 + j) |
  888. j << 10); // str ([sdq])(*),[x8,#(j * sz)]
  889. }
  890. }
  891. }
  892. tcc_free(a1);
  893. tcc_free(a);
  894. tcc_free(t);
  895. }
  896. static unsigned long arm64_func_va_list_stack;
  897. static int arm64_func_va_list_gr_offs;
  898. static int arm64_func_va_list_vr_offs;
  899. static int arm64_func_sub_sp_offset;
  900. ST_FUNC void gfunc_prolog(CType *func_type)
  901. {
  902. int n = 0;
  903. int i = 0;
  904. Sym *sym;
  905. CType **t;
  906. unsigned long *a;
  907. // Why doesn't the caller (gen_function) set func_vt?
  908. func_vt = func_type->ref->type;
  909. func_vc = 144; // offset of where x8 is stored
  910. for (sym = func_type->ref; sym; sym = sym->next)
  911. ++n;
  912. t = tcc_malloc(n * sizeof(*t));
  913. a = tcc_malloc(n * sizeof(*a));
  914. for (sym = func_type->ref; sym; sym = sym->next)
  915. t[i++] = &sym->type;
  916. arm64_func_va_list_stack = arm64_pcs(n - 1, t, a);
  917. o(0xa9b27bfd); // stp x29,x30,[sp,#-224]!
  918. o(0xad0087e0); // stp q0,q1,[sp,#16]
  919. o(0xad018fe2); // stp q2,q3,[sp,#48]
  920. o(0xad0297e4); // stp q4,q5,[sp,#80]
  921. o(0xad039fe6); // stp q6,q7,[sp,#112]
  922. o(0xa90923e8); // stp x8,x8,[sp,#144]
  923. o(0xa90a07e0); // stp x0,x1,[sp,#160]
  924. o(0xa90b0fe2); // stp x2,x3,[sp,#176]
  925. o(0xa90c17e4); // stp x4,x5,[sp,#192]
  926. o(0xa90d1fe6); // stp x6,x7,[sp,#208]
  927. arm64_func_va_list_gr_offs = -64;
  928. arm64_func_va_list_vr_offs = -128;
  929. for (i = 1, sym = func_type->ref->next; sym; i++, sym = sym->next) {
  930. int off = (a[i] < 16 ? 160 + a[i] / 2 * 8 :
  931. a[i] < 32 ? 16 + (a[i] - 16) / 2 * 16 :
  932. 224 + ((a[i] - 32) >> 1 << 1));
  933. sym_push(sym->v & ~SYM_FIELD, &sym->type,
  934. (a[i] & 1 ? VT_LLOCAL : VT_LOCAL) | lvalue_type(sym->type.t),
  935. off);
  936. if (a[i] < 16) {
  937. int align, size = type_size(&sym->type, &align);
  938. arm64_func_va_list_gr_offs = (a[i] / 2 - 7 +
  939. (!(a[i] & 1) && size > 8)) * 8;
  940. }
  941. else if (a[i] < 32) {
  942. uint32_t hfa = arm64_hfa(&sym->type, 0);
  943. arm64_func_va_list_vr_offs = (a[i] / 2 - 16 +
  944. (hfa ? hfa : 1)) * 16;
  945. }
  946. // HFAs of float and double need to be written differently:
  947. if (16 <= a[i] && a[i] < 32 && (sym->type.t & VT_BTYPE) == VT_STRUCT) {
  948. uint32_t j, sz, k = arm64_hfa(&sym->type, &sz);
  949. if (sz < 16)
  950. for (j = 0; j < k; j++) {
  951. o(0x3d0003e0 | -(sz & 8) << 27 | (sz & 4) << 29 |
  952. ((a[i] - 16) / 2 + j) | (off / sz + j) << 10);
  953. // str ([sdq])(*),[sp,#(j * sz)]
  954. }
  955. }
  956. }
  957. tcc_free(a);
  958. tcc_free(t);
  959. o(0x910003fd); // mov x29,sp
  960. arm64_func_sub_sp_offset = ind;
  961. // In gfunc_epilog these will be replaced with code to decrement SP:
  962. o(0xd503201f); // nop
  963. o(0xd503201f); // nop
  964. loc = 0;
  965. }
  966. ST_FUNC void gen_va_start(void)
  967. {
  968. int r;
  969. --vtop; // we don't need the "arg"
  970. gaddrof();
  971. r = intr(gv(RC_INT));
  972. if (arm64_func_va_list_stack) {
  973. //xx could use add (immediate) here
  974. arm64_movimm(30, arm64_func_va_list_stack + 224);
  975. o(0x8b1e03be); // add x30,x29,x30
  976. }
  977. else
  978. o(0x910383be); // add x30,x29,#224
  979. o(0xf900001e | r << 5); // str x30,[x(r)]
  980. if (arm64_func_va_list_gr_offs) {
  981. if (arm64_func_va_list_stack)
  982. o(0x910383be); // add x30,x29,#224
  983. o(0xf900041e | r << 5); // str x30,[x(r),#8]
  984. }
  985. if (arm64_func_va_list_vr_offs) {
  986. o(0x910243be); // add x30,x29,#144
  987. o(0xf900081e | r << 5); // str x30,[x(r),#16]
  988. }
  989. arm64_movimm(30, arm64_func_va_list_gr_offs);
  990. o(0xb900181e | r << 5); // str w30,[x(r),#24]
  991. arm64_movimm(30, arm64_func_va_list_vr_offs);
  992. o(0xb9001c1e | r << 5); // str w30,[x(r),#28]
  993. --vtop;
  994. }
  995. ST_FUNC void gen_va_arg(CType *t)
  996. {
  997. int align, size = type_size(t, &align);
  998. int fsize, hfa = arm64_hfa(t, &fsize);
  999. uint32_t r0, r1;
  1000. if (is_float(t->t)) {
  1001. hfa = 1;
  1002. fsize = size;
  1003. }
  1004. gaddrof();
  1005. r0 = intr(gv(RC_INT));
  1006. r1 = get_reg(RC_INT);
  1007. vtop[0].r = r1 | lvalue_type(t->t);
  1008. r1 = intr(r1);
  1009. if (!hfa) {
  1010. uint32_t n = size > 16 ? 8 : (size + 7) & -8;
  1011. o(0xb940181e | r0 << 5); // ldr w30,[x(r0),#24] // __gr_offs
  1012. if (align == 16) {
  1013. assert(0); // this path untested but needed for __uint128_t
  1014. o(0x11003fde); // add w30,w30,#15
  1015. o(0x121c6fde); // and w30,w30,#-16
  1016. }
  1017. o(0x310003c0 | r1 | n << 10); // adds w(r1),w30,#(n)
  1018. o(0x540000ad); // b.le .+20
  1019. o(0xf9400000 | r1 | r0 << 5); // ldr x(r1),[x(r0)] // __stack
  1020. o(0x9100001e | r1 << 5 | n << 10); // add x30,x(r1),#(n)
  1021. o(0xf900001e | r0 << 5); // str x30,[x(r0)] // __stack
  1022. o(0x14000004); // b .+16
  1023. o(0xb9001800 | r1 | r0 << 5); // str w(r1),[x(r0),#24] // __gr_offs
  1024. o(0xf9400400 | r1 | r0 << 5); // ldr x(r1),[x(r0),#8] // __gr_top
  1025. o(0x8b3ec000 | r1 | r1 << 5); // add x(r1),x(r1),w30,sxtw
  1026. if (size > 16)
  1027. o(0xf9400000 | r1 | r1 << 5); // ldr x(r1),[x(r1)]
  1028. }
  1029. else {
  1030. uint32_t rsz = hfa << 4;
  1031. uint32_t ssz = (size + 7) & -(uint32_t)8;
  1032. uint32_t b1, b2;
  1033. o(0xb9401c1e | r0 << 5); // ldr w30,[x(r0),#28] // __vr_offs
  1034. o(0x310003c0 | r1 | rsz << 10); // adds w(r1),w30,#(rsz)
  1035. b1 = ind; o(0x5400000d); // b.le lab1
  1036. o(0xf9400000 | r1 | r0 << 5); // ldr x(r1),[x(r0)] // __stack
  1037. if (fsize == 16) {
  1038. o(0x91003c00 | r1 | r1 << 5); // add x(r1),x(r1),#15
  1039. o(0x927cec00 | r1 | r1 << 5); // and x(r1),x(r1),#-16
  1040. }
  1041. o(0x9100001e | r1 << 5 | ssz << 10); // add x30,x(r1),#(ssz)
  1042. o(0xf900001e | r0 << 5); // str x30,[x(r0)] // __stack
  1043. b2 = ind; o(0x14000000); // b lab2
  1044. // lab1:
  1045. write32le(cur_text_section->data + b1, 0x5400000d | (ind - b1) << 3);
  1046. o(0xb9001c00 | r1 | r0 << 5); // str w(r1),[x(r0),#28] // __vr_offs
  1047. o(0xf9400800 | r1 | r0 << 5); // ldr x(r1),[x(r0),#16] // __vr_top
  1048. if (hfa == 1 || fsize == 16)
  1049. o(0x8b3ec000 | r1 | r1 << 5); // add x(r1),x(r1),w30,sxtw
  1050. else {
  1051. // We need to change the layout of this HFA.
  1052. // Get some space on the stack using global variable "loc":
  1053. loc = (loc - size) & -(uint32_t)align;
  1054. o(0x8b3ec000 | 30 | r1 << 5); // add x30,x(r1),w30,sxtw
  1055. arm64_movimm(r1, loc);
  1056. o(0x8b0003a0 | r1 | r1 << 16); // add x(r1),x29,x(r1)
  1057. o(0x4c402bdc | (uint32_t)fsize << 7 |
  1058. (uint32_t)(hfa == 2) << 15 |
  1059. (uint32_t)(hfa == 3) << 14); // ld1 {v28.(4s|2d),...},[x30]
  1060. o(0x0d00801c | r1 << 5 | (fsize == 8) << 10 |
  1061. (uint32_t)(hfa != 2) << 13 |
  1062. (uint32_t)(hfa != 3) << 21); // st(hfa) {v28.(s|d),...}[0],[x(r1)]
  1063. }
  1064. // lab2:
  1065. write32le(cur_text_section->data + b2, 0x14000000 | (ind - b2) >> 2);
  1066. }
  1067. }
  1068. ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret,
  1069. int *align, int *regsize)
  1070. {
  1071. return 0;
  1072. }
  1073. ST_FUNC void gfunc_return(CType *func_type)
  1074. {
  1075. CType *t = func_type;
  1076. unsigned long a;
  1077. arm64_pcs(0, &t, &a);
  1078. switch (a) {
  1079. case -1:
  1080. break;
  1081. case 0:
  1082. if ((func_type->t & VT_BTYPE) == VT_STRUCT) {
  1083. int align, size = type_size(func_type, &align);
  1084. gaddrof();
  1085. gv(RC_R(0));
  1086. arm64_ldrs(0, size);
  1087. }
  1088. else
  1089. gv(RC_IRET);
  1090. break;
  1091. case 1: {
  1092. CType type = *func_type;
  1093. mk_pointer(&type);
  1094. vset(&type, VT_LOCAL | VT_LVAL, func_vc);
  1095. indir();
  1096. vswap();
  1097. vstore();
  1098. break;
  1099. }
  1100. case 16:
  1101. if ((func_type->t & VT_BTYPE) == VT_STRUCT) {
  1102. uint32_t j, sz, n = arm64_hfa(&vtop->type, &sz);
  1103. gaddrof();
  1104. gv(RC_R(0));
  1105. for (j = 0; j < n; j++)
  1106. o(0x3d400000 |
  1107. (sz & 16) << 19 | -(sz & 8) << 27 | (sz & 4) << 29 |
  1108. j | j << 10); // ldr ([sdq])(*),[x0,#(j * sz)]
  1109. }
  1110. else
  1111. gv(RC_FRET);
  1112. break;
  1113. default:
  1114. assert(0);
  1115. }
  1116. vtop--;
  1117. }
  1118. ST_FUNC void gfunc_epilog(void)
  1119. {
  1120. if (loc) {
  1121. // Insert instructions to subtract size of stack frame from SP.
  1122. unsigned char *ptr = cur_text_section->data + arm64_func_sub_sp_offset;
  1123. uint64_t diff = (-loc + 15) & ~15;
  1124. if (!(diff >> 24)) {
  1125. if (diff & 0xfff) // sub sp,sp,#(diff & 0xfff)
  1126. write32le(ptr, 0xd10003ff | (diff & 0xfff) << 10);
  1127. if (diff >> 12) // sub sp,sp,#(diff >> 12),lsl #12
  1128. write32le(ptr + 4, 0xd14003ff | (diff >> 12) << 10);
  1129. }
  1130. else {
  1131. // In this case we may subtract more than necessary,
  1132. // but always less than 17/16 of what we were aiming for.
  1133. int i = 0;
  1134. int j = 0;
  1135. while (diff >> 20) {
  1136. diff = (diff + 0xffff) >> 16;
  1137. ++i;
  1138. }
  1139. while (diff >> 16) {
  1140. diff = (diff + 1) >> 1;
  1141. ++j;
  1142. }
  1143. write32le(ptr, 0xd2800010 | diff << 5 | i << 21);
  1144. // mov x16,#(diff),lsl #(16 * i)
  1145. write32le(ptr + 4, 0xcb3063ff | j << 10);
  1146. // sub sp,sp,x16,lsl #(j)
  1147. }
  1148. }
  1149. o(0x910003bf); // mov sp,x29
  1150. o(0xa8ce7bfd); // ldp x29,x30,[sp],#224
  1151. o(0xd65f03c0); // ret
  1152. }
  1153. ST_FUNC void gen_fill_nops(int bytes)
  1154. {
  1155. if ((bytes & 3))
  1156. tcc_error("alignment of code section not multiple of 4");
  1157. while (bytes > 0) {
  1158. o(0xd503201f); // nop
  1159. bytes -= 4;
  1160. }
  1161. }
  1162. // Generate forward branch to label:
  1163. ST_FUNC int gjmp(int t)
  1164. {
  1165. int r = ind;
  1166. if (nocode_wanted)
  1167. return t;
  1168. o(t);
  1169. return r;
  1170. }
  1171. // Generate branch to known address:
  1172. ST_FUNC void gjmp_addr(int a)
  1173. {
  1174. assert(a - ind + 0x8000000 < 0x10000000);
  1175. o(0x14000000 | ((a - ind) >> 2 & 0x3ffffff));
  1176. }
  1177. ST_FUNC int gtst(int inv, int t)
  1178. {
  1179. int bt = vtop->type.t & VT_BTYPE;
  1180. if (bt == VT_LDOUBLE) {
  1181. uint32_t a, b, f = fltr(gv(RC_FLOAT));
  1182. a = get_reg(RC_INT);
  1183. vpushi(0);
  1184. vtop[0].r = a;
  1185. b = get_reg(RC_INT);
  1186. a = intr(a);
  1187. b = intr(b);
  1188. o(0x4e083c00 | a | f << 5); // mov x(a),v(f).d[0]
  1189. o(0x4e183c00 | b | f << 5); // mov x(b),v(f).d[1]
  1190. o(0xaa000400 | a | a << 5 | b << 16); // orr x(a),x(a),x(b),lsl #1
  1191. o(0xb4000040 | a | !!inv << 24); // cbz/cbnz x(a),.+8
  1192. --vtop;
  1193. }
  1194. else if (bt == VT_FLOAT || bt == VT_DOUBLE) {
  1195. uint32_t a = fltr(gv(RC_FLOAT));
  1196. o(0x1e202008 | a << 5 | (bt != VT_FLOAT) << 22); // fcmp
  1197. o(0x54000040 | !!inv); // b.eq/b.ne .+8
  1198. }
  1199. else {
  1200. uint32_t ll = (bt == VT_PTR || bt == VT_LLONG);
  1201. uint32_t a = intr(gv(RC_INT));
  1202. o(0x34000040 | a | !!inv << 24 | ll << 31); // cbz/cbnz wA,.+8
  1203. }
  1204. --vtop;
  1205. return gjmp(t);
  1206. }
  1207. static int arm64_iconst(uint64_t *val, SValue *sv)
  1208. {
  1209. if ((sv->r & (VT_VALMASK | VT_LVAL | VT_SYM)) != VT_CONST)
  1210. return 0;
  1211. if (val) {
  1212. int t = sv->type.t;
  1213. int bt = t & VT_BTYPE;
  1214. *val = ((bt == VT_LLONG || bt == VT_PTR) ? sv->c.i :
  1215. (uint32_t)sv->c.i |
  1216. (t & VT_UNSIGNED ? 0 : -(sv->c.i & 0x80000000)));
  1217. }
  1218. return 1;
  1219. }
  1220. static int arm64_gen_opic(int op, uint32_t l, int rev, uint64_t val,
  1221. uint32_t x, uint32_t a)
  1222. {
  1223. if (op == '-' && !rev) {
  1224. val = -val;
  1225. op = '+';
  1226. }
  1227. val = l ? val : (uint32_t)val;
  1228. switch (op) {
  1229. case '+': {
  1230. uint32_t s = l ? val >> 63 : val >> 31;
  1231. val = s ? -val : val;
  1232. val = l ? val : (uint32_t)val;
  1233. if (!(val & ~(uint64_t)0xfff))
  1234. o(0x11000000 | l << 31 | s << 30 | x | a << 5 | val << 10);
  1235. else if (!(val & ~(uint64_t)0xfff000))
  1236. o(0x11400000 | l << 31 | s << 30 | x | a << 5 | val >> 12 << 10);
  1237. else {
  1238. arm64_movimm(30, val); // use x30
  1239. o(0x0b1e0000 | l << 31 | s << 30 | x | a << 5);
  1240. }
  1241. return 1;
  1242. }
  1243. case '-':
  1244. if (!val)
  1245. o(0x4b0003e0 | l << 31 | x | a << 16); // neg
  1246. else if (val == (l ? (uint64_t)-1 : (uint32_t)-1))
  1247. o(0x2a2003e0 | l << 31 | x | a << 16); // mvn
  1248. else {
  1249. arm64_movimm(30, val); // use x30
  1250. o(0x4b0003c0 | l << 31 | x | a << 16); // sub
  1251. }
  1252. return 1;
  1253. case '^':
  1254. if (val == -1 || (val == 0xffffffff && !l)) {
  1255. o(0x2a2003e0 | l << 31 | x | a << 16); // mvn
  1256. return 1;
  1257. }
  1258. // fall through
  1259. case '&':
  1260. case '|': {
  1261. int e = arm64_encode_bimm64(l ? val : val | val << 32);
  1262. if (e < 0)
  1263. return 0;
  1264. o((op == '&' ? 0x12000000 :
  1265. op == '|' ? 0x32000000 : 0x52000000) |
  1266. l << 31 | x | a << 5 | (uint32_t)e << 10);
  1267. return 1;
  1268. }
  1269. case TOK_SAR:
  1270. case TOK_SHL:
  1271. case TOK_SHR: {
  1272. uint32_t n = 32 << l;
  1273. val = val & (n - 1);
  1274. if (rev)
  1275. return 0;
  1276. if (!val)
  1277. assert(0);
  1278. else if (op == TOK_SHL)
  1279. o(0x53000000 | l << 31 | l << 22 | x | a << 5 |
  1280. (n - val) << 16 | (n - 1 - val) << 10); // lsl
  1281. else
  1282. o(0x13000000 | (op == TOK_SHR) << 30 | l << 31 | l << 22 |
  1283. x | a << 5 | val << 16 | (n - 1) << 10); // lsr/asr
  1284. return 1;
  1285. }
  1286. }
  1287. return 0;
  1288. }
  1289. static void arm64_gen_opil(int op, uint32_t l)
  1290. {
  1291. uint32_t x, a, b;
  1292. // Special treatment for operations with a constant operand:
  1293. {
  1294. uint64_t val;
  1295. int rev = 1;
  1296. if (arm64_iconst(0, &vtop[0])) {
  1297. vswap();
  1298. rev = 0;
  1299. }
  1300. if (arm64_iconst(&val, &vtop[-1])) {
  1301. gv(RC_INT);
  1302. a = intr(vtop[0].r);
  1303. --vtop;
  1304. x = get_reg(RC_INT);
  1305. ++vtop;
  1306. if (arm64_gen_opic(op, l, rev, val, intr(x), a)) {
  1307. vtop[0].r = x;
  1308. vswap();
  1309. --vtop;
  1310. return;
  1311. }
  1312. }
  1313. if (!rev)
  1314. vswap();
  1315. }
  1316. gv2(RC_INT, RC_INT);
  1317. assert(vtop[-1].r < VT_CONST && vtop[0].r < VT_CONST);
  1318. a = intr(vtop[-1].r);
  1319. b = intr(vtop[0].r);
  1320. vtop -= 2;
  1321. x = get_reg(RC_INT);
  1322. ++vtop;
  1323. vtop[0].r = x;
  1324. x = intr(x);
  1325. switch (op) {
  1326. case '%':
  1327. // Use x30 for quotient:
  1328. o(0x1ac00c00 | l << 31 | 30 | a << 5 | b << 16); // sdiv
  1329. o(0x1b008000 | l << 31 | x | (uint32_t)30 << 5 |
  1330. b << 16 | a << 10); // msub
  1331. break;
  1332. case '&':
  1333. o(0x0a000000 | l << 31 | x | a << 5 | b << 16); // and
  1334. break;
  1335. case '*':
  1336. o(0x1b007c00 | l << 31 | x | a << 5 | b << 16); // mul
  1337. break;
  1338. case '+':
  1339. o(0x0b000000 | l << 31 | x | a << 5 | b << 16); // add
  1340. break;
  1341. case '-':
  1342. o(0x4b000000 | l << 31 | x | a << 5 | b << 16); // sub
  1343. break;
  1344. case '/':
  1345. o(0x1ac00c00 | l << 31 | x | a << 5 | b << 16); // sdiv
  1346. break;
  1347. case '^':
  1348. o(0x4a000000 | l << 31 | x | a << 5 | b << 16); // eor
  1349. break;
  1350. case '|':
  1351. o(0x2a000000 | l << 31 | x | a << 5 | b << 16); // orr
  1352. break;
  1353. case TOK_EQ:
  1354. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1355. o(0x1a9f17e0 | x); // cset wA,eq
  1356. break;
  1357. case TOK_GE:
  1358. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1359. o(0x1a9fb7e0 | x); // cset wA,ge
  1360. break;
  1361. case TOK_GT:
  1362. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1363. o(0x1a9fd7e0 | x); // cset wA,gt
  1364. break;
  1365. case TOK_LE:
  1366. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1367. o(0x1a9fc7e0 | x); // cset wA,le
  1368. break;
  1369. case TOK_LT:
  1370. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1371. o(0x1a9fa7e0 | x); // cset wA,lt
  1372. break;
  1373. case TOK_NE:
  1374. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1375. o(0x1a9f07e0 | x); // cset wA,ne
  1376. break;
  1377. case TOK_SAR:
  1378. o(0x1ac02800 | l << 31 | x | a << 5 | b << 16); // asr
  1379. break;
  1380. case TOK_SHL:
  1381. o(0x1ac02000 | l << 31 | x | a << 5 | b << 16); // lsl
  1382. break;
  1383. case TOK_SHR:
  1384. o(0x1ac02400 | l << 31 | x | a << 5 | b << 16); // lsr
  1385. break;
  1386. case TOK_UDIV:
  1387. case TOK_PDIV:
  1388. o(0x1ac00800 | l << 31 | x | a << 5 | b << 16); // udiv
  1389. break;
  1390. case TOK_UGE:
  1391. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1392. o(0x1a9f37e0 | x); // cset wA,cs
  1393. break;
  1394. case TOK_UGT:
  1395. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1396. o(0x1a9f97e0 | x); // cset wA,hi
  1397. break;
  1398. case TOK_ULT:
  1399. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1400. o(0x1a9f27e0 | x); // cset wA,cc
  1401. break;
  1402. case TOK_ULE:
  1403. o(0x6b00001f | l << 31 | a << 5 | b << 16); // cmp
  1404. o(0x1a9f87e0 | x); // cset wA,ls
  1405. break;
  1406. case TOK_UMOD:
  1407. // Use x30 for quotient:
  1408. o(0x1ac00800 | l << 31 | 30 | a << 5 | b << 16); // udiv
  1409. o(0x1b008000 | l << 31 | x | (uint32_t)30 << 5 |
  1410. b << 16 | a << 10); // msub
  1411. break;
  1412. default:
  1413. assert(0);
  1414. }
  1415. }
  1416. ST_FUNC void gen_opi(int op)
  1417. {
  1418. arm64_gen_opil(op, 0);
  1419. }
  1420. ST_FUNC void gen_opl(int op)
  1421. {
  1422. arm64_gen_opil(op, 1);
  1423. }
  1424. ST_FUNC void gen_opf(int op)
  1425. {
  1426. uint32_t x, a, b, dbl;
  1427. if (vtop[0].type.t == VT_LDOUBLE) {
  1428. CType type = vtop[0].type;
  1429. int func = 0;
  1430. int cond = -1;
  1431. switch (op) {
  1432. case '*': func = TOK___multf3; break;
  1433. case '+': func = TOK___addtf3; break;
  1434. case '-': func = TOK___subtf3; break;
  1435. case '/': func = TOK___divtf3; break;
  1436. case TOK_EQ: func = TOK___eqtf2; cond = 1; break;
  1437. case TOK_NE: func = TOK___netf2; cond = 0; break;
  1438. case TOK_LT: func = TOK___lttf2; cond = 10; break;
  1439. case TOK_GE: func = TOK___getf2; cond = 11; break;
  1440. case TOK_LE: func = TOK___letf2; cond = 12; break;
  1441. case TOK_GT: func = TOK___gttf2; cond = 13; break;
  1442. default: assert(0); break;
  1443. }
  1444. vpush_global_sym(&func_old_type, func);
  1445. vrott(3);
  1446. gfunc_call(2);
  1447. vpushi(0);
  1448. vtop->r = cond < 0 ? REG_FRET : REG_IRET;
  1449. if (cond < 0)
  1450. vtop->type = type;
  1451. else {
  1452. o(0x7100001f); // cmp w0,#0
  1453. o(0x1a9f07e0 | (uint32_t)cond << 12); // cset w0,(cond)
  1454. }
  1455. return;
  1456. }
  1457. dbl = vtop[0].type.t != VT_FLOAT;
  1458. gv2(RC_FLOAT, RC_FLOAT);
  1459. assert(vtop[-1].r < VT_CONST && vtop[0].r < VT_CONST);
  1460. a = fltr(vtop[-1].r);
  1461. b = fltr(vtop[0].r);
  1462. vtop -= 2;
  1463. switch (op) {
  1464. case TOK_EQ: case TOK_NE:
  1465. case TOK_LT: case TOK_GE: case TOK_LE: case TOK_GT:
  1466. x = get_reg(RC_INT);
  1467. ++vtop;
  1468. vtop[0].r = x;
  1469. x = intr(x);
  1470. break;
  1471. default:
  1472. x = get_reg(RC_FLOAT);
  1473. ++vtop;
  1474. vtop[0].r = x;
  1475. x = fltr(x);
  1476. break;
  1477. }
  1478. switch (op) {
  1479. case '*':
  1480. o(0x1e200800 | dbl << 22 | x | a << 5 | b << 16); // fmul
  1481. break;
  1482. case '+':
  1483. o(0x1e202800 | dbl << 22 | x | a << 5 | b << 16); // fadd
  1484. break;
  1485. case '-':
  1486. o(0x1e203800 | dbl << 22 | x | a << 5 | b << 16); // fsub
  1487. break;
  1488. case '/':
  1489. o(0x1e201800 | dbl << 22 | x | a << 5 | b << 16); // fdiv
  1490. break;
  1491. case TOK_EQ:
  1492. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1493. o(0x1a9f17e0 | x); // cset w(x),eq
  1494. break;
  1495. case TOK_GE:
  1496. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1497. o(0x1a9fb7e0 | x); // cset w(x),ge
  1498. break;
  1499. case TOK_GT:
  1500. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1501. o(0x1a9fd7e0 | x); // cset w(x),gt
  1502. break;
  1503. case TOK_LE:
  1504. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1505. o(0x1a9f87e0 | x); // cset w(x),ls
  1506. break;
  1507. case TOK_LT:
  1508. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1509. o(0x1a9f57e0 | x); // cset w(x),mi
  1510. break;
  1511. case TOK_NE:
  1512. o(0x1e202000 | dbl << 22 | a << 5 | b << 16); // fcmp
  1513. o(0x1a9f07e0 | x); // cset w(x),ne
  1514. break;
  1515. default:
  1516. assert(0);
  1517. }
  1518. }
  1519. // Generate sign extension from 32 to 64 bits:
  1520. ST_FUNC void gen_cvt_sxtw(void)
  1521. {
  1522. uint32_t r = intr(gv(RC_INT));
  1523. o(0x93407c00 | r | r << 5); // sxtw x(r),w(r)
  1524. }
  1525. ST_FUNC void gen_cvt_itof(int t)
  1526. {
  1527. if (t == VT_LDOUBLE) {
  1528. int f = vtop->type.t;
  1529. int func = (f & VT_BTYPE) == VT_LLONG ?
  1530. (f & VT_UNSIGNED ? TOK___floatunditf : TOK___floatditf) :
  1531. (f & VT_UNSIGNED ? TOK___floatunsitf : TOK___floatsitf);
  1532. vpush_global_sym(&func_old_type, func);
  1533. vrott(2);
  1534. gfunc_call(1);
  1535. vpushi(0);
  1536. vtop->type.t = t;
  1537. vtop->r = REG_FRET;
  1538. return;
  1539. }
  1540. else {
  1541. int d, n = intr(gv(RC_INT));
  1542. int s = !(vtop->type.t & VT_UNSIGNED);
  1543. uint32_t l = ((vtop->type.t & VT_BTYPE) == VT_LLONG);
  1544. --vtop;
  1545. d = get_reg(RC_FLOAT);
  1546. ++vtop;
  1547. vtop[0].r = d;
  1548. o(0x1e220000 | (uint32_t)!s << 16 |
  1549. (uint32_t)(t != VT_FLOAT) << 22 | fltr(d) |
  1550. l << 31 | n << 5); // [us]cvtf [sd](d),[wx](n)
  1551. }
  1552. }
  1553. ST_FUNC void gen_cvt_ftoi(int t)
  1554. {
  1555. if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
  1556. int func = (t & VT_BTYPE) == VT_LLONG ?
  1557. (t & VT_UNSIGNED ? TOK___fixunstfdi : TOK___fixtfdi) :
  1558. (t & VT_UNSIGNED ? TOK___fixunstfsi : TOK___fixtfsi);
  1559. vpush_global_sym(&func_old_type, func);
  1560. vrott(2);
  1561. gfunc_call(1);
  1562. vpushi(0);
  1563. vtop->type.t = t;
  1564. vtop->r = REG_IRET;
  1565. return;
  1566. }
  1567. else {
  1568. int d, n = fltr(gv(RC_FLOAT));
  1569. uint32_t l = ((vtop->type.t & VT_BTYPE) != VT_FLOAT);
  1570. --vtop;
  1571. d = get_reg(RC_INT);
  1572. ++vtop;
  1573. vtop[0].r = d;
  1574. o(0x1e380000 |
  1575. (uint32_t)!!(t & VT_UNSIGNED) << 16 |
  1576. (uint32_t)((t & VT_BTYPE) == VT_LLONG) << 31 | intr(d) |
  1577. l << 22 | n << 5); // fcvtz[su] [wx](d),[sd](n)
  1578. }
  1579. }
  1580. ST_FUNC void gen_cvt_ftof(int t)
  1581. {
  1582. int f = vtop[0].type.t;
  1583. assert(t == VT_FLOAT || t == VT_DOUBLE || t == VT_LDOUBLE);
  1584. assert(f == VT_FLOAT || f == VT_DOUBLE || f == VT_LDOUBLE);
  1585. if (t == f)
  1586. return;
  1587. if (t == VT_LDOUBLE || f == VT_LDOUBLE) {
  1588. int func = (t == VT_LDOUBLE) ?
  1589. (f == VT_FLOAT ? TOK___extendsftf2 : TOK___extenddftf2) :
  1590. (t == VT_FLOAT ? TOK___trunctfsf2 : TOK___trunctfdf2);
  1591. vpush_global_sym(&func_old_type, func);
  1592. vrott(2);
  1593. gfunc_call(1);
  1594. vpushi(0);
  1595. vtop->type.t = t;
  1596. vtop->r = REG_FRET;
  1597. }
  1598. else {
  1599. int x, a;
  1600. gv(RC_FLOAT);
  1601. assert(vtop[0].r < VT_CONST);
  1602. a = fltr(vtop[0].r);
  1603. --vtop;
  1604. x = get_reg(RC_FLOAT);
  1605. ++vtop;
  1606. vtop[0].r = x;
  1607. x = fltr(x);
  1608. if (f == VT_FLOAT)
  1609. o(0x1e22c000 | x | a << 5); // fcvt d(x),s(a)
  1610. else
  1611. o(0x1e624000 | x | a << 5); // fcvt s(x),d(a)
  1612. }
  1613. }
  1614. ST_FUNC void ggoto(void)
  1615. {
  1616. arm64_gen_bl_or_b(1);
  1617. --vtop;
  1618. }
  1619. ST_FUNC void gen_clear_cache(void)
  1620. {
  1621. uint32_t beg, end, dsz, isz, p, lab1, b1;
  1622. gv2(RC_INT, RC_INT);
  1623. vpushi(0);
  1624. vtop->r = get_reg(RC_INT);
  1625. vpushi(0);
  1626. vtop->r = get_reg(RC_INT);
  1627. vpushi(0);
  1628. vtop->r = get_reg(RC_INT);
  1629. beg = intr(vtop[-4].r); // x0
  1630. end = intr(vtop[-3].r); // x1
  1631. dsz = intr(vtop[-2].r); // x2
  1632. isz = intr(vtop[-1].r); // x3
  1633. p = intr(vtop[0].r); // x4
  1634. vtop -= 5;
  1635. o(0xd53b0020 | isz); // mrs x(isz),ctr_el0
  1636. o(0x52800080 | p); // mov w(p),#4
  1637. o(0x53104c00 | dsz | isz << 5); // ubfx w(dsz),w(isz),#16,#4
  1638. o(0x1ac02000 | dsz | p << 5 | dsz << 16); // lsl w(dsz),w(p),w(dsz)
  1639. o(0x12000c00 | isz | isz << 5); // and w(isz),w(isz),#15
  1640. o(0x1ac02000 | isz | p << 5 | isz << 16); // lsl w(isz),w(p),w(isz)
  1641. o(0x51000400 | p | dsz << 5); // sub w(p),w(dsz),#1
  1642. o(0x8a240004 | p | beg << 5 | p << 16); // bic x(p),x(beg),x(p)
  1643. b1 = ind; o(0x14000000); // b
  1644. lab1 = ind;
  1645. o(0xd50b7b20 | p); // dc cvau,x(p)
  1646. o(0x8b000000 | p | p << 5 | dsz << 16); // add x(p),x(p),x(dsz)
  1647. write32le(cur_text_section->data + b1, 0x14000000 | (ind - b1) >> 2);
  1648. o(0xeb00001f | p << 5 | end << 16); // cmp x(p),x(end)
  1649. o(0x54ffffa3 | ((lab1 - ind) << 3 & 0xffffe0)); // b.cc lab1
  1650. o(0xd5033b9f); // dsb ish
  1651. o(0x51000400 | p | isz << 5); // sub w(p),w(isz),#1
  1652. o(0x8a240004 | p | beg << 5 | p << 16); // bic x(p),x(beg),x(p)
  1653. b1 = ind; o(0x14000000); // b
  1654. lab1 = ind;
  1655. o(0xd50b7520 | p); // ic ivau,x(p)
  1656. o(0x8b000000 | p | p << 5 | isz << 16); // add x(p),x(p),x(isz)
  1657. write32le(cur_text_section->data + b1, 0x14000000 | (ind - b1) >> 2);
  1658. o(0xeb00001f | p << 5 | end << 16); // cmp x(p),x(end)
  1659. o(0x54ffffa3 | ((lab1 - ind) << 3 & 0xffffe0)); // b.cc lab1
  1660. o(0xd5033b9f); // dsb ish
  1661. o(0xd5033fdf); // isb
  1662. }
  1663. ST_FUNC void gen_vla_sp_save(int addr) {
  1664. uint32_t r = intr(get_reg(RC_INT));
  1665. o(0x910003e0 | r); // mov x(r),sp
  1666. arm64_strx(3, r, 29, addr);
  1667. }
  1668. ST_FUNC void gen_vla_sp_restore(int addr) {
  1669. // Use x30 because this function can be called when there
  1670. // is a live return value in x0 but there is nothing on
  1671. // the value stack to prevent get_reg from returning x0.
  1672. uint32_t r = 30;
  1673. arm64_ldrx(0, 3, r, 29, addr);
  1674. o(0x9100001f | r << 5); // mov sp,x(r)
  1675. }
  1676. ST_FUNC void gen_vla_alloc(CType *type, int align) {
  1677. uint32_t r = intr(gv(RC_INT));
  1678. o(0x91003c00 | r | r << 5); // add x(r),x(r),#15
  1679. o(0x927cec00 | r | r << 5); // bic x(r),x(r),#15
  1680. o(0xcb2063ff | r << 16); // sub sp,sp,x(r)
  1681. vpop();
  1682. }
  1683. /* end of A64 code generator */
  1684. /*************************************************************/
  1685. #endif
  1686. /*************************************************************/