Skip to content

L
linux
Commit f2810033 authored by Jay Cornwall's avatar Jay Cornwall Committed by Alex Deucher
Browse files
Options

drm/amdkfd: Add gfx12 trap handler support 

- HWREG changes since gfx11
- Save/restore barrier state
- get_wave_size is now reserved by assembler

v2: rebase (Alex)
Signed-off-by: default avatarJay Cornwall <jay.cornwall@amd.com>
Reviewed-by: default avatarHarish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent 385093fd
Hide whitespace changes
Inline Side-by-side
Showing with 607 additions and 42 deletions
drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
View file @ f2810033
......@@ -3645,3 +3645,468 @@ static const uint32_t cwsr_trap_gfx9_4_3_hex[] = {
0xb97a0002, 0xbf8a0000,
0xbe801f6c, 0xbf9b0000,
};
static const uint32_t cwsr_trap_gfx12_hex[] = {
0xbfa00001, 0xbfa0023b,
0xb0804009, 0xb8f8f804,
0x9178ff78, 0x00008c00,
0xb8fbf811, 0x8b6eff78,
0x00004000, 0xbfa10008,
0x8b6eff7b, 0x00000080,
0xbfa20015, 0x8b6ea07b,
0xbfa2003e, 0xbf830010,
0xb8fbf811, 0xbfa0fffb,
0x8b6eff7b, 0x00000050,
0xbfa2000d, 0xb8eef812,
0x8b6f8f7b, 0xbfa10002,
0x8c6eff6e, 0x00000080,
0xb8eff813, 0x8b6f6e6f,
0xbfa20005, 0x8b6eff6d,
0xf0000000, 0xbfa20002,
0x8b6ea07b, 0xbfa2002b,
0xbefa4d82, 0xbf89fc07,
0x84fa887a, 0xbf0d8f7b,
0xbfa10002, 0x8c7bff7b,
0xffff0000, 0xf4601bbd,
0xf8000010, 0xbf89fc07,
0x846e976e, 0x9177ff77,
0x00800000, 0x8c776e77,
0xf4603bbd, 0xf8000000,
0xbf89fc07, 0xf4603ebd,
0xf8000008, 0xbf89fc07,
0x8bee6e6e, 0xbfa10001,
0xbe80486e, 0x8b6eff6d,
0xf0000000, 0xbfa20009,
0xb8eef811, 0x8b6eff6e,
0x00000080, 0xbfa20007,
0x8c78ff78, 0x00004000,
0x80ec886c, 0x82ed806d,
0xbfa00002, 0x806c846c,
0x826d806d, 0x8b6dff6d,
0x0000ffff, 0x8bfe7e7e,
0x8bea6a6a, 0xb978f804,
0xbe804a6c, 0x8b6dff6d,
0x0000ffff, 0xbefa0080,
0xb97a0151, 0xbeee007e,
0xbeef007f, 0xbefe0180,
0xbefe4d84, 0xbf89fc07,
0x8b7aff7f, 0x04000000,
0x847a857a, 0x8c6d7a6d,
0xbefa007e, 0x8b7bff7f,
0x0000ffff, 0xbefe00c1,
0xbeff00c1, 0xee0a407a,
0x000c0000, 0x00000000,
0x7e000280, 0xbefe007a,
0xbeff007b, 0xb8fb0742,
0x847b997b, 0xb8fa3b05,
0x807a817a, 0xbf0d997b,
0xbfa20002, 0x847a897a,
0xbfa00001, 0x847a8a7a,
0xb8fb1e06, 0x847b8a7b,
0x807a7b7a, 0x8b7bff7f,
0x0000ffff, 0x807aff7a,
0x00000200, 0x807a7e7a,
0x827b807b, 0xd7610000,
0x00010870, 0xd7610000,
0x00010a71, 0xd7610000,
0x00010c72, 0xd7610000,
0x00010e73, 0xd7610000,
0x00011074, 0xd7610000,
0x00011275, 0xd7610000,
0x00011476, 0xd7610000,
0x00011677, 0xd7610000,
0x00011a79, 0xd7610000,
0x00011c7e, 0xd7610000,
0x00011e7f, 0xbefe00ff,
0x00003fff, 0xbeff0080,
0xee0a407a, 0x000c0000,
0x00004000, 0xd760007a,
0x00011d00, 0xd760007b,
0x00011f00, 0xbefe007a,
0xbeff007b, 0xbef4007e,
0x8b75ff7f, 0x0000ffff,
0x8c75ff75, 0x00040000,
0xbef60080, 0xbef700ff,
0x10807fac, 0xbef1007d,
0xbef00080, 0xb8f30742,
0x84739973, 0xbefe00c1,
0x857d9973, 0x8b7d817d,
0xbf06817d, 0xbfa20002,
0xbeff0080, 0xbfa00002,
0xbeff00c1, 0xbfa0000c,
0xbef600ff, 0x01000000,
0xc4068070, 0x008ce801,
0x00008000, 0xc4068070,
0x008ce802, 0x00010000,
0xc4068070, 0x008ce803,
0x00018000, 0xbfa0000b,
0xbef600ff, 0x01000000,
0xc4068070, 0x008ce801,
0x00010000, 0xc4068070,
0x008ce802, 0x00020000,
0xc4068070, 0x008ce803,
0x00030000, 0xb8f03b05,
0x80708170, 0xbf0d9973,
0xbfa20002, 0x84708970,
0xbfa00001, 0x84708a70,
0xb8fa1e06, 0x847a8a7a,
0x80707a70, 0x8070ff70,
0x00000200, 0xbef600ff,
0x01000000, 0x7e000280,
0x7e020280, 0x7e040280,
0xbefd0080, 0xbe804ec2,
0xbf94fffe, 0xd7610002,
0x0000fa71, 0x807d817d,
0xd7610002, 0x0000fa6c,
0x807d817d, 0x917aff6d,
0x80000000, 0xd7610002,
0x0000fa7a, 0x807d817d,
0xd7610002, 0x0000fa6e,
0x807d817d, 0xd7610002,
0x0000fa6f, 0x807d817d,
0xd7610002, 0x0000fa78,
0x807d817d, 0xb8faf811,
0xd7610002, 0x0000fa7a,
0x807d817d, 0xd7610002,
0x0000fa7b, 0x807d817d,
0xb8f1f801, 0xd7610002,
0x0000fa71, 0x807d817d,
0xb8f1f814, 0xd7610002,
0x0000fa71, 0x807d817d,
0xb8f1f815, 0xd7610002,
0x0000fa71, 0x807d817d,
0xb8f1f812, 0xd7610002,
0x0000fa71, 0x807d817d,
0xb8f1f813, 0xd7610002,
0x0000fa71, 0x807d817d,
0xb8faf802, 0xd7610002,
0x0000fa7a, 0x807d817d,
0xbefe00ff, 0x0000ffff,
0xbeff0080, 0xc4068070,
0x008ce802, 0x00000000,
0xbefe00c1, 0xb8f03b05,
0x80708170, 0xbf0d9973,
0xbfa20002, 0x84708970,
0xbfa00001, 0x84708a70,
0xb8fa1e06, 0x847a8a7a,
0x80707a70, 0xbef600ff,
0x01000000, 0xbef90080,
0xbefd0080, 0xbf800000,
0xbe804100, 0xbe824102,
0xbe844104, 0xbe864106,
0xbe884108, 0xbe8a410a,
0xbe8c410c, 0xbe8e410e,
0xd7610002, 0x0000f200,
0x80798179, 0xd7610002,
0x0000f201, 0x80798179,
0xd7610002, 0x0000f202,
0x80798179, 0xd7610002,
0x0000f203, 0x80798179,
0xd7610002, 0x0000f204,
0x80798179, 0xd7610002,
0x0000f205, 0x80798179,
0xd7610002, 0x0000f206,
0x80798179, 0xd7610002,
0x0000f207, 0x80798179,
0xd7610002, 0x0000f208,
0x80798179, 0xd7610002,
0x0000f209, 0x80798179,
0xd7610002, 0x0000f20a,
0x80798179, 0xd7610002,
0x0000f20b, 0x80798179,
0xd7610002, 0x0000f20c,
0x80798179, 0xd7610002,
0x0000f20d, 0x80798179,
0xd7610002, 0x0000f20e,
0x80798179, 0xd7610002,
0x0000f20f, 0x80798179,
0xbf06a079, 0xbfa10007,
0xc4068070, 0x008ce802,
0x00000000, 0x8070ff70,
0x00000080, 0xbef90080,
0x7e040280, 0x807d907d,
0xbf0aff7d, 0x00000060,
0xbfa2ffbb, 0xbe804100,
0xbe824102, 0xbe844104,
0xbe864106, 0xbe884108,
0xbe8a410a, 0xd7610002,
0x0000f200, 0x80798179,
0xd7610002, 0x0000f201,
0x80798179, 0xd7610002,
0x0000f202, 0x80798179,
0xd7610002, 0x0000f203,
0x80798179, 0xd7610002,
0x0000f204, 0x80798179,
0xd7610002, 0x0000f205,
0x80798179, 0xd7610002,
0x0000f206, 0x80798179,
0xd7610002, 0x0000f207,
0x80798179, 0xd7610002,
0x0000f208, 0x80798179,
0xd7610002, 0x0000f209,
0x80798179, 0xd7610002,
0x0000f20a, 0x80798179,
0xd7610002, 0x0000f20b,
0x80798179, 0xc4068070,
0x008ce802, 0x00000000,
0xbefe00c1, 0x857d9973,
0x8b7d817d, 0xbf06817d,
0xbfa20002, 0xbeff0080,
0xbfa00001, 0xbeff00c1,
0xb8fb4306, 0x8b7bc17b,
0xbfa10045, 0x8b7aff6d,
0x80000000, 0xbfa10042,
0x847b867b, 0x847b827b,
0xbef6007b, 0xb8f03b05,
0x80708170, 0xbf0d9973,
0xbfa20002, 0x84708970,
0xbfa00001, 0x84708a70,
0xb8fa1e06, 0x847a8a7a,
0x80707a70, 0x8070ff70,
0x00000200, 0x8070ff70,
0x00000080, 0xbef600ff,
0x01000000, 0xd71f0000,
0x000100c1, 0xd7200000,
0x000200c1, 0x16000084,
0x857d9973, 0x8b7d817d,
0xbf06817d, 0xbefd0080,
0xbfa20013, 0xbe8300ff,
0x00000080, 0xbf800000,
0xbf800000, 0xbf800000,
0xd8d80000, 0x01000000,
0xbf890000, 0xc4068070,
0x008ce801, 0x00000000,
0x807d037d, 0x80700370,
0xd5250000, 0x0001ff00,
0x00000080, 0xbf0a7b7d,
0xbfa2fff3, 0xbfa00012,
0xbe8300ff, 0x00000100,
0xbf800000, 0xbf800000,
0xbf800000, 0xd8d80000,
0x01000000, 0xbf890000,
0xc4068070, 0x008ce801,
0x00000000, 0x807d037d,
0x80700370, 0xd5250000,
0x0001ff00, 0x00000100,
0xbf0a7b7d, 0xbfa2fff3,
0xbefe00c1, 0x857d9973,
0x8b7d817d, 0xbf06817d,
0xbfa20004, 0xbef000ff,
0x00000200, 0xbeff0080,
0xbfa00003, 0xbef000ff,
0x00000400, 0xbeff00c1,
0xb8fb3b05, 0x807b817b,
0x847b827b, 0x857d9973,
0x8b7d817d, 0xbf06817d,
0xbfa2001b, 0xbef600ff,
0x01000000, 0xbefd0084,
0xbf0a7b7d, 0xbfa10040,
0x7e008700, 0x7e028701,
0x7e048702, 0x7e068703,
0xc4068070, 0x008ce800,
0x00000000, 0xc4068070,
0x008ce801, 0x00008000,
0xc4068070, 0x008ce802,
0x00010000, 0xc4068070,
0x008ce803, 0x00018000,
0x807d847d, 0x8070ff70,
0x00000200, 0xbf0a7b7d,
0xbfa2ffeb, 0xbfa0002a,
0xbef600ff, 0x01000000,
0xbefd0084, 0xbf0a7b7d,
0xbfa10015, 0x7e008700,
0x7e028701, 0x7e048702,
0x7e068703, 0xc4068070,
0x008ce800, 0x00000000,
0xc4068070, 0x008ce801,
0x00010000, 0xc4068070,
0x008ce802, 0x00020000,
0xc4068070, 0x008ce803,
0x00030000, 0x807d847d,
0x8070ff70, 0x00000400,
0xbf0a7b7d, 0xbfa2ffeb,
0xb8fb1e06, 0x8b7bc17b,
0xbfa1000d, 0x847b837b,
0x807b7d7b, 0xbefe00c1,
0xbeff0080, 0x7e008700,
0xc4068070, 0x008ce800,
0x00000000, 0x807d817d,
0x8070ff70, 0x00000080,
0xbf0a7b7d, 0xbfa2fff7,
0xbfa00159, 0xbef4007e,
0x8b75ff7f, 0x0000ffff,
0x8c75ff75, 0x00040000,
0xbef60080, 0xbef700ff,
0x10807fac, 0xb8f20742,
0x84729972, 0x8b6eff7f,
0x04000000, 0xbfa1003c,
0xbefe00c1, 0x857d9972,
0x8b7d817d, 0xbf06817d,
0xbfa20002, 0xbeff0080,
0xbfa00001, 0xbeff00c1,
0xb8ef4306, 0x8b6fc16f,
0xbfa10031, 0x846f866f,
0x846f826f, 0xbef6006f,
0xb8f83b05, 0x80788178,
0xbf0d9972, 0xbfa20002,
0x84788978, 0xbfa00001,
0x84788a78, 0xb8ee1e06,
0x846e8a6e, 0x80786e78,
0x8078ff78, 0x00000200,
0x8078ff78, 0x00000080,
0xbef600ff, 0x01000000,
0x857d9972, 0x8b7d817d,
0xbf06817d, 0xbefd0080,
0xbfa2000d, 0xc4050078,
0x0080e800, 0x00000000,
0xbf8903f7, 0xdac00000,
0x00000000, 0x807dff7d,
0x00000080, 0x8078ff78,
0x00000080, 0xbf0a6f7d,
0xbfa2fff4, 0xbfa0000c,
0xc4050078, 0x0080e800,
0x00000000, 0xbf8903f7,
0xdac00000, 0x00000000,
0x807dff7d, 0x00000100,
0x8078ff78, 0x00000100,
0xbf0a6f7d, 0xbfa2fff4,
0xbef80080, 0xbefe00c1,
0x857d9972, 0x8b7d817d,
0xbf06817d, 0xbfa20002,
0xbeff0080, 0xbfa00001,
0xbeff00c1, 0xb8ef3b05,
0x806f816f, 0x846f826f,
0x857d9972, 0x8b7d817d,
0xbf06817d, 0xbfa2002c,
0xbef600ff, 0x01000000,
0xbeee0078, 0x8078ff78,
0x00000200, 0xbefd0084,
0xbf0a6f7d, 0xbfa10061,
0xc4050078, 0x008ce800,
0x00000000, 0xc4050078,
0x008ce801, 0x00008000,
0xc4050078, 0x008ce802,
0x00010000, 0xc4050078,
0x008ce803, 0x00018000,
0xbf8903f7, 0x7e008500,
0x7e028501, 0x7e048502,
0x7e068503, 0x807d847d,
0x8078ff78, 0x00000200,
0xbf0a6f7d, 0xbfa2ffea,
0xc405006e, 0x008ce800,
0x00000000, 0xc405006e,
0x008ce801, 0x00008000,
0xc405006e, 0x008ce802,
0x00010000, 0xc405006e,
0x008ce803, 0x00018000,
0xbf8903f7, 0xbfa0003d,
0xbef600ff, 0x01000000,
0xbeee0078, 0x8078ff78,
0x00000400, 0xbefd0084,
0xbf0a6f7d, 0xbfa10016,
0xc4050078, 0x008ce800,
0x00000000, 0xc4050078,
0x008ce801, 0x00010000,
0xc4050078, 0x008ce802,
0x00020000, 0xc4050078,
0x008ce803, 0x00030000,
0xbf8903f7, 0x7e008500,
0x7e028501, 0x7e048502,
0x7e068503, 0x807d847d,
0x8078ff78, 0x00000400,
0xbf0a6f7d, 0xbfa2ffea,
0xb8ef1e06, 0x8b6fc16f,
0xbfa1000f, 0x846f836f,
0x806f7d6f, 0xbefe00c1,
0xbeff0080, 0xc4050078,
0x008ce800, 0x00000000,
0xbf8903f7, 0x7e008500,
0x807d817d, 0x8078ff78,
0x00000080, 0xbf0a6f7d,
0xbfa2fff6, 0xbeff00c1,
0xc405006e, 0x008ce800,
0x00000000, 0xc405006e,
0x008ce801, 0x00010000,
0xc405006e, 0x008ce802,
0x00020000, 0xc405006e,
0x008ce803, 0x00030000,
0xbf8903f7, 0xb8f83b05,
0x80788178, 0xbf0d9972,
0xbfa20002, 0x84788978,
0xbfa00001, 0x84788a78,
0xb8ee1e06, 0x846e8a6e,
0x80786e78, 0x8078ff78,
0x00000200, 0x80f8ff78,
0x00000050, 0xbef600ff,
0x01000000, 0xbefd00ff,
0x0000006c, 0x80f89078,
0xf462403a, 0xf0000000,
0xbf89fc07, 0x80fd847d,
0xbf800000, 0xbe804300,
0xbe824302, 0x80f8a078,
0xf462603a, 0xf0000000,
0xbf89fc07, 0x80fd887d,
0xbf800000, 0xbe804300,
0xbe824302, 0xbe844304,
0xbe864306, 0x80f8c078,
0xf462803a, 0xf0000000,
0xbf89fc07, 0x80fd907d,
0xbf800000, 0xbe804300,
0xbe824302, 0xbe844304,
0xbe864306, 0xbe884308,
0xbe8a430a, 0xbe8c430c,
0xbe8e430e, 0xbf06807d,
0xbfa1fff0, 0xb980f801,
0x00000000, 0xb8f83b05,
0x80788178, 0xbf0d9972,
0xbfa20002, 0x84788978,
0xbfa00001, 0x84788a78,
0xb8ee1e06, 0x846e8a6e,
0x80786e78, 0x8078ff78,
0x00000200, 0xbef600ff,
0x01000000, 0xf4621bfa,
0xf0000000, 0x80788478,
0xf4621b3a, 0xf0000000,
0x80788478, 0xf4621b7a,
0xf0000000, 0x80788478,
0xf4621c3a, 0xf0000000,
0x80788478, 0xf4621c7a,
0xf0000000, 0x80788478,
0xf4621eba, 0xf0000000,
0x80788478, 0xf4621efa,
0xf0000000, 0x80788478,
0xf4621e7a, 0xf0000000,
0x80788478, 0xf4621cfa,
0xf0000000, 0x80788478,
0xf4621bba, 0xf0000000,
0x80788478, 0xbf89fc07,
0xb96ef814, 0xf4621bba,
0xf0000000, 0x80788478,
0xbf89fc07, 0xb96ef815,
0xf4621bba, 0xf0000000,
0x80788478, 0xbf89fc07,
0xb96ef812, 0xf4621bba,
0xf0000000, 0x80788478,
0xbf89fc07, 0xb96ef813,
0xbe804ec2, 0xbf94fffe,
0xbefd006f, 0xbefe0070,
0xbeff0071, 0xb97bf811,
0xb973f801, 0xb8ee3b05,
0x806e816e, 0xbf0d9972,
0xbfa20002, 0x846e896e,
0xbfa00001, 0x846e8a6e,
0xb8ef1e06, 0x846f8a6f,
0x806e6f6e, 0x806eff6e,
0x00000200, 0x806e746e,
0x826f8075, 0x8b6fff6f,
0x0000ffff, 0xf4605c37,
0xf8000050, 0xf4605d37,
0xf8000060, 0xf4601e77,
0xf8000074, 0xbf89fc07,
0x8b6dff6d, 0x0000ffff,
0x8bfe7e7e, 0x8bea6a6a,
0xb97af804, 0xbe804a6c,
0xbfb00000, 0xbf9f0000,
0xbf9f0000, 0xbf9f0000,
0xbf9f0000, 0xbf9f0000,
};
drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
View file @ f2810033
......@@ -33,36 +33,68 @@
* gfx11:
* cpp -DASIC_FAMILY=CHIP_PLUM_BONITO cwsr_trap_handler_gfx10.asm -P -o gfx11.sp3
* sp3 gfx11.sp3 -hex gfx11.hex
*
* gfx12:
* cpp -DASIC_FAMILY=CHIP_GFX12 cwsr_trap_handler_gfx10.asm -P -o gfx12.sp3
* sp3 gfx12.sp3 -hex gfx12.hex
*/
#define CHIP_NAVI10 26
#define CHIP_SIENNA_CICHLID 30
#define CHIP_PLUM_BONITO 36
#define CHIP_GFX12 37
#define NO_SQC_STORE (ASIC_FAMILY >= CHIP_SIENNA_CICHLID)
#define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)
#define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)
#define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)
#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO)
#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO && ASIC_FAMILY < CHIP_GFX12)
#define SAVE_AFTER_XNACK_ERROR (HAVE_XNACK && !NO_SQC_STORE) // workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger
#define SINGLE_STEP_MISSED_WORKAROUND (ASIC_FAMILY < CHIP_GFX12) //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
#if ASIC_FAMILY < CHIP_GFX12
#define S_COHERENCE glc:1
#define V_COHERENCE slc:1 glc:1
#else
#define S_COHERENCE scope:SCOPE_SYS
#define V_COHERENCE scope:SCOPE_SYS
#define HW_REG_SHADER_FLAT_SCRATCH_LO HW_REG_WAVE_SCRATCH_BASE_LO
#define HW_REG_SHADER_FLAT_SCRATCH_HI HW_REG_WAVE_SCRATCH_BASE_HI
#define HW_REG_GPR_ALLOC HW_REG_WAVE_GPR_ALLOC
#define HW_REG_LDS_ALLOC HW_REG_WAVE_LDS_ALLOC
#define HW_REG_MODE HW_REG_WAVE_MODE
#endif
var SINGLE_STEP_MISSED_WORKAROUND = 1 //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
#if ASIC_FAMILY < CHIP_GFX12
var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
var SQ_WAVE_STATUS_HALT_MASK = 0x2000
var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x20000
var SQ_WAVE_STATUS_TRAP_EN_SHIFT = 6
var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11
var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1
var S_STATUS_HWREG = HW_REG_STATUS
var S_STATUS_ALWAYS_CLEAR_MASK = SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK
var S_STATUS_HALT_MASK = SQ_WAVE_STATUS_HALT_MASK
var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000
var S_SAVE_PC_HI_HT_MASK = 0x01000000
#else
var SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK = 0xC00
var SQ_WAVE_STATE_PRIV_HALT_MASK = 0x4000
var SQ_WAVE_STATE_PRIV_POISON_ERR_MASK = 0x8000
var SQ_WAVE_STATUS_WAVE64_SHIFT = 29
var SQ_WAVE_STATUS_WAVE64_SIZE = 1
var S_STATUS_HWREG = HW_REG_WAVE_STATE_PRIV
var S_STATUS_ALWAYS_CLEAR_MASK = SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK
var S_STATUS_HALT_MASK = SQ_WAVE_STATE_PRIV_HALT_MASK
var S_SAVE_PC_HI_TRAP_ID_MASK = 0xF0000000
#endif
var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8
var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24
var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4
var SQ_WAVE_IB_STS2_WAVE64_SHIFT = 11
var SQ_WAVE_IB_STS2_WAVE64_SIZE = 1
#if ASIC_FAMILY < CHIP_PLUM_BONITO
var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
......@@ -70,6 +102,7 @@ var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 8
var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12
#endif
#if ASIC_FAMILY < CHIP_GFX12
var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x400
var SQ_WAVE_TRAPSTS_EXCP_MASK = 0x1FF
var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 10
......@@ -77,12 +110,6 @@ var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK = 0x80
var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT = 7
var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x100
var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 8
var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF
var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x0
var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 10
var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF800
var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 11
var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 21
var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x800
var SQ_WAVE_TRAPSTS_EXCP_HI_MASK = 0x7000
#if ASIC_FAMILY >= CHIP_PLUM_BONITO
......@@ -111,6 +138,23 @@ var S_TRAPSTS_NON_MASKABLE_EXCP_MASK = SQ_WAVE_TRAPSTS_MEM_VIOL_MASK |\
SQ_WAVE_TRAPSTS_WAVE_END_MASK |\
SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK
#endif
var S_TRAPSTS_HWREG = HW_REG_TRAPSTS
var S_TRAPSTS_SAVE_CONTEXT_MASK = SQ_WAVE_TRAPSTS_SAVECTX_MASK
var S_TRAPSTS_SAVE_CONTEXT_SHIFT = SQ_WAVE_TRAPSTS_SAVECTX_SHIFT
#else
var SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK = 0xF
var SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK = 0x10
var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT = 5
var SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK = 0x20
var SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK = 0x40
var SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK = 0x80
var SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK = 0x80
var S_TRAPSTS_HWREG = HW_REG_WAVE_EXCP_FLAG_PRIV
var S_TRAPSTS_SAVE_CONTEXT_MASK = SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_MASK
var S_TRAPSTS_SAVE_CONTEXT_SHIFT = SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT
var S_TRAPSTS_NON_MASKABLE_EXCP_MASK = SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK|SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_MASK
#endif
// bits [31:24] unused by SPI debug data
var TTMP11_SAVE_REPLAY_W64H_SHIFT = 31
......@@ -124,8 +168,6 @@ var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000
// when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000
var S_SAVE_BUF_RSRC_WORD3_MISC = 0x10807FAC
var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000
var S_SAVE_PC_HI_HT_MASK = 0x01000000
var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000
var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26
......@@ -197,13 +239,13 @@ L_JUMP_TO_RESTORE:
s_branch L_RESTORE
L_SKIP_RESTORE:
s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC
s_getreg_b32 s_save_status, hwreg(S_STATUS_HWREG) //save STATUS since we will change SCC
// Clear SPI_PRIO: do not save with elevated priority.
// Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.
s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK
s_andn2_b32 s_save_status, s_save_status, S_STATUS_ALWAYS_CLEAR_MASK
s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
s_getreg_b32 s_save_trapsts, hwreg(S_TRAPSTS_HWREG)
#if SW_SA_TRAP
// If ttmp1[30] is set then issue s_barrier to unblock dependent waves.
......@@ -218,23 +260,27 @@ L_TRAP_NO_BARRIER:
s_cbranch_scc1 L_CHECK_SAVE
#endif
s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
s_and_b32 ttmp2, s_save_status, S_STATUS_HALT_MASK
s_cbranch_scc0 L_NOT_HALTED
L_HALTED:
// Host trap may occur while wave is halted.
#if ASIC_FAMILY < CHIP_GFX12
s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
#else
s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK
#endif
s_cbranch_scc1 L_FETCH_2ND_TRAP
L_CHECK_SAVE:
s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
s_and_b32 ttmp2, s_save_trapsts, S_TRAPSTS_SAVE_CONTEXT_MASK
s_cbranch_scc1 L_SAVE
// Wave is halted but neither host trap nor SAVECTX is raised.
// Caused by instruction fetch memory violation.
// Spin wait until context saved to prevent interrupt storm.
s_sleep 0x10
s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)
s_getreg_b32 s_save_trapsts, hwreg(S_TRAPSTS_HWREG)
s_branch L_CHECK_SAVE
L_NOT_HALTED:
......@@ -250,6 +296,7 @@ L_NOT_HALTED:
// Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.
// Maskable exceptions only cause the wave to enter the trap handler if
// their respective bit in mode.excp_en is set.
#if ASIC_FAMILY < CHIP_GFX12
s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK
s_cbranch_scc0 L_CHECK_TRAP_ID
......@@ -262,21 +309,32 @@ L_NOT_ADDR_WATCH:
s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
s_and_b32 ttmp2, ttmp2, ttmp3
s_cbranch_scc1 L_FETCH_2ND_TRAP
#else
s_getreg_b32 ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)
s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK
s_cbranch_scc0 L_NOT_ADDR_WATCH
s_or_b32 ttmp2, ttmp2, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK
L_NOT_ADDR_WATCH:
s_getreg_b32 ttmp3, hwreg(HW_REG_WAVE_TRAP_CTRL)
s_and_b32 ttmp3, ttmp3, ttmp2
s_cbranch_scc1 L_FETCH_2ND_TRAP
#endif
L_CHECK_TRAP_ID:
// Check trap_id != 0
s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
s_cbranch_scc1 L_FETCH_2ND_TRAP
if SINGLE_STEP_MISSED_WORKAROUND
#if SINGLE_STEP_MISSED_WORKAROUND
// Prioritize single step exception over context save.
// Second-level trap will halt wave and RFE, re-entering for SAVECTX.
s_getreg_b32 ttmp2, hwreg(HW_REG_MODE)
s_and_b32 ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK
s_cbranch_scc1 L_FETCH_2ND_TRAP
end
#endif
s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK
s_and_b32 ttmp2, s_save_trapsts, S_TRAPSTS_SAVE_CONTEXT_MASK
s_cbranch_scc1 L_SAVE
L_FETCH_2ND_TRAP:
......@@ -318,9 +376,18 @@ L_NO_SIGN_EXTEND_TMA:
L_NO_NEXT_TRAP:
// If not caused by trap then halt wave to prevent re-entry.
s_and_b32 ttmp2, s_save_pc_hi, (S_SAVE_PC_HI_TRAP_ID_MASK|S_SAVE_PC_HI_HT_MASK)
s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
s_cbranch_scc1 L_TRAP_CASE
s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK
// Host trap will not cause trap re-entry.
#if ASIC_FAMILY < CHIP_GFX12
s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK
#else
s_getreg_b32 ttmp2, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV)
s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK
#endif
s_cbranch_scc1 L_EXIT_TRAP
s_or_b32 s_save_status, s_save_status, S_STATUS_HALT_MASK
// If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.
// Rewind the PC to prevent this from occurring.
......@@ -330,10 +397,6 @@ L_NO_NEXT_TRAP:
s_branch L_EXIT_TRAP
L_TRAP_CASE:
// Host trap will not cause trap re-entry.
s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK
s_cbranch_scc1 L_EXIT_TRAP
// Advance past trap instruction to prevent re-entry.
s_add_u32 ttmp0, ttmp0, 0x4
s_addc_u32 ttmp1, ttmp1, 0x0
......@@ -348,14 +411,14 @@ L_EXIT_TRAP:
// Restore SQ_WAVE_STATUS.
s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status
s_setreg_b32 hwreg(S_STATUS_HWREG), s_save_status
s_rfe_b64 [ttmp0, ttmp1]
L_SAVE:
s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]
s_mov_b32 s_save_tmp, 0
s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit
s_setreg_b32 hwreg(S_TRAPSTS_HWREG, S_TRAPSTS_SAVE_CONTEXT_SHIFT, 1), s_save_tmp //clear saveCtx bit
#if HAVE_XNACK
save_and_clear_ib_sts(s_save_tmp, s_save_trapsts)
......@@ -410,7 +473,7 @@ L_SLEEP:
// Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
// ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40
get_wave_size(s_save_ttmps_hi)
get_wave_size2(s_save_ttmps_hi)
get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi)
get_svgpr_size_bytes(s_save_ttmps_hi)
s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi
......@@ -456,7 +519,7 @@ L_SLEEP:
/* global mem offset */
s_mov_b32 s_save_mem_offset, 0x0
get_wave_size(s_wave_size)
get_wave_size2(s_wave_size)
#if HAVE_XNACK
// Save and clear vector XNACK state late to free up SGPRs.
......@@ -538,6 +601,12 @@ L_SAVE_HWREG:
s_mov_b32 m0, 0x0 //Next lane of v2 to write to
#endif
#if ASIC_FAMILY >= CHIP_GFX12
// Ensure no further changes to barrier or LDS state.
s_barrier_signal -2
s_barrier_wait -2
#endif
write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)
s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
......@@ -546,7 +615,7 @@ L_SAVE_HWREG:
write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)
s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)
s_getreg_b32 s_save_tmp, hwreg(S_TRAPSTS_HWREG)
write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
// Not used on Sienna_Cichlid but keep layout same for debugger.
......@@ -561,6 +630,17 @@ L_SAVE_HWREG:
s_getreg_b32 s_save_m0, hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI)
write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
#if ASIC_FAMILY >= CHIP_GFX12
s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_EXCP_FLAG_USER)
write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_TRAP_CTRL)
write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATUS)
write_hwreg_to_mem(s_save_tmp, s_save_buf_rsrc0, s_save_mem_offset)
#endif
#if NO_SQC_STORE
// Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this.
s_mov_b32 exec_lo, 0xFFFF
......@@ -653,7 +733,9 @@ L_SAVE_LDS_NORMAL:
s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?
s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE
#if ASIC_FAMILY < CHIP_GFX12
s_barrier //LDS is used? wait for other waves in the same TG
#endif
s_and_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK
s_cbranch_scc0 L_SAVE_LDS_DONE
......@@ -920,7 +1002,7 @@ L_RESTORE:
s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC
//determine it is wave32 or wave64
get_wave_size(s_restore_size)
get_wave_size2(s_restore_size)
s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
s_cbranch_scc0 L_RESTORE_VGPR
......@@ -1154,7 +1236,9 @@ L_RESTORE_SGPR:
// s_barrier with MODE.DEBUG_EN=1, STATUS.PRIV=1 incorrectly asserts debug exception.
// Clear DEBUG_EN before and restore MODE after the barrier.
s_setreg_imm32_b32 hwreg(HW_REG_MODE), 0
#if ASIC_FAMILY < CHIP_GFX12
s_barrier //barrier to ensure the readiness of LDS before access attemps from any other wave in the same TG
#endif
/* restore HW registers */
L_RESTORE_HWREG:
......@@ -1185,20 +1269,30 @@ L_RESTORE_HWREG:
s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s_restore_flat_scratch
#if ASIC_FAMILY >= CHIP_GFX12
read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset)
s_waitcnt lgkmcnt(0)
s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_USER), s_restore_tmp
read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset)
s_waitcnt lgkmcnt(0)
s_setreg_b32 hwreg(HW_REG_WAVE_TRAP_CTRL), s_restore_tmp
L_SKIP_BARRIER_RESTORE:
// Make barrier and LDS state visible to all waves in the group.
s_barrier_signal -2
s_barrier_wait -2
#endif
s_mov_b32 m0, s_restore_m0
s_mov_b32 exec_lo, s_restore_exec_lo
s_mov_b32 exec_hi, s_restore_exec_hi
s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
#if HAVE_XNACK
s_setreg_b32 hwreg(HW_REG_SHADER_XNACK_MASK), s_restore_xnack_mask
#endif
s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
s_setreg_b32 hwreg(S_TRAPSTS_HWREG), s_restore_trapsts
s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode
// Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic
......@@ -1235,7 +1329,7 @@ L_RESTORE_HWREG:
L_RETURN_WITHOUT_PRIV:
#endif
s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
s_setreg_b32 hwreg(S_STATUS_HWREG), s_restore_status // SCC is included, which is changed by previous salu
s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution
L_END_PGM:
......@@ -1378,11 +1472,16 @@ function get_hwreg_size_bytes
return 128
end
function get_wave_size(s_reg)
function get_wave_size2(s_reg)
#if ASIC_FAMILY < CHIP_GFX12
s_getreg_b32 s_reg, hwreg(HW_REG_IB_STS2,SQ_WAVE_IB_STS2_WAVE64_SHIFT,SQ_WAVE_IB_STS2_WAVE64_SIZE)
#else
s_getreg_b32 s_reg, hwreg(HW_REG_WAVE_STATUS,SQ_WAVE_STATUS_WAVE64_SHIFT,SQ_WAVE_STATUS_WAVE64_SIZE)
#endif
s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE
end
#if HAVE_XNACK
function save_and_clear_ib_sts(tmp1, tmp2)
// Preserve and clear scalar XNACK state before issuing scalar loads.
// Save IB_STS.REPLAY_W64H[25], RCNT[21:16], FIRST_REPLAY[15] into
......@@ -1407,3 +1506,4 @@ function restore_ib_sts(tmp1, tmp2)
s_or_b32 tmp1, tmp1, tmp2
s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1
end
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7