module.c 13.4 KB
Newer Older
Will Deacon's avatar
Will Deacon committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * AArch64 loadable module support.
 *
 * Copyright (C) 2012 ARM Limited
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

#include <linux/bitops.h>
#include <linux/elf.h>
#include <linux/gfp.h>
Andrey Ryabinin's avatar
Andrey Ryabinin committed
24
#include <linux/kasan.h>
Will Deacon's avatar
Will Deacon committed
25 26 27 28
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/moduleloader.h>
#include <linux/vmalloc.h>
29
#include <asm/alternative.h>
30
#include <asm/insn.h>
31
#include <asm/sections.h>
32

Will Deacon's avatar
Will Deacon committed
33 34
void *module_alloc(unsigned long size)
{
35
	gfp_t gfp_mask = GFP_KERNEL;
Andrey Ryabinin's avatar
Andrey Ryabinin committed
36 37
	void *p;

38 39 40 41
	/* Silence the initial allocation */
	if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS))
		gfp_mask |= __GFP_NOWARN;

42 43
	p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base,
				module_alloc_base + MODULES_VSIZE,
44
				gfp_mask, PAGE_KERNEL, 0,
Andrey Ryabinin's avatar
Andrey Ryabinin committed
45 46
				NUMA_NO_NODE, __builtin_return_address(0));

47 48 49 50 51 52 53 54 55 56 57
	if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
	    !IS_ENABLED(CONFIG_KASAN))
		/*
		 * KASAN can only deal with module allocations being served
		 * from the reserved module region, since the remainder of
		 * the vmalloc region is already backed by zero shadow pages,
		 * and punching holes into it is non-trivial. Since the module
		 * region is not randomized when KASAN is enabled, it is even
		 * less likely that the module region gets exhausted, so we
		 * can simply omit this fallback in that case.
		 */
58
		p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base,
59
				module_alloc_base + SZ_2G, GFP_KERNEL,
60
				PAGE_KERNEL, 0, NUMA_NO_NODE,
61
				__builtin_return_address(0));
62

Andrey Ryabinin's avatar
Andrey Ryabinin committed
63 64 65 66 67 68
	if (p && (kasan_module_alloc(p, size) < 0)) {
		vfree(p);
		return NULL;
	}

	return p;
Will Deacon's avatar
Will Deacon committed
69 70 71 72 73 74 75 76 77
}

enum aarch64_reloc_op {
	RELOC_OP_NONE,
	RELOC_OP_ABS,
	RELOC_OP_PREL,
	RELOC_OP_PAGE,
};

78
static u64 do_reloc(enum aarch64_reloc_op reloc_op, __le32 *place, u64 val)
Will Deacon's avatar
Will Deacon committed
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
{
	switch (reloc_op) {
	case RELOC_OP_ABS:
		return val;
	case RELOC_OP_PREL:
		return val - (u64)place;
	case RELOC_OP_PAGE:
		return (val & ~0xfff) - ((u64)place & ~0xfff);
	case RELOC_OP_NONE:
		return 0;
	}

	pr_err("do_reloc: unknown relocation operation %d\n", reloc_op);
	return 0;
}

static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len)
{
	s64 sval = do_reloc(op, place, val);

99 100
	/*
	 * The ELF psABI for AArch64 documents the 16-bit and 32-bit place
101 102 103 104
	 * relative and absolute relocations as having a range of [-2^15, 2^16)
	 * or [-2^31, 2^32), respectively. However, in order to be able to
	 * detect overflows reliably, we have to choose whether we interpret
	 * such quantities as signed or as unsigned, and stick with it.
105 106 107 108 109 110
	 * The way we organize our address space requires a signed
	 * interpretation of 32-bit relative references, so let's use that
	 * for all R_AARCH64_PRELxx relocations. This means our upper
	 * bound for overflow detection should be Sxx_MAX rather than Uxx_MAX.
	 */

Will Deacon's avatar
Will Deacon committed
111 112 113
	switch (len) {
	case 16:
		*(s16 *)place = sval;
114 115 116 117 118 119 120 121 122 123 124 125 126
		switch (op) {
		case RELOC_OP_ABS:
			if (sval < 0 || sval > U16_MAX)
				return -ERANGE;
			break;
		case RELOC_OP_PREL:
			if (sval < S16_MIN || sval > S16_MAX)
				return -ERANGE;
			break;
		default:
			pr_err("Invalid 16-bit data relocation (%d)\n", op);
			return 0;
		}
Will Deacon's avatar
Will Deacon committed
127 128 129
		break;
	case 32:
		*(s32 *)place = sval;
130 131 132 133 134 135 136 137 138 139 140 141 142
		switch (op) {
		case RELOC_OP_ABS:
			if (sval < 0 || sval > U32_MAX)
				return -ERANGE;
			break;
		case RELOC_OP_PREL:
			if (sval < S32_MIN || sval > S32_MAX)
				return -ERANGE;
			break;
		default:
			pr_err("Invalid 32-bit data relocation (%d)\n", op);
			return 0;
		}
Will Deacon's avatar
Will Deacon committed
143 144 145 146 147 148 149 150 151 152 153
		break;
	case 64:
		*(s64 *)place = sval;
		break;
	default:
		pr_err("Invalid length (%d) for data relocation\n", len);
		return 0;
	}
	return 0;
}

154 155 156 157 158
enum aarch64_insn_movw_imm_type {
	AARCH64_INSN_IMM_MOVNZ,
	AARCH64_INSN_IMM_MOVKZ,
};

159
static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val,
160
			   int lsb, enum aarch64_insn_movw_imm_type imm_type)
Will Deacon's avatar
Will Deacon committed
161
{
162
	u64 imm;
163
	s64 sval;
164
	u32 insn = le32_to_cpu(*place);
Will Deacon's avatar
Will Deacon committed
165

166
	sval = do_reloc(op, place, val);
167
	imm = sval >> lsb;
168

169
	if (imm_type == AARCH64_INSN_IMM_MOVNZ) {
Will Deacon's avatar
Will Deacon committed
170 171 172 173 174 175
		/*
		 * For signed MOVW relocations, we have to manipulate the
		 * instruction encoding depending on whether or not the
		 * immediate is less than zero.
		 */
		insn &= ~(3 << 29);
176
		if (sval >= 0) {
Will Deacon's avatar
Will Deacon committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190
			/* >=0: Set the instruction to MOVZ (opcode 10b). */
			insn |= 2 << 29;
		} else {
			/*
			 * <0: Set the instruction to MOVN (opcode 00b).
			 *     Since we've masked the opcode already, we
			 *     don't need to do anything other than
			 *     inverting the new immediate field.
			 */
			imm = ~imm;
		}
	}

	/* Update the instruction with the new encoding. */
191
	insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
192
	*place = cpu_to_le32(insn);
Will Deacon's avatar
Will Deacon committed
193

194
	if (imm > U16_MAX)
Will Deacon's avatar
Will Deacon committed
195 196 197 198 199
		return -ERANGE;

	return 0;
}

200
static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val,
201
			  int lsb, int len, enum aarch64_insn_imm_type imm_type)
Will Deacon's avatar
Will Deacon committed
202 203 204
{
	u64 imm, imm_mask;
	s64 sval;
205
	u32 insn = le32_to_cpu(*place);
Will Deacon's avatar
Will Deacon committed
206 207 208 209 210 211 212 213 214 215

	/* Calculate the relocation value. */
	sval = do_reloc(op, place, val);
	sval >>= lsb;

	/* Extract the value bits and shift them to bit 0. */
	imm_mask = (BIT(lsb + len) - 1) >> lsb;
	imm = sval & imm_mask;

	/* Update the instruction's immediate field. */
216
	insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
217
	*place = cpu_to_le32(insn);
Will Deacon's avatar
Will Deacon committed
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234

	/*
	 * Extract the upper value bits (including the sign bit) and
	 * shift them to bit 0.
	 */
	sval = (s64)(sval & ~(imm_mask >> 1)) >> (len - 1);

	/*
	 * Overflow has occurred if the upper bits are not all equal to
	 * the sign bit of the value.
	 */
	if ((u64)(sval + 1) >= 2)
		return -ERANGE;

	return 0;
}

235 236
static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs,
			   __le32 *place, u64 val)
237 238 239
{
	u32 insn;

240
	if (!is_forbidden_offset_for_adrp(place))
241 242 243 244 245 246 247 248 249 250
		return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21,
				      AARCH64_INSN_IMM_ADR);

	/* patch ADRP to ADR if it is in range */
	if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21,
			    AARCH64_INSN_IMM_ADR)) {
		insn = le32_to_cpu(*place);
		insn &= ~BIT(31);
	} else {
		/* out of range for ADR -> emit a veneer */
251
		val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff);
252 253 254 255 256 257 258 259 260 261
		if (!val)
			return -ENOEXEC;
		insn = aarch64_insn_gen_branch_imm((u64)place, val,
						   AARCH64_INSN_BRANCH_NOLINK);
	}

	*place = cpu_to_le32(insn);
	return 0;
}

Will Deacon's avatar
Will Deacon committed
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
int apply_relocate_add(Elf64_Shdr *sechdrs,
		       const char *strtab,
		       unsigned int symindex,
		       unsigned int relsec,
		       struct module *me)
{
	unsigned int i;
	int ovf;
	bool overflow_check;
	Elf64_Sym *sym;
	void *loc;
	u64 val;
	Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr;

	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) {
		/* loc corresponds to P in the AArch64 ELF document. */
		loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr
			+ rel[i].r_offset;

		/* sym is the ELF symbol we're referring to. */
		sym = (Elf64_Sym *)sechdrs[symindex].sh_addr
			+ ELF64_R_SYM(rel[i].r_info);

		/* val corresponds to (S + A) in the AArch64 ELF document. */
		val = sym->st_value + rel[i].r_addend;

		/* Check for overflow by default. */
		overflow_check = true;

		/* Perform the static relocation. */
		switch (ELF64_R_TYPE(rel[i].r_info)) {
		/* Null relocations. */
		case R_ARM_NONE:
		case R_AARCH64_NONE:
			ovf = 0;
			break;

		/* Data relocations. */
		case R_AARCH64_ABS64:
			overflow_check = false;
			ovf = reloc_data(RELOC_OP_ABS, loc, val, 64);
			break;
		case R_AARCH64_ABS32:
			ovf = reloc_data(RELOC_OP_ABS, loc, val, 32);
			break;
		case R_AARCH64_ABS16:
			ovf = reloc_data(RELOC_OP_ABS, loc, val, 16);
			break;
		case R_AARCH64_PREL64:
			overflow_check = false;
			ovf = reloc_data(RELOC_OP_PREL, loc, val, 64);
			break;
		case R_AARCH64_PREL32:
			ovf = reloc_data(RELOC_OP_PREL, loc, val, 32);
			break;
		case R_AARCH64_PREL16:
			ovf = reloc_data(RELOC_OP_PREL, loc, val, 16);
			break;

		/* MOVW instruction relocations. */
		case R_AARCH64_MOVW_UABS_G0_NC:
			overflow_check = false;
		case R_AARCH64_MOVW_UABS_G0:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
326
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
327 328 329 330 331
			break;
		case R_AARCH64_MOVW_UABS_G1_NC:
			overflow_check = false;
		case R_AARCH64_MOVW_UABS_G1:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
332
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
333 334 335 336 337
			break;
		case R_AARCH64_MOVW_UABS_G2_NC:
			overflow_check = false;
		case R_AARCH64_MOVW_UABS_G2:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
338
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
339 340 341 342 343
			break;
		case R_AARCH64_MOVW_UABS_G3:
			/* We're using the top bits so we can't overflow. */
			overflow_check = false;
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48,
344
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
345 346 347
			break;
		case R_AARCH64_MOVW_SABS_G0:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
348
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
349 350 351
			break;
		case R_AARCH64_MOVW_SABS_G1:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
352
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
353 354 355
			break;
		case R_AARCH64_MOVW_SABS_G2:
			ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
356
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
357 358 359 360
			break;
		case R_AARCH64_MOVW_PREL_G0_NC:
			overflow_check = false;
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
361
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
362 363 364
			break;
		case R_AARCH64_MOVW_PREL_G0:
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
365
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
366 367 368 369
			break;
		case R_AARCH64_MOVW_PREL_G1_NC:
			overflow_check = false;
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
370
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
371 372 373
			break;
		case R_AARCH64_MOVW_PREL_G1:
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
374
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
375 376 377 378
			break;
		case R_AARCH64_MOVW_PREL_G2_NC:
			overflow_check = false;
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
379
					      AARCH64_INSN_IMM_MOVKZ);
Will Deacon's avatar
Will Deacon committed
380 381 382
			break;
		case R_AARCH64_MOVW_PREL_G2:
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
383
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
384 385 386 387 388
			break;
		case R_AARCH64_MOVW_PREL_G3:
			/* We're using the top bits so we can't overflow. */
			overflow_check = false;
			ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48,
389
					      AARCH64_INSN_IMM_MOVNZ);
Will Deacon's avatar
Will Deacon committed
390 391 392 393 394
			break;

		/* Immediate instruction relocations. */
		case R_AARCH64_LD_PREL_LO19:
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
395
					     AARCH64_INSN_IMM_19);
Will Deacon's avatar
Will Deacon committed
396 397 398
			break;
		case R_AARCH64_ADR_PREL_LO21:
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,
399
					     AARCH64_INSN_IMM_ADR);
Will Deacon's avatar
Will Deacon committed
400 401 402 403
			break;
		case R_AARCH64_ADR_PREL_PG_HI21_NC:
			overflow_check = false;
		case R_AARCH64_ADR_PREL_PG_HI21:
404
			ovf = reloc_insn_adrp(me, sechdrs, loc, val);
405 406
			if (ovf && ovf != -ERANGE)
				return ovf;
Will Deacon's avatar
Will Deacon committed
407 408 409 410 411
			break;
		case R_AARCH64_ADD_ABS_LO12_NC:
		case R_AARCH64_LDST8_ABS_LO12_NC:
			overflow_check = false;
			ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12,
412
					     AARCH64_INSN_IMM_12);
Will Deacon's avatar
Will Deacon committed
413 414 415 416
			break;
		case R_AARCH64_LDST16_ABS_LO12_NC:
			overflow_check = false;
			ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11,
417
					     AARCH64_INSN_IMM_12);
Will Deacon's avatar
Will Deacon committed
418 419 420 421
			break;
		case R_AARCH64_LDST32_ABS_LO12_NC:
			overflow_check = false;
			ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10,
422
					     AARCH64_INSN_IMM_12);
Will Deacon's avatar
Will Deacon committed
423 424 425 426
			break;
		case R_AARCH64_LDST64_ABS_LO12_NC:
			overflow_check = false;
			ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9,
427
					     AARCH64_INSN_IMM_12);
Will Deacon's avatar
Will Deacon committed
428 429 430 431
			break;
		case R_AARCH64_LDST128_ABS_LO12_NC:
			overflow_check = false;
			ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8,
432
					     AARCH64_INSN_IMM_12);
Will Deacon's avatar
Will Deacon committed
433 434 435
			break;
		case R_AARCH64_TSTBR14:
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14,
436
					     AARCH64_INSN_IMM_14);
Will Deacon's avatar
Will Deacon committed
437 438 439
			break;
		case R_AARCH64_CONDBR19:
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
440
					     AARCH64_INSN_IMM_19);
Will Deacon's avatar
Will Deacon committed
441 442 443 444
			break;
		case R_AARCH64_JUMP26:
		case R_AARCH64_CALL26:
			ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26,
445
					     AARCH64_INSN_IMM_26);
446 447 448

			if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
			    ovf == -ERANGE) {
449
				val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym);
450 451
				if (!val)
					return -ENOEXEC;
452 453 454
				ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2,
						     26, AARCH64_INSN_IMM_26);
			}
Will Deacon's avatar
Will Deacon committed
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
			break;

		default:
			pr_err("module %s: unsupported RELA relocation: %llu\n",
			       me->name, ELF64_R_TYPE(rel[i].r_info));
			return -ENOEXEC;
		}

		if (overflow_check && ovf == -ERANGE)
			goto overflow;

	}

	return 0;

overflow:
	pr_err("module %s: overflow in relocation type %d val %Lx\n",
	       me->name, (int)ELF64_R_TYPE(rel[i].r_info), val);
	return -ENOEXEC;
}
475 476 477 478 479 480 481 482 483

int module_finalize(const Elf_Ehdr *hdr,
		    const Elf_Shdr *sechdrs,
		    struct module *me)
{
	const Elf_Shdr *s, *se;
	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;

	for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
484 485
		if (strcmp(".altinstructions", secstrs + s->sh_name) == 0)
			apply_alternatives_module((void *)s->sh_addr, s->sh_size);
486 487 488 489 490
#ifdef CONFIG_ARM64_MODULE_PLTS
		if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
		    !strcmp(".text.ftrace_trampoline", secstrs + s->sh_name))
			me->arch.ftrace_trampoline = (void *)s->sh_addr;
#endif
491 492 493 494
	}

	return 0;
}