Commit 8feae131 authored by David Howells's avatar David Howells

NOMMU: Make VMAs per MM as for MMU-mode linux

Make VMAs per mm_struct as for MMU-mode linux.  This solves two problems:

 (1) In SYSV SHM where nattch for a segment does not reflect the number of
     shmat's (and forks) done.

 (2) In mmap() where the VMA's vm_mm is set to point to the parent mm by an
     exec'ing process when VM_EXECUTABLE is specified, regardless of the fact
     that a VMA might be shared and already have its vm_mm assigned to another
     process or a dead process.

A new struct (vm_region) is introduced to track a mapped region and to remember
the circumstances under which it may be shared and the vm_list_struct structure
is discarded as it's no longer required.

This patch makes the following additional changes:

 (1) Regions are now allocated with alloc_pages() rather than kmalloc() and
     with no recourse to __GFP_COMP, so the pages are not composite.  Instead,
     each page has a reference on it held by the region.  Anything else that is
     interested in such a page will have to get a reference on it to retain it.
     When the pages are released due to unmapping, each page is passed to
     put_page() and will be freed when the page usage count reaches zero.

 (2) Excess pages are trimmed after an allocation as the allocation must be
     made as a power-of-2 quantity of pages.

 (3) VMAs are added to the parent MM's R/B tree and mmap lists.  As an MM may
     end up with overlapping VMAs within the tree, the VMA struct address is
     appended to the sort key.

 (4) Non-anonymous VMAs are now added to the backing inode's prio list.

 (5) Holes may be punched in anonymous VMAs with munmap(), releasing parts of
     the backing region.  The VMA and region structs will be split if
     necessary.

 (6) sys_shmdt() only releases one attachment to a SYSV IPC shared memory
     segment instead of all the attachments at that addresss.  Multiple
     shmat()'s return the same address under NOMMU-mode instead of different
     virtual addresses as under MMU-mode.

 (7) Core dumping for ELF-FDPIC requires fewer exceptions for NOMMU-mode.

 (8) /proc/maps is now the global list of mapped regions, and may list bits
     that aren't actually mapped anywhere.

 (9) /proc/meminfo gains a line (tagged "MmapCopy") that indicates the amount
     of RAM currently allocated by mmap to hold mappable regions that can't be
     mapped directly.  These are copies of the backing device or file if not
     anonymous.

These changes make NOMMU mode more similar to MMU mode.  The downside is that
NOMMU mode requires some extra memory to track things over NOMMU without this
patch (VMAs are no longer shared, and there are now region structs).
Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
Tested-by: default avatarMike Frysinger <vapier.adi@gmail.com>
Acked-by: default avatarPaul Mundt <lethal@linux-sh.org>
parent 41836382
...@@ -109,12 +109,18 @@ and it's also much more restricted in the latter case: ...@@ -109,12 +109,18 @@ and it's also much more restricted in the latter case:
FURTHER NOTES ON NO-MMU MMAP FURTHER NOTES ON NO-MMU MMAP
============================ ============================
(*) A request for a private mapping of less than a page in size may not return (*) A request for a private mapping of a file may return a buffer that is not
a page-aligned buffer. This is because the kernel calls kmalloc() to page-aligned. This is because XIP may take place, and the data may not be
allocate the buffer, not get_free_page(). paged aligned in the backing store.
(*) A list of all the mappings on the system is visible through /proc/maps in (*) A request for an anonymous mapping will always be page aligned. If
no-MMU mode. possible the size of the request should be a power of two otherwise some
of the space may be wasted as the kernel must allocate a power-of-2
granule but will only discard the excess if appropriately configured as
this has an effect on fragmentation.
(*) A list of all the private copy and anonymous mappings on the system is
visible through /proc/maps in no-MMU mode.
(*) A list of all the mappings in use by a process is visible through (*) A list of all the mappings in use by a process is visible through
/proc/<pid>/maps in no-MMU mode. /proc/<pid>/maps in no-MMU mode.
......
...@@ -24,7 +24,6 @@ typedef struct { ...@@ -24,7 +24,6 @@ typedef struct {
* modified for 2.6 by Hyok S. Choi <hyok.choi@samsung.com> * modified for 2.6 by Hyok S. Choi <hyok.choi@samsung.com>
*/ */
typedef struct { typedef struct {
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
} mm_context_t; } mm_context_t;
......
...@@ -10,7 +10,6 @@ struct sram_list_struct { ...@@ -10,7 +10,6 @@ struct sram_list_struct {
}; };
typedef struct { typedef struct {
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
unsigned long stack_start; unsigned long stack_start;
......
...@@ -160,15 +160,15 @@ put_reg(struct task_struct *task, int regno, unsigned long data) ...@@ -160,15 +160,15 @@ put_reg(struct task_struct *task, int regno, unsigned long data)
static inline int is_user_addr_valid(struct task_struct *child, static inline int is_user_addr_valid(struct task_struct *child,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
{ {
struct vm_list_struct *vml; struct vm_area_struct *vma;
struct sram_list_struct *sraml; struct sram_list_struct *sraml;
/* overflow */ /* overflow */
if (start + len < start) if (start + len < start)
return -EIO; return -EIO;
for (vml = child->mm->context.vmlist; vml; vml = vml->next) vma = find_vma(child->mm, start);
if (start >= vml->vma->vm_start && start + len < vml->vma->vm_end) if (vma && start >= vma->vm_start && start + len <= vma->vm_end)
return 0; return 0;
for (sraml = child->mm->context.sram_list; sraml; sraml = sraml->next) for (sraml = child->mm->context.sram_list; sraml; sraml = sraml->next)
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/rbtree.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/cplb.h> #include <asm/cplb.h>
...@@ -83,6 +84,7 @@ static void decode_address(char *buf, unsigned long address) ...@@ -83,6 +84,7 @@ static void decode_address(char *buf, unsigned long address)
struct mm_struct *mm; struct mm_struct *mm;
unsigned long flags, offset; unsigned long flags, offset;
unsigned char in_atomic = (bfin_read_IPEND() & 0x10) || in_atomic(); unsigned char in_atomic = (bfin_read_IPEND() & 0x10) || in_atomic();
struct rb_node *n;
#ifdef CONFIG_KALLSYMS #ifdef CONFIG_KALLSYMS
unsigned long symsize; unsigned long symsize;
...@@ -128,9 +130,10 @@ static void decode_address(char *buf, unsigned long address) ...@@ -128,9 +130,10 @@ static void decode_address(char *buf, unsigned long address)
if (!mm) if (!mm)
continue; continue;
vml = mm->context.vmlist; for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
while (vml) { struct vm_area_struct *vma;
struct vm_area_struct *vma = vml->vma;
vma = rb_entry(n, struct vm_area_struct, vm_rb);
if (address >= vma->vm_start && address < vma->vm_end) { if (address >= vma->vm_start && address < vma->vm_end) {
char _tmpbuf[256]; char _tmpbuf[256];
...@@ -176,8 +179,6 @@ static void decode_address(char *buf, unsigned long address) ...@@ -176,8 +179,6 @@ static void decode_address(char *buf, unsigned long address)
goto done; goto done;
} }
vml = vml->next;
} }
if (!in_atomic) if (!in_atomic)
mmput(mm); mmput(mm);
......
...@@ -69,7 +69,8 @@ static inline int put_reg(struct task_struct *task, int regno, ...@@ -69,7 +69,8 @@ static inline int put_reg(struct task_struct *task, int regno,
} }
/* /*
* check that an address falls within the bounds of the target process's memory mappings * check that an address falls within the bounds of the target process's memory
* mappings
*/ */
static inline int is_user_addr_valid(struct task_struct *child, static inline int is_user_addr_valid(struct task_struct *child,
unsigned long start, unsigned long len) unsigned long start, unsigned long len)
...@@ -79,11 +80,11 @@ static inline int is_user_addr_valid(struct task_struct *child, ...@@ -79,11 +80,11 @@ static inline int is_user_addr_valid(struct task_struct *child,
return -EIO; return -EIO;
return 0; return 0;
#else #else
struct vm_list_struct *vml; struct vm_area_struct *vma;
for (vml = child->mm->context.vmlist; vml; vml = vml->next) vma = find_vma(child->mm, start);
if (start >= vml->vma->vm_start && start + len <= vml->vma->vm_end) if (vma && start >= vma->vm_start && start + len <= vma->vm_end)
return 0; return 0;
return -EIO; return -EIO;
#endif #endif
......
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */ /* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */
typedef struct { typedef struct {
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
} mm_context_t; } mm_context_t;
......
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
/* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */ /* Copyright (C) 2002, David McCullough <davidm@snapgear.com> */
typedef struct { typedef struct {
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
} mm_context_t; } mm_context_t;
......
...@@ -9,7 +9,6 @@ typedef struct { ...@@ -9,7 +9,6 @@ typedef struct {
mm_context_id_t id; mm_context_id_t id;
void *vdso; void *vdso;
#else #else
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
#endif #endif
#ifdef CONFIG_BINFMT_ELF_FDPIC #ifdef CONFIG_BINFMT_ELF_FDPIC
......
...@@ -1567,11 +1567,9 @@ static int elf_fdpic_dump_segments(struct file *file, size_t *size, ...@@ -1567,11 +1567,9 @@ static int elf_fdpic_dump_segments(struct file *file, size_t *size,
static int elf_fdpic_dump_segments(struct file *file, size_t *size, static int elf_fdpic_dump_segments(struct file *file, size_t *size,
unsigned long *limit, unsigned long mm_flags) unsigned long *limit, unsigned long mm_flags)
{ {
struct vm_list_struct *vml; struct vm_area_struct *vma;
for (vml = current->mm->context.vmlist; vml; vml = vml->next) {
struct vm_area_struct *vma = vml->vma;
for (vma = current->mm->mmap; vma; vma = vma->vm_next) {
if (!maydump(vma, mm_flags)) if (!maydump(vma, mm_flags))
continue; continue;
...@@ -1617,9 +1615,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, ...@@ -1617,9 +1615,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
elf_fpxregset_t *xfpu = NULL; elf_fpxregset_t *xfpu = NULL;
#endif #endif
int thread_status_size = 0; int thread_status_size = 0;
#ifndef CONFIG_MMU
struct vm_list_struct *vml;
#endif
elf_addr_t *auxv; elf_addr_t *auxv;
unsigned long mm_flags; unsigned long mm_flags;
...@@ -1685,13 +1680,7 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, ...@@ -1685,13 +1680,7 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
fill_prstatus(prstatus, current, signr); fill_prstatus(prstatus, current, signr);
elf_core_copy_regs(&prstatus->pr_reg, regs); elf_core_copy_regs(&prstatus->pr_reg, regs);
#ifdef CONFIG_MMU
segs = current->mm->map_count; segs = current->mm->map_count;
#else
segs = 0;
for (vml = current->mm->context.vmlist; vml; vml = vml->next)
segs++;
#endif
#ifdef ELF_CORE_EXTRA_PHDRS #ifdef ELF_CORE_EXTRA_PHDRS
segs += ELF_CORE_EXTRA_PHDRS; segs += ELF_CORE_EXTRA_PHDRS;
#endif #endif
...@@ -1766,20 +1755,10 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, ...@@ -1766,20 +1755,10 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
mm_flags = current->mm->flags; mm_flags = current->mm->flags;
/* write program headers for segments dump */ /* write program headers for segments dump */
for ( for (vma = current->mm->mmap; vma; vma = vma->vm_next) {
#ifdef CONFIG_MMU
vma = current->mm->mmap; vma; vma = vma->vm_next
#else
vml = current->mm->context.vmlist; vml; vml = vml->next
#endif
) {
struct elf_phdr phdr; struct elf_phdr phdr;
size_t sz; size_t sz;
#ifndef CONFIG_MMU
vma = vml->vma;
#endif
sz = vma->vm_end - vma->vm_start; sz = vma->vm_end - vma->vm_start;
phdr.p_type = PT_LOAD; phdr.p_type = PT_LOAD;
......
...@@ -41,8 +41,6 @@ do { \ ...@@ -41,8 +41,6 @@ do { \
(vmi)->used = 0; \ (vmi)->used = 0; \
(vmi)->largest_chunk = 0; \ (vmi)->largest_chunk = 0; \
} while(0) } while(0)
extern int nommu_vma_show(struct seq_file *, struct vm_area_struct *);
#endif #endif
extern int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns, extern int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns,
......
...@@ -73,6 +73,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v) ...@@ -73,6 +73,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
"HighFree: %8lu kB\n" "HighFree: %8lu kB\n"
"LowTotal: %8lu kB\n" "LowTotal: %8lu kB\n"
"LowFree: %8lu kB\n" "LowFree: %8lu kB\n"
#endif
#ifndef CONFIG_MMU
"MmapCopy: %8lu kB\n"
#endif #endif
"SwapTotal: %8lu kB\n" "SwapTotal: %8lu kB\n"
"SwapFree: %8lu kB\n" "SwapFree: %8lu kB\n"
...@@ -115,6 +118,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v) ...@@ -115,6 +118,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
K(i.freehigh), K(i.freehigh),
K(i.totalram-i.totalhigh), K(i.totalram-i.totalhigh),
K(i.freeram-i.freehigh), K(i.freeram-i.freehigh),
#endif
#ifndef CONFIG_MMU
K((unsigned long) atomic_read(&mmap_pages_allocated)),
#endif #endif
K(i.totalswap), K(i.totalswap),
K(i.freeswap), K(i.freeswap),
......
...@@ -33,33 +33,33 @@ ...@@ -33,33 +33,33 @@
#include "internal.h" #include "internal.h"
/* /*
* display a single VMA to a sequenced file * display a single region to a sequenced file
*/ */
int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) static int nommu_region_show(struct seq_file *m, struct vm_region *region)
{ {
unsigned long ino = 0; unsigned long ino = 0;
struct file *file; struct file *file;
dev_t dev = 0; dev_t dev = 0;
int flags, len; int flags, len;
flags = vma->vm_flags; flags = region->vm_flags;
file = vma->vm_file; file = region->vm_file;
if (file) { if (file) {
struct inode *inode = vma->vm_file->f_path.dentry->d_inode; struct inode *inode = region->vm_file->f_path.dentry->d_inode;
dev = inode->i_sb->s_dev; dev = inode->i_sb->s_dev;
ino = inode->i_ino; ino = inode->i_ino;
} }
seq_printf(m, seq_printf(m,
"%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n", "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n",
vma->vm_start, region->vm_start,
vma->vm_end, region->vm_end,
flags & VM_READ ? 'r' : '-', flags & VM_READ ? 'r' : '-',
flags & VM_WRITE ? 'w' : '-', flags & VM_WRITE ? 'w' : '-',
flags & VM_EXEC ? 'x' : '-', flags & VM_EXEC ? 'x' : '-',
flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p',
((loff_t)vma->vm_pgoff) << PAGE_SHIFT, ((loff_t)region->vm_pgoff) << PAGE_SHIFT,
MAJOR(dev), MINOR(dev), ino, &len); MAJOR(dev), MINOR(dev), ino, &len);
if (file) { if (file) {
...@@ -75,61 +75,54 @@ int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) ...@@ -75,61 +75,54 @@ int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma)
} }
/* /*
* display a list of all the VMAs the kernel knows about * display a list of all the REGIONs the kernel knows about
* - nommu kernals have a single flat list * - nommu kernals have a single flat list
*/ */
static int nommu_vma_list_show(struct seq_file *m, void *v) static int nommu_region_list_show(struct seq_file *m, void *_p)
{ {
struct vm_area_struct *vma; struct rb_node *p = _p;
vma = rb_entry((struct rb_node *) v, struct vm_area_struct, vm_rb); return nommu_region_show(m, rb_entry(p, struct vm_region, vm_rb));
return nommu_vma_show(m, vma);
} }
static void *nommu_vma_list_start(struct seq_file *m, loff_t *_pos) static void *nommu_region_list_start(struct seq_file *m, loff_t *_pos)
{ {
struct rb_node *_rb; struct rb_node *p;
loff_t pos = *_pos; loff_t pos = *_pos;
void *next = NULL;
down_read(&nommu_vma_sem); down_read(&nommu_region_sem);
for (_rb = rb_first(&nommu_vma_tree); _rb; _rb = rb_next(_rb)) { for (p = rb_first(&nommu_region_tree); p; p = rb_next(p))
if (pos == 0) { if (pos-- == 0)
next = _rb; return p;
break; return NULL;
}
pos--;
}
return next;
} }
static void nommu_vma_list_stop(struct seq_file *m, void *v) static void nommu_region_list_stop(struct seq_file *m, void *v)
{ {
up_read(&nommu_vma_sem); up_read(&nommu_region_sem);
} }
static void *nommu_vma_list_next(struct seq_file *m, void *v, loff_t *pos) static void *nommu_region_list_next(struct seq_file *m, void *v, loff_t *pos)
{ {
(*pos)++; (*pos)++;
return rb_next((struct rb_node *) v); return rb_next((struct rb_node *) v);
} }
static const struct seq_operations proc_nommu_vma_list_seqop = { static struct seq_operations proc_nommu_region_list_seqop = {
.start = nommu_vma_list_start, .start = nommu_region_list_start,
.next = nommu_vma_list_next, .next = nommu_region_list_next,
.stop = nommu_vma_list_stop, .stop = nommu_region_list_stop,
.show = nommu_vma_list_show .show = nommu_region_list_show
}; };
static int proc_nommu_vma_list_open(struct inode *inode, struct file *file) static int proc_nommu_region_list_open(struct inode *inode, struct file *file)
{ {
return seq_open(file, &proc_nommu_vma_list_seqop); return seq_open(file, &proc_nommu_region_list_seqop);
} }
static const struct file_operations proc_nommu_vma_list_operations = { static const struct file_operations proc_nommu_region_list_operations = {
.open = proc_nommu_vma_list_open, .open = proc_nommu_region_list_open,
.read = seq_read, .read = seq_read,
.llseek = seq_lseek, .llseek = seq_lseek,
.release = seq_release, .release = seq_release,
...@@ -137,7 +130,7 @@ static const struct file_operations proc_nommu_vma_list_operations = { ...@@ -137,7 +130,7 @@ static const struct file_operations proc_nommu_vma_list_operations = {
static int __init proc_nommu_init(void) static int __init proc_nommu_init(void)
{ {
proc_create("maps", S_IRUGO, NULL, &proc_nommu_vma_list_operations); proc_create("maps", S_IRUGO, NULL, &proc_nommu_region_list_operations);
return 0; return 0;
} }
......
...@@ -15,25 +15,25 @@ ...@@ -15,25 +15,25 @@
*/ */
void task_mem(struct seq_file *m, struct mm_struct *mm) void task_mem(struct seq_file *m, struct mm_struct *mm)
{ {
struct vm_list_struct *vml; struct vm_area_struct *vma;
struct rb_node *p;
unsigned long bytes = 0, sbytes = 0, slack = 0; unsigned long bytes = 0, sbytes = 0, slack = 0;
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
for (vml = mm->context.vmlist; vml; vml = vml->next) { for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
if (!vml->vma) vma = rb_entry(p, struct vm_area_struct, vm_rb);
continue;
bytes += kobjsize(vml); bytes += kobjsize(vma);
if (atomic_read(&mm->mm_count) > 1 || if (atomic_read(&mm->mm_count) > 1 ||
atomic_read(&vml->vma->vm_usage) > 1 vma->vm_region ||
) { vma->vm_flags & VM_MAYSHARE) {
sbytes += kobjsize((void *) vml->vma->vm_start); sbytes += kobjsize((void *) vma->vm_start);
sbytes += kobjsize(vml->vma); if (vma->vm_region)
sbytes += kobjsize(vma->vm_region);
} else { } else {
bytes += kobjsize((void *) vml->vma->vm_start); bytes += kobjsize((void *) vma->vm_start);
bytes += kobjsize(vml->vma); slack += kobjsize((void *) vma->vm_start) -
slack += kobjsize((void *) vml->vma->vm_start) - (vma->vm_end - vma->vm_start);
(vml->vma->vm_end - vml->vma->vm_start);
} }
} }
...@@ -70,13 +70,14 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) ...@@ -70,13 +70,14 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
unsigned long task_vsize(struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm)
{ {
struct vm_list_struct *tbp; struct vm_area_struct *vma;
struct rb_node *p;
unsigned long vsize = 0; unsigned long vsize = 0;
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
if (tbp->vma) vma = rb_entry(p, struct vm_area_struct, vm_rb);
vsize += kobjsize((void *) tbp->vma->vm_start); vsize += vma->vm_region->vm_end - vma->vm_region->vm_start;
} }
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
return vsize; return vsize;
...@@ -85,16 +86,15 @@ unsigned long task_vsize(struct mm_struct *mm) ...@@ -85,16 +86,15 @@ unsigned long task_vsize(struct mm_struct *mm)
int task_statm(struct mm_struct *mm, int *shared, int *text, int task_statm(struct mm_struct *mm, int *shared, int *text,
int *data, int *resident) int *data, int *resident)
{ {
struct vm_list_struct *tbp; struct vm_area_struct *vma;
struct rb_node *p;
int size = kobjsize(mm); int size = kobjsize(mm);
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) {
size += kobjsize(tbp); vma = rb_entry(p, struct vm_area_struct, vm_rb);
if (tbp->vma) { size += kobjsize(vma);
size += kobjsize(tbp->vma); size += kobjsize((void *) vma->vm_start);
size += kobjsize((void *) tbp->vma->vm_start);
}
} }
size += (*text = mm->end_code - mm->start_code); size += (*text = mm->end_code - mm->start_code);
...@@ -104,21 +104,63 @@ int task_statm(struct mm_struct *mm, int *shared, int *text, ...@@ -104,21 +104,63 @@ int task_statm(struct mm_struct *mm, int *shared, int *text,
return size; return size;
} }
/*
* display a single VMA to a sequenced file
*/
static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma)
{
unsigned long ino = 0;
struct file *file;
dev_t dev = 0;
int flags, len;
flags = vma->vm_flags;
file = vma->vm_file;
if (file) {
struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
dev = inode->i_sb->s_dev;
ino = inode->i_ino;
}
seq_printf(m,
"%08lx-%08lx %c%c%c%c %08lx %02x:%02x %lu %n",
vma->vm_start,
vma->vm_end,
flags & VM_READ ? 'r' : '-',
flags & VM_WRITE ? 'w' : '-',
flags & VM_EXEC ? 'x' : '-',
flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p',
vma->vm_pgoff << PAGE_SHIFT,
MAJOR(dev), MINOR(dev), ino, &len);
if (file) {
len = 25 + sizeof(void *) * 6 - len;
if (len < 1)
len = 1;
seq_printf(m, "%*c", len, ' ');
seq_path(m, &file->f_path, "");
}
seq_putc(m, '\n');
return 0;
}
/* /*
* display mapping lines for a particular process's /proc/pid/maps * display mapping lines for a particular process's /proc/pid/maps
*/ */
static int show_map(struct seq_file *m, void *_vml) static int show_map(struct seq_file *m, void *_p)
{ {
struct vm_list_struct *vml = _vml; struct rb_node *p = _p;
return nommu_vma_show(m, vml->vma); return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb));
} }
static void *m_start(struct seq_file *m, loff_t *pos) static void *m_start(struct seq_file *m, loff_t *pos)
{ {
struct proc_maps_private *priv = m->private; struct proc_maps_private *priv = m->private;
struct vm_list_struct *vml;
struct mm_struct *mm; struct mm_struct *mm;
struct rb_node *p;
loff_t n = *pos; loff_t n = *pos;
/* pin the task and mm whilst we play with them */ /* pin the task and mm whilst we play with them */
...@@ -134,9 +176,9 @@ static void *m_start(struct seq_file *m, loff_t *pos) ...@@ -134,9 +176,9 @@ static void *m_start(struct seq_file *m, loff_t *pos)
} }
/* start from the Nth VMA */ /* start from the Nth VMA */
for (vml = mm->context.vmlist; vml; vml = vml->next) for (p = rb_first(&mm->mm_rb); p; p = rb_next(p))
if (n-- == 0) if (n-- == 0)
return vml; return p;
return NULL; return NULL;
} }
...@@ -152,12 +194,12 @@ static void m_stop(struct seq_file *m, void *_vml) ...@@ -152,12 +194,12 @@ static void m_stop(struct seq_file *m, void *_vml)
} }
} }
static void *m_next(struct seq_file *m, void *_vml, loff_t *pos) static void *m_next(struct seq_file *m, void *_p, loff_t *pos)
{ {
struct vm_list_struct *vml = _vml; struct rb_node *p = _p;
(*pos)++; (*pos)++;
return vml ? vml->next : NULL; return p ? rb_next(p) : NULL;
} }
static const struct seq_operations proc_pid_maps_ops = { static const struct seq_operations proc_pid_maps_ops = {
......
...@@ -22,7 +22,6 @@ typedef struct { ...@@ -22,7 +22,6 @@ typedef struct {
unsigned long dtlb_ptd_mapping; /* [DAMR5] PTD mapping for dtlb cached PGE */ unsigned long dtlb_ptd_mapping; /* [DAMR5] PTD mapping for dtlb cached PGE */
#else #else
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
#endif #endif
......
...@@ -4,7 +4,6 @@ ...@@ -4,7 +4,6 @@
#if !defined(CONFIG_MMU) #if !defined(CONFIG_MMU)
typedef struct { typedef struct {
struct vm_list_struct *vmlist;
unsigned long end_brk; unsigned long end_brk;
} mm_context_t; } mm_context_t;
......
...@@ -56,19 +56,9 @@ extern unsigned long mmap_min_addr; ...@@ -56,19 +56,9 @@ extern unsigned long mmap_min_addr;
extern struct kmem_cache *vm_area_cachep; extern struct kmem_cache *vm_area_cachep;
/*
* This struct defines the per-mm list of VMAs for uClinux. If CONFIG_MMU is
* disabled, then there's a single shared list of VMAs maintained by the
* system, and mm's subscribe to these individually
*/
struct vm_list_struct {
struct vm_list_struct *next;
struct vm_area_struct *vma;
};
#ifndef CONFIG_MMU #ifndef CONFIG_MMU
extern struct rb_root nommu_vma_tree; extern struct rb_root nommu_region_tree;
extern struct rw_semaphore nommu_vma_sem; extern struct rw_semaphore nommu_region_sem;
extern unsigned int kobjsize(const void *objp); extern unsigned int kobjsize(const void *objp);
#endif #endif
...@@ -1061,6 +1051,7 @@ extern void memmap_init_zone(unsigned long, int, unsigned long, ...@@ -1061,6 +1051,7 @@ extern void memmap_init_zone(unsigned long, int, unsigned long,
unsigned long, enum memmap_context); unsigned long, enum memmap_context);
extern void setup_per_zone_pages_min(void); extern void setup_per_zone_pages_min(void);
extern void mem_init(void); extern void mem_init(void);
extern void __init mmap_init(void);
extern void show_mem(void); extern void show_mem(void);
extern void si_meminfo(struct sysinfo * val); extern void si_meminfo(struct sysinfo * val);
extern void si_meminfo_node(struct sysinfo *val, int nid); extern void si_meminfo_node(struct sysinfo *val, int nid);
...@@ -1072,6 +1063,9 @@ extern void setup_per_cpu_pageset(void); ...@@ -1072,6 +1063,9 @@ extern void setup_per_cpu_pageset(void);
static inline void setup_per_cpu_pageset(void) {} static inline void setup_per_cpu_pageset(void) {}
#endif #endif
/* nommu.c */
extern atomic_t mmap_pages_allocated;
/* prio_tree.c */ /* prio_tree.c */
void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old); void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *); void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);
......
...@@ -96,6 +96,22 @@ struct page { ...@@ -96,6 +96,22 @@ struct page {
#endif /* WANT_PAGE_VIRTUAL */ #endif /* WANT_PAGE_VIRTUAL */
}; };
/*
* A region containing a mapping of a non-memory backed file under NOMMU
* conditions. These are held in a global tree and are pinned by the VMAs that
* map parts of them.
*/
struct vm_region {
struct rb_node vm_rb; /* link in global region tree */
unsigned long vm_flags; /* VMA vm_flags */
unsigned long vm_start; /* start address of region */
unsigned long vm_end; /* region initialised to here */
unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */
struct file *vm_file; /* the backing file or NULL */
atomic_t vm_usage; /* region usage count */
};
/* /*
* This struct defines a memory VMM memory area. There is one of these * This struct defines a memory VMM memory area. There is one of these
* per VM-area/task. A VM area is any part of the process virtual memory * per VM-area/task. A VM area is any part of the process virtual memory
...@@ -152,7 +168,7 @@ struct vm_area_struct { ...@@ -152,7 +168,7 @@ struct vm_area_struct {
unsigned long vm_truncate_count;/* truncate_count or restart_addr */ unsigned long vm_truncate_count;/* truncate_count or restart_addr */
#ifndef CONFIG_MMU #ifndef CONFIG_MMU
atomic_t vm_usage; /* refcount (VMAs shared if !MMU) */ struct vm_region *vm_region; /* NOMMU mapping region */
#endif #endif
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
struct mempolicy *vm_policy; /* NUMA policy for the VMA */ struct mempolicy *vm_policy; /* NUMA policy for the VMA */
......
...@@ -990,6 +990,7 @@ asmlinkage long sys_shmdt(char __user *shmaddr) ...@@ -990,6 +990,7 @@ asmlinkage long sys_shmdt(char __user *shmaddr)
*/ */
vma = find_vma(mm, addr); vma = find_vma(mm, addr);
#ifdef CONFIG_MMU
while (vma) { while (vma) {
next = vma->vm_next; next = vma->vm_next;
...@@ -1034,6 +1035,17 @@ asmlinkage long sys_shmdt(char __user *shmaddr) ...@@ -1034,6 +1035,17 @@ asmlinkage long sys_shmdt(char __user *shmaddr)
vma = next; vma = next;
} }
#else /* CONFIG_MMU */
/* under NOMMU conditions, the exact address to be destroyed must be
* given */
retval = -EINVAL;
if (vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
retval = 0;
}
#endif
up_write(&mm->mmap_sem); up_write(&mm->mmap_sem);
return retval; return retval;
} }
......
...@@ -1481,12 +1481,10 @@ void __init proc_caches_init(void) ...@@ -1481,12 +1481,10 @@ void __init proc_caches_init(void)
fs_cachep = kmem_cache_create("fs_cache", fs_cachep = kmem_cache_create("fs_cache",
sizeof(struct fs_struct), 0, sizeof(struct fs_struct), 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
vm_area_cachep = kmem_cache_create("vm_area_struct",
sizeof(struct vm_area_struct), 0,
SLAB_PANIC, NULL);
mm_cachep = kmem_cache_create("mm_struct", mm_cachep = kmem_cache_create("mm_struct",
sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN,
SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
mmap_init();
} }
/* /*
......
...@@ -512,6 +512,13 @@ config DEBUG_VIRTUAL ...@@ -512,6 +512,13 @@ config DEBUG_VIRTUAL
If unsure, say N. If unsure, say N.
config DEBUG_NOMMU_REGIONS
bool "Debug the global anon/private NOMMU mapping region tree"
depends on DEBUG_KERNEL && !MMU
help
This option causes the global tree of anonymous and private mapping
regions to be regularly checked for invalid topology.
config DEBUG_WRITECOUNT config DEBUG_WRITECOUNT
bool "Debug filesystem writers count" bool "Debug filesystem writers count"
depends on DEBUG_KERNEL depends on DEBUG_KERNEL
......
...@@ -2472,3 +2472,13 @@ void mm_drop_all_locks(struct mm_struct *mm) ...@@ -2472,3 +2472,13 @@ void mm_drop_all_locks(struct mm_struct *mm)
mutex_unlock(&mm_all_locks_mutex); mutex_unlock(&mm_all_locks_mutex);
} }
/*
* initialise the VMA slab
*/
void __init mmap_init(void)
{
vm_area_cachep = kmem_cache_create("vm_area_struct",
sizeof(struct vm_area_struct), 0,
SLAB_PANIC, NULL);
}
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
* *
* See Documentation/nommu-mmap.txt * See Documentation/nommu-mmap.txt
* *
* Copyright (c) 2004-2005 David Howells <dhowells@redhat.com> * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
* Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
* Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
* Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
...@@ -33,6 +33,28 @@ ...@@ -33,6 +33,28 @@
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/tlb.h> #include <asm/tlb.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include "internal.h"
static inline __attribute__((format(printf, 1, 2)))
void no_printk(const char *fmt, ...)
{
}
#if 0
#define kenter(FMT, ...) \
printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
#define kleave(FMT, ...) \
printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
#define kdebug(FMT, ...) \
printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)
#else
#define kenter(FMT, ...) \
no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
#define kleave(FMT, ...) \
no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
#define kdebug(FMT, ...) \
no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
#endif
#include "internal.h" #include "internal.h"
...@@ -46,12 +68,15 @@ int sysctl_overcommit_ratio = 50; /* default is 50% */ ...@@ -46,12 +68,15 @@ int sysctl_overcommit_ratio = 50; /* default is 50% */
int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
int heap_stack_gap = 0; int heap_stack_gap = 0;
atomic_t mmap_pages_allocated;
EXPORT_SYMBOL(mem_map); EXPORT_SYMBOL(mem_map);
EXPORT_SYMBOL(num_physpages); EXPORT_SYMBOL(num_physpages);
/* list of shareable VMAs */ /* list of mapped, potentially shareable regions */
struct rb_root nommu_vma_tree = RB_ROOT; static struct kmem_cache *vm_region_jar;
DECLARE_RWSEM(nommu_vma_sem); struct rb_root nommu_region_tree = RB_ROOT;
DECLARE_RWSEM(nommu_region_sem);
struct vm_operations_struct generic_file_vm_ops = { struct vm_operations_struct generic_file_vm_ops = {
}; };
...@@ -400,129 +425,174 @@ asmlinkage unsigned long sys_brk(unsigned long brk) ...@@ -400,129 +425,174 @@ asmlinkage unsigned long sys_brk(unsigned long brk)
return mm->brk = brk; return mm->brk = brk;
} }
#ifdef DEBUG /*
static void show_process_blocks(void) * initialise the VMA and region record slabs
*/
void __init mmap_init(void)
{ {
struct vm_list_struct *vml; vm_region_jar = kmem_cache_create("vm_region_jar",
sizeof(struct vm_region), 0,
printk("Process blocks %d:", current->pid); SLAB_PANIC, NULL);
vm_area_cachep = kmem_cache_create("vm_area_struct",
for (vml = &current->mm->context.vmlist; vml; vml = vml->next) { sizeof(struct vm_area_struct), 0,
printk(" %p: %p", vml, vml->vma); SLAB_PANIC, NULL);
if (vml->vma)
printk(" (%d @%lx #%d)",
kobjsize((void *) vml->vma->vm_start),
vml->vma->vm_start,
atomic_read(&vml->vma->vm_usage));
printk(vml->next ? " ->" : ".\n");
}
} }
#endif /* DEBUG */
/* /*
* add a VMA into a process's mm_struct in the appropriate place in the list * validate the region tree
* - should be called with mm->mmap_sem held writelocked * - the caller must hold the region lock
*/ */
static void add_vma_to_mm(struct mm_struct *mm, struct vm_list_struct *vml) #ifdef CONFIG_DEBUG_NOMMU_REGIONS
static noinline void validate_nommu_regions(void)
{ {
struct vm_list_struct **ppv; struct vm_region *region, *last;
struct rb_node *p, *lastp;
for (ppv = &current->mm->context.vmlist; *ppv; ppv = &(*ppv)->next) lastp = rb_first(&nommu_region_tree);
if ((*ppv)->vma->vm_start > vml->vma->vm_start) if (!lastp)
break; return;
last = rb_entry(lastp, struct vm_region, vm_rb);
if (unlikely(last->vm_end <= last->vm_start))
BUG();
while ((p = rb_next(lastp))) {
region = rb_entry(p, struct vm_region, vm_rb);
last = rb_entry(lastp, struct vm_region, vm_rb);
if (unlikely(region->vm_end <= region->vm_start))
BUG();
if (unlikely(region->vm_start < last->vm_end))
BUG();
vml->next = *ppv; lastp = p;
*ppv = vml; }
} }
#else
#define validate_nommu_regions() do {} while(0)
#endif
/* /*
* look up the first VMA in which addr resides, NULL if none * add a region into the global tree
* - should be called with mm->mmap_sem at least held readlocked
*/ */
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) static void add_nommu_region(struct vm_region *region)
{ {
struct vm_list_struct *loop, *vml; struct vm_region *pregion;
struct rb_node **p, *parent;
/* search the vm_start ordered list */ validate_nommu_regions();
vml = NULL;
for (loop = mm->context.vmlist; loop; loop = loop->next) { BUG_ON(region->vm_start & ~PAGE_MASK);
if (loop->vma->vm_start > addr)
break; parent = NULL;
vml = loop; p = &nommu_region_tree.rb_node;
while (*p) {
parent = *p;
pregion = rb_entry(parent, struct vm_region, vm_rb);
if (region->vm_start < pregion->vm_start)
p = &(*p)->rb_left;
else if (region->vm_start > pregion->vm_start)
p = &(*p)->rb_right;
else if (pregion == region)
return;
else
BUG();
} }
if (vml && vml->vma->vm_end > addr) rb_link_node(&region->vm_rb, parent, p);
return vml->vma; rb_insert_color(&region->vm_rb, &nommu_region_tree);
return NULL; validate_nommu_regions();
} }
EXPORT_SYMBOL(find_vma);
/* /*
* find a VMA * delete a region from the global tree
* - we don't extend stack VMAs under NOMMU conditions
*/ */
struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) static void delete_nommu_region(struct vm_region *region)
{ {
return find_vma(mm, addr); BUG_ON(!nommu_region_tree.rb_node);
}
int expand_stack(struct vm_area_struct *vma, unsigned long address) validate_nommu_regions();
{ rb_erase(&region->vm_rb, &nommu_region_tree);
return -ENOMEM; validate_nommu_regions();
} }
/* /*
* look up the first VMA exactly that exactly matches addr * free a contiguous series of pages
* - should be called with mm->mmap_sem at least held readlocked
*/ */
static inline struct vm_area_struct *find_vma_exact(struct mm_struct *mm, static void free_page_series(unsigned long from, unsigned long to)
unsigned long addr)
{ {
struct vm_list_struct *vml; for (; from < to; from += PAGE_SIZE) {
struct page *page = virt_to_page(from);
/* search the vm_start ordered list */
for (vml = mm->context.vmlist; vml; vml = vml->next) { kdebug("- free %lx", from);
if (vml->vma->vm_start == addr) atomic_dec(&mmap_pages_allocated);
return vml->vma; if (page_count(page) != 1)
if (vml->vma->vm_start > addr) kdebug("free page %p [%d]", page, page_count(page));
break; put_page(page);
} }
return NULL;
} }
/* /*
* find a VMA in the global tree * release a reference to a region
* - the caller must hold the region semaphore, which this releases
* - the region may not have been added to the tree yet, in which case vm_end
* will equal vm_start
*/ */
static inline struct vm_area_struct *find_nommu_vma(unsigned long start) static void __put_nommu_region(struct vm_region *region)
__releases(nommu_region_sem)
{ {
struct vm_area_struct *vma; kenter("%p{%d}", region, atomic_read(&region->vm_usage));
struct rb_node *n = nommu_vma_tree.rb_node;
while (n) { BUG_ON(!nommu_region_tree.rb_node);
vma = rb_entry(n, struct vm_area_struct, vm_rb);
if (start < vma->vm_start) if (atomic_dec_and_test(&region->vm_usage)) {
n = n->rb_left; if (region->vm_end > region->vm_start)
else if (start > vma->vm_start) delete_nommu_region(region);
n = n->rb_right; up_write(&nommu_region_sem);
else
return vma; if (region->vm_file)
fput(region->vm_file);
/* IO memory and memory shared directly out of the pagecache
* from ramfs/tmpfs mustn't be released here */
if (region->vm_flags & VM_MAPPED_COPY) {
kdebug("free series");
free_page_series(region->vm_start, region->vm_end);
}
kmem_cache_free(vm_region_jar, region);
} else {
up_write(&nommu_region_sem);
} }
}
return NULL; /*
* release a reference to a region
*/
static void put_nommu_region(struct vm_region *region)
{
down_write(&nommu_region_sem);
__put_nommu_region(region);
} }
/* /*
* add a VMA in the global tree * add a VMA into a process's mm_struct in the appropriate place in the list
* and tree and add to the address space's page tree also if not an anonymous
* page
* - should be called with mm->mmap_sem held writelocked
*/ */
static void add_nommu_vma(struct vm_area_struct *vma) static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
{ {
struct vm_area_struct *pvma; struct vm_area_struct *pvma, **pp;
struct address_space *mapping; struct address_space *mapping;
struct rb_node **p = &nommu_vma_tree.rb_node; struct rb_node **p, *parent;
struct rb_node *parent = NULL;
kenter(",%p", vma);
BUG_ON(!vma->vm_region);
mm->map_count++;
vma->vm_mm = mm;
/* add the VMA to the mapping */ /* add the VMA to the mapping */
if (vma->vm_file) { if (vma->vm_file) {
...@@ -533,42 +603,62 @@ static void add_nommu_vma(struct vm_area_struct *vma) ...@@ -533,42 +603,62 @@ static void add_nommu_vma(struct vm_area_struct *vma)
flush_dcache_mmap_unlock(mapping); flush_dcache_mmap_unlock(mapping);
} }
/* add the VMA to the master list */ /* add the VMA to the tree */
parent = NULL;
p = &mm->mm_rb.rb_node;
while (*p) { while (*p) {
parent = *p; parent = *p;
pvma = rb_entry(parent, struct vm_area_struct, vm_rb); pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
if (vma->vm_start < pvma->vm_start) { /* sort by: start addr, end addr, VMA struct addr in that order
* (the latter is necessary as we may get identical VMAs) */
if (vma->vm_start < pvma->vm_start)
p = &(*p)->rb_left; p = &(*p)->rb_left;
} else if (vma->vm_start > pvma->vm_start)
else if (vma->vm_start > pvma->vm_start) {
p = &(*p)->rb_right; p = &(*p)->rb_right;
} else if (vma->vm_end < pvma->vm_end)
else { p = &(*p)->rb_left;
/* mappings are at the same address - this can only else if (vma->vm_end > pvma->vm_end)
* happen for shared-mem chardevs and shared file p = &(*p)->rb_right;
* mappings backed by ramfs/tmpfs */ else if (vma < pvma)
BUG_ON(!(pvma->vm_flags & VM_SHARED)); p = &(*p)->rb_left;
else if (vma > pvma)
if (vma < pvma) p = &(*p)->rb_right;
p = &(*p)->rb_left; else
else if (vma > pvma) BUG();
p = &(*p)->rb_right;
else
BUG();
}
} }
rb_link_node(&vma->vm_rb, parent, p); rb_link_node(&vma->vm_rb, parent, p);
rb_insert_color(&vma->vm_rb, &nommu_vma_tree); rb_insert_color(&vma->vm_rb, &mm->mm_rb);
/* add VMA to the VMA list also */
for (pp = &mm->mmap; (pvma = *pp); pp = &(*pp)->vm_next) {
if (pvma->vm_start > vma->vm_start)
break;
if (pvma->vm_start < vma->vm_start)
continue;
if (pvma->vm_end < vma->vm_end)
break;
}
vma->vm_next = *pp;
*pp = vma;
} }
/* /*
* delete a VMA from the global list * delete a VMA from its owning mm_struct and address space
*/ */
static void delete_nommu_vma(struct vm_area_struct *vma) static void delete_vma_from_mm(struct vm_area_struct *vma)
{ {
struct vm_area_struct **pp;
struct address_space *mapping; struct address_space *mapping;
struct mm_struct *mm = vma->vm_mm;
kenter("%p", vma);
mm->map_count--;
if (mm->mmap_cache == vma)
mm->mmap_cache = NULL;
/* remove the VMA from the mapping */ /* remove the VMA from the mapping */
if (vma->vm_file) { if (vma->vm_file) {
...@@ -579,8 +669,115 @@ static void delete_nommu_vma(struct vm_area_struct *vma) ...@@ -579,8 +669,115 @@ static void delete_nommu_vma(struct vm_area_struct *vma)
flush_dcache_mmap_unlock(mapping); flush_dcache_mmap_unlock(mapping);
} }
/* remove from the master list */ /* remove from the MM's tree and list */
rb_erase(&vma->vm_rb, &nommu_vma_tree); rb_erase(&vma->vm_rb, &mm->mm_rb);
for (pp = &mm->mmap; *pp; pp = &(*pp)->vm_next) {
if (*pp == vma) {
*pp = vma->vm_next;
break;
}
}
vma->vm_mm = NULL;
}
/*
* destroy a VMA record
*/
static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
{
kenter("%p", vma);
if (vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
if (vma->vm_file) {
fput(vma->vm_file);
if (vma->vm_flags & VM_EXECUTABLE)
removed_exe_file_vma(mm);
}
put_nommu_region(vma->vm_region);
kmem_cache_free(vm_area_cachep, vma);
}
/*
* look up the first VMA in which addr resides, NULL if none
* - should be called with mm->mmap_sem at least held readlocked
*/
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma;
struct rb_node *n = mm->mm_rb.rb_node;
/* check the cache first */
vma = mm->mmap_cache;
if (vma && vma->vm_start <= addr && vma->vm_end > addr)
return vma;
/* trawl the tree (there may be multiple mappings in which addr
* resides) */
for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
vma = rb_entry(n, struct vm_area_struct, vm_rb);
if (vma->vm_start > addr)
return NULL;
if (vma->vm_end > addr) {
mm->mmap_cache = vma;
return vma;
}
}
return NULL;
}
EXPORT_SYMBOL(find_vma);
/*
* find a VMA
* - we don't extend stack VMAs under NOMMU conditions
*/
struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
{
return find_vma(mm, addr);
}
/*
* expand a stack to a given address
* - not supported under NOMMU conditions
*/
int expand_stack(struct vm_area_struct *vma, unsigned long address)
{
return -ENOMEM;
}
/*
* look up the first VMA exactly that exactly matches addr
* - should be called with mm->mmap_sem at least held readlocked
*/
static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
unsigned long addr,
unsigned long len)
{
struct vm_area_struct *vma;
struct rb_node *n = mm->mm_rb.rb_node;
unsigned long end = addr + len;
/* check the cache first */
vma = mm->mmap_cache;
if (vma && vma->vm_start == addr && vma->vm_end == end)
return vma;
/* trawl the tree (there may be multiple mappings in which addr
* resides) */
for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
vma = rb_entry(n, struct vm_area_struct, vm_rb);
if (vma->vm_start < addr)
continue;
if (vma->vm_start > addr)
return NULL;
if (vma->vm_end == end) {
mm->mmap_cache = vma;
return vma;
}
}
return NULL;
} }
/* /*
...@@ -595,7 +792,7 @@ static int validate_mmap_request(struct file *file, ...@@ -595,7 +792,7 @@ static int validate_mmap_request(struct file *file,
unsigned long pgoff, unsigned long pgoff,
unsigned long *_capabilities) unsigned long *_capabilities)
{ {
unsigned long capabilities; unsigned long capabilities, rlen;
unsigned long reqprot = prot; unsigned long reqprot = prot;
int ret; int ret;
...@@ -615,12 +812,12 @@ static int validate_mmap_request(struct file *file, ...@@ -615,12 +812,12 @@ static int validate_mmap_request(struct file *file,
return -EINVAL; return -EINVAL;
/* Careful about overflows.. */ /* Careful about overflows.. */
len = PAGE_ALIGN(len); rlen = PAGE_ALIGN(len);
if (!len || len > TASK_SIZE) if (!rlen || rlen > TASK_SIZE)
return -ENOMEM; return -ENOMEM;
/* offset overflow? */ /* offset overflow? */
if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
return -EOVERFLOW; return -EOVERFLOW;
if (file) { if (file) {
...@@ -794,9 +991,10 @@ static unsigned long determine_vm_flags(struct file *file, ...@@ -794,9 +991,10 @@ static unsigned long determine_vm_flags(struct file *file,
} }
/* /*
* set up a shared mapping on a file * set up a shared mapping on a file (the driver or filesystem provides and
* pins the storage)
*/ */
static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) static int do_mmap_shared_file(struct vm_area_struct *vma)
{ {
int ret; int ret;
...@@ -814,10 +1012,14 @@ static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) ...@@ -814,10 +1012,14 @@ static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len)
/* /*
* set up a private mapping or an anonymous shared mapping * set up a private mapping or an anonymous shared mapping
*/ */
static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) static int do_mmap_private(struct vm_area_struct *vma,
struct vm_region *region,
unsigned long len)
{ {
struct page *pages;
unsigned long total, point, n, rlen;
void *base; void *base;
int ret; int ret, order;
/* invoke the file's mapping function so that it can keep track of /* invoke the file's mapping function so that it can keep track of
* shared mappings on devices or memory * shared mappings on devices or memory
...@@ -836,23 +1038,46 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) ...@@ -836,23 +1038,46 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len)
* make a private copy of the data and map that instead */ * make a private copy of the data and map that instead */
} }
rlen = PAGE_ALIGN(len);
/* allocate some memory to hold the mapping /* allocate some memory to hold the mapping
* - note that this may not return a page-aligned address if the object * - note that this may not return a page-aligned address if the object
* we're allocating is smaller than a page * we're allocating is smaller than a page
*/ */
base = kmalloc(len, GFP_KERNEL|__GFP_COMP); order = get_order(rlen);
if (!base) kdebug("alloc order %d for %lx", order, len);
pages = alloc_pages(GFP_KERNEL, order);
if (!pages)
goto enomem; goto enomem;
vma->vm_start = (unsigned long) base; /* we allocated a power-of-2 sized page set, so we need to trim off the
vma->vm_end = vma->vm_start + len; * excess */
vma->vm_flags |= VM_MAPPED_COPY; total = 1 << order;
atomic_add(total, &mmap_pages_allocated);
point = rlen >> PAGE_SHIFT;
while (total > point) {
order = ilog2(total - point);
n = 1 << order;
kdebug("shave %lu/%lu @%lu", n, total - point, total);
atomic_sub(n, &mmap_pages_allocated);
total -= n;
set_page_refcounted(pages + total);
__free_pages(pages + total, order);
}
total = rlen >> PAGE_SHIFT;
for (point = 1; point < total; point++)
set_page_refcounted(&pages[point]);
#ifdef WARN_ON_SLACK base = page_address(pages);
if (len + WARN_ON_SLACK <= kobjsize(result)) region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
printk("Allocation of %lu bytes from process %d has %lu bytes of slack\n", region->vm_start = (unsigned long) base;
len, current->pid, kobjsize(result) - len); region->vm_end = region->vm_start + rlen;
#endif
vma->vm_start = region->vm_start;
vma->vm_end = region->vm_start + len;
if (vma->vm_file) { if (vma->vm_file) {
/* read the contents of a file into the copy */ /* read the contents of a file into the copy */
...@@ -864,26 +1089,27 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) ...@@ -864,26 +1089,27 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len)
old_fs = get_fs(); old_fs = get_fs();
set_fs(KERNEL_DS); set_fs(KERNEL_DS);
ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); ret = vma->vm_file->f_op->read(vma->vm_file, base, rlen, &fpos);
set_fs(old_fs); set_fs(old_fs);
if (ret < 0) if (ret < 0)
goto error_free; goto error_free;
/* clear the last little bit */ /* clear the last little bit */
if (ret < len) if (ret < rlen)
memset(base + ret, 0, len - ret); memset(base + ret, 0, rlen - ret);
} else { } else {
/* if it's an anonymous mapping, then just clear it */ /* if it's an anonymous mapping, then just clear it */
memset(base, 0, len); memset(base, 0, rlen);
} }
return 0; return 0;
error_free: error_free:
kfree(base); free_page_series(region->vm_start, region->vm_end);
vma->vm_start = 0; region->vm_start = vma->vm_start = 0;
region->vm_end = vma->vm_end = 0;
return ret; return ret;
enomem: enomem:
...@@ -903,13 +1129,14 @@ unsigned long do_mmap_pgoff(struct file *file, ...@@ -903,13 +1129,14 @@ unsigned long do_mmap_pgoff(struct file *file,
unsigned long flags, unsigned long flags,
unsigned long pgoff) unsigned long pgoff)
{ {
struct vm_list_struct *vml = NULL; struct vm_area_struct *vma;
struct vm_area_struct *vma = NULL; struct vm_region *region;
struct rb_node *rb; struct rb_node *rb;
unsigned long capabilities, vm_flags; unsigned long capabilities, vm_flags, result;
void *result;
int ret; int ret;
kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);
if (!(flags & MAP_FIXED)) if (!(flags & MAP_FIXED))
addr = round_hint_to_min(addr); addr = round_hint_to_min(addr);
...@@ -917,73 +1144,120 @@ unsigned long do_mmap_pgoff(struct file *file, ...@@ -917,73 +1144,120 @@ unsigned long do_mmap_pgoff(struct file *file,
* mapping */ * mapping */
ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
&capabilities); &capabilities);
if (ret < 0) if (ret < 0) {
kleave(" = %d [val]", ret);
return ret; return ret;
}
/* we've determined that we can make the mapping, now translate what we /* we've determined that we can make the mapping, now translate what we
* now know into VMA flags */ * now know into VMA flags */
vm_flags = determine_vm_flags(file, prot, flags, capabilities); vm_flags = determine_vm_flags(file, prot, flags, capabilities);
/* we're going to need to record the mapping if it works */ /* we're going to need to record the mapping */
vml = kzalloc(sizeof(struct vm_list_struct), GFP_KERNEL); region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
if (!vml) if (!region)
goto error_getting_vml; goto error_getting_region;
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
if (!vma)
goto error_getting_vma;
atomic_set(&region->vm_usage, 1);
region->vm_flags = vm_flags;
region->vm_pgoff = pgoff;
down_write(&nommu_vma_sem); INIT_LIST_HEAD(&vma->anon_vma_node);
vma->vm_flags = vm_flags;
vma->vm_pgoff = pgoff;
/* if we want to share, we need to check for VMAs created by other if (file) {
region->vm_file = file;
get_file(file);
vma->vm_file = file;
get_file(file);
if (vm_flags & VM_EXECUTABLE) {
added_exe_file_vma(current->mm);
vma->vm_mm = current->mm;
}
}
down_write(&nommu_region_sem);
/* if we want to share, we need to check for regions created by other
* mmap() calls that overlap with our proposed mapping * mmap() calls that overlap with our proposed mapping
* - we can only share with an exact match on most regular files * - we can only share with a superset match on most regular files
* - shared mappings on character devices and memory backed files are * - shared mappings on character devices and memory backed files are
* permitted to overlap inexactly as far as we are concerned for in * permitted to overlap inexactly as far as we are concerned for in
* these cases, sharing is handled in the driver or filesystem rather * these cases, sharing is handled in the driver or filesystem rather
* than here * than here
*/ */
if (vm_flags & VM_MAYSHARE) { if (vm_flags & VM_MAYSHARE) {
unsigned long pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; struct vm_region *pregion;
unsigned long vmpglen; unsigned long pglen, rpglen, pgend, rpgend, start;
/* suppress VMA sharing for shared regions */ pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (vm_flags & VM_SHARED && pgend = pgoff + pglen;
capabilities & BDI_CAP_MAP_DIRECT)
goto dont_share_VMAs;
for (rb = rb_first(&nommu_vma_tree); rb; rb = rb_next(rb)) { for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
vma = rb_entry(rb, struct vm_area_struct, vm_rb); pregion = rb_entry(rb, struct vm_region, vm_rb);
if (!(vma->vm_flags & VM_MAYSHARE)) if (!(pregion->vm_flags & VM_MAYSHARE))
continue; continue;
/* search for overlapping mappings on the same file */ /* search for overlapping mappings on the same file */
if (vma->vm_file->f_path.dentry->d_inode != file->f_path.dentry->d_inode) if (pregion->vm_file->f_path.dentry->d_inode !=
file->f_path.dentry->d_inode)
continue; continue;
if (vma->vm_pgoff >= pgoff + pglen) if (pregion->vm_pgoff >= pgend)
continue; continue;
vmpglen = vma->vm_end - vma->vm_start + PAGE_SIZE - 1; rpglen = pregion->vm_end - pregion->vm_start;
vmpglen >>= PAGE_SHIFT; rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (pgoff >= vma->vm_pgoff + vmpglen) rpgend = pregion->vm_pgoff + rpglen;
if (pgoff >= rpgend)
continue; continue;
/* handle inexactly overlapping matches between mappings */ /* handle inexactly overlapping matches between
if (vma->vm_pgoff != pgoff || vmpglen != pglen) { * mappings */
if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
!(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
/* new mapping is not a subset of the region */
if (!(capabilities & BDI_CAP_MAP_DIRECT)) if (!(capabilities & BDI_CAP_MAP_DIRECT))
goto sharing_violation; goto sharing_violation;
continue; continue;
} }
/* we've found a VMA we can share */ /* we've found a region we can share */
atomic_inc(&vma->vm_usage); atomic_inc(&pregion->vm_usage);
vma->vm_region = pregion;
vml->vma = vma; start = pregion->vm_start;
result = (void *) vma->vm_start; start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
goto shared; vma->vm_start = start;
vma->vm_end = start + len;
if (pregion->vm_flags & VM_MAPPED_COPY) {
kdebug("share copy");
vma->vm_flags |= VM_MAPPED_COPY;
} else {
kdebug("share mmap");
ret = do_mmap_shared_file(vma);
if (ret < 0) {
vma->vm_region = NULL;
vma->vm_start = 0;
vma->vm_end = 0;
atomic_dec(&pregion->vm_usage);
pregion = NULL;
goto error_just_free;
}
}
fput(region->vm_file);
kmem_cache_free(vm_region_jar, region);
region = pregion;
result = start;
goto share;
} }
dont_share_VMAs:
vma = NULL;
/* obtain the address at which to make a shared mapping /* obtain the address at which to make a shared mapping
* - this is the hook for quasi-memory character devices to * - this is the hook for quasi-memory character devices to
* tell us the location of a shared mapping * tell us the location of a shared mapping
...@@ -994,102 +1268,93 @@ unsigned long do_mmap_pgoff(struct file *file, ...@@ -994,102 +1268,93 @@ unsigned long do_mmap_pgoff(struct file *file,
if (IS_ERR((void *) addr)) { if (IS_ERR((void *) addr)) {
ret = addr; ret = addr;
if (ret != (unsigned long) -ENOSYS) if (ret != (unsigned long) -ENOSYS)
goto error; goto error_just_free;
/* the driver refused to tell us where to site /* the driver refused to tell us where to site
* the mapping so we'll have to attempt to copy * the mapping so we'll have to attempt to copy
* it */ * it */
ret = (unsigned long) -ENODEV; ret = (unsigned long) -ENODEV;
if (!(capabilities & BDI_CAP_MAP_COPY)) if (!(capabilities & BDI_CAP_MAP_COPY))
goto error; goto error_just_free;
capabilities &= ~BDI_CAP_MAP_DIRECT; capabilities &= ~BDI_CAP_MAP_DIRECT;
} else {
vma->vm_start = region->vm_start = addr;
vma->vm_end = region->vm_end = addr + len;
} }
} }
} }
/* we're going to need a VMA struct as well */ vma->vm_region = region;
vma = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL);
if (!vma)
goto error_getting_vma;
INIT_LIST_HEAD(&vma->anon_vma_node);
atomic_set(&vma->vm_usage, 1);
if (file) {
get_file(file);
if (vm_flags & VM_EXECUTABLE) {
added_exe_file_vma(current->mm);
vma->vm_mm = current->mm;
}
}
vma->vm_file = file;
vma->vm_flags = vm_flags;
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_pgoff = pgoff;
vml->vma = vma;
/* set up the mapping */ /* set up the mapping */
if (file && vma->vm_flags & VM_SHARED) if (file && vma->vm_flags & VM_SHARED)
ret = do_mmap_shared_file(vma, len); ret = do_mmap_shared_file(vma);
else else
ret = do_mmap_private(vma, len); ret = do_mmap_private(vma, region, len);
if (ret < 0) if (ret < 0)
goto error; goto error_put_region;
add_nommu_region(region);
/* okay... we have a mapping; now we have to register it */ /* okay... we have a mapping; now we have to register it */
result = (void *) vma->vm_start; result = vma->vm_start;
current->mm->total_vm += len >> PAGE_SHIFT; current->mm->total_vm += len >> PAGE_SHIFT;
add_nommu_vma(vma); share:
add_vma_to_mm(current->mm, vma);
shared: up_write(&nommu_region_sem);
add_vma_to_mm(current->mm, vml);
up_write(&nommu_vma_sem);
if (prot & PROT_EXEC) if (prot & PROT_EXEC)
flush_icache_range((unsigned long) result, flush_icache_range(result, result + len);
(unsigned long) result + len);
#ifdef DEBUG kleave(" = %lx", result);
printk("do_mmap:\n"); return result;
show_process_blocks();
#endif
return (unsigned long) result; error_put_region:
__put_nommu_region(region);
error:
up_write(&nommu_vma_sem);
kfree(vml);
if (vma) { if (vma) {
if (vma->vm_file) { if (vma->vm_file) {
fput(vma->vm_file); fput(vma->vm_file);
if (vma->vm_flags & VM_EXECUTABLE) if (vma->vm_flags & VM_EXECUTABLE)
removed_exe_file_vma(vma->vm_mm); removed_exe_file_vma(vma->vm_mm);
} }
kfree(vma); kmem_cache_free(vm_area_cachep, vma);
} }
kleave(" = %d [pr]", ret);
return ret; return ret;
sharing_violation: error_just_free:
up_write(&nommu_vma_sem); up_write(&nommu_region_sem);
printk("Attempt to share mismatched mappings\n"); error:
kfree(vml); fput(region->vm_file);
return -EINVAL; kmem_cache_free(vm_region_jar, region);
fput(vma->vm_file);
if (vma->vm_flags & VM_EXECUTABLE)
removed_exe_file_vma(vma->vm_mm);
kmem_cache_free(vm_area_cachep, vma);
kleave(" = %d", ret);
return ret;
sharing_violation:
up_write(&nommu_region_sem);
printk(KERN_WARNING "Attempt to share mismatched mappings\n");
ret = -EINVAL;
goto error;
error_getting_vma: error_getting_vma:
up_write(&nommu_vma_sem); kmem_cache_free(vm_region_jar, region);
kfree(vml); printk(KERN_WARNING "Allocation of vma for %lu byte allocation"
printk("Allocation of vma for %lu byte allocation from process %d failed\n", " from process %d failed\n",
len, current->pid); len, current->pid);
show_free_areas(); show_free_areas();
return -ENOMEM; return -ENOMEM;
error_getting_vml: error_getting_region:
printk("Allocation of vml for %lu byte allocation from process %d failed\n", printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"
" from process %d failed\n",
len, current->pid); len, current->pid);
show_free_areas(); show_free_areas();
return -ENOMEM; return -ENOMEM;
...@@ -1097,77 +1362,180 @@ unsigned long do_mmap_pgoff(struct file *file, ...@@ -1097,77 +1362,180 @@ unsigned long do_mmap_pgoff(struct file *file,
EXPORT_SYMBOL(do_mmap_pgoff); EXPORT_SYMBOL(do_mmap_pgoff);
/* /*
* handle mapping disposal for uClinux * split a vma into two pieces at address 'addr', a new vma is allocated either
* for the first part or the tail.
*/ */
static void put_vma(struct mm_struct *mm, struct vm_area_struct *vma) int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{ {
if (vma) { struct vm_area_struct *new;
down_write(&nommu_vma_sem); struct vm_region *region;
unsigned long npages;
if (atomic_dec_and_test(&vma->vm_usage)) { kenter("");
delete_nommu_vma(vma);
if (vma->vm_ops && vma->vm_ops->close) /* we're only permitted to split anonymous regions that have a single
vma->vm_ops->close(vma); * owner */
if (vma->vm_file ||
atomic_read(&vma->vm_region->vm_usage) != 1)
return -ENOMEM;
/* IO memory and memory shared directly out of the pagecache from if (mm->map_count >= sysctl_max_map_count)
* ramfs/tmpfs mustn't be released here */ return -ENOMEM;
if (vma->vm_flags & VM_MAPPED_COPY)
kfree((void *) vma->vm_start);
if (vma->vm_file) { region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
fput(vma->vm_file); if (!region)
if (vma->vm_flags & VM_EXECUTABLE) return -ENOMEM;
removed_exe_file_vma(mm);
} new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
kfree(vma); if (!new) {
} kmem_cache_free(vm_region_jar, region);
return -ENOMEM;
}
/* most fields are the same, copy all, and then fixup */
*new = *vma;
*region = *vma->vm_region;
new->vm_region = region;
npages = (addr - vma->vm_start) >> PAGE_SHIFT;
if (new_below) {
region->vm_end = new->vm_end = addr;
} else {
region->vm_start = new->vm_start = addr;
region->vm_pgoff = new->vm_pgoff += npages;
}
up_write(&nommu_vma_sem); if (new->vm_ops && new->vm_ops->open)
new->vm_ops->open(new);
delete_vma_from_mm(vma);
down_write(&nommu_region_sem);
delete_nommu_region(vma->vm_region);
if (new_below) {
vma->vm_region->vm_start = vma->vm_start = addr;
vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
} else {
vma->vm_region->vm_end = vma->vm_end = addr;
} }
add_nommu_region(vma->vm_region);
add_nommu_region(new->vm_region);
up_write(&nommu_region_sem);
add_vma_to_mm(mm, vma);
add_vma_to_mm(mm, new);
return 0;
} }
/* /*
* release a mapping * shrink a VMA by removing the specified chunk from either the beginning or
* - under NOMMU conditions the parameters must match exactly to the mapping to * the end
* be removed
*/ */
int do_munmap(struct mm_struct *mm, unsigned long addr, size_t len) static int shrink_vma(struct mm_struct *mm,
struct vm_area_struct *vma,
unsigned long from, unsigned long to)
{ {
struct vm_list_struct *vml, **parent; struct vm_region *region;
unsigned long end = addr + len;
#ifdef DEBUG kenter("");
printk("do_munmap:\n");
#endif
for (parent = &mm->context.vmlist; *parent; parent = &(*parent)->next) { /* adjust the VMA's pointers, which may reposition it in the MM's tree
if ((*parent)->vma->vm_start > addr) * and list */
break; delete_vma_from_mm(vma);
if ((*parent)->vma->vm_start == addr && if (from > vma->vm_start)
((len == 0) || ((*parent)->vma->vm_end == end))) vma->vm_end = from;
goto found; else
} vma->vm_start = to;
add_vma_to_mm(mm, vma);
printk("munmap of non-mmaped memory by process %d (%s): %p\n", /* cut the backing region down to size */
current->pid, current->comm, (void *) addr); region = vma->vm_region;
return -EINVAL; BUG_ON(atomic_read(&region->vm_usage) != 1);
found: down_write(&nommu_region_sem);
vml = *parent; delete_nommu_region(region);
if (from > region->vm_start)
region->vm_end = from;
else
region->vm_start = to;
add_nommu_region(region);
up_write(&nommu_region_sem);
put_vma(mm, vml->vma); free_page_series(from, to);
return 0;
}
*parent = vml->next; /*
kfree(vml); * release a mapping
* - under NOMMU conditions the chunk to be unmapped must be backed by a single
* VMA, though it need not cover the whole VMA
*/
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
{
struct vm_area_struct *vma;
struct rb_node *rb;
unsigned long end = start + len;
int ret;
update_hiwater_vm(mm); kenter(",%lx,%zx", start, len);
mm->total_vm -= len >> PAGE_SHIFT;
#ifdef DEBUG if (len == 0)
show_process_blocks(); return -EINVAL;
#endif
/* find the first potentially overlapping VMA */
vma = find_vma(mm, start);
if (!vma) {
printk(KERN_WARNING
"munmap of memory not mmapped by process %d (%s):"
" 0x%lx-0x%lx\n",
current->pid, current->comm, start, start + len - 1);
return -EINVAL;
}
/* we're allowed to split an anonymous VMA but not a file-backed one */
if (vma->vm_file) {
do {
if (start > vma->vm_start) {
kleave(" = -EINVAL [miss]");
return -EINVAL;
}
if (end == vma->vm_end)
goto erase_whole_vma;
rb = rb_next(&vma->vm_rb);
vma = rb_entry(rb, struct vm_area_struct, vm_rb);
} while (rb);
kleave(" = -EINVAL [split file]");
return -EINVAL;
} else {
/* the chunk must be a subset of the VMA found */
if (start == vma->vm_start && end == vma->vm_end)
goto erase_whole_vma;
if (start < vma->vm_start || end > vma->vm_end) {
kleave(" = -EINVAL [superset]");
return -EINVAL;
}
if (start & ~PAGE_MASK) {
kleave(" = -EINVAL [unaligned start]");
return -EINVAL;
}
if (end != vma->vm_end && end & ~PAGE_MASK) {
kleave(" = -EINVAL [unaligned split]");
return -EINVAL;
}
if (start != vma->vm_start && end != vma->vm_end) {
ret = split_vma(mm, vma, start, 1);
if (ret < 0) {
kleave(" = %d [split]", ret);
return ret;
}
}
return shrink_vma(mm, vma, start, end);
}
erase_whole_vma:
delete_vma_from_mm(vma);
delete_vma(mm, vma);
kleave(" = 0");
return 0; return 0;
} }
EXPORT_SYMBOL(do_munmap); EXPORT_SYMBOL(do_munmap);
...@@ -1184,29 +1552,26 @@ asmlinkage long sys_munmap(unsigned long addr, size_t len) ...@@ -1184,29 +1552,26 @@ asmlinkage long sys_munmap(unsigned long addr, size_t len)
} }
/* /*
* Release all mappings * release all the mappings made in a process's VM space
*/ */
void exit_mmap(struct mm_struct * mm) void exit_mmap(struct mm_struct *mm)
{ {
struct vm_list_struct *tmp; struct vm_area_struct *vma;
if (mm) { if (!mm)
#ifdef DEBUG return;
printk("Exit_mmap:\n");
#endif
mm->total_vm = 0; kenter("");
while ((tmp = mm->context.vmlist)) { mm->total_vm = 0;
mm->context.vmlist = tmp->next;
put_vma(mm, tmp->vma);
kfree(tmp);
}
#ifdef DEBUG while ((vma = mm->mmap)) {
show_process_blocks(); mm->mmap = vma->vm_next;
#endif delete_vma_from_mm(vma);
delete_vma(mm, vma);
} }
kleave("");
} }
unsigned long do_brk(unsigned long addr, unsigned long len) unsigned long do_brk(unsigned long addr, unsigned long len)
...@@ -1219,8 +1584,8 @@ unsigned long do_brk(unsigned long addr, unsigned long len) ...@@ -1219,8 +1584,8 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
* time (controlled by the MREMAP_MAYMOVE flag and available VM space) * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
* *
* under NOMMU conditions, we only permit changing a mapping's size, and only * under NOMMU conditions, we only permit changing a mapping's size, and only
* as long as it stays within the hole allocated by the kmalloc() call in * as long as it stays within the region allocated by do_mmap_private() and the
* do_mmap_pgoff() and the block is not shareable * block is not shareable
* *
* MREMAP_FIXED is not supported under NOMMU conditions * MREMAP_FIXED is not supported under NOMMU conditions
*/ */
...@@ -1231,13 +1596,16 @@ unsigned long do_mremap(unsigned long addr, ...@@ -1231,13 +1596,16 @@ unsigned long do_mremap(unsigned long addr,
struct vm_area_struct *vma; struct vm_area_struct *vma;
/* insanity checks first */ /* insanity checks first */
if (new_len == 0) if (old_len == 0 || new_len == 0)
return (unsigned long) -EINVAL; return (unsigned long) -EINVAL;
if (addr & ~PAGE_MASK)
return -EINVAL;
if (flags & MREMAP_FIXED && new_addr != addr) if (flags & MREMAP_FIXED && new_addr != addr)
return (unsigned long) -EINVAL; return (unsigned long) -EINVAL;
vma = find_vma_exact(current->mm, addr); vma = find_vma_exact(current->mm, addr, old_len);
if (!vma) if (!vma)
return (unsigned long) -EINVAL; return (unsigned long) -EINVAL;
...@@ -1247,19 +1615,19 @@ unsigned long do_mremap(unsigned long addr, ...@@ -1247,19 +1615,19 @@ unsigned long do_mremap(unsigned long addr,
if (vma->vm_flags & VM_MAYSHARE) if (vma->vm_flags & VM_MAYSHARE)
return (unsigned long) -EPERM; return (unsigned long) -EPERM;
if (new_len > kobjsize((void *) addr)) if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
return (unsigned long) -ENOMEM; return (unsigned long) -ENOMEM;
/* all checks complete - do it */ /* all checks complete - do it */
vma->vm_end = vma->vm_start + new_len; vma->vm_end = vma->vm_start + new_len;
return vma->vm_start; return vma->vm_start;
} }
EXPORT_SYMBOL(do_mremap); EXPORT_SYMBOL(do_mremap);
asmlinkage unsigned long sys_mremap(unsigned long addr, asmlinkage
unsigned long old_len, unsigned long new_len, unsigned long sys_mremap(unsigned long addr,
unsigned long flags, unsigned long new_addr) unsigned long old_len, unsigned long new_len,
unsigned long flags, unsigned long new_addr)
{ {
unsigned long ret; unsigned long ret;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment